diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index e038643..3092fd8 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -1,6 +1,6 @@ -from enum import Enum +from enum import Enum, IntFlag -class ParticipantType(Enum): +class ParticipantType(IntFlag): """determines the type of a participant""" NONE = 0 BSMD = 1 diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index f34f345..59497e3 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -1,12 +1,26 @@ import numpy as np import pandas as pd import datetime +import typing from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times from BreCal.database.enums import ParticipantType def pandas_series_to_data_model(): return +def set_participant_type(x, participant_df)->int: + """ + when iterating over each row entry x in the shipcall_participant_map, + one can update the 'type' column by extracting the matching data from a participant dataframe + + returns: participant_type + """ + participant_id = x["participant_id"] + participant_type = participant_df.loc[participant_id, "type"] + return participant_type + + + class SQLHandler(): """ An object that reads SQL queries from the sql_connection and stores it in pandas DataFrames. The object can read all available tables @@ -95,6 +109,10 @@ class SQLHandler(): # update the 'participants' column in 'shipcall' self.initialize_shipcall_participant_list() + + # update the 'type' in shipcall_participants_map + # fully deprecated + # self.add_participant_type_to_map() return def build_full_mysql_df_dict(self, all_schemas): @@ -121,6 +139,52 @@ class SQLHandler(): lambda x: self.get_participants(x.name), axis=1) return + + def add_participant_type_to_map(self): + """ + applies a lambda function, where the 'type'-column in the shipcall_participant_map is updated by reading the + respective data from the participants. Updates the shipcall_participant_map inplace. + """ + raise Exception("deprecated! Overwriting the shipcall_participant_map may cause harm, as a participant with multi-flag might be wrongfully assigned to multiple roles simultaneously.") + #spm = self.df_dict["shipcall_participant_map"] + #participant_df = self.df_dict["participant"] + + #spm.loc[:,"type"] = spm.loc[:].apply(lambda x: set_participant_type(x, participant_df=participant_df),axis=1) + #self.df_dict["shipcall_participant_map"] = spm + return + + def get_assigned_participants(self, shipcall)->pd.DataFrame: + """return each participant of a respective shipcall, filtered by the shipcall id""" + # get the shipcall_participant_map + spm = self.df_dict["shipcall_participant_map"] + assigned_participants = spm.loc[spm["shipcall_id"]==shipcall.id] + return assigned_participants + + def get_assigned_participants_by_type(self, assigned_participants:pd.DataFrame, participant_type:ParticipantType): + """filters a dataframe of assigned_participants by the provided type enumerator""" + if isinstance(participant_type,int): + participant_type = ParticipantType(participant_type) + + assigned_participants_of_type = assigned_participants.loc[[participant_type in ParticipantType(int(pt_)) for pt_ in list(assigned_participants["type"].values)]] + #assigned_participants_of_type = assigned_participants.loc[assigned_participants["type"]==participant_type.value] + return assigned_participants_of_type + + def check_if_any_participant_of_type_is_unassigned(self, shipcall, *args:list[ParticipantType])->bool: + """ + given a list of input arguments, where item is a participant type, the function determines, whether at least one participant + was assigned for the type. Function returns a boolean, whether any of the required participants in unassigned. + + This method is extensively used for the validation rules 0001, where the header is checked beforehand to identify, whether + the respective participant type is assigned already. + """ + assigned_participants = self.get_assigned_participants(shipcall) + + unassigned = [] # becomes a list of booleans + for participant_type in args: + assignments_of_type = self.get_assigned_participants_by_type(assigned_participants, participant_type=participant_type) + unassignment = len(assignments_of_type)==0 # a participant type does not exist, when there is no match + unassigned.append(unassignment) + return any(unassigned) # returns a single boolean, whether ANY of the types is not assigned def standardize_model_str(self, model_str:str)->str: """check if the 'model_str' is valid and apply lowercasing to the string""" @@ -177,8 +241,28 @@ class SQLHandler(): data = data_model(**data) return data + def filter_df_by_participant_type(self, df, participant_type:typing.Union[int, ParticipantType])->pd.DataFrame: + """ + As ParticipantTypes are Flag objects, a dataframe's integer might resemble multiple participant types simultaneously. + This function allows for more complex filters, as the IntFlag allows more complex queries + + e.g.: + ParticipantType(6) is 2,4 (2+4 = 6) + + Participant(2) in Participant(6) = True # 6 is both, 2 and 4 + Participant(1) in Participant(6) = False # 6 is both, 2 and 4, but not 1 + """ + if isinstance(participant_type,int): + participant_type = ParticipantType(participant_type) + filtered_df = df.loc[[participant_type in ParticipantType(df_pt) for df_pt in list(df["participant_type"].values)]] + return filtered_df + def get_times_for_participant_type(self, df_times, participant_type:int): - filtered_series = df_times.loc[df_times["participant_type"]==participant_type] + filtered_series = self.filter_df_by_participant_type(df_times, participant_type) + #filtered_series = df_times.loc[df_times["participant_type"]==participant_type] + + if len(filtered_series)==0: + return None if not len(filtered_series)<=1: # correcting the error: ERROR:root:found multiple results @@ -239,7 +323,8 @@ class SQLHandler(): df_times = df_times.loc[~df_times[non_null_column].isnull()] # NOT null filter # filter by the agency participant_type - times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] + times_agency = self.filter_df_by_participant_type(df_times, ParticipantType.AGENCY.value) + #times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] return times_agency def filter_df_by_key_value(self, df, key, value)->pd.DataFrame: diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index 0e17bbf..334f3b5 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -10,18 +10,18 @@ from BreCal.database.enums import StatusFlags # a human interpretable dictionary for error messages. In this case, the English language is preferred error_message_dict = { # 0001 A-M - "validation_rule_fct_missing_time_agency_berth_eta":"The shipcall arrives in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of arrival (ETA) {Rule #0001A}", # A - "validation_rule_fct_missing_time_agency_berth_etd":"The shipcall departs in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of departure (ETD) {Rule #0001B}", # B - "validation_rule_fct_missing_time_mooring_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of arrival (ETA) {Rule #0001C}", # C - "validation_rule_fct_missing_time_mooring_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of departure (ETD) {Rule #0001D}", # D - "validation_rule_fct_missing_time_portadministration_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of arrival (ETA) {Rule #0001F}", # F - "validation_rule_fct_missing_time_portadministration_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of departure (ETD) {Rule #0001G}", # G - "validation_rule_fct_missing_time_pilot_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of arrival (ETA) {Rule #0001H}", # H - "validation_rule_fct_missing_time_pilot_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of departure (ETD) {Rule #0001I}", # I - "validation_rule_fct_missing_time_tug_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of arrival (ETA) {Rule #0001J}", # J - "validation_rule_fct_missing_time_tug_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of departure (ETD) {Rule #0001K}", # K - "validation_rule_fct_missing_time_terminal_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of arrival (ETA) {Rule #0001L}", # L - "validation_rule_fct_missing_time_terminal_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of departure (ETD) {Rule #0001M}", # M + "validation_rule_fct_missing_time_agency_berth_eta":"Shipcall arrives soon (<20 hours). The agency did not provide a time yet (ETA) {Rule #0001A}", # A + "validation_rule_fct_missing_time_agency_berth_etd":"Shipcall departs soon (<20 hours). The agency did not provide a time yet (ETD) {Rule #0001B}", # B + "validation_rule_fct_missing_time_mooring_berth_eta":"Shipcall arrives soon (<16 hours). The mooring did not provide a time yet (ETA) {Rule #0001C}", # C + "validation_rule_fct_missing_time_mooring_berth_etd":"Shipcall departs soon (<16 hours). The mooring did not provide a time yet (ETD) {Rule #0001D}", # D + "validation_rule_fct_missing_time_portadministration_berth_eta":"Shipcall arrives soon (<16 hours). The port administration did not provide a time yet (ETA) {Rule #0001F}", # F + "validation_rule_fct_missing_time_portadministration_berth_etd":"Shipcall departs soon (<20 hours). The port administration did not provide a time yet (ETD) {Rule #0001G}", # G + "validation_rule_fct_missing_time_pilot_berth_eta":"Shipcall arrives soon (<16 hours). The pilot did not provide a time yet (ETA) {Rule #0001H}", # H + "validation_rule_fct_missing_time_pilot_berth_etd":"Shipcall departs soon (<20 hours). The pilot did not provide a time yet (ETD) {Rule #0001I}", # I + "validation_rule_fct_missing_time_tug_berth_eta":"Shipcall arrives soon (<16 hours). The tugs did not provide a time yet (ETA) {Rule #0001J}", # J + "validation_rule_fct_missing_time_tug_berth_etd":"Shipcall departs soon (<20 hours). The tugs did not provide a time yet (ETD) {Rule #0001K}", # K + "validation_rule_fct_missing_time_terminal_berth_eta":"Shipcall arrives soon (<16 hours). The terminal did not provide a time yet (ETA) {Rule #0001L}", # L + "validation_rule_fct_missing_time_terminal_berth_etd":"Shipcall departs soon (<20 hours). The terminal did not provide a time yet (ETD) {Rule #0001M}", # M # 0002 A+B+C "validation_rule_fct_shipcall_incoming_participants_disagree_on_eta":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of arrival (ETA) {Rule #0002A}", @@ -214,14 +214,18 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.eta_berth is filled in. - Measures the difference between 'now' and 'shipcall.eta'. """ - # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.eta - key_time = times_agency.eta_berth + key_time = times_agency.eta_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -242,14 +246,18 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.etd_berth is filled in. - Measures the difference between 'now' and 'shipcall.etd'. """ - # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.etd - key_time = times_agency.etd_berth + key_time = times_agency.etd_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -270,16 +278,20 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & MOORING) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value]) != 1): + if not shipcall.type in [ShipcallType.INCOMING.value]: return self.get_no_violation_default_output() + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) + if unassigned: + return self.get_no_violation_default_output() + # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) - query_time = times_agency.eta_berth - key_time = times_mooring.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_mooring.eta_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -300,16 +312,20 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & MOORING) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value]) != 1): + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) + if unassigned: + return self.get_no_violation_default_output() + # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) - query_time = times_agency.etd_berth - key_time = times_mooring.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_mooring.etd_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -332,17 +348,21 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): """ if self.ignore_port_administration_flag: return self.get_no_violation_default_output() - - # check, if the header is filled in (agency & PORT_ADMINISTRATION) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value]) != 1): + + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) - query_time = times_agency.eta_berth - key_time = times_port_administration.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_port_administration.eta_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -365,17 +385,22 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): """ if self.ignore_port_administration_flag: return self.get_no_violation_default_output() - - # check, if the header is filled in (agency & PORT_ADMINISTRATION) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value]) != 1): + + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time + # when there are no times, the function returns None times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) - query_time = times_agency.etd_berth - key_time = times_port_administration.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_port_administration.etd_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -396,16 +421,20 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & PILOT) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value]) != 1): + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) - query_time = times_agency.eta_berth - key_time = times_pilot.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_pilot.eta_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -426,16 +455,20 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & PILOT) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value]) != 1): + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) - query_time = times_agency.etd_berth - key_time = times_pilot.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_pilot.etd_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -456,16 +489,20 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & TUG) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value]) != 1): + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) - query_time = times_agency.eta_berth - key_time = times_tug.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_tug.eta_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -486,16 +523,20 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & TUG) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value]) != 1): + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) - query_time = times_agency.etd_berth - key_time = times_tug.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_tug.etd_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -516,16 +557,20 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_start is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & terminal) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1): - return self.get_no_violation_default_output() # rule not applicable + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) + if unassigned: + return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - query_time = times_agency.eta_berth - key_time = times_terminal.operations_start # eta_berth does not exist in times_terminal! Instead, it is called operations_start + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_terminal.operations_start if times_terminal is not None else None # eta_berth does not exist in times_terminal! Instead, it is called operations_start threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -546,19 +591,20 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_end is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: - return self.get_no_violation_default_output() # rule not applicable - - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: - return self.get_no_violation_default_output() # rule not applicable + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) + if unassigned: + return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - query_time = times_agency.etd_berth - key_time = times_terminal.operations_end # etd_berth does not exist in times_terminal! Instead, it is called operations_end + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_terminal.operations_end if times_terminal is not None else None # etd_berth does not exist in times_terminal! Instead, it is called operations_end threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -660,6 +706,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency & terminal) if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable @@ -692,6 +741,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency & terminal) if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable @@ -724,6 +776,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return self.get_no_violation_default_output() @@ -752,6 +807,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return self.get_no_violation_default_output() @@ -866,18 +924,18 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) # when one of the two values is null, the state is GREEN - if (times_agency.pier_side is None) or (times_terminal.pier_side is None): + if (shipcall.pier_side is None) or (times_terminal.pier_side is None): return self.get_no_violation_default_output() # when one of the two values is null, the state is GREEN - if (pd.isnull(times_agency.pier_side)) or (pd.isnull(times_terminal.pier_side)): + if (pd.isnull(shipcall.pier_side)) or (pd.isnull(times_terminal.pier_side)): return self.get_no_violation_default_output() # only incoming shipcalls matter. The other ones are not relevant for the pier_side selection if shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() - violation_state = bool(times_agency.pier_side)!=bool(times_terminal.pier_side) + violation_state = bool(shipcall.pier_side)!=bool(times_terminal.pier_side) if violation_state: validation_name = "validation_rule_fct_agency_and_terminal_pier_side_disagreement" diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index b90f691..aa2ef22 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -1,4 +1,5 @@ import copy +import re import numpy as np import pandas as pd from BreCal.database.enums import StatusFlags @@ -33,6 +34,10 @@ class ValidationRules(ValidationRuleFunctions): if len(df_times)==0: return (StatusFlags.GREEN.value, []) + + spm = self.sql_handler.df_dict["shipcall_participant_map"] + if len(spm.loc[spm["shipcall_id"]==shipcall.id])==0: + return (StatusFlags.GREEN.value, []) # filter by shipcall id df_times = self.sql_handler.get_times_of_shipcall(shipcall) @@ -71,11 +76,27 @@ class ValidationRules(ValidationRuleFunctions): # unbundle individual results. evaluation_state becomes an integer, violation evaluation_state = [StatusFlags(res[0]).value for res in results] - violations = [",".join(res[1]) if len(res[1])>0 else None for res in results] + violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results] + violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] shipcall_df.loc[:,"evaluation"] = evaluation_state shipcall_df.loc[:,"evaluation_message"] = violations return shipcall_df + + def concise_evaluation_message_if_too_long(self, violation): + """ + when many validation rules are violated at once, the resulting evaluation message may exceed 512 characters (which the mysql database allows) + in these cases, use a regular expression to provide a concise message, where the 'concise' description is only the list of violated rools + """ + if violation is None: + return violation + + if len(violation)>=512: + concise = re.findall(r'{(.*?)\}', violation) + + # e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K'] + violation = f"Evaluation message too long. Violated Rules: {concise}" + return violation def determine_validation_state(self) -> str: """ diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index aac0f44..9eac8a4 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -117,6 +117,19 @@ def test_validation_rule_fct_missing_time_agency_berth_eta__shipcall_eta_dangero # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_eta(shipcall=shipcall, df_times=df_times) @@ -137,6 +150,19 @@ def test_validation_rule_fct_missing_time_agency_berth_eta__shipcall_eta_distant # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_eta(shipcall=shipcall, df_times=df_times) @@ -157,6 +183,19 @@ def test_validation_rule_fct_missing_time_agency_berth_eta__shipcall_eta_is_unde # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_eta(shipcall=shipcall, df_times=df_times) @@ -172,11 +211,25 @@ def test_validation_rule_fct_missing_time_agency_berth_etd__shipcall_etd_is_unde df_times = get_df_times(shipcall) # the shipcall etd is 'soon' + shipcall.type = ShipcallType.OUTGOING.value shipcall.etd = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.AGENCY-10) # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_etd(shipcall=shipcall, df_times=df_times) @@ -197,6 +250,21 @@ def test_validation_rule_fct_missing_time_mooring_berth_eta__shipcall_soon_but_p # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + mooring_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":mooring_participant_id, "type":ParticipantType.MOORING.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_mooring_berth_eta(shipcall=shipcall, df_times=df_times) @@ -211,6 +279,7 @@ def test_validation_rule_fct_missing_time_mooring_berth_etd__shipcall_soon_but_p vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -219,6 +288,21 @@ def test_validation_rule_fct_missing_time_mooring_berth_etd__shipcall_soon_but_p # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + mooring_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":mooring_participant_id, "type":ParticipantType.MOORING.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_mooring_berth_etd(shipcall=shipcall, df_times=df_times) @@ -241,6 +325,21 @@ def test_validation_rule_fct_missing_time_portadministration_berth_eta__shipcall # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pa_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pa_participant_id, "type":ParticipantType.PORT_ADMINISTRATION.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_portadministration_berth_eta(shipcall=shipcall, df_times=df_times) @@ -257,6 +356,7 @@ def test_validation_rule_fct_missing_time_portadministration_berth_etd__shipcall vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -265,6 +365,21 @@ def test_validation_rule_fct_missing_time_portadministration_berth_etd__shipcall # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pa_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pa_participant_id, "type":ParticipantType.PORT_ADMINISTRATION.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_portadministration_berth_etd(shipcall=shipcall, df_times=df_times) @@ -287,6 +402,21 @@ def test_validation_rule_fct_missing_time_pilot_berth_eta__shipcall_soon_but_par # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_eta(shipcall=shipcall, df_times=df_times) @@ -299,6 +429,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -307,6 +438,21 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) @@ -314,13 +460,127 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" return +def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_participant_unassigned__return_green(build_sql_proxy_connection): + """0001-I validation_rule_fct_missing_time_pilot_berth_etd""" + vr = build_sql_proxy_connection['vr'] + shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # set times agency to be undetermined + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "etd_berth"] = None + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + vr.sql_handler.read_all(vr.sql_handler.all_schemas) + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.GREEN, f"function should return 'green', because the pilot is not assigned yet" + return + +def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_participant_estimated_time_undefined_multiple_pilot_assignments_due_to_bug(build_sql_proxy_connection): + """ + 0001-I validation_rule_fct_missing_time_pilot_berth_etd. Checks, whether the function still works in case of a buggy input. When there is more than one pilot + assignment, the validation rule should still work and return 'yellow' properly. + """ + vr = build_sql_proxy_connection['vr'] + + shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # set times agency to be undetermined + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "etd_berth"] = None + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10003, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10004, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" + return + +def test_validation_rule_fct_missing_time_pilot_berth_etd__agency_and_pilot_assigned_pilot_no_times_returns_yellow(build_sql_proxy_connection): + """ + 0001-I validation_rule_fct_missing_time_pilot_berth_etd. Checks the default behaviour, where an agency's time might exist, + while a time by pilot may not exist. In these cases, a yellow state is expected. + """ + vr = build_sql_proxy_connection['vr'] + + shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + # set times of PILOT: should not exist + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value,"participant_type"] = ParticipantType.BSMD.value + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" + return def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_participant_estimated_time_undefined(build_sql_proxy_connection): """0001-J validation_rule_fct_missing_time_tug_berth_eta""" vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.INCOMING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -329,6 +589,21 @@ def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_parti # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + tug_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":tug_participant_id, "type":ParticipantType.TUG.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_tug_berth_eta(shipcall=shipcall, df_times=df_times) @@ -343,6 +618,7 @@ def test_validation_rule_fct_missing_time_tug_berth_etd__shipcall_soon_but_parti vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -351,6 +627,21 @@ def test_validation_rule_fct_missing_time_tug_berth_etd__shipcall_soon_but_parti # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + tug_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":tug_participant_id, "type":ParticipantType.TUG.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_tug_berth_etd(shipcall=shipcall, df_times=df_times) @@ -373,6 +664,21 @@ def test_validation_rule_fct_missing_time_terminal_berth_eta__shipcall_soon_but_ # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "operations_start"] = None # previously: eta_berth, which does not exist in times_terminal + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + terminal_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":terminal_participant_id, "type":ParticipantType.TERMINAL.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_terminal_berth_eta(shipcall=shipcall, df_times=df_times) @@ -387,6 +693,7 @@ def test_validation_rule_fct_missing_time_terminal_berth_etd__shipcall_soon_but_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -395,6 +702,22 @@ def test_validation_rule_fct_missing_time_terminal_berth_etd__shipcall_soon_but_ # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "operations_end"] = None # previously: etd_berth, which does not exist in times_terminal + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + terminal_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":terminal_participant_id, "type":ParticipantType.TERMINAL.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_terminal_berth_etd(shipcall=shipcall, df_times=df_times) @@ -606,6 +929,7 @@ def test_validation_rule_fct_etd_time_not_in_operation_window__times_dont_match( """0003-B validation_rule_fct_etd_time_not_in_operation_window""" vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now() # reference time for easier readability @@ -627,6 +951,7 @@ def test_validation_rule_fct_eta_time_not_in_operation_window_and_validation_rul vr = build_sql_proxy_connection['vr'] import random shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now() @@ -722,6 +1047,7 @@ def test_validation_rule_fct_etd_time_not_in_tidal_window__etd_outside_tidal_win """0004-B validation_rule_fct_etd_time_not_in_tidal_window""" vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now() @@ -805,7 +1131,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement__agency_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() df_times = get_df_times(shipcall) - df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True + shipcall.pier_side = True + # df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "pier_side"] = True (code, msg) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall=shipcall, df_times=df_times) @@ -817,7 +1144,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement__agency_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() df_times = get_df_times(shipcall) - df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True + shipcall.pier_side = True + #df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "pier_side"] = False (code, msg) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall=shipcall, df_times=df_times) @@ -829,7 +1157,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement__agency_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() df_times = get_df_times(shipcall) - df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True + shipcall.pier_side = True + # df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "pier_side"] = None (code, msg) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall=shipcall, df_times=df_times) @@ -869,7 +1198,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_agreement(build_sql_p t2.participant_type = ParticipantType.TERMINAL.value # agreement - t1.pier_side = True + shipcall.pier_side = True + # t1.pier_side = True t2.pier_side = True time_objects = [t1, t2] @@ -900,7 +1230,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement(build_sq t2.participant_type = ParticipantType.TERMINAL.value # disagreement - t1.pier_side = True + shipcall.pier_side = True + # t1.pier_side = True t2.pier_side = False time_objects = [t1, t2]