From d6a6cc00ff0d6cce78fa1d2d4af01cb8105bbd46 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Wed, 29 Nov 2023 08:52:37 +0100 Subject: [PATCH 1/4] updating validation rules 0001 A-M. Instead of filtering by times_df (which may not exist), the rules make use of the shipcall_participant_map. When one of the participants in a rule is not assigned, no violation is observed. When there are multiple entries of a participant (due to an input bug), the function still verifies properly. When critical time is observed, and there is not yet an entry for the respective key time, there will be a 'yellow' state. --- src/server/BreCal/database/sql_handler.py | 60 ++++ .../validators/validation_rule_functions.py | 71 +++-- .../test_validation_rule_functions.py | 275 ++++++++++++++++++ 3 files changed, 375 insertions(+), 31 deletions(-) diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index f34f345..188d98a 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -7,6 +7,19 @@ from BreCal.database.enums import ParticipantType def pandas_series_to_data_model(): return +def set_participant_type(x, participant_df)->int: + """ + when iterating over each row entry x in the shipcall_participant_map, + one can update the 'type' column by extracting the matching data from a participant dataframe + + returns: participant_type + """ + participant_id = x["participant_id"] + participant_type = participant_df.loc[participant_id, "type"] + return participant_type + + + class SQLHandler(): """ An object that reads SQL queries from the sql_connection and stores it in pandas DataFrames. The object can read all available tables @@ -95,6 +108,9 @@ class SQLHandler(): # update the 'participants' column in 'shipcall' self.initialize_shipcall_participant_list() + + # update the 'type' in shipcall_participants_map + self.add_participant_type_to_map() return def build_full_mysql_df_dict(self, all_schemas): @@ -121,6 +137,50 @@ class SQLHandler(): lambda x: self.get_participants(x.name), axis=1) return + + def add_participant_type_to_map(self): + """ + applies a lambda function, where the 'type'-column in the shipcall_participant_map is updated by reading the + respective data from the participants. Updates the shipcall_participant_map inplace. + """ + spm = self.df_dict["shipcall_participant_map"] + participant_df = self.df_dict["participant"] + + spm.loc[:,"type"] = spm.loc[:].apply(lambda x: set_participant_type(x, participant_df=participant_df),axis=1) + self.df_dict["shipcall_participant_map"] = spm + return + + def get_assigned_participants(self, shipcall)->pd.DataFrame: + """return each participant of a respective shipcall, filtered by the shipcall id""" + # get the shipcall_participant_map + spm = self.df_dict["shipcall_participant_map"] + assigned_participants = spm.loc[spm["shipcall_id"]==shipcall.id] + return assigned_participants + + def get_assigned_participants_by_type(self, assigned_participants:pd.DataFrame, participant_type:ParticipantType): + """filters a dataframe of assigned_participants by the provided type enumerator""" + assigned_participants_of_type = assigned_participants.loc[assigned_participants["type"]==participant_type.value] + return assigned_participants_of_type + + def check_if_any_participant_of_type_is_unassigned(self, shipcall, *args:list[ParticipantType])->bool: + """ + given a list of input arguments, where item is a participant type, the function determines, whether at least one participant + was assigned for the type. Function returns a boolean, whether any of the required participants in unassigned. + + This method is extensively used for the validation rules 0001, where the header is checked beforehand to identify, whether + the respective participant type is assigned already. + """ + print("verbosity of function check_if_any_participant_of_type_is_unassigned") + assigned_participants = self.get_assigned_participants(shipcall) + + unassigned = [] # becomes a list of booleans + for participant_type in args: + assignments_of_type = self.get_assigned_participants_by_type(assigned_participants, participant_type=participant_type) + unassignment = len(assignments_of_type)==0 # a participant type does not exist, when there is no match + unassigned.append(unassignment) + print("participant type and unassigment state", participant_type, unassignment) + return any(unassigned) # returns a single boolean, whether ANY of the types is not assigned + def standardize_model_str(self, model_str:str)->str: """check if the 'model_str' is valid and apply lowercasing to the string""" diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index c8f00c6..7148254 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -210,8 +210,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.eta_berth is filled in. - Measures the difference between 'now' and 'shipcall.eta'. """ - # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -238,8 +239,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.etd_berth is filled in. - Measures the difference between 'now' and 'shipcall.etd'. """ - # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -266,10 +268,11 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & MOORING) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) + if unassigned: return self.get_no_violation_default_output() - + # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) @@ -296,10 +299,11 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & MOORING) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) + if unassigned: return self.get_no_violation_default_output() - + # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) @@ -326,8 +330,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & PORT_ADMINISTRATION) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -356,8 +361,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & PORT_ADMINISTRATION) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -386,8 +392,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & PILOT) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -416,8 +423,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & PILOT) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -446,8 +454,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & TUG) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -476,8 +485,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & TUG) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -506,9 +516,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_start is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & terminal) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1): - return self.get_no_violation_default_output() # rule not applicable + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) + if unassigned: + return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) @@ -536,12 +547,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_end is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: - return self.get_no_violation_default_output() # rule not applicable - - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: - return self.get_no_violation_default_output() # rule not applicable + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) + if unassigned: + return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index 1847973..4b32acd 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -117,6 +117,19 @@ def test_validation_rule_fct_missing_time_agency_berth_eta__shipcall_eta_dangero # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_eta(shipcall=shipcall, df_times=df_times) @@ -137,6 +150,19 @@ def test_validation_rule_fct_missing_time_agency_berth_eta__shipcall_eta_distant # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_eta(shipcall=shipcall, df_times=df_times) @@ -157,6 +183,19 @@ def test_validation_rule_fct_missing_time_agency_berth_eta__shipcall_eta_is_unde # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_eta(shipcall=shipcall, df_times=df_times) @@ -177,6 +216,19 @@ def test_validation_rule_fct_missing_time_agency_berth_etd__shipcall_etd_is_unde # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_etd(shipcall=shipcall, df_times=df_times) @@ -197,6 +249,21 @@ def test_validation_rule_fct_missing_time_mooring_berth_eta__shipcall_soon_but_p # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + mooring_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":mooring_participant_id, "type":ParticipantType.MOORING.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_mooring_berth_eta(shipcall=shipcall, df_times=df_times) @@ -219,6 +286,21 @@ def test_validation_rule_fct_missing_time_mooring_berth_etd__shipcall_soon_but_p # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + mooring_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":mooring_participant_id, "type":ParticipantType.MOORING.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_mooring_berth_etd(shipcall=shipcall, df_times=df_times) @@ -241,6 +323,21 @@ def test_validation_rule_fct_missing_time_portadministration_berth_eta__shipcall # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pa_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pa_participant_id, "type":ParticipantType.PORT_ADMINISTRATION.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_portadministration_berth_eta(shipcall=shipcall, df_times=df_times) @@ -263,6 +360,21 @@ def test_validation_rule_fct_missing_time_portadministration_berth_etd__shipcall # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pa_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pa_participant_id, "type":ParticipantType.PORT_ADMINISTRATION.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_portadministration_berth_etd(shipcall=shipcall, df_times=df_times) @@ -283,6 +395,21 @@ def test_validation_rule_fct_missing_time_pilot_berth_eta__shipcall_soon_but_par # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_eta(shipcall=shipcall, df_times=df_times) @@ -303,6 +430,21 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) @@ -310,7 +452,79 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" return +def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_participant_unassigned__return_green(build_sql_proxy_connection): + """0001-I validation_rule_fct_missing_time_pilot_berth_etd""" + vr = build_sql_proxy_connection['vr'] + shipcall = get_shipcall_simple() + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # set times agency to be undetermined + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "etd_berth"] = None + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + vr.sql_handler.read_all(vr.sql_handler.all_schemas) + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.GREEN, f"function should return 'green', because the pilot is not assigned yet" + return + +def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_participant_estimated_time_undefined_multiple_pilot_assignments_due_to_bug(build_sql_proxy_connection): + """ + 0001-I validation_rule_fct_missing_time_pilot_berth_etd. Checks, whether the function still works in case of a buggy input. When there is more than one pilot + assignment, the validation rule should still work and return 'yellow' properly. + """ + vr = build_sql_proxy_connection['vr'] + + shipcall = get_shipcall_simple() + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # set times agency to be undetermined + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "etd_berth"] = None + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10003, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10004, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" + return def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_participant_estimated_time_undefined(build_sql_proxy_connection): """0001-J validation_rule_fct_missing_time_tug_berth_eta""" @@ -325,6 +539,21 @@ def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_parti # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + tug_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":tug_participant_id, "type":ParticipantType.TUG.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_tug_berth_eta(shipcall=shipcall, df_times=df_times) @@ -347,6 +576,21 @@ def test_validation_rule_fct_missing_time_tug_berth_etd__shipcall_soon_but_parti # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + tug_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":tug_participant_id, "type":ParticipantType.TUG.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_tug_berth_etd(shipcall=shipcall, df_times=df_times) @@ -369,6 +613,21 @@ def test_validation_rule_fct_missing_time_terminal_berth_eta__shipcall_soon_but_ # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "operations_start"] = None # previously: eta_berth, which does not exist in times_terminal + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + terminal_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":terminal_participant_id, "type":ParticipantType.TERMINAL.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_terminal_berth_eta(shipcall=shipcall, df_times=df_times) @@ -391,6 +650,22 @@ def test_validation_rule_fct_missing_time_terminal_berth_etd__shipcall_soon_but_ # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "operations_end"] = None # previously: etd_berth, which does not exist in times_terminal + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + terminal_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":terminal_participant_id, "type":ParticipantType.TERMINAL.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_terminal_berth_etd(shipcall=shipcall, df_times=df_times) From efff2fdf827487ff59c8ea1f526e92cb92904db2 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Wed, 29 Nov 2023 15:21:20 +0100 Subject: [PATCH 2/4] removing verbosity in validation rule functions, and returning 'None', when a selected times dataframe is empty. In case of empty results, the function now properly computes the delta towards a query time and returns YELLOW, when a violation is observed. This should finally fix the bugs for 0001 A-M --- src/server/BreCal/database/sql_handler.py | 6 +-- .../validators/validation_rule_functions.py | 45 ++++++++++--------- .../test_validation_rule_functions.py | 38 ++++++++++++++++ 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index 188d98a..e473a79 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -170,7 +170,6 @@ class SQLHandler(): This method is extensively used for the validation rules 0001, where the header is checked beforehand to identify, whether the respective participant type is assigned already. """ - print("verbosity of function check_if_any_participant_of_type_is_unassigned") assigned_participants = self.get_assigned_participants(shipcall) unassigned = [] # becomes a list of booleans @@ -178,9 +177,7 @@ class SQLHandler(): assignments_of_type = self.get_assigned_participants_by_type(assigned_participants, participant_type=participant_type) unassignment = len(assignments_of_type)==0 # a participant type does not exist, when there is no match unassigned.append(unassignment) - print("participant type and unassigment state", participant_type, unassignment) return any(unassigned) # returns a single boolean, whether ANY of the types is not assigned - def standardize_model_str(self, model_str:str)->str: """check if the 'model_str' is valid and apply lowercasing to the string""" @@ -240,6 +237,9 @@ class SQLHandler(): def get_times_for_participant_type(self, df_times, participant_type:int): filtered_series = df_times.loc[df_times["participant_type"]==participant_type] + if len(filtered_series)==0: + return None + if not len(filtered_series)<=1: # correcting the error: ERROR:root:found multiple results # however, a warning will still be issued diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index 7148254..2769927 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -218,7 +218,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.eta - key_time = times_agency.eta_berth + key_time = times_agency.eta_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -247,7 +247,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.etd - key_time = times_agency.etd_berth + key_time = times_agency.etd_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -277,8 +277,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) - query_time = times_agency.eta_berth - key_time = times_mooring.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_mooring.eta_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -308,8 +308,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) - query_time = times_agency.etd_berth - key_time = times_mooring.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_mooring.etd_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -339,8 +339,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) - query_time = times_agency.eta_berth - key_time = times_port_administration.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_port_administration.eta_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -367,11 +367,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time + # when there are no times, the function returns None times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) - query_time = times_agency.etd_berth - key_time = times_port_administration.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_port_administration.etd_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -401,8 +402,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) - query_time = times_agency.eta_berth - key_time = times_pilot.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_pilot.eta_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -432,8 +433,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) - query_time = times_agency.etd_berth - key_time = times_pilot.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_pilot.etd_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -463,8 +464,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) - query_time = times_agency.eta_berth - key_time = times_tug.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_tug.eta_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -494,8 +495,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) - query_time = times_agency.etd_berth - key_time = times_tug.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_tug.etd_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -525,8 +526,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - query_time = times_agency.eta_berth - key_time = times_terminal.operations_start # eta_berth does not exist in times_terminal! Instead, it is called operations_start + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_terminal.operations_start if times_terminal is not None else None # eta_berth does not exist in times_terminal! Instead, it is called operations_start threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -556,8 +557,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - query_time = times_agency.etd_berth - key_time = times_terminal.operations_end # etd_berth does not exist in times_terminal! Instead, it is called operations_end + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_terminal.operations_end if times_terminal is not None else None # etd_berth does not exist in times_terminal! Instead, it is called operations_end threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index 4b32acd..ab85502 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -526,6 +526,44 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" return +def test_validation_rule_fct_missing_time_pilot_berth_etd__agency_and_pilot_assigned_pilot_no_times_returns_yellow(build_sql_proxy_connection): + """ + 0001-I validation_rule_fct_missing_time_pilot_berth_etd. Checks the default behaviour, where an agency's time might exist, + while a time by pilot may not exist. In these cases, a yellow state is expected. + """ + vr = build_sql_proxy_connection['vr'] + + shipcall = get_shipcall_simple() + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + # set times of PILOT: should not exist + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value,"participant_type"] = ParticipantType.BSMD.value + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" + return + def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_participant_estimated_time_undefined(build_sql_proxy_connection): """0001-J validation_rule_fct_missing_time_tug_berth_eta""" vr = build_sql_proxy_connection['vr'] From a0785d012cb2f2c8a91ff08098518648f3050dac Mon Sep 17 00:00:00 2001 From: scopesorting Date: Thu, 30 Nov 2023 15:53:42 +0100 Subject: [PATCH 3/4] changing the ParticipantType to an IntFlag, so multiple roles are possible. Adapting every validation rule (0001, 0003, 0004, 0005), which may be affected by this change. Changing the filter for a participant type to properly include the change. Changing the pier_side rule (0006B), which uses the shipcall and times_terminal. New shipcalls should now be evaluated properly, unless no participant is assigned at all. If the ladder case can occur, the validation rules 0001N+0001O will be added (held back for now). --- src/server/BreCal/database/enums.py | 4 +- src/server/BreCal/database/sql_handler.py | 41 +++++++++++++++---- .../validators/validation_rule_functions.py | 6 +-- .../BreCal/validators/validation_rules.py | 4 ++ .../test_validation_rule_functions.py | 15 ++++--- 5 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index e038643..3092fd8 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -1,6 +1,6 @@ -from enum import Enum +from enum import Enum, IntFlag -class ParticipantType(Enum): +class ParticipantType(IntFlag): """determines the type of a participant""" NONE = 0 BSMD = 1 diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index e473a79..59497e3 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd import datetime +import typing from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times from BreCal.database.enums import ParticipantType @@ -110,7 +111,8 @@ class SQLHandler(): self.initialize_shipcall_participant_list() # update the 'type' in shipcall_participants_map - self.add_participant_type_to_map() + # fully deprecated + # self.add_participant_type_to_map() return def build_full_mysql_df_dict(self, all_schemas): @@ -143,11 +145,12 @@ class SQLHandler(): applies a lambda function, where the 'type'-column in the shipcall_participant_map is updated by reading the respective data from the participants. Updates the shipcall_participant_map inplace. """ - spm = self.df_dict["shipcall_participant_map"] - participant_df = self.df_dict["participant"] + raise Exception("deprecated! Overwriting the shipcall_participant_map may cause harm, as a participant with multi-flag might be wrongfully assigned to multiple roles simultaneously.") + #spm = self.df_dict["shipcall_participant_map"] + #participant_df = self.df_dict["participant"] - spm.loc[:,"type"] = spm.loc[:].apply(lambda x: set_participant_type(x, participant_df=participant_df),axis=1) - self.df_dict["shipcall_participant_map"] = spm + #spm.loc[:,"type"] = spm.loc[:].apply(lambda x: set_participant_type(x, participant_df=participant_df),axis=1) + #self.df_dict["shipcall_participant_map"] = spm return def get_assigned_participants(self, shipcall)->pd.DataFrame: @@ -159,7 +162,11 @@ class SQLHandler(): def get_assigned_participants_by_type(self, assigned_participants:pd.DataFrame, participant_type:ParticipantType): """filters a dataframe of assigned_participants by the provided type enumerator""" - assigned_participants_of_type = assigned_participants.loc[assigned_participants["type"]==participant_type.value] + if isinstance(participant_type,int): + participant_type = ParticipantType(participant_type) + + assigned_participants_of_type = assigned_participants.loc[[participant_type in ParticipantType(int(pt_)) for pt_ in list(assigned_participants["type"].values)]] + #assigned_participants_of_type = assigned_participants.loc[assigned_participants["type"]==participant_type.value] return assigned_participants_of_type def check_if_any_participant_of_type_is_unassigned(self, shipcall, *args:list[ParticipantType])->bool: @@ -234,8 +241,25 @@ class SQLHandler(): data = data_model(**data) return data + def filter_df_by_participant_type(self, df, participant_type:typing.Union[int, ParticipantType])->pd.DataFrame: + """ + As ParticipantTypes are Flag objects, a dataframe's integer might resemble multiple participant types simultaneously. + This function allows for more complex filters, as the IntFlag allows more complex queries + + e.g.: + ParticipantType(6) is 2,4 (2+4 = 6) + + Participant(2) in Participant(6) = True # 6 is both, 2 and 4 + Participant(1) in Participant(6) = False # 6 is both, 2 and 4, but not 1 + """ + if isinstance(participant_type,int): + participant_type = ParticipantType(participant_type) + filtered_df = df.loc[[participant_type in ParticipantType(df_pt) for df_pt in list(df["participant_type"].values)]] + return filtered_df + def get_times_for_participant_type(self, df_times, participant_type:int): - filtered_series = df_times.loc[df_times["participant_type"]==participant_type] + filtered_series = self.filter_df_by_participant_type(df_times, participant_type) + #filtered_series = df_times.loc[df_times["participant_type"]==participant_type] if len(filtered_series)==0: return None @@ -299,7 +323,8 @@ class SQLHandler(): df_times = df_times.loc[~df_times[non_null_column].isnull()] # NOT null filter # filter by the agency participant_type - times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] + times_agency = self.filter_df_by_participant_type(df_times, ParticipantType.AGENCY.value) + #times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] return times_agency def filter_df_by_key_value(self, df, key, value)->pd.DataFrame: diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index 2769927..a7993cd 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -866,18 +866,18 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) # when one of the two values is null, the state is GREEN - if (times_agency.pier_side is None) or (times_terminal.pier_side is None): + if (shipcall.pier_side is None) or (times_terminal.pier_side is None): return self.get_no_violation_default_output() # when one of the two values is null, the state is GREEN - if (pd.isnull(times_agency.pier_side)) or (pd.isnull(times_terminal.pier_side)): + if (pd.isnull(shipcall.pier_side)) or (pd.isnull(times_terminal.pier_side)): return self.get_no_violation_default_output() # only incoming shipcalls matter. The other ones are not relevant for the pier_side selection if shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() - violation_state = bool(times_agency.pier_side)!=bool(times_terminal.pier_side) + violation_state = bool(shipcall.pier_side)!=bool(times_terminal.pier_side) if violation_state: validation_name = "validation_rule_fct_agency_and_terminal_pier_side_disagreement" diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index b90f691..cabe5ec 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -33,6 +33,10 @@ class ValidationRules(ValidationRuleFunctions): if len(df_times)==0: return (StatusFlags.GREEN.value, []) + + spm = self.sql_handler.df_dict["shipcall_participant_map"] + if len(spm.loc[spm["shipcall_id"]==shipcall.id])==0: + return (StatusFlags.GREEN.value, []) # filter by shipcall id df_times = self.sql_handler.get_times_of_shipcall(shipcall) diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index ab85502..5c4e3a0 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -1114,7 +1114,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement__agency_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() df_times = get_df_times(shipcall) - df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True + shipcall.pier_side = True + # df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "pier_side"] = True (code, msg) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall=shipcall, df_times=df_times) @@ -1126,7 +1127,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement__agency_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() df_times = get_df_times(shipcall) - df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True + shipcall.pier_side = True + #df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "pier_side"] = False (code, msg) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall=shipcall, df_times=df_times) @@ -1138,7 +1140,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement__agency_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() df_times = get_df_times(shipcall) - df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True + shipcall.pier_side = True + # df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "pier_side"] = None (code, msg) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall=shipcall, df_times=df_times) @@ -1178,7 +1181,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_agreement(build_sql_p t2.participant_type = ParticipantType.TERMINAL.value # agreement - t1.pier_side = True + shipcall.pier_side = True + # t1.pier_side = True t2.pier_side = True time_objects = [t1, t2] @@ -1209,7 +1213,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement(build_sq t2.participant_type = ParticipantType.TERMINAL.value # disagreement - t1.pier_side = True + shipcall.pier_side = True + # t1.pier_side = True t2.pier_side = False time_objects = [t1, t2] From 377ec85ce98e70fa159537fbbd069484821ba0f8 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Thu, 30 Nov 2023 17:34:44 +0100 Subject: [PATCH 4/4] more concise evaluation messages for 0001. Adding newlines (works on Windows) when multiple evaluation messages are shown. Properly adding the ShipcallType filters for each rule (whether incoming, outgoing or shifting). Added a regular expression to abbreviate an evaluation message when 512 characters are exceeded. --- .../validators/validation_rule_functions.py | 72 +++++++++++++++---- .../BreCal/validators/validation_rules.py | 19 ++++- .../test_validation_rule_functions.py | 13 ++++ 3 files changed, 91 insertions(+), 13 deletions(-) diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index a7993cd..92bf24e 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -10,18 +10,18 @@ from BreCal.database.enums import StatusFlags # a human interpretable dictionary for error messages. In this case, the English language is preferred error_message_dict = { # 0001 A-M - "validation_rule_fct_missing_time_agency_berth_eta":"The shipcall arrives in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of arrival (ETA) {Rule #0001A}", # A - "validation_rule_fct_missing_time_agency_berth_etd":"The shipcall departs in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of departure (ETD) {Rule #0001B}", # B - "validation_rule_fct_missing_time_mooring_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of arrival (ETA) {Rule #0001C}", # C - "validation_rule_fct_missing_time_mooring_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of departure (ETD) {Rule #0001D}", # D - "validation_rule_fct_missing_time_portadministration_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of arrival (ETA) {Rule #0001F}", # F - "validation_rule_fct_missing_time_portadministration_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of departure (ETD) {Rule #0001G}", # G - "validation_rule_fct_missing_time_pilot_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of arrival (ETA) {Rule #0001H}", # H - "validation_rule_fct_missing_time_pilot_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of departure (ETD) {Rule #0001I}", # I - "validation_rule_fct_missing_time_tug_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of arrival (ETA) {Rule #0001J}", # J - "validation_rule_fct_missing_time_tug_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of departure (ETD) {Rule #0001K}", # K - "validation_rule_fct_missing_time_terminal_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of arrival (ETA) {Rule #0001L}", # L - "validation_rule_fct_missing_time_terminal_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of departure (ETD) {Rule #0001M}", # M + "validation_rule_fct_missing_time_agency_berth_eta":"Shipcall arrives soon (<20 hours). The agency did not provide a time yet (ETA) {Rule #0001A}", # A + "validation_rule_fct_missing_time_agency_berth_etd":"Shipcall departs soon (<20 hours). The agency did not provide a time yet (ETD) {Rule #0001B}", # B + "validation_rule_fct_missing_time_mooring_berth_eta":"Shipcall arrives soon (<16 hours). The mooring did not provide a time yet (ETA) {Rule #0001C}", # C + "validation_rule_fct_missing_time_mooring_berth_etd":"Shipcall departs soon (<16 hours). The mooring did not provide a time yet (ETD) {Rule #0001D}", # D + "validation_rule_fct_missing_time_portadministration_berth_eta":"Shipcall arrives soon (<16 hours). The port administration did not provide a time yet (ETA) {Rule #0001F}", # F + "validation_rule_fct_missing_time_portadministration_berth_etd":"Shipcall departs soon (<20 hours). The port administration did not provide a time yet (ETD) {Rule #0001G}", # G + "validation_rule_fct_missing_time_pilot_berth_eta":"Shipcall arrives soon (<16 hours). The pilot did not provide a time yet (ETA) {Rule #0001H}", # H + "validation_rule_fct_missing_time_pilot_berth_etd":"Shipcall departs soon (<20 hours). The pilot did not provide a time yet (ETD) {Rule #0001I}", # I + "validation_rule_fct_missing_time_tug_berth_eta":"Shipcall arrives soon (<16 hours). The tugs did not provide a time yet (ETA) {Rule #0001J}", # J + "validation_rule_fct_missing_time_tug_berth_etd":"Shipcall departs soon (<20 hours). The tugs did not provide a time yet (ETD) {Rule #0001K}", # K + "validation_rule_fct_missing_time_terminal_berth_eta":"Shipcall arrives soon (<16 hours). The terminal did not provide a time yet (ETA) {Rule #0001L}", # L + "validation_rule_fct_missing_time_terminal_berth_etd":"Shipcall departs soon (<20 hours). The terminal did not provide a time yet (ETD) {Rule #0001M}", # M # 0002 A+B+C "validation_rule_fct_shipcall_incoming_participants_disagree_on_eta":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of arrival (ETA) {Rule #0002A}", @@ -210,6 +210,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.eta_berth is filled in. - Measures the difference between 'now' and 'shipcall.eta'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) if unassigned: @@ -239,6 +242,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.etd_berth is filled in. - Measures the difference between 'now' and 'shipcall.etd'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) if unassigned: @@ -268,6 +274,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) if unassigned: @@ -299,6 +308,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) if unassigned: @@ -330,6 +342,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) if unassigned: @@ -361,6 +376,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) if unassigned: @@ -393,6 +411,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) if unassigned: @@ -424,6 +445,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) if unassigned: @@ -455,6 +479,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) if unassigned: @@ -486,6 +513,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) if unassigned: @@ -517,6 +547,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_start is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) if unassigned: @@ -548,6 +581,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_end is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) if unassigned: @@ -660,6 +696,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency & terminal) if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable @@ -692,6 +731,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency & terminal) if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable @@ -724,6 +766,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return self.get_no_violation_default_output() @@ -752,6 +797,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return self.get_no_violation_default_output() diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index cabe5ec..aa2ef22 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -1,4 +1,5 @@ import copy +import re import numpy as np import pandas as pd from BreCal.database.enums import StatusFlags @@ -75,11 +76,27 @@ class ValidationRules(ValidationRuleFunctions): # unbundle individual results. evaluation_state becomes an integer, violation evaluation_state = [StatusFlags(res[0]).value for res in results] - violations = [",".join(res[1]) if len(res[1])>0 else None for res in results] + violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results] + violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] shipcall_df.loc[:,"evaluation"] = evaluation_state shipcall_df.loc[:,"evaluation_message"] = violations return shipcall_df + + def concise_evaluation_message_if_too_long(self, violation): + """ + when many validation rules are violated at once, the resulting evaluation message may exceed 512 characters (which the mysql database allows) + in these cases, use a regular expression to provide a concise message, where the 'concise' description is only the list of violated rools + """ + if violation is None: + return violation + + if len(violation)>=512: + concise = re.findall(r'{(.*?)\}', violation) + + # e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K'] + violation = f"Evaluation message too long. Violated Rules: {concise}" + return violation def determine_validation_state(self) -> str: """ diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index 5c4e3a0..2593e85 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -211,6 +211,7 @@ def test_validation_rule_fct_missing_time_agency_berth_etd__shipcall_etd_is_unde df_times = get_df_times(shipcall) # the shipcall etd is 'soon' + shipcall.type = ShipcallType.OUTGOING.value shipcall.etd = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.AGENCY-10) # set times agency to be undetermined @@ -278,6 +279,7 @@ def test_validation_rule_fct_missing_time_mooring_berth_etd__shipcall_soon_but_p vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -352,6 +354,7 @@ def test_validation_rule_fct_missing_time_portadministration_berth_etd__shipcall vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -422,6 +425,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -457,6 +461,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -494,6 +499,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -534,6 +540,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__agency_and_pilot_assi vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -569,6 +576,7 @@ def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_parti vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.INCOMING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -606,6 +614,7 @@ def test_validation_rule_fct_missing_time_tug_berth_etd__shipcall_soon_but_parti vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -680,6 +689,7 @@ def test_validation_rule_fct_missing_time_terminal_berth_etd__shipcall_soon_but_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -915,6 +925,7 @@ def test_validation_rule_fct_etd_time_not_in_operation_window__times_dont_match( """0003-B validation_rule_fct_etd_time_not_in_operation_window""" vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now() # reference time for easier readability @@ -936,6 +947,7 @@ def test_validation_rule_fct_eta_time_not_in_operation_window_and_validation_rul vr = build_sql_proxy_connection['vr'] import random shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now() @@ -1031,6 +1043,7 @@ def test_validation_rule_fct_etd_time_not_in_tidal_window__etd_outside_tidal_win """0004-B validation_rule_fct_etd_time_not_in_tidal_window""" vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now()