From efff2fdf827487ff59c8ea1f526e92cb92904db2 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Wed, 29 Nov 2023 15:21:20 +0100 Subject: [PATCH] removing verbosity in validation rule functions, and returning 'None', when a selected times dataframe is empty. In case of empty results, the function now properly computes the delta towards a query time and returns YELLOW, when a violation is observed. This should finally fix the bugs for 0001 A-M --- src/server/BreCal/database/sql_handler.py | 6 +-- .../validators/validation_rule_functions.py | 45 ++++++++++--------- .../test_validation_rule_functions.py | 38 ++++++++++++++++ 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index 188d98a..e473a79 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -170,7 +170,6 @@ class SQLHandler(): This method is extensively used for the validation rules 0001, where the header is checked beforehand to identify, whether the respective participant type is assigned already. """ - print("verbosity of function check_if_any_participant_of_type_is_unassigned") assigned_participants = self.get_assigned_participants(shipcall) unassigned = [] # becomes a list of booleans @@ -178,9 +177,7 @@ class SQLHandler(): assignments_of_type = self.get_assigned_participants_by_type(assigned_participants, participant_type=participant_type) unassignment = len(assignments_of_type)==0 # a participant type does not exist, when there is no match unassigned.append(unassignment) - print("participant type and unassigment state", participant_type, unassignment) return any(unassigned) # returns a single boolean, whether ANY of the types is not assigned - def standardize_model_str(self, model_str:str)->str: """check if the 'model_str' is valid and apply lowercasing to the string""" @@ -240,6 +237,9 @@ class SQLHandler(): def get_times_for_participant_type(self, df_times, participant_type:int): filtered_series = df_times.loc[df_times["participant_type"]==participant_type] + if len(filtered_series)==0: + return None + if not len(filtered_series)<=1: # correcting the error: ERROR:root:found multiple results # however, a warning will still be issued diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index 7148254..2769927 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -218,7 +218,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.eta - key_time = times_agency.eta_berth + key_time = times_agency.eta_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -247,7 +247,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.etd - key_time = times_agency.etd_berth + key_time = times_agency.etd_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -277,8 +277,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) - query_time = times_agency.eta_berth - key_time = times_mooring.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_mooring.eta_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -308,8 +308,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) - query_time = times_agency.etd_berth - key_time = times_mooring.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_mooring.etd_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -339,8 +339,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) - query_time = times_agency.eta_berth - key_time = times_port_administration.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_port_administration.eta_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -367,11 +367,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time + # when there are no times, the function returns None times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) - query_time = times_agency.etd_berth - key_time = times_port_administration.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_port_administration.etd_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -401,8 +402,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) - query_time = times_agency.eta_berth - key_time = times_pilot.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_pilot.eta_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -432,8 +433,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) - query_time = times_agency.etd_berth - key_time = times_pilot.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_pilot.etd_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -463,8 +464,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) - query_time = times_agency.eta_berth - key_time = times_tug.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_tug.eta_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -494,8 +495,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) - query_time = times_agency.etd_berth - key_time = times_tug.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_tug.etd_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -525,8 +526,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - query_time = times_agency.eta_berth - key_time = times_terminal.operations_start # eta_berth does not exist in times_terminal! Instead, it is called operations_start + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_terminal.operations_start if times_terminal is not None else None # eta_berth does not exist in times_terminal! Instead, it is called operations_start threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -556,8 +557,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - query_time = times_agency.etd_berth - key_time = times_terminal.operations_end # etd_berth does not exist in times_terminal! Instead, it is called operations_end + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_terminal.operations_end if times_terminal is not None else None # etd_berth does not exist in times_terminal! Instead, it is called operations_end threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index 4b32acd..ab85502 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -526,6 +526,44 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" return +def test_validation_rule_fct_missing_time_pilot_berth_etd__agency_and_pilot_assigned_pilot_no_times_returns_yellow(build_sql_proxy_connection): + """ + 0001-I validation_rule_fct_missing_time_pilot_berth_etd. Checks the default behaviour, where an agency's time might exist, + while a time by pilot may not exist. In these cases, a yellow state is expected. + """ + vr = build_sql_proxy_connection['vr'] + + shipcall = get_shipcall_simple() + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + # set times of PILOT: should not exist + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value,"participant_type"] = ParticipantType.BSMD.value + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" + return + def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_participant_estimated_time_undefined(build_sql_proxy_connection): """0001-J validation_rule_fct_missing_time_tug_berth_eta""" vr = build_sql_proxy_connection['vr']