From 8e2e676cb0e289f6e53c76bd40c9954a615ccb04 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Tue, 31 Oct 2023 18:15:59 +0100 Subject: [PATCH] hotfixing open errors, as defined by Daniel Schick and Christin Hollman (26.10. and 27.10.). Adding descriptions for the error codes instead of using 'cryptic' function names. This should make the application much more readable. --- .../validators/validation_rule_functions.py | 90 ++++++++++++++++--- .../BreCal/validators/validation_rules.py | 3 + .../test_validation_rule_functions.py | 8 ++ 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index 81f53f3..9e51eaf 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -7,6 +7,43 @@ from BreCal.validators.time_logic import TimeLogic from BreCal.database.enums import StatusFlags #from BreCal.validators.schema_validation import validation_state_and_validation_name +# a human interpretable dictionary for error messages. In this case, the English language is preferred +error_message_dict = { + # 0001 A-M + "validation_rule_fct_missing_time_agency_berth_eta":"The shipcall arrives in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of arrival (ETA) {Rule #0001A}", # A + "validation_rule_fct_missing_time_agency_berth_etd":"The shipcall departs in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of departure (ETD) {Rule #0001B}", # B + "validation_rule_fct_missing_time_mooring_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of arrival (ETA) {Rule #0001C}", # C + "validation_rule_fct_missing_time_mooring_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of departure (ETD) {Rule #0001D}", # D + "validation_rule_fct_missing_time_portadministration_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of arrival (ETA) {Rule #0001F}", # F + "validation_rule_fct_missing_time_portadministration_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of departure (ETD) {Rule #0001G}", # G + "validation_rule_fct_missing_time_pilot_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of arrival (ETA) {Rule #0001H}", # H + "validation_rule_fct_missing_time_pilot_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of departure (ETD) {Rule #0001I}", # I + "validation_rule_fct_missing_time_tug_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of arrival (ETA) {Rule #0001J}", # J + "validation_rule_fct_missing_time_tug_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of departure (ETD) {Rule #0001K}", # K + "validation_rule_fct_missing_time_terminal_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of arrival (ETA) {Rule #0001L}", # L + "validation_rule_fct_missing_time_terminal_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of departure (ETD) {Rule #0001M}", # M + + # 0002 A+B+C + "validation_rule_fct_shipcall_incoming_participants_disagree_on_eta":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of arrival (ETA) {Rule #0002A}", + "validation_rule_fct_shipcall_outgoing_participants_disagree_on_etd":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of departure (ETD) {Rule #0002B}", + "validation_rule_fct_shipcall_shifting_participants_disagree_on_eta_or_etd":"There are deviating times between agency, mooring, port authority, pilot and tug for ETA and ETD {Rule #0002C}", + + # 0003 A+B + "validation_rule_fct_eta_time_not_in_operation_window":"The estimated time of arrival will be AFTER the planned start of operations. {Rule #0003A}", + "validation_rule_fct_etd_time_not_in_operation_window":"The estimated time of departure is supposed to be AFTER the planned end of operations. {Rule #0003D}", + + # 0004 A+B + "validation_rule_fct_eta_time_not_in_tidal_window":"The tidal window does not fit to the agency's estimated time of arrival (ETA) {Rule #0004A}", + "validation_rule_fct_etd_time_not_in_tidal_window":"The tidal window does not fit to the agency's estimated time of departure (ETD) {Rule #0004B}", + + # 0005 A+B + "validation_rule_fct_too_many_identical_eta_times":"There are more than three ships with the same planned time of arrival (ETA) {Rule #0005A}", + "validation_rule_fct_too_many_identical_etd_times":"There are more than three ships with the same planned time of departure (ETD) {Rule #0005B}", + + # 0006 A+B + "validation_rule_fct_agency_and_terminal_berth_id_disagreement":"Agency and Terminal are planning with different berths (the berth_id deviates). {Rule #0006A}", + "validation_rule_fct_agency_and_terminal_pier_side_disagreement":"Agency and Terminal are planning with different pier sides (the pier_side deviates). {Rule #0006B}", +} class ValidationRuleBaseFunctions(): """ @@ -16,6 +53,16 @@ class ValidationRuleBaseFunctions(): def __init__(self, sql_handler): self.sql_handler = sql_handler self.time_logic = TimeLogic() + self.error_message_dict = error_message_dict + + def describe_error_message(self, key)->str: + """ + Takes any error message, which typically is the validation rule's function name and returns a description of the error. + In case that the error code is not defined in self.error_message_dict, return the cryptic error code instead + + returns: string + """ + return self.error_message_dict.get(key,key) def check_time_delta_violation_query_time_to_now(self, query_time:pd.Timestamp, key_time:pd.Timestamp, threshold:float)->bool: """ @@ -36,7 +83,7 @@ class ValidationRuleBaseFunctions(): threshold: threshold where a time difference becomes crucial. When the delta is below the threshold, a violation might occur """ # rule is not applicable -> return 'GREEN' - if key_time is not None: + if self.check_is_not_a_time_or_is_none(key_time) or self.check_is_not_a_time_or_is_none(query_time): return False # otherwise, this rule applies and the difference between 'now' and the query time is measured @@ -76,7 +123,7 @@ class ValidationRuleBaseFunctions(): df_times = df_times.loc[df_times["participant_type"].isin(participant_types),:] # exclude missing entries and consider only pd.Timestamp entries (which ignores pd.NaT/null entries) - estimated_times = [type(time_) for time_ in df_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] # df_times = df_times.loc[~df_times[query].isnull(),:] + estimated_times = [time_ for time_ in df_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] # df_times = df_times.loc[~df_times[query].isnull(),:] # apply rounding. For example, the agreement of different participants may be required to match minute-wise # '15min' rounds to 'every 15 minutes'. E.g., '2023-09-22 08:18:49' becomes '2023-09-22 08:15:00' @@ -112,6 +159,11 @@ class ValidationRuleBaseFunctions(): violation_state = np.any(np.greater(counts, maximum_threshold)) return violation_state + def check_is_not_a_time_or_is_none(self, value)->bool: + """checks, if a provided value is either None or NaT""" + return (value is None) or (value is pd.NaT) + + class ValidationRuleFunctions(ValidationRuleBaseFunctions): """ @@ -466,12 +518,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): def validation_rule_fct_missing_time_terminal_berth_etd(self, shipcall, df_times, *args, **kwargs): """ - Code: #0001-K + Code: #0001-M Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs - 0001-K: + 0001-M: - Checks, if times_terminal.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ @@ -595,7 +647,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT): + if self.check_is_not_a_time_or_is_none(times_terminal.operations_start) or self.check_is_not_a_time_or_is_none(times_agency.eta_berth): return (StatusFlags.GREEN, None) # check, whether the start of operations is AFTER the estimated arrival time @@ -607,7 +659,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): else: return (StatusFlags.GREEN, None) - def validation_rule_fct_eta_time_not_in_operation_window(self, shipcall, df_times, *args, **kwargs): + def validation_rule_fct_etd_time_not_in_operation_window(self, shipcall, df_times, *args, **kwargs): """ Code: #0003-B Type: Local Rule @@ -624,7 +676,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT): + if self.check_is_not_a_time_or_is_none(times_terminal.operations_end) or self.check_is_not_a_time_or_is_none(times_agency.etd_berth): return (StatusFlags.GREEN, None) # check, whether the end of operations is AFTER the estimated departure time @@ -651,11 +703,11 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) # requirements: tidal window (from & to) is filled in - if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT): + if self.check_is_not_a_time_or_is_none(shipcall.tidal_window_from) or self.check_is_not_a_time_or_is_none(shipcall.tidal_window_to) or self.check_is_not_a_time_or_is_none(times_agency.eta_berth): # 202310310: note: this should check times_agency, shouldn't it? return (StatusFlags.GREEN, None) # check, whether the query time is between start & end time - # a violation is observed, when the is NOT between start & end + # a violation is observed, when the time is NOT between start & end violation_state = not self.time_logic.time_inbetween(query_time=times_agency.eta_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to) if violation_state: @@ -679,11 +731,11 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) # requirements: tidal window (from & to) is filled in - if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.etd_berth is pd.NaT): + if self.check_is_not_a_time_or_is_none(shipcall.tidal_window_from) or self.check_is_not_a_time_or_is_none(shipcall.tidal_window_to) or self.check_is_not_a_time_or_is_none(times_agency.etd_berth): # 202310310: note: this should check times_agency, shouldn't it? return (StatusFlags.GREEN, None) # check, whether the query time is between start & end time - # a violation is observed, when the is NOT between start & end + # a violation is observed, when the time is NOT between start & end violation_state = not self.time_logic.time_inbetween(query_time=times_agency.etd_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to) if violation_state: @@ -698,6 +750,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): Type: Global Rule Description: this validation rule checks, whether there are too many shipcalls with identical ETA times. """ + # check, if the header is filled in (agency) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + return (StatusFlags.GREEN, None) + # when ANY of the unique values exceeds the threshold, a violation is observed query = "eta_berth" violation_state = self.check_unique_shipcall_counts(query, rounding=rounding, maximum_threshold=maximum_threshold) @@ -714,6 +770,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): Type: Global Rule Description: this validation rule checks, whether there are too many shipcalls with identical ETD times. """ + # check, if the header is filled in (agency) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + return (StatusFlags.GREEN, None) + # when ANY of the unique values exceeds the threshold, a violation is observed query = "etd_berth" violation_state = self.check_unique_shipcall_counts(query, rounding=rounding, maximum_threshold=maximum_threshold) @@ -737,6 +797,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) + # when one of the two values is null, the state is GREEN + if (times_agency.berth_id is None) or (times_terminal.berth_id is None): + return (StatusFlags.GREEN, None) + violation_state = times_agency.berth_id!=times_terminal.berth_id if violation_state: @@ -758,6 +822,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) + # when one of the two values is null, the state is GREEN + if (times_agency.pier_side is None) or (times_terminal.pier_side is None): + return (StatusFlags.GREEN, None) + violation_state = times_agency.pier_side!=times_terminal.pier_side if violation_state: diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index 7e7fcef..2753446 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -41,6 +41,9 @@ class ValidationRules(ValidationRuleFunctions): # filter out all 'None' results, which indicate that no violation occured. evaluation_results = [evaluation_result for evaluation_result in evaluation_results if evaluation_result[1] is not None] + # 'translate' all error codes into readable, human-understandable format. + evaluation_results = [(state, self.describe_error_message(msg)) for (state, msg) in evaluation_results] + """ # deprecated # check, if ANY of the evaluation results (evaluation_state) is larger than the .GREEN state. This means, that .YELLOW and .RED # would return 'True'. Numpy arrays and functions are used to accelerate the comparison. diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index 00554b8..671458b 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -156,4 +156,12 @@ def test_validation_rule_fct_agency_and_terminal_berth_id_agreement(build_sql_pr +def test_all_validation_rule_fcts_have_a_description(): + from BreCal.validators.validation_rule_functions import error_message_dict, ValidationRuleFunctions + import types + vr = ValidationRuleFunctions(sql_handler=None) + assert all( + [mthd_ in list(error_message_dict.keys()) for mthd_ in dir(vr) if ('validation_rule_fct' in mthd_)] + ), f"one of the validation_rule_fcts is currently not defined in the error_message_dict and will create cryptic descriptions! Please add it to the error_message_dict BreCal.validators.validation_rule_functions" + return