From 0a9ba0b381f8b9525ad988a810daf5fb0344ff85 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Thu, 30 Nov 2023 17:34:44 +0100 Subject: [PATCH] more concise evaluation messages for 0001. Adding newlines (works on Windows) when multiple evaluation messages are shown. Properly adding the ShipcallType filters for each rule (whether incoming, outgoing or shifting). Added a regular expression to abbreviate an evaluation message when 512 characters are exceeded. --- .../validators/validation_rule_functions.py | 72 +++++++++++++++---- .../BreCal/validators/validation_rules.py | 19 ++++- .../test_validation_rule_functions.py | 13 ++++ 3 files changed, 91 insertions(+), 13 deletions(-) diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index a7993cd..92bf24e 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -10,18 +10,18 @@ from BreCal.database.enums import StatusFlags # a human interpretable dictionary for error messages. In this case, the English language is preferred error_message_dict = { # 0001 A-M - "validation_rule_fct_missing_time_agency_berth_eta":"The shipcall arrives in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of arrival (ETA) {Rule #0001A}", # A - "validation_rule_fct_missing_time_agency_berth_etd":"The shipcall departs in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of departure (ETD) {Rule #0001B}", # B - "validation_rule_fct_missing_time_mooring_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of arrival (ETA) {Rule #0001C}", # C - "validation_rule_fct_missing_time_mooring_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of departure (ETD) {Rule #0001D}", # D - "validation_rule_fct_missing_time_portadministration_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of arrival (ETA) {Rule #0001F}", # F - "validation_rule_fct_missing_time_portadministration_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of departure (ETD) {Rule #0001G}", # G - "validation_rule_fct_missing_time_pilot_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of arrival (ETA) {Rule #0001H}", # H - "validation_rule_fct_missing_time_pilot_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of departure (ETD) {Rule #0001I}", # I - "validation_rule_fct_missing_time_tug_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of arrival (ETA) {Rule #0001J}", # J - "validation_rule_fct_missing_time_tug_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of departure (ETD) {Rule #0001K}", # K - "validation_rule_fct_missing_time_terminal_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of arrival (ETA) {Rule #0001L}", # L - "validation_rule_fct_missing_time_terminal_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of departure (ETD) {Rule #0001M}", # M + "validation_rule_fct_missing_time_agency_berth_eta":"Shipcall arrives soon (<20 hours). The agency did not provide a time yet (ETA) {Rule #0001A}", # A + "validation_rule_fct_missing_time_agency_berth_etd":"Shipcall departs soon (<20 hours). The agency did not provide a time yet (ETD) {Rule #0001B}", # B + "validation_rule_fct_missing_time_mooring_berth_eta":"Shipcall arrives soon (<16 hours). The mooring did not provide a time yet (ETA) {Rule #0001C}", # C + "validation_rule_fct_missing_time_mooring_berth_etd":"Shipcall departs soon (<16 hours). The mooring did not provide a time yet (ETD) {Rule #0001D}", # D + "validation_rule_fct_missing_time_portadministration_berth_eta":"Shipcall arrives soon (<16 hours). The port administration did not provide a time yet (ETA) {Rule #0001F}", # F + "validation_rule_fct_missing_time_portadministration_berth_etd":"Shipcall departs soon (<20 hours). The port administration did not provide a time yet (ETD) {Rule #0001G}", # G + "validation_rule_fct_missing_time_pilot_berth_eta":"Shipcall arrives soon (<16 hours). The pilot did not provide a time yet (ETA) {Rule #0001H}", # H + "validation_rule_fct_missing_time_pilot_berth_etd":"Shipcall departs soon (<20 hours). The pilot did not provide a time yet (ETD) {Rule #0001I}", # I + "validation_rule_fct_missing_time_tug_berth_eta":"Shipcall arrives soon (<16 hours). The tugs did not provide a time yet (ETA) {Rule #0001J}", # J + "validation_rule_fct_missing_time_tug_berth_etd":"Shipcall departs soon (<20 hours). The tugs did not provide a time yet (ETD) {Rule #0001K}", # K + "validation_rule_fct_missing_time_terminal_berth_eta":"Shipcall arrives soon (<16 hours). The terminal did not provide a time yet (ETA) {Rule #0001L}", # L + "validation_rule_fct_missing_time_terminal_berth_etd":"Shipcall departs soon (<20 hours). The terminal did not provide a time yet (ETD) {Rule #0001M}", # M # 0002 A+B+C "validation_rule_fct_shipcall_incoming_participants_disagree_on_eta":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of arrival (ETA) {Rule #0002A}", @@ -210,6 +210,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.eta_berth is filled in. - Measures the difference between 'now' and 'shipcall.eta'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) if unassigned: @@ -239,6 +242,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.etd_berth is filled in. - Measures the difference between 'now' and 'shipcall.etd'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) if unassigned: @@ -268,6 +274,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) if unassigned: @@ -299,6 +308,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) if unassigned: @@ -330,6 +342,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) if unassigned: @@ -361,6 +376,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) if unassigned: @@ -393,6 +411,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) if unassigned: @@ -424,6 +445,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) if unassigned: @@ -455,6 +479,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) if unassigned: @@ -486,6 +513,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) if unassigned: @@ -517,6 +547,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_start is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) if unassigned: @@ -548,6 +581,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_end is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) if unassigned: @@ -660,6 +696,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency & terminal) if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable @@ -692,6 +731,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency & terminal) if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable @@ -724,6 +766,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return self.get_no_violation_default_output() @@ -752,6 +797,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return self.get_no_violation_default_output() diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index cabe5ec..aa2ef22 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -1,4 +1,5 @@ import copy +import re import numpy as np import pandas as pd from BreCal.database.enums import StatusFlags @@ -75,11 +76,27 @@ class ValidationRules(ValidationRuleFunctions): # unbundle individual results. evaluation_state becomes an integer, violation evaluation_state = [StatusFlags(res[0]).value for res in results] - violations = [",".join(res[1]) if len(res[1])>0 else None for res in results] + violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results] + violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] shipcall_df.loc[:,"evaluation"] = evaluation_state shipcall_df.loc[:,"evaluation_message"] = violations return shipcall_df + + def concise_evaluation_message_if_too_long(self, violation): + """ + when many validation rules are violated at once, the resulting evaluation message may exceed 512 characters (which the mysql database allows) + in these cases, use a regular expression to provide a concise message, where the 'concise' description is only the list of violated rools + """ + if violation is None: + return violation + + if len(violation)>=512: + concise = re.findall(r'{(.*?)\}', violation) + + # e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K'] + violation = f"Evaluation message too long. Violated Rules: {concise}" + return violation def determine_validation_state(self) -> str: """ diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index 5c4e3a0..2593e85 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -211,6 +211,7 @@ def test_validation_rule_fct_missing_time_agency_berth_etd__shipcall_etd_is_unde df_times = get_df_times(shipcall) # the shipcall etd is 'soon' + shipcall.type = ShipcallType.OUTGOING.value shipcall.etd = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.AGENCY-10) # set times agency to be undetermined @@ -278,6 +279,7 @@ def test_validation_rule_fct_missing_time_mooring_berth_etd__shipcall_soon_but_p vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -352,6 +354,7 @@ def test_validation_rule_fct_missing_time_portadministration_berth_etd__shipcall vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -422,6 +425,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -457,6 +461,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -494,6 +499,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -534,6 +540,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__agency_and_pilot_assi vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -569,6 +576,7 @@ def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_parti vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.INCOMING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -606,6 +614,7 @@ def test_validation_rule_fct_missing_time_tug_berth_etd__shipcall_soon_but_parti vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -680,6 +689,7 @@ def test_validation_rule_fct_missing_time_terminal_berth_etd__shipcall_soon_but_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -915,6 +925,7 @@ def test_validation_rule_fct_etd_time_not_in_operation_window__times_dont_match( """0003-B validation_rule_fct_etd_time_not_in_operation_window""" vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now() # reference time for easier readability @@ -936,6 +947,7 @@ def test_validation_rule_fct_eta_time_not_in_operation_window_and_validation_rul vr = build_sql_proxy_connection['vr'] import random shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now() @@ -1031,6 +1043,7 @@ def test_validation_rule_fct_etd_time_not_in_tidal_window__etd_outside_tidal_win """0004-B validation_rule_fct_etd_time_not_in_tidal_window""" vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now()