import inspect import types from BreCal.database.enums import ParticipantType, ShipcallType, ParticipantwiseTimeDelta import numpy as np import pandas as pd import datetime from BreCal.validators.time_logic import TimeLogic from BreCal.database.enums import StatusFlags #from BreCal.validators.schema_validation import validation_state_and_validation_name # a human interpretable dictionary for error messages. In this case, the English language is preferred error_message_dict = { # 0001 A-M "validation_rule_fct_missing_time_agency_berth_eta":"Shipcall arrives soon (<20 hours). The agency did not provide a time yet (ETA) {Rule #0001A}", # A "validation_rule_fct_missing_time_agency_berth_etd":"Shipcall departs soon (<20 hours). The agency did not provide a time yet (ETD) {Rule #0001B}", # B "validation_rule_fct_missing_time_mooring_berth_eta":"Shipcall arrives soon (<16 hours). The mooring did not provide a time yet (ETA) {Rule #0001C}", # C "validation_rule_fct_missing_time_mooring_berth_etd":"Shipcall departs soon (<16 hours). The mooring did not provide a time yet (ETD) {Rule #0001D}", # D "validation_rule_fct_missing_time_portadministration_berth_eta":"Shipcall arrives soon (<16 hours). The port administration did not provide a time yet (ETA) {Rule #0001F}", # F "validation_rule_fct_missing_time_portadministration_berth_etd":"Shipcall departs soon (<16 hours). The port administration did not provide a time yet (ETD) {Rule #0001G}", # G "validation_rule_fct_missing_time_pilot_berth_eta":"Shipcall arrives soon (<16 hours). The pilot did not provide a time yet (ETA) {Rule #0001H}", # H "validation_rule_fct_missing_time_pilot_berth_etd":"Shipcall departs soon (<16 hours). The pilot did not provide a time yet (ETD) {Rule #0001I}", # I "validation_rule_fct_missing_time_tug_berth_eta":"Shipcall arrives soon (<16 hours). The tugs did not provide a time yet (ETA) {Rule #0001J}", # J "validation_rule_fct_missing_time_tug_berth_etd":"Shipcall departs soon (<16 hours). The tugs did not provide a time yet (ETD) {Rule #0001K}", # K "validation_rule_fct_missing_time_terminal_berth_eta":"Shipcall arrives soon (<16 hours). The terminal did not provide a time yet (ETA) {Rule #0001L}", # L "validation_rule_fct_missing_time_terminal_berth_etd":"Shipcall departs soon (<16 hours). The terminal did not provide a time yet (ETD) {Rule #0001M}", # M # 0002 A+B+C "validation_rule_fct_shipcall_incoming_participants_disagree_on_eta":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of arrival (ETA) {Rule #0002A}", "validation_rule_fct_shipcall_outgoing_participants_disagree_on_etd":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of departure (ETD) {Rule #0002B}", "validation_rule_fct_shipcall_shifting_participants_disagree_on_etd":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of departure (ETD) {Rule #0002C}", # 0003 A+B "validation_rule_fct_eta_time_not_in_operation_window":"The estimated time of arrival will be AFTER the planned start of operations. {Rule #0003A}", "validation_rule_fct_etd_time_not_in_operation_window":"The estimated time of departure is supposed to be AFTER the planned end of operations. {Rule #0003B}", # 0004 A+B "validation_rule_fct_eta_time_not_in_tidal_window":"The tidal window does not fit to the agency's estimated time of arrival (ETA) {Rule #0004A}", "validation_rule_fct_etd_time_not_in_tidal_window":"The tidal window does not fit to the agency's estimated time of departure (ETD) {Rule #0004B}", # 0005 A+B "validation_rule_fct_too_many_identical_eta_times":"More than three shipcalls are planned at the same time as the defined ETA {Rule #0005A}", "validation_rule_fct_too_many_identical_etd_times":"More than three shipcalls are planned at the same time as the defined ETD {Rule #0005B}", # 0006 A+B "validation_rule_fct_agency_and_terminal_berth_id_disagreement":"Agency and Terminal are planning with different berths (the berth_id deviates). {Rule #0006A}", "validation_rule_fct_agency_and_terminal_pier_side_disagreement":"Agency and Terminal are planning with different pier sides (the pier_side deviates). {Rule #0006B}", } class ValidationRuleBaseFunctions(): """ Base object with individual functions, which the {ValidationRuleFunctions}-child refers to. This parent class provides base functions and helps to restructure the code in a more comprehensible way. """ def __init__(self, sql_handler): self.sql_handler = sql_handler self.time_logic = TimeLogic() self.error_message_dict = error_message_dict # as of 23 dec. 2023 port authority validation is temporarily disabled self.ignore_port_administration_flag = True # flag to disable all port administration validation rules self.ignore_terminal_flag = True # flag to disable Terminal validation rules 0001-L & 0001-M def describe_error_message(self, key)->str: """ Takes any error message, which typically is the validation rule's function name and returns a description of the error. In case that the error code is not defined in self.error_message_dict, return the cryptic error code instead returns: string """ return self.error_message_dict.get(key,key) def get_no_violation_default_output(self): """return the default output of a validation function with no validation: a tuple of (GREEN state, None)""" return (StatusFlags.GREEN, None) def check_if_header_exists(self, df_times:pd.DataFrame, participant_type:ParticipantType)->bool: """ Given a pandas DataFrame, which contains times entries for a specific shipcall id, this function checks, whether one of the times entries belongs to the requested ParticipantType. returns bool """ # empty DataFrames form a special case, as they might miss the 'participant_type' column. if len(df_times)==0: return False return participant_type in df_times.loc[:,"participant_type"].values def check_time_delta_violation_query_time_to_now(self, query_time:pd.Timestamp, key_time:pd.Timestamp, threshold:float)->bool: """ # base function for all validation rules in the group {0001} A-L measures the time between NOW and query_time. When the query_time lays in the past, the delta is negative when the query_time lays in the future, the delta is positive returns a violation state depending on whether the delta is Violation, if: 0 >= delta <= threshold When the key time is defined (not None), there is no violation. Returns False options: query_time: will be used to measure the time difference of 'now' until the query time key_time: will be used to check, whether the respective key already has a value threshold: threshold where a time difference becomes crucial. When the delta is below the threshold, a violation might occur (minutes) """ # rule is not applicable -> return 'GREEN' # rule is only applicable, when 'key_time' is not defined (neither None, nor pd.NaT) if (key_time is not None) and (key_time is not pd.NaT): return False # when query_time is not valid, the rule cannot be applied if self.check_is_not_a_time_or_is_none(query_time): return False # otherwise, this rule applies and the difference between 'now' and the query time is measured delta = self.time_logic.time_delta_from_now_to_tgt(tgt_time=query_time, unit="m") # a violation occurs, when the delta (in minutes) exceeds the specified threshold of a participant # Violation, if delta <= threshold violation_state = (delta<=threshold) return violation_state def check_participants_agree_on_estimated_time(self, shipcall, query, df_times, applicable_shipcall_type, threshold:int=3660)->bool: """ # base function for all validation rules in the group {0002} A-C compares, whether the participants agree on the estimated time (of arrival or departure), depending on whether the shipcall type is incoming, outgoing or shifting. No violations are observed, when - the shipcall belongs to a different type than the rule expects - there are no matching times for the provided {query} (e.g., "eta_berth") This method computes the absolute time difference between all time entries. A threshold (in seconds) is used to identify, when the time differences are so large, that participants essentially disagree on the times. This circumvents previous instabilities, which stem from rounding the pd.Timestamp elements. options: threshold: integer. Determines the threshold in seconds, when two Timestamps differ 'too much' returns: violation_state (bool) """ # shipcall type filter: consider only shipcalls, where the type matches if shipcall.type != applicable_shipcall_type.value: violation_state = False return violation_state # filter by participant types of interest (agency, mooring, portauthority/administration, pilot, tug) if not self.ignore_port_administration_flag: participant_types = [ParticipantType.AGENCY.value, ParticipantType.MOORING.value, ParticipantType.PORT_ADMINISTRATION.value, ParticipantType.PILOT.value, ParticipantType.TUG.value] else: participant_types = [ParticipantType.AGENCY.value, ParticipantType.MOORING.value, ParticipantType.PILOT.value, ParticipantType.TUG.value] agency_times = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value,:] if len(agency_times)==0: violation_state = False return violation_state df_times = df_times.loc[df_times["participant_type"].isin(participant_types),:] # for the given query, e.g., 'eta_berth', sample all times from the pandas DataFrame # exclude missing entries and consider only pd.Timestamp entries (which ignores pd.NaT/null entries) estimated_times = [time_ for time_ in df_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] # df_times = df_times.loc[~df_times[query].isnull(),:] # when there are no entries left (no entries are provided), skip if len(estimated_times)==0: violation_state = False return violation_state # for the given query, e.g., 'eta_berth', sample all times from the pandas DataFrame estimated_times = [time_ for time_ in df_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] # consider only pandas Timestamp objects # measure the time difference between all pairs. # for each pair of times, the absolute timedifference in seconds (float) is measured time_absolute_differences = [[abs(time_.to_pydatetime()-time__.to_pydatetime()).total_seconds() for j_, time__ in enumerate(estimated_times) if j_ != i_] for i_, time_ in enumerate(estimated_times)] # list of lists: for each element in the list, create a boolean that indicates, whether the threshold is exceeded time_difference_exceeds_threshold = [[time__ > threshold for time__ in time_] for time_ in time_absolute_differences] # list of booleans for each time entry separately time_difference_exceeds_threshold = [any(time_) for time_ in time_difference_exceeds_threshold] # if *any* of these entries exceeds the threshold, the times are too distinct. In those case, a rule violation occurs violation_state = any(time_difference_exceeds_threshold) # this (previous) solution compares times to the reference (agency) time and checks if the difference is greater than 15 minutes # agency_time = [time_ for time_ in agency_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] # violation_state = ((np.max(estimated_times) - agency_time[0]) > pd.Timedelta("15min")) or ((agency_time[0] - np.min(estimated_times)) > pd.Timedelta("15min")) # this solution to the rule compares all times to each other. When there is a total difference of more than 15 minutes, a violation occurs # Consequently, it treats all times as equally important # difference = np.max(estimated_times) - np.min(estimated_times) # violation_state = difference > pd.Timedelta("15min") # this solution clamps the times to 15 minute intervals and compares these values. When there is a single time difference, a violation occurs # the drawback is that in some cases if there is a minimal difference say of 1 minute (:22 and :23 minutes after the hour) the violation is # triggered even though the times are very close to each other # apply rounding. For example, the agreement of different participants may be required to match minute-wise # '15min' rounds to 'every 15 minutes'. E.g., '2023-09-22 08:18:49' becomes '2023-09-22 08:15:00' # estimated_times = [time_.round("15min") for time_ in estimated_times] # there should only be one eta_berth, when all participants have provided the same time # this equates to the same criteria as checking, whether # times_agency.eta_berth==times_mooring.eta_berth==times_portadministration.eta_berth==times_pilot.eta_berth==times_tug.eta_berth # n_unique_times = len(np.unique(estimated_times)) # violation_state = n_unique_times!=1 return violation_state def check_unique_shipcall_counts(self, query:str, times_agency:pd.DataFrame, rounding="min", maximum_threshold=3, all_times_agency=None)->bool: """ # base function for all validation rules in the group {0005} A&B compares how many unique times are found for the provided {query} (e.g., "eta_berth") This function rounds the results, counts the unique values and returns a boolean state, whether the {maximum_threshold} is exceeded """ # filter the df: keep only times_agents # filter out all NaN and NaT entries if all_times_agency is None: all_times_agency = self.sql_handler.get_times_for_agency(non_null_column=query) # get values and optionally round the values (internally) counts = self.sql_handler.get_unique_ship_counts(all_df_times=all_times_agency, times_agency=times_agency, query=query, rounding=rounding, maximum_threshold=maximum_threshold) # when ANY of the unique values exceeds the threshold, a violation is observed violation_state = np.any(np.greater(counts, maximum_threshold)) return violation_state def check_is_not_a_time_or_is_none(self, value)->bool: """checks, if a provided value is either None or NaT""" return (value is None) or (value is pd.NaT) class ValidationRuleFunctions(ValidationRuleBaseFunctions): """ an accumulation object that makes sure, that any validation rule is translated to a function with default naming convention and return types. Each function should return a ValidationRuleState enumeration object and a description string to which validation rule the result belongs. These are returned as tuples (ValidationRuleState, validation_name) Each rule should have the same input arguments (self, shipcall, df_times, *args, **kwargs) The object makes heavy use of calls from an SQLHandler object, which provides functions for dataframe access and filtering. each validation_name is generated by calling the function inside a method validation_name = inspect.currentframe().f_code.co_name # validation_name then returns the name of the method from where 'currentframe()' was called. # example: #def validation_rule_fct_example(self, shipcall, df_times): #validation_name = inspect.currentframe().f_code.co_name #return (ValidationRuleState.NONE, validation_name) """ def __init__(self, sql_handler): super().__init__(sql_handler) return def get_validation_rule_functions(self): """return a list of all methods in this object, which are all validation rule functions.""" return [self.__getattribute__(mthd_) for mthd_ in dir(self) if ('validation_rule_fct' in mthd_) and (isinstance(self.__getattribute__(mthd_), types.MethodType))] def validation_rule_fct_missing_time_agency_berth_eta(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-A Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-A: - Checks, if times_agency.eta_berth is filled in. - Measures the difference between 'now' and 'shipcall.eta'. """ if not shipcall.type in [ShipcallType.INCOMING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.eta key_time = times_agency.eta_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_agency_berth_eta" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_agency_berth_etd(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-B Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-B: - Checks, if times_agency.etd_berth is filled in. - Measures the difference between 'now' and 'shipcall.etd'. """ if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.etd key_time = times_agency.etd_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_agency_berth_etd" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_mooring_berth_eta(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-C Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-C: - Checks, if times_mooring.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ if not shipcall.type in [ShipcallType.INCOMING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) query_time = times_agency.eta_berth if times_agency is not None else None key_time = times_mooring.eta_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_mooring_berth_eta" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_mooring_berth_etd(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-D Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-D: - Checks, if times_mooring.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) query_time = times_agency.etd_berth if times_agency is not None else None key_time = times_mooring.etd_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_mooring_berth_etd" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_portadministration_berth_eta(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-F Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-F: - Checks, if times_port_administration.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ if self.ignore_port_administration_flag: return self.get_no_violation_default_output() if not shipcall.type in [ShipcallType.INCOMING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) query_time = times_agency.eta_berth if times_agency is not None else None key_time = times_port_administration.eta_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_portadministration_berth_eta" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_portadministration_berth_etd(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-G Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-G: - Checks, if times_port_administration.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ if self.ignore_port_administration_flag: return self.get_no_violation_default_output() if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time # when there are no times, the function returns None times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) query_time = times_agency.etd_berth if times_agency is not None else None key_time = times_port_administration.etd_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_portadministration_berth_etd" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_pilot_berth_eta(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-H Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-H: - Checks, if times_pilot.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ if not shipcall.type in [ShipcallType.INCOMING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) query_time = times_agency.eta_berth if times_agency is not None else None key_time = times_pilot.eta_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_pilot_berth_eta" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_pilot_berth_etd(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-I Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-I: - Checks, if times_pilot.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) query_time = times_agency.etd_berth if times_agency is not None else None key_time = times_pilot.etd_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_pilot_berth_etd" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_tug_berth_eta(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-J Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-J: - Checks, if times_tug.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ if not shipcall.type in [ShipcallType.INCOMING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) query_time = times_agency.eta_berth if times_agency is not None else None key_time = times_tug.eta_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_tug_berth_eta" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_tug_berth_etd(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-K Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-K: - Checks, if times_tug.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) query_time = times_agency.etd_berth if times_agency is not None else None key_time = times_tug.etd_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_tug_berth_etd" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_terminal_berth_eta(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-L Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-L: - Checks, if times_terminal.operations_start is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ if self.ignore_terminal_flag: # this feature flag may disable the validation rule for Terminals return self.get_no_violation_default_output() if not shipcall.type in [ShipcallType.INCOMING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) query_time = times_agency.eta_berth if times_agency is not None else None key_time = times_terminal.operations_start if times_terminal is not None else None # eta_berth does not exist in times_terminal! Instead, it is called operations_start threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_terminal_berth_eta" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_missing_time_terminal_berth_etd(self, shipcall, df_times, *args, **kwargs): """ Code: #0001-M Type: Local Rule Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below a certain threshold (e.g., 20 hours), a violation occurs 0001-M: - Checks, if times_terminal.operations_end is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ if self.ignore_terminal_flag: # this feature flag may disable the validation rule for Terminals return self.get_no_violation_default_output() if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) query_time = times_agency.etd_berth if times_agency is not None else None key_time = times_terminal.operations_end if times_terminal is not None else None # etd_berth does not exist in times_terminal! Instead, it is called operations_end threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) if violation_state: validation_name = "validation_rule_fct_missing_time_terminal_berth_etd" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_shipcall_incoming_participants_disagree_on_eta(self, shipcall, df_times, *args, **kwargs): """ Code: #0002-A Type: Local Rule Description: this validation checks, whether the participants expect different ETA times Filter: only applies to incoming shipcalls """ query = "eta_berth" violation_state = self.check_participants_agree_on_estimated_time( shipcall = shipcall, query=query, df_times=df_times, applicable_shipcall_type=ShipcallType.INCOMING ) if violation_state: validation_name = "validation_rule_fct_shipcall_incoming_participants_disagree_on_eta" return (StatusFlags.RED, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_shipcall_outgoing_participants_disagree_on_etd(self, shipcall, df_times, *args, **kwargs): """ Code: #0002-B Type: Local Rule Description: this validation checks, whether the participants expect different ETA times Filter: only applies to outgoing shipcalls """ query = "etd_berth" violation_state = self.check_participants_agree_on_estimated_time( shipcall = shipcall, query=query, df_times=df_times, applicable_shipcall_type=ShipcallType.OUTGOING ) if violation_state: validation_name = "validation_rule_fct_shipcall_outgoing_participants_disagree_on_etd" return (StatusFlags.RED, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_shipcall_shifting_participants_disagree_on_etd(self, shipcall, df_times, *args, **kwargs): """ Code: #0002-C Type: Local Rule Description: this validation checks, whether the participants expect different ETD times Filter: only applies to shifting shipcalls """ violation_state_etd = self.check_participants_agree_on_estimated_time( shipcall = shipcall, query="etd_berth", df_times=df_times, applicable_shipcall_type=ShipcallType.SHIFTING ) # apply 'etd_berth' # violation: if either 'etd_berth' is violated # functionally, this is the same as individually comparing all times for the participants # times_agency.etd_berth==times_mooring.etd_berth==times_portadministration.etd_berth==times_pilot.etd_berth==times_tug.etd_berth violation_state = (violation_state_etd) if violation_state: validation_name = "validation_rule_fct_shipcall_shifting_participants_disagree_on_etd" return (StatusFlags.RED, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_eta_time_not_in_operation_window(self, shipcall, df_times, *args, **kwargs): """ Code: #0003-A Type: Local Rule Description: this validation checks, whether the ETA time is between the provided operations window of the terminal query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ if self.ignore_terminal_flag: # this feature flag may disable the validation rule for Terminals return self.get_no_violation_default_output() if not shipcall.type in [ShipcallType.INCOMING.value]: return self.get_no_violation_default_output() # check, if the header is filled in (agency & terminal) if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): #if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: return self.get_no_violation_default_output() # rule not applicable # get agency & terminal times times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) if self.check_is_not_a_time_or_is_none(times_terminal.operations_start) or self.check_is_not_a_time_or_is_none(times_agency.eta_berth): return self.get_no_violation_default_output() # check, whether the end of operations is BEFORE the estimated arrival time if isinstance(times_terminal.operations_start, (pd.Timestamp, datetime.datetime)): times_terminal.operations_start = times_terminal.operations_start.replace(second=0, microsecond=0) if isinstance(times_agency.eta_berth, (pd.Timestamp, datetime.datetime)): times_agency.eta_berth = times_agency.eta_berth.replace(second=0, microsecond=0) violation_state = times_terminal.operations_start < times_agency.eta_berth if violation_state: validation_name = "validation_rule_fct_eta_time_not_in_operation_window" return (StatusFlags.RED, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_etd_time_not_in_operation_window(self, shipcall, df_times, *args, **kwargs): """ Code: #0003-B Type: Local Rule Description: this validation checks, whether the ETD time is between the provided operations window of the terminal query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ if self.ignore_terminal_flag: # this feature flag may disable the validation rule for Terminals return self.get_no_violation_default_output() if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() # check, if the header is filled in (agency & terminal) # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() # rule not applicable # if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): return self.get_no_violation_default_output() # rule not applicable # get agency & terminal times times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) if self.check_is_not_a_time_or_is_none(times_terminal.operations_end) or self.check_is_not_a_time_or_is_none(times_agency.etd_berth): return self.get_no_violation_default_output() # check, whether the end of operations is AFTER the estimated departure time if isinstance(times_terminal.operations_end, (pd.Timestamp, datetime.datetime)): times_terminal.operations_end = times_terminal.operations_end.replace(second=0, microsecond=0) if isinstance(times_agency.etd_berth, (pd.Timestamp, datetime.datetime)): times_agency.etd_berth = times_agency.etd_berth.replace(second=0, microsecond=0) violation_state = times_terminal.operations_end > times_agency.etd_berth if violation_state: validation_name = "validation_rule_fct_etd_time_not_in_operation_window" return (StatusFlags.RED, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_eta_time_not_in_tidal_window(self, shipcall, df_times, *args, **kwargs): """ Code: #0004-A Type: Local Rule Description: this validation checks, whether the ETA time is between the provided tidal window query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ if not shipcall.type in [ShipcallType.INCOMING.value]: return self.get_no_violation_default_output() # check, if the header is filled in (agency) # if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) # requirements: tidal window (from & to) is filled in if self.check_is_not_a_time_or_is_none(shipcall.tidal_window_from) or self.check_is_not_a_time_or_is_none(shipcall.tidal_window_to) or self.check_is_not_a_time_or_is_none(times_agency.eta_berth): # 202310310: note: this should check times_agency, shouldn't it? return self.get_no_violation_default_output() # check, whether the query time is between start & end time # a violation is observed, when the time is NOT between start & end violation_state = not self.time_logic.time_inbetween(query_time=times_agency.eta_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to) if violation_state: validation_name = "validation_rule_fct_eta_time_not_in_tidal_window" return (StatusFlags.RED, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_etd_time_not_in_tidal_window(self, shipcall, df_times, *args, **kwargs): """ Code: #0004-B Type: Local Rule Description: this validation checks, whether the ETD time is between the provided tidal window query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() # check, if the header is filled in (agency) # if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) # requirements: tidal window (from & to) is filled in if self.check_is_not_a_time_or_is_none(shipcall.tidal_window_from) or self.check_is_not_a_time_or_is_none(shipcall.tidal_window_to) or self.check_is_not_a_time_or_is_none(times_agency.etd_berth): # 202310310: note: this should check times_agency, shouldn't it? return self.get_no_violation_default_output() # check, whether the query time is between start & end time # a violation is observed, when the time is NOT between start & end violation_state = not self.time_logic.time_inbetween(query_time=times_agency.etd_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to) if violation_state: validation_name = "validation_rule_fct_etd_time_not_in_tidal_window" return (StatusFlags.RED, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_too_many_identical_eta_times(self, shipcall, df_times, rounding = "min", maximum_threshold = 3, all_times_agency=None, *args, **kwargs): """ Code: #0005-A Type: Global Rule Description: this validation rule checks, whether there are too many shipcalls with identical times to the query ETA. """ if all_times_agency is None: all_times_agency = self.sql_handler.get_times_for_agency(non_null_column="eta_berth") # check, if the header is filled in (agency) if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): # if len(times_agency) != 1: return self.get_no_violation_default_output() # get the agency's query time times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] query_time = times_agency.iloc[0].eta_berth # count the number of times, where a times entry is very close to the query time (uses an internal threshold, such as 15 minutes) counts = self.sql_handler.count_synchronous_shipcall_times(query_time, all_df_times=all_times_agency) violation_state = counts > maximum_threshold if violation_state: validation_name = "validation_rule_fct_too_many_identical_eta_times" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_too_many_identical_etd_times(self, shipcall, df_times, rounding = "min", maximum_threshold = 3, all_times_agency=None, *args, **kwargs): """ Code: #0005-B Type: Global Rule Description: this validation rule checks, whether there are too many shipcalls with identical times to the query ETD. """ if all_times_agency is None: all_times_agency = self.sql_handler.get_times_for_agency(non_null_column="etd_berth") # check, if the header is filled in (agency) if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): #if len(times_agency) != 1: return self.get_no_violation_default_output() # get the agency's query time times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] query_time = times_agency.iloc[0].etd_berth # count the number of times, where a times entry is very close to the query time (uses an internal threshold, such as 15 minutes) counts = self.sql_handler.count_synchronous_shipcall_times(query_time, all_df_times=all_times_agency) violation_state = counts > maximum_threshold if violation_state: validation_name = "validation_rule_fct_too_many_identical_etd_times" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_agency_and_terminal_berth_id_disagreement(self, shipcall, df_times, *args, **kwargs): """ Code: #0006-A Type: Local Rule Description: This validation rule checks, whether agency and terminal agree with their designated berth place by checking berth_id. """ # check, if the header is filled in (agency & terminal) # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() # rule not applicable # if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): return self.get_no_violation_default_output() # rule not applicable times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) # when one of the two values is null, the state is GREEN if (times_agency.berth_id is None) or (times_terminal.berth_id is None): return self.get_no_violation_default_output() # when one of the two values is null, the state is GREEN if (pd.isnull(times_agency.berth_id)) or (pd.isnull(times_terminal.berth_id)): return self.get_no_violation_default_output() if shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() # only incoming shipcalls matter. The other ones are not relevant for the berth selection violation_state = times_agency.berth_id!=times_terminal.berth_id if violation_state: validation_name = "validation_rule_fct_agency_and_terminal_berth_id_disagreement" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output() def validation_rule_fct_agency_and_terminal_pier_side_disagreement(self, shipcall, df_times, *args, **kwargs): """ Code: #0006-B Type: Local Rule Description: This validation rule checks, whether agency and terminal agree with their designated pier side by checking pier_side. """ # check, if the header is filled in (agency & terminal) # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() # rule not applicable # if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): return self.get_no_violation_default_output() # rule not applicable times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) # when one of the two values is null, the state is GREEN if (shipcall.pier_side is None) or (times_terminal.pier_side is None): return self.get_no_violation_default_output() # when one of the two values is null, the state is GREEN if (pd.isnull(shipcall.pier_side)) or (pd.isnull(times_terminal.pier_side)): return self.get_no_violation_default_output() # only incoming shipcalls matter. The other ones are not relevant for the pier_side selection if shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() violation_state = bool(shipcall.pier_side)!=bool(times_terminal.pier_side) if violation_state: validation_name = "validation_rule_fct_agency_and_terminal_pier_side_disagreement" return (StatusFlags.YELLOW, validation_name) else: return self.get_no_violation_default_output()