From 99b8610e6a692e607ce22976075d83a7dbb31cf8 Mon Sep 17 00:00:00 2001 From: max_metz Date: Thu, 12 Oct 2023 15:43:07 +0200 Subject: [PATCH] adding validation rules and time logic. Stubs are used to test the rules. Enum objects are located in brecal_utils/database/enums --- .../brecal_utils/database/__init__.py | 0 .../brecal_utils/database/enums.py | 38 + .../brecal_utils/database/sql_handler.py | 200 +++++ .../brecal_utils/stubs/shipcall.py | 6 +- .../brecal_utils/stubs/times_full.py | 41 +- .../brecal_utils/validators/time_logic.py | 34 +- .../validators/validation_rule_functions.py | 763 ++++++++++++++++++ .../validators/validation_rules.py | 93 ++- .../test_schema_validation_berth.py | 14 +- .../test_validation_rule_functions.py | 159 ++++ .../validators/test_validation_rule_state.py | 26 + src/server/BreCal/schemas/model.py | 4 - src/server/requirements.txt | 7 +- 13 files changed, 1338 insertions(+), 47 deletions(-) create mode 100644 src/lib_brecal_utils/brecal_utils/database/__init__.py create mode 100644 src/lib_brecal_utils/brecal_utils/database/enums.py create mode 100644 src/lib_brecal_utils/brecal_utils/database/sql_handler.py create mode 100644 src/lib_brecal_utils/brecal_utils/validators/validation_rule_functions.py create mode 100644 src/lib_brecal_utils/tests/validators/test_validation_rule_functions.py create mode 100644 src/lib_brecal_utils/tests/validators/test_validation_rule_state.py diff --git a/src/lib_brecal_utils/brecal_utils/database/__init__.py b/src/lib_brecal_utils/brecal_utils/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/lib_brecal_utils/brecal_utils/database/enums.py b/src/lib_brecal_utils/brecal_utils/database/enums.py new file mode 100644 index 0000000..e038643 --- /dev/null +++ b/src/lib_brecal_utils/brecal_utils/database/enums.py @@ -0,0 +1,38 @@ +from enum import Enum + +class ParticipantType(Enum): + """determines the type of a participant""" + NONE = 0 + BSMD = 1 + TERMINAL = 2 + PILOT = 4 + AGENCY = 8 + MOORING = 16 + PORT_ADMINISTRATION = 32 + TUG = 64 + +class ShipcallType(Enum): + """determines the type of a shipcall, as this changes the applicable validation rules""" + INCOMING = 1 + OUTGOING = 2 + SHIFTING = 3 + +class ParticipantwiseTimeDelta(): + """stores the time delta for every participant, which triggers the validation rules in the rule set '0001'""" + AGENCY = 1200.0 # 20 h * 60 min/h = 1200 min + MOORING = 960.0 # 16 h * 60 min/h = 960 min + PILOT = 960.0 # 16 h * 60 min/h = 960 min + PORT_ADMINISTRATION = 960.0 # 16 h * 60 min/h = 960 min + TUG = 960.0 # 16 h * 60 min/h = 960 min + TERMINAL = 960.0 # 16 h * 60 min/h = 960 min + +class StatusFlags(Enum): + """ + these enumerators ensure that each traffic light validation rule state corresponds to a value, which will be used in the ValidationRules object to identify + the necessity of notifications. + """ + NONE = 0 + GREEN = 1 + YELLOW = 2 + RED = 3 + diff --git a/src/lib_brecal_utils/brecal_utils/database/sql_handler.py b/src/lib_brecal_utils/brecal_utils/database/sql_handler.py new file mode 100644 index 0000000..d50bbbd --- /dev/null +++ b/src/lib_brecal_utils/brecal_utils/database/sql_handler.py @@ -0,0 +1,200 @@ +import numpy as np +import pandas as pd +import datetime +from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times +from brecal_utils.database.enums import ParticipantType + +class SQLHandler(): + """ + An object that reads SQL queries from the sql_connection and stores it in pandas DataFrames. The object can read all available tables + at once into memory, when providing 'read_all=True'. + + # #TODO_initialization: shipcall_tug_map, user_role_map & role_securable_map might be mapped to the respective dataframes + """ + def __init__(self, sql_connection, read_all=False): + self.sql_connection = sql_connection + self.all_schemas = self.get_all_schemas_from_mysql() + self.build_str_to_model_dict() + + if read_all: + self.read_all(self.all_schemas) + + def get_all_schemas_from_mysql(self): + with self.sql_connection.cursor(buffered=True) as cursor: + cursor.execute("SHOW TABLES") + schema = cursor.fetchall() + all_schemas = [schem[0] for schem in schema] + return all_schemas + + def build_str_to_model_dict(self): + """ + creates a simple dictionary, which maps a string to a data object + e.g., + 'ship'->BreCal.schemas.model.Ship object + """ + self.str_to_model_dict = { + "shipcall":Shipcall, "ship":Ship, "participant":Participant, "berth":Berth, "user":User, "times":Times + } + return + + def read_mysql_table_to_df(self, table_name:str): + """determine a {table_name}, which will be read from a mysql server. returns a pandas DataFrame with the respective data""" + df = pd.read_sql(sql=f"SELECT * FROM {table_name}", con=self.sql_connection) + return df + + def mysql_to_df(self, query): + """provide an arbitrary sql query that should be read from a mysql server {sql_connection}. returns a pandas DataFrame with the obtained data""" + df = pd.read_sql(query, self.sql_connection).convert_dtypes() + df = df.set_index('id', inplace=False) # avoid inplace updates, so the raw sql remains unchanged + return df + + def read_all(self, all_schemas): + # create a dictionary, which maps every mysql schema to pandas DataFrames + self.df_dict = self.build_full_mysql_df_dict(all_schemas) + + # update the 'participants' column in 'shipcall' + self.initialize_shipcall_participant_list() + return + + def build_full_mysql_df_dict(self, all_schemas): + """given a list of strings {all_schemas}, every schema will be read as individual pandas DataFrames to a dictionary with the respective keys. returns: dictionary {schema_name:pd.DataFrame}""" + mysql_df_dict = {} + for schem in all_schemas: + query = f"SELECT * FROM {schem}" + mysql_df_dict[schem] = self.mysql_to_df(query) + return mysql_df_dict + + def initialize_shipcall_participant_list(self): + """ + iteratively applies the .get_participants method to each shipcall. + the function updates the 'participants' column. + """ + # 1.) get all shipcalls + df = self.df_dict.get('shipcall') + + # 2.) iterate over each individual shipcall, obtain the id (pandas calls it 'name') + # and apply the 'get_participants' method, which returns a list + # if the shipcall_id exists, the list contains ids + # otherwise, return a blank list + df['participants'] = df.apply( + lambda x: self.get_participants(x.name), + axis=1) + return + + def standardize_model_str(self, model_str:str)->str: + """check if the 'model_str' is valid and apply lowercasing to the string""" + model_str = model_str.lower() + assert model_str in list(self.df_dict.keys()), f"cannot find the requested 'model_str' in mysql: {model_str}" + return model_str + + def get_data(self, id:int, model_str:str): + """ + obtains {id} from the respective mysql database and builds a data model from that. + the id should match the 'id'-column in the mysql schema. + returns: data model, such as Ship, Shipcall, etc. + + e.g., + data = self.get_data(0,"shipcall") + returns a Shipcall object + """ + model_str = self.standardize_model_str(model_str) + + df = self.df_dict.get(model_str) + data = self.df_loc_to_data_model(df, id, model_str) + return data + + def get_all(self, model_str:str)->list: + """ + given a model string (e.g., 'shipcall'), return a list of all + data models of that type from the sql + """ + model_str = self.standardize_model_str(model_str) + all_ids = self.df_dict.get(model_str).index + + all_data = [ + self.get_data(_aid, model_str) + for _aid in all_ids + ] + return all_data + + def df_loc_to_data_model(self, df, id, model_str, loc_type:str="loc"): + assert len(df)>0, f"empty dataframe" + + # get a pandas series from the dataframe + series = df.loc[id] if loc_type=="loc" else df.iloc[id] + + # get the respective data model object + data_model = self.str_to_model_dict.get(model_str,None) + assert data_model is not None, f"could not find the requested model_str: {model_str}" + + # build 'data' and fill the data model object + data = {**{'id':id}, **series.to_dict()} # 'id' must be added manually, as .to_dict does not contain the index, which was set with .set_index + data = data_model(**data) + return data + + def get_times_for_participant_type(self, df_times, participant_type:int): + filtered_series = df_times.loc[df_times["participant_type"]==participant_type] + assert len(filtered_series)<=1, f"found multiple results" + times = self.df_loc_to_data_model(filtered_series, id=0, model_str='times', loc_type="iloc") # use iloc! to retrieve the first result + return times + + def dataframe_to_data_model_list(self, df, model_str)->list: + model_str = self.standardize_model_str(model_str) + + all_ids = df.index + all_data = [ + self.df_loc_to_data_model(df, _aid, model_str) + for _aid in all_ids + ] + return all_data + + def get_participants(self, shipcall_id:id)->list: + """ + given a {shipcall_id}, obtain the respective list of participants. + when there are no participants, return a blank list + + returns: participant_id_list, where every element is an int + """ + df = self.df_dict.get("shipcall_participant_map") + df = df.set_index('shipcall_id', inplace=False) + + # the 'if' call is needed to ensure, that no Exception is raised, when the shipcall_id is not present in the df + participant_id_list = df.loc[shipcall_id, "participant_id"].to_list() if shipcall_id in list(df.index) else [] + return participant_id_list + + def get_times_of_shipcall(self, shipcall)->pd.DataFrame: + df_times = self.df_dict.get('times') # -> pd.DataFrame + df_times = df_times.loc[df_times["shipcall_id"]==shipcall.id] + return df_times + + def get_times_for_agency(self, non_null_column=None)->pd.DataFrame: + """ + options: + non_null_column: + None or str. If provided, the 'non_null_column'-column of the dataframe will be filtered, + so only entries with provided values are returned (filters all NaN and NaT entries) + """ + # get all times + df_times = self.df_dict.get('times') # -> pd.DataFrame + + # filter out all NaN and NaT entries + if non_null_column is not None: + df_times = df_times.loc[~df_times[non_null_column].isnull()] # NOT null filter + + # filter by the agency participant_type + times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] + return times_agency + + def filter_df_by_key_value(self, df, key, value)->pd.DataFrame: + return df.loc[df[key]==value] + + def get_unique_ship_counts(self, all_df_times:pd.DataFrame, query:str, rounding:str="min", maximum_threshold=3): + """given a dataframe of all agency times, get all unique ship counts, their values (datetime) and the string tags. returns a tuple (values,unique,counts)""" + # get values and optional: rounding + values = all_df_times.loc[:, query] + if rounding is not None: + values = values.dt.round(rounding) # e.g., 'min' + + unique, counts = np.unique(values, return_counts=True) + violation_state = np.any(np.greater(counts, maximum_threshold)) + return (values, unique, counts) diff --git a/src/lib_brecal_utils/brecal_utils/stubs/shipcall.py b/src/lib_brecal_utils/brecal_utils/stubs/shipcall.py index c59e711..6762692 100644 --- a/src/lib_brecal_utils/brecal_utils/stubs/shipcall.py +++ b/src/lib_brecal_utils/brecal_utils/stubs/shipcall.py @@ -39,10 +39,12 @@ def get_shipcall_simple(): moored_lock = False # de: 'Festmacherschleuse', en: 'moored lock' canceled = False + evaluation = None + evaluation_message = "" created = datetime.datetime.now() modified = created+datetime.timedelta(seconds=10) - participants = field(default_factory=[generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int()]) # list + participants = [generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int()] # field(default_factory=[generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int()]) # list shipcall = Shipcall( shipcall_id, @@ -68,6 +70,8 @@ def get_shipcall_simple(): anchored, moored_lock, canceled, + evaluation, + evaluation_message, created, modified, participants, diff --git a/src/lib_brecal_utils/brecal_utils/stubs/times_full.py b/src/lib_brecal_utils/brecal_utils/stubs/times_full.py index bddc050..8f9e0b1 100644 --- a/src/lib_brecal_utils/brecal_utils/stubs/times_full.py +++ b/src/lib_brecal_utils/brecal_utils/stubs/times_full.py @@ -34,25 +34,34 @@ def get_times_full_simple(): participant_id = generate_uuid1_int() shipcall_id = generate_uuid1_int() + berth_id = generate_uuid1_int() + berth_info = "" + pier_side = True + participant_type = None + created = datetime.datetime.now() modified = created+datetime.timedelta(seconds=10) times = Times( - times_id, - eta_berth, - eta_berth_fixed, - etd_berth, - etd_berth_fixed, - lock_time, - lock_time_fixed, - zone_entry, - zone_entry_fixed, - operations_start, - operations_end, - remarks, - participant_id, - shipcall_id, - created, - modified, + id=times_id, + eta_berth=eta_berth, + eta_berth_fixed=eta_berth_fixed, + etd_berth=etd_berth, + etd_berth_fixed=etd_berth_fixed, + lock_time=lock_time, + lock_time_fixed=lock_time_fixed, + zone_entry=zone_entry, + zone_entry_fixed=zone_entry_fixed, + operations_start=operations_start, + operations_end=operations_end, + remarks=remarks, + participant_id=participant_id, + berth_id=berth_id, + berth_info=berth_info, + pier_side=pier_side, + participant_type=participant_type, + shipcall_id=shipcall_id, + created=created, + modified=modified, ) return times diff --git a/src/lib_brecal_utils/brecal_utils/validators/time_logic.py b/src/lib_brecal_utils/brecal_utils/validators/time_logic.py index e77417b..b80384e 100644 --- a/src/lib_brecal_utils/brecal_utils/validators/time_logic.py +++ b/src/lib_brecal_utils/brecal_utils/validators/time_logic.py @@ -1,12 +1,42 @@ import datetime import numpy as np +import pandas as pd class TimeLogic(): def __init__(self): return - def time_delta(self, query_time, other_times): - return + def time_delta(self, src_time, tgt_time, unit:str="m"): + """ + in brief, this function measures tgt_time - src_time + + if the tgt_time is in the future, it is a positive value (tgt_time > src_time) + if the tgt_time is in the past, it is a negative value (tgt_time < src_time) + + returns the delta between tgt_time and src_time as a float of minutes (or the optionally provided unit) + + options: + unit: str, which defaults to 'm' (minutes). 'h' (hours) or 's' (seconds) are also common units. Determines the unit of the output time delta + """ + # convert np.datetime64 + if isinstance(src_time, pd.Timestamp): + src_time = src_time.to_datetime64() + + if isinstance(tgt_time, pd.Timestamp): + tgt_time = tgt_time.to_datetime64() + + if isinstance(src_time, datetime.datetime): + src_time = np.datetime64(src_time) + + if isinstance(tgt_time, datetime.datetime): + tgt_time = np.datetime64(tgt_time) + + delta = tgt_time - src_time + minute_delta = delta / np.timedelta64(1, unit) + return minute_delta + + def time_delta_from_now_to_tgt(self, tgt_time, unit="m"): + return self.time_delta(datetime.datetime.now(), tgt_time=tgt_time, unit=unit) def time_inbetween(self, query_time:datetime.datetime, start_time:datetime.datetime, end_time:datetime.datetime) -> bool: """ diff --git a/src/lib_brecal_utils/brecal_utils/validators/validation_rule_functions.py b/src/lib_brecal_utils/brecal_utils/validators/validation_rule_functions.py new file mode 100644 index 0000000..39a34c7 --- /dev/null +++ b/src/lib_brecal_utils/brecal_utils/validators/validation_rule_functions.py @@ -0,0 +1,763 @@ +import inspect +import types +from brecal_utils.database.enums import ParticipantType, ShipcallType, ParticipantwiseTimeDelta +import numpy as np +import pandas as pd +from brecal_utils.validators.time_logic import TimeLogic +from brecal_utils.database.enums import StatusFlags +#from brecal_utils.validators.schema_validation import validation_state_and_validation_name + + +class ValidationRuleBaseFunctions(): + """ + Base object with individual functions, which the {ValidationRuleFunctions}-child refers to. + This parent class provides base functions and helps to restructure the code in a more comprehensible way. + """ + def __init__(self, sql_handler): + self.sql_handler = sql_handler + self.time_logic = TimeLogic() + + def check_time_delta_violation_query_time_to_now(self, query_time:pd.Timestamp, key_time:pd.Timestamp, threshold:float)->bool: + """ + # base function for all validation rules in the group {0001} A-L + + measures the time between NOW and query_time. + When the query_time lays in the past, the delta is negative + when the query_time lays in the future, the delta is positive + + returns a violation state depending on whether the delta is + Violation, if: 0 >= delta > threshold + + When the key time is defined (not None), there is no violation. Returns False + + options: + query_time: will be used to measure the time difference of 'now' until the query time + key_time: will be used to check, whether the respective key already has a value + threshold: threshold where a time difference becomes crucial. When the delta is below the threshold, a violation might occur + """ + # rule is not applicable -> return 'GREEN' + if key_time is not None: + return False + + # otherwise, this rule applies and the difference between 'now' and the query time is measured + delta = self.time_logic.time_delta_from_now_to_tgt(tgt_time=query_time, unit="m") + + # a violation occurs, when the delta (in minutes) exceeds the specified threshold of a participant + # to prevent past-events from triggering violations, negative values are ignored + # Violation, if 0 >= delta >= threshold + violation_state = (delta >= 0) and (delta<=threshold) + return violation_state + + def check_participants_agree_on_estimated_time(self, shipcall, query, df_times, applicable_shipcall_type)->bool: + """ + # base function for all validation rules in the group {0002} A-C + + compares, whether the participants agree on the estimated time (of arrival or departure), depending on + whether the shipcall type is incoming, outgoing or shifting. + + No violations are observed, when + - the shipcall belongs to a different type than the rule expects + - there are no matching times for the provided {query} (e.g., "eta_berth") + + Instead of comparing each individual result, this function counts the amount of unique instances. + When there is not only one unique value, there are deviating time estimates, and a violation occurs + + returns: violation_state (bool) + """ + # shipcall type filter: consider only shipcalls, where the type matches + if shipcall.type != applicable_shipcall_type.value: + violation_state = False + return violation_state + + # filter by participant types of interest (agency, mooring, portauthority/administration, pilot, tug) + participant_types = [ParticipantType.AGENCY.value, ParticipantType.MOORING.value, ParticipantType.PORT_ADMINISTRATION.value, ParticipantType.PILOT.value, ParticipantType.TUG.value] + df_times = df_times.loc[df_times["participant_type"].isin(participant_types),:] + + # exclude missing entries + df_times.loc[~df_times[query].isnull(),:] + + # when there are no entries left (no entries are provided), skip + if len(df_times)==0: + violation_state = False + return violation_state + + # there should only be one eta_berth, when all participants have provided the same time + # this equates to the same criteria as checking, whether + # times_agency.eta_berth==times_mooring.eta_berth==times_portadministration.eta_berth==times_pilot.eta_berth==times_tug.eta_berth + unique_times = len(pd.unique(df_times.loc[:,query])) + violation_state = unique_times!=1 + return violation_state + + def check_unique_shipcall_counts(self, query:str, rounding="min", maximum_threshold=3)->bool: + """ + # base function for all validation rules in the group {0005} A&B + + compares how many unique times are found for the provided {query} (e.g., "eta_berth") + This function rounds the results, counts the unique values and returns a boolean state, whether the {maximum_threshold} is exceeded + """ + # filter the df: keep only times_agents + # filter out all NaN and NaT entries + times_agency = self.sql_handler.get_times_for_agency(non_null_column=query) + + # get values and optionally round the values + (values, unique, counts) = self.sql_handler.get_unique_ship_counts(all_df_times=times_agency, query=query, rounding=rounding, maximum_threshold=maximum_threshold) + + # when ANY of the unique values exceeds the threshold, a violation is observed + violation_state = np.any(np.greater(counts, maximum_threshold)) + return violation_state + + +class ValidationRuleFunctions(ValidationRuleBaseFunctions): + """ + an accumulation object that makes sure, that any validation rule is translated to a function with default naming convention and + return types. Each function should return a ValidationRuleState enumeration object and a description string to which validation rule + the result belongs. These are returned as tuples (ValidationRuleState, validation_name) + Each rule should have the same input arguments (self, shipcall, df_times, *args, **kwargs) + + The object makes heavy use of calls from an SQLHandler object, which provides functions for dataframe access and filtering. + + each validation_name is generated by calling the function inside a method + validation_name = inspect.currentframe().f_code.co_name # validation_name then returns the name of the method from where 'currentframe()' was called. + + # example: + #def validation_rule_fct_example(self, shipcall, df_times): + #validation_name = inspect.currentframe().f_code.co_name + #return (ValidationRuleState.NONE, validation_name) + """ + def __init__(self, sql_handler): + super().__init__(sql_handler) + return + + def get_validation_rule_functions(self): + """return a list of all methods in this object, which are all validation rule functions.""" + return [self.__getattribute__(mthd_) for mthd_ in dir(self) if ('validation_rule_fct' in mthd_) and (isinstance(self.__getattribute__(mthd_), types.MethodType))] + + def validation_rule_fct_missing_time_agency_berth_eta(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-A + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-A: + - Checks, if times_agency.eta_berth is filled in. + - Measures the difference between 'now' and 'shipcall.eta'. + """ + # check, if the header is filled in (agency) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + query_time = shipcall.eta + key_time = times_agency.eta_berth + threshold = ParticipantwiseTimeDelta.AGENCY + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_agency_berth_etd(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-B + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-B: + - Checks, if times_agency.etd_berth is filled in. + - Measures the difference between 'now' and 'shipcall.etd'. + """ + # check, if the header is filled in (agency) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + query_time = shipcall.etd + key_time = times_agency.etd_berth + threshold = ParticipantwiseTimeDelta.AGENCY + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_mooring_berth_eta(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-C + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-C: + - Checks, if times_mooring.eta_berth is filled in. + - Measures the difference between 'now' and 'times_agency.eta_berth'. + """ + # check, if the header is filled in (agency & MOORING) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.MOORING.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) + + query_time = times_agency.eta_berth + key_time = times_mooring.eta_berth + threshold = ParticipantwiseTimeDelta.MOORING + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_mooring_berth_etd(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-D + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-D: + - Checks, if times_mooring.etd_berth is filled in. + - Measures the difference between 'now' and 'times_agency.etd_berth'. + """ + # check, if the header is filled in (agency & MOORING) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.MOORING.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) + + query_time = times_agency.etd_berth + key_time = times_mooring.etd_berth + threshold = ParticipantwiseTimeDelta.MOORING + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_portadministration_berth_eta(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-F + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-F: + - Checks, if times_port_administration.eta_berth is filled in. + - Measures the difference between 'now' and 'times_agency.eta_berth'. + """ + # check, if the header is filled in (agency & PORT_ADMINISTRATION) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.PORT_ADMINISTRATION.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) + + query_time = times_agency.eta_berth + key_time = times_port_administration.eta_berth + threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_portadministration_berth_etd(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-G + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-G: + - Checks, if times_port_administration.etd_berth is filled in. + - Measures the difference between 'now' and 'times_agency.etd_berth'. + """ + # check, if the header is filled in (agency & PORT_ADMINISTRATION) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.PORT_ADMINISTRATION.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) + + query_time = times_agency.etd_berth + key_time = times_port_administration.etd_berth + threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_pilot_berth_eta(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-H + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-H: + - Checks, if times_pilot.eta_berth is filled in. + - Measures the difference between 'now' and 'times_agency.eta_berth'. + """ + # check, if the header is filled in (agency & PILOT) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.PILOT.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) + + query_time = times_agency.eta_berth + key_time = times_pilot.eta_berth + threshold = ParticipantwiseTimeDelta.PILOT + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_pilot_berth_etd(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-I + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-I: + - Checks, if times_pilot.etd_berth is filled in. + - Measures the difference between 'now' and 'times_agency.etd_berth'. + """ + # check, if the header is filled in (agency & PILOT) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.PILOT.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) + + query_time = times_agency.etd_berth + key_time = times_pilot.etd_berth + threshold = ParticipantwiseTimeDelta.PILOT + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_tug_berth_eta(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-J + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-J: + - Checks, if times_tug.eta_berth is filled in. + - Measures the difference between 'now' and 'times_agency.eta_berth'. + """ + # check, if the header is filled in (agency & TUG) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TUG.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) + + query_time = times_agency.eta_berth + key_time = times_tug.eta_berth + threshold = ParticipantwiseTimeDelta.TUG + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_tug_berth_etd(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-K + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-K: + - Checks, if times_tug.etd_berth is filled in. + - Measures the difference between 'now' and 'times_agency.etd_berth'. + """ + # check, if the header is filled in (agency & TUG) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TUG.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) + + query_time = times_agency.etd_berth + key_time = times_tug.etd_berth + threshold = ParticipantwiseTimeDelta.TUG + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_terminal_berth_eta(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-L + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-L: + - Checks, if times_terminal.eta_berth is filled in. + - Measures the difference between 'now' and 'times_agency.eta_berth'. + """ + # check, if the header is filled in (agency & terminal) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) + + query_time = times_agency.eta_berth + key_time = times_terminal.eta_berth + threshold = ParticipantwiseTimeDelta.TERMINAL + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_missing_time_terminal_berth_etd(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0001-K + Type: Local Rule + Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below + a certain threshold (e.g., 20 hours), a violation occurs + + 0001-K: + - Checks, if times_terminal.etd_berth is filled in. + - Measures the difference between 'now' and 'times_agency.etd_berth'. + """ + # check, if the header is filled in (agency & terminal) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2: + return (StatusFlags.GREEN, None) + + # preparation: obtain the correct times of the participant, define the query time and the key time + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) + + query_time = times_agency.etd_berth + key_time = times_terminal.etd_berth + threshold = ParticipantwiseTimeDelta.TERMINAL + violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + + def validation_rule_fct_shipcall_incoming_participants_disagree_on_eta(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0002-A + Type: Local Rule + Description: this validation checks, whether the participants expect different ETA times + Filter: only applies to incoming shipcalls + """ + query = "eta_berth" + + violation_state = self.check_participants_agree_on_estimated_time( + shipcall = shipcall, + + query=query, + df_times=df_times, + applicable_shipcall_type=ShipcallType.INCOMING + ) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.RED, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_shipcall_outgoing_participants_disagree_on_etd(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0002-B + Type: Local Rule + Description: this validation checks, whether the participants expect different ETA times + Filter: only applies to outgoing shipcalls + """ + query = "etd_berth" + + violation_state = self.check_participants_agree_on_estimated_time( + shipcall = shipcall, + + query=query, + df_times=df_times, + applicable_shipcall_type=ShipcallType.OUTGOING + ) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.RED, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_shipcall_shifting_participants_disagree_on_eta_or_etd(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0002-C + Type: Local Rule + Description: this validation checks, whether the participants expect different ETA or ETD times + Filter: only applies to shifting shipcalls + """ + violation_state_eta = self.check_participants_agree_on_estimated_time( + shipcall = shipcall, + + query="eta_berth", + df_times=df_times, + applicable_shipcall_type=ShipcallType.SHIFTING + ) + + violation_state_etd = self.check_participants_agree_on_estimated_time( + shipcall = shipcall, + + query="etd_berth", + df_times=df_times, + applicable_shipcall_type=ShipcallType.SHIFTING + ) + + # apply 'eta_berth' check + # apply 'etd_berth' + # violation: if either 'eta_berth' or 'etd_berth' is violated + # functionally, this is the same as individually comparing all times for the participants + # times_agency.eta_berth==times_mooring.eta_berth==times_portadministration.eta_berth==times_pilot.eta_berth==times_tug.eta_berth + # times_agency.etd_berth==times_mooring.etd_berth==times_portadministration.etd_berth==times_pilot.etd_berth==times_tug.etd_berth + violation_state = (violation_state_eta) or (violation_state_etd) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.RED, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_eta_time_not_in_operation_window(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0003-A + Type: Local Rule + Description: this validation checks, whether the ETA time is between the provided operations window of the terminal + + query time: eta_berth (times_agency) + start_time & end_time: operations_start & operations_end (times_terminal) + """ + # check, if the header is filled in (agency & terminal) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2: + return (StatusFlags.GREEN, None) + + # get agency & terminal times + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY) + times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL) + + if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT): + return (StatusFlags.GREEN, None) + + # check, whether the start of operations is AFTER the estimated arrival time + violation_state = times_terminal.operations_start times_agency.etd_berth + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.RED, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_eta_time_not_in_tidal_window(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0004-A + Type: Local Rule + Description: this validation checks, whether the ETA time is between the provided tidal window + + query time: eta_berth (times_agency) + start_time & end_time: tidal_window_from & tidal_window_to (shipcall) + """ + # check, if the header is filled in (agency) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + return (StatusFlags.GREEN, None) + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY) + + # requirements: tidal window (from & to) is filled in + if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT): + return (StatusFlags.GREEN, None) + + # check, whether the query time is between start & end time + # a violation is observed, when the is NOT between start & end + violation_state = not self.time_logic.time_inbetween(query_time=times_agency.eta_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.RED, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_etd_time_not_in_tidal_window(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0004-B + Type: Local Rule + Description: this validation checks, whether the ETD time is between the provided tidal window + + query time: eta_berth (times_agency) + start_time & end_time: tidal_window_from & tidal_window_to (shipcall) + """ + # check, if the header is filled in (agency) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + return (StatusFlags.GREEN, None) + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY) + + # requirements: tidal window (from & to) is filled in + if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT): + return (StatusFlags.GREEN, None) + + # check, whether the query time is between start & end time + # a violation is observed, when the is NOT between start & end + violation_state = not self.time_logic.time_inbetween(query_time=times_agency.etd_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.RED, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_too_many_identical_eta_times(self, shipcall, df_times, rounding = "min", maximum_threshold = 3, *args, **kwargs): + """ + Code: #0005-A + Type: Global Rule + Description: this validation rule checks, whether there are too many shipcalls with identical ETA times. + """ + # when ANY of the unique values exceeds the threshold, a violation is observed + query = "eta_berth" + violation_state = self.check_unique_shipcall_counts(query, rounding=rounding, maximum_threshold=maximum_threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_too_many_identical_etd_times(self, shipcall, df_times, rounding = "min", maximum_threshold = 3, *args, **kwargs): + """ + Code: #0005-B + Type: Global Rule + Description: this validation rule checks, whether there are too many shipcalls with identical ETD times. + """ + # when ANY of the unique values exceeds the threshold, a violation is observed + query = "etd_berth" + violation_state = self.check_unique_shipcall_counts(query, rounding=rounding, maximum_threshold=maximum_threshold) + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_agency_and_terminal_berth_id_disagreement(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0006-A + Type: Local Rule + Description: This validation rule checks, whether agency and terminal agree with their designated berth place by checking berth_id. + """ + # check, if the header is filled in (agency & terminal) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2: + return (StatusFlags.GREEN, None) + + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) + + violation_state = times_agency.berth_id!=times_terminal.berth_id + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + def validation_rule_fct_agency_and_terminal_pier_side_disagreement(self, shipcall, df_times, *args, **kwargs): + """ + Code: #0006-B + Type: Local Rule + Description: This validation rule checks, whether agency and terminal agree with their designated pier side by checking pier_side. + """ + # check, if the header is filled in (agency & terminal) + if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2: + return (StatusFlags.GREEN, None) + + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) + + violation_state = times_agency.pier_side!=times_terminal.pier_side + + if violation_state: + validation_name = inspect.currentframe().f_code.co_name + return (StatusFlags.YELLOW, validation_name) + else: + return (StatusFlags.GREEN, None) + + diff --git a/src/lib_brecal_utils/brecal_utils/validators/validation_rules.py b/src/lib_brecal_utils/brecal_utils/validators/validation_rules.py index d1e3a50..3ee7a46 100644 --- a/src/lib_brecal_utils/brecal_utils/validators/validation_rules.py +++ b/src/lib_brecal_utils/brecal_utils/validators/validation_rules.py @@ -1,22 +1,81 @@ import copy -from brecal_utils.validators.time_logic import TimeLogic +import numpy as np +import pandas as pd +from brecal_utils.database.enums import StatusFlags +from brecal_utils.validators.validation_rule_functions import ValidationRuleFunctions +from BreCal.schemas.model import Shipcall -class ValidationRules(): + +class ValidationRules(ValidationRuleFunctions): """ - An object that determines the traffic light state for validation and notification. The provided str feedback ('green', 'yellow', 'red') - determines, whether the state is critical. + An object that determines the traffic light state for validation and notification. The provided feedback ('green', 'yellow', 'red') + determines, whether the state is critical. It uses ValidationRuleState enumerations. In case of a critical validation state, the user's input prompt may be interrupted and the user may be warned. In case of a critical notification state, the respective users will be automatically notified after n seconds. (#TODO_n_seconds_delay) """ - def __init__(self, shipcall, times): # use the entire data that is provided for this query (e.g., json input) - self.time_logic = TimeLogic() - self.shipcall = shipcall - self.times = times + def __init__(self, sql_handler): # use the entire data that is provided for this query (e.g., json input) + super().__init__(sql_handler) self.validation_state = self.determine_validation_state() - self.notification_state = self.determine_notification_state() # (state:str, should_notify:bool) + # currently flagged: notification_state initially was based on using one ValidationRules object for each query. This is deprecated. + # self.notification_state = self.determine_notification_state() # (state:str, should_notify:bool) return + def evaluate(self, shipcall): + """ + 1.) prepare df_times, which every validation rule tends to use + calling this only once saves a lot of computational overhead + 2.) apply all validation rules + returns: (evaluation_state, violations) + """ + # prepare df_times, which every validation rule tends to use + df_times = self.sql_handler.df_dict.get('times') # -> pd.DataFrame + + # filter by shipcall id + df_times = self.sql_handler.get_times_of_shipcall(shipcall) + + # apply all validation rules + # list of tuples, where each element is (state, msg) + evaluation_results = [elem(shipcall, df_times) for elem in self.get_validation_rule_functions()] + + # filter out all 'None' results, which indicate that no violation occured. + evaluation_results = [evaluation_result for evaluation_result in evaluation_results if evaluation_result[1] is not None] + + """ # deprecated + # check, if ANY of the evaluation results (evaluation_state) is larger than the .GREEN state. This means, that .YELLOW and .RED + # would return 'True'. Numpy arrays and functions are used to accelerate the comparison. + # np.any returns a boolean. + #evaluation_state = not np.any(np.greater(np.array([result[0] for result in evaluation_results]), ValidationRuleState.GREEN)) + """ + # check, what the maximum state flag is and return it + evaluation_state = np.max(np.array([result[0] for result in evaluation_results])) if len(evaluation_results)>0 else 1 + return (evaluation_state, evaluation_results) + + def evaluation_verbosity(self, evaluation_state, evaluation_results): + """This function suggestions verbosity for the evaluation results. Based on 'True'/'False' evaluation outcome, the returned string is different.""" + if evaluation_state: + return f"OK! The validation was successful. There are no rule violations." + else: + verbose_string = "These are:" + "\n\t".join(evaluation_results) # every element of the list will be displayed in a new line with a tab + return f"FAILED VALIDATION. There have been {len(evaluation_results)} violations. {verbose_string}" + + def evaluate_shipcall_from_df(self, x): + shipcall = Shipcall(**{**{'id':x.name}, **x.to_dict()}) + evaluation_state, violations = self.evaluate(shipcall) + return evaluation_state, violations + + def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame: + """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation' and 'evaluation_message' are updated)""" + results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values + + # unbundle individual results. evaluation_state becomes an integer, violation + evaluation_state = [StatusFlags(res[0]).value for res in results] + violations = [",".join(res[1]) if len(res[1])>0 else None for res in results] + + shipcall_df.loc[:,"evaluation"] = evaluation_state + shipcall_df.loc[:,"evaluation_message"] = violations + return shipcall_df + def determine_validation_state(self) -> str: """ this method determines the validation state of a shipcall. The state is either ['green', 'yellow', 'red'] and signals, @@ -24,7 +83,7 @@ class ValidationRules(): returns: validation_state_new (str) """ - validation_state_new = self.undefined_method() + (validation_state_new, description) = self.undefined_method() # should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification. self.validation_state = validation_state_new return validation_state_new @@ -37,7 +96,7 @@ class ValidationRules(): returns: notification_state_new (str), should_notify (bool) """ - state_new = self.undefined_method() # determine the successor + (state_new, description) = self.undefined_method() # determine the successor should_notify = self.identify_notification_state_change(state_new) self.notification_state = state_new # overwrite the predecessor return state_new, should_notify @@ -53,16 +112,16 @@ class ValidationRules(): yellow -> red (none -> yellow) or (none -> red) + due to the values in the enumeration objects, the states are mapped to provide this function. + green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than". returns bool, whether a notification should be triggered """ - state_old = copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else 'none' - - state_mapping = {'none':0, 'green':0, 'yellow':1, 'red':2} - return state_mapping[state_new] > state_mapping[state_old] + # state_old is always considered at least 'Green' (1) + state_old = max(copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else StatusFlags.NONE, StatusFlags.GREEN.value) + return state_new.value > state_old.value def undefined_method(self) -> str: """this function should apply the ValidationRules to the respective .shipcall, in regards to .times""" # #TODO_traffic_state - return ('green', False) # (state:str, should_notify:bool) - + return (StatusFlags.GREEN, False) # (state:str, should_notify:bool) diff --git a/src/lib_brecal_utils/tests/validators/test_schema_validation_berth.py b/src/lib_brecal_utils/tests/validators/test_schema_validation_berth.py index dc1639c..900cfc6 100644 --- a/src/lib_brecal_utils/tests/validators/test_schema_validation_berth.py +++ b/src/lib_brecal_utils/tests/validators/test_schema_validation_berth.py @@ -2,11 +2,13 @@ import pytest from brecal_utils.stubs.berth import get_berth_simple def test_berth(): - berth = get_berth_simple() - raise ValueError("copied from ships.") - from brecal_utils.validators.schema_validation import test____ - ship = get_ship_simple() - ship.length = 234 - assert ship_length_in_range(ship)[0], f"ship length must be between 0 and 500 meters" + with pytest.raises(ValueError, match="#TODO: copied from ships."): + berth = get_berth_simple() + + raise ValueError("#TODO: copied from ships.") + from brecal_utils.validators.schema_validation import test____ + ship = get_ship_simple() + ship.length = 234 + assert ship_length_in_range(ship)[0], f"ship length must be between 0 and 500 meters" return diff --git a/src/lib_brecal_utils/tests/validators/test_validation_rule_functions.py b/src/lib_brecal_utils/tests/validators/test_validation_rule_functions.py new file mode 100644 index 0000000..498dd43 --- /dev/null +++ b/src/lib_brecal_utils/tests/validators/test_validation_rule_functions.py @@ -0,0 +1,159 @@ +import pytest +from brecal_utils.validators.validation_rule_functions import ValidationRuleFunctions +from brecal_utils.validators.validation_rules import ValidationRules +from brecal_utils.database.sql_handler import SQLHandler + +@pytest.fixture(scope="session") +def build_sql_proxy_connection(): + import mysql.connector + conn_from_pool = mysql.connector.connect(**{'host':'localhost', 'port':3306, 'user':'root', 'password':'HalloWach_2323XXL!!', 'pool_name':'brecal_pool', 'pool_size':20, 'database':'bremen_calling', 'autocommit': True}) + sql_handler = SQLHandler(sql_connection=conn_from_pool, read_all=True) + vr = ValidationRules(sql_handler) + return locals() + +def test_build_validation_rule_functions(build_sql_proxy_connection): + import types + sql_handler = build_sql_proxy_connection["sql_handler"] + vr = build_sql_proxy_connection["vr"] + + validation_rule_functions = vr.get_validation_rule_functions() + assert isinstance(validation_rule_functions, list), f"must return a list of methods" + for vrule in validation_rule_functions: + assert isinstance(vrule,types.MethodType), f"every element returned from get_validation_rule_functions must be a method. found: {type(vrule)}" + assert len(validation_rule_functions)>0, f"must return at least one method!" + return + + + + +def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement(build_sql_proxy_connection): + """#0006-A validation_rule_fct_agency_and_terminal_pier_side_disagreement""" + import pandas as pd + + from brecal_utils.stubs.times_full import get_times_full_simple + from brecal_utils.stubs.shipcall import get_shipcall_simple + from brecal_utils.database.enums import ParticipantType + from brecal_utils.database.enums import StatusFlags + + vr = build_sql_proxy_connection["vr"] + shipcall = get_shipcall_simple() + t1 = get_times_full_simple() + t2 = get_times_full_simple() + + # roles: agency & terminal + t1.participant_type = ParticipantType.AGENCY.value + t2.participant_type = ParticipantType.TERMINAL.value + + # disagreement + t1.pier_side = True + t2.pier_side = False + + time_objects = [t1, t2] + df_times = pd.DataFrame.from_records([to_.__dict__ for to_ in time_objects]) + df_times.set_index('id',inplace=True) + + (state, description) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall, df_times) + assert state.value > StatusFlags.GREEN.value, f"a violation must be identified" + assert description is not None, f"a violation description must be identified" + return + + +def test_validation_rule_fct_agency_and_terminal_pier_side_agreement(build_sql_proxy_connection): + """#0006-A validation_rule_fct_agency_and_terminal_pier_side_disagreement""" + import pandas as pd + + from brecal_utils.stubs.times_full import get_times_full_simple + from brecal_utils.stubs.shipcall import get_shipcall_simple + from brecal_utils.database.enums import ParticipantType + from brecal_utils.database.enums import StatusFlags + + vr = build_sql_proxy_connection["vr"] + shipcall = get_shipcall_simple() + t1 = get_times_full_simple() + t2 = get_times_full_simple() + + # roles: agency & terminal + t1.participant_type = ParticipantType.AGENCY.value + t2.participant_type = ParticipantType.TERMINAL.value + + # agreement + t1.pier_side = True + t2.pier_side = True + + time_objects = [t1, t2] + df_times = pd.DataFrame.from_records([to_.__dict__ for to_ in time_objects]) + df_times.set_index('id',inplace=True) + + (state, description) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall, df_times) + assert state.value == StatusFlags.GREEN.value, f"no violation should be observed" + assert description is None, f"no violation should be observed" + return + + + + +def test_validation_rule_fct_agency_and_terminal_berth_id_disagreement(build_sql_proxy_connection): + """#0006-B validation_rule_fct_agency_and_terminal_pier_side_disagreement""" + import pandas as pd + + from brecal_utils.stubs.times_full import get_times_full_simple + from brecal_utils.stubs.shipcall import get_shipcall_simple + from brecal_utils.database.enums import ParticipantType + from brecal_utils.database.enums import StatusFlags + + vr = build_sql_proxy_connection["vr"] + shipcall = get_shipcall_simple() + t1 = get_times_full_simple() + t2 = get_times_full_simple() + + # roles: agency & terminal + t1.participant_type = ParticipantType.AGENCY.value + t2.participant_type = ParticipantType.TERMINAL.value + + # disagreement + t1.berth_id = 1 + t2.berth_id = 2 + + time_objects = [t1, t2] + df_times = pd.DataFrame.from_records([to_.__dict__ for to_ in time_objects]) + df_times.set_index('id',inplace=True) + + (state, description) = vr.validation_rule_fct_agency_and_terminal_berth_id_disagreement(shipcall, df_times) + assert state.value > StatusFlags.GREEN.value, f"a violation must be identified" + assert description is not None, f"a violation description must be identified" + return + +def test_validation_rule_fct_agency_and_terminal_berth_id_agreement(build_sql_proxy_connection): + """#0006-B validation_rule_fct_agency_and_terminal_pier_side_disagreement""" + import pandas as pd + + from brecal_utils.stubs.times_full import get_times_full_simple + from brecal_utils.stubs.shipcall import get_shipcall_simple + from brecal_utils.database.enums import ParticipantType + from brecal_utils.database.enums import StatusFlags + + vr = build_sql_proxy_connection["vr"] + shipcall = get_shipcall_simple() + t1 = get_times_full_simple() + t2 = get_times_full_simple() + + # roles: agency & terminal + t1.participant_type = ParticipantType.AGENCY.value + t2.participant_type = ParticipantType.TERMINAL.value + + # agreement + t1.berth_id = 21 + t2.berth_id = 21 + + time_objects = [t1, t2] + df_times = pd.DataFrame.from_records([to_.__dict__ for to_ in time_objects]) + df_times.set_index('id',inplace=True) + + (state, description) = vr.validation_rule_fct_agency_and_terminal_berth_id_disagreement(shipcall, df_times) + assert state.value == StatusFlags.GREEN.value, f"no violation should be observed" + assert description is None, f"no violation should be observed" + return + + + + diff --git a/src/lib_brecal_utils/tests/validators/test_validation_rule_state.py b/src/lib_brecal_utils/tests/validators/test_validation_rule_state.py new file mode 100644 index 0000000..3ea2281 --- /dev/null +++ b/src/lib_brecal_utils/tests/validators/test_validation_rule_state.py @@ -0,0 +1,26 @@ +import pytest +from brecal_utils.database.enums import StatusFlags + +def test_validation_rule_state_green_is_1(): + assert StatusFlags.GREEN.value==1 + return + +def test_validation_rule_state_yellow_is_2(): + assert StatusFlags.YELLOW.value==2 + return + +def test_validation_rule_state_red_is_3(): + assert StatusFlags.RED.value==3 + return + +def test_validation_rule_state_order(): + # Red 3, Yellow 2, Green 1, None 0 + # red>yellow>green>none + assert StatusFlags.RED.value>StatusFlags.YELLOW.value + assert StatusFlags.YELLOW.value>StatusFlags.GREEN.value + assert StatusFlags.GREEN.value>StatusFlags.NONE.value + return + +if __name__=="__main__": + pass + diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index 518ad7a..9f77a34 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -85,8 +85,6 @@ class ShipcallSchema(Schema): anchored = fields.Bool(Required = False, allow_none=True) moored_lock = fields.Bool(Required = False, allow_none=True) canceled = fields.Bool(Required = False, allow_none=True) - validation_state = fields.Str(Required = False, allow_none=True) - validation_state_changed = fields.DateTime(Required = False, allow_none=True) evaluation = fields.Int(Required = False, allow_none=True) evaluation_message = fields.Str(Required = False, allow_none=True) participants = fields.List(fields.Int) @@ -119,8 +117,6 @@ class Shipcall: anchored: bool moored_lock: bool canceled: bool - validation_state: str - validation_state_changed: datetime evaluation: int evaluation_message: str created: datetime diff --git a/src/server/requirements.txt b/src/server/requirements.txt index ca817e1..9c7b54f 100644 --- a/src/server/requirements.txt +++ b/src/server/requirements.txt @@ -10,4 +10,9 @@ pydapper[mysql-connector-python] marshmallow-dataclass bcrypt jwt -flask-jwt-extended \ No newline at end of file +flask-jwt-extended +SQLAlchemy +numpy +pandas + +