adding validation rules and time logic. Stubs are used to test the rules. Enum objects are located in brecal_utils/database/enums

src/lib_brecal_utils/brecal_utils/database/enums.py

@@ -0,0 +1,38 @@
+from enum import Enum
+
+class ParticipantType(Enum):
+    """determines the type of a participant"""
+    NONE = 0
+    BSMD = 1
+    TERMINAL = 2
+    PILOT = 4
+    AGENCY = 8
+    MOORING = 16
+    PORT_ADMINISTRATION = 32
+    TUG = 64
+
+class ShipcallType(Enum):
+    """determines the type of a shipcall, as this changes the applicable validation rules"""
+    INCOMING = 1
+    OUTGOING = 2
+    SHIFTING = 3
+
+class ParticipantwiseTimeDelta():
+    """stores the time delta for every participant, which triggers the validation rules in the rule set '0001'"""
+    AGENCY = 1200.0 # 20 h * 60 min/h = 1200 min
+    MOORING = 960.0 # 16 h * 60 min/h = 960 min
+    PILOT = 960.0 # 16 h * 60 min/h = 960 min
+    PORT_ADMINISTRATION = 960.0 # 16 h * 60 min/h = 960 min
+    TUG = 960.0 # 16 h * 60 min/h = 960 min
+    TERMINAL = 960.0 # 16 h * 60 min/h = 960 min
+
+class StatusFlags(Enum):
+    """
+    these enumerators ensure that each traffic light validation rule state corresponds to a value, which will be used in the ValidationRules object to identify
+    the necessity of notifications.
+    """
+    NONE = 0
+    GREEN = 1
+    YELLOW = 2
+    RED = 3
+

src/lib_brecal_utils/brecal_utils/database/sql_handler.py

@@ -0,0 +1,200 @@
+import numpy as np
+import pandas as pd
+import datetime
+from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times
+from brecal_utils.database.enums import ParticipantType
+
+class SQLHandler():
+    """
+    An object that reads SQL queries from the sql_connection and stores it in pandas DataFrames. The object can read all available tables
+    at once into memory, when providing 'read_all=True'.
+
+    # #TODO_initialization: shipcall_tug_map, user_role_map & role_securable_map might be mapped to the respective dataframes
+    """
+    def __init__(self, sql_connection, read_all=False):
+        self.sql_connection = sql_connection
+        self.all_schemas = self.get_all_schemas_from_mysql()
+        self.build_str_to_model_dict()
+
+        if read_all:
+            self.read_all(self.all_schemas)
+
+    def get_all_schemas_from_mysql(self):
+        with self.sql_connection.cursor(buffered=True) as cursor:
+            cursor.execute("SHOW TABLES")
+            schema = cursor.fetchall()
+        all_schemas = [schem[0] for schem in schema]
+        return all_schemas
+    
+    def build_str_to_model_dict(self):
+        """
+        creates a simple dictionary, which maps a string to a data object
+        e.g.,
+            'ship'->BreCal.schemas.model.Ship object
+        """
+        self.str_to_model_dict = {
+            "shipcall":Shipcall, "ship":Ship, "participant":Participant, "berth":Berth, "user":User, "times":Times
+        }
+        return
+
+    def read_mysql_table_to_df(self, table_name:str):
+        """determine a {table_name}, which will be read from a mysql server. returns a pandas DataFrame with the respective data"""
+        df = pd.read_sql(sql=f"SELECT * FROM {table_name}", con=self.sql_connection)
+        return df
+
+    def mysql_to_df(self, query):
+        """provide an arbitrary sql query that should be read from a mysql server {sql_connection}. returns a pandas DataFrame with the obtained data"""
+        df = pd.read_sql(query, self.sql_connection).convert_dtypes()
+        df = df.set_index('id', inplace=False) # avoid inplace updates, so the raw sql remains unchanged
+        return df
+    
+    def read_all(self, all_schemas):
+        # create a dictionary, which maps every mysql schema to pandas DataFrames
+        self.df_dict = self.build_full_mysql_df_dict(all_schemas)
+
+        # update the 'participants' column in 'shipcall'
+        self.initialize_shipcall_participant_list()
+        return
+
+    def build_full_mysql_df_dict(self, all_schemas):
+        """given a list of strings {all_schemas}, every schema will be read as individual pandas DataFrames to a dictionary with the respective keys. returns: dictionary {schema_name:pd.DataFrame}"""
+        mysql_df_dict = {}
+        for schem in all_schemas:
+            query = f"SELECT * FROM {schem}"
+            mysql_df_dict[schem] = self.mysql_to_df(query)
+        return mysql_df_dict
+    
+    def initialize_shipcall_participant_list(self):
+        """
+        iteratively applies the .get_participants method to each shipcall.
+        the function updates the 'participants' column.
+        """
+        # 1.) get all shipcalls
+        df = self.df_dict.get('shipcall')
+
+        # 2.) iterate over each individual shipcall, obtain the id (pandas calls it 'name')
+        # and apply the 'get_participants' method, which returns a list
+        # if the shipcall_id exists, the list contains ids
+        # otherwise, return a blank list
+        df['participants'] = df.apply(
+            lambda x: self.get_participants(x.name), 
+            axis=1)
+        return
+    
+    def standardize_model_str(self, model_str:str)->str:
+        """check if the 'model_str' is valid and apply lowercasing to the string"""
+        model_str = model_str.lower()
+        assert model_str in list(self.df_dict.keys()), f"cannot find the requested 'model_str' in mysql: {model_str}"
+        return model_str
+        
+    def get_data(self, id:int, model_str:str):
+        """
+        obtains {id} from the respective mysql database and builds a data model from that.
+        the id should match the 'id'-column in the mysql schema.
+        returns: data model, such as Ship, Shipcall, etc.
+
+        e.g.,
+            data = self.get_data(0,"shipcall")
+            returns a Shipcall object
+        """
+        model_str = self.standardize_model_str(model_str)
+        
+        df = self.df_dict.get(model_str)
+        data = self.df_loc_to_data_model(df, id, model_str)
+        return data
+        
+    def get_all(self, model_str:str)->list:
+        """
+        given a model string (e.g., 'shipcall'), return a list of all
+        data models of that type from the sql
+        """
+        model_str = self.standardize_model_str(model_str)
+        all_ids = self.df_dict.get(model_str).index
+
+        all_data = [
+            self.get_data(_aid, model_str)
+            for _aid in all_ids
+        ]
+        return all_data
+    
+    def df_loc_to_data_model(self, df, id, model_str, loc_type:str="loc"):
+        assert len(df)>0, f"empty dataframe"
+        
+        # get a pandas series from the dataframe
+        series = df.loc[id] if loc_type=="loc" else df.iloc[id]
+
+        # get the respective data model object
+        data_model = self.str_to_model_dict.get(model_str,None)
+        assert data_model is not None, f"could not find the requested model_str: {model_str}"
+
+        # build 'data' and fill the data model object
+        data = {**{'id':id}, **series.to_dict()} # 'id' must be added manually, as .to_dict does not contain the index, which was set with .set_index
+        data = data_model(**data)
+        return data
+    
+    def get_times_for_participant_type(self, df_times, participant_type:int):
+        filtered_series = df_times.loc[df_times["participant_type"]==participant_type]
+        assert len(filtered_series)<=1, f"found multiple results"
+        times = self.df_loc_to_data_model(filtered_series, id=0, model_str='times', loc_type="iloc") # use iloc! to retrieve the first result
+        return times
+    
+    def dataframe_to_data_model_list(self, df, model_str)->list:
+        model_str = self.standardize_model_str(model_str)
+
+        all_ids = df.index
+        all_data = [
+            self.df_loc_to_data_model(df, _aid, model_str)
+            for _aid in all_ids
+        ]
+        return all_data
+
+    def get_participants(self, shipcall_id:id)->list:
+        """
+        given a {shipcall_id}, obtain the respective list of participants.
+        when there are no participants, return a blank list
+
+        returns: participant_id_list, where every element is an int
+        """
+        df = self.df_dict.get("shipcall_participant_map")
+        df = df.set_index('shipcall_id', inplace=False)
+        
+        # the 'if' call is needed to ensure, that no Exception is raised, when the shipcall_id is not present in the df
+        participant_id_list = df.loc[shipcall_id, "participant_id"].to_list() if shipcall_id in list(df.index) else []
+        return participant_id_list
+    
+    def get_times_of_shipcall(self, shipcall)->pd.DataFrame:
+        df_times = self.df_dict.get('times') # -> pd.DataFrame
+        df_times = df_times.loc[df_times["shipcall_id"]==shipcall.id]
+        return df_times
+    
+    def get_times_for_agency(self, non_null_column=None)->pd.DataFrame:
+        """
+        options:
+            non_null_column: 
+                None or str. If provided, the 'non_null_column'-column of the dataframe will be filtered,
+                so only entries with provided values are returned (filters all NaN and NaT entries)
+        """
+        # get all times
+        df_times = self.df_dict.get('times') # -> pd.DataFrame
+
+        # filter out all NaN and NaT entries
+        if non_null_column is not None:
+            df_times = df_times.loc[~df_times[non_null_column].isnull()] # NOT null filter
+
+        # filter by the agency participant_type
+        times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]
+        return times_agency
+    
+    def filter_df_by_key_value(self, df, key, value)->pd.DataFrame:
+        return df.loc[df[key]==value]
+    
+    def get_unique_ship_counts(self, all_df_times:pd.DataFrame, query:str, rounding:str="min", maximum_threshold=3):
+        """given a dataframe of all agency times, get all unique ship counts, their values (datetime) and the string tags. returns a tuple (values,unique,counts)"""
+        # get values and optional: rounding
+        values = all_df_times.loc[:, query]
+        if rounding is not None:
+            values = values.dt.round(rounding) # e.g., 'min'
+
+        unique, counts = np.unique(values, return_counts=True)
+        violation_state = np.any(np.greater(counts, maximum_threshold))
+        return (values, unique, counts)

src/lib_brecal_utils/brecal_utils/stubs/shipcall.py

@@ -39,10 +39,12 @@ def get_shipcall_simple():
     moored_lock = False # de: 'Festmacherschleuse', en: 'moored lock'
 
     canceled = False
+    evaluation = None
+    evaluation_message = ""
     created = datetime.datetime.now()
     modified = created+datetime.timedelta(seconds=10)
 
-    participants = field(default_factory=[generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int()]) # list
+    participants = [generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int()] # field(default_factory=[generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int(), generate_uuid1_int()]) # list
 
     shipcall = Shipcall(
         shipcall_id, 
@@ -68,6 +70,8 @@ def get_shipcall_simple():
         anchored, 
         moored_lock, 
         canceled, 
+        evaluation,
+        evaluation_message, 
         created, 
         modified, 
         participants, 

src/lib_brecal_utils/brecal_utils/stubs/times_full.py

@@ -34,25 +34,34 @@ def get_times_full_simple():
     participant_id = generate_uuid1_int()
     shipcall_id = generate_uuid1_int()
 
+    berth_id = generate_uuid1_int()
+    berth_info = ""
+    pier_side = True
+    participant_type = None
+
     created = datetime.datetime.now()
     modified = created+datetime.timedelta(seconds=10)
 
     times = Times(
-        times_id, 
+        id=times_id, 
-        eta_berth, 
+        eta_berth=eta_berth, 
-        eta_berth_fixed, 
+        eta_berth_fixed=eta_berth_fixed, 
-        etd_berth, 
+        etd_berth=etd_berth, 
-        etd_berth_fixed,
+        etd_berth_fixed=etd_berth_fixed,
-        lock_time,
+        lock_time=lock_time,
-        lock_time_fixed,
+        lock_time_fixed=lock_time_fixed,
-        zone_entry,
+        zone_entry=zone_entry,
-        zone_entry_fixed,
+        zone_entry_fixed=zone_entry_fixed,
-        operations_start,
+        operations_start=operations_start,
-        operations_end,
+        operations_end=operations_end,
-        remarks, 
+        remarks=remarks, 
-        participant_id,
+        participant_id=participant_id,
-        shipcall_id,
+        berth_id=berth_id,
-        created,
+        berth_info=berth_info,
-        modified,
+        pier_side=pier_side,
+        participant_type=participant_type,
+        shipcall_id=shipcall_id,
+        created=created,
+        modified=modified,
     )   
     return times

src/lib_brecal_utils/brecal_utils/validators/time_logic.py

@@ -1,12 +1,42 @@
 import datetime
 import numpy as np
+import pandas as pd
 
 class TimeLogic():
     def __init__(self):
         return
     
-    def time_delta(self, query_time, other_times):
+    def time_delta(self, src_time, tgt_time, unit:str="m"):
-        return
+        """
+        in brief, this function measures tgt_time - src_time
+
+        if the tgt_time is in the future, it is a positive value (tgt_time > src_time)
+        if the tgt_time is in the past, it is a negative value (tgt_time < src_time)
+
+        returns the delta between tgt_time and src_time as a float of minutes (or the optionally provided unit)
+
+        options:
+            unit: str, which defaults to 'm' (minutes). 'h' (hours) or 's' (seconds) are also common units. Determines the unit of the output time delta
+        """
+        # convert np.datetime64
+        if isinstance(src_time, pd.Timestamp):
+            src_time = src_time.to_datetime64()
+
+        if isinstance(tgt_time, pd.Timestamp):
+            tgt_time = tgt_time.to_datetime64()
+
+        if isinstance(src_time, datetime.datetime):
+            src_time = np.datetime64(src_time)
+
+        if isinstance(tgt_time, datetime.datetime):
+            tgt_time = np.datetime64(tgt_time)
+
+        delta = tgt_time - src_time
+        minute_delta = delta / np.timedelta64(1, unit)
+        return minute_delta
+    
+    def time_delta_from_now_to_tgt(self, tgt_time, unit="m"):
+        return self.time_delta(datetime.datetime.now(), tgt_time=tgt_time, unit=unit)
     
     def time_inbetween(self, query_time:datetime.datetime, start_time:datetime.datetime, end_time:datetime.datetime) -> bool:
         """

src/lib_brecal_utils/brecal_utils/validators/validation_rule_functions.py

@@ -0,0 +1,763 @@
+import inspect
+import types
+from brecal_utils.database.enums import ParticipantType, ShipcallType, ParticipantwiseTimeDelta
+import numpy as np
+import pandas as pd
+from brecal_utils.validators.time_logic import TimeLogic
+from brecal_utils.database.enums import StatusFlags
+#from brecal_utils.validators.schema_validation import validation_state_and_validation_name
+
+
+class ValidationRuleBaseFunctions():
+    """
+    Base object with individual functions, which the {ValidationRuleFunctions}-child refers to.
+    This parent class provides base functions and helps to restructure the code in a more comprehensible way.
+    """
+    def __init__(self, sql_handler):
+        self.sql_handler = sql_handler
+        self.time_logic = TimeLogic()
+
+    def check_time_delta_violation_query_time_to_now(self, query_time:pd.Timestamp, key_time:pd.Timestamp, threshold:float)->bool:
+        """
+        # base function for all validation rules in the group {0001} A-L
+
+        measures the time between NOW and query_time. 
+        When the query_time lays in the past, the delta is negative
+        when the query_time lays in the future, the delta is positive
+
+        returns a violation state depending on whether the delta is
+            Violation, if: 0 >= delta > threshold
+
+        When the key time is defined (not None), there is no violation. Returns False
+
+        options:
+            query_time: will be used to measure the time difference of 'now' until the query time
+            key_time: will be used to check, whether the respective key already has a value
+            threshold: threshold where a time difference becomes crucial. When the delta is below the threshold, a violation might occur
+        """
+        # rule is not applicable -> return 'GREEN'
+        if key_time is not None:
+            return False
+        
+        # otherwise, this rule applies and the difference between 'now' and the query time is measured
+        delta = self.time_logic.time_delta_from_now_to_tgt(tgt_time=query_time, unit="m")
+
+        # a violation occurs, when the delta (in minutes) exceeds the specified threshold of a participant
+        # to prevent past-events from triggering violations, negative values are ignored
+        # Violation, if 0 >= delta >= threshold
+        violation_state = (delta >= 0) and (delta<=threshold)
+        return violation_state
+    
+    def check_participants_agree_on_estimated_time(self, shipcall, query, df_times, applicable_shipcall_type)->bool:
+        """
+        # base function for all validation rules in the group {0002} A-C
+
+        compares, whether the participants agree on the estimated time (of arrival or departure), depending on
+        whether the shipcall type is incoming, outgoing or shifting.
+
+        No violations are observed, when
+            - the shipcall belongs to a different type than the rule expects
+            - there are no matching times for the provided {query} (e.g., "eta_berth")
+            
+        Instead of comparing each individual result, this function counts the amount of unique instances.
+        When there is not only one unique value, there are deviating time estimates, and a violation occurs
+
+        returns: violation_state (bool)
+        """
+        # shipcall type filter: consider only shipcalls, where the type matches
+        if shipcall.type != applicable_shipcall_type.value:
+            violation_state = False
+            return violation_state
+
+        # filter by participant types of interest (agency, mooring, portauthority/administration, pilot, tug)
+        participant_types = [ParticipantType.AGENCY.value, ParticipantType.MOORING.value, ParticipantType.PORT_ADMINISTRATION.value, ParticipantType.PILOT.value, ParticipantType.TUG.value]
+        df_times = df_times.loc[df_times["participant_type"].isin(participant_types),:]
+
+        # exclude missing entries
+        df_times.loc[~df_times[query].isnull(),:]
+
+        # when there are no entries left (no entries are provided), skip
+        if len(df_times)==0:
+            violation_state = False
+            return violation_state
+
+        # there should only be one eta_berth, when all participants have provided the same time
+        # this equates to the same criteria as checking, whether
+        #    times_agency.eta_berth==times_mooring.eta_berth==times_portadministration.eta_berth==times_pilot.eta_berth==times_tug.eta_berth
+        unique_times = len(pd.unique(df_times.loc[:,query]))
+        violation_state = unique_times!=1
+        return violation_state
+    
+    def check_unique_shipcall_counts(self, query:str, rounding="min", maximum_threshold=3)->bool:
+        """
+        # base function for all validation rules in the group {0005} A&B
+
+        compares how many unique times are found for the provided {query} (e.g., "eta_berth")
+        This function rounds the results, counts the unique values and returns a boolean state, whether the {maximum_threshold} is exceeded
+        """
+        # filter the df: keep only times_agents
+        # filter out all NaN and NaT entries
+        times_agency = self.sql_handler.get_times_for_agency(non_null_column=query)
+
+        # get values and optionally round the values
+        (values, unique, counts) = self.sql_handler.get_unique_ship_counts(all_df_times=times_agency, query=query, rounding=rounding, maximum_threshold=maximum_threshold)
+
+        # when ANY of the unique values exceeds the threshold, a violation is observed
+        violation_state = np.any(np.greater(counts, maximum_threshold))
+        return violation_state
+
+
+class ValidationRuleFunctions(ValidationRuleBaseFunctions):
+    """
+    an accumulation object that makes sure, that any validation rule is translated to a function with default naming convention and 
+    return types. Each function should return a ValidationRuleState enumeration object and a description string to which validation rule
+    the result belongs. These are returned as tuples (ValidationRuleState, validation_name)
+    Each rule should have the same input arguments (self, shipcall, df_times, *args, **kwargs)
+
+    The object makes heavy use of calls from an SQLHandler object, which provides functions for dataframe access and filtering.
+    
+    each validation_name is generated by calling the function inside a method
+        validation_name = inspect.currentframe().f_code.co_name # validation_name then returns the name of the method from where 'currentframe()' was called.
+
+    # example:
+    #def validation_rule_fct_example(self, shipcall, df_times):
+        #validation_name = inspect.currentframe().f_code.co_name
+        #return (ValidationRuleState.NONE, validation_name)
+    """
+    def __init__(self, sql_handler):
+        super().__init__(sql_handler)
+        return 
+    
+    def get_validation_rule_functions(self):
+        """return a list of all methods in this object, which are all validation rule functions."""
+        return [self.__getattribute__(mthd_) for mthd_ in dir(self) if ('validation_rule_fct' in mthd_) and (isinstance(self.__getattribute__(mthd_), types.MethodType))]
+    
+    def validation_rule_fct_missing_time_agency_berth_eta(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-A
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-A:
+            - Checks, if times_agency.eta_berth is filled in. 
+            - Measures the difference between 'now' and 'shipcall.eta'.
+        """
+        # check, if the header is filled in (agency)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        query_time = shipcall.eta
+        key_time = times_agency.eta_berth
+        threshold = ParticipantwiseTimeDelta.AGENCY
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_agency_berth_etd(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-B
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-B:
+            - Checks, if times_agency.etd_berth is filled in. 
+            - Measures the difference between 'now' and 'shipcall.etd'.
+        """
+        # check, if the header is filled in (agency)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        query_time = shipcall.etd
+        key_time = times_agency.etd_berth
+        threshold = ParticipantwiseTimeDelta.AGENCY
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_mooring_berth_eta(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-C
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-C:
+            - Checks, if times_mooring.eta_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.eta_berth'.
+        """
+        # check, if the header is filled in (agency & MOORING)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.MOORING.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value)
+
+        query_time = times_agency.eta_berth
+        key_time = times_mooring.eta_berth
+        threshold = ParticipantwiseTimeDelta.MOORING
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_mooring_berth_etd(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-D
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-D:
+            - Checks, if times_mooring.etd_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.etd_berth'.
+        """
+        # check, if the header is filled in (agency & MOORING)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.MOORING.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value)
+
+        query_time = times_agency.etd_berth
+        key_time = times_mooring.etd_berth
+        threshold = ParticipantwiseTimeDelta.MOORING
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_portadministration_berth_eta(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-F
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-F:
+            - Checks, if times_port_administration.eta_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.eta_berth'.
+        """
+        # check, if the header is filled in (agency & PORT_ADMINISTRATION)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.PORT_ADMINISTRATION.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value)
+
+        query_time = times_agency.eta_berth
+        key_time = times_port_administration.eta_berth
+        threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_portadministration_berth_etd(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-G
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-G:
+            - Checks, if times_port_administration.etd_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.etd_berth'.
+        """
+        # check, if the header is filled in (agency & PORT_ADMINISTRATION)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.PORT_ADMINISTRATION.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value)
+
+        query_time = times_agency.etd_berth
+        key_time = times_port_administration.etd_berth
+        threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_pilot_berth_eta(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-H
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-H:
+            - Checks, if times_pilot.eta_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.eta_berth'.
+        """
+        # check, if the header is filled in (agency & PILOT)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.PILOT.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value)
+
+        query_time = times_agency.eta_berth
+        key_time = times_pilot.eta_berth
+        threshold = ParticipantwiseTimeDelta.PILOT
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+        
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_pilot_berth_etd(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-I
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-I:
+            - Checks, if times_pilot.etd_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.etd_berth'.
+        """
+        # check, if the header is filled in (agency & PILOT)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.PILOT.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value)
+
+        query_time = times_agency.etd_berth
+        key_time = times_pilot.etd_berth
+        threshold = ParticipantwiseTimeDelta.PILOT
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+        
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_tug_berth_eta(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-J
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-J:
+            - Checks, if times_tug.eta_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.eta_berth'.
+        """
+        # check, if the header is filled in (agency & TUG)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TUG.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value)
+
+        query_time = times_agency.eta_berth
+        key_time = times_tug.eta_berth
+        threshold = ParticipantwiseTimeDelta.TUG
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+        
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_tug_berth_etd(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-K
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-K:
+            - Checks, if times_tug.etd_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.etd_berth'.
+        """
+        # check, if the header is filled in (agency & TUG)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TUG.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value)
+
+        query_time = times_agency.etd_berth
+        key_time = times_tug.etd_berth
+        threshold = ParticipantwiseTimeDelta.TUG
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+        
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_terminal_berth_eta(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-L
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-L:
+            - Checks, if times_terminal.eta_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.eta_berth'.
+        """
+        # check, if the header is filled in (agency & terminal)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
+
+        query_time = times_agency.eta_berth
+        key_time = times_terminal.eta_berth
+        threshold = ParticipantwiseTimeDelta.TERMINAL
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+        
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_missing_time_terminal_berth_etd(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0001-K
+        Type: Local Rule
+        Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
+        a certain threshold (e.g., 20 hours), a violation occurs
+
+        0001-K:
+            - Checks, if times_terminal.etd_berth is filled in. 
+            - Measures the difference between 'now' and 'times_agency.etd_berth'.
+        """
+        # check, if the header is filled in (agency & terminal)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # preparation: obtain the correct times of the participant, define the query time and the key time
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
+
+        query_time = times_agency.etd_berth
+        key_time = times_terminal.etd_berth
+        threshold = ParticipantwiseTimeDelta.TERMINAL
+        violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold)
+        
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+
+    
+    def validation_rule_fct_shipcall_incoming_participants_disagree_on_eta(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0002-A
+        Type: Local Rule
+        Description: this validation checks, whether the participants expect different ETA times
+        Filter: only applies to incoming shipcalls
+        """
+        query = "eta_berth"
+        
+        violation_state = self.check_participants_agree_on_estimated_time(
+            shipcall = shipcall, 
+
+            query=query, 
+            df_times=df_times, 
+            applicable_shipcall_type=ShipcallType.INCOMING
+            )
+        
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.RED, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_shipcall_outgoing_participants_disagree_on_etd(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0002-B
+        Type: Local Rule
+        Description: this validation checks, whether the participants expect different ETA times
+        Filter: only applies to outgoing shipcalls
+        """
+        query = "etd_berth"
+        
+        violation_state = self.check_participants_agree_on_estimated_time(
+            shipcall = shipcall, 
+
+            query=query, 
+            df_times=df_times, 
+            applicable_shipcall_type=ShipcallType.OUTGOING
+            )
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.RED, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_shipcall_shifting_participants_disagree_on_eta_or_etd(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0002-C
+        Type: Local Rule
+        Description: this validation checks, whether the participants expect different ETA or ETD times
+        Filter: only applies to shifting shipcalls
+        """
+        violation_state_eta = self.check_participants_agree_on_estimated_time(
+            shipcall = shipcall, 
+
+            query="eta_berth", 
+            df_times=df_times, 
+            applicable_shipcall_type=ShipcallType.SHIFTING
+            )
+        
+        violation_state_etd = self.check_participants_agree_on_estimated_time(
+            shipcall = shipcall, 
+
+            query="etd_berth", 
+            df_times=df_times, 
+            applicable_shipcall_type=ShipcallType.SHIFTING
+            )
+        
+        # apply 'eta_berth' check
+        # apply 'etd_berth'
+        # violation: if either 'eta_berth' or 'etd_berth' is violated
+        # functionally, this is the same as individually comparing all times for the participants
+        #    times_agency.eta_berth==times_mooring.eta_berth==times_portadministration.eta_berth==times_pilot.eta_berth==times_tug.eta_berth
+        #    times_agency.etd_berth==times_mooring.etd_berth==times_portadministration.etd_berth==times_pilot.etd_berth==times_tug.etd_berth
+        violation_state = (violation_state_eta) or (violation_state_etd)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.RED, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_eta_time_not_in_operation_window(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0003-A
+        Type: Local Rule
+        Description: this validation checks, whether the ETA time is between the provided operations window of the terminal
+
+        query time: eta_berth (times_agency)
+        start_time & end_time: operations_start & operations_end (times_terminal)
+        """
+        # check, if the header is filled in (agency & terminal)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # get agency & terminal times
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY)
+        times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL)
+
+        if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT):
+            return (StatusFlags.GREEN, None)
+
+        # check, whether the start of operations is AFTER the estimated arrival time
+        violation_state = times_terminal.operations_start<times_agency.eta_berth
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.RED, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_eta_time_not_in_operation_window(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0003-B
+        Type: Local Rule
+        Description: this validation checks, whether the ETD time is between the provided operations window of the terminal
+
+        query time: eta_berth (times_agency)
+        start_time & end_time: operations_start & operations_end (times_terminal)
+        """
+        # check, if the header is filled in (agency & terminal)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        # get agency & terminal times
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY)
+        times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL)
+
+        if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT):
+            return (StatusFlags.GREEN, None)
+
+        # check, whether the end of operations is AFTER the estimated departure time
+        violation_state = times_terminal.operations_end > times_agency.etd_berth
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.RED, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_eta_time_not_in_tidal_window(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0004-A
+        Type: Local Rule
+        Description: this validation checks, whether the ETA time is between the provided tidal window
+
+        query time: eta_berth (times_agency)
+        start_time & end_time: tidal_window_from & tidal_window_to (shipcall)
+        """
+        # check, if the header is filled in (agency)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1:
+            return (StatusFlags.GREEN, None)
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY)
+
+        # requirements: tidal window (from & to) is filled in
+        if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT):
+            return (StatusFlags.GREEN, None)
+
+        # check, whether the query time is between start & end time
+        # a violation is observed, when the is NOT between start & end
+        violation_state = not self.time_logic.time_inbetween(query_time=times_agency.eta_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.RED, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_etd_time_not_in_tidal_window(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0004-B
+        Type: Local Rule
+        Description: this validation checks, whether the ETD time is between the provided tidal window
+
+        query time: eta_berth (times_agency)
+        start_time & end_time: tidal_window_from & tidal_window_to (shipcall)
+        """
+        # check, if the header is filled in (agency)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1:
+            return (StatusFlags.GREEN, None)
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY)
+
+        # requirements: tidal window (from & to) is filled in
+        if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT):
+            return (StatusFlags.GREEN, None)
+
+        # check, whether the query time is between start & end time
+        # a violation is observed, when the is NOT between start & end
+        violation_state = not self.time_logic.time_inbetween(query_time=times_agency.etd_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.RED, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_too_many_identical_eta_times(self, shipcall, df_times, rounding = "min", maximum_threshold = 3, *args, **kwargs):
+        """
+        Code: #0005-A
+        Type: Global Rule
+        Description: this validation rule checks, whether there are too many shipcalls with identical ETA times.
+        """
+        # when ANY of the unique values exceeds the threshold, a violation is observed
+        query = "eta_berth"
+        violation_state = self.check_unique_shipcall_counts(query, rounding=rounding, maximum_threshold=maximum_threshold)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_too_many_identical_etd_times(self, shipcall, df_times, rounding = "min", maximum_threshold = 3, *args, **kwargs):
+        """
+        Code: #0005-B
+        Type: Global Rule
+        Description: this validation rule checks, whether there are too many shipcalls with identical ETD times.
+        """
+        # when ANY of the unique values exceeds the threshold, a violation is observed
+        query = "etd_berth"
+        violation_state = self.check_unique_shipcall_counts(query, rounding=rounding, maximum_threshold=maximum_threshold)
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_agency_and_terminal_berth_id_disagreement(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0006-A
+        Type: Local Rule
+        Description: This validation rule checks, whether agency and terminal agree with their designated berth place by checking berth_id.
+        """
+        # check, if the header is filled in (agency & terminal)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
+
+        violation_state = times_agency.berth_id!=times_terminal.berth_id
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+    
+    def validation_rule_fct_agency_and_terminal_pier_side_disagreement(self, shipcall, df_times, *args, **kwargs):
+        """
+        Code: #0006-B
+        Type: Local Rule
+        Description: This validation rule checks, whether agency and terminal agree with their designated pier side by checking pier_side.
+        """
+        # check, if the header is filled in (agency & terminal)
+        if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value, ParticipantType.TERMINAL.value])]) != 2:
+            return (StatusFlags.GREEN, None)
+        
+        times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
+        times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
+
+        violation_state = times_agency.pier_side!=times_terminal.pier_side
+
+        if violation_state:
+            validation_name = inspect.currentframe().f_code.co_name
+            return (StatusFlags.YELLOW, validation_name)
+        else:
+            return (StatusFlags.GREEN, None)
+
+

src/lib_brecal_utils/brecal_utils/validators/validation_rules.py

@@ -1,22 +1,81 @@
 import copy
-from brecal_utils.validators.time_logic import TimeLogic
+import numpy as np
+import pandas as pd
+from brecal_utils.database.enums import StatusFlags
+from brecal_utils.validators.validation_rule_functions import ValidationRuleFunctions
+from BreCal.schemas.model import Shipcall
 
-class ValidationRules():
+
+class ValidationRules(ValidationRuleFunctions):
     """
-    An object that determines the traffic light state for validation and notification. The provided str feedback ('green', 'yellow', 'red')
+    An object that determines the traffic light state for validation and notification. The provided feedback ('green', 'yellow', 'red')
-    determines, whether the state is critical. 
+    determines, whether the state is critical. It uses ValidationRuleState enumerations.
         In case of a critical validation state, the user's input prompt may be interrupted and the user may be warned.
         In case of a critical notification state, the respective users will be automatically notified after n seconds. (#TODO_n_seconds_delay)
     """
-    def __init__(self, shipcall, times): # use the entire data that is provided for this query (e.g., json input)
+    def __init__(self, sql_handler): # use the entire data that is provided for this query (e.g., json input)
-        self.time_logic = TimeLogic()
+        super().__init__(sql_handler)
-        self.shipcall = shipcall
-        self.times = times
 
         self.validation_state = self.determine_validation_state()
-        self.notification_state = self.determine_notification_state() # (state:str, should_notify:bool)
+        # currently flagged: notification_state initially was based on using one ValidationRules object for each query. This is deprecated.
+        # self.notification_state = self.determine_notification_state() # (state:str, should_notify:bool)
         return
     
+    def evaluate(self, shipcall):
+        """
+        1.) prepare df_times, which every validation rule tends to use
+            calling this only once saves a lot of computational overhead
+        2.) apply all validation rules
+        returns: (evaluation_state, violations)
+        """
+        # prepare df_times, which every validation rule tends to use
+        df_times = self.sql_handler.df_dict.get('times') # -> pd.DataFrame
+
+        # filter by shipcall id
+        df_times = self.sql_handler.get_times_of_shipcall(shipcall)
+
+        # apply all validation rules
+        # list of tuples, where each element is (state, msg)
+        evaluation_results = [elem(shipcall, df_times) for elem in self.get_validation_rule_functions()]
+
+        # filter out all 'None' results, which indicate that no violation occured.
+        evaluation_results = [evaluation_result for evaluation_result in evaluation_results if evaluation_result[1] is not None]
+
+        """ # deprecated
+        # check, if ANY of the evaluation results (evaluation_state) is larger than the .GREEN state. This means, that .YELLOW and .RED
+        # would return 'True'. Numpy arrays and functions are used to accelerate the comparison. 
+        # np.any returns a boolean. 
+        #evaluation_state = not np.any(np.greater(np.array([result[0] for result in evaluation_results]), ValidationRuleState.GREEN))
+        """
+        # check, what the maximum state flag is and return it
+        evaluation_state = np.max(np.array([result[0] for result in evaluation_results])) if len(evaluation_results)>0 else 1
+        return (evaluation_state, evaluation_results)
+    
+    def evaluation_verbosity(self, evaluation_state, evaluation_results):
+        """This function suggestions verbosity for the evaluation results. Based on 'True'/'False' evaluation outcome, the returned string is different."""
+        if evaluation_state:
+            return f"OK! The validation was successful. There are no rule violations."
+        else:
+            verbose_string = "These are:" + "\n\t".join(evaluation_results) # every element of the list will be displayed in a new line with a tab
+            return f"FAILED VALIDATION. There have been {len(evaluation_results)} violations. {verbose_string}"
+        
+    def evaluate_shipcall_from_df(self, x):
+        shipcall = Shipcall(**{**{'id':x.name}, **x.to_dict()})
+        evaluation_state, violations = self.evaluate(shipcall)
+        return evaluation_state, violations
+        
+    def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame:
+        """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation' and 'evaluation_message' are updated)"""
+        results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values
+
+        # unbundle individual results. evaluation_state becomes an integer, violation 
+        evaluation_state = [StatusFlags(res[0]).value for res in results]
+        violations = [",".join(res[1]) if len(res[1])>0 else None for res in results]
+
+        shipcall_df.loc[:,"evaluation"] = evaluation_state
+        shipcall_df.loc[:,"evaluation_message"] = violations
+        return shipcall_df
+            
     def determine_validation_state(self) -> str:
         """
         this method determines the validation state of a shipcall. The state is either ['green', 'yellow', 'red'] and signals,
@@ -24,7 +83,7 @@ class ValidationRules():
         
         returns: validation_state_new (str)
         """
-        validation_state_new = self.undefined_method()
+        (validation_state_new, description) = self.undefined_method()
         # should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification. 
         self.validation_state = validation_state_new
         return validation_state_new
@@ -37,7 +96,7 @@ class ValidationRules():
 
         returns: notification_state_new (str), should_notify (bool)
         """
-        state_new = self.undefined_method() # determine the successor
+        (state_new, description) = self.undefined_method() # determine the successor
         should_notify = self.identify_notification_state_change(state_new)
         self.notification_state = state_new # overwrite the predecessor
         return state_new, should_notify
@@ -53,16 +112,16 @@ class ValidationRules():
             yellow -> red
 
             (none -> yellow) or (none -> red)
+            due to the values in the enumeration objects, the states are mapped to provide this function.
+            green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than". 
 
         returns bool, whether a notification should be triggered
         """
-        state_old = copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else 'none'
+        # state_old is always considered at least 'Green' (1)
-
+        state_old = max(copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else StatusFlags.NONE, StatusFlags.GREEN.value)
-        state_mapping = {'none':0, 'green':0, 'yellow':1, 'red':2}
+        return state_new.value > state_old.value
-        return state_mapping[state_new] > state_mapping[state_old]
     
     def undefined_method(self) -> str:
         """this function should apply the ValidationRules to the respective .shipcall, in regards to .times"""
         # #TODO_traffic_state
-        return ('green', False) # (state:str, should_notify:bool)
+        return (StatusFlags.GREEN, False) # (state:str, should_notify:bool)
-

src/lib_brecal_utils/tests/validators/test_schema_validation_berth.py

@@ -2,11 +2,13 @@ import pytest
 from brecal_utils.stubs.berth import get_berth_simple
 
 def test_berth():
-    berth = get_berth_simple()
+    with pytest.raises(ValueError, match="#TODO: copied from ships."):
-    raise ValueError("copied from ships.")
+        berth = get_berth_simple()
-    from brecal_utils.validators.schema_validation import test____
+        
-    ship = get_ship_simple()
+        raise ValueError("#TODO: copied from ships.")
-    ship.length = 234
+        from brecal_utils.validators.schema_validation import test____
-    assert ship_length_in_range(ship)[0], f"ship length must be between 0 and 500 meters"
+        ship = get_ship_simple()
+        ship.length = 234
+        assert ship_length_in_range(ship)[0], f"ship length must be between 0 and 500 meters"
     return
 

src/lib_brecal_utils/tests/validators/test_validation_rule_functions.py

@@ -0,0 +1,159 @@
+import pytest
+from brecal_utils.validators.validation_rule_functions import ValidationRuleFunctions
+from brecal_utils.validators.validation_rules import ValidationRules
+from brecal_utils.database.sql_handler import SQLHandler
+
+@pytest.fixture(scope="session")
+def build_sql_proxy_connection():
+    import mysql.connector
+    conn_from_pool = mysql.connector.connect(**{'host':'localhost', 'port':3306, 'user':'root', 'password':'HalloWach_2323XXL!!', 'pool_name':'brecal_pool', 'pool_size':20, 'database':'bremen_calling', 'autocommit': True})
+    sql_handler = SQLHandler(sql_connection=conn_from_pool, read_all=True)
+    vr = ValidationRules(sql_handler)
+    return locals()
+
+def test_build_validation_rule_functions(build_sql_proxy_connection):
+    import types
+    sql_handler = build_sql_proxy_connection["sql_handler"]
+    vr = build_sql_proxy_connection["vr"]
+
+    validation_rule_functions = vr.get_validation_rule_functions()
+    assert isinstance(validation_rule_functions, list), f"must return a list of methods"
+    for vrule in validation_rule_functions:
+        assert isinstance(vrule,types.MethodType), f"every element returned from get_validation_rule_functions must be a method. found: {type(vrule)}"
+    assert len(validation_rule_functions)>0, f"must return at least one method!"
+    return
+
+
+
+
+def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement(build_sql_proxy_connection):
+    """#0006-A validation_rule_fct_agency_and_terminal_pier_side_disagreement"""
+    import pandas as pd
+
+    from brecal_utils.stubs.times_full import get_times_full_simple
+    from brecal_utils.stubs.shipcall import get_shipcall_simple
+    from brecal_utils.database.enums import ParticipantType
+    from brecal_utils.database.enums import StatusFlags
+
+    vr = build_sql_proxy_connection["vr"]
+    shipcall = get_shipcall_simple()
+    t1 = get_times_full_simple()
+    t2 = get_times_full_simple()
+
+    # roles: agency & terminal
+    t1.participant_type = ParticipantType.AGENCY.value
+    t2.participant_type = ParticipantType.TERMINAL.value
+
+    # disagreement
+    t1.pier_side = True
+    t2.pier_side = False
+
+    time_objects = [t1, t2]
+    df_times = pd.DataFrame.from_records([to_.__dict__ for to_ in time_objects])
+    df_times.set_index('id',inplace=True)
+
+    (state, description) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall, df_times)
+    assert state.value > StatusFlags.GREEN.value, f"a violation must be identified"
+    assert description is not None, f"a violation description must be identified"
+    return
+
+
+def test_validation_rule_fct_agency_and_terminal_pier_side_agreement(build_sql_proxy_connection):
+    """#0006-A validation_rule_fct_agency_and_terminal_pier_side_disagreement"""
+    import pandas as pd
+
+    from brecal_utils.stubs.times_full import get_times_full_simple
+    from brecal_utils.stubs.shipcall import get_shipcall_simple
+    from brecal_utils.database.enums import ParticipantType
+    from brecal_utils.database.enums import StatusFlags
+
+    vr = build_sql_proxy_connection["vr"]
+    shipcall = get_shipcall_simple()
+    t1 = get_times_full_simple()
+    t2 = get_times_full_simple()
+
+    # roles: agency & terminal
+    t1.participant_type = ParticipantType.AGENCY.value
+    t2.participant_type = ParticipantType.TERMINAL.value
+
+    # agreement
+    t1.pier_side = True
+    t2.pier_side = True
+
+    time_objects = [t1, t2]
+    df_times = pd.DataFrame.from_records([to_.__dict__ for to_ in time_objects])
+    df_times.set_index('id',inplace=True)
+
+    (state, description) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall, df_times)
+    assert state.value == StatusFlags.GREEN.value, f"no violation should be observed"
+    assert description is None, f"no violation should be observed"
+    return
+
+
+
+
+def test_validation_rule_fct_agency_and_terminal_berth_id_disagreement(build_sql_proxy_connection):
+    """#0006-B validation_rule_fct_agency_and_terminal_pier_side_disagreement"""
+    import pandas as pd
+
+    from brecal_utils.stubs.times_full import get_times_full_simple
+    from brecal_utils.stubs.shipcall import get_shipcall_simple
+    from brecal_utils.database.enums import ParticipantType
+    from brecal_utils.database.enums import StatusFlags
+
+    vr = build_sql_proxy_connection["vr"]
+    shipcall = get_shipcall_simple()
+    t1 = get_times_full_simple()
+    t2 = get_times_full_simple()
+
+    # roles: agency & terminal
+    t1.participant_type = ParticipantType.AGENCY.value
+    t2.participant_type = ParticipantType.TERMINAL.value
+
+    # disagreement
+    t1.berth_id = 1
+    t2.berth_id = 2
+
+    time_objects = [t1, t2]
+    df_times = pd.DataFrame.from_records([to_.__dict__ for to_ in time_objects])
+    df_times.set_index('id',inplace=True)
+
+    (state, description) = vr.validation_rule_fct_agency_and_terminal_berth_id_disagreement(shipcall, df_times)
+    assert state.value > StatusFlags.GREEN.value, f"a violation must be identified"
+    assert description is not None, f"a violation description must be identified"
+    return
+
+def test_validation_rule_fct_agency_and_terminal_berth_id_agreement(build_sql_proxy_connection):
+    """#0006-B validation_rule_fct_agency_and_terminal_pier_side_disagreement"""
+    import pandas as pd
+
+    from brecal_utils.stubs.times_full import get_times_full_simple
+    from brecal_utils.stubs.shipcall import get_shipcall_simple
+    from brecal_utils.database.enums import ParticipantType
+    from brecal_utils.database.enums import StatusFlags
+
+    vr = build_sql_proxy_connection["vr"]
+    shipcall = get_shipcall_simple()
+    t1 = get_times_full_simple()
+    t2 = get_times_full_simple()
+
+    # roles: agency & terminal
+    t1.participant_type = ParticipantType.AGENCY.value
+    t2.participant_type = ParticipantType.TERMINAL.value
+
+    # agreement
+    t1.berth_id = 21
+    t2.berth_id = 21
+
+    time_objects = [t1, t2]
+    df_times = pd.DataFrame.from_records([to_.__dict__ for to_ in time_objects])
+    df_times.set_index('id',inplace=True)
+
+    (state, description) = vr.validation_rule_fct_agency_and_terminal_berth_id_disagreement(shipcall, df_times)
+    assert state.value == StatusFlags.GREEN.value, f"no violation should be observed"
+    assert description is None, f"no violation should be observed"
+    return
+
+
+
+

src/lib_brecal_utils/tests/validators/test_validation_rule_state.py

@@ -0,0 +1,26 @@
+import pytest
+from brecal_utils.database.enums import StatusFlags
+
+def test_validation_rule_state_green_is_1():
+    assert StatusFlags.GREEN.value==1
+    return
+
+def test_validation_rule_state_yellow_is_2():
+    assert StatusFlags.YELLOW.value==2
+    return
+
+def test_validation_rule_state_red_is_3():
+    assert StatusFlags.RED.value==3
+    return
+
+def test_validation_rule_state_order():
+    # Red 3, Yellow 2, Green 1, None 0
+    # red>yellow>green>none
+    assert StatusFlags.RED.value>StatusFlags.YELLOW.value
+    assert StatusFlags.YELLOW.value>StatusFlags.GREEN.value
+    assert StatusFlags.GREEN.value>StatusFlags.NONE.value
+    return
+
+if __name__=="__main__":
+    pass
+

src/server/BreCal/schemas/model.py

@@ -85,8 +85,6 @@ class ShipcallSchema(Schema):
     anchored = fields.Bool(Required = False, allow_none=True)
     moored_lock = fields.Bool(Required = False, allow_none=True)
     canceled = fields.Bool(Required = False, allow_none=True)
-    validation_state = fields.Str(Required = False, allow_none=True)
-    validation_state_changed = fields.DateTime(Required = False, allow_none=True)
     evaluation = fields.Int(Required = False, allow_none=True)
     evaluation_message = fields.Str(Required = False, allow_none=True)
     participants = fields.List(fields.Int)
@@ -119,8 +117,6 @@ class Shipcall:
     anchored: bool
     moored_lock: bool
     canceled: bool
-    validation_state: str
-    validation_state_changed: datetime
     evaluation: int
     evaluation_message: str
     created: datetime

src/server/requirements.txt

@@ -11,3 +11,8 @@ marshmallow-dataclass
 bcrypt
 jwt
 flask-jwt-extended
+SQLAlchemy
+numpy
+pandas
+
+