From 2b43dc49d964462fe56150bdaa097a522d0c1b8a Mon Sep 17 00:00:00 2001 From: Daniel Schick Date: Mon, 27 Nov 2023 14:37:23 +0100 Subject: [PATCH 01/11] extended gitignore for local flask --- .gitignore | 162 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/.gitignore b/.gitignore index 83afcd3..005dd7b 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,165 @@ __pycache__/ # Local History for Visual Studio Code .history/ + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file From 81a45f57cf94772aeff7c28802429e8205d1fdae Mon Sep 17 00:00:00 2001 From: Daniel Schick Date: Mon, 27 Nov 2023 17:26:37 +0100 Subject: [PATCH 02/11] Catch exception if one occurrs during saving --- src/BreCalClient/MainWindow.xaml.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/BreCalClient/MainWindow.xaml.cs b/src/BreCalClient/MainWindow.xaml.cs index 3402794..9389a70 100644 --- a/src/BreCalClient/MainWindow.xaml.cs +++ b/src/BreCalClient/MainWindow.xaml.cs @@ -189,8 +189,15 @@ namespace BreCalClient esc.ShipcallModel.Shipcall?.Participants.Clear(); foreach (ParticipantAssignment pa in esc.ShipcallModel.AssignedParticipants.Values) esc.ShipcallModel.Shipcall?.Participants.Add(pa); - this._api.ShipcallsPost(esc.ShipcallModel.Shipcall); // save new ship call - this.AddShipcall(esc.ShipcallModel); + try + { + this._api.ShipcallsPost(esc.ShipcallModel.Shipcall); // save new ship call + this.AddShipcall(esc.ShipcallModel); + } + catch(Exception ex) + { + this.ShowErrorDialog(ex.ToString(), ex.Message); + } _refreshImmediately = true; // set flag to avoid timer loop termination _tokenSource.Cancel(); // force timer loop end From c36e9134725d84abccbf66103f4f0e79c8bfde09 Mon Sep 17 00:00:00 2001 From: Daniel Schick Date: Tue, 28 Nov 2023 11:23:46 +0100 Subject: [PATCH 03/11] added trace output and fixed a bug when saving shipcalls without times --- src/server/BreCal/database/sql_handler.py | 2 ++ src/server/BreCal/impl/shipcalls.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index f34f345..c228e8c 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -218,6 +218,8 @@ class SQLHandler(): def get_times_of_shipcall(self, shipcall)->pd.DataFrame: df_times = self.df_dict.get('times') # -> pd.DataFrame + if df_times is None: + return None df_times = df_times.loc[df_times["shipcall_id"]==shipcall.id] return df_times diff --git a/src/server/BreCal/impl/shipcalls.py b/src/server/BreCal/impl/shipcalls.py index dd9cc3f..452a8be 100644 --- a/src/server/BreCal/impl/shipcalls.py +++ b/src/server/BreCal/impl/shipcalls.py @@ -1,6 +1,6 @@ import json import logging - +import traceback import pydapper from ..schemas import model @@ -36,6 +36,7 @@ def GetShipcalls(options): pooledConnection.close() except Exception as ex: + logging.error(traceback.format_exc()) logging.error(ex) print(ex) result = {} @@ -115,6 +116,7 @@ def PostShipcalls(schemaModel): return json.dumps({"id" : new_id}), 201, {'Content-Type': 'application/json; charset=utf-8'} except Exception as ex: + logging.error(traceback.format_exc()) logging.error(ex) print(ex) result = {} @@ -199,6 +201,7 @@ def PutShipcalls(schemaModel): return json.dumps({"id" : schemaModel["id"]}), 200 except Exception as ex: + logging.error(traceback.format_exc()) logging.error(ex) print(ex) result = {} From 8027620f22a85ed9555386ad1953bffd5fde363e Mon Sep 17 00:00:00 2001 From: scopesorting Date: Tue, 28 Nov 2023 14:17:07 +0100 Subject: [PATCH 04/11] fixing the 'KeyError' when using an empty times dataframe. Returning 'green' --- src/server/BreCal/database/sql_handler.py | 2 -- src/server/BreCal/validators/validation_rules.py | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index c228e8c..f34f345 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -218,8 +218,6 @@ class SQLHandler(): def get_times_of_shipcall(self, shipcall)->pd.DataFrame: df_times = self.df_dict.get('times') # -> pd.DataFrame - if df_times is None: - return None df_times = df_times.loc[df_times["shipcall_id"]==shipcall.id] return df_times diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index 2753446..febbff4 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -31,6 +31,9 @@ class ValidationRules(ValidationRuleFunctions): # prepare df_times, which every validation rule tends to use df_times = self.sql_handler.df_dict.get('times') # -> pd.DataFrame + if len(df_times)==0: + return (StatusFlags.GREEN.value, []) + # filter by shipcall id df_times = self.sql_handler.get_times_of_shipcall(shipcall) From 834da84786f690ed3b069bb2d6d510cf4db5886c Mon Sep 17 00:00:00 2001 From: scopesorting Date: Tue, 28 Nov 2023 14:26:38 +0100 Subject: [PATCH 05/11] minot adjustments & refactoring --- src/server/BreCal/validators/validation_rules.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index febbff4..ba808e8 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -46,15 +46,9 @@ class ValidationRules(ValidationRuleFunctions): # 'translate' all error codes into readable, human-understandable format. evaluation_results = [(state, self.describe_error_message(msg)) for (state, msg) in evaluation_results] - - """ # deprecated - # check, if ANY of the evaluation results (evaluation_state) is larger than the .GREEN state. This means, that .YELLOW and .RED - # would return 'True'. Numpy arrays and functions are used to accelerate the comparison. - # np.any returns a boolean. - #evaluation_state = not np.any(np.greater(np.array([result[0] for result in evaluation_results]), ValidationRuleState.GREEN)) - """ + # check, what the maximum state flag is and return it - evaluation_state = np.max(np.array([result[0].value for result in evaluation_results])) if len(evaluation_results)>0 else 1 + evaluation_state = np.max(np.array([result[0].value for result in evaluation_results])) if len(evaluation_results)>0 else StatusFlags.GREEN.value evaluation_verbosity = [result[1] for result in evaluation_results] return (evaluation_state, evaluation_verbosity) From 4859ff6803ee0e23974b0155cda6a77a57f087bc Mon Sep 17 00:00:00 2001 From: scopesorting Date: Tue, 28 Nov 2023 14:49:40 +0100 Subject: [PATCH 06/11] ensuring that len(df_times) always works. Preventing 'None' from occuring --- src/server/BreCal/validators/validation_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index ba808e8..b90f691 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -29,7 +29,7 @@ class ValidationRules(ValidationRuleFunctions): returns: (evaluation_state, violations) """ # prepare df_times, which every validation rule tends to use - df_times = self.sql_handler.df_dict.get('times') # -> pd.DataFrame + df_times = self.sql_handler.df_dict.get('times', pd.DataFrame()) # -> pd.DataFrame if len(df_times)==0: return (StatusFlags.GREEN.value, []) From 9f61e2af38db6a27167a78f423334f600e056d1c Mon Sep 17 00:00:00 2001 From: scopesorting Date: Tue, 28 Nov 2023 16:48:20 +0100 Subject: [PATCH 07/11] updating check_time_delta_violation_query_time_to_now: no longer ignoring events of the past (delta<=0) --- src/server/BreCal/validators/validation_rule_functions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index e1c59ff..c8f00c6 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -99,9 +99,8 @@ class ValidationRuleBaseFunctions(): delta = self.time_logic.time_delta_from_now_to_tgt(tgt_time=query_time, unit="m") # a violation occurs, when the delta (in minutes) exceeds the specified threshold of a participant - # to prevent past-events from triggering violations, negative values are ignored - # Violation, if 0 <= delta <= threshold - violation_state = (delta >= 0) and (delta<=threshold) + # Violation, if delta <= threshold + violation_state = (delta<=threshold) return violation_state def check_participants_agree_on_estimated_time(self, shipcall, query, df_times, applicable_shipcall_type)->bool: From 9a12d74e7746f358bd9fffb517116b8221099a3a Mon Sep 17 00:00:00 2001 From: scopesorting Date: Wed, 29 Nov 2023 08:52:37 +0100 Subject: [PATCH 08/11] updating validation rules 0001 A-M. Instead of filtering by times_df (which may not exist), the rules make use of the shipcall_participant_map. When one of the participants in a rule is not assigned, no violation is observed. When there are multiple entries of a participant (due to an input bug), the function still verifies properly. When critical time is observed, and there is not yet an entry for the respective key time, there will be a 'yellow' state. --- src/server/BreCal/database/sql_handler.py | 60 ++++ .../validators/validation_rule_functions.py | 71 +++-- .../test_validation_rule_functions.py | 275 ++++++++++++++++++ 3 files changed, 375 insertions(+), 31 deletions(-) diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index f34f345..188d98a 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -7,6 +7,19 @@ from BreCal.database.enums import ParticipantType def pandas_series_to_data_model(): return +def set_participant_type(x, participant_df)->int: + """ + when iterating over each row entry x in the shipcall_participant_map, + one can update the 'type' column by extracting the matching data from a participant dataframe + + returns: participant_type + """ + participant_id = x["participant_id"] + participant_type = participant_df.loc[participant_id, "type"] + return participant_type + + + class SQLHandler(): """ An object that reads SQL queries from the sql_connection and stores it in pandas DataFrames. The object can read all available tables @@ -95,6 +108,9 @@ class SQLHandler(): # update the 'participants' column in 'shipcall' self.initialize_shipcall_participant_list() + + # update the 'type' in shipcall_participants_map + self.add_participant_type_to_map() return def build_full_mysql_df_dict(self, all_schemas): @@ -121,6 +137,50 @@ class SQLHandler(): lambda x: self.get_participants(x.name), axis=1) return + + def add_participant_type_to_map(self): + """ + applies a lambda function, where the 'type'-column in the shipcall_participant_map is updated by reading the + respective data from the participants. Updates the shipcall_participant_map inplace. + """ + spm = self.df_dict["shipcall_participant_map"] + participant_df = self.df_dict["participant"] + + spm.loc[:,"type"] = spm.loc[:].apply(lambda x: set_participant_type(x, participant_df=participant_df),axis=1) + self.df_dict["shipcall_participant_map"] = spm + return + + def get_assigned_participants(self, shipcall)->pd.DataFrame: + """return each participant of a respective shipcall, filtered by the shipcall id""" + # get the shipcall_participant_map + spm = self.df_dict["shipcall_participant_map"] + assigned_participants = spm.loc[spm["shipcall_id"]==shipcall.id] + return assigned_participants + + def get_assigned_participants_by_type(self, assigned_participants:pd.DataFrame, participant_type:ParticipantType): + """filters a dataframe of assigned_participants by the provided type enumerator""" + assigned_participants_of_type = assigned_participants.loc[assigned_participants["type"]==participant_type.value] + return assigned_participants_of_type + + def check_if_any_participant_of_type_is_unassigned(self, shipcall, *args:list[ParticipantType])->bool: + """ + given a list of input arguments, where item is a participant type, the function determines, whether at least one participant + was assigned for the type. Function returns a boolean, whether any of the required participants in unassigned. + + This method is extensively used for the validation rules 0001, where the header is checked beforehand to identify, whether + the respective participant type is assigned already. + """ + print("verbosity of function check_if_any_participant_of_type_is_unassigned") + assigned_participants = self.get_assigned_participants(shipcall) + + unassigned = [] # becomes a list of booleans + for participant_type in args: + assignments_of_type = self.get_assigned_participants_by_type(assigned_participants, participant_type=participant_type) + unassignment = len(assignments_of_type)==0 # a participant type does not exist, when there is no match + unassigned.append(unassignment) + print("participant type and unassigment state", participant_type, unassignment) + return any(unassigned) # returns a single boolean, whether ANY of the types is not assigned + def standardize_model_str(self, model_str:str)->str: """check if the 'model_str' is valid and apply lowercasing to the string""" diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index c8f00c6..7148254 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -210,8 +210,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.eta_berth is filled in. - Measures the difference between 'now' and 'shipcall.eta'. """ - # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -238,8 +239,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.etd_berth is filled in. - Measures the difference between 'now' and 'shipcall.etd'. """ - # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -266,10 +268,11 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & MOORING) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) + if unassigned: return self.get_no_violation_default_output() - + # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) @@ -296,10 +299,11 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & MOORING) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) + if unassigned: return self.get_no_violation_default_output() - + # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) @@ -326,8 +330,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & PORT_ADMINISTRATION) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -356,8 +361,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & PORT_ADMINISTRATION) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -386,8 +392,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & PILOT) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -416,8 +423,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & PILOT) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -446,8 +454,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & TUG) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -476,8 +485,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & TUG) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value]) != 1): + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) + if unassigned: return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time @@ -506,9 +516,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_start is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ - # check, if the header is filled in (agency & terminal) - if (len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1) or (len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1): - return self.get_no_violation_default_output() # rule not applicable + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) + if unassigned: + return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) @@ -536,12 +547,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_end is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ - # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: - return self.get_no_violation_default_output() # rule not applicable - - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: - return self.get_no_violation_default_output() # rule not applicable + # check, if the header is filled in + unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) + if unassigned: + return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index 1847973..4b32acd 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -117,6 +117,19 @@ def test_validation_rule_fct_missing_time_agency_berth_eta__shipcall_eta_dangero # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_eta(shipcall=shipcall, df_times=df_times) @@ -137,6 +150,19 @@ def test_validation_rule_fct_missing_time_agency_berth_eta__shipcall_eta_distant # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_eta(shipcall=shipcall, df_times=df_times) @@ -157,6 +183,19 @@ def test_validation_rule_fct_missing_time_agency_berth_eta__shipcall_eta_is_unde # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_eta(shipcall=shipcall, df_times=df_times) @@ -177,6 +216,19 @@ def test_validation_rule_fct_missing_time_agency_berth_etd__shipcall_etd_is_unde # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_agency_berth_etd(shipcall=shipcall, df_times=df_times) @@ -197,6 +249,21 @@ def test_validation_rule_fct_missing_time_mooring_berth_eta__shipcall_soon_but_p # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + mooring_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":mooring_participant_id, "type":ParticipantType.MOORING.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_mooring_berth_eta(shipcall=shipcall, df_times=df_times) @@ -219,6 +286,21 @@ def test_validation_rule_fct_missing_time_mooring_berth_etd__shipcall_soon_but_p # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + mooring_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.MOORING.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":mooring_participant_id, "type":ParticipantType.MOORING.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_mooring_berth_etd(shipcall=shipcall, df_times=df_times) @@ -241,6 +323,21 @@ def test_validation_rule_fct_missing_time_portadministration_berth_eta__shipcall # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pa_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pa_participant_id, "type":ParticipantType.PORT_ADMINISTRATION.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_portadministration_berth_eta(shipcall=shipcall, df_times=df_times) @@ -263,6 +360,21 @@ def test_validation_rule_fct_missing_time_portadministration_berth_etd__shipcall # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pa_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PORT_ADMINISTRATION.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pa_participant_id, "type":ParticipantType.PORT_ADMINISTRATION.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_portadministration_berth_etd(shipcall=shipcall, df_times=df_times) @@ -283,6 +395,21 @@ def test_validation_rule_fct_missing_time_pilot_berth_eta__shipcall_soon_but_par # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_eta(shipcall=shipcall, df_times=df_times) @@ -303,6 +430,21 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) @@ -310,7 +452,79 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" return +def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_participant_unassigned__return_green(build_sql_proxy_connection): + """0001-I validation_rule_fct_missing_time_pilot_berth_etd""" + vr = build_sql_proxy_connection['vr'] + shipcall = get_shipcall_simple() + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # set times agency to be undetermined + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "etd_berth"] = None + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + + vr.sql_handler.read_all(vr.sql_handler.all_schemas) + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.GREEN, f"function should return 'green', because the pilot is not assigned yet" + return + +def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_participant_estimated_time_undefined_multiple_pilot_assignments_due_to_bug(build_sql_proxy_connection): + """ + 0001-I validation_rule_fct_missing_time_pilot_berth_etd. Checks, whether the function still works in case of a buggy input. When there is more than one pilot + assignment, the validation rule should still work and return 'yellow' properly. + """ + vr = build_sql_proxy_connection['vr'] + + shipcall = get_shipcall_simple() + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # set times agency to be undetermined + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "etd_berth"] = None + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10003, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10004, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" + return def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_participant_estimated_time_undefined(build_sql_proxy_connection): """0001-J validation_rule_fct_missing_time_tug_berth_eta""" @@ -325,6 +539,21 @@ def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_parti # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "eta_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + tug_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":tug_participant_id, "type":ParticipantType.TUG.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_tug_berth_eta(shipcall=shipcall, df_times=df_times) @@ -347,6 +576,21 @@ def test_validation_rule_fct_missing_time_tug_berth_etd__shipcall_soon_but_parti # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "etd_berth"] = None + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + tug_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TUG.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":tug_participant_id, "type":ParticipantType.TUG.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_tug_berth_etd(shipcall=shipcall, df_times=df_times) @@ -369,6 +613,21 @@ def test_validation_rule_fct_missing_time_terminal_berth_eta__shipcall_soon_but_ # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "operations_start"] = None # previously: eta_berth, which does not exist in times_terminal + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + terminal_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":terminal_participant_id, "type":ParticipantType.TERMINAL.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_terminal_berth_eta(shipcall=shipcall, df_times=df_times) @@ -391,6 +650,22 @@ def test_validation_rule_fct_missing_time_terminal_berth_etd__shipcall_soon_but_ # set times agency to be undetermined df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "operations_end"] = None # previously: etd_berth, which does not exist in times_terminal + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + terminal_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "participant_id"].iloc[0] + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":terminal_participant_id, "type":ParticipantType.TERMINAL.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + # apply the validation rule (state, msg) = vr.validation_rule_fct_missing_time_terminal_berth_etd(shipcall=shipcall, df_times=df_times) From 6f9b4a6b5afd6f8161b5d3911a185e3e2b8e3421 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Wed, 29 Nov 2023 15:21:20 +0100 Subject: [PATCH 09/11] removing verbosity in validation rule functions, and returning 'None', when a selected times dataframe is empty. In case of empty results, the function now properly computes the delta towards a query time and returns YELLOW, when a violation is observed. This should finally fix the bugs for 0001 A-M --- src/server/BreCal/database/sql_handler.py | 6 +-- .../validators/validation_rule_functions.py | 45 ++++++++++--------- .../test_validation_rule_functions.py | 38 ++++++++++++++++ 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index 188d98a..e473a79 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -170,7 +170,6 @@ class SQLHandler(): This method is extensively used for the validation rules 0001, where the header is checked beforehand to identify, whether the respective participant type is assigned already. """ - print("verbosity of function check_if_any_participant_of_type_is_unassigned") assigned_participants = self.get_assigned_participants(shipcall) unassigned = [] # becomes a list of booleans @@ -178,9 +177,7 @@ class SQLHandler(): assignments_of_type = self.get_assigned_participants_by_type(assigned_participants, participant_type=participant_type) unassignment = len(assignments_of_type)==0 # a participant type does not exist, when there is no match unassigned.append(unassignment) - print("participant type and unassigment state", participant_type, unassignment) return any(unassigned) # returns a single boolean, whether ANY of the types is not assigned - def standardize_model_str(self, model_str:str)->str: """check if the 'model_str' is valid and apply lowercasing to the string""" @@ -240,6 +237,9 @@ class SQLHandler(): def get_times_for_participant_type(self, df_times, participant_type:int): filtered_series = df_times.loc[df_times["participant_type"]==participant_type] + if len(filtered_series)==0: + return None + if not len(filtered_series)<=1: # correcting the error: ERROR:root:found multiple results # however, a warning will still be issued diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index 7148254..2769927 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -218,7 +218,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.eta - key_time = times_agency.eta_berth + key_time = times_agency.eta_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -247,7 +247,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): # preparation: obtain the correct times of the participant, define the query time and the key time times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) query_time = shipcall.etd - key_time = times_agency.etd_berth + key_time = times_agency.etd_berth if times_agency is not None else None threshold = ParticipantwiseTimeDelta.AGENCY violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -277,8 +277,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) - query_time = times_agency.eta_berth - key_time = times_mooring.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_mooring.eta_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -308,8 +308,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_mooring = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.MOORING.value) - query_time = times_agency.etd_berth - key_time = times_mooring.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_mooring.etd_berth if times_mooring is not None else None threshold = ParticipantwiseTimeDelta.MOORING violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -339,8 +339,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) - query_time = times_agency.eta_berth - key_time = times_port_administration.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_port_administration.eta_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -367,11 +367,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # preparation: obtain the correct times of the participant, define the query time and the key time + # when there are no times, the function returns None times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_port_administration = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PORT_ADMINISTRATION.value) - query_time = times_agency.etd_berth - key_time = times_port_administration.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_port_administration.etd_berth if times_port_administration is not None else None threshold = ParticipantwiseTimeDelta.PORT_ADMINISTRATION violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -401,8 +402,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) - query_time = times_agency.eta_berth - key_time = times_pilot.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_pilot.eta_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -432,8 +433,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_pilot = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.PILOT.value) - query_time = times_agency.etd_berth - key_time = times_pilot.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_pilot.etd_berth if times_pilot is not None else None threshold = ParticipantwiseTimeDelta.PILOT violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -463,8 +464,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) - query_time = times_agency.eta_berth - key_time = times_tug.eta_berth + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_tug.eta_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -494,8 +495,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_tug = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TUG.value) - query_time = times_agency.etd_berth - key_time = times_tug.etd_berth + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_tug.etd_berth if times_tug is not None else None threshold = ParticipantwiseTimeDelta.TUG violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -525,8 +526,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - query_time = times_agency.eta_berth - key_time = times_terminal.operations_start # eta_berth does not exist in times_terminal! Instead, it is called operations_start + query_time = times_agency.eta_berth if times_agency is not None else None + key_time = times_terminal.operations_start if times_terminal is not None else None # eta_berth does not exist in times_terminal! Instead, it is called operations_start threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) @@ -556,8 +557,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) - query_time = times_agency.etd_berth - key_time = times_terminal.operations_end # etd_berth does not exist in times_terminal! Instead, it is called operations_end + query_time = times_agency.etd_berth if times_agency is not None else None + key_time = times_terminal.operations_end if times_terminal is not None else None # etd_berth does not exist in times_terminal! Instead, it is called operations_end threshold = ParticipantwiseTimeDelta.TERMINAL violation_state = self.check_time_delta_violation_query_time_to_now(query_time=query_time, key_time=key_time, threshold=threshold) diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index 4b32acd..ab85502 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -526,6 +526,44 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" return +def test_validation_rule_fct_missing_time_pilot_berth_etd__agency_and_pilot_assigned_pilot_no_times_returns_yellow(build_sql_proxy_connection): + """ + 0001-I validation_rule_fct_missing_time_pilot_berth_etd. Checks the default behaviour, where an agency's time might exist, + while a time by pilot may not exist. In these cases, a yellow state is expected. + """ + vr = build_sql_proxy_connection['vr'] + + shipcall = get_shipcall_simple() + df_times = get_df_times(shipcall) + + # according to the agency, a shipcall takes place soon (ETA/ETD) + df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "etd_berth"] = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.PILOT-10) + + # must adapt the shipcall_participant_map, so it suits the test + agency_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "participant_id"].iloc[0] + pilot_participant_id = df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value, "participant_id"].iloc[0] + + # set times of PILOT: should not exist + df_times.loc[df_times["participant_type"]==ParticipantType.PILOT.value,"participant_type"] = ParticipantType.BSMD.value + + spm = vr.sql_handler.df_dict["shipcall_participant_map"] + df = pd.DataFrame( + [ + {"id":10001, "shipcall_id":shipcall.id, "participant_id":agency_participant_id, "type":ParticipantType.AGENCY.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None}, + {"id":10002, "shipcall_id":shipcall.id, "participant_id":pilot_participant_id, "type":ParticipantType.PILOT.value, "created":pd.Timestamp(datetime.datetime.now().isoformat()), "modified":None} + ] + ) + df.set_index("id", inplace=True) + spm = pd.concat([spm, df], axis=0, ignore_index=True) + vr.sql_handler.df_dict["shipcall_participant_map"] = spm + + # apply the validation rule + (state, msg) = vr.validation_rule_fct_missing_time_pilot_berth_etd(shipcall=shipcall, df_times=df_times) + + # expectation: green state, no msg + assert state==StatusFlags.YELLOW, f"function should return 'yellow', because the participant did not provide a time and the shipcall takes place soon (according to the agency)" + return + def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_participant_estimated_time_undefined(build_sql_proxy_connection): """0001-J validation_rule_fct_missing_time_tug_berth_eta""" vr = build_sql_proxy_connection['vr'] From fce8ce0c6885cd455ba5b6dcf157b1ec160e4743 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Thu, 30 Nov 2023 15:53:42 +0100 Subject: [PATCH 10/11] changing the ParticipantType to an IntFlag, so multiple roles are possible. Adapting every validation rule (0001, 0003, 0004, 0005), which may be affected by this change. Changing the filter for a participant type to properly include the change. Changing the pier_side rule (0006B), which uses the shipcall and times_terminal. New shipcalls should now be evaluated properly, unless no participant is assigned at all. If the ladder case can occur, the validation rules 0001N+0001O will be added (held back for now). --- src/server/BreCal/database/enums.py | 4 +- src/server/BreCal/database/sql_handler.py | 41 +++++++++++++++---- .../validators/validation_rule_functions.py | 6 +-- .../BreCal/validators/validation_rules.py | 4 ++ .../test_validation_rule_functions.py | 15 ++++--- 5 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index e038643..3092fd8 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -1,6 +1,6 @@ -from enum import Enum +from enum import Enum, IntFlag -class ParticipantType(Enum): +class ParticipantType(IntFlag): """determines the type of a participant""" NONE = 0 BSMD = 1 diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index e473a79..59497e3 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd import datetime +import typing from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times from BreCal.database.enums import ParticipantType @@ -110,7 +111,8 @@ class SQLHandler(): self.initialize_shipcall_participant_list() # update the 'type' in shipcall_participants_map - self.add_participant_type_to_map() + # fully deprecated + # self.add_participant_type_to_map() return def build_full_mysql_df_dict(self, all_schemas): @@ -143,11 +145,12 @@ class SQLHandler(): applies a lambda function, where the 'type'-column in the shipcall_participant_map is updated by reading the respective data from the participants. Updates the shipcall_participant_map inplace. """ - spm = self.df_dict["shipcall_participant_map"] - participant_df = self.df_dict["participant"] + raise Exception("deprecated! Overwriting the shipcall_participant_map may cause harm, as a participant with multi-flag might be wrongfully assigned to multiple roles simultaneously.") + #spm = self.df_dict["shipcall_participant_map"] + #participant_df = self.df_dict["participant"] - spm.loc[:,"type"] = spm.loc[:].apply(lambda x: set_participant_type(x, participant_df=participant_df),axis=1) - self.df_dict["shipcall_participant_map"] = spm + #spm.loc[:,"type"] = spm.loc[:].apply(lambda x: set_participant_type(x, participant_df=participant_df),axis=1) + #self.df_dict["shipcall_participant_map"] = spm return def get_assigned_participants(self, shipcall)->pd.DataFrame: @@ -159,7 +162,11 @@ class SQLHandler(): def get_assigned_participants_by_type(self, assigned_participants:pd.DataFrame, participant_type:ParticipantType): """filters a dataframe of assigned_participants by the provided type enumerator""" - assigned_participants_of_type = assigned_participants.loc[assigned_participants["type"]==participant_type.value] + if isinstance(participant_type,int): + participant_type = ParticipantType(participant_type) + + assigned_participants_of_type = assigned_participants.loc[[participant_type in ParticipantType(int(pt_)) for pt_ in list(assigned_participants["type"].values)]] + #assigned_participants_of_type = assigned_participants.loc[assigned_participants["type"]==participant_type.value] return assigned_participants_of_type def check_if_any_participant_of_type_is_unassigned(self, shipcall, *args:list[ParticipantType])->bool: @@ -234,8 +241,25 @@ class SQLHandler(): data = data_model(**data) return data + def filter_df_by_participant_type(self, df, participant_type:typing.Union[int, ParticipantType])->pd.DataFrame: + """ + As ParticipantTypes are Flag objects, a dataframe's integer might resemble multiple participant types simultaneously. + This function allows for more complex filters, as the IntFlag allows more complex queries + + e.g.: + ParticipantType(6) is 2,4 (2+4 = 6) + + Participant(2) in Participant(6) = True # 6 is both, 2 and 4 + Participant(1) in Participant(6) = False # 6 is both, 2 and 4, but not 1 + """ + if isinstance(participant_type,int): + participant_type = ParticipantType(participant_type) + filtered_df = df.loc[[participant_type in ParticipantType(df_pt) for df_pt in list(df["participant_type"].values)]] + return filtered_df + def get_times_for_participant_type(self, df_times, participant_type:int): - filtered_series = df_times.loc[df_times["participant_type"]==participant_type] + filtered_series = self.filter_df_by_participant_type(df_times, participant_type) + #filtered_series = df_times.loc[df_times["participant_type"]==participant_type] if len(filtered_series)==0: return None @@ -299,7 +323,8 @@ class SQLHandler(): df_times = df_times.loc[~df_times[non_null_column].isnull()] # NOT null filter # filter by the agency participant_type - times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] + times_agency = self.filter_df_by_participant_type(df_times, ParticipantType.AGENCY.value) + #times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] return times_agency def filter_df_by_key_value(self, df, key, value)->pd.DataFrame: diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index 2769927..a7993cd 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -866,18 +866,18 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) # when one of the two values is null, the state is GREEN - if (times_agency.pier_side is None) or (times_terminal.pier_side is None): + if (shipcall.pier_side is None) or (times_terminal.pier_side is None): return self.get_no_violation_default_output() # when one of the two values is null, the state is GREEN - if (pd.isnull(times_agency.pier_side)) or (pd.isnull(times_terminal.pier_side)): + if (pd.isnull(shipcall.pier_side)) or (pd.isnull(times_terminal.pier_side)): return self.get_no_violation_default_output() # only incoming shipcalls matter. The other ones are not relevant for the pier_side selection if shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: return self.get_no_violation_default_output() - violation_state = bool(times_agency.pier_side)!=bool(times_terminal.pier_side) + violation_state = bool(shipcall.pier_side)!=bool(times_terminal.pier_side) if violation_state: validation_name = "validation_rule_fct_agency_and_terminal_pier_side_disagreement" diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index b90f691..cabe5ec 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -33,6 +33,10 @@ class ValidationRules(ValidationRuleFunctions): if len(df_times)==0: return (StatusFlags.GREEN.value, []) + + spm = self.sql_handler.df_dict["shipcall_participant_map"] + if len(spm.loc[spm["shipcall_id"]==shipcall.id])==0: + return (StatusFlags.GREEN.value, []) # filter by shipcall id df_times = self.sql_handler.get_times_of_shipcall(shipcall) diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index ab85502..5c4e3a0 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -1114,7 +1114,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement__agency_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() df_times = get_df_times(shipcall) - df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True + shipcall.pier_side = True + # df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "pier_side"] = True (code, msg) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall=shipcall, df_times=df_times) @@ -1126,7 +1127,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement__agency_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() df_times = get_df_times(shipcall) - df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True + shipcall.pier_side = True + #df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "pier_side"] = False (code, msg) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall=shipcall, df_times=df_times) @@ -1138,7 +1140,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement__agency_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() df_times = get_df_times(shipcall) - df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True + shipcall.pier_side = True + # df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value, "pier_side"] = True df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value, "pier_side"] = None (code, msg) = vr.validation_rule_fct_agency_and_terminal_pier_side_disagreement(shipcall=shipcall, df_times=df_times) @@ -1178,7 +1181,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_agreement(build_sql_p t2.participant_type = ParticipantType.TERMINAL.value # agreement - t1.pier_side = True + shipcall.pier_side = True + # t1.pier_side = True t2.pier_side = True time_objects = [t1, t2] @@ -1209,7 +1213,8 @@ def test_validation_rule_fct_agency_and_terminal_pier_side_disagreement(build_sq t2.participant_type = ParticipantType.TERMINAL.value # disagreement - t1.pier_side = True + shipcall.pier_side = True + # t1.pier_side = True t2.pier_side = False time_objects = [t1, t2] From ad593ff2a2ce64bfd3368f231b4ac0a102437b8a Mon Sep 17 00:00:00 2001 From: scopesorting Date: Thu, 30 Nov 2023 17:34:44 +0100 Subject: [PATCH 11/11] more concise evaluation messages for 0001. Adding newlines (works on Windows) when multiple evaluation messages are shown. Properly adding the ShipcallType filters for each rule (whether incoming, outgoing or shifting). Added a regular expression to abbreviate an evaluation message when 512 characters are exceeded. --- .../validators/validation_rule_functions.py | 72 +++++++++++++++---- .../BreCal/validators/validation_rules.py | 19 ++++- .../test_validation_rule_functions.py | 13 ++++ 3 files changed, 91 insertions(+), 13 deletions(-) diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index a7993cd..92bf24e 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -10,18 +10,18 @@ from BreCal.database.enums import StatusFlags # a human interpretable dictionary for error messages. In this case, the English language is preferred error_message_dict = { # 0001 A-M - "validation_rule_fct_missing_time_agency_berth_eta":"The shipcall arrives in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of arrival (ETA) {Rule #0001A}", # A - "validation_rule_fct_missing_time_agency_berth_etd":"The shipcall departs in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of departure (ETD) {Rule #0001B}", # B - "validation_rule_fct_missing_time_mooring_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of arrival (ETA) {Rule #0001C}", # C - "validation_rule_fct_missing_time_mooring_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of departure (ETD) {Rule #0001D}", # D - "validation_rule_fct_missing_time_portadministration_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of arrival (ETA) {Rule #0001F}", # F - "validation_rule_fct_missing_time_portadministration_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of departure (ETD) {Rule #0001G}", # G - "validation_rule_fct_missing_time_pilot_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of arrival (ETA) {Rule #0001H}", # H - "validation_rule_fct_missing_time_pilot_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of departure (ETD) {Rule #0001I}", # I - "validation_rule_fct_missing_time_tug_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of arrival (ETA) {Rule #0001J}", # J - "validation_rule_fct_missing_time_tug_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of departure (ETD) {Rule #0001K}", # K - "validation_rule_fct_missing_time_terminal_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of arrival (ETA) {Rule #0001L}", # L - "validation_rule_fct_missing_time_terminal_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of departure (ETD) {Rule #0001M}", # M + "validation_rule_fct_missing_time_agency_berth_eta":"Shipcall arrives soon (<20 hours). The agency did not provide a time yet (ETA) {Rule #0001A}", # A + "validation_rule_fct_missing_time_agency_berth_etd":"Shipcall departs soon (<20 hours). The agency did not provide a time yet (ETD) {Rule #0001B}", # B + "validation_rule_fct_missing_time_mooring_berth_eta":"Shipcall arrives soon (<16 hours). The mooring did not provide a time yet (ETA) {Rule #0001C}", # C + "validation_rule_fct_missing_time_mooring_berth_etd":"Shipcall departs soon (<16 hours). The mooring did not provide a time yet (ETD) {Rule #0001D}", # D + "validation_rule_fct_missing_time_portadministration_berth_eta":"Shipcall arrives soon (<16 hours). The port administration did not provide a time yet (ETA) {Rule #0001F}", # F + "validation_rule_fct_missing_time_portadministration_berth_etd":"Shipcall departs soon (<20 hours). The port administration did not provide a time yet (ETD) {Rule #0001G}", # G + "validation_rule_fct_missing_time_pilot_berth_eta":"Shipcall arrives soon (<16 hours). The pilot did not provide a time yet (ETA) {Rule #0001H}", # H + "validation_rule_fct_missing_time_pilot_berth_etd":"Shipcall departs soon (<20 hours). The pilot did not provide a time yet (ETD) {Rule #0001I}", # I + "validation_rule_fct_missing_time_tug_berth_eta":"Shipcall arrives soon (<16 hours). The tugs did not provide a time yet (ETA) {Rule #0001J}", # J + "validation_rule_fct_missing_time_tug_berth_etd":"Shipcall departs soon (<20 hours). The tugs did not provide a time yet (ETD) {Rule #0001K}", # K + "validation_rule_fct_missing_time_terminal_berth_eta":"Shipcall arrives soon (<16 hours). The terminal did not provide a time yet (ETA) {Rule #0001L}", # L + "validation_rule_fct_missing_time_terminal_berth_etd":"Shipcall departs soon (<20 hours). The terminal did not provide a time yet (ETD) {Rule #0001M}", # M # 0002 A+B+C "validation_rule_fct_shipcall_incoming_participants_disagree_on_eta":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of arrival (ETA) {Rule #0002A}", @@ -210,6 +210,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.eta_berth is filled in. - Measures the difference between 'now' and 'shipcall.eta'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) if unassigned: @@ -239,6 +242,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_agency.etd_berth is filled in. - Measures the difference between 'now' and 'shipcall.etd'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY]) if unassigned: @@ -268,6 +274,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) if unassigned: @@ -299,6 +308,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_mooring.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.MOORING]) if unassigned: @@ -330,6 +342,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) if unassigned: @@ -361,6 +376,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_port_administration.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PORT_ADMINISTRATION]) if unassigned: @@ -393,6 +411,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) if unassigned: @@ -424,6 +445,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_pilot.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.PILOT]) if unassigned: @@ -455,6 +479,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.eta_berth is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) if unassigned: @@ -486,6 +513,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_tug.etd_berth is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TUG]) if unassigned: @@ -517,6 +547,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_start is filled in. - Measures the difference between 'now' and 'times_agency.eta_berth'. """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) if unassigned: @@ -548,6 +581,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): - Checks, if times_terminal.operations_end is filled in. - Measures the difference between 'now' and 'times_agency.etd_berth'. """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in unassigned = self.sql_handler.check_if_any_participant_of_type_is_unassigned(shipcall, *[ParticipantType.AGENCY, ParticipantType.TERMINAL]) if unassigned: @@ -660,6 +696,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency & terminal) if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable @@ -692,6 +731,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: operations_start & operations_end (times_terminal) """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency & terminal) if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable @@ -724,6 +766,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ + if not shipcall.type in [ShipcallType.INCOMING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return self.get_no_violation_default_output() @@ -752,6 +797,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): query time: eta_berth (times_agency) start_time & end_time: tidal_window_from & tidal_window_to (shipcall) """ + if not shipcall.type in [ShipcallType.OUTGOING.value, ShipcallType.SHIFTING.value]: + return self.get_no_violation_default_output() + # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return self.get_no_violation_default_output() diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index cabe5ec..aa2ef22 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -1,4 +1,5 @@ import copy +import re import numpy as np import pandas as pd from BreCal.database.enums import StatusFlags @@ -75,11 +76,27 @@ class ValidationRules(ValidationRuleFunctions): # unbundle individual results. evaluation_state becomes an integer, violation evaluation_state = [StatusFlags(res[0]).value for res in results] - violations = [",".join(res[1]) if len(res[1])>0 else None for res in results] + violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results] + violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] shipcall_df.loc[:,"evaluation"] = evaluation_state shipcall_df.loc[:,"evaluation_message"] = violations return shipcall_df + + def concise_evaluation_message_if_too_long(self, violation): + """ + when many validation rules are violated at once, the resulting evaluation message may exceed 512 characters (which the mysql database allows) + in these cases, use a regular expression to provide a concise message, where the 'concise' description is only the list of violated rools + """ + if violation is None: + return violation + + if len(violation)>=512: + concise = re.findall(r'{(.*?)\}', violation) + + # e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K'] + violation = f"Evaluation message too long. Violated Rules: {concise}" + return violation def determine_validation_state(self) -> str: """ diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index 5c4e3a0..2593e85 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -211,6 +211,7 @@ def test_validation_rule_fct_missing_time_agency_berth_etd__shipcall_etd_is_unde df_times = get_df_times(shipcall) # the shipcall etd is 'soon' + shipcall.type = ShipcallType.OUTGOING.value shipcall.etd = datetime.datetime.now() + datetime.timedelta(minutes=ParticipantwiseTimeDelta.AGENCY-10) # set times agency to be undetermined @@ -278,6 +279,7 @@ def test_validation_rule_fct_missing_time_mooring_berth_etd__shipcall_soon_but_p vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -352,6 +354,7 @@ def test_validation_rule_fct_missing_time_portadministration_berth_etd__shipcall vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -422,6 +425,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -457,6 +461,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -494,6 +499,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__shipcall_soon_but_par vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -534,6 +540,7 @@ def test_validation_rule_fct_missing_time_pilot_berth_etd__agency_and_pilot_assi vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -569,6 +576,7 @@ def test_validation_rule_fct_missing_time_tug_berth_eta__shipcall_soon_but_parti vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.INCOMING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -606,6 +614,7 @@ def test_validation_rule_fct_missing_time_tug_berth_etd__shipcall_soon_but_parti vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -680,6 +689,7 @@ def test_validation_rule_fct_missing_time_terminal_berth_etd__shipcall_soon_but_ vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) # according to the agency, a shipcall takes place soon (ETA/ETD) @@ -915,6 +925,7 @@ def test_validation_rule_fct_etd_time_not_in_operation_window__times_dont_match( """0003-B validation_rule_fct_etd_time_not_in_operation_window""" vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now() # reference time for easier readability @@ -936,6 +947,7 @@ def test_validation_rule_fct_eta_time_not_in_operation_window_and_validation_rul vr = build_sql_proxy_connection['vr'] import random shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.SHIFTING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now() @@ -1031,6 +1043,7 @@ def test_validation_rule_fct_etd_time_not_in_tidal_window__etd_outside_tidal_win """0004-B validation_rule_fct_etd_time_not_in_tidal_window""" vr = build_sql_proxy_connection['vr'] shipcall = get_shipcall_simple() + shipcall.type = ShipcallType.OUTGOING.value df_times = get_df_times(shipcall) t0_time = datetime.datetime.now()