From f996a8a74e7879f4cb9f11a78a016edf08d343d6 Mon Sep 17 00:00:00 2001 From: max_metz Date: Fri, 13 Oct 2023 10:02:26 +0200 Subject: [PATCH] adding the shipcall evaluation (traffic state). Provided a one-line function to connect, evaluate and update the shipcalls in a database --- .../brecal_utils/database/sql_handler.py | 6 +- .../brecal_utils/database/update_database.py | 95 +++++++++++++++++++ .../validators/validation_rule_functions.py | 12 +-- .../validators/validation_rules.py | 5 +- src/server/requirements.txt | 1 - 5 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 src/lib_brecal_utils/brecal_utils/database/update_database.py diff --git a/src/lib_brecal_utils/brecal_utils/database/sql_handler.py b/src/lib_brecal_utils/brecal_utils/database/sql_handler.py index d50bbbd..91f714a 100644 --- a/src/lib_brecal_utils/brecal_utils/database/sql_handler.py +++ b/src/lib_brecal_utils/brecal_utils/database/sql_handler.py @@ -4,6 +4,9 @@ import datetime from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times from brecal_utils.database.enums import ParticipantType +def pandas_series_to_data_model(): + return + class SQLHandler(): """ An object that reads SQL queries from the sql_connection and stores it in pandas DataFrames. The object can read all available tables @@ -128,7 +131,8 @@ class SQLHandler(): assert data_model is not None, f"could not find the requested model_str: {model_str}" # build 'data' and fill the data model object - data = {**{'id':id}, **series.to_dict()} # 'id' must be added manually, as .to_dict does not contain the index, which was set with .set_index + # convert the 'id' to an integer, so the np.uint64 (used by pandas) is convertible to mysql + data = {**{'id':int(id)}, **series.to_dict()} # 'id' must be added manually, as .to_dict does not contain the index, which was set with .set_index data = data_model(**data) return data diff --git a/src/lib_brecal_utils/brecal_utils/database/update_database.py b/src/lib_brecal_utils/brecal_utils/database/update_database.py new file mode 100644 index 0000000..0ed293d --- /dev/null +++ b/src/lib_brecal_utils/brecal_utils/database/update_database.py @@ -0,0 +1,95 @@ +import json +import pydapper +import pandas as pd +import mysql.connector + +from brecal_utils.database.sql_handler import SQLHandler +from brecal_utils.validators.validation_rules import ValidationRules +from BreCal.schemas.model import Shipcall + +def update_shipcall_in_mysql_database(sql_connection, shipcall:Shipcall, relevant_keys:list = ["evaluation", "evaluation_message"]): + """ + given an individual schemaModel (e.g., Shipcall.__dict__), update the entry within the mysql database + + options: + sql_connection: an instance of mysql.connector.connect + shipcall: a Shipcall data model + relevant_keys: a list of the keys to be updated. Should never contain 'id', which is immutable + + returns: (query, affected_rows) + """ + assert not "id" in relevant_keys, f"the 'id' key should never be updated." + schemaModel = shipcall.__dict__ + + commands = pydapper.using(sql_connection) + sentinel = object() + theshipcall = commands.query_single_or_default("SELECT * FROM shipcall where id = ?id?", sentinel, param={"id" : schemaModel["id"]}) + + if theshipcall is sentinel: + return json.dumps("no such record"), 404, {'Content-Type': 'application/json; charset=utf-8'} + + query = build_mysql_query_to_update_shipcall(shipcall=shipcall, relevant_keys=relevant_keys) + affected_rows = commands.execute(query, param=schemaModel) + return (query, affected_rows) + +def build_mysql_query_to_update_shipcall(shipcall, relevant_keys:list): + """builds a mysql query, which updates the shipcall table. In particular, the provided shipcall will be updated for each key in {relevant_keys}""" + schemaModel = shipcall.__dict__ + + # prepare prefix and suffix. Then build the body of the query + prefix = "UPDATE shipcall SET " + suffix = "where id = ?id?" + body = ", ".join([f"{key} = ?{key}? " for key in schemaModel.keys() if (key in relevant_keys)]) # .join ignores the first ', ', which equals the 'isNotFirst' boolean-loop + + # build query + query = f"{prefix}{body}{suffix}" + return query + +def update_all_shipcalls_in_mysql_database(sql_connection, sql_handler:SQLHandler, shipcall_df:pd.DataFrame)->None: + """ + iterates over each shipcall_id in a shipcall dataframe, builds Shipcall data models and updates those in the sql database, which + is located in {sql_connection} + + options: + sql_connection: an instance of mysql.connector.connect + sql_handler: an SQLHandler instance + shipcall_df: dataframe, which stores the data that is used to retrieve the shipcall data models (that are then updated in the database) + """ + for shipcall_id in shipcall_df.index: + shipcall = sql_handler.df_loc_to_data_model(df=shipcall_df, id=shipcall_id, model_str="shipcall") + update_shipcall_in_mysql_database(sql_connection, shipcall=shipcall, relevant_keys = ["evaluation", "evaluation_message"]) + return + +def run_validation_rules(mysql_connector_instance, debug=False)->pd.DataFrame: + """ + options: + mysql_connector_instance: an instance created by the mysql.connector.connect() call. It is advised to use Python's context manager to close the connection after finished. + e.g., + with mysql.connector.connect(**mysql_connection_data) as mysql_connector_instance: + run_validation_rules(mysql_connector_instance) + returns None + + """ + sql_handler = SQLHandler(sql_connection=mysql_connector_instance, read_all=True) + vr = ValidationRules(sql_handler) + + shipcall_df = sql_handler.df_dict.get("shipcall") + # placeholder: filter shipcalls. For example, exclude historic entries. + shipcall_df = vr.evaluate_shipcalls(shipcall_df) + + if debug: + return shipcall_df + + # iterate over each shipcall in shipcall_df and update the respective entry in the mysql database + update_all_shipcalls_in_mysql_database(sql_connection=mysql_connector_instance, sql_handler=sql_handler, shipcall_df=shipcall_df) + return shipcall_df + +def update_shipcall_evaluation_state(mysql_connection_data)->pd.DataFrame: + """ + single line function to connect to a mysql database (using the {mysql_connection_data}), evaluate each shipcall (bei traffic state) + and finally, update those in the database. + """ + with mysql.connector.connect(**mysql_connection_data) as mysql_connector_instance: + shipcall_df = run_validation_rules(mysql_connector_instance=mysql_connector_instance, debug=False) + return shipcall_df + diff --git a/src/lib_brecal_utils/brecal_utils/validators/validation_rule_functions.py b/src/lib_brecal_utils/brecal_utils/validators/validation_rule_functions.py index 39a34c7..bb42a60 100644 --- a/src/lib_brecal_utils/brecal_utils/validators/validation_rule_functions.py +++ b/src/lib_brecal_utils/brecal_utils/validators/validation_rule_functions.py @@ -586,8 +586,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return (StatusFlags.GREEN, None) # get agency & terminal times - times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY) - times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL) + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT): return (StatusFlags.GREEN, None) @@ -615,8 +615,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return (StatusFlags.GREEN, None) # get agency & terminal times - times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY) - times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL) + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) + times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT): return (StatusFlags.GREEN, None) @@ -642,7 +642,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return (StatusFlags.GREEN, None) - times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY) + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) # requirements: tidal window (from & to) is filled in if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT): @@ -670,7 +670,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): # check, if the header is filled in (agency) if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: return (StatusFlags.GREEN, None) - times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY) + times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) # requirements: tidal window (from & to) is filled in if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT): diff --git a/src/lib_brecal_utils/brecal_utils/validators/validation_rules.py b/src/lib_brecal_utils/brecal_utils/validators/validation_rules.py index 3ee7a46..0d4aa36 100644 --- a/src/lib_brecal_utils/brecal_utils/validators/validation_rules.py +++ b/src/lib_brecal_utils/brecal_utils/validators/validation_rules.py @@ -48,8 +48,9 @@ class ValidationRules(ValidationRuleFunctions): #evaluation_state = not np.any(np.greater(np.array([result[0] for result in evaluation_results]), ValidationRuleState.GREEN)) """ # check, what the maximum state flag is and return it - evaluation_state = np.max(np.array([result[0] for result in evaluation_results])) if len(evaluation_results)>0 else 1 - return (evaluation_state, evaluation_results) + evaluation_state = np.max(np.array([result[0].value for result in evaluation_results])) if len(evaluation_results)>0 else 1 + evaluation_verbosity = [result[1] for result in evaluation_results] + return (evaluation_state, evaluation_verbosity) def evaluation_verbosity(self, evaluation_state, evaluation_results): """This function suggestions verbosity for the evaluation results. Based on 'True'/'False' evaluation outcome, the returned string is different.""" diff --git a/src/server/requirements.txt b/src/server/requirements.txt index 9c7b54f..2f648ee 100644 --- a/src/server/requirements.txt +++ b/src/server/requirements.txt @@ -11,7 +11,6 @@ marshmallow-dataclass bcrypt jwt flask-jwt-extended -SQLAlchemy numpy pandas