adding the shipcall evaluation (traffic state). Provided a one-line function to connect, evaluate and update the shipcalls in a database
This commit is contained in:
max_metz
parent
43b9552c90
commit
f996a8a74e
@ -4,6 +4,9 @@ import datetime
|
||||
from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times
|
||||
from brecal_utils.database.enums import ParticipantType
|
||||
|
||||
def pandas_series_to_data_model():
|
||||
return
|
||||
|
||||
class SQLHandler():
|
||||
"""
|
||||
An object that reads SQL queries from the sql_connection and stores it in pandas DataFrames. The object can read all available tables
|
||||
@ -128,7 +131,8 @@ class SQLHandler():
|
||||
assert data_model is not None, f"could not find the requested model_str: {model_str}"
|
||||
|
||||
# build 'data' and fill the data model object
|
||||
data = {**{'id':id}, **series.to_dict()} # 'id' must be added manually, as .to_dict does not contain the index, which was set with .set_index
|
||||
# convert the 'id' to an integer, so the np.uint64 (used by pandas) is convertible to mysql
|
||||
data = {**{'id':int(id)}, **series.to_dict()} # 'id' must be added manually, as .to_dict does not contain the index, which was set with .set_index
|
||||
data = data_model(**data)
|
||||
return data
|
||||
|
||||
|
||||
@ -0,0 +1,95 @@
|
||||
import json
|
||||
import pydapper
|
||||
import pandas as pd
|
||||
import mysql.connector
|
||||
|
||||
from brecal_utils.database.sql_handler import SQLHandler
|
||||
from brecal_utils.validators.validation_rules import ValidationRules
|
||||
from BreCal.schemas.model import Shipcall
|
||||
|
||||
def update_shipcall_in_mysql_database(sql_connection, shipcall:Shipcall, relevant_keys:list = ["evaluation", "evaluation_message"]):
|
||||
"""
|
||||
given an individual schemaModel (e.g., Shipcall.__dict__), update the entry within the mysql database
|
||||
|
||||
options:
|
||||
sql_connection: an instance of mysql.connector.connect
|
||||
shipcall: a Shipcall data model
|
||||
relevant_keys: a list of the keys to be updated. Should never contain 'id', which is immutable
|
||||
|
||||
returns: (query, affected_rows)
|
||||
"""
|
||||
assert not "id" in relevant_keys, f"the 'id' key should never be updated."
|
||||
schemaModel = shipcall.__dict__
|
||||
|
||||
commands = pydapper.using(sql_connection)
|
||||
sentinel = object()
|
||||
theshipcall = commands.query_single_or_default("SELECT * FROM shipcall where id = ?id?", sentinel, param={"id" : schemaModel["id"]})
|
||||
|
||||
if theshipcall is sentinel:
|
||||
return json.dumps("no such record"), 404, {'Content-Type': 'application/json; charset=utf-8'}
|
||||
|
||||
query = build_mysql_query_to_update_shipcall(shipcall=shipcall, relevant_keys=relevant_keys)
|
||||
affected_rows = commands.execute(query, param=schemaModel)
|
||||
return (query, affected_rows)
|
||||
|
||||
def build_mysql_query_to_update_shipcall(shipcall, relevant_keys:list):
|
||||
"""builds a mysql query, which updates the shipcall table. In particular, the provided shipcall will be updated for each key in {relevant_keys}"""
|
||||
schemaModel = shipcall.__dict__
|
||||
|
||||
# prepare prefix and suffix. Then build the body of the query
|
||||
prefix = "UPDATE shipcall SET "
|
||||
suffix = "where id = ?id?"
|
||||
body = ", ".join([f"{key} = ?{key}? " for key in schemaModel.keys() if (key in relevant_keys)]) # .join ignores the first ', ', which equals the 'isNotFirst' boolean-loop
|
||||
|
||||
# build query
|
||||
query = f"{prefix}{body}{suffix}"
|
||||
return query
|
||||
|
||||
def update_all_shipcalls_in_mysql_database(sql_connection, sql_handler:SQLHandler, shipcall_df:pd.DataFrame)->None:
|
||||
"""
|
||||
iterates over each shipcall_id in a shipcall dataframe, builds Shipcall data models and updates those in the sql database, which
|
||||
is located in {sql_connection}
|
||||
|
||||
options:
|
||||
sql_connection: an instance of mysql.connector.connect
|
||||
sql_handler: an SQLHandler instance
|
||||
shipcall_df: dataframe, which stores the data that is used to retrieve the shipcall data models (that are then updated in the database)
|
||||
"""
|
||||
for shipcall_id in shipcall_df.index:
|
||||
shipcall = sql_handler.df_loc_to_data_model(df=shipcall_df, id=shipcall_id, model_str="shipcall")
|
||||
update_shipcall_in_mysql_database(sql_connection, shipcall=shipcall, relevant_keys = ["evaluation", "evaluation_message"])
|
||||
return
|
||||
|
||||
def run_validation_rules(mysql_connector_instance, debug=False)->pd.DataFrame:
|
||||
"""
|
||||
options:
|
||||
mysql_connector_instance: an instance created by the mysql.connector.connect() call. It is advised to use Python's context manager to close the connection after finished.
|
||||
e.g.,
|
||||
with mysql.connector.connect(**mysql_connection_data) as mysql_connector_instance:
|
||||
run_validation_rules(mysql_connector_instance)
|
||||
returns None
|
||||
|
||||
"""
|
||||
sql_handler = SQLHandler(sql_connection=mysql_connector_instance, read_all=True)
|
||||
vr = ValidationRules(sql_handler)
|
||||
|
||||
shipcall_df = sql_handler.df_dict.get("shipcall")
|
||||
# placeholder: filter shipcalls. For example, exclude historic entries.
|
||||
shipcall_df = vr.evaluate_shipcalls(shipcall_df)
|
||||
|
||||
if debug:
|
||||
return shipcall_df
|
||||
|
||||
# iterate over each shipcall in shipcall_df and update the respective entry in the mysql database
|
||||
update_all_shipcalls_in_mysql_database(sql_connection=mysql_connector_instance, sql_handler=sql_handler, shipcall_df=shipcall_df)
|
||||
return shipcall_df
|
||||
|
||||
def update_shipcall_evaluation_state(mysql_connection_data)->pd.DataFrame:
|
||||
"""
|
||||
single line function to connect to a mysql database (using the {mysql_connection_data}), evaluate each shipcall (bei traffic state)
|
||||
and finally, update those in the database.
|
||||
"""
|
||||
with mysql.connector.connect(**mysql_connection_data) as mysql_connector_instance:
|
||||
shipcall_df = run_validation_rules(mysql_connector_instance=mysql_connector_instance, debug=False)
|
||||
return shipcall_df
|
||||
|
||||
@ -586,8 +586,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
|
||||
return (StatusFlags.GREEN, None)
|
||||
|
||||
# get agency & terminal times
|
||||
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY)
|
||||
times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL)
|
||||
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
|
||||
times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
|
||||
|
||||
if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT):
|
||||
return (StatusFlags.GREEN, None)
|
||||
@ -615,8 +615,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
|
||||
return (StatusFlags.GREEN, None)
|
||||
|
||||
# get agency & terminal times
|
||||
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY)
|
||||
times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL)
|
||||
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
|
||||
times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
|
||||
|
||||
if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT):
|
||||
return (StatusFlags.GREEN, None)
|
||||
@ -642,7 +642,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
|
||||
# check, if the header is filled in (agency)
|
||||
if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1:
|
||||
return (StatusFlags.GREEN, None)
|
||||
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY)
|
||||
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
|
||||
|
||||
# requirements: tidal window (from & to) is filled in
|
||||
if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT):
|
||||
@ -670,7 +670,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
|
||||
# check, if the header is filled in (agency)
|
||||
if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1:
|
||||
return (StatusFlags.GREEN, None)
|
||||
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY)
|
||||
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
|
||||
|
||||
# requirements: tidal window (from & to) is filled in
|
||||
if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT):
|
||||
|
||||
@ -48,8 +48,9 @@ class ValidationRules(ValidationRuleFunctions):
|
||||
#evaluation_state = not np.any(np.greater(np.array([result[0] for result in evaluation_results]), ValidationRuleState.GREEN))
|
||||
"""
|
||||
# check, what the maximum state flag is and return it
|
||||
evaluation_state = np.max(np.array([result[0] for result in evaluation_results])) if len(evaluation_results)>0 else 1
|
||||
return (evaluation_state, evaluation_results)
|
||||
evaluation_state = np.max(np.array([result[0].value for result in evaluation_results])) if len(evaluation_results)>0 else 1
|
||||
evaluation_verbosity = [result[1] for result in evaluation_results]
|
||||
return (evaluation_state, evaluation_verbosity)
|
||||
|
||||
def evaluation_verbosity(self, evaluation_state, evaluation_results):
|
||||
"""This function suggestions verbosity for the evaluation results. Based on 'True'/'False' evaluation outcome, the returned string is different."""
|
||||
|
||||
@ -11,7 +11,6 @@ marshmallow-dataclass
|
||||
bcrypt
|
||||
jwt
|
||||
flask-jwt-extended
|
||||
SQLAlchemy
|
||||
numpy
|
||||
pandas
|
||||
|
||||
|
||||
Reference in New Issue
Block a user