dschick/git_brcal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

set schedule logging to INFO and using correct query for schedule test calls

Browse Source
This commit is contained in:
Daniel Schick 2024-05-02 09:10:37 +02:00
parent db0bcea485
commit 3734e672fc
2 changed files with 31 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/server/BreCal/services/schedule_routines.py
View File

@ -25,11 +25,15 @@ def UpdateShipcalls(options:dict = {'past_days':2}):
try: try:
pooledConnection = getPoolConnection() pooledConnection = getPoolConnection()
commands = pydapper.using(pooledConnection) commands = pydapper.using(pooledConnection)
query = ("SELECT id, ship_id, type, eta, voyage, etd, arrival_berth_id, departure_berth_id, tug_required, pilot_required, " query = ("SELECT id, ship_id, type, eta, voyage, etd, arrival_berth_id, departure_berth_id, tug_required, pilot_required, "
"flags, pier_side, bunkering, replenishing_terminal, replenishing_lock, draft, tidal_window_from, tidal_window_to, rain_sensitive_cargo, recommended_tugs, " "flags, pier_side, bunkering, replenishing_terminal, replenishing_lock, draft, tidal_window_from, tidal_window_to, rain_sensitive_cargo, recommended_tugs, "
"anchored, moored_lock, canceled, evaluation, evaluation_message, evaluation_notifications_sent, evaluation_time, created, modified, time_ref_point FROM shipcall WHERE ((type = 1 OR type = 3) AND eta >= DATE(NOW() - INTERVAL %d DAY)" "anchored, moored_lock, canceled, evaluation, evaluation_message, evaluation_notifications_sent, evaluation_time, created, modified, time_ref_point FROM shipcall s " +
"OR (type = 2 AND etd >= DATE(NOW() - INTERVAL %d DAY))) " "LEFT JOIN times t ON t.shipcall_id = s.id AND t.participant_type = 8 "
"ORDER BY eta") % (options["past_days"], options["past_days"]) "WHERE "
"(type = 1 AND (COALESCE(t.eta_berth, eta) >= DATE(NOW() - INTERVAL %d DAY))) OR "
"((type = 2 OR type = 3) AND (COALESCE(t.etd_berth, etd) >= DATE(NOW() - INTERVAL %d DAY)))"
"ORDER BY s.id") % (options["past_days"], options["past_days"])
# obtain data from the MYSQL database # obtain data from the MYSQL database
data = commands.query(query, model=model.Shipcall) data = commands.query(query, model=model.Shipcall)
@ -54,6 +58,9 @@ def add_function_to_schedule__update_shipcalls(interval_in_minutes:int, options:
return return
def setup_schedule(update_shipcalls_interval_in_minutes:int=60): def setup_schedule(update_shipcalls_interval_in_minutes:int=60):
logging.getLogger('schedule').setLevel(logging.INFO); # set the logging level of the schedule module to INFO
schedule.clear() # clear all routine jobs. This prevents jobs from being created multiple times schedule.clear() # clear all routine jobs. This prevents jobs from being created multiple times
# update the evaluation state in every recent shipcall # update the evaluation state in every recent shipcall

39
src/server/BreCal/validators/validation_rules.py
View File

@ -1,4 +1,5 @@
import copy import copy
import logging
import re import re
import numpy as np import numpy as np
import pandas as pd import pandas as pd
@ -21,7 +22,7 @@ class ValidationRules(ValidationRuleFunctions):
# currently flagged: notification_state initially was based on using one ValidationRules object for each query. This is deprecated. # currently flagged: notification_state initially was based on using one ValidationRules object for each query. This is deprecated.
# self.notification_state = self.determine_notification_state() # (state:str, should_notify:bool) # self.notification_state = self.determine_notification_state() # (state:str, should_notify:bool)
return return
def evaluate(self, shipcall): def evaluate(self, shipcall):
""" """
1.) prepare df_times, which every validation rule tends to use 1.) prepare df_times, which every validation rule tends to use
@ -34,7 +35,7 @@ class ValidationRules(ValidationRuleFunctions):
if len(df_times)==0: if len(df_times)==0:
return (StatusFlags.GREEN.value, []) return (StatusFlags.GREEN.value, [])
spm = self.sql_handler.df_dict["shipcall_participant_map"] spm = self.sql_handler.df_dict["shipcall_participant_map"]
if len(spm.loc[spm["shipcall_id"]==shipcall.id])==0: if len(spm.loc[spm["shipcall_id"]==shipcall.id])==0:
return (StatusFlags.GREEN.value, []) return (StatusFlags.GREEN.value, [])
@ -51,12 +52,14 @@ class ValidationRules(ValidationRuleFunctions):
# 'translate' all error codes into readable, human-understandable format. # 'translate' all error codes into readable, human-understandable format.
evaluation_results = [(state, self.describe_error_message(msg)) for (state, msg) in evaluation_results] evaluation_results = [(state, self.describe_error_message(msg)) for (state, msg) in evaluation_results]
logging.info(f"Validation results for shipcall {shipcall.id}: {evaluation_results}")
# check, what the maximum state flag is and return it # check, what the maximum state flag is and return it
evaluation_state = np.max(np.array([result[0].value for result in evaluation_results])) if len(evaluation_results)>0 else StatusFlags.GREEN.value evaluation_state = np.max(np.array([result[0].value for result in evaluation_results])) if len(evaluation_results)>0 else StatusFlags.GREEN.value
evaluation_verbosity = [result[1] for result in evaluation_results] evaluation_verbosity = [result[1] for result in evaluation_results]
return (evaluation_state, evaluation_verbosity) return (evaluation_state, evaluation_verbosity)
def evaluation_verbosity(self, evaluation_state, evaluation_results): def evaluation_verbosity(self, evaluation_state, evaluation_results):
"""This function suggestions verbosity for the evaluation results. Based on 'True'/'False' evaluation outcome, the returned string is different.""" """This function suggestions verbosity for the evaluation results. Based on 'True'/'False' evaluation outcome, the returned string is different."""
if evaluation_state: if evaluation_state:
@ -64,17 +67,17 @@ class ValidationRules(ValidationRuleFunctions):
else: else:
verbose_string = "These are:" + "\n\t".join(evaluation_results) # every element of the list will be displayed in a new line with a tab verbose_string = "These are:" + "\n\t".join(evaluation_results) # every element of the list will be displayed in a new line with a tab
return f"FAILED VALIDATION. There have been {len(evaluation_results)} violations. {verbose_string}" return f"FAILED VALIDATION. There have been {len(evaluation_results)} violations. {verbose_string}"
def evaluate_shipcall_from_df(self, x): def evaluate_shipcall_from_df(self, x):
shipcall = Shipcall(**{**{'id':x.name}, **x.to_dict()}) shipcall = Shipcall(**{**{'id':x.name}, **x.to_dict()})
evaluation_state, violations = self.evaluate(shipcall) evaluation_state, violations = self.evaluate(shipcall)
return evaluation_state, violations return evaluation_state, violations
def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame: def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame:
"""apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation' and 'evaluation_message' are updated)""" """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation' and 'evaluation_message' are updated)"""
results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values
# unbundle individual results. evaluation_state becomes an integer, violation # unbundle individual results. evaluation_state becomes an integer, violation
evaluation_state = [StatusFlags(res[0]).value for res in results] evaluation_state = [StatusFlags(res[0]).value for res in results]
violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results] violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results]
violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations]
@ -90,31 +93,31 @@ class ValidationRules(ValidationRuleFunctions):
""" """
if violation is None: if violation is None:
return violation return violation
if len(violation)>=512: if len(violation)>=512:
concise = re.findall(r'{(.*?)\}', violation) concise = re.findall(r'{(.*?)\}', violation)
# e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K'] # e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K']
violation = f"Evaluation message too long. Violated Rules: {concise}" violation = f"Evaluation message too long. Violated Rules: {concise}"
return violation return violation
def determine_validation_state(self) -> str: def determine_validation_state(self) -> str:
""" """
this method determines the validation state of a shipcall. The state is either ['green', 'yellow', 'red'] and signals, this method determines the validation state of a shipcall. The state is either ['green', 'yellow', 'red'] and signals,
whether an entry causes issues within the workflow of users. whether an entry causes issues within the workflow of users.
returns: validation_state_new (str) returns: validation_state_new (str)
""" """
(validation_state_new, description) = self.undefined_method() (validation_state_new, description) = self.undefined_method()
# should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification. # should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification.
self.validation_state = validation_state_new self.validation_state = validation_state_new
return validation_state_new return validation_state_new
def determine_notification_state(self) -> (str, bool): def determine_notification_state(self) -> (str, bool):
""" """
this method determines state changes in the notification state. When the state is changed to yellow or red, this method determines state changes in the notification state. When the state is changed to yellow or red,
a user is notified about it. The only exception for this rule is when the state was yellow or red before, a user is notified about it. The only exception for this rule is when the state was yellow or red before,
as the user has then already been notified. as the user has then already been notified.
returns: notification_state_new (str), should_notify (bool) returns: notification_state_new (str), should_notify (bool)
""" """
@ -122,10 +125,10 @@ class ValidationRules(ValidationRuleFunctions):
should_notify = self.identify_notification_state_change(state_new) should_notify = self.identify_notification_state_change(state_new)
self.notification_state = state_new # overwrite the predecessor self.notification_state = state_new # overwrite the predecessor
return state_new, should_notify return state_new, should_notify
def identify_notification_state_change(self, state_new) -> bool: def identify_notification_state_change(self, state_new) -> bool:
""" """
determines, whether the observed state change should trigger a notification. determines, whether the observed state change should trigger a notification.
internally, this function maps a color string to an integer and determines, if the successor state is more severe than the predecessor. internally, this function maps a color string to an integer and determines, if the successor state is more severe than the predecessor.
state changes trigger a notification in the following cases: state changes trigger a notification in the following cases:
@ -135,14 +138,14 @@ class ValidationRules(ValidationRuleFunctions):
(none -> yellow) or (none -> red) (none -> yellow) or (none -> red)
due to the values in the enumeration objects, the states are mapped to provide this function. due to the values in the enumeration objects, the states are mapped to provide this function.
green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than". green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than".
returns bool, whether a notification should be triggered returns bool, whether a notification should be triggered
""" """
# state_old is always considered at least 'Green' (1) # state_old is always considered at least 'Green' (1)
state_old = max(copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else StatusFlags.NONE, StatusFlags.GREEN.value) state_old = max(copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else StatusFlags.NONE, StatusFlags.GREEN.value)
return state_new.value > state_old.value return state_new.value > state_old.value
def undefined_method(self) -> str: def undefined_method(self) -> str:
"""this function should apply the ValidationRules to the respective .shipcall, in regards to .times""" """this function should apply the ValidationRules to the respective .shipcall, in regards to .times"""
# #TODO_traffic_state # #TODO_traffic_state