dschick/git_brcal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

set schedule logging to INFO and using correct query for schedule test calls

Browse Source
This commit is contained in:
Daniel Schick 2024-05-02 09:10:37 +02:00
parent 707ffd0d59
commit 3b01dbb7aa
2 changed files with 31 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/server/BreCal/services/schedule_routines.py
View File

@ -25,11 +25,15 @@ def UpdateShipcalls(options:dict = {'past_days':2}):
try:
pooledConnection = getPoolConnection()
commands = pydapper.using(pooledConnection)
query = ("SELECT id, ship_id, type, eta, voyage, etd, arrival_berth_id, departure_berth_id, tug_required, pilot_required, "
"flags, pier_side, bunkering, replenishing_terminal, replenishing_lock, draft, tidal_window_from, tidal_window_to, rain_sensitive_cargo, recommended_tugs, "
"anchored, moored_lock, canceled, evaluation, evaluation_message, evaluation_notifications_sent, evaluation_time, created, modified, time_ref_point FROM shipcall WHERE ((type = 1 OR type = 3) AND eta >= DATE(NOW() - INTERVAL %d DAY)"
"OR (type = 2 AND etd >= DATE(NOW() - INTERVAL %d DAY))) "
"ORDER BY eta") % (options["past_days"], options["past_days"])
"anchored, moored_lock, canceled, evaluation, evaluation_message, evaluation_notifications_sent, evaluation_time, created, modified, time_ref_point FROM shipcall s " +
"LEFT JOIN times t ON t.shipcall_id = s.id AND t.participant_type = 8 "
"WHERE "
"(type = 1 AND (COALESCE(t.eta_berth, eta) >= DATE(NOW() - INTERVAL %d DAY))) OR "
"((type = 2 OR type = 3) AND (COALESCE(t.etd_berth, etd) >= DATE(NOW() - INTERVAL %d DAY)))"
"ORDER BY s.id") % (options["past_days"], options["past_days"])
# obtain data from the MYSQL database
data = commands.query(query, model=model.Shipcall)
@ -54,6 +58,9 @@ def add_function_to_schedule__update_shipcalls(interval_in_minutes:int, options:
return
def setup_schedule(update_shipcalls_interval_in_minutes:int=60):
logging.getLogger('schedule').setLevel(logging.INFO); # set the logging level of the schedule module to INFO
schedule.clear() # clear all routine jobs. This prevents jobs from being created multiple times
# update the evaluation state in every recent shipcall

39
src/server/BreCal/validators/validation_rules.py
View File

@ -1,4 +1,5 @@
import copy
import logging
import re
import numpy as np
import pandas as pd
@ -21,7 +22,7 @@ class ValidationRules(ValidationRuleFunctions):
# currently flagged: notification_state initially was based on using one ValidationRules object for each query. This is deprecated.
# self.notification_state = self.determine_notification_state() # (state:str, should_notify:bool)
return
def evaluate(self, shipcall):
"""
1.) prepare df_times, which every validation rule tends to use
@ -34,7 +35,7 @@ class ValidationRules(ValidationRuleFunctions):
if len(df_times)==0:
return (StatusFlags.GREEN.value, [])
spm = self.sql_handler.df_dict["shipcall_participant_map"]
if len(spm.loc[spm["shipcall_id"]==shipcall.id])==0:
return (StatusFlags.GREEN.value, [])
@ -51,12 +52,14 @@ class ValidationRules(ValidationRuleFunctions):
# 'translate' all error codes into readable, human-understandable format.
evaluation_results = [(state, self.describe_error_message(msg)) for (state, msg) in evaluation_results]
logging.info(f"Validation results for shipcall {shipcall.id}: {evaluation_results}")
# check, what the maximum state flag is and return it
evaluation_state = np.max(np.array([result[0].value for result in evaluation_results])) if len(evaluation_results)>0 else StatusFlags.GREEN.value
evaluation_verbosity = [result[1] for result in evaluation_results]
return (evaluation_state, evaluation_verbosity)
def evaluation_verbosity(self, evaluation_state, evaluation_results):
"""This function suggestions verbosity for the evaluation results. Based on 'True'/'False' evaluation outcome, the returned string is different."""
if evaluation_state:
@ -64,17 +67,17 @@ class ValidationRules(ValidationRuleFunctions):
else:
verbose_string = "These are:" + "\n\t".join(evaluation_results) # every element of the list will be displayed in a new line with a tab
return f"FAILED VALIDATION. There have been {len(evaluation_results)} violations. {verbose_string}"
def evaluate_shipcall_from_df(self, x):
shipcall = Shipcall(**{**{'id':x.name}, **x.to_dict()})
evaluation_state, violations = self.evaluate(shipcall)
return evaluation_state, violations
def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame:
"""apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation' and 'evaluation_message' are updated)"""
results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values
# unbundle individual results. evaluation_state becomes an integer, violation
# unbundle individual results. evaluation_state becomes an integer, violation
evaluation_state = [StatusFlags(res[0]).value for res in results]
violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results]
violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations]
@ -90,31 +93,31 @@ class ValidationRules(ValidationRuleFunctions):
"""
if violation is None:
return violation
if len(violation)>=512:
concise = re.findall(r'{(.*?)\}', violation)
# e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K']
violation = f"Evaluation message too long. Violated Rules: {concise}"
return violation
def determine_validation_state(self) -> str:
"""
this method determines the validation state of a shipcall. The state is either ['green', 'yellow', 'red'] and signals,
whether an entry causes issues within the workflow of users.
whether an entry causes issues within the workflow of users.
returns: validation_state_new (str)
"""
(validation_state_new, description) = self.undefined_method()
# should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification.
# should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification.
self.validation_state = validation_state_new
return validation_state_new
def determine_notification_state(self) -> (str, bool):
"""
this method determines state changes in the notification state. When the state is changed to yellow or red,
a user is notified about it. The only exception for this rule is when the state was yellow or red before,
as the user has then already been notified.
as the user has then already been notified.
returns: notification_state_new (str), should_notify (bool)
"""
@ -122,10 +125,10 @@ class ValidationRules(ValidationRuleFunctions):
should_notify = self.identify_notification_state_change(state_new)
self.notification_state = state_new # overwrite the predecessor
return state_new, should_notify
def identify_notification_state_change(self, state_new) -> bool:
"""
determines, whether the observed state change should trigger a notification.
determines, whether the observed state change should trigger a notification.
internally, this function maps a color string to an integer and determines, if the successor state is more severe than the predecessor.
state changes trigger a notification in the following cases:
@ -135,14 +138,14 @@ class ValidationRules(ValidationRuleFunctions):
(none -> yellow) or (none -> red)
due to the values in the enumeration objects, the states are mapped to provide this function.
green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than".
green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than".
returns bool, whether a notification should be triggered
"""
# state_old is always considered at least 'Green' (1)
state_old = max(copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else StatusFlags.NONE, StatusFlags.GREEN.value)
return state_new.value > state_old.value
def undefined_method(self) -> str:
"""this function should apply the ValidationRules to the respective .shipcall, in regards to .times"""
# #TODO_traffic_state