diff --git a/src/server/BreCal/services/schedule_routines.py b/src/server/BreCal/services/schedule_routines.py index 2ff4f0d..f2cb496 100644 --- a/src/server/BreCal/services/schedule_routines.py +++ b/src/server/BreCal/services/schedule_routines.py @@ -25,11 +25,15 @@ def UpdateShipcalls(options:dict = {'past_days':2}): try: pooledConnection = getPoolConnection() commands = pydapper.using(pooledConnection) + query = ("SELECT id, ship_id, type, eta, voyage, etd, arrival_berth_id, departure_berth_id, tug_required, pilot_required, " "flags, pier_side, bunkering, replenishing_terminal, replenishing_lock, draft, tidal_window_from, tidal_window_to, rain_sensitive_cargo, recommended_tugs, " - "anchored, moored_lock, canceled, evaluation, evaluation_message, evaluation_notifications_sent, evaluation_time, created, modified, time_ref_point FROM shipcall WHERE ((type = 1 OR type = 3) AND eta >= DATE(NOW() - INTERVAL %d DAY)" - "OR (type = 2 AND etd >= DATE(NOW() - INTERVAL %d DAY))) " - "ORDER BY eta") % (options["past_days"], options["past_days"]) + "anchored, moored_lock, canceled, evaluation, evaluation_message, evaluation_notifications_sent, evaluation_time, created, modified, time_ref_point FROM shipcall s " + + "LEFT JOIN times t ON t.shipcall_id = s.id AND t.participant_type = 8 " + "WHERE " + "(type = 1 AND (COALESCE(t.eta_berth, eta) >= DATE(NOW() - INTERVAL %d DAY))) OR " + "((type = 2 OR type = 3) AND (COALESCE(t.etd_berth, etd) >= DATE(NOW() - INTERVAL %d DAY)))" + "ORDER BY s.id") % (options["past_days"], options["past_days"]) # obtain data from the MYSQL database data = commands.query(query, model=model.Shipcall) @@ -54,6 +58,9 @@ def add_function_to_schedule__update_shipcalls(interval_in_minutes:int, options: return def setup_schedule(update_shipcalls_interval_in_minutes:int=60): + + logging.getLogger('schedule').setLevel(logging.INFO); # set the logging level of the schedule module to INFO + schedule.clear() # clear all routine jobs. This prevents jobs from being created multiple times # update the evaluation state in every recent shipcall diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index aa2ef22..405b0b6 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -1,4 +1,5 @@ import copy +import logging import re import numpy as np import pandas as pd @@ -21,7 +22,7 @@ class ValidationRules(ValidationRuleFunctions): # currently flagged: notification_state initially was based on using one ValidationRules object for each query. This is deprecated. # self.notification_state = self.determine_notification_state() # (state:str, should_notify:bool) return - + def evaluate(self, shipcall): """ 1.) prepare df_times, which every validation rule tends to use @@ -34,7 +35,7 @@ class ValidationRules(ValidationRuleFunctions): if len(df_times)==0: return (StatusFlags.GREEN.value, []) - + spm = self.sql_handler.df_dict["shipcall_participant_map"] if len(spm.loc[spm["shipcall_id"]==shipcall.id])==0: return (StatusFlags.GREEN.value, []) @@ -51,12 +52,14 @@ class ValidationRules(ValidationRuleFunctions): # 'translate' all error codes into readable, human-understandable format. evaluation_results = [(state, self.describe_error_message(msg)) for (state, msg) in evaluation_results] - + + logging.info(f"Validation results for shipcall {shipcall.id}: {evaluation_results}") + # check, what the maximum state flag is and return it evaluation_state = np.max(np.array([result[0].value for result in evaluation_results])) if len(evaluation_results)>0 else StatusFlags.GREEN.value evaluation_verbosity = [result[1] for result in evaluation_results] return (evaluation_state, evaluation_verbosity) - + def evaluation_verbosity(self, evaluation_state, evaluation_results): """This function suggestions verbosity for the evaluation results. Based on 'True'/'False' evaluation outcome, the returned string is different.""" if evaluation_state: @@ -64,17 +67,17 @@ class ValidationRules(ValidationRuleFunctions): else: verbose_string = "These are:" + "\n\t".join(evaluation_results) # every element of the list will be displayed in a new line with a tab return f"FAILED VALIDATION. There have been {len(evaluation_results)} violations. {verbose_string}" - + def evaluate_shipcall_from_df(self, x): shipcall = Shipcall(**{**{'id':x.name}, **x.to_dict()}) evaluation_state, violations = self.evaluate(shipcall) return evaluation_state, violations - + def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame: """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation' and 'evaluation_message' are updated)""" results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values - # unbundle individual results. evaluation_state becomes an integer, violation + # unbundle individual results. evaluation_state becomes an integer, violation evaluation_state = [StatusFlags(res[0]).value for res in results] violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results] violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] @@ -90,31 +93,31 @@ class ValidationRules(ValidationRuleFunctions): """ if violation is None: return violation - + if len(violation)>=512: concise = re.findall(r'{(.*?)\}', violation) # e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K'] violation = f"Evaluation message too long. Violated Rules: {concise}" return violation - + def determine_validation_state(self) -> str: """ this method determines the validation state of a shipcall. The state is either ['green', 'yellow', 'red'] and signals, - whether an entry causes issues within the workflow of users. - + whether an entry causes issues within the workflow of users. + returns: validation_state_new (str) """ (validation_state_new, description) = self.undefined_method() - # should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification. + # should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification. self.validation_state = validation_state_new return validation_state_new - + def determine_notification_state(self) -> (str, bool): """ this method determines state changes in the notification state. When the state is changed to yellow or red, a user is notified about it. The only exception for this rule is when the state was yellow or red before, - as the user has then already been notified. + as the user has then already been notified. returns: notification_state_new (str), should_notify (bool) """ @@ -122,10 +125,10 @@ class ValidationRules(ValidationRuleFunctions): should_notify = self.identify_notification_state_change(state_new) self.notification_state = state_new # overwrite the predecessor return state_new, should_notify - + def identify_notification_state_change(self, state_new) -> bool: """ - determines, whether the observed state change should trigger a notification. + determines, whether the observed state change should trigger a notification. internally, this function maps a color string to an integer and determines, if the successor state is more severe than the predecessor. state changes trigger a notification in the following cases: @@ -135,14 +138,14 @@ class ValidationRules(ValidationRuleFunctions): (none -> yellow) or (none -> red) due to the values in the enumeration objects, the states are mapped to provide this function. - green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than". + green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than". returns bool, whether a notification should be triggered """ # state_old is always considered at least 'Green' (1) state_old = max(copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else StatusFlags.NONE, StatusFlags.GREEN.value) return state_new.value > state_old.value - + def undefined_method(self) -> str: """this function should apply the ValidationRules to the respective .shipcall, in regards to .times""" # #TODO_traffic_state