dschick/git_brcal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #11 from puls200/feature/hotfix_20231031_traffic_lights

Browse Source 
Validation rule bug fixes as noted in test version 0.9.4
This commit is contained in:
Daniel Schick 2023-11-01 16:07:38 +01:00 committed by GitHub
commit ead190d219
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 90 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
src/server/BreCal/validators/validation_rule_functions.py
View File

@ -7,6 +7,43 @@ from BreCal.validators.time_logic import TimeLogic
from BreCal.database.enums import StatusFlags
#from BreCal.validators.schema_validation import validation_state_and_validation_name
# a human interpretable dictionary for error messages. In this case, the English language is preferred
error_message_dict = {
# 0001 A-M
"validation_rule_fct_missing_time_agency_berth_eta":"The shipcall arrives in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of arrival (ETA) {Rule #0001A}", # A
"validation_rule_fct_missing_time_agency_berth_etd":"The shipcall departs in less than 20 hours, but there are still missing times by the agency. Please add the estimated time of departure (ETD) {Rule #0001B}", # B
"validation_rule_fct_missing_time_mooring_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of arrival (ETA) {Rule #0001C}", # C
"validation_rule_fct_missing_time_mooring_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the mooring. Please add the estimated time of departure (ETD) {Rule #0001D}", # D
"validation_rule_fct_missing_time_portadministration_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of arrival (ETA) {Rule #0001F}", # F
"validation_rule_fct_missing_time_portadministration_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the port administration. Please add the estimated time of departure (ETD) {Rule #0001G}", # G
"validation_rule_fct_missing_time_pilot_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of arrival (ETA) {Rule #0001H}", # H
"validation_rule_fct_missing_time_pilot_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the pilot. Please add the estimated time of departure (ETD) {Rule #0001I}", # I
"validation_rule_fct_missing_time_tug_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of arrival (ETA) {Rule #0001J}", # J
"validation_rule_fct_missing_time_tug_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the tugs. Please add the estimated time of departure (ETD) {Rule #0001K}", # K
"validation_rule_fct_missing_time_terminal_berth_eta":"The shipcall arrives in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of arrival (ETA) {Rule #0001L}", # L
"validation_rule_fct_missing_time_terminal_berth_etd":"The shipcall departs in less than 16 hours, but there are still missing times by the terminal. Please add the estimated time of departure (ETD) {Rule #0001M}", # M
# 0002 A+B+C
"validation_rule_fct_shipcall_incoming_participants_disagree_on_eta":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of arrival (ETA) {Rule #0002A}",
"validation_rule_fct_shipcall_outgoing_participants_disagree_on_etd":"There are deviating times between agency, mooring, port authority, pilot and tug for the estimated time of departure (ETD) {Rule #0002B}",
"validation_rule_fct_shipcall_shifting_participants_disagree_on_eta_or_etd":"There are deviating times between agency, mooring, port authority, pilot and tug for ETA and ETD {Rule #0002C}",
# 0003 A+B
"validation_rule_fct_eta_time_not_in_operation_window":"The estimated time of arrival will be AFTER the planned start of operations. {Rule #0003A}",
"validation_rule_fct_etd_time_not_in_operation_window":"The estimated time of departure is supposed to be AFTER the planned end of operations. {Rule #0003D}",
# 0004 A+B
"validation_rule_fct_eta_time_not_in_tidal_window":"The tidal window does not fit to the agency's estimated time of arrival (ETA) {Rule #0004A}",
"validation_rule_fct_etd_time_not_in_tidal_window":"The tidal window does not fit to the agency's estimated time of departure (ETD) {Rule #0004B}",
# 0005 A+B
"validation_rule_fct_too_many_identical_eta_times":"There are more than three ships with the same planned time of arrival (ETA) {Rule #0005A}",
"validation_rule_fct_too_many_identical_etd_times":"There are more than three ships with the same planned time of departure (ETD) {Rule #0005B}",
# 0006 A+B
"validation_rule_fct_agency_and_terminal_berth_id_disagreement":"Agency and Terminal are planning with different berths (the berth_id deviates). {Rule #0006A}",
"validation_rule_fct_agency_and_terminal_pier_side_disagreement":"Agency and Terminal are planning with different pier sides (the pier_side deviates). {Rule #0006B}",
}
class ValidationRuleBaseFunctions():
"""
@ -16,6 +53,16 @@ class ValidationRuleBaseFunctions():
def __init__(self, sql_handler):
self.sql_handler = sql_handler
self.time_logic = TimeLogic()
self.error_message_dict = error_message_dict
def describe_error_message(self, key)->str:
"""
Takes any error message, which typically is the validation rule's function name and returns a description of the error.
In case that the error code is not defined in self.error_message_dict, return the cryptic error code instead
returns: string
"""
return self.error_message_dict.get(key,key)
def check_time_delta_violation_query_time_to_now(self, query_time:pd.Timestamp, key_time:pd.Timestamp, threshold:float)->bool:
"""
@ -36,7 +83,7 @@ class ValidationRuleBaseFunctions():
threshold: threshold where a time difference becomes crucial. When the delta is below the threshold, a violation might occur
"""
# rule is not applicable -> return 'GREEN'
if key_time is not None:
if self.check_is_not_a_time_or_is_none(key_time) or self.check_is_not_a_time_or_is_none(query_time):
return False
# otherwise, this rule applies and the difference between 'now' and the query time is measured
@ -76,7 +123,7 @@ class ValidationRuleBaseFunctions():
df_times = df_times.loc[df_times["participant_type"].isin(participant_types),:]
# exclude missing entries and consider only pd.Timestamp entries (which ignores pd.NaT/null entries)
estimated_times = [type(time_) for time_ in df_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] # df_times = df_times.loc[~df_times[query].isnull(),:]
estimated_times = [time_ for time_ in df_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] # df_times = df_times.loc[~df_times[query].isnull(),:]
# apply rounding. For example, the agreement of different participants may be required to match minute-wise
# '15min' rounds to 'every 15 minutes'. E.g., '2023-09-22 08:18:49' becomes '2023-09-22 08:15:00'
@ -112,6 +159,11 @@ class ValidationRuleBaseFunctions():
violation_state = np.any(np.greater(counts, maximum_threshold))
return violation_state
def check_is_not_a_time_or_is_none(self, value)->bool:
"""checks, if a provided value is either None or NaT"""
return (value is None) or (value is pd.NaT)
class ValidationRuleFunctions(ValidationRuleBaseFunctions):
"""
@ -466,12 +518,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
def validation_rule_fct_missing_time_terminal_berth_etd(self, shipcall, df_times, *args, **kwargs):
"""
Code: #0001-K
Code: #0001-M
Type: Local Rule
Description: this validation checks, whether there is a missing time. When the difference between an event (e.g., the shipcall eta) is below
a certain threshold (e.g., 20 hours), a violation occurs
0001-K:
0001-M:
- Checks, if times_terminal.etd_berth is filled in.
- Measures the difference between 'now' and 'times_agency.etd_berth'.
"""
@ -595,7 +647,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT):
if self.check_is_not_a_time_or_is_none(times_terminal.operations_start) or self.check_is_not_a_time_or_is_none(times_agency.eta_berth):
return (StatusFlags.GREEN, None)
# check, whether the start of operations is AFTER the estimated arrival time
@ -607,7 +659,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
else:
return (StatusFlags.GREEN, None)
def validation_rule_fct_eta_time_not_in_operation_window(self, shipcall, df_times, *args, **kwargs):
def validation_rule_fct_etd_time_not_in_operation_window(self, shipcall, df_times, *args, **kwargs):
"""
Code: #0003-B
Type: Local Rule
@ -624,7 +676,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
if (times_terminal.operations_end is pd.NaT) or (times_agency.etd_berth is pd.NaT):
if self.check_is_not_a_time_or_is_none(times_terminal.operations_end) or self.check_is_not_a_time_or_is_none(times_agency.etd_berth):
return (StatusFlags.GREEN, None)
# check, whether the end of operations is AFTER the estimated departure time
@ -651,11 +703,11 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
# requirements: tidal window (from & to) is filled in
if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.eta_berth is pd.NaT):
if self.check_is_not_a_time_or_is_none(shipcall.tidal_window_from) or self.check_is_not_a_time_or_is_none(shipcall.tidal_window_to) or self.check_is_not_a_time_or_is_none(times_agency.eta_berth): # 202310310: note: this should check times_agency, shouldn't it?
return (StatusFlags.GREEN, None)
# check, whether the query time is between start & end time
# a violation is observed, when the is NOT between start & end
# a violation is observed, when the time is NOT between start & end
violation_state = not self.time_logic.time_inbetween(query_time=times_agency.eta_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to)
if violation_state:
@ -679,11 +731,11 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
# requirements: tidal window (from & to) is filled in
if (shipcall.tidal_window_from is pd.NaT) or (shipcall.tidal_window_to is pd.NaT) or (df_times.etd_berth is pd.NaT):
if self.check_is_not_a_time_or_is_none(shipcall.tidal_window_from) or self.check_is_not_a_time_or_is_none(shipcall.tidal_window_to) or self.check_is_not_a_time_or_is_none(times_agency.etd_berth): # 202310310: note: this should check times_agency, shouldn't it?
return (StatusFlags.GREEN, None)
# check, whether the query time is between start & end time
# a violation is observed, when the is NOT between start & end
# a violation is observed, when the time is NOT between start & end
violation_state = not self.time_logic.time_inbetween(query_time=times_agency.etd_berth, start_time=shipcall.tidal_window_from, end_time=shipcall.tidal_window_to)
if violation_state:
@ -698,6 +750,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
Type: Global Rule
Description: this validation rule checks, whether there are too many shipcalls with identical ETA times.
"""
# check, if the header is filled in (agency)
if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1:
return (StatusFlags.GREEN, None)
# when ANY of the unique values exceeds the threshold, a violation is observed
query = "eta_berth"
violation_state = self.check_unique_shipcall_counts(query, rounding=rounding, maximum_threshold=maximum_threshold)
@ -714,6 +770,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
Type: Global Rule
Description: this validation rule checks, whether there are too many shipcalls with identical ETD times.
"""
# check, if the header is filled in (agency)
if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1:
return (StatusFlags.GREEN, None)
# when ANY of the unique values exceeds the threshold, a violation is observed
query = "etd_berth"
violation_state = self.check_unique_shipcall_counts(query, rounding=rounding, maximum_threshold=maximum_threshold)
@ -737,6 +797,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
# when one of the two values is null, the state is GREEN
if (times_agency.berth_id is None) or (times_terminal.berth_id is None):
return (StatusFlags.GREEN, None)
violation_state = times_agency.berth_id!=times_terminal.berth_id
if violation_state:
@ -758,6 +822,10 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value)
times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value)
# when one of the two values is null, the state is GREEN
if (times_agency.pier_side is None) or (times_terminal.pier_side is None):
return (StatusFlags.GREEN, None)
violation_state = times_agency.pier_side!=times_terminal.pier_side
if violation_state:

3
src/server/BreCal/validators/validation_rules.py
View File

@ -41,6 +41,9 @@ class ValidationRules(ValidationRuleFunctions):
# filter out all 'None' results, which indicate that no violation occured.
evaluation_results = [evaluation_result for evaluation_result in evaluation_results if evaluation_result[1] is not None]
# 'translate' all error codes into readable, human-understandable format.
evaluation_results = [(state, self.describe_error_message(msg)) for (state, msg) in evaluation_results]
""" # deprecated
# check, if ANY of the evaluation results (evaluation_state) is larger than the .GREEN state. This means, that .YELLOW and .RED
# would return 'True'. Numpy arrays and functions are used to accelerate the comparison.

8
src/server/tests/validators/test_validation_rule_functions.py
View File

@ -156,4 +156,12 @@ def test_validation_rule_fct_agency_and_terminal_berth_id_agreement(build_sql_pr
def test_all_validation_rule_fcts_have_a_description():
from BreCal.validators.validation_rule_functions import error_message_dict, ValidationRuleFunctions
import types
vr = ValidationRuleFunctions(sql_handler=None)
assert all(
[mthd_ in list(error_message_dict.keys()) for mthd_ in dir(vr) if ('validation_rule_fct' in mthd_)]
), f"one of the validation_rule_fcts is currently not defined in the error_message_dict and will create cryptic descriptions! Please add it to the error_message_dict BreCal.validators.validation_rule_functions"
return