From b7078f8d8ebf6fe5de9b63e7c80f80c77b396fbb Mon Sep 17 00:00:00 2001 From: Max Metz Date: Mon, 29 Apr 2024 16:46:50 +0200 Subject: [PATCH] implementing POST-request input validation for shipcalls. Creating many tests and applying slight updates to the Notifier (not implemented yet) --- src/server/BreCal/api/shipcalls.py | 32 +---- src/server/BreCal/database/update_database.py | 1 + .../notifications/notification_functions.py | 2 +- src/server/BreCal/schemas/model.py | 9 +- src/server/BreCal/stubs/shipcall.py | 18 +++ .../BreCal/validators/input_validation.py | 133 +++++++++++++++++- .../BreCal/validators/validation_rules.py | 7 +- 7 files changed, 165 insertions(+), 37 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 8adde2f..c1c6e5d 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -4,7 +4,7 @@ from marshmallow import Schema, fields, ValidationError from ..schemas import model from .. import impl from ..services.auth_guard import auth_guard, check_jwt -from BreCal.validators.input_validation import check_if_user_is_bsmd_type, check_if_user_data_has_valid_ship_id, check_if_user_data_has_valid_berth_id, check_if_user_data_has_valid_participant_id +from BreCal.validators.input_validation import validate_posted_shipcall_data import logging import json @@ -41,35 +41,15 @@ def PostShipcalls(): try: content = request.get_json(force=True) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + logging.log(20, loadedModel) + logging.log(20, "dev. above: loaded model, below: content") + logging.log(20, content) # read the user data from the JWT token (set when login is performed) user_data = check_jwt() - # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD - # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. - is_bsmd = check_if_user_is_bsmd_type(user_data) - if not is_bsmd: - raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") - - import logging - logging.log(20, loadedModel) - logging.log(20, "metz development") - """ - # loadedModel ... - loadedModel.get("ship_id", 0) - - 2024-04-22 18:21:03,982 | root | INFO | {'ship_id': 1, - 'type': 1, 'eta': datetime.datetime(2023, 7, 23, 7, 18, 19), - 'voyage': '43B', 'tug_required': False, 'pilot_required': True, - 'flags': 0, 'pier_side': False, 'bunkering': True, 'recommended_tugs': 2, - 'type_value': 1, 'evaluation_value': 0} - - - valid_ship_id = check_if_user_data_has_valid_ship_id(ship_id) - valid_berth_id = check_if_user_data_has_valid_berth_id(berth_id) - valid_participant_id = check_if_user_data_has_valid_participant_id(participant_id) - """ - + # validate the posted shipcall data + validate_posted_shipcall_data(user_data, loadedModel, content) except ValidationError as ex: logging.error(ex) diff --git a/src/server/BreCal/database/update_database.py b/src/server/BreCal/database/update_database.py index 7b7639b..4c120f3 100644 --- a/src/server/BreCal/database/update_database.py +++ b/src/server/BreCal/database/update_database.py @@ -55,6 +55,7 @@ def update_all_shipcalls_in_mysql_database(sql_connection, sql_handler:SQLHandle sql_handler: an SQLHandler instance shipcall_df: dataframe, which stores the data that is used to retrieve the shipcall data models (that are then updated in the database) """ + print(shipcall_df) for shipcall_id in shipcall_df.index: shipcall = sql_handler.df_loc_to_data_model(df=shipcall_df, id=shipcall_id, model_str="shipcall") update_shipcall_in_mysql_database(sql_connection, shipcall=shipcall, relevant_keys = ["evaluation", "evaluation_message"]) diff --git a/src/server/BreCal/notifications/notification_functions.py b/src/server/BreCal/notifications/notification_functions.py index 9bbc373..86956f0 100644 --- a/src/server/BreCal/notifications/notification_functions.py +++ b/src/server/BreCal/notifications/notification_functions.py @@ -110,6 +110,6 @@ class Notifier(): def get_notification_states(self, evaluation_states_old, evaluation_states_new)->list[bool]: """# build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created""" - evaluation_notifications_sent = [self.notifier.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)] + evaluation_notifications_sent = [self.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)] return evaluation_notifications_sent diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index 33638fd..f346783 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -188,9 +188,9 @@ class ShipcallSchema(Schema): pass id = fields.Integer() - ship_id = fields.Integer() + ship_id = fields.Integer(metadata={'required':True}) #type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator - type = fields.Integer() # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator + type = fields.Integer(metadata={'required':True}) # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator eta = fields.DateTime(metadata={'required':False}, allow_none=True) voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} etd = fields.DateTime(metadata={'required':False}, allow_none=True) @@ -207,7 +207,7 @@ class ShipcallSchema(Schema): tidal_window_from = fields.DateTime(metadata={'required':False}, allow_none=True) tidal_window_to = fields.DateTime(metadata={'required':False}, allow_none=True) rain_sensitive_cargo = fields.Bool(metadata={'required':False}, allow_none=True) - recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True) + recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True, validate=[validate.Range(min=0, max=10, min_inclusive=True, max_inclusive=True)]) anchored = fields.Bool(metadata={'required':False}, allow_none=True) moored_lock = fields.Bool(metadata={'required':False}, allow_none=True) canceled = fields.Bool(metadata={'required':False}, allow_none=True) @@ -251,6 +251,9 @@ class Participant_Assignment: participant_id: int type: int # a variant would be to use the IntFlag type (with appropriate serialization) + def to_json(self): + return self.__dict__ + @dataclass class Shipcall: diff --git a/src/server/BreCal/stubs/shipcall.py b/src/server/BreCal/stubs/shipcall.py index 3f5c062..48c81f7 100644 --- a/src/server/BreCal/stubs/shipcall.py +++ b/src/server/BreCal/stubs/shipcall.py @@ -85,4 +85,22 @@ def get_shipcall_simple(): ) return shipcall +def create_postman_stub_shipcall(): + """ + this function returns the common stub, which is used to POST data to shipcalls via POSTMAN. However, + the stub-function is updated with a dynamic ETA in the future, so the POST-request does not fail. + """ + shipcall = { + 'ship_id': 1, + 'type': 1, + 'eta': (datetime.datetime.now()+datetime.timedelta(hours=3)).isoformat(), + 'voyage': '43B', + 'tug_required': False, + 'pilot_required': True, + 'flags': 0, + 'pier_side': False, + 'bunkering': True, + 'recommended_tugs': 2 + } + return shipcall diff --git a/src/server/BreCal/validators/input_validation.py b/src/server/BreCal/validators/input_validation.py index 2248779..2652f11 100644 --- a/src/server/BreCal/validators/input_validation.py +++ b/src/server/BreCal/validators/input_validation.py @@ -2,14 +2,31 @@ ####################################### InputValidation ####################################### import json +import datetime from abc import ABC, abstractmethod -from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant +from marshmallow import ValidationError +from string import ascii_letters, digits + +from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant, ShipcallType from BreCal.impl.participant import GetParticipant from BreCal.impl.ships import GetShips from BreCal.impl.berths import GetBerths from BreCal.database.enums import ParticipantType +def check_if_string_has_special_characters(text:str): + """ + check, whether there are any characters within the provided string, which are not found in the ascii letters or digits + ascii_letters: abcd (...) and ABCD (...) + digits: 0123 (...) + + Source: https://stackoverflow.com/questions/57062794/is-there-a-way-to-check-if-a-string-contains-special-characters + User: https://stackoverflow.com/users/10035985/andrej-kesely + returns bool + """ + return bool(set(text).difference(ascii_letters + digits)) + + def get_participant_id_dictionary(): # get all participants response,status_code,header = GetParticipant(options={}) @@ -61,7 +78,11 @@ def check_if_user_is_bsmd_type(user_data:dict)->bool: is_bsmd = ParticipantType.BSMD in ParticipantType(participant.get("type",0)) return is_bsmd -def check_if_user_data_has_valid_ship_id(ship_id): +def check_if_ship_id_is_valid(ship_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if ship_id is None: + return True + # build a dictionary of id:item pairs, so one can select the respective participant ships = get_ship_id_dictionary() @@ -69,7 +90,11 @@ def check_if_user_data_has_valid_ship_id(ship_id): ship_id_is_valid = ship_id in list(ships.keys()) return ship_id_is_valid -def check_if_user_data_has_valid_berth_id(berth_id): +def check_if_berth_id_is_valid(berth_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if berth_id is None: + return True + # build a dictionary of id:item pairs, so one can select the respective participant berths = get_berth_id_dictionary() @@ -77,7 +102,11 @@ def check_if_user_data_has_valid_berth_id(berth_id): berth_id_is_valid = berth_id in list(berths.keys()) return berth_id_is_valid -def check_if_user_data_has_valid_participant_id(participant_id): +def check_if_participant_id_is_valid(participant_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if participant_id is None: + return True + # build a dictionary of id:item pairs, so one can select the respective participant participants = get_participant_id_dictionary() @@ -85,6 +114,102 @@ def check_if_user_data_has_valid_participant_id(participant_id): participant_id_is_valid = participant_id in list(participants.keys()) return participant_id_is_valid +def check_if_participant_ids_are_valid(participant_ids): + # check each participant id individually + valid_participant_ids = [check_if_participant_id_is_valid(participant_id) for participant_id in participant_ids] + + # boolean check, whether all participant ids are valid + return all(valid_participant_ids) + + +def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict): + """this function applies more complex validation functions to data, which is sent to a post-request of shipcalls""" + # #TODO_refactor: this function is pretty complex. One may instead build an object, which calls the methods separately. + + import logging + logging.log(20, "dev") + logging.log(20, user_data) + logging.log(20, loadedModel) + logging.log(20, content) + ##### Section 1: check user_data ##### + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + + ##### Section 2: check loadedModel ##### + valid_ship_id = check_if_ship_id_is_valid(ship_id=loadedModel.get("ship_id", None)) + if not valid_ship_id: + raise ValidationError(f"provided an invalid ship id, which is not found in the database: {loadedModel.get('ship_id', None)}") + + valid_arrival_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("arrival_berth_id", None)) + if not valid_arrival_berth_id: + raise ValidationError(f"provided an invalid arrival berth id, which is not found in the database: {loadedModel.get('arrival_berth_id', None)}") + + valid_departure_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("departure_berth_id", None)) + if not valid_departure_berth_id: + raise ValidationError(f"provided an invalid departure berth id, which is not found in the database: {loadedModel.get('departure_berth_id', None)}") + + valid_participant_ids = check_if_participant_ids_are_valid(participant_ids=loadedModel.get("participants",[])) + if not valid_participant_ids: + raise ValidationError(f"one of the provided participant ids is invalid. Could not find one of these in the database: {loadedModel.get('participants', None)}") + + + ##### Section 3: check content ##### + # loadedModel fills missing values, sometimes using optional values. Hence, check content + + # the following keys should not be set in a POST-request. + for forbidden_key in ["canceled", "evaluation", "evaluation_message"]: + value = content.get(forbidden_key, None) + if value is not None: + raise ValidationError(f"'{forbidden_key}' may not be set on POST. Found: {value}") + + voyage_str_is_invalid = check_if_string_has_special_characters(text=content.get("voyage","")) + if voyage_str_is_invalid: + raise ValidationError(f"there are invalid characters in the 'voyage'-string. Please use only digits and ASCII letters. Allowed: {ascii_letters+digits}. Found: {content.get('voyage')}") + + + ##### Section 4: check loadedModel & content ##### + # #TODO_refactor: these methods should be placed in separate locations + + # existance checks in content + # datetime checks in loadedModel (datetime.datetime objects). Dates should be in the future. + time_now = datetime.datetime.now() + type_ = loadedModel.get("type", int(ShipcallType.undefined)) + if int(type_)==int(ShipcallType.undefined): + raise ValidationError(f"providing 'type' is mandatory. Missing key!") + elif int(type_)==int(ShipcallType.arrival): + eta = loadedModel.get("eta") + if (content.get("eta", None) is None): + raise ValidationError(f"providing 'eta' is mandatory. Missing key!") + if content.get("arrival_berth_id", None) is None: + raise ValidationError(f"providing 'arrival_berth_id' is mandatory. Missing key!") + if not eta >= time_now: + raise ValidationError(f"'eta' must be in the future. Incorrect datetime provided.") + elif int(type_)==int(ShipcallType.departure): + etd = loadedModel.get("etd") + if (content.get("etd", None) is None): + raise ValidationError(f"providing 'etd' is mandatory. Missing key!") + if content.get("departure_berth_id", None) is None: + raise ValidationError(f"providing 'departure_berth_id' is mandatory. Missing key!") + if not etd >= time_now: + raise ValidationError(f"'etd' must be in the future. Incorrect datetime provided.") + elif int(type_)==int(ShipcallType.shifting): + eta = loadedModel.get("eta") + etd = loadedModel.get("etd") + # * arrival_berth_id / departure_berth_id (depending on type, see above) + if (content.get("eta", None) is None) or (content.get("etd", None) is None): + raise ValidationError(f"providing 'eta' and 'etd' is mandatory. Missing one of those keys!") + if (content.get("arrival_berth_id", None) is None) or (content.get("departure_berth_id", None) is None): + raise ValidationError(f"providing 'arrival_berth_id' & 'departure_berth_id' is mandatory. Missing key!") + if (not eta >= time_now) or (not etd >= time_now) or (not eta >= etd): + raise ValidationError(f"'eta' and 'etd' must be in the future. Incorrect datetime provided.") + + + # #TODO: len of participants > 0, if agency + # * assigned participant for agency + return class InputValidation(): diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index 69cc5ee..7d5448c 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -75,6 +75,7 @@ class ValidationRules(ValidationRuleFunctions): def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame: """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation', 'evaluation_message', 'evaluation_time' and 'evaluation_notifications_sent' are updated)""" evaluation_states_old = [state_old for state_old in shipcall_df.loc[:,"evaluation"]] + evaluation_states_old = [state_old if not pd.isna(state_old) else 0 for state_old in evaluation_states_old] results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values # returns tuple (state, message) # unbundle individual results. evaluation_states becomes an integer, violation @@ -83,14 +84,14 @@ class ValidationRules(ValidationRuleFunctions): violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] # build the list of evaluation times ('now', as isoformat) - evaluation_times = self.notifier.get_notification_times(evaluation_states_new) + evaluation_time = self.notifier.get_notification_times(evaluation_states_new) # build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created - evaluation_notifications_sent = self.get_notification_states(evaluation_states_old, evaluation_states_new) + evaluation_notifications_sent = self.notifier.get_notification_states(evaluation_states_old, evaluation_states_new) shipcall_df.loc[:,"evaluation"] = evaluation_states_new shipcall_df.loc[:,"evaluation_message"] = violations - shipcall_df.loc[:,"evaluation_times"] = evaluation_times + shipcall_df.loc[:,"evaluation_time"] = evaluation_time shipcall_df.loc[:,"evaluation_notifications_sent"] = evaluation_notifications_sent return shipcall_df