implementing POST-request input validation for shipcalls. Creating many tests and applying slight updates to the Notifier (not implemented yet)

src/server/BreCal/api/shipcalls.py

@@ -4,7 +4,7 @@ from marshmallow import Schema, fields, ValidationError
 from ..schemas import model
 from .. import impl
 from ..services.auth_guard import auth_guard, check_jwt
-from BreCal.validators.input_validation import check_if_user_is_bsmd_type, check_if_user_data_has_valid_ship_id, check_if_user_data_has_valid_berth_id, check_if_user_data_has_valid_participant_id
+from BreCal.validators.input_validation import validate_posted_shipcall_data
 
 import logging
 import json
@@ -41,35 +41,15 @@ def PostShipcalls():
     try:
         content = request.get_json(force=True)
         loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True)
+        logging.log(20, loadedModel)
+        logging.log(20, "dev. above: loaded model, below: content")
+        logging.log(20, content)
 
         # read the user data from the JWT token (set when login is performed)
         user_data = check_jwt()
 
-        # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD
+        # validate the posted shipcall data
-        # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated.
+        validate_posted_shipcall_data(user_data, loadedModel, content)
-        is_bsmd = check_if_user_is_bsmd_type(user_data)
-        if not is_bsmd:
-            raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}")
-        
-        import logging
-        logging.log(20, loadedModel)
-        logging.log(20, "metz development")        
-        """
-        # loadedModel ...
-        loadedModel.get("ship_id", 0)
-
-        2024-04-22 18:21:03,982 | root | INFO | {'ship_id': 1, 
-        'type': 1, 'eta': datetime.datetime(2023, 7, 23, 7, 18, 19),
-        'voyage': '43B', 'tug_required': False, 'pilot_required': True, 
-        'flags': 0, 'pier_side': False, 'bunkering': True, 'recommended_tugs': 2,
-        'type_value': 1, 'evaluation_value': 0}
-
-
-        valid_ship_id = check_if_user_data_has_valid_ship_id(ship_id)
-        valid_berth_id = check_if_user_data_has_valid_berth_id(berth_id)
-        valid_participant_id = check_if_user_data_has_valid_participant_id(participant_id)
-        """
-
         
     except ValidationError as ex:
         logging.error(ex)

src/server/BreCal/database/update_database.py

@@ -55,6 +55,7 @@ def update_all_shipcalls_in_mysql_database(sql_connection, sql_handler:SQLHandle
         sql_handler: an SQLHandler instance
         shipcall_df: dataframe, which stores the data that is used to retrieve the shipcall data models (that are then updated in the database)
     """
+    print(shipcall_df)
     for shipcall_id in shipcall_df.index:
         shipcall = sql_handler.df_loc_to_data_model(df=shipcall_df, id=shipcall_id, model_str="shipcall")
         update_shipcall_in_mysql_database(sql_connection, shipcall=shipcall, relevant_keys = ["evaluation", "evaluation_message"])

src/server/BreCal/notifications/notification_functions.py

@@ -110,6 +110,6 @@ class Notifier():
     
     def get_notification_states(self, evaluation_states_old, evaluation_states_new)->list[bool]:
         """# build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created"""
-        evaluation_notifications_sent = [self.notifier.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)]
+        evaluation_notifications_sent = [self.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)]
         return evaluation_notifications_sent
 

src/server/BreCal/schemas/model.py

@@ -188,9 +188,9 @@ class ShipcallSchema(Schema):
         pass
 
     id = fields.Integer()
-    ship_id = fields.Integer()
+    ship_id = fields.Integer(metadata={'required':True})
     #type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator
-    type = fields.Integer() # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator
+    type = fields.Integer(metadata={'required':True}) # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator
     eta = fields.DateTime(metadata={'required':False}, allow_none=True)
     voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False}
     etd = fields.DateTime(metadata={'required':False}, allow_none=True)
@@ -207,7 +207,7 @@ class ShipcallSchema(Schema):
     tidal_window_from = fields.DateTime(metadata={'required':False}, allow_none=True)
     tidal_window_to = fields.DateTime(metadata={'required':False}, allow_none=True)
     rain_sensitive_cargo = fields.Bool(metadata={'required':False}, allow_none=True)
-    recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True)
+    recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True, validate=[validate.Range(min=0, max=10, min_inclusive=True, max_inclusive=True)])
     anchored = fields.Bool(metadata={'required':False}, allow_none=True)
     moored_lock = fields.Bool(metadata={'required':False}, allow_none=True)
     canceled = fields.Bool(metadata={'required':False}, allow_none=True)
@@ -251,6 +251,9 @@ class Participant_Assignment:
     participant_id: int
     type: int # a variant would be to use the IntFlag type (with appropriate serialization)
 
+    def to_json(self):
+        return self.__dict__
+
 
 @dataclass
 class Shipcall:

src/server/BreCal/stubs/shipcall.py

@@ -85,4 +85,22 @@ def get_shipcall_simple():
     )
     return shipcall
 
+def create_postman_stub_shipcall():
+    """
+    this function returns the common stub, which is used to POST data to shipcalls via POSTMAN. However,
+    the stub-function is updated with a dynamic ETA in the future, so the POST-request does not fail. 
+    """
+    shipcall = {
+        'ship_id': 1,
+        'type': 1,
+        'eta': (datetime.datetime.now()+datetime.timedelta(hours=3)).isoformat(),
+        'voyage': '43B',
+        'tug_required': False,
+        'pilot_required': True,
+        'flags': 0,
+        'pier_side': False,
+        'bunkering': True,
+        'recommended_tugs': 2
+    }
+    return shipcall
 

src/server/BreCal/validators/input_validation.py

@@ -2,14 +2,31 @@
 ####################################### InputValidation #######################################
 
 import json
+import datetime
 from abc import ABC, abstractmethod
-from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant
+from marshmallow import ValidationError
+from string import ascii_letters, digits
+
+from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant, ShipcallType
 from BreCal.impl.participant import GetParticipant
 from BreCal.impl.ships import GetShips
 from BreCal.impl.berths import GetBerths
 
 from BreCal.database.enums import ParticipantType
 
+def check_if_string_has_special_characters(text:str):
+    """
+    check, whether there are any characters within the provided string, which are not found in the ascii letters or digits
+        ascii_letters: abcd (...) and ABCD (...)
+        digits: 0123 (...)
+
+    Source: https://stackoverflow.com/questions/57062794/is-there-a-way-to-check-if-a-string-contains-special-characters
+    User: https://stackoverflow.com/users/10035985/andrej-kesely
+    returns bool
+    """
+    return bool(set(text).difference(ascii_letters + digits))
+
+
 def get_participant_id_dictionary():
     # get all participants
     response,status_code,header = GetParticipant(options={})
@@ -61,7 +78,11 @@ def check_if_user_is_bsmd_type(user_data:dict)->bool:
     is_bsmd = ParticipantType.BSMD in ParticipantType(participant.get("type",0)) 
     return is_bsmd
 
-def check_if_user_data_has_valid_ship_id(ship_id):
+def check_if_ship_id_is_valid(ship_id):
+    """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once"""
+    if ship_id is None:
+        return True
+    
     # build a dictionary of id:item pairs, so one can select the respective participant
     ships = get_ship_id_dictionary()
     
@@ -69,7 +90,11 @@ def check_if_user_data_has_valid_ship_id(ship_id):
     ship_id_is_valid = ship_id in list(ships.keys())
     return ship_id_is_valid
 
-def check_if_user_data_has_valid_berth_id(berth_id):
+def check_if_berth_id_is_valid(berth_id):
+    """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once"""
+    if berth_id is None:
+        return True
+    
     # build a dictionary of id:item pairs, so one can select the respective participant
     berths = get_berth_id_dictionary()
 
@@ -77,7 +102,11 @@ def check_if_user_data_has_valid_berth_id(berth_id):
     berth_id_is_valid = berth_id in list(berths.keys())
     return berth_id_is_valid
 
-def check_if_user_data_has_valid_participant_id(participant_id):
+def check_if_participant_id_is_valid(participant_id):
+    """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once"""
+    if participant_id is None:
+        return True
+    
     # build a dictionary of id:item pairs, so one can select the respective participant
     participants = get_participant_id_dictionary()
 
@@ -85,6 +114,102 @@ def check_if_user_data_has_valid_participant_id(participant_id):
     participant_id_is_valid = participant_id in list(participants.keys())
     return participant_id_is_valid
 
+def check_if_participant_ids_are_valid(participant_ids):
+    # check each participant id individually
+    valid_participant_ids = [check_if_participant_id_is_valid(participant_id) for participant_id in participant_ids]
+
+    # boolean check, whether all participant ids are valid
+    return all(valid_participant_ids)
+
+
+def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict):
+    """this function applies more complex validation functions to data, which is sent to a post-request of shipcalls"""
+    # #TODO_refactor: this function is pretty complex. One may instead build an object, which calls the methods separately. 
+
+    import logging
+    logging.log(20, "dev")
+    logging.log(20, user_data)
+    logging.log(20, loadedModel)
+    logging.log(20, content)
+    ##### Section 1: check user_data #####
+    # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD
+    # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated.
+    is_bsmd = check_if_user_is_bsmd_type(user_data)
+    if not is_bsmd:
+        raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}")
+    
+    ##### Section 2: check loadedModel #####
+    valid_ship_id = check_if_ship_id_is_valid(ship_id=loadedModel.get("ship_id", None))
+    if not valid_ship_id:
+        raise ValidationError(f"provided an invalid ship id, which is not found in the database: {loadedModel.get('ship_id', None)}")
+    
+    valid_arrival_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("arrival_berth_id", None))
+    if not valid_arrival_berth_id:
+        raise ValidationError(f"provided an invalid arrival berth id, which is not found in the database: {loadedModel.get('arrival_berth_id', None)}")
+    
+    valid_departure_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("departure_berth_id", None))
+    if not valid_departure_berth_id:
+        raise ValidationError(f"provided an invalid departure berth id, which is not found in the database: {loadedModel.get('departure_berth_id', None)}")
+    
+    valid_participant_ids = check_if_participant_ids_are_valid(participant_ids=loadedModel.get("participants",[]))
+    if not valid_participant_ids:
+        raise ValidationError(f"one of the provided participant ids is invalid. Could not find one of these in the database: {loadedModel.get('participants', None)}")   
+
+
+    ##### Section 3: check content #####
+    # loadedModel fills missing values, sometimes using optional values. Hence, check content
+
+    # the following keys should not be set in a POST-request.
+    for forbidden_key in ["canceled", "evaluation", "evaluation_message"]:
+        value = content.get(forbidden_key, None)
+        if value is not None:
+            raise ValidationError(f"'{forbidden_key}' may not be set on POST. Found: {value}")
+        
+    voyage_str_is_invalid = check_if_string_has_special_characters(text=content.get("voyage",""))
+    if voyage_str_is_invalid:
+        raise ValidationError(f"there are invalid characters in the 'voyage'-string. Please use only digits and ASCII letters. Allowed: {ascii_letters+digits}. Found: {content.get('voyage')}")   
+    
+
+    ##### Section 4: check loadedModel & content #####
+    # #TODO_refactor: these methods should be placed in separate locations
+
+    # existance checks in content
+    # datetime checks in loadedModel (datetime.datetime objects). Dates should be in the future. 
+    time_now = datetime.datetime.now()
+    type_ = loadedModel.get("type", int(ShipcallType.undefined))
+    if int(type_)==int(ShipcallType.undefined):
+        raise ValidationError(f"providing 'type' is mandatory. Missing key!")
+    elif int(type_)==int(ShipcallType.arrival):
+        eta = loadedModel.get("eta")
+        if (content.get("eta", None) is None):
+            raise ValidationError(f"providing 'eta' is mandatory. Missing key!")
+        if content.get("arrival_berth_id", None) is None:
+            raise ValidationError(f"providing 'arrival_berth_id' is mandatory. Missing key!")
+        if not eta >= time_now:
+            raise ValidationError(f"'eta' must be in the future. Incorrect datetime provided.")
+    elif int(type_)==int(ShipcallType.departure):
+        etd = loadedModel.get("etd")
+        if (content.get("etd", None) is None):
+            raise ValidationError(f"providing 'etd' is mandatory. Missing key!")
+        if content.get("departure_berth_id", None) is None:
+            raise ValidationError(f"providing 'departure_berth_id' is mandatory. Missing key!")
+        if not etd >= time_now:
+            raise ValidationError(f"'etd' must be in the future. Incorrect datetime provided.")
+    elif int(type_)==int(ShipcallType.shifting):
+        eta = loadedModel.get("eta")
+        etd = loadedModel.get("etd")
+        # * arrival_berth_id / departure_berth_id (depending on type, see above)
+        if (content.get("eta", None) is None) or (content.get("etd", None) is None):
+            raise ValidationError(f"providing 'eta' and 'etd' is mandatory. Missing one of those keys!")
+        if (content.get("arrival_berth_id", None) is None) or (content.get("departure_berth_id", None) is None):
+            raise ValidationError(f"providing 'arrival_berth_id' & 'departure_berth_id' is mandatory. Missing key!")
+        if (not eta >= time_now) or (not etd >= time_now) or (not eta >= etd):
+            raise ValidationError(f"'eta' and 'etd' must be in the future. Incorrect datetime provided.")
+    
+    
+    # #TODO: len of participants > 0, if agency
+    # * assigned participant for agency
+    return
 
 
 class InputValidation():

src/server/BreCal/validators/validation_rules.py

@@ -75,6 +75,7 @@ class ValidationRules(ValidationRuleFunctions):
     def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame:
         """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation', 'evaluation_message', 'evaluation_time' and 'evaluation_notifications_sent' are updated)"""
         evaluation_states_old = [state_old for state_old in shipcall_df.loc[:,"evaluation"]]
+        evaluation_states_old = [state_old if not pd.isna(state_old) else 0 for state_old in evaluation_states_old]
         results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values # returns tuple (state, message)
 
         # unbundle individual results. evaluation_states becomes an integer, violation 
@@ -83,14 +84,14 @@ class ValidationRules(ValidationRuleFunctions):
         violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations]
 
         # build the list of evaluation times ('now', as isoformat)
-        evaluation_times = self.notifier.get_notification_times(evaluation_states_new)
+        evaluation_time = self.notifier.get_notification_times(evaluation_states_new)
 
         # build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created
-        evaluation_notifications_sent = self.get_notification_states(evaluation_states_old, evaluation_states_new)
+        evaluation_notifications_sent = self.notifier.get_notification_states(evaluation_states_old, evaluation_states_new)
 
         shipcall_df.loc[:,"evaluation"] = evaluation_states_new
         shipcall_df.loc[:,"evaluation_message"] = violations
-        shipcall_df.loc[:,"evaluation_times"] = evaluation_times
+        shipcall_df.loc[:,"evaluation_time"] = evaluation_time
         shipcall_df.loc[:,"evaluation_notifications_sent"] = evaluation_notifications_sent
         return shipcall_df