Merge pull request #38 from puls200/hotfix/api_input_validation_20240812

marshmallow.fields incorrectly resolved the 'required' field. Adapted…

src/server/BreCal/api/__init__.py

@@ -0,0 +1,15 @@
+
+
+import json
+import logging
+from flask import request
+
+def verify_if_request_is_json(request):
+    """
+    when a request contains invalid JSON data, this function raises a 400 error (bad request) and returns an error description.
+    this function avoids less precise 500 Internal Server Error messages.
+    """
+    if request.is_json:
+        # when invalid json data is posted, a JSONDecodeError will be raised
+        json.loads(request.data)
+    return

src/server/BreCal/api/shipcalls.py

@@ -7,6 +7,7 @@ from ..services.auth_guard import auth_guard, check_jwt
 from BreCal.validators.input_validation import validate_posted_shipcall_data, check_if_user_is_bsmd_type
 from BreCal.validators.input_validation_shipcall import InputValidationShipcall
 from BreCal.database.sql_handler import execute_sql_query_standalone
+from . import verify_if_request_is_json
 
 import logging
 import json
@@ -43,6 +44,8 @@ def GetShipcalls():
 def PostShipcalls():
 
     try:
+        verify_if_request_is_json(request)
+
         content = request.get_json(force=True)
         loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True)
 
@@ -71,6 +74,8 @@ def PostShipcalls():
 def PutShipcalls():
 
     try:
+        verify_if_request_is_json(request)
+        
         content = request.get_json(force=True)
         loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True)
 

src/server/BreCal/api/ships.py

@@ -5,6 +5,7 @@ from marshmallow import EXCLUDE, ValidationError
 from ..schemas import model
 import json
 import logging
+from . import verify_if_request_is_json
 
 from BreCal.validators.input_validation import check_if_user_is_bsmd_type
 from BreCal.validators.input_validation_ship import InputValidationShip
@@ -27,6 +28,8 @@ def GetShips():
 def PostShip():
 
     try:
+        verify_if_request_is_json(request)
+
         # read the user data from the JWT token (set when login is performed)
         user_data = check_jwt()
 
@@ -55,6 +58,8 @@ def PostShip():
 def PutShip():
 
     try:
+        verify_if_request_is_json(request)
+
         # read the user data from the JWT token (set when login is performed)
         user_data = check_jwt()
 
@@ -77,6 +82,8 @@ def PutShip():
 def DeleteShip():
 
     try:
+        verify_if_request_is_json(request)
+        
         # read the user data from the JWT token (set when login is performed)
         user_data = check_jwt()
         ship_id = request.args.get("id")

src/server/BreCal/api/times.py

@@ -6,6 +6,7 @@ import json
 import logging
 from marshmallow import ValidationError
 from BreCal.validators.input_validation_times import InputValidationTimes
+from . import verify_if_request_is_json
 
 bp = Blueprint('times', __name__)
 
@@ -25,6 +26,8 @@ def GetTimes():
 def PostTimes():
 
     try:
+        verify_if_request_is_json(request)
+
         # print (request.is_json)
         content = request.get_json(force=True) # force gets us json even if the content-type was wrong
 
@@ -56,6 +59,8 @@ def PostTimes():
 def PutTimes():
 
     try:
+        verify_if_request_is_json(request)
+        
         content = request.get_json(force=True)
         loadedModel = model.TimesSchema().load(data=content, many=False, partial=True)
 

src/server/BreCal/api/user.py

@@ -5,6 +5,7 @@ from ..services.auth_guard import auth_guard
 import json
 import logging
 from marshmallow import ValidationError
+from . import verify_if_request_is_json
 
 bp = Blueprint('user', __name__)
 
@@ -13,6 +14,8 @@ bp = Blueprint('user', __name__)
 def PutUser():
 
     try:
+        verify_if_request_is_json(request)
+        
         content = request.get_json(force=True)
         loadedModel = model.UserSchema().load(data=content, many=False, partial=True)
         

src/server/BreCal/database/enums.py

@@ -11,6 +11,10 @@ class ParticipantType(IntFlag):
     PORT_ADMINISTRATION = 32
     TUG = 64
 
+    @classmethod
+    def _missing_(cls, value):
+        return cls.undefined
+
 class ShipcallType(IntEnum):
     """determines the type of a shipcall, as this changes the applicable validation rules"""
     undefined = 0

src/server/BreCal/database/sql_handler.py

@@ -33,7 +33,9 @@ def get_synchronous_shipcall_times_standalone(query_time:pd.Timestamp, all_df_ti
 
     returns: counts
     """
-    assert isinstance(query_time,pd.Timestamp)
+    assert (isinstance(query_time,pd.Timestamp)) or (pd.isnull(query_time)), f"expected a timestamp. Found type: {type(query_time)} with value: {query_time}"
+    if pd.isnull(query_time):
+        return 0
 
     # get a timedelta for each valid (not Null) time entry
     time_deltas_eta = [(query_time.to_pydatetime()-time_.to_pydatetime()) for time_ in all_df_times.loc[:,"eta_berth"] if not pd.isnull(time_)]
@@ -442,4 +444,6 @@ class SQLHandler():
 
     def count_synchronous_shipcall_times(self, query_time:pd.Timestamp, all_df_times:pd.DataFrame, delta_threshold=900)->int:
         """count all times entries, which are too close to the query_time. The {delta_threshold} determines the threshold. returns counts (int)"""
+        if all_df_times is None:
+            all_df_times = self.df_dict.get("times")
         return get_synchronous_shipcall_times_standalone(query_time, all_df_times, delta_threshold)

src/server/BreCal/schemas/model.py

@@ -119,7 +119,7 @@ class History:
         return self(id, participant_id, shipcall_id, timestamp, eta, ObjectType(type), OperationType(operation))
 
 class Error(Schema):
-    message = fields.String(metadata={'required':True})
+    message = fields.String(required=True)
 
 
 class GetVerifyInlineResp(Schema):
@@ -203,37 +203,37 @@ class ShipcallSchema(Schema):
         super().__init__(unknown=None)
         pass
 
-    id = fields.Integer(metadata={'required':True})
-    ship_id = fields.Integer(metadata={'required':True})
+    id = fields.Integer(required=True)
+    ship_id = fields.Integer(required=True)
     type = fields.Enum(ShipcallType, default=ShipcallType.undefined)
-    eta = fields.DateTime(metadata={'required':False}, allow_none=True)
-    voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False}
-    etd = fields.DateTime(metadata={'required':False}, allow_none=True)
-    arrival_berth_id = fields.Integer(metadata={'required':False}, allow_none=True)
-    departure_berth_id = fields.Integer(metadata={'required':False}, allow_none=True)
-    tug_required = fields.Bool(metadata={'required':False}, allow_none=True)
-    pilot_required = fields.Bool(metadata={'required':False}, allow_none=True)
-    flags = fields.Integer(metadata={'required':False}, allow_none=True)
-    pier_side = fields.Bool(metadata={'required':False}, allow_none=True)
-    bunkering = fields.Bool(metadata={'required':False}, allow_none=True)
-    replenishing_terminal = fields.Bool(metadata={'required':False}, allow_none=True)
-    replenishing_lock = fields.Bool(metadata={'required':False}, allow_none=True)
-    draft = fields.Float(metadata={'required':False}, allow_none=True, validate=[validate.Range(min=0, max=20, min_inclusive=False, max_inclusive=True)])
-    tidal_window_from = fields.DateTime(metadata={'required':False}, allow_none=True)
-    tidal_window_to = fields.DateTime(metadata={'required':False}, allow_none=True)
-    rain_sensitive_cargo = fields.Bool(metadata={'required':False}, allow_none=True)
-    recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True, validate=[validate.Range(min=0, max=10, min_inclusive=True, max_inclusive=True)])
-    anchored = fields.Bool(metadata={'required':False}, allow_none=True)
-    moored_lock = fields.Bool(metadata={'required':False}, allow_none=True)
-    canceled = fields.Bool(metadata={'required':False}, allow_none=True)
-    evaluation = fields.Enum(EvaluationType,  metadata={'required':False}, allow_none=True, default=EvaluationType.undefined)
-    evaluation_message = fields.Str(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False}
-    evaluation_time = fields.DateTime(metadata={'required':False}, allow_none=True)
-    evaluation_notifications_sent = fields.Bool(metadata={'required':False}, allow_none=True)
-    time_ref_point = fields.Integer(metadata={'required':False}, allow_none=True)
+    eta = fields.DateTime(required=False, allow_none=True)
+    voyage = fields.String(allow_none=True, required=False, validate=[validate.Length(max=16)])
+    etd = fields.DateTime(required=False, allow_none=True)
+    arrival_berth_id = fields.Integer(required=False, allow_none=True)
+    departure_berth_id = fields.Integer(required=False, allow_none=True)
+    tug_required = fields.Bool(required=False, allow_none=True)
+    pilot_required = fields.Bool(required=False, allow_none=True)
+    flags = fields.Integer(required=False, allow_none=True)
+    pier_side = fields.Bool(required=False, allow_none=True)
+    bunkering = fields.Bool(required=False, allow_none=True)
+    replenishing_terminal = fields.Bool(required=False, allow_none=True)
+    replenishing_lock = fields.Bool(required=False, allow_none=True)
+    draft = fields.Float(required=False, allow_none=True, validate=[validate.Range(min=0, max=20, min_inclusive=False, max_inclusive=True)])
+    tidal_window_from = fields.DateTime(required=False, allow_none=True)
+    tidal_window_to = fields.DateTime(required=False, allow_none=True)
+    rain_sensitive_cargo = fields.Bool(required=False, allow_none=True)
+    recommended_tugs = fields.Integer(required=False, allow_none=True, validate=[validate.Range(min=0, max=10, min_inclusive=True, max_inclusive=True)])
+    anchored = fields.Bool(required=False, allow_none=True)
+    moored_lock = fields.Bool(required=False, allow_none=True)
+    canceled = fields.Bool(required=False, allow_none=True)
+    evaluation = fields.Enum(EvaluationType,  required=False, allow_none=True, default=EvaluationType.undefined)
+    evaluation_message = fields.Str(allow_none=True, required=False)
+    evaluation_time = fields.DateTime(required=False, allow_none=True)
+    evaluation_notifications_sent = fields.Bool(required=False, allow_none=True)
+    time_ref_point = fields.Integer(required=False, allow_none=True)
     participants = fields.List(fields.Nested(ParticipantAssignmentSchema))
-    created = fields.DateTime(metadata={'required':False}, allow_none=True)
-    modified = fields.DateTime(metadata={'required':False}, allow_none=True)
+    created = fields.DateTime(required=False, allow_none=True)
+    modified = fields.DateTime(required=False, allow_none=True)
 
     @post_load
     def make_shipcall(self, data, **kwargs):
@@ -357,30 +357,30 @@ class TimesSchema(Schema):
         super().__init__(unknown=None)
         pass
 
-    id = fields.Integer(metadata={'required':False})
-    eta_berth = fields.DateTime(metadata={'required':False}, allow_none=True)
-    eta_berth_fixed = fields.Bool(metadata={'required':False}, allow_none=True)
-    etd_berth = fields.DateTime(metadata={'required':False}, allow_none=True)
-    etd_berth_fixed = fields.Bool(metadata={'required':False}, allow_none=True)
-    lock_time = fields.DateTime(metadata={'required':False}, allow_none=True)
-    lock_time_fixed = fields.Bool(metadata={'required':False}, allow_none=True)
-    zone_entry = fields.DateTime(metadata={'required':False}, allow_none=True)
-    zone_entry_fixed = fields.Bool(metadata={'required':False}, allow_none=True)
-    operations_start = fields.DateTime(metadata={'required':False}, allow_none=True)
-    operations_end  = fields.DateTime(metadata={'required':False}, allow_none=True)
-    remarks = fields.String(metadata={'required':False}, allow_none=True, validate=[validate.Length(max=512)])
-    participant_id = fields.Integer(metadata={'required':True})
-    berth_id = fields.Integer(metadata={'required':False}, allow_none = True)
-    berth_info = fields.String(metadata={'required':False}, allow_none=True, validate=[validate.Length(max=512)])
-    pier_side = fields.Bool(metadata={'required':False}, allow_none = True)
-    shipcall_id = fields.Integer(metadata={'required':True})
-    participant_type = fields.Integer(Required = False, allow_none=True)# TODO: could become Enum. fields.Enum(ParticipantType, metadata={'required':False}, allow_none=True, default=ParticipantType.undefined) #fields.Integer(metadata={'required':False}, allow_none=True)
-    ata = fields.DateTime(metadata={'required':False}, allow_none=True)
-    atd = fields.DateTime(metadata={'required':False}, allow_none=True)
-    eta_interval_end = fields.DateTime(metadata={'required':False}, allow_none=True)
-    etd_interval_end = fields.DateTime(metadata={'required':False}, allow_none=True)
-    created = fields.DateTime(metadata={'required':False}, allow_none=True)
-    modified = fields.DateTime(metadata={'required':False}, allow_none=True)
+    id = fields.Integer(required=False)
+    eta_berth = fields.DateTime(required=False, allow_none=True)
+    eta_berth_fixed = fields.Bool(required=False, allow_none=True)
+    etd_berth = fields.DateTime(required=False, allow_none=True)
+    etd_berth_fixed = fields.Bool(required=False, allow_none=True)
+    lock_time = fields.DateTime(required=False, allow_none=True)
+    lock_time_fixed = fields.Bool(required=False, allow_none=True)
+    zone_entry = fields.DateTime(required=False, allow_none=True)
+    zone_entry_fixed = fields.Bool(required=False, allow_none=True)
+    operations_start = fields.DateTime(required=False, allow_none=True)
+    operations_end  = fields.DateTime(required=False, allow_none=True)
+    remarks = fields.String(required=False, allow_none=True, validate=[validate.Length(max=512)])
+    participant_id = fields.Integer(required=True)
+    berth_id = fields.Integer(required=False, allow_none = True)
+    berth_info = fields.String(required=False, allow_none=True, validate=[validate.Length(max=512)])
+    pier_side = fields.Bool(required=False, allow_none = True)
+    shipcall_id = fields.Integer(required=True)
+    participant_type = fields.Integer(Required = False, allow_none=True)# TODO: could become Enum. # participant_type = fields.Enum(ParticipantType, required=False, allow_none=True, default=ParticipantType.undefined) #fields.Integer(required=False, allow_none=True)
+    ata = fields.DateTime(required=False, allow_none=True)
+    atd = fields.DateTime(required=False, allow_none=True)
+    eta_interval_end = fields.DateTime(required=False, allow_none=True)
+    etd_interval_end = fields.DateTime(required=False, allow_none=True)
+    created = fields.DateTime(required=False, allow_none=True)
+    modified = fields.DateTime(required=False, allow_none=True)
 
     @validates("participant_type")
     def validate_participant_type(self, value):
@@ -443,13 +443,13 @@ class UserSchema(Schema):
     def __init__(self):
         super().__init__(unknown=None)
         pass
-    id = fields.Integer(metadata={'required':True})
-    first_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)])
-    last_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)])
-    user_phone = fields.String(allow_none=True, metadata={'Required':False})
-    user_email = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)])
-    old_password = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=128)])
-    new_password = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(min=6, max=128)])
+    id = fields.Integer(required=True)
+    first_name = fields.String(allow_none=True, required=False, validate=[validate.Length(max=64)])
+    last_name = fields.String(allow_none=True, required=False, validate=[validate.Length(max=64)])
+    user_phone = fields.String(allow_none=True, required=False)
+    user_email = fields.String(allow_none=True, required=False, validate=[validate.Length(max=64)])
+    old_password = fields.String(allow_none=True, required=False, validate=[validate.Length(max=128)])
+    new_password = fields.String(allow_none=True, required=False, validate=[validate.Length(min=6, max=128)])
     # #TODO: the user schema does not (yet) include the 'notify_' fields
 
     @validates("user_phone")
@@ -532,19 +532,19 @@ class ShipSchema(Schema):
         super().__init__(unknown=None)
         pass
 
-    id = fields.Int(metadata={'required':False})
-    name = fields.String(allow_none=False, metadata={'Required':True})
-    imo = fields.Int(allow_none=False, metadata={'Required':True})
-    callsign = fields.String(allow_none=True, metadata={'Required':False})
-    participant_id = fields.Int(allow_none=True, metadata={'Required':False})
-    length = fields.Float(allow_none=True, metadata={'Required':False}, validate=[validate.Range(min=0, max=1000, min_inclusive=False, max_inclusive=False)])
-    width = fields.Float(allow_none=True, metadata={'Required':False}, validate=[validate.Range(min=0, max=100, min_inclusive=False, max_inclusive=False)])
-    is_tug = fields.Bool(allow_none=True, metadata={'Required':False}, default=False)
-    bollard_pull = fields.Int(allow_none=True, metadata={'Required':False})
-    eni = fields.Int(allow_none=True, metadata={'Required':False})
-    created = fields.DateTime(allow_none=True, metadata={'Required':False})
-    modified = fields.DateTime(allow_none=True, metadata={'Required':False})
-    deleted = fields.Bool(allow_none=True, metadata={'Required':False}, default=False)
+    id = fields.Int(required=False)
+    name = fields.String(allow_none=False, required=True)
+    imo = fields.Int(allow_none=False, required=True)
+    callsign = fields.String(allow_none=True, required=False)
+    participant_id = fields.Int(allow_none=True, required=False)
+    length = fields.Float(allow_none=True, required=False, validate=[validate.Range(min=0, max=1000, min_inclusive=False, max_inclusive=False)])
+    width = fields.Float(allow_none=True, required=False, validate=[validate.Range(min=0, max=100, min_inclusive=False, max_inclusive=False)])
+    is_tug = fields.Bool(allow_none=True, required=False, default=False)
+    bollard_pull = fields.Int(allow_none=True, required=False)
+    eni = fields.Int(allow_none=True, required=False)
+    created = fields.DateTime(allow_none=True, required=False)
+    modified = fields.DateTime(allow_none=True, required=False)
+    deleted = fields.Bool(allow_none=True, required=False, default=False)
 
     @validates("name")
     def validate_name(self, value):

src/server/BreCal/validators/validation_rule_functions.py

@@ -920,6 +920,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
         Type: Global Rule
         Description: this validation rule checks, whether there are too many shipcalls with identical times to the query ETA.
         """
+        if all_times_agency is None:
+            all_times_agency = self.sql_handler.get_times_for_agency(non_null_column="eta_berth")
+
         # check, if the header is filled in (agency)
         if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): # if len(times_agency) != 1:
             return self.get_no_violation_default_output()
@@ -929,6 +932,7 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
         query_time = times_agency.iloc[0].eta_berth
 
         # count the number of times, where a times entry is very close to the query time (uses an internal threshold, such as 15 minutes)
+
         counts = self.sql_handler.count_synchronous_shipcall_times(query_time, all_df_times=all_times_agency)
         violation_state = counts > maximum_threshold
 
@@ -944,6 +948,9 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions):
         Type: Global Rule
         Description: this validation rule checks, whether there are too many shipcalls with identical times to the query ETD.
         """
+        if all_times_agency is None:
+            all_times_agency = self.sql_handler.get_times_for_agency(non_null_column="etd_berth")
+
         # check, if the header is filled in (agency)
         if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): #if len(times_agency) != 1:
             return self.get_no_violation_default_output()