implementing POST-request input validation for shipcalls. Creating many tests and applying slight updates to the Notifier (not implemented yet)
This commit is contained in:
Max Metz
parent
d0753f0b32
commit
b7078f8d8e
@ -4,7 +4,7 @@ from marshmallow import Schema, fields, ValidationError
|
||||
from ..schemas import model
|
||||
from .. import impl
|
||||
from ..services.auth_guard import auth_guard, check_jwt
|
||||
from BreCal.validators.input_validation import check_if_user_is_bsmd_type, check_if_user_data_has_valid_ship_id, check_if_user_data_has_valid_berth_id, check_if_user_data_has_valid_participant_id
|
||||
from BreCal.validators.input_validation import validate_posted_shipcall_data
|
||||
|
||||
import logging
|
||||
import json
|
||||
@ -41,35 +41,15 @@ def PostShipcalls():
|
||||
try:
|
||||
content = request.get_json(force=True)
|
||||
loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True)
|
||||
logging.log(20, loadedModel)
|
||||
logging.log(20, "dev. above: loaded model, below: content")
|
||||
logging.log(20, content)
|
||||
|
||||
# read the user data from the JWT token (set when login is performed)
|
||||
user_data = check_jwt()
|
||||
|
||||
# check, whether the user belongs to a participant, which is of type ParticipantType.BSMD
|
||||
# as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated.
|
||||
is_bsmd = check_if_user_is_bsmd_type(user_data)
|
||||
if not is_bsmd:
|
||||
raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}")
|
||||
|
||||
import logging
|
||||
logging.log(20, loadedModel)
|
||||
logging.log(20, "metz development")
|
||||
"""
|
||||
# loadedModel ...
|
||||
loadedModel.get("ship_id", 0)
|
||||
|
||||
2024-04-22 18:21:03,982 | root | INFO | {'ship_id': 1,
|
||||
'type': 1, 'eta': datetime.datetime(2023, 7, 23, 7, 18, 19),
|
||||
'voyage': '43B', 'tug_required': False, 'pilot_required': True,
|
||||
'flags': 0, 'pier_side': False, 'bunkering': True, 'recommended_tugs': 2,
|
||||
'type_value': 1, 'evaluation_value': 0}
|
||||
|
||||
|
||||
valid_ship_id = check_if_user_data_has_valid_ship_id(ship_id)
|
||||
valid_berth_id = check_if_user_data_has_valid_berth_id(berth_id)
|
||||
valid_participant_id = check_if_user_data_has_valid_participant_id(participant_id)
|
||||
"""
|
||||
|
||||
# validate the posted shipcall data
|
||||
validate_posted_shipcall_data(user_data, loadedModel, content)
|
||||
|
||||
except ValidationError as ex:
|
||||
logging.error(ex)
|
||||
|
||||
@ -55,6 +55,7 @@ def update_all_shipcalls_in_mysql_database(sql_connection, sql_handler:SQLHandle
|
||||
sql_handler: an SQLHandler instance
|
||||
shipcall_df: dataframe, which stores the data that is used to retrieve the shipcall data models (that are then updated in the database)
|
||||
"""
|
||||
print(shipcall_df)
|
||||
for shipcall_id in shipcall_df.index:
|
||||
shipcall = sql_handler.df_loc_to_data_model(df=shipcall_df, id=shipcall_id, model_str="shipcall")
|
||||
update_shipcall_in_mysql_database(sql_connection, shipcall=shipcall, relevant_keys = ["evaluation", "evaluation_message"])
|
||||
|
||||
@ -110,6 +110,6 @@ class Notifier():
|
||||
|
||||
def get_notification_states(self, evaluation_states_old, evaluation_states_new)->list[bool]:
|
||||
"""# build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created"""
|
||||
evaluation_notifications_sent = [self.notifier.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)]
|
||||
evaluation_notifications_sent = [self.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)]
|
||||
return evaluation_notifications_sent
|
||||
|
||||
|
||||
@ -188,9 +188,9 @@ class ShipcallSchema(Schema):
|
||||
pass
|
||||
|
||||
id = fields.Integer()
|
||||
ship_id = fields.Integer()
|
||||
ship_id = fields.Integer(metadata={'required':True})
|
||||
#type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator
|
||||
type = fields.Integer() # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator
|
||||
type = fields.Integer(metadata={'required':True}) # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator
|
||||
eta = fields.DateTime(metadata={'required':False}, allow_none=True)
|
||||
voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False}
|
||||
etd = fields.DateTime(metadata={'required':False}, allow_none=True)
|
||||
@ -207,7 +207,7 @@ class ShipcallSchema(Schema):
|
||||
tidal_window_from = fields.DateTime(metadata={'required':False}, allow_none=True)
|
||||
tidal_window_to = fields.DateTime(metadata={'required':False}, allow_none=True)
|
||||
rain_sensitive_cargo = fields.Bool(metadata={'required':False}, allow_none=True)
|
||||
recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True)
|
||||
recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True, validate=[validate.Range(min=0, max=10, min_inclusive=True, max_inclusive=True)])
|
||||
anchored = fields.Bool(metadata={'required':False}, allow_none=True)
|
||||
moored_lock = fields.Bool(metadata={'required':False}, allow_none=True)
|
||||
canceled = fields.Bool(metadata={'required':False}, allow_none=True)
|
||||
@ -251,6 +251,9 @@ class Participant_Assignment:
|
||||
participant_id: int
|
||||
type: int # a variant would be to use the IntFlag type (with appropriate serialization)
|
||||
|
||||
def to_json(self):
|
||||
return self.__dict__
|
||||
|
||||
|
||||
@dataclass
|
||||
class Shipcall:
|
||||
|
||||
@ -85,4 +85,22 @@ def get_shipcall_simple():
|
||||
)
|
||||
return shipcall
|
||||
|
||||
def create_postman_stub_shipcall():
|
||||
"""
|
||||
this function returns the common stub, which is used to POST data to shipcalls via POSTMAN. However,
|
||||
the stub-function is updated with a dynamic ETA in the future, so the POST-request does not fail.
|
||||
"""
|
||||
shipcall = {
|
||||
'ship_id': 1,
|
||||
'type': 1,
|
||||
'eta': (datetime.datetime.now()+datetime.timedelta(hours=3)).isoformat(),
|
||||
'voyage': '43B',
|
||||
'tug_required': False,
|
||||
'pilot_required': True,
|
||||
'flags': 0,
|
||||
'pier_side': False,
|
||||
'bunkering': True,
|
||||
'recommended_tugs': 2
|
||||
}
|
||||
return shipcall
|
||||
|
||||
|
||||
@ -2,14 +2,31 @@
|
||||
####################################### InputValidation #######################################
|
||||
|
||||
import json
|
||||
import datetime
|
||||
from abc import ABC, abstractmethod
|
||||
from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant
|
||||
from marshmallow import ValidationError
|
||||
from string import ascii_letters, digits
|
||||
|
||||
from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant, ShipcallType
|
||||
from BreCal.impl.participant import GetParticipant
|
||||
from BreCal.impl.ships import GetShips
|
||||
from BreCal.impl.berths import GetBerths
|
||||
|
||||
from BreCal.database.enums import ParticipantType
|
||||
|
||||
def check_if_string_has_special_characters(text:str):
|
||||
"""
|
||||
check, whether there are any characters within the provided string, which are not found in the ascii letters or digits
|
||||
ascii_letters: abcd (...) and ABCD (...)
|
||||
digits: 0123 (...)
|
||||
|
||||
Source: https://stackoverflow.com/questions/57062794/is-there-a-way-to-check-if-a-string-contains-special-characters
|
||||
User: https://stackoverflow.com/users/10035985/andrej-kesely
|
||||
returns bool
|
||||
"""
|
||||
return bool(set(text).difference(ascii_letters + digits))
|
||||
|
||||
|
||||
def get_participant_id_dictionary():
|
||||
# get all participants
|
||||
response,status_code,header = GetParticipant(options={})
|
||||
@ -61,7 +78,11 @@ def check_if_user_is_bsmd_type(user_data:dict)->bool:
|
||||
is_bsmd = ParticipantType.BSMD in ParticipantType(participant.get("type",0))
|
||||
return is_bsmd
|
||||
|
||||
def check_if_user_data_has_valid_ship_id(ship_id):
|
||||
def check_if_ship_id_is_valid(ship_id):
|
||||
"""check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once"""
|
||||
if ship_id is None:
|
||||
return True
|
||||
|
||||
# build a dictionary of id:item pairs, so one can select the respective participant
|
||||
ships = get_ship_id_dictionary()
|
||||
|
||||
@ -69,7 +90,11 @@ def check_if_user_data_has_valid_ship_id(ship_id):
|
||||
ship_id_is_valid = ship_id in list(ships.keys())
|
||||
return ship_id_is_valid
|
||||
|
||||
def check_if_user_data_has_valid_berth_id(berth_id):
|
||||
def check_if_berth_id_is_valid(berth_id):
|
||||
"""check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once"""
|
||||
if berth_id is None:
|
||||
return True
|
||||
|
||||
# build a dictionary of id:item pairs, so one can select the respective participant
|
||||
berths = get_berth_id_dictionary()
|
||||
|
||||
@ -77,7 +102,11 @@ def check_if_user_data_has_valid_berth_id(berth_id):
|
||||
berth_id_is_valid = berth_id in list(berths.keys())
|
||||
return berth_id_is_valid
|
||||
|
||||
def check_if_user_data_has_valid_participant_id(participant_id):
|
||||
def check_if_participant_id_is_valid(participant_id):
|
||||
"""check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once"""
|
||||
if participant_id is None:
|
||||
return True
|
||||
|
||||
# build a dictionary of id:item pairs, so one can select the respective participant
|
||||
participants = get_participant_id_dictionary()
|
||||
|
||||
@ -85,6 +114,102 @@ def check_if_user_data_has_valid_participant_id(participant_id):
|
||||
participant_id_is_valid = participant_id in list(participants.keys())
|
||||
return participant_id_is_valid
|
||||
|
||||
def check_if_participant_ids_are_valid(participant_ids):
|
||||
# check each participant id individually
|
||||
valid_participant_ids = [check_if_participant_id_is_valid(participant_id) for participant_id in participant_ids]
|
||||
|
||||
# boolean check, whether all participant ids are valid
|
||||
return all(valid_participant_ids)
|
||||
|
||||
|
||||
def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict):
|
||||
"""this function applies more complex validation functions to data, which is sent to a post-request of shipcalls"""
|
||||
# #TODO_refactor: this function is pretty complex. One may instead build an object, which calls the methods separately.
|
||||
|
||||
import logging
|
||||
logging.log(20, "dev")
|
||||
logging.log(20, user_data)
|
||||
logging.log(20, loadedModel)
|
||||
logging.log(20, content)
|
||||
##### Section 1: check user_data #####
|
||||
# check, whether the user belongs to a participant, which is of type ParticipantType.BSMD
|
||||
# as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated.
|
||||
is_bsmd = check_if_user_is_bsmd_type(user_data)
|
||||
if not is_bsmd:
|
||||
raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}")
|
||||
|
||||
##### Section 2: check loadedModel #####
|
||||
valid_ship_id = check_if_ship_id_is_valid(ship_id=loadedModel.get("ship_id", None))
|
||||
if not valid_ship_id:
|
||||
raise ValidationError(f"provided an invalid ship id, which is not found in the database: {loadedModel.get('ship_id', None)}")
|
||||
|
||||
valid_arrival_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("arrival_berth_id", None))
|
||||
if not valid_arrival_berth_id:
|
||||
raise ValidationError(f"provided an invalid arrival berth id, which is not found in the database: {loadedModel.get('arrival_berth_id', None)}")
|
||||
|
||||
valid_departure_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("departure_berth_id", None))
|
||||
if not valid_departure_berth_id:
|
||||
raise ValidationError(f"provided an invalid departure berth id, which is not found in the database: {loadedModel.get('departure_berth_id', None)}")
|
||||
|
||||
valid_participant_ids = check_if_participant_ids_are_valid(participant_ids=loadedModel.get("participants",[]))
|
||||
if not valid_participant_ids:
|
||||
raise ValidationError(f"one of the provided participant ids is invalid. Could not find one of these in the database: {loadedModel.get('participants', None)}")
|
||||
|
||||
|
||||
##### Section 3: check content #####
|
||||
# loadedModel fills missing values, sometimes using optional values. Hence, check content
|
||||
|
||||
# the following keys should not be set in a POST-request.
|
||||
for forbidden_key in ["canceled", "evaluation", "evaluation_message"]:
|
||||
value = content.get(forbidden_key, None)
|
||||
if value is not None:
|
||||
raise ValidationError(f"'{forbidden_key}' may not be set on POST. Found: {value}")
|
||||
|
||||
voyage_str_is_invalid = check_if_string_has_special_characters(text=content.get("voyage",""))
|
||||
if voyage_str_is_invalid:
|
||||
raise ValidationError(f"there are invalid characters in the 'voyage'-string. Please use only digits and ASCII letters. Allowed: {ascii_letters+digits}. Found: {content.get('voyage')}")
|
||||
|
||||
|
||||
##### Section 4: check loadedModel & content #####
|
||||
# #TODO_refactor: these methods should be placed in separate locations
|
||||
|
||||
# existance checks in content
|
||||
# datetime checks in loadedModel (datetime.datetime objects). Dates should be in the future.
|
||||
time_now = datetime.datetime.now()
|
||||
type_ = loadedModel.get("type", int(ShipcallType.undefined))
|
||||
if int(type_)==int(ShipcallType.undefined):
|
||||
raise ValidationError(f"providing 'type' is mandatory. Missing key!")
|
||||
elif int(type_)==int(ShipcallType.arrival):
|
||||
eta = loadedModel.get("eta")
|
||||
if (content.get("eta", None) is None):
|
||||
raise ValidationError(f"providing 'eta' is mandatory. Missing key!")
|
||||
if content.get("arrival_berth_id", None) is None:
|
||||
raise ValidationError(f"providing 'arrival_berth_id' is mandatory. Missing key!")
|
||||
if not eta >= time_now:
|
||||
raise ValidationError(f"'eta' must be in the future. Incorrect datetime provided.")
|
||||
elif int(type_)==int(ShipcallType.departure):
|
||||
etd = loadedModel.get("etd")
|
||||
if (content.get("etd", None) is None):
|
||||
raise ValidationError(f"providing 'etd' is mandatory. Missing key!")
|
||||
if content.get("departure_berth_id", None) is None:
|
||||
raise ValidationError(f"providing 'departure_berth_id' is mandatory. Missing key!")
|
||||
if not etd >= time_now:
|
||||
raise ValidationError(f"'etd' must be in the future. Incorrect datetime provided.")
|
||||
elif int(type_)==int(ShipcallType.shifting):
|
||||
eta = loadedModel.get("eta")
|
||||
etd = loadedModel.get("etd")
|
||||
# * arrival_berth_id / departure_berth_id (depending on type, see above)
|
||||
if (content.get("eta", None) is None) or (content.get("etd", None) is None):
|
||||
raise ValidationError(f"providing 'eta' and 'etd' is mandatory. Missing one of those keys!")
|
||||
if (content.get("arrival_berth_id", None) is None) or (content.get("departure_berth_id", None) is None):
|
||||
raise ValidationError(f"providing 'arrival_berth_id' & 'departure_berth_id' is mandatory. Missing key!")
|
||||
if (not eta >= time_now) or (not etd >= time_now) or (not eta >= etd):
|
||||
raise ValidationError(f"'eta' and 'etd' must be in the future. Incorrect datetime provided.")
|
||||
|
||||
|
||||
# #TODO: len of participants > 0, if agency
|
||||
# * assigned participant for agency
|
||||
return
|
||||
|
||||
|
||||
class InputValidation():
|
||||
|
||||
@ -75,6 +75,7 @@ class ValidationRules(ValidationRuleFunctions):
|
||||
def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame:
|
||||
"""apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation', 'evaluation_message', 'evaluation_time' and 'evaluation_notifications_sent' are updated)"""
|
||||
evaluation_states_old = [state_old for state_old in shipcall_df.loc[:,"evaluation"]]
|
||||
evaluation_states_old = [state_old if not pd.isna(state_old) else 0 for state_old in evaluation_states_old]
|
||||
results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values # returns tuple (state, message)
|
||||
|
||||
# unbundle individual results. evaluation_states becomes an integer, violation
|
||||
@ -83,14 +84,14 @@ class ValidationRules(ValidationRuleFunctions):
|
||||
violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations]
|
||||
|
||||
# build the list of evaluation times ('now', as isoformat)
|
||||
evaluation_times = self.notifier.get_notification_times(evaluation_states_new)
|
||||
evaluation_time = self.notifier.get_notification_times(evaluation_states_new)
|
||||
|
||||
# build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created
|
||||
evaluation_notifications_sent = self.get_notification_states(evaluation_states_old, evaluation_states_new)
|
||||
evaluation_notifications_sent = self.notifier.get_notification_states(evaluation_states_old, evaluation_states_new)
|
||||
|
||||
shipcall_df.loc[:,"evaluation"] = evaluation_states_new
|
||||
shipcall_df.loc[:,"evaluation_message"] = violations
|
||||
shipcall_df.loc[:,"evaluation_times"] = evaluation_times
|
||||
shipcall_df.loc[:,"evaluation_time"] = evaluation_time
|
||||
shipcall_df.loc[:,"evaluation_notifications_sent"] = evaluation_notifications_sent
|
||||
return shipcall_df
|
||||
|
||||
|
||||
Reference in New Issue
Block a user