From c72575b27ada77437a4b49b3a175b89c9cb18740 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Fri, 19 Jan 2024 17:33:43 +0100 Subject: [PATCH 01/30] slight adjustments to prepare the authentification validation --- src/server/BreCal/api/shipcalls.py | 11 ++++++++++- src/server/BreCal/database/enums.py | 5 ++++- src/server/BreCal/local_db.py | 4 ++-- src/server/BreCal/schemas/model.py | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 396dc93..73a06c1 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -14,7 +14,16 @@ bp = Blueprint('shipcalls', __name__) @auth_guard() # no restriction by role def GetShipcalls(): if 'Authorization' in request.headers: - token = request.headers.get('Authorization') + token = request.headers.get('Authorization') # see impl/login to see the token encoding, which is a JWT token. + + """ + from BreCal.services.jwt_handler import decode_jwt + jwt = token.split('Bearer ')[1] # string key + payload = decode_jwt(jwt) # dictionary, which includes 'id' (user id) and 'participant_id' + + # oneline: + payload = decode_jwt(request.headers.get("Authorization").split("Bearer ")[-1]) + """ options = {} options["participant_id"] = request.args.get("participant_id") options["past_days"] = request.args.get("past_days", default=1, type=int) diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index 3092fd8..fe6b37f 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -1,4 +1,4 @@ -from enum import Enum, IntFlag +from enum import IntEnum, Enum, IntFlag class ParticipantType(IntFlag): """determines the type of a participant""" @@ -36,3 +36,6 @@ class StatusFlags(Enum): YELLOW = 2 RED = 3 +class PierSide(IntEnum): + PORTSIDE = 0 # Port/Backbord + STARBOARD_SIDE = 1 # Starboard / Steuerbord diff --git a/src/server/BreCal/local_db.py b/src/server/BreCal/local_db.py index 85dff68..4293ff3 100644 --- a/src/server/BreCal/local_db.py +++ b/src/server/BreCal/local_db.py @@ -16,7 +16,7 @@ def initPool(instancePath): print (config_path) if not os.path.exists(config_path): - print ('cannot find ' + config_path) + print ('cannot find ' + os.path.abspath(config_path)) print("instance path", instancePath) sys.exit(1) @@ -39,4 +39,4 @@ def getPoolConnection(): global config_path f = open(config_path); connection_data = json.load(f) - return mysql.connector.connect(**connection_data) \ No newline at end of file + return mysql.connector.connect(**connection_data) diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index bf157aa..f45bc11 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -230,7 +230,7 @@ class Shipcall: tug_required: bool pilot_required: bool flags: int - pier_side: bool + pier_side: bool # enumerator object in database/enum/PierSide bunkering: bool replenishing_terminal: bool replenishing_lock: bool From afbc56b4edbd59609111a4b03a59c083fe96d40a Mon Sep 17 00:00:00 2001 From: scopesorting Date: Thu, 7 Dec 2023 12:01:41 +0100 Subject: [PATCH 02/30] adapting shipcall, times and user to include ValidationError (marshmallow). Adjusting the Schemas for User, Times and Shipcall to be validated with additional input validators. Creating a set of tests for the input validations. --- brecal.code-workspace | 8 +++ src/server/BreCal/api/shipcalls.py | 14 ++++- src/server/BreCal/api/times.py | 11 ++++ src/server/BreCal/api/user.py | 6 ++ src/server/BreCal/schemas/model.py | 73 ++++++++++++++--------- src/server/tests/schemas/test_model.py | 80 ++++++++++++++++++++++++++ 6 files changed, 165 insertions(+), 27 deletions(-) create mode 100644 brecal.code-workspace create mode 100644 src/server/tests/schemas/test_model.py diff --git a/brecal.code-workspace b/brecal.code-workspace new file mode 100644 index 0000000..876a149 --- /dev/null +++ b/brecal.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 73a06c1..fbf552a 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -1,6 +1,6 @@ from flask import Blueprint, request from webargs.flaskparser import parser -from marshmallow import Schema, fields +from marshmallow import Schema, fields, ValidationError from ..schemas import model from .. import impl from ..services.auth_guard import auth_guard @@ -40,6 +40,12 @@ def PostShipcalls(): try: content = request.get_json(force=True) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 + except Exception as ex: logging.error(ex) print(ex) @@ -56,6 +62,12 @@ def PutShipcalls(): content = request.get_json(force=True) logging.info(content) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 + except Exception as ex: logging.error(ex) print(ex) diff --git a/src/server/BreCal/api/times.py b/src/server/BreCal/api/times.py index 2c90397..a333064 100644 --- a/src/server/BreCal/api/times.py +++ b/src/server/BreCal/api/times.py @@ -4,6 +4,7 @@ from .. import impl from ..services.auth_guard import auth_guard import json import logging +from marshmallow import ValidationError bp = Blueprint('times', __name__) @@ -29,6 +30,11 @@ def PostTimes(): # print (content) # body = parser.parse(schema, request, location='json') loadedModel = model.TimesSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 except Exception as ex: logging.error(ex) @@ -45,6 +51,11 @@ def PutTimes(): try: content = request.get_json(force=True) loadedModel = model.TimesSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 except Exception as ex: logging.error(ex) diff --git a/src/server/BreCal/api/user.py b/src/server/BreCal/api/user.py index bbd5b4b..2c3c1a0 100644 --- a/src/server/BreCal/api/user.py +++ b/src/server/BreCal/api/user.py @@ -4,6 +4,7 @@ from .. import impl from ..services.auth_guard import auth_guard import json import logging +from marshmallow import ValidationError bp = Blueprint('user', __name__) @@ -14,6 +15,11 @@ def PutUser(): try: content = request.get_json(force=True) loadedModel = model.UserSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 except Exception as ex: logging.error(ex) diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index f45bc11..f6957cc 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -1,5 +1,5 @@ from dataclasses import field, dataclass -from marshmallow import Schema, fields, post_load, INCLUDE, ValidationError +from marshmallow import Schema, fields, INCLUDE, ValidationError, validate, validates from marshmallow.fields import Field from marshmallow_enum import EnumField from enum import IntEnum @@ -9,6 +9,7 @@ from typing import List import json import datetime +from BreCal.validators.time_logic import validate_time_exceeds_threshold def obj_dict(obj): if isinstance(obj, datetime.datetime): @@ -152,34 +153,34 @@ class ParticipantList(Participant): pass class ParticipantAssignmentSchema(Schema): - participant_id = fields.Int() - type = fields.Int() + participant_id = fields.Integer() + type = fields.Integer() class ShipcallSchema(Schema): def __init__(self): super().__init__(unknown=None) pass - id = fields.Int() - ship_id = fields.Int() - type = fields.Enum(ShipcallType, required=True) + id = fields.Integer() + ship_id = fields.Integer() + type = fields.Integer() eta = fields.DateTime(Required = False, allow_none=True) - voyage = fields.Str(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} + voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} etd = fields.DateTime(Required = False, allow_none=True) - arrival_berth_id = fields.Int(Required = False, allow_none=True) - departure_berth_id = fields.Int(Required = False, allow_none=True) + arrival_berth_id = fields.Integer(Required = False, allow_none=True) + departure_berth_id = fields.Integer(Required = False, allow_none=True) tug_required = fields.Bool(Required = False, allow_none=True) pilot_required = fields.Bool(Required = False, allow_none=True) - flags = fields.Int(Required = False, allow_none=True) + flags = fields.Integer(Required = False, allow_none=True) pier_side = fields.Bool(Required = False, allow_none=True) bunkering = fields.Bool(Required = False, allow_none=True) replenishing_terminal = fields.Bool(Required = False, allow_none=True) replenishing_lock = fields.Bool(Required = False, allow_none=True) - draft = fields.Float(Required = False, allow_none=True) + draft = fields.Float(Required = False, allow_none=True, validate=[validate.Range(min=0, max=20, min_inclusive=False, max_inclusive=True)]) tidal_window_from = fields.DateTime(Required = False, allow_none=True) tidal_window_to = fields.DateTime(Required = False, allow_none=True) rain_sensitive_cargo = fields.Bool(Required = False, allow_none=True) - recommended_tugs = fields.Int(Required = False, allow_none=True) + recommended_tugs = fields.Integer(Required = False, allow_none=True) anchored = fields.Bool(Required = False, allow_none=True) moored_lock = fields.Bool(Required = False, allow_none=True) canceled = fields.Bool(Required = False, allow_none=True) @@ -297,12 +298,13 @@ class ShipcallId(Schema): # this is the way! + class TimesSchema(Schema): def __init__(self): super().__init__(unknown=None) pass - id = fields.Int(Required=False) + id = fields.Integer(Required=False) eta_berth = fields.DateTime(Required = False, allow_none=True) eta_berth_fixed = fields.Bool(Required = False, allow_none=True) etd_berth = fields.DateTime(Required = False, allow_none=True) @@ -313,13 +315,13 @@ class TimesSchema(Schema): zone_entry_fixed = fields.Bool(Required = False, allow_none=True) operations_start = fields.DateTime(Required = False, allow_none=True) operations_end = fields.DateTime(Required = False, allow_none=True) - remarks = fields.String(Required = False, allow_none=True) - participant_id = fields.Int(Required = True) - berth_id = fields.Int(Required = False, allow_none = True) - berth_info = fields.String(Required = False, allow_none=True) + remarks = fields.String(Required = False, allow_none=True, validate=[validate.Length(max=256)]) + participant_id = fields.Integer(Required = True) + berth_id = fields.Integer(Required = False, allow_none = True) + berth_info = fields.String(Required = False, allow_none=True, validate=[validate.Length(max=256)]) pier_side = fields.Bool(Required = False, allow_none = True) - shipcall_id = fields.Int(Required = True) - participant_type = fields.Int(Required = False, allow_none=True) + shipcall_id = fields.Integer(Required = True) + participant_type = fields.Integer(Required = False, allow_none=True) ata = fields.DateTime(Required = False, allow_none=True) atd = fields.DateTime(Required = False, allow_none=True) eta_interval_end = fields.DateTime(Required = False, allow_none=True) @@ -327,19 +329,38 @@ class TimesSchema(Schema): created = fields.DateTime(Required = False, allow_none=True) modified = fields.DateTime(Required = False, allow_none=True) + @validates("eta_berth") + def validate_eta_berth(self, value): + threshold_exceeded = validate_time_exceeds_threshold(value, months=12) + print(threshold_exceeded, value) + if threshold_exceeded: + raise ValidationError(f"the provided time exceeds the twelve month threshold.") + # deserialize PUT object target class UserSchema(Schema): def __init__(self): super().__init__(unknown=None) pass - id = fields.Int(required=True) - first_name = fields.Str(allow_none=True, metadata={'Required':False}) - last_name = fields.Str(allow_none=True, metadata={'Required':False}) - user_phone = fields.Str(allow_none=True, metadata={'Required':False}) - user_email = fields.Str(allow_none=True, metadata={'Required':False}) - old_password = fields.Str(allow_none=True, metadata={'Required':False}) - new_password = fields.Str(allow_none=True, metadata={'Required':False}) + id = fields.Integer(required=True) + first_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) + last_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) + user_phone = fields.String(allow_none=True, metadata={'Required':False}) + user_email = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) + old_password = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=128)]) + new_password = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(min=6, max=128)]) + + @validates("user_phone") + def validate_user_phone(self, value): + valid_characters = list(map(str,range(0,10)))+["+", " "] + if not all([v in valid_characters for v in value]): + raise ValidationError(f"one of the phone number values is not valid.") + + @validates("user_email") + def validate_user_email(self, value): + if not "@" in value: + raise ValidationError(f"invalid email address") + @dataclass class Times: diff --git a/src/server/tests/schemas/test_model.py b/src/server/tests/schemas/test_model.py new file mode 100644 index 0000000..8944f45 --- /dev/null +++ b/src/server/tests/schemas/test_model.py @@ -0,0 +1,80 @@ +from marshmallow import ValidationError +import pytest +from BreCal.schemas.model import ShipcallSchema + + +@pytest.fixture(scope="function") # function: destroy fixture at the end of each test +def prepare_shipcall_content(): + import datetime + from BreCal.stubs.shipcall import get_shipcall_simple + shipcall_stub = get_shipcall_simple() + content = shipcall_stub.__dict__ + content["participants"] = [] + content = {k:v.isoformat() if isinstance(v, datetime.datetime) else v for k,v in content.items()} + return locals() + +def test_shipcall_input_validation_draft(prepare_shipcall_content): + content = prepare_shipcall_content["content"] + content["draft"] = 24.11 + + schemaModel = ShipcallSchema() + with pytest.raises(ValidationError, match="Must be greater than 0 and less than or equal to 20."): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + +def test_shipcall_input_validation_voyage(prepare_shipcall_content): + content = prepare_shipcall_content["content"] + content["voyage"] = "".join(list(map(str,list(range(0,24))))) # 38 characters + + schemaModel = ShipcallSchema() + with pytest.raises(ValidationError, match="Longer than maximum length "): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + + +@pytest.fixture(scope="function") # function: destroy fixture at the end of each test +def prepare_user_content(): + import datetime + from BreCal.stubs.user import get_user_simple + from BreCal.schemas.model import UserSchema + schemaModel = UserSchema() + + user_stub = get_user_simple() + content = user_stub.__dict__ + content = {k:v.isoformat() if isinstance(v, datetime.datetime) else v for k,v in content.items()} + content = {k:v for k,v in content.items() if k in list(schemaModel.fields.keys())} + content["old_password"] = "myfavoritedog123" + content["new_password"] = "SecuRepassW0rd!" + return locals() + + +def test_input_validation_berth_phone_number_is_valid(prepare_user_content): + content, schemaModel = prepare_user_content["content"], prepare_user_content["schemaModel"] + content["user_phone"] = "+49123 45678912" # whitespace and + are valid + + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + +def test_input_validation_berth_phone_number_is_invalid(prepare_user_content): + content, schemaModel = prepare_user_content["content"], prepare_user_content["schemaModel"] + content["user_phone"] = "+49123 45678912!" # ! is invalid + + with pytest.raises(ValidationError, match="one of the phone number values is not valid."): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + +def test_input_validation_new_password_too_short(prepare_user_content): + content, schemaModel = prepare_user_content["content"], prepare_user_content["schemaModel"] + content["new_password"] = "1234" # must have between 6 and 128 characters + + with pytest.raises(ValidationError, match="Length must be between 6 and 128."): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + +def test_input_validation_user_email_invalid(prepare_user_content): + content, schemaModel = prepare_user_content["content"], prepare_user_content["schemaModel"] + content["user_email"] = "userbrecal.com" # forgot @ -> invalid + + with pytest.raises(ValidationError, match="invalid email address"): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return From 49d12b96c8ec0dfd6902463633b5282460887beb Mon Sep 17 00:00:00 2001 From: scopesorting Date: Tue, 12 Dec 2023 17:07:09 +0100 Subject: [PATCH 03/30] partial commit of integrating the input validation (references and mandatory fields) --- src/server/BreCal/__init__.py | 1 + src/server/BreCal/database/update_database.py | 1 - src/server/BreCal/impl/shipcalls.py | 22 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/server/BreCal/__init__.py b/src/server/BreCal/__init__.py index eb832f5..1f37ffc 100644 --- a/src/server/BreCal/__init__.py +++ b/src/server/BreCal/__init__.py @@ -46,6 +46,7 @@ def create_app(test_config=None): app.config.from_mapping(test_config) try: + import os print(f'Instance path = {app.instance_path}') os.makedirs(app.instance_path) except OSError: diff --git a/src/server/BreCal/database/update_database.py b/src/server/BreCal/database/update_database.py index 0e5e5ac..7b7639b 100644 --- a/src/server/BreCal/database/update_database.py +++ b/src/server/BreCal/database/update_database.py @@ -68,7 +68,6 @@ def evaluate_shipcall_state(mysql_connector_instance, shipcall_id:int=None, debu with mysql.connector.connect(**mysql_connection_data) as mysql_connector_instance: evaluate_shipcall_state(mysql_connector_instance) returns None - """ sql_handler = SQLHandler(sql_connection=mysql_connector_instance, read_all=True) vr = ValidationRules(sql_handler) diff --git a/src/server/BreCal/impl/shipcalls.py b/src/server/BreCal/impl/shipcalls.py index 59f9311..8d15983 100644 --- a/src/server/BreCal/impl/shipcalls.py +++ b/src/server/BreCal/impl/shipcalls.py @@ -123,6 +123,17 @@ def PostShipcalls(schemaModel): query += "?" + param_key + "?" query += ")" + # #TODO: enter completeness validation here. Only shipcalls, where all required fields are filled in are valid + # "Pflichtfelder: z.B. die Felder bei der Neuanlage eines shipcalls müssen entsprechend des Anlauf-Typs belegt sein, damit gespeichert werden kann" - Daniel Schick + + # #TODO: enter role validation here. Only users of correct type should be allowed to post a shipcall + + # #TODO: enter reference validation here + # full_id_existances = check_id_existances(mysql_connector_instance=sql_connection, schemaModel=schemaModel, debug=False) + # if not full_id_existances: + # pooledConnection.close() + # return json.dumps({"message" : "call failed. missing mandatory keywords."}), 500, {'Content-Type': 'application/json; charset=utf-8'} + commands.execute(query, schemaModel) new_id = commands.execute_scalar("select last_insert_id()") @@ -206,6 +217,17 @@ def PutShipcalls(schemaModel): query += key + " = ?" + param_key + "? " query += "WHERE id = ?id?" + + # #TODO: enter role validation here. + + # #TODO: enter completeness validation here. Only shipcalls, where all required fields are filled in are valid + + + # #TODO: enter reference validation here + # full_id_existances = check_id_existances(mysql_connector_instance=sql_connection, schemaModel=schemaModel, debug=False) + # if not full_id_existances: + # return ... (500) + affected_rows = commands.execute(query, param=schemaModel) pquery = "SELECT id, participant_id, type FROM shipcall_participant_map where shipcall_id = ?id?" From 2cfda62932d5f5111ffa5510c4a1f34d585fa3f8 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Fri, 15 Dec 2023 17:37:27 +0100 Subject: [PATCH 04/30] implementing an EmailHandler to send out emails (potentially with attachment). --- src/server/BreCal/services/email_handling.py | 174 +++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 src/server/BreCal/services/email_handling.py diff --git a/src/server/BreCal/services/email_handling.py b/src/server/BreCal/services/email_handling.py new file mode 100644 index 0000000..e06021d --- /dev/null +++ b/src/server/BreCal/services/email_handling.py @@ -0,0 +1,174 @@ +import os +import typing +import smtplib +from getpass import getpass +from email.message import EmailMessage +import mimetypes + +import email +# from email.mime.base import MIMEBase +# from email.mime.image import MIMEImage +# from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +from email.mime.application import MIMEApplication + + +class EmailHandler(): + """ + Creates an EmailHandler, which is capable of connecting to a mail server at a respective port, + as well as logging into a specific user's mail address. + Upon creating messages, these can be sent via this handler. + + Options: + mail_server: address of the server, such as 'smtp.gmail.com' or 'w01d5503.kasserver.com + mail_port: + 25 - SMTP Port, to send emails + 110 - POP3 Port, to receive emails + 143 - IMAP Port, to receive from IMAP + 465 - SSL Port of SMTP + 587 - alternative SMTP Port + 993 - SSL/TLS-Port of IMAP + 995 - SSL/TLS-Port of POP3 + mail_address: a specific user's Email address, which will be used to send Emails. Example: "my_user@gmail.com" + """ + def __init__(self, mail_server:str, mail_port:int, mail_address:str): + self.mail_server = mail_server + self.mail_port = mail_port + self.mail_address = mail_address + + self.server = smtplib.SMTP_SSL(self.mail_server, self.mail_port) # alternatively, SMTP + + def check_state(self): + """check, whether the server login took place and is open.""" + try: + (status_code, status_msg) = self.server.noop() + return status_code==250 # 250: b'2.0.0 Ok' + except smtplib.SMTPServerDisconnected: + return False + + def check_connection(self): + """check, whether the server object is connected to the server. If not, connect it. """ + try: + self.server.ehlo() + except smtplib.SMTPServerDisconnected: + self.server.connect(self.mail_server, self.mail_port) + return + + def check_login(self)->bool: + """check, whether the server object is logged in as a user""" + user = self.server.__dict__.get("user",None) + return user is not None + + def login(self, interactive:bool=True): + """ + login on the determined mail server's mail address. By default, this function opens an interactive window to + type the password without echoing (printing '*******' instead of readable characters). + + returns (status_code, status_msg) + """ + self.check_connection() + if interactive: + (status_code, status_msg) = self.server.login(self.mail_address, password=getpass()) + else: + # fernet + password file + raise NotImplementedError() + return (status_code, status_msg) # should be: (235, b'2.7.0 Authentication successful') + + def create_email(self, subject:str, message_body:str)->EmailMessage: + """ + Create an EmailMessage object, which contains the Email's header ("Subject"), content ("Message Body") and the sender's address ("From"). + The EmailMessage object does not contain the recipients yet, as these will be defined upon sending the Email. + """ + msg = EmailMessage() + msg["Subject"] = subject + msg["From"] = self.mail_address + #msg["To"] = email_tgts # will be defined in self.send_email + msg.set_content(message_body) + return msg + + def build_recipients(self, email_tgts:list[str]): + """ + email formatting does not support lists. Instead, items are joined into a comma-space-separated string. + Example: + [mail1@mail.com, mail2@mail.com] becomes + 'mail1@mail.com, mail2@mail.com' + """ + return ', '.join(email_tgts) + + def open_mime_application(self, path:str)->MIMEApplication: + """open a local file, read the bytes into a MIMEApplication object, which is built with the proper subtype (based on the file extension)""" + with open(path, 'rb') as file: + attachment = MIMEApplication(file.read(), _subtype=mimetypes.MimeTypes().guess_type(path)) + + attachment.add_header('Content-Disposition','attachment',filename=str(os.path.basename(path))) + return attachment + + def attach_file(self, path:str, msg:email.mime.multipart.MIMEMultipart)->None: + """ + attach a file to the message. This function opens the file, reads its bytes, defines the mime type by the + path extension. The filename is appended as the header. + + mimetypes: # https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types + """ + attachment = self.open_mime_application(path) + msg.attach(attachment) + return + + def send_email(self, msg:EmailMessage, email_tgts:list[str], cc_tgts:typing.Optional[list[str]]=None, bcc_tgts:typing.Optional[list[str]]=None, debug:bool=False)->typing.Union[dict,EmailMessage]: + """ + send a prepared email message to recipients (email_tgts), copy (cc_tgts) and blind copy (bcc_tgts). + Returns a dictionary of feedback, which is commonly empty and the EmailMessage. + + When failing, this function returns an SMTP error instead of returning the default outputs. + """ + # Set the Recipients + msg["To"] = self.build_recipients(email_tgts) + + # optionally, add CC and BCC (copy and blind-copy) + if cc_tgts is not None: + msg["Cc"] = self.build_recipients(cc_tgts) + if bcc_tgts is not None: + msg["Bcc"] = self.build_recipients(bcc_tgts) + + # when debugging, do not send the Email, but return the EmailMessage. + if debug: + return {}, msg + + assert self.check_login(), f"currently not logged in. Cannot send an Email. Make sure to properly use self.login first. " + # send the prepared EmailMessage via the server. + feedback = self.server.send_message(msg) + return feedback, msg + + def translate_mail_to_multipart(self, msg:EmailMessage): + """EmailMessage does not support HTML and attachments. Hence, one can convert an EmailMessage object.""" + if msg.is_multipart(): + return msg + + # create a novel MIMEMultipart email + msg_new = MIMEMultipart("mixed") + headers = list((k, v) for (k, v) in msg.items() if k not in ("Content-Type", "Content-Transfer-Encoding")) + + # add the headers of msg to the new message + for k,v in headers: + msg_new[k] = v + + # delete the headers from msg + for k,v in headers: + del msg[k] + + # attach the remainder of the msg, such as the body, to the MIMEMultipart + msg_new.attach(msg) + return msg_new + + def print_email_attachments(self, msg:MIMEMultipart)->list[str]: + """return a list of lines of an Email, which contain 'filename=' as a list. """ + return [line_ for line_ in msg.as_string().split("\n") if "filename=" in line_] + + def close(self): + self.server.__dict__.pop("user",None) + self.server.__dict__.pop("password",None) + + # quit the server connection (internally uses .close) + self.server.quit() + return + From 7d3e8b769341893ac21947dc4f03eff7fddc27bc Mon Sep 17 00:00:00 2001 From: scopesorting Date: Fri, 19 Jan 2024 14:22:54 +0100 Subject: [PATCH 05/30] implementing notifications, working on input validation --- src/server/BreCal/database/enums.py | 9 ++ src/server/BreCal/notifications/__init__.py | 0 .../notifications/notification_functions.py | 115 ++++++++++++++++++ src/server/BreCal/schemas/model.py | 9 ++ .../BreCal/services/schedule_routines.py | 5 + src/server/BreCal/stubs/shipcall.py | 4 +- .../BreCal/validators/validation_rules.py | 96 +++++++-------- 7 files changed, 181 insertions(+), 57 deletions(-) create mode 100644 src/server/BreCal/notifications/__init__.py create mode 100644 src/server/BreCal/notifications/notification_functions.py diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index fe6b37f..7f8fc3d 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -26,6 +26,8 @@ class ParticipantwiseTimeDelta(): TUG = 960.0 # 16 h * 60 min/h = 960 min TERMINAL = 960.0 # 16 h * 60 min/h = 960 min + NOTIFICATION = 10.0 # after n minutes, an evaluation may rise a notification + class StatusFlags(Enum): """ these enumerators ensure that each traffic light validation rule state corresponds to a value, which will be used in the ValidationRules object to identify @@ -39,3 +41,10 @@ class StatusFlags(Enum): class PierSide(IntEnum): PORTSIDE = 0 # Port/Backbord STARBOARD_SIDE = 1 # Starboard / Steuerbord + +class NotificationType(IntFlag): + """determines the method by which a notification is distributed to users. Flagging allows selecting multiple notification types.""" + UNDEFINED = 0 + EMAIL = 1 + POPUP = 2 + MESSENGER = 4 diff --git a/src/server/BreCal/notifications/__init__.py b/src/server/BreCal/notifications/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server/BreCal/notifications/notification_functions.py b/src/server/BreCal/notifications/notification_functions.py new file mode 100644 index 0000000..9bbc373 --- /dev/null +++ b/src/server/BreCal/notifications/notification_functions.py @@ -0,0 +1,115 @@ +import datetime +import pandas as pd +from BreCal.schemas.model import Notification +from BreCal.database.enums import NotificationType, ParticipantType, ShipcallType, StatusFlags + +def create_notification(id, times_id, message, level, notification_type:NotificationType, created=None, modified=None): + created = (datetime.datetime.now()).isoformat() or created + + notification = Notification( + id=id, + times_id=times_id, acknowledged=False, level=level, type=notification_type.value, message=message, created=created, modified=modified + ) + return notification + + + + + +#### Verbosity Functions #### + +def get_default_header()->str: + # HEADER (greeting and default message) + header = "Dear Sir or Madam\n\nThank you for participating in the project 'Bremen Calling'. During analysis, our software has identified an event, which may be worth a second look. Here is the summary. \n\n" + return header + +def get_default_footer()->str: + # FOOTER (signature) + footer = "\n\nWe would kindly ask you to have a look at the shipcall and verify, if any action is required from your side. \n\nKind regards\nThe 'Bremen Calling' Team" + return footer + +def get_agency_name(sql_handler, times_df): + times_agency = times_df.loc[times_df["participant_type"]==ParticipantType.AGENCY.value,"participant_id"] + if len(times_agency)==0: + agency_name = "" + else: + agency_participant_id = times_agency.iloc[0] + agency_name = sql_handler.df_dict.get("participant").loc[agency_participant_id,"name"] + return agency_name + +def get_ship_name(sql_handler, shipcall): + ship = sql_handler.df_dict.get("ship").loc[shipcall.ship_id] + ship_name = ship.loc["name"] # when calling ship.name, the ID is returned (pandas syntax) + return ship_name + + +def create_notification_body(sql_handler, times_df, shipcall, result)->str: + # #TODO: add 'Link zum Anlauf' + # URL: https://trello.com/c/qenZyJxR/75-als-bsmd-m%C3%B6chte-ich-%C3%BCber-gelbe-und-rote-ampeln-informiert-werden-um-die-systembeteiligung-zu-st%C3%A4rken + header = get_default_header() + footer = get_default_footer() + + agency_name = get_agency_name(sql_handler, times_df) + ship_name = get_ship_name(sql_handler, shipcall) + + verbosity_introduction = f"Respective Shipcall:\n" + traffic_state_verbosity = f"\tTraffic Light State: {StatusFlags(result[0]).name}\n" + ship_name_verbosity = f"\tShip: {ship_name} (the ship is {ShipcallType(shipcall.type).name.lower()})\n" + agency_name_verbosity = f"\tResponsible Agency: {agency_name}\n" + eta_verbosity = f"\tEstimated Arrival Time: {shipcall.eta.isoformat()}\n" if not pd.isna(shipcall.eta) else "" + etd_verbosity = f"\tEstimated Departure Time: {shipcall.etd.isoformat()}\n" if not pd.isna(shipcall.etd) else "" + error_verbosity = f"\nError Description:\n\t" + "\n\t".join(result[1]) + + message_body = "".join([header, verbosity_introduction, traffic_state_verbosity, ship_name_verbosity, agency_name_verbosity, eta_verbosity, etd_verbosity, error_verbosity, footer]) + return message_body + + +class Notifier(): + """An object that helps with the logic of selecting eligible shipcalls to create the correct notifications for the respective users.""" + def __init__(self)->None: + pass + + def determine_notification_state(self, state_old, state_new): + """ + this method determines state changes in the notification state. When the state increases, a user is notified about it. + state order: (NONE = GREEN < YELLOW < RED) + """ + # identify a state increase + should_notify = self.identify_notification_state_change(state_old=state_old, state_new=state_new) + + # when a state increases, a notification must be sent. Thereby, the field should be set to False ({evaluation_notifications_sent}) + evaluation_notifications_sent = False if bool(should_notify) else None + return evaluation_notifications_sent + + def identify_notification_state_change(self, state_old, state_new) -> bool: + """ + determines, whether the observed state change should trigger a notification. + internally, this function maps StatusFlags to an integer and determines, if the successor state is more severe than the predecessor. + + state changes trigger a notification in the following cases: + green -> yellow + green -> red + yellow -> red + + (none -> yellow) or (none -> red) + due to the values in the enumeration objects, the states are mapped to provide this function. + green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than". + + returns bool, whether a notification should be triggered + """ + # state_old is always considered at least 'Green' (1) + if state_old is None: + state_old = StatusFlags.NONE.value + state_old = max(int(state_old), StatusFlags.GREEN.value) + return int(state_new) > int(state_old) + + def get_notification_times(self, evaluation_states_new)->list[datetime.datetime]: + """# build the list of evaluation times ('now', as isoformat)""" + evaluation_times = [datetime.datetime.now().isoformat() for _i in range(len(evaluation_states_new))] + return evaluation_times + + def get_notification_states(self, evaluation_states_old, evaluation_states_new)->list[bool]: + """# build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created""" + evaluation_notifications_sent = [self.notifier.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)] + return evaluation_notifications_sent + diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index f6957cc..c429777 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -184,11 +184,18 @@ class ShipcallSchema(Schema): anchored = fields.Bool(Required = False, allow_none=True) moored_lock = fields.Bool(Required = False, allow_none=True) canceled = fields.Bool(Required = False, allow_none=True) +<<<<<<< HEAD evaluation = fields.Enum(EvaluationType, required=False, allow_none=True, default=EvaluationType.undefined) evaluation_message = fields.Str(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} evaluation_time = fields.DateTime(Required = False, allow_none=True) evaluation_notifications_sent = fields.Bool(Required = False, allow_none=True) time_ref_point = fields.Int(Required = False, allow_none=True) +======= + evaluation = fields.Integer(Required = False, allow_none=True) + evaluation_message = fields.String(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} + evaluation_time = fields.DateTime(Required = False, allow_none=True) + evaluation_notifications_sent = fields.Bool(Required = False, allow_none=True) +>>>>>>> a5284a4 (implementing notifications, working on input validation) participants = fields.List(fields.Nested(ParticipantAssignmentSchema)) created = fields.DateTime(Required = False, allow_none=True) modified = fields.DateTime(Required = False, allow_none=True) @@ -251,6 +258,8 @@ class Shipcall: created: datetime modified: datetime participants: List[Participant_Assignment] = field(default_factory=list) + evaluation_time : datetime = None + evaluation_notifications_sent : bool = None def to_json(self): return { diff --git a/src/server/BreCal/services/schedule_routines.py b/src/server/BreCal/services/schedule_routines.py index f83b631..308cf7b 100644 --- a/src/server/BreCal/services/schedule_routines.py +++ b/src/server/BreCal/services/schedule_routines.py @@ -53,6 +53,11 @@ def add_function_to_schedule__update_shipcalls(interval_in_minutes:int, options: schedule.every(interval_in_minutes).minutes.do(UpdateShipcalls, **kwargs_) return +def add_function_to_schedule__send_notifications(vr, interval_in_minutes:int=10): + schedule.every(interval_in_minutes).minutes.do(vr.notifier.send_notifications) + return + + def setup_schedule(update_shipcalls_interval_in_minutes:int=60): schedule.clear() # clear all routine jobs. This prevents jobs from being created multiple times diff --git a/src/server/BreCal/stubs/shipcall.py b/src/server/BreCal/stubs/shipcall.py index e86d379..2e4e154 100644 --- a/src/server/BreCal/stubs/shipcall.py +++ b/src/server/BreCal/stubs/shipcall.py @@ -37,12 +37,12 @@ def get_shipcall_simple(): recommended_tugs = 2 # assert 0pd.DataFrame: - """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation' and 'evaluation_message' are updated)""" - results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values + """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation', 'evaluation_message', 'evaluation_time' and 'evaluation_notifications_sent' are updated)""" + evaluation_states_old = [state_old for state_old in shipcall_df.loc[:,"evaluation"]] + results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values # returns tuple (state, message) - # unbundle individual results. evaluation_state becomes an integer, violation - evaluation_state = [StatusFlags(res[0]).value for res in results] + # unbundle individual results. evaluation_states becomes an integer, violation + evaluation_states_new = [StatusFlags(res[0]).value for res in results] violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results] violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] - shipcall_df.loc[:,"evaluation"] = evaluation_state + # build the list of evaluation times ('now', as isoformat) + evaluation_times = self.notifier.get_notification_times(evaluation_states_new) + + # build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created + evaluation_notifications_sent = self.get_notification_states(evaluation_states_old, evaluation_states_new) + + shipcall_df.loc[:,"evaluation"] = evaluation_states_new shipcall_df.loc[:,"evaluation_message"] = violations + shipcall_df.loc[:,"evaluation_times"] = evaluation_times + shipcall_df.loc[:,"evaluation_notifications_sent"] = evaluation_notifications_sent return shipcall_df def concise_evaluation_message_if_too_long(self, violation): @@ -97,53 +105,31 @@ class ValidationRules(ValidationRuleFunctions): # e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K'] violation = f"Evaluation message too long. Violated Rules: {concise}" return violation - - def determine_validation_state(self) -> str: - """ - this method determines the validation state of a shipcall. The state is either ['green', 'yellow', 'red'] and signals, - whether an entry causes issues within the workflow of users. - - returns: validation_state_new (str) - """ - (validation_state_new, description) = self.undefined_method() - # should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification. - self.validation_state = validation_state_new - return validation_state_new - - def determine_notification_state(self) -> (str, bool): - """ - this method determines state changes in the notification state. When the state is changed to yellow or red, - a user is notified about it. The only exception for this rule is when the state was yellow or red before, - as the user has then already been notified. - - returns: notification_state_new (str), should_notify (bool) - """ - (state_new, description) = self.undefined_method() # determine the successor - should_notify = self.identify_notification_state_change(state_new) - self.notification_state = state_new # overwrite the predecessor - return state_new, should_notify - - def identify_notification_state_change(self, state_new) -> bool: - """ - determines, whether the observed state change should trigger a notification. - internally, this function maps a color string to an integer and determines, if the successor state is more severe than the predecessor. - - state changes trigger a notification in the following cases: - green -> yellow - green -> red - yellow -> red - - (none -> yellow) or (none -> red) - due to the values in the enumeration objects, the states are mapped to provide this function. - green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than". - - returns bool, whether a notification should be triggered - """ - # state_old is always considered at least 'Green' (1) - state_old = max(copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else StatusFlags.NONE, StatusFlags.GREEN.value) - return state_new.value > state_old.value def undefined_method(self) -> str: """this function should apply the ValidationRules to the respective .shipcall, in regards to .times""" - # #TODO_traffic_state return (StatusFlags.GREEN, False) # (state:str, should_notify:bool) + + +def inspect_shipcall_evaluation(vr, sql_handler, shipcall_id): + """ + # debug only! + + a simple debugging function, which serves in inspecting an evaluation function for a single shipcall id. It returns the result and all related data. + returns: result, shipcall_df (filtered by shipcall id), shipcall, spm (shipcall participant map, filtered by shipcall id), times_df (filtered by shipcall id) + """ + shipcall_df = sql_handler.df_dict.get("shipcall").loc[shipcall_id:shipcall_id,:] + + shipcall = Shipcall(**{**{"id":shipcall_id},**sql_handler.df_dict.get("shipcall").loc[shipcall_id].to_dict()}) + result = vr.evaluate(shipcall=shipcall) + notification_state = vr.identify_notification_state_change(state_old=int(shipcall.evaluation), state_new=int(result[0])) + print(f"Previous state: {int(shipcall.evaluation)}, New State: {result[0]}, Notification State: {notification_state}") + + times_df = sql_handler.df_dict.get("times") + times_df = times_df.loc[times_df["shipcall_id"]==shipcall_id] + + + spm = sql_handler.df_dict["shipcall_participant_map"] + spm = spm.loc[spm["shipcall_id"]==shipcall_id] + + return result, shipcall_df, shipcall, spm, times_df \ No newline at end of file From 2aaedb2ea5e0ad13bd672afd85c3f9a5d846c226 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Fri, 19 Jan 2024 18:07:17 +0100 Subject: [PATCH 06/30] enumerators are now IntEnum objects, which provides simpler typing. --- src/server/BreCal/database/enums.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index 7f8fc3d..90726b3 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -11,7 +11,7 @@ class ParticipantType(IntFlag): PORT_ADMINISTRATION = 32 TUG = 64 -class ShipcallType(Enum): +class ShipcallType(IntEnum): """determines the type of a shipcall, as this changes the applicable validation rules""" INCOMING = 1 OUTGOING = 2 @@ -28,7 +28,7 @@ class ParticipantwiseTimeDelta(): NOTIFICATION = 10.0 # after n minutes, an evaluation may rise a notification -class StatusFlags(Enum): +class StatusFlags(IntEnum): """ these enumerators ensure that each traffic light validation rule state corresponds to a value, which will be used in the ValidationRules object to identify the necessity of notifications. @@ -39,6 +39,7 @@ class StatusFlags(Enum): RED = 3 class PierSide(IntEnum): + """These enumerators determine the pier side of a shipcall.""" PORTSIDE = 0 # Port/Backbord STARBOARD_SIDE = 1 # Starboard / Steuerbord From a45526a42bc49ec87bf90bdb203d91ca1b613ab4 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Mon, 15 Apr 2024 08:19:08 +0200 Subject: [PATCH 07/30] git ignoring VSCode --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5ce71e8..cfac68e 100644 --- a/.gitignore +++ b/.gitignore @@ -289,3 +289,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. src/notebooks_metz/ src/server/editable_requirements.txt +brecal.code-workspace From 81c93412ad39464c0469d14d85e1b9235c1099f6 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Mon, 15 Apr 2024 12:06:48 +0200 Subject: [PATCH 08/30] updating STUB objects, slightly adapting data models --- src/server/BreCal/schemas/model.py | 115 +++++++++++------------- src/server/BreCal/stubs/notification.py | 2 - src/server/BreCal/stubs/times_full.py | 9 ++ src/server/BreCal/stubs/user.py | 9 ++ 4 files changed, 71 insertions(+), 64 deletions(-) diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index c429777..0df42c8 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -1,5 +1,5 @@ from dataclasses import field, dataclass -from marshmallow import Schema, fields, INCLUDE, ValidationError, validate, validates +from marshmallow import Schema, fields, INCLUDE, ValidationError, validate, validates, post_load from marshmallow.fields import Field from marshmallow_enum import EnumField from enum import IntEnum @@ -105,7 +105,7 @@ class History: return self(id, participant_id, shipcall_id, timestamp, eta, ObjectType(type), OperationType(operation)) class Error(Schema): - message = fields.String(required=True) + message = fields.String(metadata={'required':True}) class GetVerifyInlineResp(Schema): @@ -164,41 +164,34 @@ class ShipcallSchema(Schema): id = fields.Integer() ship_id = fields.Integer() type = fields.Integer() - eta = fields.DateTime(Required = False, allow_none=True) + eta = fields.DateTime(metadata={'required':False}, allow_none=True) voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} - etd = fields.DateTime(Required = False, allow_none=True) - arrival_berth_id = fields.Integer(Required = False, allow_none=True) - departure_berth_id = fields.Integer(Required = False, allow_none=True) - tug_required = fields.Bool(Required = False, allow_none=True) - pilot_required = fields.Bool(Required = False, allow_none=True) - flags = fields.Integer(Required = False, allow_none=True) - pier_side = fields.Bool(Required = False, allow_none=True) - bunkering = fields.Bool(Required = False, allow_none=True) - replenishing_terminal = fields.Bool(Required = False, allow_none=True) - replenishing_lock = fields.Bool(Required = False, allow_none=True) - draft = fields.Float(Required = False, allow_none=True, validate=[validate.Range(min=0, max=20, min_inclusive=False, max_inclusive=True)]) - tidal_window_from = fields.DateTime(Required = False, allow_none=True) - tidal_window_to = fields.DateTime(Required = False, allow_none=True) - rain_sensitive_cargo = fields.Bool(Required = False, allow_none=True) - recommended_tugs = fields.Integer(Required = False, allow_none=True) - anchored = fields.Bool(Required = False, allow_none=True) - moored_lock = fields.Bool(Required = False, allow_none=True) - canceled = fields.Bool(Required = False, allow_none=True) -<<<<<<< HEAD - evaluation = fields.Enum(EvaluationType, required=False, allow_none=True, default=EvaluationType.undefined) + etd = fields.DateTime(metadata={'required':False}, allow_none=True) + arrival_berth_id = fields.Integer(metadata={'required':False}, allow_none=True) + departure_berth_id = fields.Integer(metadata={'required':False}, allow_none=True) + tug_required = fields.Bool(metadata={'required':False}, allow_none=True) + pilot_required = fields.Bool(metadata={'required':False}, allow_none=True) + flags = fields.Integer(metadata={'required':False}, allow_none=True) + pier_side = fields.Bool(metadata={'required':False}, allow_none=True) + bunkering = fields.Bool(metadata={'required':False}, allow_none=True) + replenishing_terminal = fields.Bool(metadata={'required':False}, allow_none=True) + replenishing_lock = fields.Bool(metadata={'required':False}, allow_none=True) + draft = fields.Float(metadata={'required':False}, allow_none=True, validate=[validate.Range(min=0, max=20, min_inclusive=False, max_inclusive=True)]) + tidal_window_from = fields.DateTime(metadata={'required':False}, allow_none=True) + tidal_window_to = fields.DateTime(metadata={'required':False}, allow_none=True) + rain_sensitive_cargo = fields.Bool(metadata={'required':False}, allow_none=True) + recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True) + anchored = fields.Bool(metadata={'required':False}, allow_none=True) + moored_lock = fields.Bool(metadata={'required':False}, allow_none=True) + canceled = fields.Bool(metadata={'required':False}, allow_none=True) + evaluation = fields.Enum(EvaluationType, metadata={'required':False}, allow_none=True, default=EvaluationType.undefined) evaluation_message = fields.Str(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} - evaluation_time = fields.DateTime(Required = False, allow_none=True) - evaluation_notifications_sent = fields.Bool(Required = False, allow_none=True) - time_ref_point = fields.Int(Required = False, allow_none=True) -======= - evaluation = fields.Integer(Required = False, allow_none=True) - evaluation_message = fields.String(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} - evaluation_time = fields.DateTime(Required = False, allow_none=True) - evaluation_notifications_sent = fields.Bool(Required = False, allow_none=True) ->>>>>>> a5284a4 (implementing notifications, working on input validation) + evaluation_time = fields.DateTime(metadata={'required':False}, allow_none=True) + evaluation_notifications_sent = fields.Bool(metadata={'required':False}, allow_none=True) + time_ref_point = fields.Integer(metadata={'required':False}, allow_none=True) participants = fields.List(fields.Nested(ParticipantAssignmentSchema)) - created = fields.DateTime(Required = False, allow_none=True) - modified = fields.DateTime(Required = False, allow_none=True) + created = fields.DateTime(metadata={'required':False}, allow_none=True) + modified = fields.DateTime(metadata={'required':False}, allow_none=True) @post_load def make_shipcall(self, data, **kwargs): @@ -258,8 +251,6 @@ class Shipcall: created: datetime modified: datetime participants: List[Participant_Assignment] = field(default_factory=list) - evaluation_time : datetime = None - evaluation_notifications_sent : bool = None def to_json(self): return { @@ -313,30 +304,30 @@ class TimesSchema(Schema): super().__init__(unknown=None) pass - id = fields.Integer(Required=False) - eta_berth = fields.DateTime(Required = False, allow_none=True) - eta_berth_fixed = fields.Bool(Required = False, allow_none=True) - etd_berth = fields.DateTime(Required = False, allow_none=True) - etd_berth_fixed = fields.Bool(Required = False, allow_none=True) - lock_time = fields.DateTime(Required = False, allow_none=True) - lock_time_fixed = fields.Bool(Required = False, allow_none=True) - zone_entry = fields.DateTime(Required = False, allow_none=True) - zone_entry_fixed = fields.Bool(Required = False, allow_none=True) - operations_start = fields.DateTime(Required = False, allow_none=True) - operations_end = fields.DateTime(Required = False, allow_none=True) - remarks = fields.String(Required = False, allow_none=True, validate=[validate.Length(max=256)]) - participant_id = fields.Integer(Required = True) - berth_id = fields.Integer(Required = False, allow_none = True) - berth_info = fields.String(Required = False, allow_none=True, validate=[validate.Length(max=256)]) - pier_side = fields.Bool(Required = False, allow_none = True) - shipcall_id = fields.Integer(Required = True) - participant_type = fields.Integer(Required = False, allow_none=True) - ata = fields.DateTime(Required = False, allow_none=True) - atd = fields.DateTime(Required = False, allow_none=True) - eta_interval_end = fields.DateTime(Required = False, allow_none=True) - etd_interval_end = fields.DateTime(Required = False, allow_none=True) - created = fields.DateTime(Required = False, allow_none=True) - modified = fields.DateTime(Required = False, allow_none=True) + id = fields.Integer(metadata={'required':False}) + eta_berth = fields.DateTime(metadata={'required':False}, allow_none=True) + eta_berth_fixed = fields.Bool(metadata={'required':False}, allow_none=True) + etd_berth = fields.DateTime(metadata={'required':False}, allow_none=True) + etd_berth_fixed = fields.Bool(metadata={'required':False}, allow_none=True) + lock_time = fields.DateTime(metadata={'required':False}, allow_none=True) + lock_time_fixed = fields.Bool(metadata={'required':False}, allow_none=True) + zone_entry = fields.DateTime(metadata={'required':False}, allow_none=True) + zone_entry_fixed = fields.Bool(metadata={'required':False}, allow_none=True) + operations_start = fields.DateTime(metadata={'required':False}, allow_none=True) + operations_end = fields.DateTime(metadata={'required':False}, allow_none=True) + remarks = fields.String(metadata={'required':False}, allow_none=True, validate=[validate.Length(max=256)]) + participant_id = fields.Integer(metadata={'required':True}) + berth_id = fields.Integer(metadata={'required':False}, allow_none = True) + berth_info = fields.String(metadata={'required':False}, allow_none=True, validate=[validate.Length(max=256)]) + pier_side = fields.Bool(metadata={'required':False}, allow_none = True) + shipcall_id = fields.Integer(metadata={'required':True}) + participant_type = fields.Integer(metadata={'required':False}, allow_none=True) + ata = fields.DateTime(metadata={'required':False}, allow_none=True) + atd = fields.DateTime(metadata={'required':False}, allow_none=True) + eta_interval_end = fields.DateTime(metadata={'required':False}, allow_none=True) + etd_interval_end = fields.DateTime(metadata={'required':False}, allow_none=True) + created = fields.DateTime(metadata={'required':False}, allow_none=True) + modified = fields.DateTime(metadata={'required':False}, allow_none=True) @validates("eta_berth") def validate_eta_berth(self, value): @@ -351,7 +342,7 @@ class UserSchema(Schema): def __init__(self): super().__init__(unknown=None) pass - id = fields.Integer(required=True) + id = fields.Integer(metadata={'required':True}) first_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) last_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) user_phone = fields.String(allow_none=True, metadata={'Required':False}) @@ -439,7 +430,7 @@ class ShipSchema(Schema): super().__init__(unknown=None) pass - id = fields.Int(Required=False) + id = fields.Int(metadata={'required':False}) name = fields.String(allow_none=False, metadata={'Required':True}) imo = fields.Int(allow_none=False, metadata={'Required':True}) callsign = fields.String(allow_none=True, metadata={'Required':False}) diff --git a/src/server/BreCal/stubs/notification.py b/src/server/BreCal/stubs/notification.py index 63df90f..8971a76 100644 --- a/src/server/BreCal/stubs/notification.py +++ b/src/server/BreCal/stubs/notification.py @@ -7,7 +7,6 @@ def get_notification_simple(): """creates a default notification, where 'created' is now, and modified is now+10 seconds""" notification_id = generate_uuid1_int() # uid? times_id = generate_uuid1_int() # uid? - acknowledged = False level = 10 type = 0 message = "hello world" @@ -17,7 +16,6 @@ def get_notification_simple(): notification = Notification( notification_id, times_id, - acknowledged, level, type, message, diff --git a/src/server/BreCal/stubs/times_full.py b/src/server/BreCal/stubs/times_full.py index f0176a1..774cc9f 100644 --- a/src/server/BreCal/stubs/times_full.py +++ b/src/server/BreCal/stubs/times_full.py @@ -27,6 +27,11 @@ def get_times_full_simple(): zone_entry = etd_berth+datetime.timedelta(hours=0, minutes=15) zone_entry_fixed = False + + ata = eta_berth+datetime.timedelta(hours=0, minutes=15) + atd = etd_berth+datetime.timedelta(hours=0, minutes=15) + eta_interval_end = eta_berth + datetime.timedelta(hours=0, minutes=25) + etd_interval_end = etd_berth + datetime.timedelta(hours=0, minutes=25) operations_start = zone_entry+datetime.timedelta(hours=1, minutes=30) operations_end = operations_start+datetime.timedelta(hours=4, minutes=30) @@ -63,6 +68,10 @@ def get_times_full_simple(): pier_side=pier_side, participant_type=participant_type, shipcall_id=shipcall_id, + ata=ata, + atd=atd, + eta_interval_end=eta_interval_end, + etd_interval_end=etd_interval_end, created=created, modified=modified, ) diff --git a/src/server/BreCal/stubs/user.py b/src/server/BreCal/stubs/user.py index e469c55..908f512 100644 --- a/src/server/BreCal/stubs/user.py +++ b/src/server/BreCal/stubs/user.py @@ -18,6 +18,11 @@ def get_user_simple(): created = datetime.datetime.now() modified = created+datetime.timedelta(seconds=10) + + notify_email = True + notify_whatsapp = True + notify_signal = True + notify_popup = True user = User( user_id, @@ -29,6 +34,10 @@ def get_user_simple(): user_phone, password_hash, api_key, + notify_email, + notify_whatsapp, + notify_signal, + notify_popup, created, modified ) From 046e1c7d70652f123d61dfa52efd99ff81ec7afb Mon Sep 17 00:00:00 2001 From: scopesorting Date: Mon, 15 Apr 2024 12:22:00 +0200 Subject: [PATCH 09/30] removing workspace file from VSCode --- brecal.code-workspace | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 brecal.code-workspace diff --git a/brecal.code-workspace b/brecal.code-workspace deleted file mode 100644 index 876a149..0000000 --- a/brecal.code-workspace +++ /dev/null @@ -1,8 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ], - "settings": {} -} \ No newline at end of file From c98af22299dc5fae5f99413d3e7fb034aef3a0be Mon Sep 17 00:00:00 2001 From: Max Metz Date: Wed, 24 Apr 2024 08:26:37 +0200 Subject: [PATCH 10/30] setting up a local mysql database and running the API locally, which requires slight adaptations. Implementing input validation for POST requests of shipcalls and adapting enumerators, as well as data models. --- src/server/BreCal/api/shipcalls.py | 23 ++++- src/server/BreCal/database/enums.py | 22 ++++- src/server/BreCal/database/sql_handler.py | 19 +++-- src/server/BreCal/impl/shipcalls.py | 8 +- src/server/BreCal/local_db.py | 4 +- src/server/BreCal/schemas/model.py | 68 +++++++++++++-- src/server/BreCal/services/auth_guard.py | 1 + src/server/BreCal/services/jwt_handler.py | 15 ++++ src/server/BreCal/stubs/shipcall.py | 17 ++-- .../BreCal/validators/input_validation.py | 83 +++++++++++++++++++ src/server/requirements.txt | 3 +- src/server/tests/test_create_app.py | 2 +- .../test_validation_rule_functions.py | 12 ++- 13 files changed, 244 insertions(+), 33 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index fbf552a..61a7a7c 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -3,7 +3,8 @@ from webargs.flaskparser import parser from marshmallow import Schema, fields, ValidationError from ..schemas import model from .. import impl -from ..services.auth_guard import auth_guard +from ..services.auth_guard import auth_guard, check_jwt +from BreCal.validators.input_validation import check_if_user_is_bsmd_type, check_if_user_data_has_valid_ship_id, check_if_user_data_has_valid_berth_id, check_if_user_data_has_valid_participant_id import logging import json @@ -40,6 +41,26 @@ def PostShipcalls(): try: content = request.get_json(force=True) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + + import logging + logging.log(20, loadedModel) + logging.log(20, "metz development") + """ + # loadedModel ... + valid_ship_id = check_if_user_data_has_valid_ship_id(ship_id) + valid_berth_id = check_if_user_data_has_valid_berth_id(berth_id) + valid_participant_id = check_if_user_data_has_valid_participant_id(participant_id) + """ + except ValidationError as ex: logging.error(ex) diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index 90726b3..5d28e65 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -2,7 +2,7 @@ from enum import IntEnum, Enum, IntFlag class ParticipantType(IntFlag): """determines the type of a participant""" - NONE = 0 + undefined = 0 BSMD = 1 TERMINAL = 2 PILOT = 4 @@ -13,10 +13,15 @@ class ParticipantType(IntFlag): class ShipcallType(IntEnum): """determines the type of a shipcall, as this changes the applicable validation rules""" + undefined = 0 INCOMING = 1 OUTGOING = 2 SHIFTING = 3 + @classmethod + def _missing_(cls, value): + return cls.undefined + class ParticipantwiseTimeDelta(): """stores the time delta for every participant, which triggers the validation rules in the rule set '0001'""" AGENCY = 1200.0 # 20 h * 60 min/h = 1200 min @@ -42,10 +47,23 @@ class PierSide(IntEnum): """These enumerators determine the pier side of a shipcall.""" PORTSIDE = 0 # Port/Backbord STARBOARD_SIDE = 1 # Starboard / Steuerbord - + class NotificationType(IntFlag): """determines the method by which a notification is distributed to users. Flagging allows selecting multiple notification types.""" UNDEFINED = 0 EMAIL = 1 POPUP = 2 MESSENGER = 4 + +class ParticipantFlag(IntFlag): + """ + | 1 | If this flag is set on a shipcall record with participant type Agency (8), + all participants of type BSMD (1) may edit the record. + """ + undefined = 0 + BSMD = 1 + + @classmethod + def _missing_(cls, value): + return cls.undefined + diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index 59497e3..929f558 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -2,7 +2,7 @@ import numpy as np import pandas as pd import datetime import typing -from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times +from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times, ShipcallParticipantMap from BreCal.database.enums import ParticipantType def pandas_series_to_data_model(): @@ -50,7 +50,8 @@ class SQLHandler(): 'ship'->BreCal.schemas.model.Ship object """ self.str_to_model_dict = { - "shipcall":Shipcall, "ship":Ship, "participant":Participant, "berth":Berth, "user":User, "times":Times + "shipcall":Shipcall, "ship":Ship, "participant":Participant, "berth":Berth, "user":User, "times":Times, + "shipcall_participant_map":ShipcallParticipantMap } return @@ -70,12 +71,16 @@ class SQLHandler(): data = [{k:v for k,v in zip(column_names, dat)} for dat in data] # 4.) build a dataframe from the respective data models (which ensures the correct data type) + df = self.build_df_from_data_and_name(data, table_name) + return df + + def build_df_from_data_and_name(self, data, table_name): data_model = self.str_to_model_dict.get(table_name) if data_model is not None: - df = pd.DataFrame([data_model(**dat) for dat in data]) + df = pd.DataFrame([data_model(**dat) for dat in data], columns=list(data_model.__annotations__.keys())) else: df = pd.DataFrame([dat for dat in data]) - return df + return df def mysql_to_df(self, query, table_name): """provide an arbitrary sql query that should be read from a mysql server {sql_connection}. returns a pandas DataFrame with the obtained data""" @@ -94,11 +99,7 @@ class SQLHandler(): # 4.) build a dataframe from the respective data models (which ensures the correct data type) data_model = self.str_to_model_dict.get(table_name) - if data_model is not None: - df = pd.DataFrame([data_model(**dat) for dat in data]) - else: - df = pd.DataFrame([dat for dat in data]) - + df = self.build_df_from_data_and_name(data, table_name) if 'id' in df.columns: df = df.set_index('id', inplace=False) # avoid inplace updates, so the raw sql remains unchanged return df diff --git a/src/server/BreCal/impl/shipcalls.py b/src/server/BreCal/impl/shipcalls.py index 8d15983..252c817 100644 --- a/src/server/BreCal/impl/shipcalls.py +++ b/src/server/BreCal/impl/shipcalls.py @@ -60,8 +60,13 @@ def PostShipcalls(schemaModel): """ :param schemaModel: The deserialized dict of the request + e.g., + { + 'ship_id': 1, 'type': 1, 'eta': datetime.datetime(2023, 7, 23, 7, 18, 19), + 'voyage': '43B', 'tug_required': False, 'pilot_required': True, 'flags': 0, + 'pier_side': False, 'bunkering': True, 'recommended_tugs': 2, 'type_value': 1, 'evaluation_value': 0} + } """ - # TODO: Validate the upload data # This creates a *new* entry @@ -133,7 +138,6 @@ def PostShipcalls(schemaModel): # if not full_id_existances: # pooledConnection.close() # return json.dumps({"message" : "call failed. missing mandatory keywords."}), 500, {'Content-Type': 'application/json; charset=utf-8'} - commands.execute(query, schemaModel) new_id = commands.execute_scalar("select last_insert_id()") diff --git a/src/server/BreCal/local_db.py b/src/server/BreCal/local_db.py index 4293ff3..b4e02cd 100644 --- a/src/server/BreCal/local_db.py +++ b/src/server/BreCal/local_db.py @@ -7,11 +7,11 @@ import sys config_path = None -def initPool(instancePath): +def initPool(instancePath, connection_filename="connection_data_devel.json"): try: global config_path if(config_path == None): - config_path = os.path.join(instancePath,'../../../secure/connection_data_devel.json'); + config_path = os.path.join(instancePath,f'../../../../secure/{connection_filename}') #connection_data_devel.json'); print (config_path) diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index 0df42c8..33638fd 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -10,6 +10,9 @@ from typing import List import json import datetime from BreCal.validators.time_logic import validate_time_exceeds_threshold +from BreCal.database.enums import ParticipantType, ParticipantFlag + +# from BreCal. ... import check_if_user_is_bsmd_type def obj_dict(obj): if isinstance(obj, datetime.datetime): @@ -54,6 +57,7 @@ class NotificationType(IntEnum): undefined = 0 email = 1 push = 2 + @classmethod def _missing_(cls, value): return cls.undefined @@ -143,12 +147,34 @@ class Participant(Schema): street: str postal_code: str city: str - type: int + type: int # fields.Enum(ParticipantType ...) flags: int created: datetime modified: datetime deleted: bool + @validates("type") + def validate_type(self, value): + # e.g., when an IntFlag has the values 1,2,4; the maximum valid value is 7 + max_int = sum([int(val) for val in list(ParticipantType._value2member_map_.values())]) + min_int = 0 + + valid_type = 0 <= value < max_int + if not valid_type: + raise ValidationError(f"the provided integer is not supported for default behaviour of the ParticipantType IntFlag. Your choice: {value}. Supported values are: 0 <= value {max_int}") + + + @validates("flags") + def validate_type(self, value): + # e.g., when an IntFlag has the values 1,2,4; the maximum valid value is 7 + max_int = sum([int(val) for val in list(ParticipantFlag._value2member_map_.values())]) + min_int = 0 + + valid_type = 0 <= value < max_int + if not valid_type: + raise ValidationError(f"the provided integer is not supported for default behaviour of the ParticipantFlag IntFlag. Your choice: {value}. Supported values are: 0 <= value {max_int}") + + class ParticipantList(Participant): pass @@ -163,7 +189,8 @@ class ShipcallSchema(Schema): id = fields.Integer() ship_id = fields.Integer() - type = fields.Integer() + #type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator + type = fields.Integer() # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator eta = fields.DateTime(metadata={'required':False}, allow_none=True) voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} etd = fields.DateTime(metadata={'required':False}, allow_none=True) @@ -196,15 +223,23 @@ class ShipcallSchema(Schema): @post_load def make_shipcall(self, data, **kwargs): if 'type' in data: - data['type_value'] = data['type'].value + data['type_value'] = int(data['type']) else: - data['type_value'] = ShipcallType.undefined + data['type_value'] = int(ShipcallType.undefined) if 'evaluation' in data: if data['evaluation']: - data['evaluation_value'] = data['evaluation'].value + data['evaluation_value'] = int(data['evaluation']) else: - data['evaluation_value'] = EvaluationType.undefined + data['evaluation_value'] = int(EvaluationType.undefined) return data + + @validates("type") + def validate_type(self, value): + valid_shipcall_type = int(value) in [item.value for item in ShipcallType] + + if not valid_shipcall_type: + raise ValidationError(f"the provided type is not a valid shipcall type.") + @dataclass class Participant_Assignment: @@ -321,7 +356,7 @@ class TimesSchema(Schema): berth_info = fields.String(metadata={'required':False}, allow_none=True, validate=[validate.Length(max=256)]) pier_side = fields.Bool(metadata={'required':False}, allow_none = True) shipcall_id = fields.Integer(metadata={'required':True}) - participant_type = fields.Integer(metadata={'required':False}, allow_none=True) + participant_type = fields.Enum(ParticipantType, metadata={'required':False}, allow_none=True, default=ParticipantType.undefined) #fields.Integer(metadata={'required':False}, allow_none=True) ata = fields.DateTime(metadata={'required':False}, allow_none=True) atd = fields.DateTime(metadata={'required':False}, allow_none=True) eta_interval_end = fields.DateTime(metadata={'required':False}, allow_none=True) @@ -463,3 +498,22 @@ class Shipcalls(Shipcall): class TimesList(Times): pass + +@dataclass +class ShipcallParticipantMap: + id: int + shipcall_id: int + participant_id: int + type : ShipcallType + created: datetime + modified: datetime + + def to_json(self): + return { + "id": self.id, + "shipcall_id": self.shipcall_id, + "participant_id": self.participant_id, + "type": self.type.name, + "created": self.created.isoformat() if self.created else "", + "modified": self.modified.isoformat() if self.modified else "", + } diff --git a/src/server/BreCal/services/auth_guard.py b/src/server/BreCal/services/auth_guard.py index 9c5824d..9bd27c6 100644 --- a/src/server/BreCal/services/auth_guard.py +++ b/src/server/BreCal/services/auth_guard.py @@ -9,6 +9,7 @@ def check_jwt(): if not token: raise Exception('Missing access token') jwt = token.split('Bearer ')[1] + try: return decode_jwt(jwt) except Exception as e: diff --git a/src/server/BreCal/services/jwt_handler.py b/src/server/BreCal/services/jwt_handler.py index 66c0f43..b3c80e7 100644 --- a/src/server/BreCal/services/jwt_handler.py +++ b/src/server/BreCal/services/jwt_handler.py @@ -7,11 +7,26 @@ def create_api_key(): return secrets.token_urlsafe(16) def generate_jwt(payload, lifetime=None): + """ + creates an encoded token, which is based on the 'SECRET_KEY' environment variable. The environment variable + is set when the .wsgi application is started or can theoretically be set on system-level. + + args: + payload: + json-dictionary with key:value pairs. + + lifetime: + When a 'lifetime' (integer) is provided, the payload will be extended by an expiration key 'exp', which is + valid for the next {lifetime} minutes. + + returns: token, a JWT-encoded string + """ if lifetime: payload['exp'] = (datetime.datetime.now() + datetime.timedelta(minutes=lifetime)).timestamp() return jwt.encode(payload, os.environ.get('SECRET_KEY'), algorithm="HS256") def decode_jwt(token): + """this function reverts the {generate_jwt} function. An encoded JWT token is decoded into a JSON dictionary.""" return jwt.decode(token, os.environ.get('SECRET_KEY'), algorithms=["HS256"]) diff --git a/src/server/BreCal/stubs/shipcall.py b/src/server/BreCal/stubs/shipcall.py index 2e4e154..3f5c062 100644 --- a/src/server/BreCal/stubs/shipcall.py +++ b/src/server/BreCal/stubs/shipcall.py @@ -7,21 +7,21 @@ def get_shipcall_simple(): # only used for the stub base_time = datetime.datetime.now() - shipcall_id = generate_uuid1_int() - ship_id = generate_uuid1_int() + shipcall_id = 124 # generate_uuid1_int() + ship_id = 5 # generate_uuid1_int() eta = base_time+datetime.timedelta(hours=3, minutes=12) role_type = 1 voyage = "987654321" etd = base_time+datetime.timedelta(hours=6, minutes=12) # should never be before eta - arrival_berth_id = generate_uuid1_int() - departure_berth_id = generate_uuid1_int() + arrival_berth_id = 140 #generate_uuid1_int() + departure_berth_id = 140 #generate_uuid1_int() tug_required = False pilot_required = False - flags = 0 # #TODO_shipcall_flags. What is meant here? What should be tested? + flags = 0 # #TODO_shipcall_flags. What is meant here? What should be tested? pier_side = False # whether a ship will be fixated on the pier side. en: pier side, de: Anlegestelle. From 'BremenCalling_Datenmodell.xlsx': gedreht/ungedreht bunkering = False # #TODO_bunkering_unclear replenishing_terminal = False # en: replenishing terminal, de: Nachfüll-Liegeplatz @@ -38,11 +38,13 @@ def get_shipcall_simple(): anchored = False moored_lock = False # de: 'Festmacherschleuse', en: 'moored lock' canceled = False + + time_ref_point = 0 evaluation = None evaluation_message = "" - evaluation_time = None - evaluation_notifications_sent = None + evaluation_time = datetime.datetime.now() + evaluation_notifications_sent = False created = datetime.datetime.now() modified = created+datetime.timedelta(seconds=10) @@ -76,6 +78,7 @@ def get_shipcall_simple(): evaluation_message, evaluation_time, evaluation_notifications_sent, + time_ref_point, created, modified, participants, diff --git a/src/server/BreCal/validators/input_validation.py b/src/server/BreCal/validators/input_validation.py index 188e5e4..2248779 100644 --- a/src/server/BreCal/validators/input_validation.py +++ b/src/server/BreCal/validators/input_validation.py @@ -1,8 +1,91 @@ ####################################### InputValidation ####################################### +import json from abc import ABC, abstractmethod from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant +from BreCal.impl.participant import GetParticipant +from BreCal.impl.ships import GetShips +from BreCal.impl.berths import GetBerths + +from BreCal.database.enums import ParticipantType + +def get_participant_id_dictionary(): + # get all participants + response,status_code,header = GetParticipant(options={}) + + # build a dictionary of id:item pairs, so one can select the respective participant + participants = json.loads(response) + participants = {items.get("id"):items for items in participants} + return participants + +def get_ship_id_dictionary(): + # get all ships + response,status_code,header = GetShips(token=None) + + # build a dictionary of id:item pairs, so one can select the respective participant + ships = json.loads(response) + ships = {items.get("id"):items for items in ships} + return ships + +def get_berth_id_dictionary(): + # get all berths + response,status_code,header = GetBerths(token=None) + + # build a dictionary of id:item pairs, so one can select the respective participant + berths = json.loads(response) + berths = {items.get("id"):items for items in berths} + return berths + +def check_if_user_is_bsmd_type(user_data:dict)->bool: + """ + given a dictionary of user data, determine the respective participant id and read, whether + that participant is a .BSMD-type + + Note: ParticipantType is an IntFlag. + Hence, ParticipantType(1) is ParticipantType.BSMD, + and ParticipantType(7) is [ParticipantType.BSMD, ParticipantType.TERMINAL, ParticipantType.PILOT] + + both would return 'True' + + returns: boolean. Whether the participant id is a .BSMD type element + """ + # user_data = decode token + participant_id = user_data.get("participant_id") + + # build a dictionary of id:item pairs, so one can select the respective participant + participants = get_participant_id_dictionary() + participant = participants.get(participant_id,{}) + + # boolean check: is the participant of type .BSMD? + is_bsmd = ParticipantType.BSMD in ParticipantType(participant.get("type",0)) + return is_bsmd + +def check_if_user_data_has_valid_ship_id(ship_id): + # build a dictionary of id:item pairs, so one can select the respective participant + ships = get_ship_id_dictionary() + + # boolean check + ship_id_is_valid = ship_id in list(ships.keys()) + return ship_id_is_valid + +def check_if_user_data_has_valid_berth_id(berth_id): + # build a dictionary of id:item pairs, so one can select the respective participant + berths = get_berth_id_dictionary() + + # boolean check + berth_id_is_valid = berth_id in list(berths.keys()) + return berth_id_is_valid + +def check_if_user_data_has_valid_participant_id(participant_id): + # build a dictionary of id:item pairs, so one can select the respective participant + participants = get_participant_id_dictionary() + + # boolean check + participant_id_is_valid = participant_id in list(participants.keys()) + return participant_id_is_valid + + class InputValidation(): def __init__(self): diff --git a/src/server/requirements.txt b/src/server/requirements.txt index 8b7f3ad..23bf276 100644 --- a/src/server/requirements.txt +++ b/src/server/requirements.txt @@ -8,6 +8,7 @@ webargs==6.1.1 Werkzeug==1.0.1 pydapper[mysql-connector-python] marshmallow-dataclass +marshmallow-enum bcrypt pyjwt flask-jwt-extended @@ -20,4 +21,4 @@ pytest pytest-cov coverage -../server/. +-e ../server/. diff --git a/src/server/tests/test_create_app.py b/src/server/tests/test_create_app.py index 652ae3d..46ede88 100644 --- a/src/server/tests/test_create_app.py +++ b/src/server/tests/test_create_app.py @@ -8,7 +8,7 @@ def test_create_app(): import sys from BreCal import get_project_root - project_root = get_project_root("brecal") + project_root = os.path.join(os.path.expanduser("~"), "brecal") lib_location = os.path.join(project_root, "src", "server") sys.path.append(lib_location) diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index fdd8152..3a08735 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -12,7 +12,17 @@ from BreCal.stubs.df_times import get_df_times, random_time_perturbation, get_df @pytest.fixture(scope="session") def build_sql_proxy_connection(): import mysql.connector - conn_from_pool = mysql.connector.connect(**{'host':'localhost', 'port':3306, 'user':'root', 'password':'HalloWach_2323XXL!!', 'pool_name':'brecal_pool', 'pool_size':20, 'database':'bremen_calling', 'autocommit': True}) + import os + import json + connection_data_path = os.path.join(os.path.expanduser("~"),"secure","connection_data_local.json") + assert os.path.exists(connection_data_path) + + with open(connection_data_path, "r") as jr: + connection_data = json.load(jr) + connection_data = {k:v for k,v in connection_data.items() if k in ["host", "port", "user", "password", "pool_size", "pool_name", "database"]} + + conn_from_pool = mysql.connector.connect(**connection_data) + #conn_from_pool = mysql.connector.connect(**{'host':'localhost', 'port':3306, 'user':'root', 'password':'HalloWach_2323XXL!!', 'pool_name':'brecal_pool', 'pool_size':20, 'database':'bremen_calling_local', 'autocommit': True}) sql_handler = SQLHandler(sql_connection=conn_from_pool, read_all=True) vr = ValidationRules(sql_handler) return locals() From 68a1e00477e4151b8c2641654e2e6701ccb29c13 Mon Sep 17 00:00:00 2001 From: Max Metz Date: Mon, 29 Apr 2024 11:30:24 +0200 Subject: [PATCH 11/30] adapting rule 0005A and refactoring header-checks --- src/server/BreCal/api/shipcalls.py | 9 +++ src/server/BreCal/database/sql_handler.py | 38 +++++++++ .../validators/validation_rule_functions.py | 81 +++++++++++++------ .../BreCal/validators/validation_rules.py | 4 +- 4 files changed, 104 insertions(+), 28 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 61a7a7c..8adde2f 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -56,6 +56,15 @@ def PostShipcalls(): logging.log(20, "metz development") """ # loadedModel ... + loadedModel.get("ship_id", 0) + + 2024-04-22 18:21:03,982 | root | INFO | {'ship_id': 1, + 'type': 1, 'eta': datetime.datetime(2023, 7, 23, 7, 18, 19), + 'voyage': '43B', 'tug_required': False, 'pilot_required': True, + 'flags': 0, 'pier_side': False, 'bunkering': True, 'recommended_tugs': 2, + 'type_value': 1, 'evaluation_value': 0} + + valid_ship_id = check_if_user_data_has_valid_ship_id(ship_id) valid_berth_id = check_if_user_data_has_valid_berth_id(berth_id) valid_participant_id = check_if_user_data_has_valid_participant_id(participant_id) diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index 929f558..741e631 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -19,7 +19,37 @@ def set_participant_type(x, participant_df)->int: participant_type = participant_df.loc[participant_id, "type"] return participant_type +def get_synchronous_shipcall_times_standalone(query_time:pd.Timestamp, all_df_times:pd.DataFrame, delta_threshold=900)->int: + """ + This function counts all entries in {all_df_times}, which have the same timestamp as {query_time}. + It does so by: + 1.) selecting all eta_berth & etd_berth entries + 2.) measuring the timedelta towards {query_time} + 3.) converting the timedelta to total absolute seconds (positive or negative time differences do not matter) + 4.) applying a {delta_threshold} to identify, whether two times are too closely together + 5.) counting the times, where the timedelta is below the threshold + returns: counts + """ + assert isinstance(query_time,pd.Timestamp) + + # get a timedelta for each valid (not Null) time entry + time_deltas_eta = [(query_time.to_pydatetime()-time_.to_pydatetime()) for time_ in all_df_times.loc[:,"eta_berth"] if not pd.isnull(time_)] + time_deltas_etd = [(query_time.to_pydatetime()-time_.to_pydatetime()) for time_ in all_df_times.loc[:,"etd_berth"] if not pd.isnull(time_)] + + # consider both, eta and etd times + time_deltas = time_deltas_eta + time_deltas_etd + + # convert the timedelta to absolute total seconds + time_deltas = [abs(delta.total_seconds()) for delta in time_deltas] + + # consider only those time deltas, which are <= the determined threshold + # create a list of booleans + time_deltas_filtered = [delta <= delta_threshold for delta in time_deltas] + + # booleans can be added/counted in Python by using sum() + counts = sum(time_deltas_filtered) # int + return counts class SQLHandler(): """ @@ -333,6 +363,10 @@ class SQLHandler(): def get_unique_ship_counts(self, all_df_times:pd.DataFrame, times_agency:pd.DataFrame, query:str, rounding:str="min", maximum_threshold=3): """given a dataframe of all agency times, get all unique ship counts, their values (datetime) and the string tags. returns a tuple (values,unique,counts)""" + # #deprecated! + import warnings + warnings.warn(f"SQLHandler.get_unique_ship_counts is deprecated. Instead, please use SQLHandler.count_synchronous_shipcall_times") + # optional: rounding if rounding is not None: all_df_times.loc[:, query] = pd.to_datetime(all_df_times.loc[:, query]).dt.round(rounding) # e.g., 'min' --- # correcting the error: 'AttributeError: Can only use .dt accessor with datetimelike values' @@ -348,3 +382,7 @@ class SQLHandler(): # get unique entries and counts counts = len(values) # unique, counts = np.unique(values, return_counts=True) return counts # (values, unique, counts) + + def count_synchronous_shipcall_times(self, query_time:pd.Timestamp, all_df_times:pd.DataFrame, delta_threshold=900)->int: + """count all times entries, which are too close to the query_time. The {delta_threshold} determines the threshold. returns counts (int)""" + return get_synchronous_shipcall_times_standalone(query_time, all_df_times, delta_threshold) diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index ef44f08..36ea933 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -38,14 +38,16 @@ error_message_dict = { "validation_rule_fct_etd_time_not_in_tidal_window":"The tidal window does not fit to the agency's estimated time of departure (ETD) {Rule #0004B}", # 0005 A+B - "validation_rule_fct_too_many_identical_eta_times":"There are more than three ships with the same planned time of arrival (ETA) {Rule #0005A}", - "validation_rule_fct_too_many_identical_etd_times":"There are more than three ships with the same planned time of departure (ETD) {Rule #0005B}", + "validation_rule_fct_too_many_identical_eta_times":"More than three shipcalls are planned at the same time as the defined ETA {Rule #0005A}", + "validation_rule_fct_too_many_identical_etd_times":"More than three shipcalls are planned at the same time as the defined ETD {Rule #0005B}", # 0006 A+B "validation_rule_fct_agency_and_terminal_berth_id_disagreement":"Agency and Terminal are planning with different berths (the berth_id deviates). {Rule #0006A}", "validation_rule_fct_agency_and_terminal_pier_side_disagreement":"Agency and Terminal are planning with different pier sides (the pier_side deviates). {Rule #0006B}", } + + class ValidationRuleBaseFunctions(): """ Base object with individual functions, which the {ValidationRuleFunctions}-child refers to. @@ -70,6 +72,18 @@ class ValidationRuleBaseFunctions(): def get_no_violation_default_output(self): """return the default output of a validation function with no validation: a tuple of (GREEN state, None)""" return (StatusFlags.GREEN, None) + + def check_if_header_exists(self, df_times:pd.DataFrame, participant_type:ParticipantType)->bool: + """ + Given a pandas DataFrame, which contains times entries for a specific shipcall id, + this function checks, whether one of the times entries belongs to the requested ParticipantType. + + returns bool + """ + # empty DataFrames form a special case, as they might miss the 'participant_type' column. + if len(df_times)==0: + return False + return participant_type in df_times.loc[:,"participant_type"].values def check_time_delta_violation_query_time_to_now(self, query_time:pd.Timestamp, key_time:pd.Timestamp, threshold:float)->bool: """ @@ -140,7 +154,7 @@ class ValidationRuleBaseFunctions(): agency_time = [time_ for time_ in agency_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] - if not len(agency_time): + if not len(agency_time): # if len(agency_time) == 0 violation_state = False return violation_state @@ -734,10 +748,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): + #if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: return self.get_no_violation_default_output() # rule not applicable # get agency & terminal times @@ -774,10 +790,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() # rule not applicable - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): return self.get_no_violation_default_output() # rule not applicable # get agency & terminal times @@ -814,7 +832,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + # if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) @@ -845,7 +864,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + # if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) @@ -867,16 +887,19 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): """ Code: #0005-A Type: Global Rule - Description: this validation rule checks, whether there are too many shipcalls with identical ETA times. + Description: this validation rule checks, whether there are too many shipcalls with identical times to the query ETA. """ - times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] # check, if the header is filled in (agency) - if len(times_agency) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): # if len(times_agency) != 1: return self.get_no_violation_default_output() - # when ANY of the unique values exceeds the threshold, a violation is observed - query = "eta_berth" - violation_state = self.check_unique_shipcall_counts(query, times_agency=times_agency, rounding=rounding, maximum_threshold=maximum_threshold, all_times_agency=all_times_agency) + # get the agency's query time + times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] + query_time = times_agency.iloc[0].eta_berth + + # count the number of times, where a times entry is very close to the query time (uses an internal threshold, such as 15 minutes) + counts = self.sql_handler.count_synchronous_shipcall_times(query_time, all_df_times=all_times_agency) + violation_state = counts > maximum_threshold if violation_state: validation_name = "validation_rule_fct_too_many_identical_eta_times" @@ -888,16 +911,19 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): """ Code: #0005-B Type: Global Rule - Description: this validation rule checks, whether there are too many shipcalls with identical ETD times. + Description: this validation rule checks, whether there are too many shipcalls with identical times to the query ETD. """ - times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] # check, if the header is filled in (agency) - if len(times_agency) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): #if len(times_agency) != 1: return self.get_no_violation_default_output() - # when ANY of the unique values exceeds the threshold, a violation is observed - query = "etd_berth" - violation_state = self.check_unique_shipcall_counts(query, times_agency=times_agency, rounding=rounding, maximum_threshold=maximum_threshold, all_times_agency=all_times_agency) + # get the agency's query time + times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] + query_time = times_agency.iloc[0].etd_berth + + # count the number of times, where a times entry is very close to the query time (uses an internal threshold, such as 15 minutes) + counts = self.sql_handler.count_synchronous_shipcall_times(query_time, all_df_times=all_times_agency) + violation_state = counts > maximum_threshold if violation_state: validation_name = "validation_rule_fct_too_many_identical_etd_times" @@ -912,10 +938,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): Description: This validation rule checks, whether agency and terminal agree with their designated berth place by checking berth_id. """ # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() # rule not applicable - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): return self.get_no_violation_default_output() # rule not applicable times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) @@ -948,13 +976,14 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): Description: This validation rule checks, whether agency and terminal agree with their designated pier side by checking pier_side. """ # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() # rule not applicable - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): return self.get_no_violation_default_output() # rule not applicable - times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) # when one of the two values is null, the state is GREEN diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index eb0e7d6..08944d4 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -29,9 +29,9 @@ class ValidationRules(ValidationRuleFunctions): returns: (evaluation_state, violations) """ # prepare df_times, which every validation rule tends to use - df_times = self.sql_handler.df_dict.get('times', pd.DataFrame()) # -> pd.DataFrame + all_df_times = self.sql_handler.df_dict.get('times', pd.DataFrame()) # -> pd.DataFrame - if len(df_times)==0: + if len(all_df_times)==0: return (StatusFlags.GREEN.value, []) spm = self.sql_handler.df_dict["shipcall_participant_map"] From 78d7fcbd5beb4b2ac55b66dc2cdf14c21d704863 Mon Sep 17 00:00:00 2001 From: Max Metz Date: Mon, 29 Apr 2024 16:46:50 +0200 Subject: [PATCH 12/30] implementing POST-request input validation for shipcalls. Creating many tests and applying slight updates to the Notifier (not implemented yet) --- src/server/BreCal/api/shipcalls.py | 32 +---- src/server/BreCal/database/update_database.py | 1 + .../notifications/notification_functions.py | 2 +- src/server/BreCal/schemas/model.py | 9 +- src/server/BreCal/stubs/shipcall.py | 18 +++ .../BreCal/validators/input_validation.py | 133 +++++++++++++++++- .../BreCal/validators/validation_rules.py | 7 +- 7 files changed, 165 insertions(+), 37 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 8adde2f..c1c6e5d 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -4,7 +4,7 @@ from marshmallow import Schema, fields, ValidationError from ..schemas import model from .. import impl from ..services.auth_guard import auth_guard, check_jwt -from BreCal.validators.input_validation import check_if_user_is_bsmd_type, check_if_user_data_has_valid_ship_id, check_if_user_data_has_valid_berth_id, check_if_user_data_has_valid_participant_id +from BreCal.validators.input_validation import validate_posted_shipcall_data import logging import json @@ -41,35 +41,15 @@ def PostShipcalls(): try: content = request.get_json(force=True) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + logging.log(20, loadedModel) + logging.log(20, "dev. above: loaded model, below: content") + logging.log(20, content) # read the user data from the JWT token (set when login is performed) user_data = check_jwt() - # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD - # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. - is_bsmd = check_if_user_is_bsmd_type(user_data) - if not is_bsmd: - raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") - - import logging - logging.log(20, loadedModel) - logging.log(20, "metz development") - """ - # loadedModel ... - loadedModel.get("ship_id", 0) - - 2024-04-22 18:21:03,982 | root | INFO | {'ship_id': 1, - 'type': 1, 'eta': datetime.datetime(2023, 7, 23, 7, 18, 19), - 'voyage': '43B', 'tug_required': False, 'pilot_required': True, - 'flags': 0, 'pier_side': False, 'bunkering': True, 'recommended_tugs': 2, - 'type_value': 1, 'evaluation_value': 0} - - - valid_ship_id = check_if_user_data_has_valid_ship_id(ship_id) - valid_berth_id = check_if_user_data_has_valid_berth_id(berth_id) - valid_participant_id = check_if_user_data_has_valid_participant_id(participant_id) - """ - + # validate the posted shipcall data + validate_posted_shipcall_data(user_data, loadedModel, content) except ValidationError as ex: logging.error(ex) diff --git a/src/server/BreCal/database/update_database.py b/src/server/BreCal/database/update_database.py index 7b7639b..4c120f3 100644 --- a/src/server/BreCal/database/update_database.py +++ b/src/server/BreCal/database/update_database.py @@ -55,6 +55,7 @@ def update_all_shipcalls_in_mysql_database(sql_connection, sql_handler:SQLHandle sql_handler: an SQLHandler instance shipcall_df: dataframe, which stores the data that is used to retrieve the shipcall data models (that are then updated in the database) """ + print(shipcall_df) for shipcall_id in shipcall_df.index: shipcall = sql_handler.df_loc_to_data_model(df=shipcall_df, id=shipcall_id, model_str="shipcall") update_shipcall_in_mysql_database(sql_connection, shipcall=shipcall, relevant_keys = ["evaluation", "evaluation_message"]) diff --git a/src/server/BreCal/notifications/notification_functions.py b/src/server/BreCal/notifications/notification_functions.py index 9bbc373..86956f0 100644 --- a/src/server/BreCal/notifications/notification_functions.py +++ b/src/server/BreCal/notifications/notification_functions.py @@ -110,6 +110,6 @@ class Notifier(): def get_notification_states(self, evaluation_states_old, evaluation_states_new)->list[bool]: """# build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created""" - evaluation_notifications_sent = [self.notifier.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)] + evaluation_notifications_sent = [self.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)] return evaluation_notifications_sent diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index 33638fd..f346783 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -188,9 +188,9 @@ class ShipcallSchema(Schema): pass id = fields.Integer() - ship_id = fields.Integer() + ship_id = fields.Integer(metadata={'required':True}) #type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator - type = fields.Integer() # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator + type = fields.Integer(metadata={'required':True}) # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator eta = fields.DateTime(metadata={'required':False}, allow_none=True) voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} etd = fields.DateTime(metadata={'required':False}, allow_none=True) @@ -207,7 +207,7 @@ class ShipcallSchema(Schema): tidal_window_from = fields.DateTime(metadata={'required':False}, allow_none=True) tidal_window_to = fields.DateTime(metadata={'required':False}, allow_none=True) rain_sensitive_cargo = fields.Bool(metadata={'required':False}, allow_none=True) - recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True) + recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True, validate=[validate.Range(min=0, max=10, min_inclusive=True, max_inclusive=True)]) anchored = fields.Bool(metadata={'required':False}, allow_none=True) moored_lock = fields.Bool(metadata={'required':False}, allow_none=True) canceled = fields.Bool(metadata={'required':False}, allow_none=True) @@ -251,6 +251,9 @@ class Participant_Assignment: participant_id: int type: int # a variant would be to use the IntFlag type (with appropriate serialization) + def to_json(self): + return self.__dict__ + @dataclass class Shipcall: diff --git a/src/server/BreCal/stubs/shipcall.py b/src/server/BreCal/stubs/shipcall.py index 3f5c062..48c81f7 100644 --- a/src/server/BreCal/stubs/shipcall.py +++ b/src/server/BreCal/stubs/shipcall.py @@ -85,4 +85,22 @@ def get_shipcall_simple(): ) return shipcall +def create_postman_stub_shipcall(): + """ + this function returns the common stub, which is used to POST data to shipcalls via POSTMAN. However, + the stub-function is updated with a dynamic ETA in the future, so the POST-request does not fail. + """ + shipcall = { + 'ship_id': 1, + 'type': 1, + 'eta': (datetime.datetime.now()+datetime.timedelta(hours=3)).isoformat(), + 'voyage': '43B', + 'tug_required': False, + 'pilot_required': True, + 'flags': 0, + 'pier_side': False, + 'bunkering': True, + 'recommended_tugs': 2 + } + return shipcall diff --git a/src/server/BreCal/validators/input_validation.py b/src/server/BreCal/validators/input_validation.py index 2248779..2652f11 100644 --- a/src/server/BreCal/validators/input_validation.py +++ b/src/server/BreCal/validators/input_validation.py @@ -2,14 +2,31 @@ ####################################### InputValidation ####################################### import json +import datetime from abc import ABC, abstractmethod -from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant +from marshmallow import ValidationError +from string import ascii_letters, digits + +from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant, ShipcallType from BreCal.impl.participant import GetParticipant from BreCal.impl.ships import GetShips from BreCal.impl.berths import GetBerths from BreCal.database.enums import ParticipantType +def check_if_string_has_special_characters(text:str): + """ + check, whether there are any characters within the provided string, which are not found in the ascii letters or digits + ascii_letters: abcd (...) and ABCD (...) + digits: 0123 (...) + + Source: https://stackoverflow.com/questions/57062794/is-there-a-way-to-check-if-a-string-contains-special-characters + User: https://stackoverflow.com/users/10035985/andrej-kesely + returns bool + """ + return bool(set(text).difference(ascii_letters + digits)) + + def get_participant_id_dictionary(): # get all participants response,status_code,header = GetParticipant(options={}) @@ -61,7 +78,11 @@ def check_if_user_is_bsmd_type(user_data:dict)->bool: is_bsmd = ParticipantType.BSMD in ParticipantType(participant.get("type",0)) return is_bsmd -def check_if_user_data_has_valid_ship_id(ship_id): +def check_if_ship_id_is_valid(ship_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if ship_id is None: + return True + # build a dictionary of id:item pairs, so one can select the respective participant ships = get_ship_id_dictionary() @@ -69,7 +90,11 @@ def check_if_user_data_has_valid_ship_id(ship_id): ship_id_is_valid = ship_id in list(ships.keys()) return ship_id_is_valid -def check_if_user_data_has_valid_berth_id(berth_id): +def check_if_berth_id_is_valid(berth_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if berth_id is None: + return True + # build a dictionary of id:item pairs, so one can select the respective participant berths = get_berth_id_dictionary() @@ -77,7 +102,11 @@ def check_if_user_data_has_valid_berth_id(berth_id): berth_id_is_valid = berth_id in list(berths.keys()) return berth_id_is_valid -def check_if_user_data_has_valid_participant_id(participant_id): +def check_if_participant_id_is_valid(participant_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if participant_id is None: + return True + # build a dictionary of id:item pairs, so one can select the respective participant participants = get_participant_id_dictionary() @@ -85,6 +114,102 @@ def check_if_user_data_has_valid_participant_id(participant_id): participant_id_is_valid = participant_id in list(participants.keys()) return participant_id_is_valid +def check_if_participant_ids_are_valid(participant_ids): + # check each participant id individually + valid_participant_ids = [check_if_participant_id_is_valid(participant_id) for participant_id in participant_ids] + + # boolean check, whether all participant ids are valid + return all(valid_participant_ids) + + +def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict): + """this function applies more complex validation functions to data, which is sent to a post-request of shipcalls""" + # #TODO_refactor: this function is pretty complex. One may instead build an object, which calls the methods separately. + + import logging + logging.log(20, "dev") + logging.log(20, user_data) + logging.log(20, loadedModel) + logging.log(20, content) + ##### Section 1: check user_data ##### + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + + ##### Section 2: check loadedModel ##### + valid_ship_id = check_if_ship_id_is_valid(ship_id=loadedModel.get("ship_id", None)) + if not valid_ship_id: + raise ValidationError(f"provided an invalid ship id, which is not found in the database: {loadedModel.get('ship_id', None)}") + + valid_arrival_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("arrival_berth_id", None)) + if not valid_arrival_berth_id: + raise ValidationError(f"provided an invalid arrival berth id, which is not found in the database: {loadedModel.get('arrival_berth_id', None)}") + + valid_departure_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("departure_berth_id", None)) + if not valid_departure_berth_id: + raise ValidationError(f"provided an invalid departure berth id, which is not found in the database: {loadedModel.get('departure_berth_id', None)}") + + valid_participant_ids = check_if_participant_ids_are_valid(participant_ids=loadedModel.get("participants",[])) + if not valid_participant_ids: + raise ValidationError(f"one of the provided participant ids is invalid. Could not find one of these in the database: {loadedModel.get('participants', None)}") + + + ##### Section 3: check content ##### + # loadedModel fills missing values, sometimes using optional values. Hence, check content + + # the following keys should not be set in a POST-request. + for forbidden_key in ["canceled", "evaluation", "evaluation_message"]: + value = content.get(forbidden_key, None) + if value is not None: + raise ValidationError(f"'{forbidden_key}' may not be set on POST. Found: {value}") + + voyage_str_is_invalid = check_if_string_has_special_characters(text=content.get("voyage","")) + if voyage_str_is_invalid: + raise ValidationError(f"there are invalid characters in the 'voyage'-string. Please use only digits and ASCII letters. Allowed: {ascii_letters+digits}. Found: {content.get('voyage')}") + + + ##### Section 4: check loadedModel & content ##### + # #TODO_refactor: these methods should be placed in separate locations + + # existance checks in content + # datetime checks in loadedModel (datetime.datetime objects). Dates should be in the future. + time_now = datetime.datetime.now() + type_ = loadedModel.get("type", int(ShipcallType.undefined)) + if int(type_)==int(ShipcallType.undefined): + raise ValidationError(f"providing 'type' is mandatory. Missing key!") + elif int(type_)==int(ShipcallType.arrival): + eta = loadedModel.get("eta") + if (content.get("eta", None) is None): + raise ValidationError(f"providing 'eta' is mandatory. Missing key!") + if content.get("arrival_berth_id", None) is None: + raise ValidationError(f"providing 'arrival_berth_id' is mandatory. Missing key!") + if not eta >= time_now: + raise ValidationError(f"'eta' must be in the future. Incorrect datetime provided.") + elif int(type_)==int(ShipcallType.departure): + etd = loadedModel.get("etd") + if (content.get("etd", None) is None): + raise ValidationError(f"providing 'etd' is mandatory. Missing key!") + if content.get("departure_berth_id", None) is None: + raise ValidationError(f"providing 'departure_berth_id' is mandatory. Missing key!") + if not etd >= time_now: + raise ValidationError(f"'etd' must be in the future. Incorrect datetime provided.") + elif int(type_)==int(ShipcallType.shifting): + eta = loadedModel.get("eta") + etd = loadedModel.get("etd") + # * arrival_berth_id / departure_berth_id (depending on type, see above) + if (content.get("eta", None) is None) or (content.get("etd", None) is None): + raise ValidationError(f"providing 'eta' and 'etd' is mandatory. Missing one of those keys!") + if (content.get("arrival_berth_id", None) is None) or (content.get("departure_berth_id", None) is None): + raise ValidationError(f"providing 'arrival_berth_id' & 'departure_berth_id' is mandatory. Missing key!") + if (not eta >= time_now) or (not etd >= time_now) or (not eta >= etd): + raise ValidationError(f"'eta' and 'etd' must be in the future. Incorrect datetime provided.") + + + # #TODO: len of participants > 0, if agency + # * assigned participant for agency + return class InputValidation(): diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index 08944d4..14db51e 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -72,6 +72,7 @@ class ValidationRules(ValidationRuleFunctions): def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame: """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation', 'evaluation_message', 'evaluation_time' and 'evaluation_notifications_sent' are updated)""" evaluation_states_old = [state_old for state_old in shipcall_df.loc[:,"evaluation"]] + evaluation_states_old = [state_old if not pd.isna(state_old) else 0 for state_old in evaluation_states_old] results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values # returns tuple (state, message) # unbundle individual results. evaluation_states becomes an integer, violation @@ -80,14 +81,14 @@ class ValidationRules(ValidationRuleFunctions): violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] # build the list of evaluation times ('now', as isoformat) - evaluation_times = self.notifier.get_notification_times(evaluation_states_new) + evaluation_time = self.notifier.get_notification_times(evaluation_states_new) # build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created - evaluation_notifications_sent = self.get_notification_states(evaluation_states_old, evaluation_states_new) + evaluation_notifications_sent = self.notifier.get_notification_states(evaluation_states_old, evaluation_states_new) shipcall_df.loc[:,"evaluation"] = evaluation_states_new shipcall_df.loc[:,"evaluation_message"] = violations - shipcall_df.loc[:,"evaluation_times"] = evaluation_times + shipcall_df.loc[:,"evaluation_time"] = evaluation_time shipcall_df.loc[:,"evaluation_notifications_sent"] = evaluation_notifications_sent return shipcall_df From 6349e4a73cbd271f7ed6a35e37329548a9225409 Mon Sep 17 00:00:00 2001 From: Max Metz Date: Mon, 29 Apr 2024 18:50:46 +0200 Subject: [PATCH 13/30] implementing more input-validation-functions for shipcalls and ships. Beginning to refactor some of the validation functions into more readable Python classes. --- src/server/BreCal/api/shipcalls.py | 14 +++++--- src/server/BreCal/api/ships.py | 35 +++++++++++++++++-- src/server/BreCal/stubs/shipcall.py | 3 ++ .../BreCal/validators/input_validation.py | 20 +++++++---- .../validators/input_validation_shipcall.py | 0 5 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 src/server/BreCal/validators/input_validation_shipcall.py diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index c1c6e5d..e43f7ec 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -4,7 +4,7 @@ from marshmallow import Schema, fields, ValidationError from ..schemas import model from .. import impl from ..services.auth_guard import auth_guard, check_jwt -from BreCal.validators.input_validation import validate_posted_shipcall_data +from BreCal.validators.input_validation import validate_posted_shipcall_data, check_if_user_is_bsmd_type import logging import json @@ -41,9 +41,6 @@ def PostShipcalls(): try: content = request.get_json(force=True) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) - logging.log(20, loadedModel) - logging.log(20, "dev. above: loaded model, below: content") - logging.log(20, content) # read the user data from the JWT token (set when login is performed) user_data = check_jwt() @@ -72,6 +69,15 @@ def PutShipcalls(): content = request.get_json(force=True) logging.info(content) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") except ValidationError as ex: logging.error(ex) diff --git a/src/server/BreCal/api/ships.py b/src/server/BreCal/api/ships.py index e31147e..5fc3c50 100644 --- a/src/server/BreCal/api/ships.py +++ b/src/server/BreCal/api/ships.py @@ -1,11 +1,14 @@ from flask import Blueprint, request from .. import impl -from ..services.auth_guard import auth_guard -from marshmallow import EXCLUDE +from ..services.auth_guard import auth_guard, check_jwt +from marshmallow import EXCLUDE, ValidationError from ..schemas import model import json import logging +from BreCal.validators.input_validation import check_if_user_is_bsmd_type + + bp = Blueprint('ships', __name__) @bp.route('/ships', methods=['get']) @@ -24,6 +27,15 @@ def GetShips(): def PostShip(): try: + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + content = request.get_json(force=True) loadedModel = model.ShipSchema().load(data=content, many=False, partial=True) except Exception as ex: @@ -39,6 +51,15 @@ def PostShip(): def PutShip(): try: + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + content = request.get_json(force=True) loadedModel = model.ShipSchema().load(data=content, many=False, partial=True, unknown=EXCLUDE) except Exception as ex: @@ -53,8 +74,16 @@ def PutShip(): @auth_guard() # no restriction by role def DeleteShip(): - # TODO check if I am allowed to delete this thing by deriving the participant from the bearer token try: + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + if 'id' in request.args: options = {} options["id"] = request.args.get("id") diff --git a/src/server/BreCal/stubs/shipcall.py b/src/server/BreCal/stubs/shipcall.py index 48c81f7..9efab32 100644 --- a/src/server/BreCal/stubs/shipcall.py +++ b/src/server/BreCal/stubs/shipcall.py @@ -89,12 +89,15 @@ def create_postman_stub_shipcall(): """ this function returns the common stub, which is used to POST data to shipcalls via POSTMAN. However, the stub-function is updated with a dynamic ETA in the future, so the POST-request does not fail. + + Also provides a stub arrival_berth_id, so the POST-request succeeds. """ shipcall = { 'ship_id': 1, 'type': 1, 'eta': (datetime.datetime.now()+datetime.timedelta(hours=3)).isoformat(), 'voyage': '43B', + 'arrival_berth_id':142, 'tug_required': False, 'pilot_required': True, 'flags': 0, diff --git a/src/server/BreCal/validators/input_validation.py b/src/server/BreCal/validators/input_validation.py index 2652f11..048d594 100644 --- a/src/server/BreCal/validators/input_validation.py +++ b/src/server/BreCal/validators/input_validation.py @@ -26,6 +26,10 @@ def check_if_string_has_special_characters(text:str): """ return bool(set(text).difference(ascii_letters + digits)) +def check_if_int_is_valid_flag(value, enum_object): + # e.g., when an IntFlag has the values 1,2,4; the maximum valid value is 7 + max_int = sum([int(val) for val in list(enum_object._value2member_map_.values())]) + return 0 < value <= max_int def get_participant_id_dictionary(): # get all participants @@ -126,11 +130,6 @@ def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict """this function applies more complex validation functions to data, which is sent to a post-request of shipcalls""" # #TODO_refactor: this function is pretty complex. One may instead build an object, which calls the methods separately. - import logging - logging.log(20, "dev") - logging.log(20, user_data) - logging.log(20, loadedModel) - logging.log(20, content) ##### Section 1: check user_data ##### # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. @@ -205,7 +204,16 @@ def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict raise ValidationError(f"providing 'arrival_berth_id' & 'departure_berth_id' is mandatory. Missing key!") if (not eta >= time_now) or (not etd >= time_now) or (not eta >= etd): raise ValidationError(f"'eta' and 'etd' must be in the future. Incorrect datetime provided.") - + + tidal_window_from = loadedModel.get("tidal_window_from", None) + tidal_window_to = loadedModel.get("tidal_window_to", None) + if tidal_window_to is not None: + if not tidal_window_to >= time_now: + raise ValidationError(f"'tidal_window_to' must be in the future. Incorrect datetime provided.") + + if tidal_window_from is not None: + if not tidal_window_from >= time_now: + raise ValidationError(f"'tidal_window_from' must be in the future. Incorrect datetime provided.") # #TODO: len of participants > 0, if agency # * assigned participant for agency diff --git a/src/server/BreCal/validators/input_validation_shipcall.py b/src/server/BreCal/validators/input_validation_shipcall.py new file mode 100644 index 0000000..e69de29 From 56628a3c45996b92405ba1916cb67ef6a39faa5d Mon Sep 17 00:00:00 2001 From: scopesorting Date: Fri, 19 Jan 2024 17:33:43 +0100 Subject: [PATCH 14/30] slight adjustments to prepare the authentification validation --- src/server/BreCal/api/shipcalls.py | 11 ++++++++++- src/server/BreCal/database/enums.py | 5 ++++- src/server/BreCal/local_db.py | 4 ++-- src/server/BreCal/schemas/model.py | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 396dc93..73a06c1 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -14,7 +14,16 @@ bp = Blueprint('shipcalls', __name__) @auth_guard() # no restriction by role def GetShipcalls(): if 'Authorization' in request.headers: - token = request.headers.get('Authorization') + token = request.headers.get('Authorization') # see impl/login to see the token encoding, which is a JWT token. + + """ + from BreCal.services.jwt_handler import decode_jwt + jwt = token.split('Bearer ')[1] # string key + payload = decode_jwt(jwt) # dictionary, which includes 'id' (user id) and 'participant_id' + + # oneline: + payload = decode_jwt(request.headers.get("Authorization").split("Bearer ")[-1]) + """ options = {} options["participant_id"] = request.args.get("participant_id") options["past_days"] = request.args.get("past_days", default=1, type=int) diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index 3092fd8..fe6b37f 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -1,4 +1,4 @@ -from enum import Enum, IntFlag +from enum import IntEnum, Enum, IntFlag class ParticipantType(IntFlag): """determines the type of a participant""" @@ -36,3 +36,6 @@ class StatusFlags(Enum): YELLOW = 2 RED = 3 +class PierSide(IntEnum): + PORTSIDE = 0 # Port/Backbord + STARBOARD_SIDE = 1 # Starboard / Steuerbord diff --git a/src/server/BreCal/local_db.py b/src/server/BreCal/local_db.py index 85dff68..4293ff3 100644 --- a/src/server/BreCal/local_db.py +++ b/src/server/BreCal/local_db.py @@ -16,7 +16,7 @@ def initPool(instancePath): print (config_path) if not os.path.exists(config_path): - print ('cannot find ' + config_path) + print ('cannot find ' + os.path.abspath(config_path)) print("instance path", instancePath) sys.exit(1) @@ -39,4 +39,4 @@ def getPoolConnection(): global config_path f = open(config_path); connection_data = json.load(f) - return mysql.connector.connect(**connection_data) \ No newline at end of file + return mysql.connector.connect(**connection_data) diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index bf157aa..f45bc11 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -230,7 +230,7 @@ class Shipcall: tug_required: bool pilot_required: bool flags: int - pier_side: bool + pier_side: bool # enumerator object in database/enum/PierSide bunkering: bool replenishing_terminal: bool replenishing_lock: bool From 9b0a08551064937c1b13bd0e208a1e6401e757a8 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Thu, 7 Dec 2023 12:01:41 +0100 Subject: [PATCH 15/30] adapting shipcall, times and user to include ValidationError (marshmallow). Adjusting the Schemas for User, Times and Shipcall to be validated with additional input validators. Creating a set of tests for the input validations. --- brecal.code-workspace | 8 +++ src/server/BreCal/api/shipcalls.py | 14 ++++- src/server/BreCal/api/times.py | 11 ++++ src/server/BreCal/api/user.py | 6 ++ src/server/BreCal/schemas/model.py | 73 ++++++++++++++--------- src/server/tests/schemas/test_model.py | 80 ++++++++++++++++++++++++++ 6 files changed, 165 insertions(+), 27 deletions(-) create mode 100644 brecal.code-workspace create mode 100644 src/server/tests/schemas/test_model.py diff --git a/brecal.code-workspace b/brecal.code-workspace new file mode 100644 index 0000000..876a149 --- /dev/null +++ b/brecal.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 73a06c1..fbf552a 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -1,6 +1,6 @@ from flask import Blueprint, request from webargs.flaskparser import parser -from marshmallow import Schema, fields +from marshmallow import Schema, fields, ValidationError from ..schemas import model from .. import impl from ..services.auth_guard import auth_guard @@ -40,6 +40,12 @@ def PostShipcalls(): try: content = request.get_json(force=True) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 + except Exception as ex: logging.error(ex) print(ex) @@ -56,6 +62,12 @@ def PutShipcalls(): content = request.get_json(force=True) logging.info(content) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 + except Exception as ex: logging.error(ex) print(ex) diff --git a/src/server/BreCal/api/times.py b/src/server/BreCal/api/times.py index 2c90397..a333064 100644 --- a/src/server/BreCal/api/times.py +++ b/src/server/BreCal/api/times.py @@ -4,6 +4,7 @@ from .. import impl from ..services.auth_guard import auth_guard import json import logging +from marshmallow import ValidationError bp = Blueprint('times', __name__) @@ -29,6 +30,11 @@ def PostTimes(): # print (content) # body = parser.parse(schema, request, location='json') loadedModel = model.TimesSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 except Exception as ex: logging.error(ex) @@ -45,6 +51,11 @@ def PutTimes(): try: content = request.get_json(force=True) loadedModel = model.TimesSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 except Exception as ex: logging.error(ex) diff --git a/src/server/BreCal/api/user.py b/src/server/BreCal/api/user.py index bbd5b4b..2c3c1a0 100644 --- a/src/server/BreCal/api/user.py +++ b/src/server/BreCal/api/user.py @@ -4,6 +4,7 @@ from .. import impl from ..services.auth_guard import auth_guard import json import logging +from marshmallow import ValidationError bp = Blueprint('user', __name__) @@ -14,6 +15,11 @@ def PutUser(): try: content = request.get_json(force=True) loadedModel = model.UserSchema().load(data=content, many=False, partial=True) + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 except Exception as ex: logging.error(ex) diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index f45bc11..f6957cc 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -1,5 +1,5 @@ from dataclasses import field, dataclass -from marshmallow import Schema, fields, post_load, INCLUDE, ValidationError +from marshmallow import Schema, fields, INCLUDE, ValidationError, validate, validates from marshmallow.fields import Field from marshmallow_enum import EnumField from enum import IntEnum @@ -9,6 +9,7 @@ from typing import List import json import datetime +from BreCal.validators.time_logic import validate_time_exceeds_threshold def obj_dict(obj): if isinstance(obj, datetime.datetime): @@ -152,34 +153,34 @@ class ParticipantList(Participant): pass class ParticipantAssignmentSchema(Schema): - participant_id = fields.Int() - type = fields.Int() + participant_id = fields.Integer() + type = fields.Integer() class ShipcallSchema(Schema): def __init__(self): super().__init__(unknown=None) pass - id = fields.Int() - ship_id = fields.Int() - type = fields.Enum(ShipcallType, required=True) + id = fields.Integer() + ship_id = fields.Integer() + type = fields.Integer() eta = fields.DateTime(Required = False, allow_none=True) - voyage = fields.Str(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} + voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} etd = fields.DateTime(Required = False, allow_none=True) - arrival_berth_id = fields.Int(Required = False, allow_none=True) - departure_berth_id = fields.Int(Required = False, allow_none=True) + arrival_berth_id = fields.Integer(Required = False, allow_none=True) + departure_berth_id = fields.Integer(Required = False, allow_none=True) tug_required = fields.Bool(Required = False, allow_none=True) pilot_required = fields.Bool(Required = False, allow_none=True) - flags = fields.Int(Required = False, allow_none=True) + flags = fields.Integer(Required = False, allow_none=True) pier_side = fields.Bool(Required = False, allow_none=True) bunkering = fields.Bool(Required = False, allow_none=True) replenishing_terminal = fields.Bool(Required = False, allow_none=True) replenishing_lock = fields.Bool(Required = False, allow_none=True) - draft = fields.Float(Required = False, allow_none=True) + draft = fields.Float(Required = False, allow_none=True, validate=[validate.Range(min=0, max=20, min_inclusive=False, max_inclusive=True)]) tidal_window_from = fields.DateTime(Required = False, allow_none=True) tidal_window_to = fields.DateTime(Required = False, allow_none=True) rain_sensitive_cargo = fields.Bool(Required = False, allow_none=True) - recommended_tugs = fields.Int(Required = False, allow_none=True) + recommended_tugs = fields.Integer(Required = False, allow_none=True) anchored = fields.Bool(Required = False, allow_none=True) moored_lock = fields.Bool(Required = False, allow_none=True) canceled = fields.Bool(Required = False, allow_none=True) @@ -297,12 +298,13 @@ class ShipcallId(Schema): # this is the way! + class TimesSchema(Schema): def __init__(self): super().__init__(unknown=None) pass - id = fields.Int(Required=False) + id = fields.Integer(Required=False) eta_berth = fields.DateTime(Required = False, allow_none=True) eta_berth_fixed = fields.Bool(Required = False, allow_none=True) etd_berth = fields.DateTime(Required = False, allow_none=True) @@ -313,13 +315,13 @@ class TimesSchema(Schema): zone_entry_fixed = fields.Bool(Required = False, allow_none=True) operations_start = fields.DateTime(Required = False, allow_none=True) operations_end = fields.DateTime(Required = False, allow_none=True) - remarks = fields.String(Required = False, allow_none=True) - participant_id = fields.Int(Required = True) - berth_id = fields.Int(Required = False, allow_none = True) - berth_info = fields.String(Required = False, allow_none=True) + remarks = fields.String(Required = False, allow_none=True, validate=[validate.Length(max=256)]) + participant_id = fields.Integer(Required = True) + berth_id = fields.Integer(Required = False, allow_none = True) + berth_info = fields.String(Required = False, allow_none=True, validate=[validate.Length(max=256)]) pier_side = fields.Bool(Required = False, allow_none = True) - shipcall_id = fields.Int(Required = True) - participant_type = fields.Int(Required = False, allow_none=True) + shipcall_id = fields.Integer(Required = True) + participant_type = fields.Integer(Required = False, allow_none=True) ata = fields.DateTime(Required = False, allow_none=True) atd = fields.DateTime(Required = False, allow_none=True) eta_interval_end = fields.DateTime(Required = False, allow_none=True) @@ -327,19 +329,38 @@ class TimesSchema(Schema): created = fields.DateTime(Required = False, allow_none=True) modified = fields.DateTime(Required = False, allow_none=True) + @validates("eta_berth") + def validate_eta_berth(self, value): + threshold_exceeded = validate_time_exceeds_threshold(value, months=12) + print(threshold_exceeded, value) + if threshold_exceeded: + raise ValidationError(f"the provided time exceeds the twelve month threshold.") + # deserialize PUT object target class UserSchema(Schema): def __init__(self): super().__init__(unknown=None) pass - id = fields.Int(required=True) - first_name = fields.Str(allow_none=True, metadata={'Required':False}) - last_name = fields.Str(allow_none=True, metadata={'Required':False}) - user_phone = fields.Str(allow_none=True, metadata={'Required':False}) - user_email = fields.Str(allow_none=True, metadata={'Required':False}) - old_password = fields.Str(allow_none=True, metadata={'Required':False}) - new_password = fields.Str(allow_none=True, metadata={'Required':False}) + id = fields.Integer(required=True) + first_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) + last_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) + user_phone = fields.String(allow_none=True, metadata={'Required':False}) + user_email = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) + old_password = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=128)]) + new_password = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(min=6, max=128)]) + + @validates("user_phone") + def validate_user_phone(self, value): + valid_characters = list(map(str,range(0,10)))+["+", " "] + if not all([v in valid_characters for v in value]): + raise ValidationError(f"one of the phone number values is not valid.") + + @validates("user_email") + def validate_user_email(self, value): + if not "@" in value: + raise ValidationError(f"invalid email address") + @dataclass class Times: diff --git a/src/server/tests/schemas/test_model.py b/src/server/tests/schemas/test_model.py new file mode 100644 index 0000000..8944f45 --- /dev/null +++ b/src/server/tests/schemas/test_model.py @@ -0,0 +1,80 @@ +from marshmallow import ValidationError +import pytest +from BreCal.schemas.model import ShipcallSchema + + +@pytest.fixture(scope="function") # function: destroy fixture at the end of each test +def prepare_shipcall_content(): + import datetime + from BreCal.stubs.shipcall import get_shipcall_simple + shipcall_stub = get_shipcall_simple() + content = shipcall_stub.__dict__ + content["participants"] = [] + content = {k:v.isoformat() if isinstance(v, datetime.datetime) else v for k,v in content.items()} + return locals() + +def test_shipcall_input_validation_draft(prepare_shipcall_content): + content = prepare_shipcall_content["content"] + content["draft"] = 24.11 + + schemaModel = ShipcallSchema() + with pytest.raises(ValidationError, match="Must be greater than 0 and less than or equal to 20."): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + +def test_shipcall_input_validation_voyage(prepare_shipcall_content): + content = prepare_shipcall_content["content"] + content["voyage"] = "".join(list(map(str,list(range(0,24))))) # 38 characters + + schemaModel = ShipcallSchema() + with pytest.raises(ValidationError, match="Longer than maximum length "): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + + +@pytest.fixture(scope="function") # function: destroy fixture at the end of each test +def prepare_user_content(): + import datetime + from BreCal.stubs.user import get_user_simple + from BreCal.schemas.model import UserSchema + schemaModel = UserSchema() + + user_stub = get_user_simple() + content = user_stub.__dict__ + content = {k:v.isoformat() if isinstance(v, datetime.datetime) else v for k,v in content.items()} + content = {k:v for k,v in content.items() if k in list(schemaModel.fields.keys())} + content["old_password"] = "myfavoritedog123" + content["new_password"] = "SecuRepassW0rd!" + return locals() + + +def test_input_validation_berth_phone_number_is_valid(prepare_user_content): + content, schemaModel = prepare_user_content["content"], prepare_user_content["schemaModel"] + content["user_phone"] = "+49123 45678912" # whitespace and + are valid + + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + +def test_input_validation_berth_phone_number_is_invalid(prepare_user_content): + content, schemaModel = prepare_user_content["content"], prepare_user_content["schemaModel"] + content["user_phone"] = "+49123 45678912!" # ! is invalid + + with pytest.raises(ValidationError, match="one of the phone number values is not valid."): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + +def test_input_validation_new_password_too_short(prepare_user_content): + content, schemaModel = prepare_user_content["content"], prepare_user_content["schemaModel"] + content["new_password"] = "1234" # must have between 6 and 128 characters + + with pytest.raises(ValidationError, match="Length must be between 6 and 128."): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return + +def test_input_validation_user_email_invalid(prepare_user_content): + content, schemaModel = prepare_user_content["content"], prepare_user_content["schemaModel"] + content["user_email"] = "userbrecal.com" # forgot @ -> invalid + + with pytest.raises(ValidationError, match="invalid email address"): + loadedModel = schemaModel.load(data=content, many=False, partial=True) + return From 06bad205dece467d5e9ebf080fd2ae298b3495a8 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Tue, 12 Dec 2023 17:07:09 +0100 Subject: [PATCH 16/30] partial commit of integrating the input validation (references and mandatory fields) --- src/server/BreCal/__init__.py | 1 + src/server/BreCal/database/update_database.py | 1 - src/server/BreCal/impl/shipcalls.py | 22 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/server/BreCal/__init__.py b/src/server/BreCal/__init__.py index eb832f5..1f37ffc 100644 --- a/src/server/BreCal/__init__.py +++ b/src/server/BreCal/__init__.py @@ -46,6 +46,7 @@ def create_app(test_config=None): app.config.from_mapping(test_config) try: + import os print(f'Instance path = {app.instance_path}') os.makedirs(app.instance_path) except OSError: diff --git a/src/server/BreCal/database/update_database.py b/src/server/BreCal/database/update_database.py index 0e5e5ac..7b7639b 100644 --- a/src/server/BreCal/database/update_database.py +++ b/src/server/BreCal/database/update_database.py @@ -68,7 +68,6 @@ def evaluate_shipcall_state(mysql_connector_instance, shipcall_id:int=None, debu with mysql.connector.connect(**mysql_connection_data) as mysql_connector_instance: evaluate_shipcall_state(mysql_connector_instance) returns None - """ sql_handler = SQLHandler(sql_connection=mysql_connector_instance, read_all=True) vr = ValidationRules(sql_handler) diff --git a/src/server/BreCal/impl/shipcalls.py b/src/server/BreCal/impl/shipcalls.py index 59f9311..8d15983 100644 --- a/src/server/BreCal/impl/shipcalls.py +++ b/src/server/BreCal/impl/shipcalls.py @@ -123,6 +123,17 @@ def PostShipcalls(schemaModel): query += "?" + param_key + "?" query += ")" + # #TODO: enter completeness validation here. Only shipcalls, where all required fields are filled in are valid + # "Pflichtfelder: z.B. die Felder bei der Neuanlage eines shipcalls müssen entsprechend des Anlauf-Typs belegt sein, damit gespeichert werden kann" - Daniel Schick + + # #TODO: enter role validation here. Only users of correct type should be allowed to post a shipcall + + # #TODO: enter reference validation here + # full_id_existances = check_id_existances(mysql_connector_instance=sql_connection, schemaModel=schemaModel, debug=False) + # if not full_id_existances: + # pooledConnection.close() + # return json.dumps({"message" : "call failed. missing mandatory keywords."}), 500, {'Content-Type': 'application/json; charset=utf-8'} + commands.execute(query, schemaModel) new_id = commands.execute_scalar("select last_insert_id()") @@ -206,6 +217,17 @@ def PutShipcalls(schemaModel): query += key + " = ?" + param_key + "? " query += "WHERE id = ?id?" + + # #TODO: enter role validation here. + + # #TODO: enter completeness validation here. Only shipcalls, where all required fields are filled in are valid + + + # #TODO: enter reference validation here + # full_id_existances = check_id_existances(mysql_connector_instance=sql_connection, schemaModel=schemaModel, debug=False) + # if not full_id_existances: + # return ... (500) + affected_rows = commands.execute(query, param=schemaModel) pquery = "SELECT id, participant_id, type FROM shipcall_participant_map where shipcall_id = ?id?" From f3818a1b2feea4d78b7a3de4c3bbc353bf77f5c3 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Fri, 15 Dec 2023 17:37:27 +0100 Subject: [PATCH 17/30] implementing an EmailHandler to send out emails (potentially with attachment). --- src/server/BreCal/services/email_handling.py | 174 +++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 src/server/BreCal/services/email_handling.py diff --git a/src/server/BreCal/services/email_handling.py b/src/server/BreCal/services/email_handling.py new file mode 100644 index 0000000..e06021d --- /dev/null +++ b/src/server/BreCal/services/email_handling.py @@ -0,0 +1,174 @@ +import os +import typing +import smtplib +from getpass import getpass +from email.message import EmailMessage +import mimetypes + +import email +# from email.mime.base import MIMEBase +# from email.mime.image import MIMEImage +# from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +from email.mime.application import MIMEApplication + + +class EmailHandler(): + """ + Creates an EmailHandler, which is capable of connecting to a mail server at a respective port, + as well as logging into a specific user's mail address. + Upon creating messages, these can be sent via this handler. + + Options: + mail_server: address of the server, such as 'smtp.gmail.com' or 'w01d5503.kasserver.com + mail_port: + 25 - SMTP Port, to send emails + 110 - POP3 Port, to receive emails + 143 - IMAP Port, to receive from IMAP + 465 - SSL Port of SMTP + 587 - alternative SMTP Port + 993 - SSL/TLS-Port of IMAP + 995 - SSL/TLS-Port of POP3 + mail_address: a specific user's Email address, which will be used to send Emails. Example: "my_user@gmail.com" + """ + def __init__(self, mail_server:str, mail_port:int, mail_address:str): + self.mail_server = mail_server + self.mail_port = mail_port + self.mail_address = mail_address + + self.server = smtplib.SMTP_SSL(self.mail_server, self.mail_port) # alternatively, SMTP + + def check_state(self): + """check, whether the server login took place and is open.""" + try: + (status_code, status_msg) = self.server.noop() + return status_code==250 # 250: b'2.0.0 Ok' + except smtplib.SMTPServerDisconnected: + return False + + def check_connection(self): + """check, whether the server object is connected to the server. If not, connect it. """ + try: + self.server.ehlo() + except smtplib.SMTPServerDisconnected: + self.server.connect(self.mail_server, self.mail_port) + return + + def check_login(self)->bool: + """check, whether the server object is logged in as a user""" + user = self.server.__dict__.get("user",None) + return user is not None + + def login(self, interactive:bool=True): + """ + login on the determined mail server's mail address. By default, this function opens an interactive window to + type the password without echoing (printing '*******' instead of readable characters). + + returns (status_code, status_msg) + """ + self.check_connection() + if interactive: + (status_code, status_msg) = self.server.login(self.mail_address, password=getpass()) + else: + # fernet + password file + raise NotImplementedError() + return (status_code, status_msg) # should be: (235, b'2.7.0 Authentication successful') + + def create_email(self, subject:str, message_body:str)->EmailMessage: + """ + Create an EmailMessage object, which contains the Email's header ("Subject"), content ("Message Body") and the sender's address ("From"). + The EmailMessage object does not contain the recipients yet, as these will be defined upon sending the Email. + """ + msg = EmailMessage() + msg["Subject"] = subject + msg["From"] = self.mail_address + #msg["To"] = email_tgts # will be defined in self.send_email + msg.set_content(message_body) + return msg + + def build_recipients(self, email_tgts:list[str]): + """ + email formatting does not support lists. Instead, items are joined into a comma-space-separated string. + Example: + [mail1@mail.com, mail2@mail.com] becomes + 'mail1@mail.com, mail2@mail.com' + """ + return ', '.join(email_tgts) + + def open_mime_application(self, path:str)->MIMEApplication: + """open a local file, read the bytes into a MIMEApplication object, which is built with the proper subtype (based on the file extension)""" + with open(path, 'rb') as file: + attachment = MIMEApplication(file.read(), _subtype=mimetypes.MimeTypes().guess_type(path)) + + attachment.add_header('Content-Disposition','attachment',filename=str(os.path.basename(path))) + return attachment + + def attach_file(self, path:str, msg:email.mime.multipart.MIMEMultipart)->None: + """ + attach a file to the message. This function opens the file, reads its bytes, defines the mime type by the + path extension. The filename is appended as the header. + + mimetypes: # https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types + """ + attachment = self.open_mime_application(path) + msg.attach(attachment) + return + + def send_email(self, msg:EmailMessage, email_tgts:list[str], cc_tgts:typing.Optional[list[str]]=None, bcc_tgts:typing.Optional[list[str]]=None, debug:bool=False)->typing.Union[dict,EmailMessage]: + """ + send a prepared email message to recipients (email_tgts), copy (cc_tgts) and blind copy (bcc_tgts). + Returns a dictionary of feedback, which is commonly empty and the EmailMessage. + + When failing, this function returns an SMTP error instead of returning the default outputs. + """ + # Set the Recipients + msg["To"] = self.build_recipients(email_tgts) + + # optionally, add CC and BCC (copy and blind-copy) + if cc_tgts is not None: + msg["Cc"] = self.build_recipients(cc_tgts) + if bcc_tgts is not None: + msg["Bcc"] = self.build_recipients(bcc_tgts) + + # when debugging, do not send the Email, but return the EmailMessage. + if debug: + return {}, msg + + assert self.check_login(), f"currently not logged in. Cannot send an Email. Make sure to properly use self.login first. " + # send the prepared EmailMessage via the server. + feedback = self.server.send_message(msg) + return feedback, msg + + def translate_mail_to_multipart(self, msg:EmailMessage): + """EmailMessage does not support HTML and attachments. Hence, one can convert an EmailMessage object.""" + if msg.is_multipart(): + return msg + + # create a novel MIMEMultipart email + msg_new = MIMEMultipart("mixed") + headers = list((k, v) for (k, v) in msg.items() if k not in ("Content-Type", "Content-Transfer-Encoding")) + + # add the headers of msg to the new message + for k,v in headers: + msg_new[k] = v + + # delete the headers from msg + for k,v in headers: + del msg[k] + + # attach the remainder of the msg, such as the body, to the MIMEMultipart + msg_new.attach(msg) + return msg_new + + def print_email_attachments(self, msg:MIMEMultipart)->list[str]: + """return a list of lines of an Email, which contain 'filename=' as a list. """ + return [line_ for line_ in msg.as_string().split("\n") if "filename=" in line_] + + def close(self): + self.server.__dict__.pop("user",None) + self.server.__dict__.pop("password",None) + + # quit the server connection (internally uses .close) + self.server.quit() + return + From 73d13d4d6236bf0ecbaddfae0e98ad47d62807f8 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Fri, 19 Jan 2024 14:22:54 +0100 Subject: [PATCH 18/30] implementing notifications, working on input validation. rebase. --- src/server/BreCal/database/enums.py | 9 ++ src/server/BreCal/notifications/__init__.py | 0 .../notifications/notification_functions.py | 115 ++++++++++++++++++ src/server/BreCal/schemas/model.py | 9 ++ .../BreCal/services/schedule_routines.py | 5 + src/server/BreCal/stubs/shipcall.py | 4 +- .../BreCal/validators/validation_rules.py | 98 +++++++-------- 7 files changed, 182 insertions(+), 58 deletions(-) create mode 100644 src/server/BreCal/notifications/__init__.py create mode 100644 src/server/BreCal/notifications/notification_functions.py diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index fe6b37f..7f8fc3d 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -26,6 +26,8 @@ class ParticipantwiseTimeDelta(): TUG = 960.0 # 16 h * 60 min/h = 960 min TERMINAL = 960.0 # 16 h * 60 min/h = 960 min + NOTIFICATION = 10.0 # after n minutes, an evaluation may rise a notification + class StatusFlags(Enum): """ these enumerators ensure that each traffic light validation rule state corresponds to a value, which will be used in the ValidationRules object to identify @@ -39,3 +41,10 @@ class StatusFlags(Enum): class PierSide(IntEnum): PORTSIDE = 0 # Port/Backbord STARBOARD_SIDE = 1 # Starboard / Steuerbord + +class NotificationType(IntFlag): + """determines the method by which a notification is distributed to users. Flagging allows selecting multiple notification types.""" + UNDEFINED = 0 + EMAIL = 1 + POPUP = 2 + MESSENGER = 4 diff --git a/src/server/BreCal/notifications/__init__.py b/src/server/BreCal/notifications/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/server/BreCal/notifications/notification_functions.py b/src/server/BreCal/notifications/notification_functions.py new file mode 100644 index 0000000..9bbc373 --- /dev/null +++ b/src/server/BreCal/notifications/notification_functions.py @@ -0,0 +1,115 @@ +import datetime +import pandas as pd +from BreCal.schemas.model import Notification +from BreCal.database.enums import NotificationType, ParticipantType, ShipcallType, StatusFlags + +def create_notification(id, times_id, message, level, notification_type:NotificationType, created=None, modified=None): + created = (datetime.datetime.now()).isoformat() or created + + notification = Notification( + id=id, + times_id=times_id, acknowledged=False, level=level, type=notification_type.value, message=message, created=created, modified=modified + ) + return notification + + + + + +#### Verbosity Functions #### + +def get_default_header()->str: + # HEADER (greeting and default message) + header = "Dear Sir or Madam\n\nThank you for participating in the project 'Bremen Calling'. During analysis, our software has identified an event, which may be worth a second look. Here is the summary. \n\n" + return header + +def get_default_footer()->str: + # FOOTER (signature) + footer = "\n\nWe would kindly ask you to have a look at the shipcall and verify, if any action is required from your side. \n\nKind regards\nThe 'Bremen Calling' Team" + return footer + +def get_agency_name(sql_handler, times_df): + times_agency = times_df.loc[times_df["participant_type"]==ParticipantType.AGENCY.value,"participant_id"] + if len(times_agency)==0: + agency_name = "" + else: + agency_participant_id = times_agency.iloc[0] + agency_name = sql_handler.df_dict.get("participant").loc[agency_participant_id,"name"] + return agency_name + +def get_ship_name(sql_handler, shipcall): + ship = sql_handler.df_dict.get("ship").loc[shipcall.ship_id] + ship_name = ship.loc["name"] # when calling ship.name, the ID is returned (pandas syntax) + return ship_name + + +def create_notification_body(sql_handler, times_df, shipcall, result)->str: + # #TODO: add 'Link zum Anlauf' + # URL: https://trello.com/c/qenZyJxR/75-als-bsmd-m%C3%B6chte-ich-%C3%BCber-gelbe-und-rote-ampeln-informiert-werden-um-die-systembeteiligung-zu-st%C3%A4rken + header = get_default_header() + footer = get_default_footer() + + agency_name = get_agency_name(sql_handler, times_df) + ship_name = get_ship_name(sql_handler, shipcall) + + verbosity_introduction = f"Respective Shipcall:\n" + traffic_state_verbosity = f"\tTraffic Light State: {StatusFlags(result[0]).name}\n" + ship_name_verbosity = f"\tShip: {ship_name} (the ship is {ShipcallType(shipcall.type).name.lower()})\n" + agency_name_verbosity = f"\tResponsible Agency: {agency_name}\n" + eta_verbosity = f"\tEstimated Arrival Time: {shipcall.eta.isoformat()}\n" if not pd.isna(shipcall.eta) else "" + etd_verbosity = f"\tEstimated Departure Time: {shipcall.etd.isoformat()}\n" if not pd.isna(shipcall.etd) else "" + error_verbosity = f"\nError Description:\n\t" + "\n\t".join(result[1]) + + message_body = "".join([header, verbosity_introduction, traffic_state_verbosity, ship_name_verbosity, agency_name_verbosity, eta_verbosity, etd_verbosity, error_verbosity, footer]) + return message_body + + +class Notifier(): + """An object that helps with the logic of selecting eligible shipcalls to create the correct notifications for the respective users.""" + def __init__(self)->None: + pass + + def determine_notification_state(self, state_old, state_new): + """ + this method determines state changes in the notification state. When the state increases, a user is notified about it. + state order: (NONE = GREEN < YELLOW < RED) + """ + # identify a state increase + should_notify = self.identify_notification_state_change(state_old=state_old, state_new=state_new) + + # when a state increases, a notification must be sent. Thereby, the field should be set to False ({evaluation_notifications_sent}) + evaluation_notifications_sent = False if bool(should_notify) else None + return evaluation_notifications_sent + + def identify_notification_state_change(self, state_old, state_new) -> bool: + """ + determines, whether the observed state change should trigger a notification. + internally, this function maps StatusFlags to an integer and determines, if the successor state is more severe than the predecessor. + + state changes trigger a notification in the following cases: + green -> yellow + green -> red + yellow -> red + + (none -> yellow) or (none -> red) + due to the values in the enumeration objects, the states are mapped to provide this function. + green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than". + + returns bool, whether a notification should be triggered + """ + # state_old is always considered at least 'Green' (1) + if state_old is None: + state_old = StatusFlags.NONE.value + state_old = max(int(state_old), StatusFlags.GREEN.value) + return int(state_new) > int(state_old) + + def get_notification_times(self, evaluation_states_new)->list[datetime.datetime]: + """# build the list of evaluation times ('now', as isoformat)""" + evaluation_times = [datetime.datetime.now().isoformat() for _i in range(len(evaluation_states_new))] + return evaluation_times + + def get_notification_states(self, evaluation_states_old, evaluation_states_new)->list[bool]: + """# build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created""" + evaluation_notifications_sent = [self.notifier.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)] + return evaluation_notifications_sent + diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index f6957cc..c429777 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -184,11 +184,18 @@ class ShipcallSchema(Schema): anchored = fields.Bool(Required = False, allow_none=True) moored_lock = fields.Bool(Required = False, allow_none=True) canceled = fields.Bool(Required = False, allow_none=True) +<<<<<<< HEAD evaluation = fields.Enum(EvaluationType, required=False, allow_none=True, default=EvaluationType.undefined) evaluation_message = fields.Str(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} evaluation_time = fields.DateTime(Required = False, allow_none=True) evaluation_notifications_sent = fields.Bool(Required = False, allow_none=True) time_ref_point = fields.Int(Required = False, allow_none=True) +======= + evaluation = fields.Integer(Required = False, allow_none=True) + evaluation_message = fields.String(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} + evaluation_time = fields.DateTime(Required = False, allow_none=True) + evaluation_notifications_sent = fields.Bool(Required = False, allow_none=True) +>>>>>>> a5284a4 (implementing notifications, working on input validation) participants = fields.List(fields.Nested(ParticipantAssignmentSchema)) created = fields.DateTime(Required = False, allow_none=True) modified = fields.DateTime(Required = False, allow_none=True) @@ -251,6 +258,8 @@ class Shipcall: created: datetime modified: datetime participants: List[Participant_Assignment] = field(default_factory=list) + evaluation_time : datetime = None + evaluation_notifications_sent : bool = None def to_json(self): return { diff --git a/src/server/BreCal/services/schedule_routines.py b/src/server/BreCal/services/schedule_routines.py index 542b213..848364d 100644 --- a/src/server/BreCal/services/schedule_routines.py +++ b/src/server/BreCal/services/schedule_routines.py @@ -57,6 +57,11 @@ def add_function_to_schedule__update_shipcalls(interval_in_minutes:int, options: schedule.every(interval_in_minutes).minutes.do(UpdateShipcalls, **kwargs_) return +def add_function_to_schedule__send_notifications(vr, interval_in_minutes:int=10): + schedule.every(interval_in_minutes).minutes.do(vr.notifier.send_notifications) + return + + def setup_schedule(update_shipcalls_interval_in_minutes:int=60): logging.getLogger('schedule').setLevel(logging.INFO); # set the logging level of the schedule module to INFO diff --git a/src/server/BreCal/stubs/shipcall.py b/src/server/BreCal/stubs/shipcall.py index e86d379..2e4e154 100644 --- a/src/server/BreCal/stubs/shipcall.py +++ b/src/server/BreCal/stubs/shipcall.py @@ -37,12 +37,12 @@ def get_shipcall_simple(): recommended_tugs = 2 # assert 0pd.DataFrame: - """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation' and 'evaluation_message' are updated)""" - results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values + """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation', 'evaluation_message', 'evaluation_time' and 'evaluation_notifications_sent' are updated)""" + evaluation_states_old = [state_old for state_old in shipcall_df.loc[:,"evaluation"]] + results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values # returns tuple (state, message) - # unbundle individual results. evaluation_state becomes an integer, violation - evaluation_state = [StatusFlags(res[0]).value for res in results] + # unbundle individual results. evaluation_states becomes an integer, violation + evaluation_states_new = [StatusFlags(res[0]).value for res in results] violations = [",\r\n".join(res[1]) if len(res[1])>0 else None for res in results] violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] - shipcall_df.loc[:,"evaluation"] = evaluation_state + # build the list of evaluation times ('now', as isoformat) + evaluation_times = self.notifier.get_notification_times(evaluation_states_new) + + # build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created + evaluation_notifications_sent = self.get_notification_states(evaluation_states_old, evaluation_states_new) + + shipcall_df.loc[:,"evaluation"] = evaluation_states_new shipcall_df.loc[:,"evaluation_message"] = violations + shipcall_df.loc[:,"evaluation_times"] = evaluation_times + shipcall_df.loc[:,"evaluation_notifications_sent"] = evaluation_notifications_sent return shipcall_df def concise_evaluation_message_if_too_long(self, violation): @@ -100,53 +108,31 @@ class ValidationRules(ValidationRuleFunctions): # e.g.: Evaluation message too long. Violated Rules: ['Rule #0001C', 'Rule #0001H', 'Rule #0001F', 'Rule #0001G', 'Rule #0001L', 'Rule #0001M', 'Rule #0001J', 'Rule #0001K'] violation = f"Evaluation message too long. Violated Rules: {concise}" return violation - - def determine_validation_state(self) -> str: - """ - this method determines the validation state of a shipcall. The state is either ['green', 'yellow', 'red'] and signals, - whether an entry causes issues within the workflow of users. - - returns: validation_state_new (str) - """ - (validation_state_new, description) = self.undefined_method() - # should there also be notifications for critical validation states? In principle, the traffic light itself provides that notification. - self.validation_state = validation_state_new - return validation_state_new - - def determine_notification_state(self) -> (str, bool): - """ - this method determines state changes in the notification state. When the state is changed to yellow or red, - a user is notified about it. The only exception for this rule is when the state was yellow or red before, - as the user has then already been notified. - - returns: notification_state_new (str), should_notify (bool) - """ - (state_new, description) = self.undefined_method() # determine the successor - should_notify = self.identify_notification_state_change(state_new) - self.notification_state = state_new # overwrite the predecessor - return state_new, should_notify - - def identify_notification_state_change(self, state_new) -> bool: - """ - determines, whether the observed state change should trigger a notification. - internally, this function maps a color string to an integer and determines, if the successor state is more severe than the predecessor. - - state changes trigger a notification in the following cases: - green -> yellow - green -> red - yellow -> red - - (none -> yellow) or (none -> red) - due to the values in the enumeration objects, the states are mapped to provide this function. - green=1, yellow=2, red=3, none=1. Hence, critical changes can be observed by simply checking with "greater than". - - returns bool, whether a notification should be triggered - """ - # state_old is always considered at least 'Green' (1) - state_old = max(copy.copy(self.notification_state) if "notification_state" in list(self.__dict__.keys()) else StatusFlags.NONE, StatusFlags.GREEN.value) - return state_new.value > state_old.value - + def undefined_method(self) -> str: """this function should apply the ValidationRules to the respective .shipcall, in regards to .times""" - # #TODO_traffic_state return (StatusFlags.GREEN, False) # (state:str, should_notify:bool) + + +def inspect_shipcall_evaluation(vr, sql_handler, shipcall_id): + """ + # debug only! + + a simple debugging function, which serves in inspecting an evaluation function for a single shipcall id. It returns the result and all related data. + returns: result, shipcall_df (filtered by shipcall id), shipcall, spm (shipcall participant map, filtered by shipcall id), times_df (filtered by shipcall id) + """ + shipcall_df = sql_handler.df_dict.get("shipcall").loc[shipcall_id:shipcall_id,:] + + shipcall = Shipcall(**{**{"id":shipcall_id},**sql_handler.df_dict.get("shipcall").loc[shipcall_id].to_dict()}) + result = vr.evaluate(shipcall=shipcall) + notification_state = vr.identify_notification_state_change(state_old=int(shipcall.evaluation), state_new=int(result[0])) + print(f"Previous state: {int(shipcall.evaluation)}, New State: {result[0]}, Notification State: {notification_state}") + + times_df = sql_handler.df_dict.get("times") + times_df = times_df.loc[times_df["shipcall_id"]==shipcall_id] + + + spm = sql_handler.df_dict["shipcall_participant_map"] + spm = spm.loc[spm["shipcall_id"]==shipcall_id] + + return result, shipcall_df, shipcall, spm, times_df From f684b2fd95a6ac850b32f0612def0f48ba5ecc6b Mon Sep 17 00:00:00 2001 From: scopesorting Date: Fri, 19 Jan 2024 18:07:17 +0100 Subject: [PATCH 19/30] enumerators are now IntEnum objects, which provides simpler typing. --- src/server/BreCal/database/enums.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index 7f8fc3d..90726b3 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -11,7 +11,7 @@ class ParticipantType(IntFlag): PORT_ADMINISTRATION = 32 TUG = 64 -class ShipcallType(Enum): +class ShipcallType(IntEnum): """determines the type of a shipcall, as this changes the applicable validation rules""" INCOMING = 1 OUTGOING = 2 @@ -28,7 +28,7 @@ class ParticipantwiseTimeDelta(): NOTIFICATION = 10.0 # after n minutes, an evaluation may rise a notification -class StatusFlags(Enum): +class StatusFlags(IntEnum): """ these enumerators ensure that each traffic light validation rule state corresponds to a value, which will be used in the ValidationRules object to identify the necessity of notifications. @@ -39,6 +39,7 @@ class StatusFlags(Enum): RED = 3 class PierSide(IntEnum): + """These enumerators determine the pier side of a shipcall.""" PORTSIDE = 0 # Port/Backbord STARBOARD_SIDE = 1 # Starboard / Steuerbord From 2ce96a4fd6c9e1093f264f31b9735c12ef22e299 Mon Sep 17 00:00:00 2001 From: scopesorting Date: Mon, 15 Apr 2024 08:19:08 +0200 Subject: [PATCH 20/30] git ignoring VSCode --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5ce71e8..cfac68e 100644 --- a/.gitignore +++ b/.gitignore @@ -289,3 +289,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. src/notebooks_metz/ src/server/editable_requirements.txt +brecal.code-workspace From 90aefaeb99d04c02d82e98b206e6bbe0e1ddb42f Mon Sep 17 00:00:00 2001 From: scopesorting Date: Mon, 15 Apr 2024 12:06:48 +0200 Subject: [PATCH 21/30] updating STUB objects, slightly adapting data models --- src/server/BreCal/schemas/model.py | 115 +++++++++++------------- src/server/BreCal/stubs/notification.py | 2 - src/server/BreCal/stubs/times_full.py | 9 ++ src/server/BreCal/stubs/user.py | 9 ++ 4 files changed, 71 insertions(+), 64 deletions(-) diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index c429777..0df42c8 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -1,5 +1,5 @@ from dataclasses import field, dataclass -from marshmallow import Schema, fields, INCLUDE, ValidationError, validate, validates +from marshmallow import Schema, fields, INCLUDE, ValidationError, validate, validates, post_load from marshmallow.fields import Field from marshmallow_enum import EnumField from enum import IntEnum @@ -105,7 +105,7 @@ class History: return self(id, participant_id, shipcall_id, timestamp, eta, ObjectType(type), OperationType(operation)) class Error(Schema): - message = fields.String(required=True) + message = fields.String(metadata={'required':True}) class GetVerifyInlineResp(Schema): @@ -164,41 +164,34 @@ class ShipcallSchema(Schema): id = fields.Integer() ship_id = fields.Integer() type = fields.Integer() - eta = fields.DateTime(Required = False, allow_none=True) + eta = fields.DateTime(metadata={'required':False}, allow_none=True) voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} - etd = fields.DateTime(Required = False, allow_none=True) - arrival_berth_id = fields.Integer(Required = False, allow_none=True) - departure_berth_id = fields.Integer(Required = False, allow_none=True) - tug_required = fields.Bool(Required = False, allow_none=True) - pilot_required = fields.Bool(Required = False, allow_none=True) - flags = fields.Integer(Required = False, allow_none=True) - pier_side = fields.Bool(Required = False, allow_none=True) - bunkering = fields.Bool(Required = False, allow_none=True) - replenishing_terminal = fields.Bool(Required = False, allow_none=True) - replenishing_lock = fields.Bool(Required = False, allow_none=True) - draft = fields.Float(Required = False, allow_none=True, validate=[validate.Range(min=0, max=20, min_inclusive=False, max_inclusive=True)]) - tidal_window_from = fields.DateTime(Required = False, allow_none=True) - tidal_window_to = fields.DateTime(Required = False, allow_none=True) - rain_sensitive_cargo = fields.Bool(Required = False, allow_none=True) - recommended_tugs = fields.Integer(Required = False, allow_none=True) - anchored = fields.Bool(Required = False, allow_none=True) - moored_lock = fields.Bool(Required = False, allow_none=True) - canceled = fields.Bool(Required = False, allow_none=True) -<<<<<<< HEAD - evaluation = fields.Enum(EvaluationType, required=False, allow_none=True, default=EvaluationType.undefined) + etd = fields.DateTime(metadata={'required':False}, allow_none=True) + arrival_berth_id = fields.Integer(metadata={'required':False}, allow_none=True) + departure_berth_id = fields.Integer(metadata={'required':False}, allow_none=True) + tug_required = fields.Bool(metadata={'required':False}, allow_none=True) + pilot_required = fields.Bool(metadata={'required':False}, allow_none=True) + flags = fields.Integer(metadata={'required':False}, allow_none=True) + pier_side = fields.Bool(metadata={'required':False}, allow_none=True) + bunkering = fields.Bool(metadata={'required':False}, allow_none=True) + replenishing_terminal = fields.Bool(metadata={'required':False}, allow_none=True) + replenishing_lock = fields.Bool(metadata={'required':False}, allow_none=True) + draft = fields.Float(metadata={'required':False}, allow_none=True, validate=[validate.Range(min=0, max=20, min_inclusive=False, max_inclusive=True)]) + tidal_window_from = fields.DateTime(metadata={'required':False}, allow_none=True) + tidal_window_to = fields.DateTime(metadata={'required':False}, allow_none=True) + rain_sensitive_cargo = fields.Bool(metadata={'required':False}, allow_none=True) + recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True) + anchored = fields.Bool(metadata={'required':False}, allow_none=True) + moored_lock = fields.Bool(metadata={'required':False}, allow_none=True) + canceled = fields.Bool(metadata={'required':False}, allow_none=True) + evaluation = fields.Enum(EvaluationType, metadata={'required':False}, allow_none=True, default=EvaluationType.undefined) evaluation_message = fields.Str(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} - evaluation_time = fields.DateTime(Required = False, allow_none=True) - evaluation_notifications_sent = fields.Bool(Required = False, allow_none=True) - time_ref_point = fields.Int(Required = False, allow_none=True) -======= - evaluation = fields.Integer(Required = False, allow_none=True) - evaluation_message = fields.String(allow_none=True, metadata={'Required':False}) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} - evaluation_time = fields.DateTime(Required = False, allow_none=True) - evaluation_notifications_sent = fields.Bool(Required = False, allow_none=True) ->>>>>>> a5284a4 (implementing notifications, working on input validation) + evaluation_time = fields.DateTime(metadata={'required':False}, allow_none=True) + evaluation_notifications_sent = fields.Bool(metadata={'required':False}, allow_none=True) + time_ref_point = fields.Integer(metadata={'required':False}, allow_none=True) participants = fields.List(fields.Nested(ParticipantAssignmentSchema)) - created = fields.DateTime(Required = False, allow_none=True) - modified = fields.DateTime(Required = False, allow_none=True) + created = fields.DateTime(metadata={'required':False}, allow_none=True) + modified = fields.DateTime(metadata={'required':False}, allow_none=True) @post_load def make_shipcall(self, data, **kwargs): @@ -258,8 +251,6 @@ class Shipcall: created: datetime modified: datetime participants: List[Participant_Assignment] = field(default_factory=list) - evaluation_time : datetime = None - evaluation_notifications_sent : bool = None def to_json(self): return { @@ -313,30 +304,30 @@ class TimesSchema(Schema): super().__init__(unknown=None) pass - id = fields.Integer(Required=False) - eta_berth = fields.DateTime(Required = False, allow_none=True) - eta_berth_fixed = fields.Bool(Required = False, allow_none=True) - etd_berth = fields.DateTime(Required = False, allow_none=True) - etd_berth_fixed = fields.Bool(Required = False, allow_none=True) - lock_time = fields.DateTime(Required = False, allow_none=True) - lock_time_fixed = fields.Bool(Required = False, allow_none=True) - zone_entry = fields.DateTime(Required = False, allow_none=True) - zone_entry_fixed = fields.Bool(Required = False, allow_none=True) - operations_start = fields.DateTime(Required = False, allow_none=True) - operations_end = fields.DateTime(Required = False, allow_none=True) - remarks = fields.String(Required = False, allow_none=True, validate=[validate.Length(max=256)]) - participant_id = fields.Integer(Required = True) - berth_id = fields.Integer(Required = False, allow_none = True) - berth_info = fields.String(Required = False, allow_none=True, validate=[validate.Length(max=256)]) - pier_side = fields.Bool(Required = False, allow_none = True) - shipcall_id = fields.Integer(Required = True) - participant_type = fields.Integer(Required = False, allow_none=True) - ata = fields.DateTime(Required = False, allow_none=True) - atd = fields.DateTime(Required = False, allow_none=True) - eta_interval_end = fields.DateTime(Required = False, allow_none=True) - etd_interval_end = fields.DateTime(Required = False, allow_none=True) - created = fields.DateTime(Required = False, allow_none=True) - modified = fields.DateTime(Required = False, allow_none=True) + id = fields.Integer(metadata={'required':False}) + eta_berth = fields.DateTime(metadata={'required':False}, allow_none=True) + eta_berth_fixed = fields.Bool(metadata={'required':False}, allow_none=True) + etd_berth = fields.DateTime(metadata={'required':False}, allow_none=True) + etd_berth_fixed = fields.Bool(metadata={'required':False}, allow_none=True) + lock_time = fields.DateTime(metadata={'required':False}, allow_none=True) + lock_time_fixed = fields.Bool(metadata={'required':False}, allow_none=True) + zone_entry = fields.DateTime(metadata={'required':False}, allow_none=True) + zone_entry_fixed = fields.Bool(metadata={'required':False}, allow_none=True) + operations_start = fields.DateTime(metadata={'required':False}, allow_none=True) + operations_end = fields.DateTime(metadata={'required':False}, allow_none=True) + remarks = fields.String(metadata={'required':False}, allow_none=True, validate=[validate.Length(max=256)]) + participant_id = fields.Integer(metadata={'required':True}) + berth_id = fields.Integer(metadata={'required':False}, allow_none = True) + berth_info = fields.String(metadata={'required':False}, allow_none=True, validate=[validate.Length(max=256)]) + pier_side = fields.Bool(metadata={'required':False}, allow_none = True) + shipcall_id = fields.Integer(metadata={'required':True}) + participant_type = fields.Integer(metadata={'required':False}, allow_none=True) + ata = fields.DateTime(metadata={'required':False}, allow_none=True) + atd = fields.DateTime(metadata={'required':False}, allow_none=True) + eta_interval_end = fields.DateTime(metadata={'required':False}, allow_none=True) + etd_interval_end = fields.DateTime(metadata={'required':False}, allow_none=True) + created = fields.DateTime(metadata={'required':False}, allow_none=True) + modified = fields.DateTime(metadata={'required':False}, allow_none=True) @validates("eta_berth") def validate_eta_berth(self, value): @@ -351,7 +342,7 @@ class UserSchema(Schema): def __init__(self): super().__init__(unknown=None) pass - id = fields.Integer(required=True) + id = fields.Integer(metadata={'required':True}) first_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) last_name = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=64)]) user_phone = fields.String(allow_none=True, metadata={'Required':False}) @@ -439,7 +430,7 @@ class ShipSchema(Schema): super().__init__(unknown=None) pass - id = fields.Int(Required=False) + id = fields.Int(metadata={'required':False}) name = fields.String(allow_none=False, metadata={'Required':True}) imo = fields.Int(allow_none=False, metadata={'Required':True}) callsign = fields.String(allow_none=True, metadata={'Required':False}) diff --git a/src/server/BreCal/stubs/notification.py b/src/server/BreCal/stubs/notification.py index 63df90f..8971a76 100644 --- a/src/server/BreCal/stubs/notification.py +++ b/src/server/BreCal/stubs/notification.py @@ -7,7 +7,6 @@ def get_notification_simple(): """creates a default notification, where 'created' is now, and modified is now+10 seconds""" notification_id = generate_uuid1_int() # uid? times_id = generate_uuid1_int() # uid? - acknowledged = False level = 10 type = 0 message = "hello world" @@ -17,7 +16,6 @@ def get_notification_simple(): notification = Notification( notification_id, times_id, - acknowledged, level, type, message, diff --git a/src/server/BreCal/stubs/times_full.py b/src/server/BreCal/stubs/times_full.py index f0176a1..774cc9f 100644 --- a/src/server/BreCal/stubs/times_full.py +++ b/src/server/BreCal/stubs/times_full.py @@ -27,6 +27,11 @@ def get_times_full_simple(): zone_entry = etd_berth+datetime.timedelta(hours=0, minutes=15) zone_entry_fixed = False + + ata = eta_berth+datetime.timedelta(hours=0, minutes=15) + atd = etd_berth+datetime.timedelta(hours=0, minutes=15) + eta_interval_end = eta_berth + datetime.timedelta(hours=0, minutes=25) + etd_interval_end = etd_berth + datetime.timedelta(hours=0, minutes=25) operations_start = zone_entry+datetime.timedelta(hours=1, minutes=30) operations_end = operations_start+datetime.timedelta(hours=4, minutes=30) @@ -63,6 +68,10 @@ def get_times_full_simple(): pier_side=pier_side, participant_type=participant_type, shipcall_id=shipcall_id, + ata=ata, + atd=atd, + eta_interval_end=eta_interval_end, + etd_interval_end=etd_interval_end, created=created, modified=modified, ) diff --git a/src/server/BreCal/stubs/user.py b/src/server/BreCal/stubs/user.py index e469c55..908f512 100644 --- a/src/server/BreCal/stubs/user.py +++ b/src/server/BreCal/stubs/user.py @@ -18,6 +18,11 @@ def get_user_simple(): created = datetime.datetime.now() modified = created+datetime.timedelta(seconds=10) + + notify_email = True + notify_whatsapp = True + notify_signal = True + notify_popup = True user = User( user_id, @@ -29,6 +34,10 @@ def get_user_simple(): user_phone, password_hash, api_key, + notify_email, + notify_whatsapp, + notify_signal, + notify_popup, created, modified ) From fcb889d2bc0c5ad22c634f57e6024cdecfc777ca Mon Sep 17 00:00:00 2001 From: scopesorting Date: Mon, 15 Apr 2024 12:22:00 +0200 Subject: [PATCH 22/30] removing workspace file from VSCode --- brecal.code-workspace | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 brecal.code-workspace diff --git a/brecal.code-workspace b/brecal.code-workspace deleted file mode 100644 index 876a149..0000000 --- a/brecal.code-workspace +++ /dev/null @@ -1,8 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ], - "settings": {} -} \ No newline at end of file From b32b466f740a0a0d5b82554fa3780893436e4eaf Mon Sep 17 00:00:00 2001 From: Max Metz Date: Wed, 24 Apr 2024 08:26:37 +0200 Subject: [PATCH 23/30] setting up a local mysql database and running the API locally, which requires slight adaptations. Implementing input validation for POST requests of shipcalls and adapting enumerators, as well as data models. --- src/server/BreCal/api/shipcalls.py | 23 ++++- src/server/BreCal/database/enums.py | 22 ++++- src/server/BreCal/database/sql_handler.py | 19 +++-- src/server/BreCal/impl/shipcalls.py | 8 +- src/server/BreCal/local_db.py | 4 +- src/server/BreCal/schemas/model.py | 68 +++++++++++++-- src/server/BreCal/services/auth_guard.py | 1 + src/server/BreCal/services/jwt_handler.py | 15 ++++ src/server/BreCal/stubs/shipcall.py | 17 ++-- .../BreCal/validators/input_validation.py | 83 +++++++++++++++++++ src/server/requirements.txt | 3 +- src/server/tests/test_create_app.py | 2 +- .../test_validation_rule_functions.py | 12 ++- 13 files changed, 244 insertions(+), 33 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index fbf552a..61a7a7c 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -3,7 +3,8 @@ from webargs.flaskparser import parser from marshmallow import Schema, fields, ValidationError from ..schemas import model from .. import impl -from ..services.auth_guard import auth_guard +from ..services.auth_guard import auth_guard, check_jwt +from BreCal.validators.input_validation import check_if_user_is_bsmd_type, check_if_user_data_has_valid_ship_id, check_if_user_data_has_valid_berth_id, check_if_user_data_has_valid_participant_id import logging import json @@ -40,6 +41,26 @@ def PostShipcalls(): try: content = request.get_json(force=True) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + + import logging + logging.log(20, loadedModel) + logging.log(20, "metz development") + """ + # loadedModel ... + valid_ship_id = check_if_user_data_has_valid_ship_id(ship_id) + valid_berth_id = check_if_user_data_has_valid_berth_id(berth_id) + valid_participant_id = check_if_user_data_has_valid_participant_id(participant_id) + """ + except ValidationError as ex: logging.error(ex) diff --git a/src/server/BreCal/database/enums.py b/src/server/BreCal/database/enums.py index 90726b3..5d28e65 100644 --- a/src/server/BreCal/database/enums.py +++ b/src/server/BreCal/database/enums.py @@ -2,7 +2,7 @@ from enum import IntEnum, Enum, IntFlag class ParticipantType(IntFlag): """determines the type of a participant""" - NONE = 0 + undefined = 0 BSMD = 1 TERMINAL = 2 PILOT = 4 @@ -13,10 +13,15 @@ class ParticipantType(IntFlag): class ShipcallType(IntEnum): """determines the type of a shipcall, as this changes the applicable validation rules""" + undefined = 0 INCOMING = 1 OUTGOING = 2 SHIFTING = 3 + @classmethod + def _missing_(cls, value): + return cls.undefined + class ParticipantwiseTimeDelta(): """stores the time delta for every participant, which triggers the validation rules in the rule set '0001'""" AGENCY = 1200.0 # 20 h * 60 min/h = 1200 min @@ -42,10 +47,23 @@ class PierSide(IntEnum): """These enumerators determine the pier side of a shipcall.""" PORTSIDE = 0 # Port/Backbord STARBOARD_SIDE = 1 # Starboard / Steuerbord - + class NotificationType(IntFlag): """determines the method by which a notification is distributed to users. Flagging allows selecting multiple notification types.""" UNDEFINED = 0 EMAIL = 1 POPUP = 2 MESSENGER = 4 + +class ParticipantFlag(IntFlag): + """ + | 1 | If this flag is set on a shipcall record with participant type Agency (8), + all participants of type BSMD (1) may edit the record. + """ + undefined = 0 + BSMD = 1 + + @classmethod + def _missing_(cls, value): + return cls.undefined + diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index 59497e3..929f558 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -2,7 +2,7 @@ import numpy as np import pandas as pd import datetime import typing -from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times +from BreCal.schemas.model import Shipcall, Ship, Participant, Berth, User, Times, ShipcallParticipantMap from BreCal.database.enums import ParticipantType def pandas_series_to_data_model(): @@ -50,7 +50,8 @@ class SQLHandler(): 'ship'->BreCal.schemas.model.Ship object """ self.str_to_model_dict = { - "shipcall":Shipcall, "ship":Ship, "participant":Participant, "berth":Berth, "user":User, "times":Times + "shipcall":Shipcall, "ship":Ship, "participant":Participant, "berth":Berth, "user":User, "times":Times, + "shipcall_participant_map":ShipcallParticipantMap } return @@ -70,12 +71,16 @@ class SQLHandler(): data = [{k:v for k,v in zip(column_names, dat)} for dat in data] # 4.) build a dataframe from the respective data models (which ensures the correct data type) + df = self.build_df_from_data_and_name(data, table_name) + return df + + def build_df_from_data_and_name(self, data, table_name): data_model = self.str_to_model_dict.get(table_name) if data_model is not None: - df = pd.DataFrame([data_model(**dat) for dat in data]) + df = pd.DataFrame([data_model(**dat) for dat in data], columns=list(data_model.__annotations__.keys())) else: df = pd.DataFrame([dat for dat in data]) - return df + return df def mysql_to_df(self, query, table_name): """provide an arbitrary sql query that should be read from a mysql server {sql_connection}. returns a pandas DataFrame with the obtained data""" @@ -94,11 +99,7 @@ class SQLHandler(): # 4.) build a dataframe from the respective data models (which ensures the correct data type) data_model = self.str_to_model_dict.get(table_name) - if data_model is not None: - df = pd.DataFrame([data_model(**dat) for dat in data]) - else: - df = pd.DataFrame([dat for dat in data]) - + df = self.build_df_from_data_and_name(data, table_name) if 'id' in df.columns: df = df.set_index('id', inplace=False) # avoid inplace updates, so the raw sql remains unchanged return df diff --git a/src/server/BreCal/impl/shipcalls.py b/src/server/BreCal/impl/shipcalls.py index 8d15983..252c817 100644 --- a/src/server/BreCal/impl/shipcalls.py +++ b/src/server/BreCal/impl/shipcalls.py @@ -60,8 +60,13 @@ def PostShipcalls(schemaModel): """ :param schemaModel: The deserialized dict of the request + e.g., + { + 'ship_id': 1, 'type': 1, 'eta': datetime.datetime(2023, 7, 23, 7, 18, 19), + 'voyage': '43B', 'tug_required': False, 'pilot_required': True, 'flags': 0, + 'pier_side': False, 'bunkering': True, 'recommended_tugs': 2, 'type_value': 1, 'evaluation_value': 0} + } """ - # TODO: Validate the upload data # This creates a *new* entry @@ -133,7 +138,6 @@ def PostShipcalls(schemaModel): # if not full_id_existances: # pooledConnection.close() # return json.dumps({"message" : "call failed. missing mandatory keywords."}), 500, {'Content-Type': 'application/json; charset=utf-8'} - commands.execute(query, schemaModel) new_id = commands.execute_scalar("select last_insert_id()") diff --git a/src/server/BreCal/local_db.py b/src/server/BreCal/local_db.py index 4293ff3..b4e02cd 100644 --- a/src/server/BreCal/local_db.py +++ b/src/server/BreCal/local_db.py @@ -7,11 +7,11 @@ import sys config_path = None -def initPool(instancePath): +def initPool(instancePath, connection_filename="connection_data_devel.json"): try: global config_path if(config_path == None): - config_path = os.path.join(instancePath,'../../../secure/connection_data_devel.json'); + config_path = os.path.join(instancePath,f'../../../../secure/{connection_filename}') #connection_data_devel.json'); print (config_path) diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index 0df42c8..33638fd 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -10,6 +10,9 @@ from typing import List import json import datetime from BreCal.validators.time_logic import validate_time_exceeds_threshold +from BreCal.database.enums import ParticipantType, ParticipantFlag + +# from BreCal. ... import check_if_user_is_bsmd_type def obj_dict(obj): if isinstance(obj, datetime.datetime): @@ -54,6 +57,7 @@ class NotificationType(IntEnum): undefined = 0 email = 1 push = 2 + @classmethod def _missing_(cls, value): return cls.undefined @@ -143,12 +147,34 @@ class Participant(Schema): street: str postal_code: str city: str - type: int + type: int # fields.Enum(ParticipantType ...) flags: int created: datetime modified: datetime deleted: bool + @validates("type") + def validate_type(self, value): + # e.g., when an IntFlag has the values 1,2,4; the maximum valid value is 7 + max_int = sum([int(val) for val in list(ParticipantType._value2member_map_.values())]) + min_int = 0 + + valid_type = 0 <= value < max_int + if not valid_type: + raise ValidationError(f"the provided integer is not supported for default behaviour of the ParticipantType IntFlag. Your choice: {value}. Supported values are: 0 <= value {max_int}") + + + @validates("flags") + def validate_type(self, value): + # e.g., when an IntFlag has the values 1,2,4; the maximum valid value is 7 + max_int = sum([int(val) for val in list(ParticipantFlag._value2member_map_.values())]) + min_int = 0 + + valid_type = 0 <= value < max_int + if not valid_type: + raise ValidationError(f"the provided integer is not supported for default behaviour of the ParticipantFlag IntFlag. Your choice: {value}. Supported values are: 0 <= value {max_int}") + + class ParticipantList(Participant): pass @@ -163,7 +189,8 @@ class ShipcallSchema(Schema): id = fields.Integer() ship_id = fields.Integer() - type = fields.Integer() + #type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator + type = fields.Integer() # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator eta = fields.DateTime(metadata={'required':False}, allow_none=True) voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} etd = fields.DateTime(metadata={'required':False}, allow_none=True) @@ -196,15 +223,23 @@ class ShipcallSchema(Schema): @post_load def make_shipcall(self, data, **kwargs): if 'type' in data: - data['type_value'] = data['type'].value + data['type_value'] = int(data['type']) else: - data['type_value'] = ShipcallType.undefined + data['type_value'] = int(ShipcallType.undefined) if 'evaluation' in data: if data['evaluation']: - data['evaluation_value'] = data['evaluation'].value + data['evaluation_value'] = int(data['evaluation']) else: - data['evaluation_value'] = EvaluationType.undefined + data['evaluation_value'] = int(EvaluationType.undefined) return data + + @validates("type") + def validate_type(self, value): + valid_shipcall_type = int(value) in [item.value for item in ShipcallType] + + if not valid_shipcall_type: + raise ValidationError(f"the provided type is not a valid shipcall type.") + @dataclass class Participant_Assignment: @@ -321,7 +356,7 @@ class TimesSchema(Schema): berth_info = fields.String(metadata={'required':False}, allow_none=True, validate=[validate.Length(max=256)]) pier_side = fields.Bool(metadata={'required':False}, allow_none = True) shipcall_id = fields.Integer(metadata={'required':True}) - participant_type = fields.Integer(metadata={'required':False}, allow_none=True) + participant_type = fields.Enum(ParticipantType, metadata={'required':False}, allow_none=True, default=ParticipantType.undefined) #fields.Integer(metadata={'required':False}, allow_none=True) ata = fields.DateTime(metadata={'required':False}, allow_none=True) atd = fields.DateTime(metadata={'required':False}, allow_none=True) eta_interval_end = fields.DateTime(metadata={'required':False}, allow_none=True) @@ -463,3 +498,22 @@ class Shipcalls(Shipcall): class TimesList(Times): pass + +@dataclass +class ShipcallParticipantMap: + id: int + shipcall_id: int + participant_id: int + type : ShipcallType + created: datetime + modified: datetime + + def to_json(self): + return { + "id": self.id, + "shipcall_id": self.shipcall_id, + "participant_id": self.participant_id, + "type": self.type.name, + "created": self.created.isoformat() if self.created else "", + "modified": self.modified.isoformat() if self.modified else "", + } diff --git a/src/server/BreCal/services/auth_guard.py b/src/server/BreCal/services/auth_guard.py index 9c5824d..9bd27c6 100644 --- a/src/server/BreCal/services/auth_guard.py +++ b/src/server/BreCal/services/auth_guard.py @@ -9,6 +9,7 @@ def check_jwt(): if not token: raise Exception('Missing access token') jwt = token.split('Bearer ')[1] + try: return decode_jwt(jwt) except Exception as e: diff --git a/src/server/BreCal/services/jwt_handler.py b/src/server/BreCal/services/jwt_handler.py index 66c0f43..b3c80e7 100644 --- a/src/server/BreCal/services/jwt_handler.py +++ b/src/server/BreCal/services/jwt_handler.py @@ -7,11 +7,26 @@ def create_api_key(): return secrets.token_urlsafe(16) def generate_jwt(payload, lifetime=None): + """ + creates an encoded token, which is based on the 'SECRET_KEY' environment variable. The environment variable + is set when the .wsgi application is started or can theoretically be set on system-level. + + args: + payload: + json-dictionary with key:value pairs. + + lifetime: + When a 'lifetime' (integer) is provided, the payload will be extended by an expiration key 'exp', which is + valid for the next {lifetime} minutes. + + returns: token, a JWT-encoded string + """ if lifetime: payload['exp'] = (datetime.datetime.now() + datetime.timedelta(minutes=lifetime)).timestamp() return jwt.encode(payload, os.environ.get('SECRET_KEY'), algorithm="HS256") def decode_jwt(token): + """this function reverts the {generate_jwt} function. An encoded JWT token is decoded into a JSON dictionary.""" return jwt.decode(token, os.environ.get('SECRET_KEY'), algorithms=["HS256"]) diff --git a/src/server/BreCal/stubs/shipcall.py b/src/server/BreCal/stubs/shipcall.py index 2e4e154..3f5c062 100644 --- a/src/server/BreCal/stubs/shipcall.py +++ b/src/server/BreCal/stubs/shipcall.py @@ -7,21 +7,21 @@ def get_shipcall_simple(): # only used for the stub base_time = datetime.datetime.now() - shipcall_id = generate_uuid1_int() - ship_id = generate_uuid1_int() + shipcall_id = 124 # generate_uuid1_int() + ship_id = 5 # generate_uuid1_int() eta = base_time+datetime.timedelta(hours=3, minutes=12) role_type = 1 voyage = "987654321" etd = base_time+datetime.timedelta(hours=6, minutes=12) # should never be before eta - arrival_berth_id = generate_uuid1_int() - departure_berth_id = generate_uuid1_int() + arrival_berth_id = 140 #generate_uuid1_int() + departure_berth_id = 140 #generate_uuid1_int() tug_required = False pilot_required = False - flags = 0 # #TODO_shipcall_flags. What is meant here? What should be tested? + flags = 0 # #TODO_shipcall_flags. What is meant here? What should be tested? pier_side = False # whether a ship will be fixated on the pier side. en: pier side, de: Anlegestelle. From 'BremenCalling_Datenmodell.xlsx': gedreht/ungedreht bunkering = False # #TODO_bunkering_unclear replenishing_terminal = False # en: replenishing terminal, de: Nachfüll-Liegeplatz @@ -38,11 +38,13 @@ def get_shipcall_simple(): anchored = False moored_lock = False # de: 'Festmacherschleuse', en: 'moored lock' canceled = False + + time_ref_point = 0 evaluation = None evaluation_message = "" - evaluation_time = None - evaluation_notifications_sent = None + evaluation_time = datetime.datetime.now() + evaluation_notifications_sent = False created = datetime.datetime.now() modified = created+datetime.timedelta(seconds=10) @@ -76,6 +78,7 @@ def get_shipcall_simple(): evaluation_message, evaluation_time, evaluation_notifications_sent, + time_ref_point, created, modified, participants, diff --git a/src/server/BreCal/validators/input_validation.py b/src/server/BreCal/validators/input_validation.py index 188e5e4..2248779 100644 --- a/src/server/BreCal/validators/input_validation.py +++ b/src/server/BreCal/validators/input_validation.py @@ -1,8 +1,91 @@ ####################################### InputValidation ####################################### +import json from abc import ABC, abstractmethod from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant +from BreCal.impl.participant import GetParticipant +from BreCal.impl.ships import GetShips +from BreCal.impl.berths import GetBerths + +from BreCal.database.enums import ParticipantType + +def get_participant_id_dictionary(): + # get all participants + response,status_code,header = GetParticipant(options={}) + + # build a dictionary of id:item pairs, so one can select the respective participant + participants = json.loads(response) + participants = {items.get("id"):items for items in participants} + return participants + +def get_ship_id_dictionary(): + # get all ships + response,status_code,header = GetShips(token=None) + + # build a dictionary of id:item pairs, so one can select the respective participant + ships = json.loads(response) + ships = {items.get("id"):items for items in ships} + return ships + +def get_berth_id_dictionary(): + # get all berths + response,status_code,header = GetBerths(token=None) + + # build a dictionary of id:item pairs, so one can select the respective participant + berths = json.loads(response) + berths = {items.get("id"):items for items in berths} + return berths + +def check_if_user_is_bsmd_type(user_data:dict)->bool: + """ + given a dictionary of user data, determine the respective participant id and read, whether + that participant is a .BSMD-type + + Note: ParticipantType is an IntFlag. + Hence, ParticipantType(1) is ParticipantType.BSMD, + and ParticipantType(7) is [ParticipantType.BSMD, ParticipantType.TERMINAL, ParticipantType.PILOT] + + both would return 'True' + + returns: boolean. Whether the participant id is a .BSMD type element + """ + # user_data = decode token + participant_id = user_data.get("participant_id") + + # build a dictionary of id:item pairs, so one can select the respective participant + participants = get_participant_id_dictionary() + participant = participants.get(participant_id,{}) + + # boolean check: is the participant of type .BSMD? + is_bsmd = ParticipantType.BSMD in ParticipantType(participant.get("type",0)) + return is_bsmd + +def check_if_user_data_has_valid_ship_id(ship_id): + # build a dictionary of id:item pairs, so one can select the respective participant + ships = get_ship_id_dictionary() + + # boolean check + ship_id_is_valid = ship_id in list(ships.keys()) + return ship_id_is_valid + +def check_if_user_data_has_valid_berth_id(berth_id): + # build a dictionary of id:item pairs, so one can select the respective participant + berths = get_berth_id_dictionary() + + # boolean check + berth_id_is_valid = berth_id in list(berths.keys()) + return berth_id_is_valid + +def check_if_user_data_has_valid_participant_id(participant_id): + # build a dictionary of id:item pairs, so one can select the respective participant + participants = get_participant_id_dictionary() + + # boolean check + participant_id_is_valid = participant_id in list(participants.keys()) + return participant_id_is_valid + + class InputValidation(): def __init__(self): diff --git a/src/server/requirements.txt b/src/server/requirements.txt index 8b7f3ad..23bf276 100644 --- a/src/server/requirements.txt +++ b/src/server/requirements.txt @@ -8,6 +8,7 @@ webargs==6.1.1 Werkzeug==1.0.1 pydapper[mysql-connector-python] marshmallow-dataclass +marshmallow-enum bcrypt pyjwt flask-jwt-extended @@ -20,4 +21,4 @@ pytest pytest-cov coverage -../server/. +-e ../server/. diff --git a/src/server/tests/test_create_app.py b/src/server/tests/test_create_app.py index 652ae3d..46ede88 100644 --- a/src/server/tests/test_create_app.py +++ b/src/server/tests/test_create_app.py @@ -8,7 +8,7 @@ def test_create_app(): import sys from BreCal import get_project_root - project_root = get_project_root("brecal") + project_root = os.path.join(os.path.expanduser("~"), "brecal") lib_location = os.path.join(project_root, "src", "server") sys.path.append(lib_location) diff --git a/src/server/tests/validators/test_validation_rule_functions.py b/src/server/tests/validators/test_validation_rule_functions.py index fdd8152..3a08735 100644 --- a/src/server/tests/validators/test_validation_rule_functions.py +++ b/src/server/tests/validators/test_validation_rule_functions.py @@ -12,7 +12,17 @@ from BreCal.stubs.df_times import get_df_times, random_time_perturbation, get_df @pytest.fixture(scope="session") def build_sql_proxy_connection(): import mysql.connector - conn_from_pool = mysql.connector.connect(**{'host':'localhost', 'port':3306, 'user':'root', 'password':'HalloWach_2323XXL!!', 'pool_name':'brecal_pool', 'pool_size':20, 'database':'bremen_calling', 'autocommit': True}) + import os + import json + connection_data_path = os.path.join(os.path.expanduser("~"),"secure","connection_data_local.json") + assert os.path.exists(connection_data_path) + + with open(connection_data_path, "r") as jr: + connection_data = json.load(jr) + connection_data = {k:v for k,v in connection_data.items() if k in ["host", "port", "user", "password", "pool_size", "pool_name", "database"]} + + conn_from_pool = mysql.connector.connect(**connection_data) + #conn_from_pool = mysql.connector.connect(**{'host':'localhost', 'port':3306, 'user':'root', 'password':'HalloWach_2323XXL!!', 'pool_name':'brecal_pool', 'pool_size':20, 'database':'bremen_calling_local', 'autocommit': True}) sql_handler = SQLHandler(sql_connection=conn_from_pool, read_all=True) vr = ValidationRules(sql_handler) return locals() From d0753f0b32df81d58c7ba67ed62e288779df0203 Mon Sep 17 00:00:00 2001 From: Max Metz Date: Mon, 29 Apr 2024 11:30:24 +0200 Subject: [PATCH 24/30] adapting rule 0005A and refactoring header-checks. Solving a conflict between versions, where there was a premature exit for time-agreement-rules. --- src/server/BreCal/api/shipcalls.py | 9 +++ src/server/BreCal/database/sql_handler.py | 38 +++++++++ .../validators/validation_rule_functions.py | 81 +++++++++++++------ .../BreCal/validators/validation_rules.py | 4 +- 4 files changed, 104 insertions(+), 28 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 61a7a7c..8adde2f 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -56,6 +56,15 @@ def PostShipcalls(): logging.log(20, "metz development") """ # loadedModel ... + loadedModel.get("ship_id", 0) + + 2024-04-22 18:21:03,982 | root | INFO | {'ship_id': 1, + 'type': 1, 'eta': datetime.datetime(2023, 7, 23, 7, 18, 19), + 'voyage': '43B', 'tug_required': False, 'pilot_required': True, + 'flags': 0, 'pier_side': False, 'bunkering': True, 'recommended_tugs': 2, + 'type_value': 1, 'evaluation_value': 0} + + valid_ship_id = check_if_user_data_has_valid_ship_id(ship_id) valid_berth_id = check_if_user_data_has_valid_berth_id(berth_id) valid_participant_id = check_if_user_data_has_valid_participant_id(participant_id) diff --git a/src/server/BreCal/database/sql_handler.py b/src/server/BreCal/database/sql_handler.py index 929f558..741e631 100644 --- a/src/server/BreCal/database/sql_handler.py +++ b/src/server/BreCal/database/sql_handler.py @@ -19,7 +19,37 @@ def set_participant_type(x, participant_df)->int: participant_type = participant_df.loc[participant_id, "type"] return participant_type +def get_synchronous_shipcall_times_standalone(query_time:pd.Timestamp, all_df_times:pd.DataFrame, delta_threshold=900)->int: + """ + This function counts all entries in {all_df_times}, which have the same timestamp as {query_time}. + It does so by: + 1.) selecting all eta_berth & etd_berth entries + 2.) measuring the timedelta towards {query_time} + 3.) converting the timedelta to total absolute seconds (positive or negative time differences do not matter) + 4.) applying a {delta_threshold} to identify, whether two times are too closely together + 5.) counting the times, where the timedelta is below the threshold + returns: counts + """ + assert isinstance(query_time,pd.Timestamp) + + # get a timedelta for each valid (not Null) time entry + time_deltas_eta = [(query_time.to_pydatetime()-time_.to_pydatetime()) for time_ in all_df_times.loc[:,"eta_berth"] if not pd.isnull(time_)] + time_deltas_etd = [(query_time.to_pydatetime()-time_.to_pydatetime()) for time_ in all_df_times.loc[:,"etd_berth"] if not pd.isnull(time_)] + + # consider both, eta and etd times + time_deltas = time_deltas_eta + time_deltas_etd + + # convert the timedelta to absolute total seconds + time_deltas = [abs(delta.total_seconds()) for delta in time_deltas] + + # consider only those time deltas, which are <= the determined threshold + # create a list of booleans + time_deltas_filtered = [delta <= delta_threshold for delta in time_deltas] + + # booleans can be added/counted in Python by using sum() + counts = sum(time_deltas_filtered) # int + return counts class SQLHandler(): """ @@ -333,6 +363,10 @@ class SQLHandler(): def get_unique_ship_counts(self, all_df_times:pd.DataFrame, times_agency:pd.DataFrame, query:str, rounding:str="min", maximum_threshold=3): """given a dataframe of all agency times, get all unique ship counts, their values (datetime) and the string tags. returns a tuple (values,unique,counts)""" + # #deprecated! + import warnings + warnings.warn(f"SQLHandler.get_unique_ship_counts is deprecated. Instead, please use SQLHandler.count_synchronous_shipcall_times") + # optional: rounding if rounding is not None: all_df_times.loc[:, query] = pd.to_datetime(all_df_times.loc[:, query]).dt.round(rounding) # e.g., 'min' --- # correcting the error: 'AttributeError: Can only use .dt accessor with datetimelike values' @@ -348,3 +382,7 @@ class SQLHandler(): # get unique entries and counts counts = len(values) # unique, counts = np.unique(values, return_counts=True) return counts # (values, unique, counts) + + def count_synchronous_shipcall_times(self, query_time:pd.Timestamp, all_df_times:pd.DataFrame, delta_threshold=900)->int: + """count all times entries, which are too close to the query_time. The {delta_threshold} determines the threshold. returns counts (int)""" + return get_synchronous_shipcall_times_standalone(query_time, all_df_times, delta_threshold) diff --git a/src/server/BreCal/validators/validation_rule_functions.py b/src/server/BreCal/validators/validation_rule_functions.py index 98fff6e..24bbc62 100644 --- a/src/server/BreCal/validators/validation_rule_functions.py +++ b/src/server/BreCal/validators/validation_rule_functions.py @@ -38,14 +38,16 @@ error_message_dict = { "validation_rule_fct_etd_time_not_in_tidal_window":"The tidal window does not fit to the agency's estimated time of departure (ETD) {Rule #0004B}", # 0005 A+B - "validation_rule_fct_too_many_identical_eta_times":"There are more than three ships with the same planned time of arrival (ETA) {Rule #0005A}", - "validation_rule_fct_too_many_identical_etd_times":"There are more than three ships with the same planned time of departure (ETD) {Rule #0005B}", + "validation_rule_fct_too_many_identical_eta_times":"More than three shipcalls are planned at the same time as the defined ETA {Rule #0005A}", + "validation_rule_fct_too_many_identical_etd_times":"More than three shipcalls are planned at the same time as the defined ETD {Rule #0005B}", # 0006 A+B "validation_rule_fct_agency_and_terminal_berth_id_disagreement":"Agency and Terminal are planning with different berths (the berth_id deviates). {Rule #0006A}", "validation_rule_fct_agency_and_terminal_pier_side_disagreement":"Agency and Terminal are planning with different pier sides (the pier_side deviates). {Rule #0006B}", } + + class ValidationRuleBaseFunctions(): """ Base object with individual functions, which the {ValidationRuleFunctions}-child refers to. @@ -71,6 +73,18 @@ class ValidationRuleBaseFunctions(): def get_no_violation_default_output(self): """return the default output of a validation function with no validation: a tuple of (GREEN state, None)""" return (StatusFlags.GREEN, None) + + def check_if_header_exists(self, df_times:pd.DataFrame, participant_type:ParticipantType)->bool: + """ + Given a pandas DataFrame, which contains times entries for a specific shipcall id, + this function checks, whether one of the times entries belongs to the requested ParticipantType. + + returns bool + """ + # empty DataFrames form a special case, as they might miss the 'participant_type' column. + if len(df_times)==0: + return False + return participant_type in df_times.loc[:,"participant_type"].values def check_time_delta_violation_query_time_to_now(self, query_time:pd.Timestamp, key_time:pd.Timestamp, threshold:float)->bool: """ @@ -144,7 +158,6 @@ class ValidationRuleBaseFunctions(): return violation_state df_times = df_times.loc[df_times["participant_type"].isin(participant_types),:] - agency_time = [time_ for time_ in agency_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] # for the given query, e.g., 'eta_berth', sample all times from the pandas DataFrame # exclude missing entries and consider only pd.Timestamp entries (which ignores pd.NaT/null entries) @@ -172,6 +185,7 @@ class ValidationRuleBaseFunctions(): violation_state = any(time_difference_exceeds_threshold) # this (previous) solution compares times to the reference (agency) time and checks if the difference is greater than 15 minutes + # agency_time = [time_ for time_ in agency_times.loc[:,query].tolist() if isinstance(time_, pd.Timestamp)] # violation_state = ((np.max(estimated_times) - agency_time[0]) > pd.Timedelta("15min")) or ((agency_time[0] - np.min(estimated_times)) > pd.Timedelta("15min")) # this solution to the rule compares all times to each other. When there is a total difference of more than 15 minutes, a violation occurs @@ -762,10 +776,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: return self.get_no_violation_default_output() # rule not applicable - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): + #if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: return self.get_no_violation_default_output() # rule not applicable # get agency & terminal times @@ -805,10 +821,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() # rule not applicable - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): return self.get_no_violation_default_output() # rule not applicable # get agency & terminal times @@ -845,7 +863,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + # if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) @@ -876,7 +895,8 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): return self.get_no_violation_default_output() # check, if the header is filled in (agency) - if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + # if len(df_times.loc[df_times["participant_type"].isin([ParticipantType.AGENCY.value])]) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) @@ -898,16 +918,19 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): """ Code: #0005-A Type: Global Rule - Description: this validation rule checks, whether there are too many shipcalls with identical ETA times. + Description: this validation rule checks, whether there are too many shipcalls with identical times to the query ETA. """ - times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] # check, if the header is filled in (agency) - if len(times_agency) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): # if len(times_agency) != 1: return self.get_no_violation_default_output() - # when ANY of the unique values exceeds the threshold, a violation is observed - query = "eta_berth" - violation_state = self.check_unique_shipcall_counts(query, times_agency=times_agency, rounding=rounding, maximum_threshold=maximum_threshold, all_times_agency=all_times_agency) + # get the agency's query time + times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] + query_time = times_agency.iloc[0].eta_berth + + # count the number of times, where a times entry is very close to the query time (uses an internal threshold, such as 15 minutes) + counts = self.sql_handler.count_synchronous_shipcall_times(query_time, all_df_times=all_times_agency) + violation_state = counts > maximum_threshold if violation_state: validation_name = "validation_rule_fct_too_many_identical_eta_times" @@ -919,16 +942,19 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): """ Code: #0005-B Type: Global Rule - Description: this validation rule checks, whether there are too many shipcalls with identical ETD times. + Description: this validation rule checks, whether there are too many shipcalls with identical times to the query ETD. """ - times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] # check, if the header is filled in (agency) - if len(times_agency) != 1: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): #if len(times_agency) != 1: return self.get_no_violation_default_output() - # when ANY of the unique values exceeds the threshold, a violation is observed - query = "etd_berth" - violation_state = self.check_unique_shipcall_counts(query, times_agency=times_agency, rounding=rounding, maximum_threshold=maximum_threshold, all_times_agency=all_times_agency) + # get the agency's query time + times_agency = df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value] + query_time = times_agency.iloc[0].etd_berth + + # count the number of times, where a times entry is very close to the query time (uses an internal threshold, such as 15 minutes) + counts = self.sql_handler.count_synchronous_shipcall_times(query_time, all_df_times=all_times_agency) + violation_state = counts > maximum_threshold if violation_state: validation_name = "validation_rule_fct_too_many_identical_etd_times" @@ -943,10 +969,12 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): Description: This validation rule checks, whether agency and terminal agree with their designated berth place by checking berth_id. """ # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() # rule not applicable - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): return self.get_no_violation_default_output() # rule not applicable times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) @@ -979,13 +1007,14 @@ class ValidationRuleFunctions(ValidationRuleBaseFunctions): Description: This validation rule checks, whether agency and terminal agree with their designated pier side by checking pier_side. """ # check, if the header is filled in (agency & terminal) - if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.AGENCY.value]) == 0: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.AGENCY): return self.get_no_violation_default_output() # rule not applicable - if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: + # if len(df_times.loc[df_times["participant_type"]==ParticipantType.TERMINAL.value]) == 0: + if not self.check_if_header_exists(df_times, participant_type=ParticipantType.TERMINAL): return self.get_no_violation_default_output() # rule not applicable - times_agency = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.AGENCY.value) times_terminal = self.sql_handler.get_times_for_participant_type(df_times, participant_type=ParticipantType.TERMINAL.value) # when one of the two values is null, the state is GREEN diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index 4b56e9b..69cc5ee 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -30,9 +30,9 @@ class ValidationRules(ValidationRuleFunctions): returns: (evaluation_state, violations) """ # prepare df_times, which every validation rule tends to use - df_times = self.sql_handler.df_dict.get('times', pd.DataFrame()) # -> pd.DataFrame + all_df_times = self.sql_handler.df_dict.get('times', pd.DataFrame()) # -> pd.DataFrame - if len(df_times)==0: + if len(all_df_times)==0: return (StatusFlags.GREEN.value, []) spm = self.sql_handler.df_dict["shipcall_participant_map"] From b7078f8d8ebf6fe5de9b63e7c80f80c77b396fbb Mon Sep 17 00:00:00 2001 From: Max Metz Date: Mon, 29 Apr 2024 16:46:50 +0200 Subject: [PATCH 25/30] implementing POST-request input validation for shipcalls. Creating many tests and applying slight updates to the Notifier (not implemented yet) --- src/server/BreCal/api/shipcalls.py | 32 +---- src/server/BreCal/database/update_database.py | 1 + .../notifications/notification_functions.py | 2 +- src/server/BreCal/schemas/model.py | 9 +- src/server/BreCal/stubs/shipcall.py | 18 +++ .../BreCal/validators/input_validation.py | 133 +++++++++++++++++- .../BreCal/validators/validation_rules.py | 7 +- 7 files changed, 165 insertions(+), 37 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 8adde2f..c1c6e5d 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -4,7 +4,7 @@ from marshmallow import Schema, fields, ValidationError from ..schemas import model from .. import impl from ..services.auth_guard import auth_guard, check_jwt -from BreCal.validators.input_validation import check_if_user_is_bsmd_type, check_if_user_data_has_valid_ship_id, check_if_user_data_has_valid_berth_id, check_if_user_data_has_valid_participant_id +from BreCal.validators.input_validation import validate_posted_shipcall_data import logging import json @@ -41,35 +41,15 @@ def PostShipcalls(): try: content = request.get_json(force=True) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + logging.log(20, loadedModel) + logging.log(20, "dev. above: loaded model, below: content") + logging.log(20, content) # read the user data from the JWT token (set when login is performed) user_data = check_jwt() - # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD - # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. - is_bsmd = check_if_user_is_bsmd_type(user_data) - if not is_bsmd: - raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") - - import logging - logging.log(20, loadedModel) - logging.log(20, "metz development") - """ - # loadedModel ... - loadedModel.get("ship_id", 0) - - 2024-04-22 18:21:03,982 | root | INFO | {'ship_id': 1, - 'type': 1, 'eta': datetime.datetime(2023, 7, 23, 7, 18, 19), - 'voyage': '43B', 'tug_required': False, 'pilot_required': True, - 'flags': 0, 'pier_side': False, 'bunkering': True, 'recommended_tugs': 2, - 'type_value': 1, 'evaluation_value': 0} - - - valid_ship_id = check_if_user_data_has_valid_ship_id(ship_id) - valid_berth_id = check_if_user_data_has_valid_berth_id(berth_id) - valid_participant_id = check_if_user_data_has_valid_participant_id(participant_id) - """ - + # validate the posted shipcall data + validate_posted_shipcall_data(user_data, loadedModel, content) except ValidationError as ex: logging.error(ex) diff --git a/src/server/BreCal/database/update_database.py b/src/server/BreCal/database/update_database.py index 7b7639b..4c120f3 100644 --- a/src/server/BreCal/database/update_database.py +++ b/src/server/BreCal/database/update_database.py @@ -55,6 +55,7 @@ def update_all_shipcalls_in_mysql_database(sql_connection, sql_handler:SQLHandle sql_handler: an SQLHandler instance shipcall_df: dataframe, which stores the data that is used to retrieve the shipcall data models (that are then updated in the database) """ + print(shipcall_df) for shipcall_id in shipcall_df.index: shipcall = sql_handler.df_loc_to_data_model(df=shipcall_df, id=shipcall_id, model_str="shipcall") update_shipcall_in_mysql_database(sql_connection, shipcall=shipcall, relevant_keys = ["evaluation", "evaluation_message"]) diff --git a/src/server/BreCal/notifications/notification_functions.py b/src/server/BreCal/notifications/notification_functions.py index 9bbc373..86956f0 100644 --- a/src/server/BreCal/notifications/notification_functions.py +++ b/src/server/BreCal/notifications/notification_functions.py @@ -110,6 +110,6 @@ class Notifier(): def get_notification_states(self, evaluation_states_old, evaluation_states_new)->list[bool]: """# build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created""" - evaluation_notifications_sent = [self.notifier.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)] + evaluation_notifications_sent = [self.determine_notification_state(state_old=int(state_old), state_new=int(state_new)) for state_old, state_new in zip(evaluation_states_old, evaluation_states_new)] return evaluation_notifications_sent diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index 33638fd..f346783 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -188,9 +188,9 @@ class ShipcallSchema(Schema): pass id = fields.Integer() - ship_id = fields.Integer() + ship_id = fields.Integer(metadata={'required':True}) #type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator - type = fields.Integer() # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator + type = fields.Integer(metadata={'required':True}) # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator eta = fields.DateTime(metadata={'required':False}, allow_none=True) voyage = fields.String(allow_none=True, metadata={'Required':False}, validate=[validate.Length(max=16)]) # Solving: RemovedInMarshmallow4Warning: Passing field metadata as keyword arguments is deprecated. Use the explicit `metadata=...` argument instead. Additional metadata: {'Required': False} etd = fields.DateTime(metadata={'required':False}, allow_none=True) @@ -207,7 +207,7 @@ class ShipcallSchema(Schema): tidal_window_from = fields.DateTime(metadata={'required':False}, allow_none=True) tidal_window_to = fields.DateTime(metadata={'required':False}, allow_none=True) rain_sensitive_cargo = fields.Bool(metadata={'required':False}, allow_none=True) - recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True) + recommended_tugs = fields.Integer(metadata={'required':False}, allow_none=True, validate=[validate.Range(min=0, max=10, min_inclusive=True, max_inclusive=True)]) anchored = fields.Bool(metadata={'required':False}, allow_none=True) moored_lock = fields.Bool(metadata={'required':False}, allow_none=True) canceled = fields.Bool(metadata={'required':False}, allow_none=True) @@ -251,6 +251,9 @@ class Participant_Assignment: participant_id: int type: int # a variant would be to use the IntFlag type (with appropriate serialization) + def to_json(self): + return self.__dict__ + @dataclass class Shipcall: diff --git a/src/server/BreCal/stubs/shipcall.py b/src/server/BreCal/stubs/shipcall.py index 3f5c062..48c81f7 100644 --- a/src/server/BreCal/stubs/shipcall.py +++ b/src/server/BreCal/stubs/shipcall.py @@ -85,4 +85,22 @@ def get_shipcall_simple(): ) return shipcall +def create_postman_stub_shipcall(): + """ + this function returns the common stub, which is used to POST data to shipcalls via POSTMAN. However, + the stub-function is updated with a dynamic ETA in the future, so the POST-request does not fail. + """ + shipcall = { + 'ship_id': 1, + 'type': 1, + 'eta': (datetime.datetime.now()+datetime.timedelta(hours=3)).isoformat(), + 'voyage': '43B', + 'tug_required': False, + 'pilot_required': True, + 'flags': 0, + 'pier_side': False, + 'bunkering': True, + 'recommended_tugs': 2 + } + return shipcall diff --git a/src/server/BreCal/validators/input_validation.py b/src/server/BreCal/validators/input_validation.py index 2248779..2652f11 100644 --- a/src/server/BreCal/validators/input_validation.py +++ b/src/server/BreCal/validators/input_validation.py @@ -2,14 +2,31 @@ ####################################### InputValidation ####################################### import json +import datetime from abc import ABC, abstractmethod -from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant +from marshmallow import ValidationError +from string import ascii_letters, digits + +from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant, ShipcallType from BreCal.impl.participant import GetParticipant from BreCal.impl.ships import GetShips from BreCal.impl.berths import GetBerths from BreCal.database.enums import ParticipantType +def check_if_string_has_special_characters(text:str): + """ + check, whether there are any characters within the provided string, which are not found in the ascii letters or digits + ascii_letters: abcd (...) and ABCD (...) + digits: 0123 (...) + + Source: https://stackoverflow.com/questions/57062794/is-there-a-way-to-check-if-a-string-contains-special-characters + User: https://stackoverflow.com/users/10035985/andrej-kesely + returns bool + """ + return bool(set(text).difference(ascii_letters + digits)) + + def get_participant_id_dictionary(): # get all participants response,status_code,header = GetParticipant(options={}) @@ -61,7 +78,11 @@ def check_if_user_is_bsmd_type(user_data:dict)->bool: is_bsmd = ParticipantType.BSMD in ParticipantType(participant.get("type",0)) return is_bsmd -def check_if_user_data_has_valid_ship_id(ship_id): +def check_if_ship_id_is_valid(ship_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if ship_id is None: + return True + # build a dictionary of id:item pairs, so one can select the respective participant ships = get_ship_id_dictionary() @@ -69,7 +90,11 @@ def check_if_user_data_has_valid_ship_id(ship_id): ship_id_is_valid = ship_id in list(ships.keys()) return ship_id_is_valid -def check_if_user_data_has_valid_berth_id(berth_id): +def check_if_berth_id_is_valid(berth_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if berth_id is None: + return True + # build a dictionary of id:item pairs, so one can select the respective participant berths = get_berth_id_dictionary() @@ -77,7 +102,11 @@ def check_if_user_data_has_valid_berth_id(berth_id): berth_id_is_valid = berth_id in list(berths.keys()) return berth_id_is_valid -def check_if_user_data_has_valid_participant_id(participant_id): +def check_if_participant_id_is_valid(participant_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if participant_id is None: + return True + # build a dictionary of id:item pairs, so one can select the respective participant participants = get_participant_id_dictionary() @@ -85,6 +114,102 @@ def check_if_user_data_has_valid_participant_id(participant_id): participant_id_is_valid = participant_id in list(participants.keys()) return participant_id_is_valid +def check_if_participant_ids_are_valid(participant_ids): + # check each participant id individually + valid_participant_ids = [check_if_participant_id_is_valid(participant_id) for participant_id in participant_ids] + + # boolean check, whether all participant ids are valid + return all(valid_participant_ids) + + +def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict): + """this function applies more complex validation functions to data, which is sent to a post-request of shipcalls""" + # #TODO_refactor: this function is pretty complex. One may instead build an object, which calls the methods separately. + + import logging + logging.log(20, "dev") + logging.log(20, user_data) + logging.log(20, loadedModel) + logging.log(20, content) + ##### Section 1: check user_data ##### + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + + ##### Section 2: check loadedModel ##### + valid_ship_id = check_if_ship_id_is_valid(ship_id=loadedModel.get("ship_id", None)) + if not valid_ship_id: + raise ValidationError(f"provided an invalid ship id, which is not found in the database: {loadedModel.get('ship_id', None)}") + + valid_arrival_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("arrival_berth_id", None)) + if not valid_arrival_berth_id: + raise ValidationError(f"provided an invalid arrival berth id, which is not found in the database: {loadedModel.get('arrival_berth_id', None)}") + + valid_departure_berth_id = check_if_berth_id_is_valid(berth_id=loadedModel.get("departure_berth_id", None)) + if not valid_departure_berth_id: + raise ValidationError(f"provided an invalid departure berth id, which is not found in the database: {loadedModel.get('departure_berth_id', None)}") + + valid_participant_ids = check_if_participant_ids_are_valid(participant_ids=loadedModel.get("participants",[])) + if not valid_participant_ids: + raise ValidationError(f"one of the provided participant ids is invalid. Could not find one of these in the database: {loadedModel.get('participants', None)}") + + + ##### Section 3: check content ##### + # loadedModel fills missing values, sometimes using optional values. Hence, check content + + # the following keys should not be set in a POST-request. + for forbidden_key in ["canceled", "evaluation", "evaluation_message"]: + value = content.get(forbidden_key, None) + if value is not None: + raise ValidationError(f"'{forbidden_key}' may not be set on POST. Found: {value}") + + voyage_str_is_invalid = check_if_string_has_special_characters(text=content.get("voyage","")) + if voyage_str_is_invalid: + raise ValidationError(f"there are invalid characters in the 'voyage'-string. Please use only digits and ASCII letters. Allowed: {ascii_letters+digits}. Found: {content.get('voyage')}") + + + ##### Section 4: check loadedModel & content ##### + # #TODO_refactor: these methods should be placed in separate locations + + # existance checks in content + # datetime checks in loadedModel (datetime.datetime objects). Dates should be in the future. + time_now = datetime.datetime.now() + type_ = loadedModel.get("type", int(ShipcallType.undefined)) + if int(type_)==int(ShipcallType.undefined): + raise ValidationError(f"providing 'type' is mandatory. Missing key!") + elif int(type_)==int(ShipcallType.arrival): + eta = loadedModel.get("eta") + if (content.get("eta", None) is None): + raise ValidationError(f"providing 'eta' is mandatory. Missing key!") + if content.get("arrival_berth_id", None) is None: + raise ValidationError(f"providing 'arrival_berth_id' is mandatory. Missing key!") + if not eta >= time_now: + raise ValidationError(f"'eta' must be in the future. Incorrect datetime provided.") + elif int(type_)==int(ShipcallType.departure): + etd = loadedModel.get("etd") + if (content.get("etd", None) is None): + raise ValidationError(f"providing 'etd' is mandatory. Missing key!") + if content.get("departure_berth_id", None) is None: + raise ValidationError(f"providing 'departure_berth_id' is mandatory. Missing key!") + if not etd >= time_now: + raise ValidationError(f"'etd' must be in the future. Incorrect datetime provided.") + elif int(type_)==int(ShipcallType.shifting): + eta = loadedModel.get("eta") + etd = loadedModel.get("etd") + # * arrival_berth_id / departure_berth_id (depending on type, see above) + if (content.get("eta", None) is None) or (content.get("etd", None) is None): + raise ValidationError(f"providing 'eta' and 'etd' is mandatory. Missing one of those keys!") + if (content.get("arrival_berth_id", None) is None) or (content.get("departure_berth_id", None) is None): + raise ValidationError(f"providing 'arrival_berth_id' & 'departure_berth_id' is mandatory. Missing key!") + if (not eta >= time_now) or (not etd >= time_now) or (not eta >= etd): + raise ValidationError(f"'eta' and 'etd' must be in the future. Incorrect datetime provided.") + + + # #TODO: len of participants > 0, if agency + # * assigned participant for agency + return class InputValidation(): diff --git a/src/server/BreCal/validators/validation_rules.py b/src/server/BreCal/validators/validation_rules.py index 69cc5ee..7d5448c 100644 --- a/src/server/BreCal/validators/validation_rules.py +++ b/src/server/BreCal/validators/validation_rules.py @@ -75,6 +75,7 @@ class ValidationRules(ValidationRuleFunctions): def evaluate_shipcalls(self, shipcall_df:pd.DataFrame)->pd.DataFrame: """apply 'evaluate_shipcall_from_df' to each individual shipcall in {shipcall_df}. Returns shipcall_df ('evaluation', 'evaluation_message', 'evaluation_time' and 'evaluation_notifications_sent' are updated)""" evaluation_states_old = [state_old for state_old in shipcall_df.loc[:,"evaluation"]] + evaluation_states_old = [state_old if not pd.isna(state_old) else 0 for state_old in evaluation_states_old] results = shipcall_df.apply(lambda x: self.evaluate_shipcall_from_df(x), axis=1).values # returns tuple (state, message) # unbundle individual results. evaluation_states becomes an integer, violation @@ -83,14 +84,14 @@ class ValidationRules(ValidationRuleFunctions): violations = [self.concise_evaluation_message_if_too_long(violation) for violation in violations] # build the list of evaluation times ('now', as isoformat) - evaluation_times = self.notifier.get_notification_times(evaluation_states_new) + evaluation_time = self.notifier.get_notification_times(evaluation_states_new) # build the list of 'evaluation_notifications_sent'. The value is 'False', when a notification should be created - evaluation_notifications_sent = self.get_notification_states(evaluation_states_old, evaluation_states_new) + evaluation_notifications_sent = self.notifier.get_notification_states(evaluation_states_old, evaluation_states_new) shipcall_df.loc[:,"evaluation"] = evaluation_states_new shipcall_df.loc[:,"evaluation_message"] = violations - shipcall_df.loc[:,"evaluation_times"] = evaluation_times + shipcall_df.loc[:,"evaluation_time"] = evaluation_time shipcall_df.loc[:,"evaluation_notifications_sent"] = evaluation_notifications_sent return shipcall_df From ba031e6d144df88f7accec304fcc6c9b9764ab69 Mon Sep 17 00:00:00 2001 From: Max Metz Date: Mon, 29 Apr 2024 18:50:46 +0200 Subject: [PATCH 26/30] implementing more input-validation-functions for shipcalls and ships. Beginning to refactor some of the validation functions into more readable Python classes. --- src/server/BreCal/api/shipcalls.py | 14 +++++--- src/server/BreCal/api/ships.py | 35 +++++++++++++++++-- src/server/BreCal/stubs/shipcall.py | 3 ++ .../BreCal/validators/input_validation.py | 20 +++++++---- .../validators/input_validation_shipcall.py | 0 5 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 src/server/BreCal/validators/input_validation_shipcall.py diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index c1c6e5d..e43f7ec 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -4,7 +4,7 @@ from marshmallow import Schema, fields, ValidationError from ..schemas import model from .. import impl from ..services.auth_guard import auth_guard, check_jwt -from BreCal.validators.input_validation import validate_posted_shipcall_data +from BreCal.validators.input_validation import validate_posted_shipcall_data, check_if_user_is_bsmd_type import logging import json @@ -41,9 +41,6 @@ def PostShipcalls(): try: content = request.get_json(force=True) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) - logging.log(20, loadedModel) - logging.log(20, "dev. above: loaded model, below: content") - logging.log(20, content) # read the user data from the JWT token (set when login is performed) user_data = check_jwt() @@ -72,6 +69,15 @@ def PutShipcalls(): content = request.get_json(force=True) logging.info(content) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) + + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") except ValidationError as ex: logging.error(ex) diff --git a/src/server/BreCal/api/ships.py b/src/server/BreCal/api/ships.py index e31147e..5fc3c50 100644 --- a/src/server/BreCal/api/ships.py +++ b/src/server/BreCal/api/ships.py @@ -1,11 +1,14 @@ from flask import Blueprint, request from .. import impl -from ..services.auth_guard import auth_guard -from marshmallow import EXCLUDE +from ..services.auth_guard import auth_guard, check_jwt +from marshmallow import EXCLUDE, ValidationError from ..schemas import model import json import logging +from BreCal.validators.input_validation import check_if_user_is_bsmd_type + + bp = Blueprint('ships', __name__) @bp.route('/ships', methods=['get']) @@ -24,6 +27,15 @@ def GetShips(): def PostShip(): try: + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + content = request.get_json(force=True) loadedModel = model.ShipSchema().load(data=content, many=False, partial=True) except Exception as ex: @@ -39,6 +51,15 @@ def PostShip(): def PutShip(): try: + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + content = request.get_json(force=True) loadedModel = model.ShipSchema().load(data=content, many=False, partial=True, unknown=EXCLUDE) except Exception as ex: @@ -53,8 +74,16 @@ def PutShip(): @auth_guard() # no restriction by role def DeleteShip(): - # TODO check if I am allowed to delete this thing by deriving the participant from the bearer token try: + # read the user data from the JWT token (set when login is performed) + user_data = check_jwt() + + # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + if 'id' in request.args: options = {} options["id"] = request.args.get("id") diff --git a/src/server/BreCal/stubs/shipcall.py b/src/server/BreCal/stubs/shipcall.py index 48c81f7..9efab32 100644 --- a/src/server/BreCal/stubs/shipcall.py +++ b/src/server/BreCal/stubs/shipcall.py @@ -89,12 +89,15 @@ def create_postman_stub_shipcall(): """ this function returns the common stub, which is used to POST data to shipcalls via POSTMAN. However, the stub-function is updated with a dynamic ETA in the future, so the POST-request does not fail. + + Also provides a stub arrival_berth_id, so the POST-request succeeds. """ shipcall = { 'ship_id': 1, 'type': 1, 'eta': (datetime.datetime.now()+datetime.timedelta(hours=3)).isoformat(), 'voyage': '43B', + 'arrival_berth_id':142, 'tug_required': False, 'pilot_required': True, 'flags': 0, diff --git a/src/server/BreCal/validators/input_validation.py b/src/server/BreCal/validators/input_validation.py index 2652f11..048d594 100644 --- a/src/server/BreCal/validators/input_validation.py +++ b/src/server/BreCal/validators/input_validation.py @@ -26,6 +26,10 @@ def check_if_string_has_special_characters(text:str): """ return bool(set(text).difference(ascii_letters + digits)) +def check_if_int_is_valid_flag(value, enum_object): + # e.g., when an IntFlag has the values 1,2,4; the maximum valid value is 7 + max_int = sum([int(val) for val in list(enum_object._value2member_map_.values())]) + return 0 < value <= max_int def get_participant_id_dictionary(): # get all participants @@ -126,11 +130,6 @@ def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict """this function applies more complex validation functions to data, which is sent to a post-request of shipcalls""" # #TODO_refactor: this function is pretty complex. One may instead build an object, which calls the methods separately. - import logging - logging.log(20, "dev") - logging.log(20, user_data) - logging.log(20, loadedModel) - logging.log(20, content) ##### Section 1: check user_data ##### # check, whether the user belongs to a participant, which is of type ParticipantType.BSMD # as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. @@ -205,7 +204,16 @@ def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict raise ValidationError(f"providing 'arrival_berth_id' & 'departure_berth_id' is mandatory. Missing key!") if (not eta >= time_now) or (not etd >= time_now) or (not eta >= etd): raise ValidationError(f"'eta' and 'etd' must be in the future. Incorrect datetime provided.") - + + tidal_window_from = loadedModel.get("tidal_window_from", None) + tidal_window_to = loadedModel.get("tidal_window_to", None) + if tidal_window_to is not None: + if not tidal_window_to >= time_now: + raise ValidationError(f"'tidal_window_to' must be in the future. Incorrect datetime provided.") + + if tidal_window_from is not None: + if not tidal_window_from >= time_now: + raise ValidationError(f"'tidal_window_from' must be in the future. Incorrect datetime provided.") # #TODO: len of participants > 0, if agency # * assigned participant for agency diff --git a/src/server/BreCal/validators/input_validation_shipcall.py b/src/server/BreCal/validators/input_validation_shipcall.py new file mode 100644 index 0000000..e69de29 From 2671bbbd05099e014fc22b377bec1c565444f9eb Mon Sep 17 00:00:00 2001 From: Max Metz Date: Tue, 14 May 2024 12:19:25 +0200 Subject: [PATCH 27/30] refactoring SQL get-query for shipcall into a separate utility-section, so it becomes reusable --- src/server/BreCal/database/sql_queries.py | 46 +++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/server/BreCal/database/sql_queries.py diff --git a/src/server/BreCal/database/sql_queries.py b/src/server/BreCal/database/sql_queries.py new file mode 100644 index 0000000..551aa91 --- /dev/null +++ b/src/server/BreCal/database/sql_queries.py @@ -0,0 +1,46 @@ + + +def create_sql_query_shipcall_get(options:dict)->str: + """ + creates an SQL query, which selects all shipcalls from the mysql database. + the agency eta times are used to order the entries. + + args: + options : dict. A dictionary, which must contains the 'past_days' key (int). Determines the range + by which shipcalls are filtered. + """ + query = ("SELECT s.id as id, ship_id, type, eta, voyage, etd, arrival_berth_id, departure_berth_id, tug_required, pilot_required, " + + "flags, s.pier_side, bunkering, replenishing_terminal, replenishing_lock, draft, tidal_window_from, " + + "tidal_window_to, rain_sensitive_cargo, recommended_tugs, anchored, moored_lock, canceled, evaluation, " + + "evaluation_message, evaluation_time, evaluation_notifications_sent, s.created as created, s.modified as modified, time_ref_point " + + "FROM shipcall s " + + "LEFT JOIN times t ON t.shipcall_id = s.id AND t.participant_type = 8 " + + "WHERE " + + "(type = 1 AND " + + "((t.id IS NOT NULL AND t.eta_berth >= DATE(NOW() - INTERVAL %d DAY)) OR " + + "(eta >= DATE(NOW() - INTERVAL %d DAY)))) OR " + + "((type = 2 OR type = 3) AND " + + "((t.id IS NOT NULL AND t.etd_berth >= DATE(NOW() - INTERVAL %d DAY)) OR " + + "(etd >= DATE(NOW() - INTERVAL %d DAY)))) " + + "ORDER BY eta") % (options["past_days"], options["past_days"], options["past_days"], options["past_days"]) + + """ + alternatively, f-strings could be used. + query_two = ("SELECT s.id as id, ship_id, type, eta, voyage, etd, arrival_berth_id, departure_berth_id, tug_required, pilot_required, " + + "flags, s.pier_side, bunkering, replenishing_terminal, replenishing_lock, draft, tidal_window_from, " + + "tidal_window_to, rain_sensitive_cargo, recommended_tugs, anchored, moored_lock, canceled, evaluation, " + + "evaluation_message, evaluation_time, evaluation_notifications_sent, s.created as created, s.modified as modified, time_ref_point " + + "FROM shipcall s " + + "LEFT JOIN times t ON t.shipcall_id = s.id AND t.participant_type = 8 " + + "WHERE " + + "(type = 1 AND " + + f"((t.id IS NOT NULL AND t.eta_berth >= DATE(NOW() - INTERVAL {options['past_days']} DAY)) OR " + + f"(eta >= DATE(NOW() - INTERVAL {options['past_days']} DAY)))) OR " + + "((type = 2 OR type = 3) AND " + + f"((t.id IS NOT NULL AND t.etd_berth >= DATE(NOW() - INTERVAL {options['past_days']} DAY)) OR " + + f"(etd >= DATE(NOW() - INTERVAL {options['past_days']} DAY)))) " + + "ORDER BY eta") + + assert query==query_two + """ + return query From 6966ba65e37f55c5f576c41b6f4191fcb7bdf05d Mon Sep 17 00:00:00 2001 From: Max Metz Date: Tue, 14 May 2024 12:44:54 +0200 Subject: [PATCH 28/30] refactored the shipcall's GET mysql-query into a separate file and reused it in the evaluation routine. --- src/server/BreCal/impl/shipcalls.py | 16 ++-------------- src/server/BreCal/services/schedule_routines.py | 13 +++---------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/src/server/BreCal/impl/shipcalls.py b/src/server/BreCal/impl/shipcalls.py index 252c817..de60788 100644 --- a/src/server/BreCal/impl/shipcalls.py +++ b/src/server/BreCal/impl/shipcalls.py @@ -8,6 +8,7 @@ from .. import local_db from ..services.auth_guard import check_jwt from BreCal.database.update_database import evaluate_shipcall_state +from BreCal.database.sql_queries import create_sql_query_shipcall_get def GetShipcalls(options): """ @@ -18,20 +19,7 @@ def GetShipcalls(options): pooledConnection = local_db.getPoolConnection() commands = pydapper.using(pooledConnection) - query = ("SELECT s.id as id, ship_id, type, eta, voyage, etd, arrival_berth_id, departure_berth_id, tug_required, pilot_required, " + - "flags, s.pier_side, bunkering, replenishing_terminal, replenishing_lock, draft, tidal_window_from, " + - "tidal_window_to, rain_sensitive_cargo, recommended_tugs, anchored, moored_lock, canceled, evaluation, " + - "evaluation_message, evaluation_time, evaluation_notifications_sent, s.created as created, s.modified as modified, time_ref_point " + - "FROM shipcall s " + - "LEFT JOIN times t ON t.shipcall_id = s.id AND t.participant_type = 8 " + - "WHERE " + - "(type = 1 AND " + - "((t.id IS NOT NULL AND t.eta_berth >= DATE(NOW() - INTERVAL %d DAY)) OR " + - "(eta >= DATE(NOW() - INTERVAL %d DAY)))) OR " + - "((type = 2 OR type = 3) AND " + - "((t.id IS NOT NULL AND t.etd_berth >= DATE(NOW() - INTERVAL %d DAY)) OR " + - "(etd >= DATE(NOW() - INTERVAL %d DAY)))) " + - "ORDER BY eta") % (options["past_days"], options["past_days"], options["past_days"], options["past_days"]) + query = create_sql_query_shipcall_get(options) data = commands.query(query, model=model.Shipcall.from_query_row, buffered=True) for shipcall in data: diff --git a/src/server/BreCal/services/schedule_routines.py b/src/server/BreCal/services/schedule_routines.py index 848364d..a36a1fc 100644 --- a/src/server/BreCal/services/schedule_routines.py +++ b/src/server/BreCal/services/schedule_routines.py @@ -3,6 +3,7 @@ import pydapper from BreCal.schemas import model from BreCal.local_db import getPoolConnection from BreCal.database.update_database import evaluate_shipcall_state +from BreCal.database.sql_queries import create_sql_query_shipcall_get import threading import schedule @@ -26,16 +27,8 @@ def UpdateShipcalls(options:dict = {'past_days':2}): pooledConnection = getPoolConnection() commands = pydapper.using(pooledConnection) - query = ("SELECT s.id as id, ship_id, type, eta, voyage, etd, arrival_berth_id, departure_berth_id, tug_required, pilot_required, " - "flags, s.pier_side, bunkering, replenishing_terminal, replenishing_lock, draft, tidal_window_from, tidal_window_to, rain_sensitive_cargo, recommended_tugs, " - "anchored, moored_lock, canceled, evaluation, evaluation_message, evaluation_notifications_sent, evaluation_time, s.created as created, s.modified as modified, time_ref_point FROM shipcall s " + - "LEFT JOIN times t ON t.shipcall_id = s.id AND t.participant_type = 8 " - "WHERE " - "(type = 1 AND (COALESCE(t.eta_berth, eta) >= DATE(NOW() - INTERVAL %d DAY))) OR " - "((type = 2 OR type = 3) AND (COALESCE(t.etd_berth, etd) >= DATE(NOW() - INTERVAL %d DAY)))" - "ORDER BY s.id") % (options["past_days"], options["past_days"]) - - # obtain data from the MYSQL database + # obtain data from the MYSQL database (uses 'options' to filter the resulting data by the ETA, considering those entries of 'past_days'-range) + query = create_sql_query_shipcall_get(options) data = commands.query(query, model=model.Shipcall) # get the shipcall ids, which are of interest From 79e22d86f228a4876397a34348c1503c60de0575 Mon Sep 17 00:00:00 2001 From: Max Metz Date: Tue, 14 May 2024 15:42:20 +0200 Subject: [PATCH 29/30] refactoring 'validate_post_shipcall_data' into a novel object InputValidationShipcall. Implemented the majority of rules for POST and PUT requests. Unit tests have not been created & run yet --- src/server/BreCal/api/shipcalls.py | 3 + src/server/BreCal/database/sql_queries.py | 99 ++++++ src/server/BreCal/impl/shipcalls.py | 14 +- src/server/BreCal/schemas/model.py | 2 +- .../BreCal/validators/input_validation.py | 110 +----- .../validators/input_validation_shipcall.py | 330 ++++++++++++++++++ .../validators/input_validation_utils.py | 146 ++++++++ 7 files changed, 595 insertions(+), 109 deletions(-) create mode 100644 src/server/BreCal/validators/input_validation_utils.py diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index e43f7ec..6dfef58 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -5,6 +5,7 @@ from ..schemas import model from .. import impl from ..services.auth_guard import auth_guard, check_jwt from BreCal.validators.input_validation import validate_posted_shipcall_data, check_if_user_is_bsmd_type +from BreCal.validators.input_validation_shipcall import InputValidationShipcall import logging import json @@ -47,6 +48,7 @@ def PostShipcalls(): # validate the posted shipcall data validate_posted_shipcall_data(user_data, loadedModel, content) + # InputValidationShipcall.evaluate_post_data(user_data, loadedModel, content) except ValidationError as ex: logging.error(ex) @@ -78,6 +80,7 @@ def PutShipcalls(): is_bsmd = check_if_user_is_bsmd_type(user_data) if not is_bsmd: raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + # InputValidationShipcall.evaluate_put_data(user_data, loadedModel, content) except ValidationError as ex: logging.error(ex) diff --git a/src/server/BreCal/database/sql_queries.py b/src/server/BreCal/database/sql_queries.py index 551aa91..0940254 100644 --- a/src/server/BreCal/database/sql_queries.py +++ b/src/server/BreCal/database/sql_queries.py @@ -44,3 +44,102 @@ def create_sql_query_shipcall_get(options:dict)->str: assert query==query_two """ return query + + +def create_sql_query_shipcall_post(schemaModel:dict)->str: + query = "INSERT INTO shipcall (" + isNotFirst = False + for key in schemaModel.keys(): + if key == "id": + continue + if key == "participants": + continue + if key == "created": + continue + if key == "modified": + continue + if key == "evaluation": + continue + if key == "evaluation_message": + continue + if key == "type_value": + continue + if key == "evaluation_value": + continue + if isNotFirst: + query += "," + isNotFirst = True + query += key + query += ") VALUES (" + isNotFirst = False + for key in schemaModel.keys(): + param_key = key + if key == "id": + continue + if key == "participants": + continue + if key == "created": + continue + if key == "modified": + continue + if key == "evaluation": + continue + if key == "evaluation_message": + continue + if key == "type": + param_key = "type_value" + if key == "type_value": + continue + if key == "evaluation": + param_key = "evaluation_value" + if key == "evaluation_value": + continue + if isNotFirst: + query += "," + isNotFirst = True + query += "?" + param_key + "?" + query += ")" + return + +def create_sql_query_shipcall_put(schemaModel:dict)->str: + query = "UPDATE shipcall SET " + isNotFirst = False + for key in schemaModel.keys(): + param_key = key + if key == "id": + continue + if key == "participants": + continue + if key == "created": + continue + if key == "modified": + continue + if key == "evaluation": + continue + if key == "evaluation_message": + continue + if key == "type": + param_key = "type_value" + if key == "type_value": + continue + if key == "evaluation": + param_key = "evaluation_value" + if key == "evaluation_value": + continue + if isNotFirst: + query += ", " + isNotFirst = True + query += key + " = ?" + param_key + "? " + + query += "WHERE id = ?id?" + return query + + +def create_sql_query_history_post()->str: + query = "INSERT INTO history (participant_id, shipcall_id, user_id, timestamp, eta, type, operation) VALUES (?pid?, ?scid?, ?uid?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 1, 1)" + return query + +def create_sql_query_history_put()->str: + query = "INSERT INTO history (participant_id, shipcall_id, user_id, timestamp, eta, type, operation) VALUES (?pid?, ?scid?, ?uid?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 1, 2)" + return query + diff --git a/src/server/BreCal/impl/shipcalls.py b/src/server/BreCal/impl/shipcalls.py index de60788..0dfcf74 100644 --- a/src/server/BreCal/impl/shipcalls.py +++ b/src/server/BreCal/impl/shipcalls.py @@ -8,7 +8,7 @@ from .. import local_db from ..services.auth_guard import check_jwt from BreCal.database.update_database import evaluate_shipcall_state -from BreCal.database.sql_queries import create_sql_query_shipcall_get +from BreCal.database.sql_queries import create_sql_query_shipcall_get, create_sql_query_shipcall_post, create_sql_query_shipcall_put def GetShipcalls(options): """ @@ -63,6 +63,7 @@ def PostShipcalls(schemaModel): pooledConnection = local_db.getPoolConnection() commands = pydapper.using(pooledConnection) + # query = create_sql_query_shipcall_post(schemaModel) query = "INSERT INTO shipcall (" isNotFirst = False for key in schemaModel.keys(): @@ -145,6 +146,11 @@ def PostShipcalls(schemaModel): commands.execute(query, {"scid" : new_id, "pid" : user_data["participant_id"], "uid" : user_data["id"]}) return json.dumps({"id" : new_id}), 201, {'Content-Type': 'application/json; charset=utf-8'} + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 except Exception as ex: logging.error(traceback.format_exc()) @@ -179,6 +185,7 @@ def PutShipcalls(schemaModel): pooledConnection.close() return json.dumps("no such record"), 404, {'Content-Type': 'application/json; charset=utf-8'} + # query = create_sql_query_shipcall_put(schemaModel) query = "UPDATE shipcall SET " isNotFirst = False for key in schemaModel.keys(): @@ -258,6 +265,11 @@ def PutShipcalls(schemaModel): commands.execute(query, {"scid" : schemaModel["id"], "pid" : user_data["participant_id"], "uid" : user_data["id"]}) return json.dumps({"id" : schemaModel["id"]}), 200 + + except ValidationError as ex: + logging.error(ex) + print(ex) + return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 except Exception as ex: logging.error(traceback.format_exc()) diff --git a/src/server/BreCal/schemas/model.py b/src/server/BreCal/schemas/model.py index f346783..1eb75b8 100644 --- a/src/server/BreCal/schemas/model.py +++ b/src/server/BreCal/schemas/model.py @@ -187,7 +187,7 @@ class ShipcallSchema(Schema): super().__init__(unknown=None) pass - id = fields.Integer() + id = fields.Integer(metadata={'required':True}) ship_id = fields.Integer(metadata={'required':True}) #type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator type = fields.Integer(metadata={'required':True}) # make enum: shipcall type. add validator # type = fields.Enum(ShipcallType, default=ShipcallType.undefined) # type = fields.Integer() # make enum: shipcall type. add validator diff --git a/src/server/BreCal/validators/input_validation.py b/src/server/BreCal/validators/input_validation.py index 048d594..6cddf47 100644 --- a/src/server/BreCal/validators/input_validation.py +++ b/src/server/BreCal/validators/input_validation.py @@ -14,116 +14,10 @@ from BreCal.impl.berths import GetBerths from BreCal.database.enums import ParticipantType -def check_if_string_has_special_characters(text:str): - """ - check, whether there are any characters within the provided string, which are not found in the ascii letters or digits - ascii_letters: abcd (...) and ABCD (...) - digits: 0123 (...) - Source: https://stackoverflow.com/questions/57062794/is-there-a-way-to-check-if-a-string-contains-special-characters - User: https://stackoverflow.com/users/10035985/andrej-kesely - returns bool - """ - return bool(set(text).difference(ascii_letters + digits)) +from BreCal.validators.input_validation_utils import check_if_user_is_bsmd_type, get_participant_id_dictionary, check_if_ship_id_is_valid, check_if_berth_id_is_valid, check_if_participant_ids_are_valid, get_berth_id_dictionary, check_if_string_has_special_characters, get_ship_id_dictionary, check_if_int_is_valid_flag -def check_if_int_is_valid_flag(value, enum_object): - # e.g., when an IntFlag has the values 1,2,4; the maximum valid value is 7 - max_int = sum([int(val) for val in list(enum_object._value2member_map_.values())]) - return 0 < value <= max_int -def get_participant_id_dictionary(): - # get all participants - response,status_code,header = GetParticipant(options={}) - - # build a dictionary of id:item pairs, so one can select the respective participant - participants = json.loads(response) - participants = {items.get("id"):items for items in participants} - return participants - -def get_ship_id_dictionary(): - # get all ships - response,status_code,header = GetShips(token=None) - - # build a dictionary of id:item pairs, so one can select the respective participant - ships = json.loads(response) - ships = {items.get("id"):items for items in ships} - return ships - -def get_berth_id_dictionary(): - # get all berths - response,status_code,header = GetBerths(token=None) - - # build a dictionary of id:item pairs, so one can select the respective participant - berths = json.loads(response) - berths = {items.get("id"):items for items in berths} - return berths - -def check_if_user_is_bsmd_type(user_data:dict)->bool: - """ - given a dictionary of user data, determine the respective participant id and read, whether - that participant is a .BSMD-type - - Note: ParticipantType is an IntFlag. - Hence, ParticipantType(1) is ParticipantType.BSMD, - and ParticipantType(7) is [ParticipantType.BSMD, ParticipantType.TERMINAL, ParticipantType.PILOT] - - both would return 'True' - - returns: boolean. Whether the participant id is a .BSMD type element - """ - # user_data = decode token - participant_id = user_data.get("participant_id") - - # build a dictionary of id:item pairs, so one can select the respective participant - participants = get_participant_id_dictionary() - participant = participants.get(participant_id,{}) - - # boolean check: is the participant of type .BSMD? - is_bsmd = ParticipantType.BSMD in ParticipantType(participant.get("type",0)) - return is_bsmd - -def check_if_ship_id_is_valid(ship_id): - """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" - if ship_id is None: - return True - - # build a dictionary of id:item pairs, so one can select the respective participant - ships = get_ship_id_dictionary() - - # boolean check - ship_id_is_valid = ship_id in list(ships.keys()) - return ship_id_is_valid - -def check_if_berth_id_is_valid(berth_id): - """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" - if berth_id is None: - return True - - # build a dictionary of id:item pairs, so one can select the respective participant - berths = get_berth_id_dictionary() - - # boolean check - berth_id_is_valid = berth_id in list(berths.keys()) - return berth_id_is_valid - -def check_if_participant_id_is_valid(participant_id): - """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" - if participant_id is None: - return True - - # build a dictionary of id:item pairs, so one can select the respective participant - participants = get_participant_id_dictionary() - - # boolean check - participant_id_is_valid = participant_id in list(participants.keys()) - return participant_id_is_valid - -def check_if_participant_ids_are_valid(participant_ids): - # check each participant id individually - valid_participant_ids = [check_if_participant_id_is_valid(participant_id) for participant_id in participant_ids] - - # boolean check, whether all participant ids are valid - return all(valid_participant_ids) def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict): @@ -214,6 +108,8 @@ def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict if tidal_window_from is not None: if not tidal_window_from >= time_now: raise ValidationError(f"'tidal_window_from' must be in the future. Incorrect datetime provided.") + + # #TODO: assert tidal_window_from > tidal_window_to # #TODO: len of participants > 0, if agency # * assigned participant for agency diff --git a/src/server/BreCal/validators/input_validation_shipcall.py b/src/server/BreCal/validators/input_validation_shipcall.py index e69de29..d1ded52 100644 --- a/src/server/BreCal/validators/input_validation_shipcall.py +++ b/src/server/BreCal/validators/input_validation_shipcall.py @@ -0,0 +1,330 @@ +import json +import datetime +from abc import ABC, abstractmethod +from marshmallow import ValidationError +from string import ascii_letters, digits + +from BreCal.schemas.model import Ship, Shipcall, Berth, User, Participant, ShipcallType +from BreCal.impl.participant import GetParticipant +from BreCal.impl.ships import GetShips +from BreCal.impl.berths import GetBerths + +from BreCal.database.enums import ParticipantType, ParticipantFlag +from BreCal.validators.input_validation_utils import check_if_user_is_bsmd_type, check_if_ship_id_is_valid, check_if_berth_id_is_valid, check_if_participant_ids_are_valid, check_if_string_has_special_characters, get_shipcall_id_dictionary, get_participant_type_from_user_data, check_if_int_is_valid_flag + + +class InputValidationShipcall(): + """ + This class combines a complex set of individual input validation functions into a joint object. + It uses static methods, so the object does not need to be instantiated, but functions can be called immediately. + + Example: + InputValidationShipcall.evaluate(user_data, loadedModel, content) + + When the data violates one of the rules, a marshmallow.ValidationError is raised, which details the issues. + + """ + def __init__(self) -> None: + pass + + @staticmethod + def evaluate_post_data(user_data:dict, loadedModel:dict, content:dict): + """ + this function combines multiple validation functions to verify data, which is sent to the API as a shipcall's POST-request + + checks: + 1. permission: only participants that belong to the BSMD group are allowed to POST shipcalls + 2. reference checks: all refered objects within the Shipcall must exist + 3. reasonable values: validates the values within the Shipcall + 4. existance of required fields + """ + # check for permission (only BSMD-type participants) + InputValidationShipcall.check_user_is_bsmd_type(user_data) + + # check references (referred IDs must exist) + InputValidationShipcall.check_referenced_ids(loadedModel) + + # check for reasonable values in the shipcall fields + InputValidationShipcall.check_shipcall_values(loadedModel, content, forbidden_keys=["canceled", "evaluation", "evaluation_message"]) + + # POST-request only: check the existance of required fields based on the ShipcallType + InputValidationShipcall.check_required_fields_exist_based_on_type(loadedModel, content) + + # POST-request only: check the existance of a participant list, when the user is of type agency + InputValidationShipcall.check_participant_list_not_empty_when_user_is_agency(user_data, loadedModel) + return + + @staticmethod + def evaluate_put_data(user_data:dict, loadedModel:dict, content:dict): + """ + this function combines multiple validation functions to verify data, which is sent to the API as a shipcall's PUT-request + + checks: + 1. whether the user belongs to participant group type BSMD + 2. users of the agency may edit the shipcall, when the shipcall-participant-map entry lists them + 3. all value-rules of the POST evaluation + 4. a canceled shipcall may not be changed + 5. existance of required fields + """ + # check for permission (only BSMD-type participants) + InputValidationShipcall.check_user_is_bsmd_type(user_data) + + # check, whether an agency is listed in the shipcall-participant-map + # InputValidationShipcall.check_agency_in_shipcall_participant_map() # args? + + # check for reasonable values in the shipcall fields and checks for forbidden keys. Note: 'canceled' is allowed in PUT-requests. + InputValidationShipcall.check_shipcall_values(loadedModel, content, forbidden_keys=["evaluation", "evaluation_message"]) + + # a canceled shipcall cannot be selected + InputValidationShipcall.check_shipcall_is_canceled(loadedModel, content) + + # the ID field is required, all missing fields will be ignored in the update + InputValidationShipcall.check_required_fields_of_put_request(content) + return + + @staticmethod + def check_shipcall_values(loadedModel:dict, content:dict, forbidden_keys:list=["canceled", "evaluation", "evaluation_message"]): + """ + individually checks each value provided in the loadedModel/content. + This function validates, whether the values are reasonable. + + Also, some data may not be set in a POST-request. + """ + # Note: BreCal.schemas.model.ShipcallSchema has an internal validation, which the marshmallow library provides. This is used + # to verify values individually, when the schema is loaded with data. + # This function focuses on more complex input validation, which may require more sophisticated methods + + # loadedModel fills missing values, sometimes using optional values. Hence, the 'content'-variable is prefered for some of these verifications + # voyage shall not contain special characters + voyage_str_is_invalid = check_if_string_has_special_characters(text=content.get("voyage","")) + if voyage_str_is_invalid: + raise ValidationError(f"there are invalid characters in the 'voyage'-string. Please use only digits and ASCII letters. Allowed: {ascii_letters+digits}. Found: {content.get('voyage')}") + + # the 'flags' integer must be valid + flags_value = content.get("flags", 0) + if check_if_int_is_valid_flag(flags_value, enum_object=ParticipantFlag): + raise ValidationError(f"incorrect value provided for 'flags'. Must be a valid combination of the flags.") + + # time values must use future-dates + InputValidationShipcall.check_times_are_in_future(loadedModel, content) + + # some arguments must not be provided + InputValidationShipcall.check_forbidden_arguments(content, forbidden_keys=forbidden_keys) + return + + @staticmethod + def check_agency_in_shipcall_participant_map(): # args? + return + + @staticmethod + def check_user_is_bsmd_type(user_data): + """ + check, whether the user belongs to a participant, which is of type ParticipantType.BSMD + as ParticipantType is an IntFlag, a user belonging to multiple groups is properly evaluated. + """ + is_bsmd = check_if_user_is_bsmd_type(user_data) + if not is_bsmd: + raise ValidationError(f"current user does not belong to BSMD. Cannot post shipcalls. Found user data: {user_data}") + return + + @staticmethod + def check_referenced_ids(loadedModel): + """ + check, whether the referenced entries exist (e.g., when a Ship ID is referenced, but does not exist, the validation fails) + """ + # get all IDs from the loadedModel + ship_id = loadedModel.get("ship_id", None) + arrival_berth_id = loadedModel.get("arrival_berth_id", None) + departure_berth_id = loadedModel.get("departure_berth_id", None) + participant_ids = loadedModel.get("participants",[]) + + valid_ship_id = check_if_ship_id_is_valid(ship_id=ship_id) + if not valid_ship_id: + raise ValidationError(f"provided an invalid ship id, which is not found in the database: {ship_id}") + + valid_arrival_berth_id = check_if_berth_id_is_valid(berth_id=arrival_berth_id) + if not valid_arrival_berth_id: + raise ValidationError(f"provided an invalid arrival berth id, which is not found in the database: {arrival_berth_id}") + + valid_departure_berth_id = check_if_berth_id_is_valid(berth_id=departure_berth_id) + if not valid_departure_berth_id: + raise ValidationError(f"provided an invalid departure berth id, which is not found in the database: {departure_berth_id}") + + valid_participant_ids = check_if_participant_ids_are_valid(participant_ids=participant_ids) + if not valid_participant_ids: + raise ValidationError(f"one of the provided participant ids is invalid. Could not find one of these in the database: {participant_ids}") + return + + @staticmethod + def check_forbidden_arguments(content:dict, forbidden_keys=["canceled", "evaluation", "evaluation_message"]): + """ + a post-request must not contain the arguments 'canceled', 'evaluation', 'evaluation_message'. + a put-request must not contain the arguments 'evaluation', 'evaluation_message' + + """ + # the following keys should not be set in a POST-request. + for forbidden_key in forbidden_keys: + value = content.get(forbidden_key, None) + if value is not None: + raise ValidationError(f"'{forbidden_key}' may not be set on POST. Found: {value}") + return + + @staticmethod + def check_required_fields_exist_based_on_type(loadedModel:dict, content:dict): + """ + depending on the ShipcallType, some fields are *required* in a POST-request + """ + type_ = loadedModel.get("type", int(ShipcallType.undefined)) + eta = content.get("eta", None) + etd = content.get("etd", None) + arrival_berth_id = content.get("arrival_berth_id", None) + departure_berth_id = content.get("departure_berth_id", None) + + if int(type_)==int(ShipcallType.undefined): + raise ValidationError(f"providing 'type' is mandatory. Missing key!") + + # arrival: arrival_berth_id & eta must exist + elif int(type_)==int(ShipcallType.arrival): + if eta is None: + raise ValidationError(f"providing 'eta' is mandatory. Missing key!") + + if arrival_berth_id is None: + raise ValidationError(f"providing 'arrival_berth_id' is mandatory. Missing key!") + + # departure: departive_berth_id and etd must exist + elif int(type_)==int(ShipcallType.departure): + if etd is None: + raise ValidationError(f"providing 'etd' is mandatory. Missing key!") + + if departure_berth_id is None: + raise ValidationError(f"providing 'departure_berth_id' is mandatory. Missing key!") + + # shifting: arrival_berth_id, departure_berth_id, eta and etd must exist + elif int(type_)==int(ShipcallType.shifting): + if (eta is None) or (etd is None): + raise ValidationError(f"providing 'eta' and 'etd' is mandatory. Missing one of those keys!") + if (arrival_berth_id is None) or (departure_berth_id is None): + raise ValidationError(f"providing 'arrival_berth_id' & 'departure_berth_id' is mandatory. Missing key!") + + else: + raise ValidationError(f"incorrect 'type' provided!") + return + + @staticmethod + def check_times_are_in_future(loadedModel:dict, content:dict): + """ + Dates should be in the future. Depending on the ShipcallType, specific values should be checked + Perfornms datetime checks in the loadedModel (datetime.datetime objects). + """ + # obtain the current datetime to check, whether the provided values are in the future + time_now = datetime.datetime.now() + + type_ = loadedModel.get("type", int(ShipcallType.undefined)) + eta = loadedModel.get("eta") + etd = loadedModel.get("etd") + tidal_window_from = loadedModel.get("tidal_window_from", None) + tidal_window_to = loadedModel.get("tidal_window_to", None) + + # Estimated arrival or departure times + InputValidationShipcall.check_times_in_future_based_on_type(type_, time_now, eta, etd) + + # Tidal Window + InputValidationShipcall.check_tidal_window_in_future(time_now, tidal_window_from, tidal_window_to) + return + + @staticmethod + def check_times_in_future_based_on_type(type_, time_now, eta, etd): + """ + checks, whether the ETA & ETD times are in the future. + based on the type, this function checks: + arrival: eta + departure: etd + shifting: eta & etd + """ + if int(type_)==int(ShipcallType.undefined): + raise ValidationError(f"providing 'type' is mandatory. Missing key!") + elif int(type_)==int(ShipcallType.arrival): + if not eta >= time_now: + raise ValidationError(f"'eta' must be in the future. Incorrect datetime provided. Current Time: {time_now}. ETA: {eta}.") + elif int(type_)==int(ShipcallType.departure): + if not etd >= time_now: + raise ValidationError(f"'etd' must be in the future. Incorrect datetime provided. Current Time: {time_now}. ETD: {etd}.") + elif int(type_)==int(ShipcallType.shifting): + if (not eta >= time_now) or (not etd >= time_now): + raise ValidationError(f"'eta' and 'etd' must be in the future. Incorrect datetime provided. Current Time: {time_now}. ETA: {eta}. ETD: {etd}") + if (not eta >= etd): + raise ValidationError(f"'etd' must be larger than 'eta'. The ship cannot depart, before it has arrived. Found: ETA {eta}, ETA: {etd}") + return + + @staticmethod + def check_tidal_window_in_future(time_now, tidal_window_from, tidal_window_to): + if tidal_window_to is not None: + if not tidal_window_to >= time_now: + raise ValidationError(f"'tidal_window_to' must be in the future. Incorrect datetime provided.") + + if tidal_window_from is not None: + if not tidal_window_from >= time_now: + raise ValidationError(f"'tidal_window_from' must be in the future. Incorrect datetime provided.") + + if (tidal_window_to is not None) and (tidal_window_from is not None): + if tidal_window_to < tidal_window_from: + raise ValidationError(f"'tidal_window_to' must take place after 'tidal_window_from'. Incorrect datetime provided. Found 'tidal_window_to': {tidal_window_to}, 'tidal_window_from': {tidal_window_to}.") + return + + @staticmethod + def check_participant_list_not_empty_when_user_is_agency(user_data, loadedModel): + """ + participant types use an IntFlag to assign multiple roles to a user. When a user is assigned to the + AGENCY role, the user must provide a non-empty list of participants in POST-requests. + """ + participant_type = get_participant_type_from_user_data(user_data) + + if int(ParticipantType.AGENCY) in int(participant_type): + participants = loadedModel.get("participants",[]) + + if len(participants)==0: + raise ValidationError(f"A user of type 'ParticipantType.AGENCY' is required to provide a list of valid participants.") + return + + @staticmethod + def check_shipcall_is_canceled(loadedModel, content): + # read the shipcall_id from the PUT data + shipcall_id = loadedModel.get("id") + + # get all shipcalls in the database + shipcalls = get_shipcall_id_dictionary() + + # search for the matching shipcall in the database + shipcall = shipcalls.get(shipcall_id,{}) + + # if the *existing* shipcall in the database is canceled, it may not be changed + if shipcall.get("canceled", False): + raise ValidationError(f"The shipcall with id 'shipcall_id' is canceled. A canceled shipcall may not be changed.") + return + + @staticmethod + def check_required_fields_of_put_request(content:dict): + shipcall_id = content.get("id", None) + if shipcall_id is None: + raise ValidationError(f"A PUT request requires an 'id' to refer to.") + + + +""" +# copy +def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict): + ##### Section 1: check user_data ##### + # DONE: refactored + + ##### Section 2: check loadedModel ##### + # DONE: refactored + + ##### Section 3: check content ##### + # DONE: refactored + + + ##### Section 4: check loadedModel & content ##### + # DONE: refactored ET and BERTH ID existance check + # DONE: refactored 'time in future' checks + return +""" \ No newline at end of file diff --git a/src/server/BreCal/validators/input_validation_utils.py b/src/server/BreCal/validators/input_validation_utils.py new file mode 100644 index 0000000..dd38a4a --- /dev/null +++ b/src/server/BreCal/validators/input_validation_utils.py @@ -0,0 +1,146 @@ +import json +from string import ascii_letters, digits + +from BreCal.impl.participant import GetParticipant +from BreCal.impl.ships import GetShips +from BreCal.impl.berths import GetBerths +from BreCal.impl.shipcalls import GetShipcalls + +from BreCal.database.enums import ParticipantType + +def get_participant_id_dictionary(): + """ + get a dictionary of all participants, where the key is the participant's id, and the value is a dictionary + of common participant data (not a data model). + """ + # get all participants + response,status_code,header = GetParticipant(options={}) + + # build a dictionary of id:item pairs, so one can select the respective participant + participants = json.loads(response) + participants = {items.get("id"):items for items in participants} + return participants + +def get_berth_id_dictionary(): + # get all berths + response,status_code,header = GetBerths(token=None) + + # build a dictionary of id:item pairs, so one can select the respective participant + berths = json.loads(response) + berths = {items.get("id"):items for items in berths} + return berths + +def get_ship_id_dictionary(): + # get all ships + response,status_code,header = GetShips(token=None) + + # build a dictionary of id:item pairs, so one can select the respective participant + ships = json.loads(response) + ships = {items.get("id"):items for items in ships} + return ships + +def get_shipcall_id_dictionary(): + # get all ships + response,status_code,header = GetShipcalls(token=None) + + # build a dictionary of id:item pairs, so one can select the respective participant + shipcalls = json.loads(response) + shipcalls = {items.get("id"):items for items in shipcalls} + return shipcalls + + +def get_participant_type_from_user_data(user_data:dict)->ParticipantType: + # user_data = decode token + participant_id = user_data.get("participant_id") + + # build a dictionary of id:item pairs, so one can select the respective participant + participants = get_participant_id_dictionary() + participant = participants.get(participant_id,{}) + participant_type = ParticipantType(participant.get("type",0)) + return participant_type + +def check_if_user_is_bsmd_type(user_data:dict)->bool: + """ + given a dictionary of user data, determine the respective participant id and read, whether + that participant is a .BSMD-type + + Note: ParticipantType is an IntFlag. + Hence, ParticipantType(1) is ParticipantType.BSMD, + and ParticipantType(7) is [ParticipantType.BSMD, ParticipantType.TERMINAL, ParticipantType.PILOT] + + both would return 'True' + + returns: boolean. Whether the participant id is a .BSMD type element + """ + # use the decoded JWT token and extract the participant type + participant_type = get_participant_type_from_user_data(user_data) + + # boolean check: is the participant of type .BSMD? + is_bsmd = ParticipantType.BSMD in participant_type + return is_bsmd + + +def check_if_ship_id_is_valid(ship_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if ship_id is None: + return True + + # build a dictionary of id:item pairs, so one can select the respective participant + ships = get_ship_id_dictionary() + + # boolean check + ship_id_is_valid = ship_id in list(ships.keys()) + return ship_id_is_valid + +def check_if_berth_id_is_valid(berth_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + if berth_id is None: + return True + + # build a dictionary of id:item pairs, so one can select the respective participant + berths = get_berth_id_dictionary() + + # boolean check + berth_id_is_valid = berth_id in list(berths.keys()) + return berth_id_is_valid + +def check_if_participant_id_is_valid(participant_id): + """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" + # #TODO1: Daniel Schick: 'types may only appear once and must not include type "BSMD"' + + if participant_id is None: + return True + + # build a dictionary of id:item pairs, so one can select the respective participant + participants = get_participant_id_dictionary() + + # boolean check + participant_id_is_valid = participant_id in list(participants.keys()) + return participant_id_is_valid + +def check_if_participant_ids_are_valid(participant_ids): + # check each participant id individually + valid_participant_ids = [check_if_participant_id_is_valid(participant_id) for participant_id in participant_ids] + + # boolean check, whether all participant ids are valid + return all(valid_participant_ids) + + +def check_if_string_has_special_characters(text:str): + """ + check, whether there are any characters within the provided string, which are not found in the ascii letters or digits + ascii_letters: abcd (...) and ABCD (...) + digits: 0123 (...) + + Source: https://stackoverflow.com/questions/57062794/is-there-a-way-to-check-if-a-string-contains-special-characters + User: https://stackoverflow.com/users/10035985/andrej-kesely + returns bool + """ + return bool(set(text).difference(ascii_letters + digits)) + +def check_if_int_is_valid_flag(value, enum_object): + # e.g., when an IntFlag has the values 1,2,4; the maximum valid value is 7 + max_int = sum([int(val) for val in list(enum_object._value2member_map_.values())]) + return 0 < value <= max_int + + From 07c735a3f3423055f9bebea9ae97263d7742e63f Mon Sep 17 00:00:00 2001 From: Max Metz Date: Wed, 15 May 2024 00:31:14 +0200 Subject: [PATCH 30/30] extending the capabilities of InputValidationShipcall and performing unit tests to check proper implementation. --- src/server/BreCal/api/shipcalls.py | 11 +-- src/server/BreCal/database/sql_queries.py | 2 + src/server/BreCal/impl/shipcalls.py | 8 ++- .../BreCal/validators/input_validation.py | 3 +- .../validators/input_validation_shipcall.py | 68 ++++++++++--------- .../validators/input_validation_utils.py | 41 +++++++++-- 6 files changed, 88 insertions(+), 45 deletions(-) diff --git a/src/server/BreCal/api/shipcalls.py b/src/server/BreCal/api/shipcalls.py index 6dfef58..f22fed0 100644 --- a/src/server/BreCal/api/shipcalls.py +++ b/src/server/BreCal/api/shipcalls.py @@ -9,6 +9,7 @@ from BreCal.validators.input_validation_shipcall import InputValidationShipcall import logging import json +import traceback bp = Blueprint('shipcalls', __name__) @@ -47,16 +48,17 @@ def PostShipcalls(): user_data = check_jwt() # validate the posted shipcall data - validate_posted_shipcall_data(user_data, loadedModel, content) - # InputValidationShipcall.evaluate_post_data(user_data, loadedModel, content) + # validate_posted_shipcall_data(user_data, loadedModel, content) + InputValidationShipcall.evaluate_post_data(user_data, loadedModel, content) except ValidationError as ex: logging.error(ex) print(ex) - return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 + return json.dumps({"message":f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"}), 400 except Exception as ex: logging.error(ex) + logging.error(traceback.format_exc()) print(ex) return json.dumps("bad format"), 400 @@ -69,7 +71,6 @@ def PutShipcalls(): try: content = request.get_json(force=True) - logging.info(content) loadedModel = model.ShipcallSchema().load(data=content, many=False, partial=True) # read the user data from the JWT token (set when login is performed) @@ -85,7 +86,7 @@ def PutShipcalls(): except ValidationError as ex: logging.error(ex) print(ex) - return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 + return json.dumps({"message":f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"}), 400 except Exception as ex: logging.error(ex) diff --git a/src/server/BreCal/database/sql_queries.py b/src/server/BreCal/database/sql_queries.py index 0940254..55ddf90 100644 --- a/src/server/BreCal/database/sql_queries.py +++ b/src/server/BreCal/database/sql_queries.py @@ -1,3 +1,4 @@ +import logging def create_sql_query_shipcall_get(options:dict)->str: @@ -9,6 +10,7 @@ def create_sql_query_shipcall_get(options:dict)->str: options : dict. A dictionary, which must contains the 'past_days' key (int). Determines the range by which shipcalls are filtered. """ + logging.info(options) query = ("SELECT s.id as id, ship_id, type, eta, voyage, etd, arrival_berth_id, departure_berth_id, tug_required, pilot_required, " + "flags, s.pier_side, bunkering, replenishing_terminal, replenishing_lock, draft, tidal_window_from, " + "tidal_window_to, rain_sensitive_cargo, recommended_tugs, anchored, moored_lock, canceled, evaluation, " + diff --git a/src/server/BreCal/impl/shipcalls.py b/src/server/BreCal/impl/shipcalls.py index 0dfcf74..54f85cd 100644 --- a/src/server/BreCal/impl/shipcalls.py +++ b/src/server/BreCal/impl/shipcalls.py @@ -8,7 +8,7 @@ from .. import local_db from ..services.auth_guard import check_jwt from BreCal.database.update_database import evaluate_shipcall_state -from BreCal.database.sql_queries import create_sql_query_shipcall_get, create_sql_query_shipcall_post, create_sql_query_shipcall_put +from BreCal.database.sql_queries import create_sql_query_shipcall_get, create_sql_query_shipcall_post, create_sql_query_shipcall_put, create_sql_query_history_post, create_sql_query_history_put def GetShipcalls(options): """ @@ -142,6 +142,7 @@ def PostShipcalls(schemaModel): # save history data # TODO: set ETA properly user_data = check_jwt() + # query = create_sql_query_history_post() query = "INSERT INTO history (participant_id, shipcall_id, user_id, timestamp, eta, type, operation) VALUES (?pid?, ?scid?, ?uid?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 1, 1)" commands.execute(query, {"scid" : new_id, "pid" : user_data["participant_id"], "uid" : user_data["id"]}) @@ -150,7 +151,7 @@ def PostShipcalls(schemaModel): except ValidationError as ex: logging.error(ex) print(ex) - return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 + return json.dumps({"message":f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"}), 400 except Exception as ex: logging.error(traceback.format_exc()) @@ -261,6 +262,7 @@ def PutShipcalls(schemaModel): # save history data # TODO: set ETA properly user_data = check_jwt() + # query = create_sql_query_history_put() query = "INSERT INTO history (participant_id, shipcall_id, user_id, timestamp, eta, type, operation) VALUES (?pid?, ?scid?, ?uid?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 1, 2)" commands.execute(query, {"scid" : schemaModel["id"], "pid" : user_data["participant_id"], "uid" : user_data["id"]}) @@ -269,7 +271,7 @@ def PutShipcalls(schemaModel): except ValidationError as ex: logging.error(ex) print(ex) - return json.dumps(f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"), 400 + return json.dumps({"message":f"bad format. \nError Messages: {ex.messages}. \nValid Data: {ex.valid_data}"}), 400 except Exception as ex: logging.error(traceback.format_exc()) diff --git a/src/server/BreCal/validators/input_validation.py b/src/server/BreCal/validators/input_validation.py index 6cddf47..3e5a801 100644 --- a/src/server/BreCal/validators/input_validation.py +++ b/src/server/BreCal/validators/input_validation.py @@ -22,6 +22,7 @@ from BreCal.validators.input_validation_utils import check_if_user_is_bsmd_type, def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict): """this function applies more complex validation functions to data, which is sent to a post-request of shipcalls""" + # DEPRECATED: this function has been refactored into InputValidationShipcall (see methods for POST and PUT evaluation) # #TODO_refactor: this function is pretty complex. One may instead build an object, which calls the methods separately. ##### Section 1: check user_data ##### @@ -44,7 +45,7 @@ def validate_posted_shipcall_data(user_data:dict, loadedModel:dict, content:dict if not valid_departure_berth_id: raise ValidationError(f"provided an invalid departure berth id, which is not found in the database: {loadedModel.get('departure_berth_id', None)}") - valid_participant_ids = check_if_participant_ids_are_valid(participant_ids=loadedModel.get("participants",[])) + valid_participant_ids = check_if_participant_ids_are_valid(participants=loadedModel.get("participants",[])) if not valid_participant_ids: raise ValidationError(f"one of the provided participant ids is invalid. Could not find one of these in the database: {loadedModel.get('participants', None)}") diff --git a/src/server/BreCal/validators/input_validation_shipcall.py b/src/server/BreCal/validators/input_validation_shipcall.py index d1ded52..4fdae69 100644 --- a/src/server/BreCal/validators/input_validation_shipcall.py +++ b/src/server/BreCal/validators/input_validation_shipcall.py @@ -10,7 +10,7 @@ from BreCal.impl.ships import GetShips from BreCal.impl.berths import GetBerths from BreCal.database.enums import ParticipantType, ParticipantFlag -from BreCal.validators.input_validation_utils import check_if_user_is_bsmd_type, check_if_ship_id_is_valid, check_if_berth_id_is_valid, check_if_participant_ids_are_valid, check_if_string_has_special_characters, get_shipcall_id_dictionary, get_participant_type_from_user_data, check_if_int_is_valid_flag +from BreCal.validators.input_validation_utils import check_if_user_is_bsmd_type, check_if_ship_id_is_valid, check_if_berth_id_is_valid, check_if_participant_ids_are_valid, check_if_participant_ids_and_types_are_valid, check_if_string_has_special_characters, get_shipcall_id_dictionary, get_participant_type_from_user_data, check_if_int_is_valid_flag class InputValidationShipcall(): @@ -35,8 +35,8 @@ class InputValidationShipcall(): checks: 1. permission: only participants that belong to the BSMD group are allowed to POST shipcalls 2. reference checks: all refered objects within the Shipcall must exist - 3. reasonable values: validates the values within the Shipcall - 4. existance of required fields + 3. existance of required fields + 4. reasonable values: validates the values within the Shipcall """ # check for permission (only BSMD-type participants) InputValidationShipcall.check_user_is_bsmd_type(user_data) @@ -44,14 +44,14 @@ class InputValidationShipcall(): # check references (referred IDs must exist) InputValidationShipcall.check_referenced_ids(loadedModel) - # check for reasonable values in the shipcall fields - InputValidationShipcall.check_shipcall_values(loadedModel, content, forbidden_keys=["canceled", "evaluation", "evaluation_message"]) - # POST-request only: check the existance of required fields based on the ShipcallType InputValidationShipcall.check_required_fields_exist_based_on_type(loadedModel, content) # POST-request only: check the existance of a participant list, when the user is of type agency - InputValidationShipcall.check_participant_list_not_empty_when_user_is_agency(user_data, loadedModel) + InputValidationShipcall.check_participant_list_not_empty_when_user_is_agency(loadedModel) + + # check for reasonable values in the shipcall fields + InputValidationShipcall.check_shipcall_values(loadedModel, content, forbidden_keys=["canceled", "evaluation", "evaluation_message"]) return @staticmethod @@ -62,9 +62,9 @@ class InputValidationShipcall(): checks: 1. whether the user belongs to participant group type BSMD 2. users of the agency may edit the shipcall, when the shipcall-participant-map entry lists them - 3. all value-rules of the POST evaluation - 4. a canceled shipcall may not be changed - 5. existance of required fields + 3. existance of required fields + 4. all value-rules of the POST evaluation + 5. a canceled shipcall may not be changed """ # check for permission (only BSMD-type participants) InputValidationShipcall.check_user_is_bsmd_type(user_data) @@ -72,14 +72,14 @@ class InputValidationShipcall(): # check, whether an agency is listed in the shipcall-participant-map # InputValidationShipcall.check_agency_in_shipcall_participant_map() # args? + # the ID field is required, all missing fields will be ignored in the update + InputValidationShipcall.check_required_fields_of_put_request(content) + # check for reasonable values in the shipcall fields and checks for forbidden keys. Note: 'canceled' is allowed in PUT-requests. InputValidationShipcall.check_shipcall_values(loadedModel, content, forbidden_keys=["evaluation", "evaluation_message"]) # a canceled shipcall cannot be selected InputValidationShipcall.check_shipcall_is_canceled(loadedModel, content) - - # the ID field is required, all missing fields will be ignored in the update - InputValidationShipcall.check_required_fields_of_put_request(content) return @staticmethod @@ -136,7 +136,7 @@ class InputValidationShipcall(): ship_id = loadedModel.get("ship_id", None) arrival_berth_id = loadedModel.get("arrival_berth_id", None) departure_berth_id = loadedModel.get("departure_berth_id", None) - participant_ids = loadedModel.get("participants",[]) + participants = loadedModel.get("participants",[]) valid_ship_id = check_if_ship_id_is_valid(ship_id=ship_id) if not valid_ship_id: @@ -150,11 +150,14 @@ class InputValidationShipcall(): if not valid_departure_berth_id: raise ValidationError(f"provided an invalid departure berth id, which is not found in the database: {departure_berth_id}") - valid_participant_ids = check_if_participant_ids_are_valid(participant_ids=participant_ids) + valid_participant_ids = check_if_participant_ids_are_valid(participants=participants) if not valid_participant_ids: - raise ValidationError(f"one of the provided participant ids is invalid. Could not find one of these in the database: {participant_ids}") - return - + raise ValidationError(f"one of the provided participant ids are invalid. Could not find one of these in the database: {participants}") + + valid_participant_types = check_if_participant_ids_and_types_are_valid(participants=participants) + if not valid_participant_types: + raise ValidationError(f"every participant id and type should be listed only once. Found multiple entries for one of the participants.") + @staticmethod def check_forbidden_arguments(content:dict, forbidden_keys=["canceled", "evaluation", "evaluation_message"]): """ @@ -175,11 +178,15 @@ class InputValidationShipcall(): depending on the ShipcallType, some fields are *required* in a POST-request """ type_ = loadedModel.get("type", int(ShipcallType.undefined)) + ship_id = content.get("ship_id", None) eta = content.get("eta", None) etd = content.get("etd", None) arrival_berth_id = content.get("arrival_berth_id", None) departure_berth_id = content.get("departure_berth_id", None) + if ship_id is None: + raise ValidationError(f"providing 'ship_id' is mandatory. Missing key!") + if int(type_)==int(ShipcallType.undefined): raise ValidationError(f"providing 'type' is mandatory. Missing key!") @@ -244,16 +251,16 @@ class InputValidationShipcall(): if int(type_)==int(ShipcallType.undefined): raise ValidationError(f"providing 'type' is mandatory. Missing key!") elif int(type_)==int(ShipcallType.arrival): - if not eta >= time_now: + if not eta > time_now: raise ValidationError(f"'eta' must be in the future. Incorrect datetime provided. Current Time: {time_now}. ETA: {eta}.") elif int(type_)==int(ShipcallType.departure): - if not etd >= time_now: + if not etd > time_now: raise ValidationError(f"'etd' must be in the future. Incorrect datetime provided. Current Time: {time_now}. ETD: {etd}.") elif int(type_)==int(ShipcallType.shifting): - if (not eta >= time_now) or (not etd >= time_now): + if (not eta > time_now) or (not etd > time_now): raise ValidationError(f"'eta' and 'etd' must be in the future. Incorrect datetime provided. Current Time: {time_now}. ETA: {eta}. ETD: {etd}") - if (not eta >= etd): - raise ValidationError(f"'etd' must be larger than 'eta'. The ship cannot depart, before it has arrived. Found: ETA {eta}, ETA: {etd}") + if (not etd > eta): + raise ValidationError(f"'etd' must be larger than 'eta'. The ship cannot depart, before it has arrived. Found: ETA {eta}, ETD: {etd}") return @staticmethod @@ -272,18 +279,15 @@ class InputValidationShipcall(): return @staticmethod - def check_participant_list_not_empty_when_user_is_agency(user_data, loadedModel): + def check_participant_list_not_empty_when_user_is_agency(loadedModel): """ - participant types use an IntFlag to assign multiple roles to a user. When a user is assigned to the - AGENCY role, the user must provide a non-empty list of participants in POST-requests. + For each POST request, one of the participants in the list must be assigned as a ParticipantType.AGENCY """ - participant_type = get_participant_type_from_user_data(user_data) + participants = loadedModel.get("participants", []) + is_agency_participant = [ParticipantType.AGENCY in ParticipantType(participant.get("type")) for participant in participants] - if int(ParticipantType.AGENCY) in int(participant_type): - participants = loadedModel.get("participants",[]) - - if len(participants)==0: - raise ValidationError(f"A user of type 'ParticipantType.AGENCY' is required to provide a list of valid participants.") + if not any(is_agency_participant): + raise ValidationError(f"One of the assigned participants *must* be of type 'ParticipantType.AGENCY'. Found list of participants: {participants}") return @staticmethod diff --git a/src/server/BreCal/validators/input_validation_utils.py b/src/server/BreCal/validators/input_validation_utils.py index dd38a4a..4476041 100644 --- a/src/server/BreCal/validators/input_validation_utils.py +++ b/src/server/BreCal/validators/input_validation_utils.py @@ -1,5 +1,7 @@ +import logging import json from string import ascii_letters, digits +from collections import Counter from BreCal.impl.participant import GetParticipant from BreCal.impl.ships import GetShips @@ -104,9 +106,16 @@ def check_if_berth_id_is_valid(berth_id): berth_id_is_valid = berth_id in list(berths.keys()) return berth_id_is_valid -def check_if_participant_id_is_valid(participant_id): - """check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once""" +def check_if_participant_id_is_valid(participant:dict): + """ + check, whether the provided ID is valid. If it is 'None', it will be considered valid. This is, because a shipcall POST-request, does not have to include all IDs at once + + Following the common BreCal.schemas.model.ParticipantAssignmentSchema, a participant dictionary contains the keys: + 'participant_id' : int + 'type' : ParticipantType + """ # #TODO1: Daniel Schick: 'types may only appear once and must not include type "BSMD"' + participant_id = participant.get("participant_id", None) if participant_id is None: return True @@ -118,13 +127,37 @@ def check_if_participant_id_is_valid(participant_id): participant_id_is_valid = participant_id in list(participants.keys()) return participant_id_is_valid -def check_if_participant_ids_are_valid(participant_ids): +def check_if_participant_ids_are_valid(participants:list[dict]): + """ + + args: + participants (list of participant-elements) + Following the common BreCal.schemas.model.ParticipantAssignmentSchema, a participant dictionary contains the keys: + 'participant_id' : int + 'type' : ParticipantType + """ # check each participant id individually - valid_participant_ids = [check_if_participant_id_is_valid(participant_id) for participant_id in participant_ids] + valid_participant_ids = [check_if_participant_id_is_valid(participant) for participant in participants] # boolean check, whether all participant ids are valid return all(valid_participant_ids) +def check_if_participant_ids_and_types_are_valid(participants:list[dict[str,int]]): + # creates a Counter object, which counts the number of unique elements + # key of counter: type, value of counter: number of listings in 'participants' + # e.g., {1: 4, 2: 1, 8: 1} (type 1 occurs 4 times in this example) + counter_type = Counter([participant.get("type") for participant in participants]) + counter_id = Counter([participant.get("type") for participant in participants]) + + # obtains the maximum count from the counter's values + max_count_type = max(list(counter_type.values())) if len(list(counter_type.values()))>0 else 0 + max_count_ids = max(list(counter_id.values())) if len(list(counter_id.values()))>0 else 0 + + # when 0 or 1 count for the participant ids or types, return true. Return false, when there is more than one entry. + return max_count_type <= 1 and max_count_ids <= 1 + + + def check_if_string_has_special_characters(text:str): """