Initializing pool connection variable with None.

Release pool connection handle und all circumstances especially also when a query fails
before the call is finished. This should avoid connection starvation.

fix prod. link

production fix

Fixed application path

.vscode/launch.json

@@ -12,7 +12,8 @@
             "env": {
                 "FLASK_APP": "src/server/BreCal",
                 "FLASK_DEBUG": "1",
-                "SECRET_KEY" : "zdiTz8P3jXOc7jztIQAoelK4zztyuCpJ" // https://randomkeygen.com/
+                "SECRET_KEY" : "zdiTz8P3jXOc7jztIQAoelK4zztyuCpJ", // https://randomkeygen.com/
+                "FLASK_RUN_PORT": "5000"
             },
             "args": [
                 "run",

src/server/BreCal/__init__.py

@@ -69,7 +69,7 @@ def create_app(test_config=None, instance_path=None):
     app.register_blueprint(history.bp)
     app.register_blueprint(ports.bp)
 
-    logging.basicConfig(filename='brecaldevel.log', level=logging.DEBUG, format='%(asctime)s | %(name)s | %(levelname)s | %(message)s')
+    logging.basicConfig(filename='brecal.log', level=logging.WARNING, format='%(asctime)s | %(name)s | %(levelname)s | %(message)s')
     local_db.initPool(os.path.dirname(app.instance_path))
     logging.info('App started')
 

src/server/BreCal/impl/berths.py

@@ -10,6 +10,7 @@ def GetBerths(options):
     No parameters, gets all entries
     """
 
+    pooledConnection = None
     try:
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)

src/server/BreCal/impl/history.py

@@ -16,6 +16,8 @@ def GetHistory(options):
         options["shipcall_id"]: **Id of shipcall**.
     """
 
+    pooledConnection = None
+    data = []
     try:
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
@@ -26,10 +28,6 @@ def GetHistory(options):
             data = commands.query("SELECT id, participant_id, shipcall_id, timestamp, eta, type, operation FROM history WHERE shipcall_id = ?shipcallid?",
                                   model=History.from_query_row,
                                   param={"shipcallid" : options["shipcall_id"]})
-
-
-        pooledConnection.close()
-
     except Exception as ex:
         pdb.pm()
         logging.error(ex)
@@ -37,6 +35,9 @@ def GetHistory(options):
         result = {}
         result["error_field"] = "call failed"
         return json.dumps("call failed"), 500
+    finally:
+        if pooledConnection is not None:
+            pooledConnection.close()
 
     return json.dumps(data, default=model.obj_dict), 200, {'Content-Type': 'application/json; charset=utf-8'}
 

src/server/BreCal/impl/login.py

@@ -6,14 +6,15 @@ import bcrypt
 from ..schemas import model
 from .. import local_db
 from ..services import jwt_handler
-from BreCal.database.sql_queries import SQLQuery
 
 
 def GetUser(options):
 
+    pooledConnection = None
+    
     try:
         if "password" in options and "username" in options:
-            hash = bcrypt.hashpw(options["password"].encode('utf-8'), bcrypt.gensalt( 12 )).decode('utf8')
+            
             pooledConnection = local_db.getPoolConnection()
             commands = pydapper.using(pooledConnection)
             # query = SQLQuery.get_user()
@@ -63,7 +64,3 @@ def GetUser(options):
     finally:
         if pooledConnection is not None:
             pooledConnection.close()
-
-# $2b$12$uWLE0r32IrtCV30WkMbVwOdltgeibymZyYAf4ZnQb2Bip8hrkGGwG
-# $2b$12$.vEapj9xU8z0RK0IpIGeYuRIl0ktdMt4XdJQBhVn.3K2hmvm7qD3y
-# $2b$12$yL3PiseU70ciwEuMVM4OtuMwR6tNuIT9vvBiBG/uyMrPxa16E2Zqu

src/server/BreCal/impl/notifications.py

@@ -11,13 +11,12 @@ def GetNotifications(token):
     No parameters, gets all entries
     """
 
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
         data = commands.query("SELECT id, shipcall_id, participant_id, level, type, message, created, modified FROM notification " +
                               "WHERE level = 2", model=model.Notification.from_query_row)
-        pooledConnection.close()
 
     except Exception as ex:
         logging.error(ex)
@@ -25,6 +24,9 @@ def GetNotifications(token):
         result = {}
         result["error_field"] = "call failed"
         return json.dumps(result), 500, {'Content-Type': 'application/json; charset=utf-8'}
+    finally:
+        if pooledConnection is not None:
+            pooledConnection.close()
 
     return json.dumps(data, default=model.obj_dict), 200, {'Content-Type': 'application/json; charset=utf-8'}
 

src/server/BreCal/impl/participant.py

@@ -12,6 +12,7 @@ def GetParticipant(options):
         options["user_id"]: **Id of user**. *Example: 2*. User id returned by login call.
     """
 
+    pooledConnection = None
     try:
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)

src/server/BreCal/impl/ports.py

@@ -11,6 +11,7 @@ def GetPorts(token):
     No parameters, gets all entries
     """
 
+    pooledConnection = None
     try:
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)

src/server/BreCal/impl/shipcalls.py

@@ -18,8 +18,8 @@ def GetShipcalls(options):
     No parameters, gets all entries
     """
 
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
         # query = SQLQuery.get_shipcalls(options)
@@ -70,8 +70,8 @@ def PostShipcalls(schemaModel):
     """
 
     # This creates a *new* entry
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
 
@@ -192,8 +192,8 @@ def PutShipcalls(schemaModel):
     """
 
      # This updates an *existing* entry
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
 
@@ -205,7 +205,6 @@ def PutShipcalls(schemaModel):
 
         theshipcall = commands.query_single_or_default("SELECT * FROM shipcall where id = ?id?", sentinel, param={"id" : schemaModel["id"]})
         if theshipcall is sentinel:
-            pooledConnection.close()
             return json.dumps("no such record"), 404, {'Content-Type': 'application/json; charset=utf-8'}
         
         was_canceled = theshipcall["canceled"]

src/server/BreCal/impl/ships.py

@@ -11,8 +11,8 @@ def GetShips(token):
     No parameters, gets all entries
     """
 
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
         # query = SQLQuery.get_ships()
@@ -44,8 +44,8 @@ def PostShip(schemaModel):
     # TODO: Validate the incoming data
 
     # This creates a *new* entry
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
 
@@ -83,8 +83,6 @@ def PostShip(schemaModel):
         # new_id = commands.execute_scalar(nquery)
         new_id = commands.execute_scalar("select last_insert_id()")
 
-        pooledConnection.close()
-
         return json.dumps({"id" : new_id}), 201, {'Content-Type': 'application/json; charset=utf-8'}
 
     except Exception as ex:
@@ -93,6 +91,9 @@ def PostShip(schemaModel):
         result = {}
         result["error_field"] = "call failed"
         return json.dumps(result), 500, {'Content-Type': 'application/json; charset=utf-8'}
+    finally:
+        if pooledConnection is not None:
+            pooledConnection.close()
 
 
 def PutShip(schemaModel):
@@ -101,8 +102,8 @@ def PutShip(schemaModel):
     """
 
     # This updates an *existing* entry
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
 
@@ -125,8 +126,6 @@ def PutShip(schemaModel):
 
         affected_rows = commands.execute(query, param=schemaModel)
 
-        pooledConnection.close()
-
         return json.dumps({"id" : schemaModel["id"]}), 200, {'Content-Type': 'application/json; charset=utf-8'}
 
     except Exception as ex:
@@ -135,6 +134,9 @@ def PutShip(schemaModel):
         result = {}
         result["error_field"] = "call failed"
         return json.dumps(result), 500, {'Content-Type': 'application/json; charset=utf-8'}
+    finally:
+        if pooledConnection is not None:
+            pooledConnection.close()
 
 
 def DeleteShip(options):
@@ -143,16 +145,14 @@ def DeleteShip(options):
         options["id"]
 
     """
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
         # query = SQLQuery.get_ship_delete_by_id()
         # affected_rows = commands.execute(query, param={"id" : options["id"]})
         affected_rows = commands.execute("UPDATE ship SET deleted = 1 WHERE id = ?id?", param={"id" : options["id"]})
 
-        pooledConnection.close()
-
         if affected_rows == 1:
             return json.dumps({"id" : options["id"]}), 200, {'Content-Type': 'application/json; charset=utf-8'}
 
@@ -166,3 +166,6 @@ def DeleteShip(options):
         result = {}
         result["error_field"] = "call failed"
         return json.dumps(result), 500, {'Content-Type': 'application/json; charset=utf-8'}
+    finally:
+        if pooledConnection is not None:
+            pooledConnection.close()

src/server/BreCal/impl/times.py

@@ -18,8 +18,8 @@ def GetTimes(options):
 
     """
 
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
         # query = SQLQuery.get_times()
@@ -28,7 +28,6 @@ def GetTimes(options):
                               "zone_entry, zone_entry_fixed, operations_start, operations_end, remarks, shipcall_id, participant_id, " +
                               "berth_id, berth_info, pier_side, participant_type, created, modified, ata, atd, eta_interval_end, etd_interval_end FROM times " +
                               "WHERE times.shipcall_id = ?scid?", model=model.Times, param={"scid" : options["shipcall_id"]})
-        pooledConnection.close()
 
     except Exception as ex:
         logging.error(traceback.format_exc())
@@ -38,6 +37,10 @@ def GetTimes(options):
         result["error_field"] = "call failed"
         return json.dumps(result), 500, {'Content-Type': 'application/json; charset=utf-8'}
 
+    finally:
+        if pooledConnection is not None:
+            pooledConnection.close()
+
     return json.dumps(data, default=model.obj_dict), 200, {'Content-Type': 'application/json; charset=utf-8'}
 
 
@@ -51,8 +54,8 @@ def PostTimes(schemaModel):
     # TODO: Validate the upload data
 
     # This creates a *new* entry
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
 
@@ -119,8 +122,8 @@ def PutTimes(schemaModel):
     """
 
     # This updates an *existing* entry
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
 
@@ -177,8 +180,8 @@ def DeleteTimes(options):
         options["id"]
 
     """
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
         shipcall_id = commands.execute_scalar("SELECT shipcall_id FROM times WHERE id = ?id?", param={"id" : options["id"]})

src/server/BreCal/impl/user.py

@@ -14,8 +14,8 @@ def PutUser(schemaModel):
     """
 
      # This updates an *existing* entry
+    pooledConnection = None
     try:
-
         pooledConnection = local_db.getPoolConnection()
         commands = pydapper.using(pooledConnection)
 
@@ -26,7 +26,6 @@ def PutUser(schemaModel):
         # theuser = commands.query_single_or_default(query, sentinel, param={"id" : schemaModel["id"]}, model=model.User)
         theuser = commands.query_single_or_default("SELECT * FROM user where id = ?id?", sentinel, param={"id" : schemaModel["id"]}, model=model.User)
         if theuser is sentinel:
-            pooledConnection.close()
             # #TODO: result = {"message":"no such record"} -> json.dumps
             return json.dumps("no such record"), 404, {'Content-Type': 'application/json; charset=utf-8'}
 

src/server/BreCal/local_db.py

@@ -1,58 +1,83 @@
 import mysql.connector
+from mysql.connector import pooling
 import pydapper
 import logging
 import json
 import os
 import sys
 from BreCal.schemas import defs
+
 config_path = None
+_connection_pool = None
 
-def initPool(instancePath, connection_filename="connection_data_devel.json"):
+
+def _load_json(path):
+    with open(path, encoding="utf-8") as fh:
+        return json.load(fh)
+
+
+def _build_pool_config(connection_data, pool_name, pool_size):
+    pool_config = dict(connection_data)
+    pool_config.setdefault("pool_name", pool_name)
+    pool_config.setdefault("pool_size", pool_size)
+    return pool_config
+
+
+def initPool(instancePath, connection_filename="connection_data_prod.json",
+             pool_name="brecal_pool", pool_size=10):
+    """
+    Initialize the MySQL connection pool and load email credentials.
+    """
+    global config_path, _connection_pool
     try:
-        global config_path
-        if(config_path == None):
-            config_path = os.path.join(instancePath,f'../../../secure/{connection_filename}') #connection_data_devel.json');
+        if config_path is None:
+            config_path = os.path.join(instancePath, f'../../../secure/{connection_filename}')
+            
+        # config_path = 'C:\\temp\\connection_data_test.json'
 
-        # config_path = "E:/temp/connection_data.json"
         print(config_path)
-
         if not os.path.exists(config_path):
             print('cannot find ' + os.path.abspath(config_path))
             print("instance path", instancePath)
             sys.exit(1)
 
-        f = open(config_path);
-        connection_data = json.load(f)
-        f.close()
-
-        conn_from_pool = mysql.connector.connect(**connection_data)
+        connection_data = _load_json(config_path)
+        if _connection_pool is None:
+            pool_config = _build_pool_config(connection_data, pool_name, pool_size)
+            _connection_pool = pooling.MySQLConnectionPool(**pool_config)
 
+        conn_from_pool = _connection_pool.get_connection()
+        try:
             commands = pydapper.using(conn_from_pool)
-        data = commands.query("SELECT id from `user`")
+            commands.query("SELECT id from `user` LIMIT 1")
             print("DB connection successful")
+        finally:
             conn_from_pool.close()
 
-        credentials_file = "email_credentials_devel.json"
+        credentials_file = "email_credentials_test.json"
         credentials_path = os.path.join(instancePath, f'../../../secure/{credentials_file}')
 
-        # credentials_path = "E:/temp/email_credentials_devel.json"
+        # credentials_path = 'C:\\temp\\email_credentials_test.json'
 
         if not os.path.exists(credentials_path):
             print('cannot find ' + os.path.abspath(credentials_path))
             sys.exit(1)
 
-        f = open(credentials_path);
-        defs.email_credentials = json.load(f)
-        f.close()
+        defs.email_credentials = _load_json(credentials_path)
 
     except mysql.connector.PoolError as e:
         logging.error(f"Failed to create connection pool: {e}")
         print(e)
     except Exception as e:
+        logging.error("Failed to initialize DB pool: %s", e)
         print(e)
 
+
 def getPoolConnection():
-    global config_path
-    f = open(config_path);
-    connection_data = json.load(f)
-    return mysql.connector.connect(**connection_data)
+    if _connection_pool is None:
+        raise RuntimeError("Connection pool not initialized. Call initPool first.")
+    try:
+        return _connection_pool.get_connection()
+    except mysql.connector.PoolError as exc:
+        logging.error("Connection pool exhausted: %s", exc)
+        raise

src/server/BreCal/services/schedule_routines.py

@@ -31,6 +31,7 @@ def UpdateShipcalls(options:dict = {'past_days':2}):
     options:
         key: 'past_days'. Is used to execute a filtered query of all available shipcalls. Defaults to 2 (days)
     """
+    pooledConnection = None
     try:
         pooledConnection = getPoolConnection()
         commands = pydapper.using(pooledConnection)
@@ -49,10 +50,11 @@ def UpdateShipcalls(options:dict = {'past_days':2}):
             # apply 'Traffic Light' evaluation to obtain 'GREEN', 'YELLOW' or 'RED' evaluation state. The function internally updates the mysql database
             evaluate_shipcall_state(mysql_connector_instance=pooledConnection, shipcall_id=shipcall_id) # new_id (last insert id) refers to the shipcall id
 
-        pooledConnection.close()
-
     except Exception as ex:
         logging.error(ex)
+    finally:
+        if pooledConnection is not None:
+            pooledConnection.close()
     return
 
 def UpdateNotifications(cooldown_in_mins:int=10):
@@ -61,6 +63,7 @@ def UpdateNotifications(cooldown_in_mins:int=10):
     notification is updated to state 1 and a notification is received by the user
     """
 
+    pooledConnection = None
     try:
         pooledConnection = getPoolConnection()
         commands = pydapper.using(pooledConnection)
@@ -70,32 +73,39 @@ def UpdateNotifications(cooldown_in_mins:int=10):
         for notification in data:
             commands.execute("UPDATE notification SET level = 1 WHERE id = ?id?", param={"id":notification.id})
 
-        pooledConnection.close()
     except Exception as ex:
         logging.error(ex)
+    finally:
+        if pooledConnection is not None:
+            pooledConnection.close()
 
 def ClearNotifications(max_age_in_days:int=3):
     """
     This function clears all notifications in state ("level") 2 that are older than x days
     """
 
+    pooledConnection = None
     try:
         pooledConnection = getPoolConnection()
         commands = pydapper.using(pooledConnection)
 
         query = f"DELETE FROM notification WHERE level = 2 and created < TIMESTAMP(NOW() - INTERVAL {max_age_in_days} DAY)"
         result = commands.execute(query)
-        pooledConnection.close()
         if(result > 0):
             logging.info(f"Deleted {result} notifications")
 
     except Exception as ex:
         logging.error(ex)
+    finally:
+        if pooledConnection is not None:
+            pooledConnection.close()
 
 def SendEmails(email_dict):
     """
     This function sends emails to all users in the emaildict
     """
+    pooledConnection = None
+    conn = None
     try:
         pooledConnection = getPoolConnection()
         commands = pydapper.using(pooledConnection)
@@ -177,10 +187,13 @@ def SendEmails(email_dict):
     finally:
         if conn is not None:
             conn.quit()
+        if pooledConnection is not None:
+            pooledConnection.close()
 
 
 def SendNotifications():
     # perhaps this will be moved somewhere else later
+    pooledConnection = None
     try:
         # find all notifications in level 1
         pooledConnection = getPoolConnection()
@@ -271,6 +284,7 @@ def add_function_to_schedule_send_notifications(interval_in_minutes:int=1):
     return
 
 def eval_next_24_hrs():
+    pooledConnection = None
     try:
         pooledConnection = getPoolConnection()
         commands = pydapper.using(pooledConnection)

src/server/BreCal/validators/validation_rules.py

@@ -90,6 +90,8 @@ class ValidationRules(ValidationRuleFunctions):
         if evaluation_states_old is not None and evaluation_states_new is not None:
             if len(evaluation_states_old) == 1 and len(evaluation_states_new) == 1:
                 if evaluation_states_old[0] != evaluation_states_new[0]:
+                    pooledConnection = None
+                    try:
                         pooledConnection = getPoolConnection()
                         commands = pydapper.using(pooledConnection)
                         notification_type = 3 # RED (mapped to time_conflict) 
@@ -122,7 +124,8 @@ class ValidationRules(ValidationRuleFunctions):
                             else:
                                 query = "INSERT INTO notification (shipcall_id, type, level) VALUES (?shipcall_id?, 4, 0)"
                                 commands.execute(query, param={"shipcall_id" : int(shipcall_df.index[0])})
-
+                    finally:
+                        if pooledConnection is not None:
                             pooledConnection.close()
 
 

src/server/flaskapp.wsgi

@@ -2,7 +2,7 @@ import os
 import sys
 import logging
 
-sys.path.insert(0, '/var/www/brecal_devel/src/server')
+sys.path.insert(0, '/var/www/brecal/src/server')
 sys.path.insert(0, '/var/www/venv/lib/python3.12/site-packages/')
 
 import schedule