Update Database to use QueuePool class

src/common/core/jobs/jobs/download-plugins.py

@@ -92,7 +92,7 @@ try:
         LOGGER.info("No external plugins to download")
         sys_exit(0)
 
-    db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
+    db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"))
     plugin_nbr = 0
 
     # Loop on URLs

src/common/core/letsencrypt/jobs/certbot-auth.py

@@ -26,7 +26,7 @@ try:
 
     # Cluster case
     if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
-        db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
+        db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None))
         lock = Lock()
 
         with lock:

src/common/core/letsencrypt/jobs/certbot-cleanup.py

@@ -25,7 +25,7 @@ try:
 
     # Cluster case
     if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
-        db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
+        db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None))
         lock = Lock()
         with lock:
             instances = db.get_instances()

src/common/core/letsencrypt/jobs/certbot-deploy.py

@@ -37,7 +37,7 @@ try:
         tgz.seek(0, 0)
         files = {"archive.tar.gz": tgz}
 
-        db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
+        db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None))
         lock = Lock()
 
         with lock:

src/common/core/pro/jobs/download-pro-plugins.py

@@ -94,7 +94,7 @@ def install_plugin(plugin_dir: str, db, preview: bool = True) -> bool:
 
 
 try:
-    db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
+    db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"))
     db_metadata = db.get_metadata()
     current_date = datetime.now()
 

src/common/core/realip/jobs/realip-download.py

@@ -72,7 +72,7 @@ try:
         sys_exit(0)
 
     if not urls:
-        LOGGER.error("No URL found, skipping download...")
+        LOGGER.info("No URL found, skipping download...")
         sys_exit(0)
 
     # Download and write data to temp file

src/common/db/Database.py

@@ -48,7 +48,7 @@ from sqlalchemy.exc import (
     SQLAlchemyError,
 )
 from sqlalchemy.orm import scoped_session, sessionmaker
-from sqlalchemy.pool import SingletonThreadPool
+from sqlalchemy.pool import QueuePool
 
 install_as_MySQLdb()
 
@@ -56,16 +56,13 @@ install_as_MySQLdb()
 class Database:
     DB_STRING_RX = re_compile(r"^(?P<database>(mariadb|mysql)(\+pymysql)?|sqlite(\+pysqlite)?|postgresql(\+psycopg)?):/+(?P<path>/[^\s]+)")
 
-    def __init__(
-        self,
-        logger: Logger,
-        sqlalchemy_string: Optional[str] = None,
-        *,
-        ui: bool = False,
-        pool: bool = True,
-    ) -> None:
+    def __init__(self, logger: Logger, sqlalchemy_string: Optional[str] = None, *, ui: bool = False, pool: Optional[bool] = None) -> None:
         """Initialize the database"""
         self.logger = logger
+
+        if pool:
+            self.logger.warning("The pool parameter is deprecated, it will be removed in the next version")
+
         self.__session_factory = None
         self.__sql_engine = None
 
@@ -97,7 +94,14 @@ class Database:
         self.database_uri = sqlalchemy_string
         error = False
 
-        engine_kwargs = {"future": True, "poolclass": None if pool else SingletonThreadPool, "pool_pre_ping": True, "pool_recycle": 1800}
+        engine_kwargs = {
+            "future": True,
+            "poolclass": QueuePool,
+            "pool_pre_ping": True,
+            "pool_recycle": 1800,
+            "pool_size": 20,
+            "max_overflow": 10,
+        }
 
         try:
             self.__sql_engine = create_engine(sqlalchemy_string, **engine_kwargs)
@@ -152,7 +156,6 @@ class Database:
 
         self.logger.info("✅ Database connection established")
 
-        self.__session_factory = sessionmaker(bind=self.__sql_engine, autoflush=True, expire_on_commit=False)
         self.suffix_rx = re_compile(r"_\d+$")
 
         if sqlalchemy_string.startswith("sqlite"):
@@ -171,20 +174,21 @@ class Database:
     @contextmanager
     def __db_session(self):
         try:
-            assert self.__session_factory is not None
+            assert self.__sql_engine is not None
         except AssertionError:
-            self.logger.error("The database session is not initialized")
+            self.logger.error("The database engine is not initialized")
             _exit(1)
 
-        session = scoped_session(self.__session_factory)
-
-        try:
-            yield session
-        except BaseException:
-            session.rollback()
-            raise
-        finally:
-            session.remove()
+        with self.__sql_engine.connect() as conn:
+            session_factory = sessionmaker(bind=conn, autoflush=True, expire_on_commit=False)
+            session = scoped_session(session_factory)
+            try:
+                yield session
+            except BaseException:
+                session.rollback()
+                raise
+            finally:
+                session.remove()
 
     def set_autoconf_load(self, value: bool = True) -> str:
         """Set the autoconf_loaded value"""

src/common/gen/save_config.py

@@ -52,7 +52,7 @@ def get_instance_configs_and_apis(instance: Any, db, _type="Docker"):
             tmp_config[split[0]] = split[1]
 
             if not db and split[0] == "DATABASE_URI":
-                db = Database(logger, sqlalchemy_string=split[1], pool=False)
+                db = Database(logger, sqlalchemy_string=split[1])
             elif split[0] == "API_HTTP_PORT":
                 api_http_port = split[1]
             elif split[0] == "API_SERVER_NAME":
@@ -132,7 +132,7 @@ if __name__ == "__main__":
         external_plugins = args.plugins
         pro_plugins = args.pro_plugins
         if not Path(sep, "usr", "sbin", "nginx").exists() and args.method == "ui":
-            db = Database(logger, pool=False)
+            db = Database(logger)
             external_plugins = db.get_plugins(_type="external")
             pro_plugins = db.get_plugins(_type="pro")
 
@@ -181,7 +181,7 @@ if __name__ == "__main__":
                         f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}"
                     )
 
-            db = Database(logger, config_files.get("DATABASE_URI", None), pool=False)
+            db = Database(logger, config_files.get("DATABASE_URI", None))
         else:
             docker_client = DockerClient(base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock"))
 
@@ -217,7 +217,7 @@ if __name__ == "__main__":
                         tmp_config[split[0]] = split[1]
 
                         if not db and split[0] == "DATABASE_URI":
-                            db = Database(logger, sqlalchemy_string=split[1], pool=False)
+                            db = Database(logger, sqlalchemy_string=split[1])
                         elif split[0] == "API_HTTP_PORT":
                             api_http_port = split[1]
                         elif split[0] == "API_SERVER_NAME":
@@ -231,7 +231,7 @@ if __name__ == "__main__":
                 )
 
         if not db:
-            db = Database(logger, pool=False)
+            db = Database(logger)
 
         # Compute the config
         if not config_files:

src/common/utils/jobs.py

@@ -41,7 +41,7 @@ class Job:
         if not self.db:
             from Database import Database  # type: ignore
 
-            self.db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
+            self.db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"))
         self.logger = logger or self.db.logger
 
         if not deprecated: