diff --git a/src/common/db/Database.py b/src/common/db/Database.py index d7ca2e609..55ce41c11 100644 --- a/src/common/db/Database.py +++ b/src/common/db/Database.py @@ -129,6 +129,7 @@ class Database: "pool_recycle": 1800, "pool_size": 40, "max_overflow": 20, + "pool_timeout": 5, } | kwargs try: @@ -215,16 +216,19 @@ class Database: def test_write(self): """Test the write access to the database""" self.logger.debug("Testing write access to the database ...") + self.retry_connection(pool_timeout=1) with self.__db_session() as session: table_name = uuid4().hex session.execute(text(f"CREATE TABLE IF NOT EXISTS test_{table_name} (id INT)")) session.execute(text(f"DROP TABLE IF EXISTS test_{table_name}")) session.commit() + self.retry_connection() - def retry_connection(self, *, readonly: bool = False, fallback: bool = False, **kwargs) -> None: + def retry_connection(self, *, readonly: bool = False, fallback: bool = False, log: bool = True, **kwargs) -> None: """Retry the connection to the database""" - self.logger.debug(f"Retrying the connection to the database {'in read-only mode' if readonly else ''}{' with fallback' if fallback else ''} ...") + if log: + self.logger.debug(f"Retrying the connection to the database{' in read-only mode' if readonly else ''}{' with fallback' if fallback else ''} ...") assert self.sql_engine is not None @@ -265,16 +269,19 @@ class Database: self.logger.warning("The database is read-only, retrying in read-only mode ...") try: self.retry_connection(readonly=True, pool_timeout=1) + self.retry_connection(readonly=True, log=False) except (OperationalError, DatabaseError): if self.database_uri_readonly: self.logger.warning("Can't connect to the database in read-only mode, falling back to read-only one") with suppress(OperationalError, DatabaseError): self.retry_connection(fallback=True, pool_timeout=1) + self.retry_connection(fallback=True, log=False) self.readonly = True elif isinstance(e, (ConnectionRefusedError, OperationalError)) and self.database_uri_readonly: self.logger.warning("Can't connect to the database, falling back to read-only one ...") with suppress(OperationalError, DatabaseError): self.retry_connection(fallback=True, pool_timeout=1) + self.retry_connection(fallback=True, log=False) self.readonly = True raise finally: diff --git a/src/scheduler/JobScheduler.py b/src/scheduler/JobScheduler.py index 639302dc4..018d6193e 100644 --- a/src/scheduler/JobScheduler.py +++ b/src/scheduler/JobScheduler.py @@ -367,19 +367,21 @@ class JobScheduler(ApiCaller): if self.db.database_uri and self.db.readonly: try: self.db.retry_connection(pool_timeout=1) + self.db.retry_connection(log=False) self.db.readonly = False self.__logger.info("The database is no longer read-only, defaulting to read-write mode") except BaseException: try: self.db.retry_connection(readonly=True, pool_timeout=1) + self.db.retry_connection(readonly=True, log=False) except BaseException: if self.db.database_uri_readonly: with suppress(BaseException): self.db.retry_connection(fallback=True, pool_timeout=1) + self.db.retry_connection(fallback=True, log=False) self.db.readonly = True if self.db.readonly: self.__logger.error("Database is in read-only mode, jobs will not be executed") - return True return self.db.readonly diff --git a/src/ui/main.py b/src/ui/main.py index 51809a90d..d8bae6c07 100755 --- a/src/ui/main.py +++ b/src/ui/main.py @@ -445,15 +445,18 @@ def before_request(): if app.config["DB"].database_uri and app.config["DB"].readonly: try: app.config["DB"].retry_connection(pool_timeout=1) + app.config["DB"].retry_connection(log=False) app.config["DB"].readonly = False app.logger.info("The database is no longer read-only, defaulting to read-write mode") except BaseException: try: app.config["DB"].retry_connection(readonly=True, pool_timeout=1) + app.config["DB"].retry_connection(readonly=True, log=False) except BaseException: if app.config["DB"].database_uri_readonly: with suppress(BaseException): app.config["DB"].retry_connection(fallback=True, pool_timeout=1) + app.config["DB"].retry_connection(fallback=True, log=False) app.config["DB"].readonly = True elif not app.config["DB"].readonly and request.method == "POST" and not ("/totp" in request.path or "/login" in request.path): try: