Merge pull request #1215 from bunkerity/dev

Merge branch "dev" into branch "staging"

misc/dev/docker-compose.autoconf.misc.yml

@@ -31,6 +31,7 @@ services:
       - USE_GZIP=yes
       - EXTERNAL_PLUGIN_URLS=https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip
       - CUSTOM_CONF_MODSEC_CRS_reqbody-rule=SecRuleRemoveById 200002
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -48,6 +49,7 @@ services:
       - bw-docker
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -68,6 +70,7 @@ services:
       - ./configs/server-http/hello.conf:/data/configs/server-http/hello.conf:ro
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -83,6 +86,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -97,6 +101,7 @@ services:
       - MYSQL_PASSWORD=secret
     volumes:
       - bw-db:/var/lib/mysql
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -104,6 +109,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

misc/dev/docker-compose.autoconf.ui.misc.yml

@@ -31,6 +31,7 @@ services:
       - USE_GZIP=yes
       - EXTERNAL_PLUGIN_URLS=https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip
       - CUSTOM_CONF_MODSEC_CRS_reqbody-rule=SecRuleRemoveById 200002
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -48,6 +49,7 @@ services:
       - bw-docker
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -68,6 +70,7 @@ services:
       - ./configs/server-http/hello.conf:/data/configs/server-http/hello.conf:ro
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -83,6 +86,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -106,6 +110,7 @@ services:
       ADMIN_USERNAME: "admin"
       ADMIN_PASSWORD: "P@ssw0rd"
       DEBUG: "1"
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -131,6 +136,7 @@ services:
       - MYSQL_PASSWORD=secret
     volumes:
       - bw-db:/var/lib/mysql
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -138,6 +144,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

misc/dev/docker-compose.autoconf.ui.yml

@@ -29,6 +29,7 @@ services:
       - DISABLE_DEFAULT_SERVER=yes
       - USE_CLIENT_CACHE=yes
       - USE_GZIP=yes
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -46,6 +47,7 @@ services:
       - bw-docker
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -65,6 +67,7 @@ services:
       - bw-data:/data
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -80,6 +83,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -103,6 +107,7 @@ services:
       ADMIN_USERNAME: "admin"
       ADMIN_PASSWORD: "P@ssw0rd"
       DEBUG: "1"
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -127,6 +132,7 @@ services:
       - MYSQL_PASSWORD=secret
     volumes:
       - bw-db:/var/lib/mysql
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -134,6 +140,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

misc/dev/docker-compose.autoconf.wizard.yml

@@ -29,6 +29,7 @@ services:
       - USE_CLIENT_CACHE=yes
       - USE_GZIP=yes
       - UI_HOST=http://bw-ui:7000
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -46,6 +47,7 @@ services:
       - bw-docker
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -65,6 +67,7 @@ services:
       - bw-data:/data
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -80,6 +83,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -101,6 +105,7 @@ services:
     environment:
       <<: *env
       DEBUG: "1"
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -118,6 +123,7 @@ services:
       - MYSQL_PASSWORD=secret
     volumes:
       - bw-db:/var/lib/mysql
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -125,6 +131,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

misc/dev/docker-compose.autoconf.yml

@@ -29,6 +29,7 @@ services:
       - DISABLE_DEFAULT_SERVER=yes
       - USE_CLIENT_CACHE=yes
       - USE_GZIP=yes
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -46,6 +47,7 @@ services:
       - bw-docker
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -65,6 +67,7 @@ services:
       - bw-data:/data
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -80,6 +83,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -94,6 +98,7 @@ services:
       - MYSQL_PASSWORD=secret
     volumes:
       - bw-db:/var/lib/mysql
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -101,6 +106,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

misc/dev/docker-compose.misc.yml

@@ -26,6 +26,7 @@ services:
       - REVERSE_PROXY_HOST=http://app1:8080
       - EXTERNAL_PLUGIN_URLS=https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip
       - CUSTOM_CONF_MODSEC_CRS_reqbody-suppress=SecRuleRemoveById 200002
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -47,6 +48,7 @@ services:
     environment:
       - DOCKER_HOST=tcp://bw-docker:2375
       - LOG_LEVEL=debug
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -62,6 +64,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -69,6 +72,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

misc/dev/docker-compose.ui.misc.yml

@@ -38,6 +38,7 @@ services:
       - app1.example.com_USE_REVERSE_PROXY=yes
       - app1.example.com_REVERSE_PROXY_URL=/
       - app1.example.com_REVERSE_PROXY_HOST=http://app1:8080
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -58,6 +59,7 @@ services:
       - ./configs/server-http/hello.conf:/data/configs/server-http/hello.conf:ro
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -73,6 +75,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -96,6 +99,7 @@ services:
       ADMIN_USERNAME: "admin"
       ADMIN_PASSWORD: "P@ssw0rd"
       DEBUG: "1"
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -113,6 +117,7 @@ services:
       - MYSQL_PASSWORD=secret
     volumes:
       - bw-db:/var/lib/mysql
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -120,6 +125,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

misc/dev/docker-compose.ui.yml

@@ -35,6 +35,7 @@ services:
       - app1.example.com_USE_REVERSE_PROXY=yes
       - app1.example.com_REVERSE_PROXY_URL=/
       - app1.example.com_REVERSE_PROXY_HOST=http://app1:8080
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -54,6 +55,7 @@ services:
       - bw-data:/data
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -69,6 +71,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -92,6 +95,7 @@ services:
       ADMIN_USERNAME: "admin"
       ADMIN_PASSWORD: "P@ssw0rd"
       DEBUG: "1"
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -109,6 +113,7 @@ services:
       - MYSQL_PASSWORD=secret
     volumes:
       - bw-db:/var/lib/mysql
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -116,6 +121,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

misc/dev/docker-compose.wizard.yml

@@ -30,6 +30,7 @@ services:
       - app1.example.com_USE_REVERSE_PROXY=yes
       - app1.example.com_REVERSE_PROXY_URL=/
       - app1.example.com_REVERSE_PROXY_HOST=http://app1:8080
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -49,6 +50,7 @@ services:
       - bw-data:/data
     environment:
       <<: *env
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -64,6 +66,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -85,6 +88,7 @@ services:
     environment:
       <<: *env
       DEBUG: "1"
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -102,6 +106,7 @@ services:
       - MYSQL_PASSWORD=secret
     volumes:
       - bw-db:/var/lib/mysql
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -109,6 +114,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

misc/dev/docker-compose.yml

@@ -24,6 +24,7 @@ services:
       - USE_REVERSE_PROXY=yes
       - REVERSE_PROXY_URL=/
       - REVERSE_PROXY_HOST=http://app1:8080
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -44,6 +45,7 @@ services:
     environment:
       - DOCKER_HOST=tcp://bw-docker:2375
       - LOG_LEVEL=debug
+    restart: "unless-stopped"
     networks:
       bw-universe:
         aliases:
@@ -59,6 +61,7 @@ services:
     environment:
       - CONTAINERS=1
       - LOG_LEVEL=warning
+    restart: "unless-stopped"
     networks:
       bw-docker:
         aliases:
@@ -66,6 +69,7 @@ services:
 
   app1:
     image: nginxdemos/nginx-hello
+    restart: "unless-stopped"
     networks:
       bw-services:
         aliases:

src/common/db/Database.py

@@ -68,6 +68,7 @@ def set_sqlite_pragma(dbapi_connection, _):
 
 class Database:
     DB_STRING_RX = re_compile(r"^(?P<database>(mariadb|mysql)(\+pymysql)?|sqlite(\+pysqlite)?|postgresql(\+psycopg)?):/+(?P<path>/[^\s]+)")
+    READONLY_ERROR = ("readonly", "read-only", "command denied", "Access denied")
 
     def __init__(
         self, logger: Logger, sqlalchemy_string: Optional[str] = None, *, ui: bool = False, pool: Optional[bool] = None, log: bool = True, **kwargs
@@ -75,6 +76,7 @@ class Database:
         """Initialize the database"""
         self.logger = logger
         self.readonly = False
+        self.last_connection_retry = None
 
         if pool:
             self.logger.warning("The pool parameter is deprecated, it will be removed in the next version")
@@ -185,7 +187,7 @@ class Database:
                     self.logger.error(f"Can't connect to database after {DATABASE_RETRY_TIMEOUT} seconds: {e}")
                     _exit(1)
 
-                if "readonly" in str(e) or "read-only" in str(e) or "command denied" in str(e):
+                if any(error in str(e) for error in self.READONLY_ERROR):
                     if log:
                         self.logger.warning("The database is read-only. Retrying in read-only mode in 5 seconds ...")
                     self.sql_engine.dispose(close=True)
@@ -226,6 +228,7 @@ class Database:
 
     def retry_connection(self, *, readonly: bool = False, fallback: bool = False, log: bool = True, **kwargs) -> None:
         """Retry the connection to the database"""
+        self.last_connection_retry = datetime.now()
 
         if log:
             self.logger.debug(f"Retrying the connection to the database{' in read-only mode' if readonly else ''}{' with fallback' if fallback else ''} ...")
@@ -243,9 +246,10 @@ class Database:
                 conn.execute(text("SELECT 1"))
             return
 
+        table_name = uuid4().hex
         with self.sql_engine.connect() as conn:
-            conn.execute(text("CREATE TABLE IF NOT EXISTS test (id INT)"))
-            conn.execute(text("DROP TABLE test"))
+            conn.execute(text(f"CREATE TABLE IF NOT EXISTS test_{table_name} (id INT)"))
+            conn.execute(text(f"DROP TABLE IF EXISTS test_{table_name}"))
 
     @contextmanager
     def __db_session(self) -> Any:
@@ -265,7 +269,7 @@ class Database:
             if session:
                 session.rollback()
 
-            if "readonly" in str(e) or "read-only" in str(e) or "command denied" in str(e):
+            if any(error in str(e) for error in self.READONLY_ERROR):
                 self.logger.warning("The database is read-only, retrying in read-only mode ...")
                 try:
                     self.retry_connection(readonly=True, pool_timeout=1)
@@ -1426,7 +1430,7 @@ class Database:
 
         return message
 
-    def get_config(self, methods: bool = False, with_drafts: bool = False) -> Dict[str, Any]:
+    def get_config(self, global_only: bool = False, methods: bool = False, with_drafts: bool = False) -> Dict[str, Any]:
         """Get the config from the database"""
         with self.__db_session() as session:
             config = {}
@@ -1466,7 +1470,7 @@ class Database:
             if not with_drafts:
                 services = services.filter_by(is_draft=False)
 
-            if is_multisite:
+            if not global_only and is_multisite:
                 for service in services:
                     config[f"{service.id}_IS_DRAFT"] = "yes" if service.is_draft else "no"
                     if methods:

src/common/utils/logger.py

@@ -25,8 +25,8 @@ setLoggerClass(BWLogger)
 
 default_level = _nameToLevel.get(getenv("LOG_LEVEL", "INFO").upper(), INFO)
 basicConfig(
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    datefmt="[%Y-%m-%d %H:%M:%S]",
+    format="%(asctime)s [%(name)s] [%(process)d] [%(levelname)s] - %(message)s",
+    datefmt="[%Y-%m-%d %H:%M:%S %z]",
     level=default_level,
 )
 

src/scheduler/JobScheduler.py

@@ -2,6 +2,7 @@
 
 from contextlib import suppress
 from copy import deepcopy
+from datetime import datetime
 from functools import partial
 from glob import glob
 from json import loads
@@ -363,6 +364,8 @@ class JobScheduler(ApiCaller):
             except BaseException:
                 self.db.readonly = True
                 return True
+        elif self.db.last_connection_retry and (datetime.now() - self.db.last_connection_retry).total_seconds() > 30:
+            return True
 
         if self.db.database_uri and self.db.readonly:
             try:

src/scheduler/main.py

@@ -124,7 +124,7 @@ def stop(status):
     _exit(status)
 
 
-def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Union[Path, str] = CUSTOM_CONFIGS_PATH):
+def generate_custom_configs(configs: Optional[List[Dict[str, Any]]] = None, *, original_path: Union[Path, str] = CUSTOM_CONFIGS_PATH):
     if not isinstance(original_path, Path):
         original_path = Path(original_path)
 
@@ -138,6 +138,10 @@ def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Uni
             elif file.is_dir():
                 rmtree(file, ignore_errors=True)
 
+    if configs is None:
+        assert SCHEDULER is not None
+        configs = SCHEDULER.db.get_custom_configs()
+
     if configs:
         logger.info("Generating new custom configs ...")
         original_path.mkdir(parents=True, exist_ok=True)
@@ -171,7 +175,7 @@ def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Uni
             logger.error("Sending custom configs failed, configuration will not work as expected...")
 
 
-def generate_external_plugins(plugins: Optional[List[Dict[str, Any]]], *, original_path: Union[Path, str] = EXTERNAL_PLUGINS_PATH):
+def generate_external_plugins(plugins: Optional[List[Dict[str, Any]]] = None, *, original_path: Union[Path, str] = EXTERNAL_PLUGINS_PATH):
     if not isinstance(original_path, Path):
         original_path = Path(original_path)
     pro = "pro" in original_path.parts
@@ -240,62 +244,61 @@ def generate_external_plugins(plugins: Optional[List[Dict[str, Any]]], *, origin
             logger.error(f"Sending {'pro ' if pro else ''}external plugins failed, configuration will not work as expected...")
 
 
-def generate_caches(plugins: List[Dict[str, Any]]):
+def generate_caches():
     assert SCHEDULER is not None
 
-    for plugin in plugins:
-        job_cache_files = SCHEDULER.db.get_jobs_cache_files(plugin_id=plugin["id"])
-        plugin_cache_files = set()
-        ignored_dirs = set()
-        job_path = Path(sep, "var", "cache", "bunkerweb", plugin["id"])
+    job_cache_files = SCHEDULER.db.get_jobs_cache_files()
+    plugin_cache_files = set()
+    ignored_dirs = set()
 
-        for job_cache_file in job_cache_files:
-            cache_path = job_path.joinpath(job_cache_file["service_id"] or "", job_cache_file["file_name"])
-            plugin_cache_files.add(cache_path)
+    for job_cache_file in job_cache_files:
+        job_path = Path(sep, "var", "cache", "bunkerweb", job_cache_file["plugin_id"])
+        cache_path = job_path.joinpath(job_cache_file["service_id"] or "", job_cache_file["file_name"])
+        plugin_cache_files.add(cache_path)
 
-            try:
-                if job_cache_file["file_name"].endswith(".tgz"):
-                    extract_path = cache_path.parent
-                    if job_cache_file["file_name"].startswith("folder:"):
-                        extract_path = Path(job_cache_file["file_name"].split("folder:", 1)[1].rsplit(".tgz", 1)[0])
-                    ignored_dirs.add(extract_path.as_posix())
-                    rmtree(extract_path, ignore_errors=True)
-                    extract_path.mkdir(parents=True, exist_ok=True)
-                    with tar_open(fileobj=BytesIO(job_cache_file["data"]), mode="r:gz") as tar:
-                        assert isinstance(tar, TarFile)
-                        try:
-                            for member in tar.getmembers():
-                                try:
-                                    tar.extract(member, path=extract_path)
-                                except Exception as e:
-                                    logger.error(f"Error extracting {member.name}: {e}")
-                        except Exception as e:
-                            logger.error(f"Error extracting tar file: {e}")
-                    logger.debug(f"Restored cache directory {extract_path}")
-                    continue
-                cache_path.parent.mkdir(parents=True, exist_ok=True)
-                cache_path.write_bytes(job_cache_file["data"])
-                logger.debug(f"Restored cache file {job_cache_file['file_name']}")
-            except BaseException as e:
-                logger.error(f"Exception while restoring cache file {job_cache_file['file_name']} :\n{e}")
+        try:
+            if job_cache_file["file_name"].endswith(".tgz"):
+                extract_path = cache_path.parent
+                if job_cache_file["file_name"].startswith("folder:"):
+                    extract_path = Path(job_cache_file["file_name"].split("folder:", 1)[1].rsplit(".tgz", 1)[0])
+                ignored_dirs.add(extract_path.as_posix())
+                rmtree(extract_path, ignore_errors=True)
+                extract_path.mkdir(parents=True, exist_ok=True)
+                with tar_open(fileobj=BytesIO(job_cache_file["data"]), mode="r:gz") as tar:
+                    assert isinstance(tar, TarFile)
+                    try:
+                        for member in tar.getmembers():
+                            try:
+                                tar.extract(member, path=extract_path)
+                            except Exception as e:
+                                logger.error(f"Error extracting {member.name}: {e}")
+                    except Exception as e:
+                        logger.error(f"Error extracting tar file: {e}")
+                logger.debug(f"Restored cache directory {extract_path}")
+                continue
+            cache_path.parent.mkdir(parents=True, exist_ok=True)
+            cache_path.write_bytes(job_cache_file["data"])
+            logger.debug(f"Restored cache file {job_cache_file['file_name']}")
+        except BaseException as e:
+            logger.error(f"Exception while restoring cache file {job_cache_file['file_name']} :\n{e}")
 
-        if job_path.is_dir():
-            for file in job_path.rglob("*"):
-                if file.as_posix().startswith(tuple(ignored_dirs)):
-                    continue
+    if job_path.is_dir():
+        for file in job_path.rglob("*"):
+            if file.as_posix().startswith(tuple(ignored_dirs)):
+                continue
 
-                logger.debug(f"Checking if {file} should be removed")
-                if file not in plugin_cache_files and file.is_file():
-                    logger.debug(f"Removing non-cached file {file}")
-                    file.unlink(missing_ok=True)
-                    if file.parent.is_dir() and not list(file.parent.iterdir()):
-                        logger.debug(f"Removing empty directory {file.parent}")
-                        rmtree(file.parent, ignore_errors=True)
-                        if file.parent == job_path:
-                            break
-                elif file.is_dir() and not list(file.iterdir()):
-                    logger.debug(f"Removing empty directory {file}")
-                    rmtree(file, ignore_errors=True)
+            logger.debug(f"Checking if {file} should be removed")
+            if file not in plugin_cache_files and file.is_file():
+                logger.debug(f"Removing non-cached file {file}")
+                file.unlink(missing_ok=True)
+                if file.parent.is_dir() and not list(file.parent.iterdir()):
+                    logger.debug(f"Removing empty directory {file.parent}")
+                    rmtree(file.parent, ignore_errors=True)
+                    if file.parent == job_path:
+                        break
+            elif file.is_dir() and not list(file.iterdir()):
+                logger.debug(f"Removing empty directory {file}")
+                rmtree(file, ignore_errors=True)
 
 
 def api_to_instance(api):
@@ -321,17 +324,18 @@ def run_in_slave_mode():
         sleep(5)
         env = SCHEDULER.db.get_config()
 
-    # Download plugins
-    pro_plugins = SCHEDULER.db.get_plugins(_type="pro", with_data=True)
-    generate_external_plugins(pro_plugins, original_path=PRO_PLUGINS_PATH)
-    external_plugins = SCHEDULER.db.get_plugins(_type="external", with_data=True)
-    generate_external_plugins(external_plugins)
+    threads = [
+        Thread(target=generate_custom_configs),
+        Thread(target=generate_external_plugins),
+        Thread(target=generate_external_plugins, kwargs={"original_path": PRO_PLUGINS_PATH}),
+        Thread(target=generate_caches),
+    ]
 
-    # Download custom configs
-    generate_custom_configs(SCHEDULER.db.get_custom_configs())
+    for thread in threads:
+        thread.start()
 
-    # Download caches
-    generate_caches(pro_plugins + external_plugins)
+    for thread in threads:
+        thread.join()
 
     # Gen config
     content = ""
@@ -577,9 +581,9 @@ if __name__ == "__main__":
             threads.clear()
 
             if changes["pro_plugins_changed"]:
-                threads.append(Thread(target=generate_external_plugins, args=(None,), kwargs={"original_path": PRO_PLUGINS_PATH}))
+                threads.append(Thread(target=generate_external_plugins, kwargs={"original_path": PRO_PLUGINS_PATH}))
             if changes["external_plugins_changed"]:
-                threads.append(Thread(target=generate_external_plugins, args=(None,)))
+                threads.append(Thread(target=generate_external_plugins))
 
             for thread in threads:
                 thread.start()
@@ -670,7 +674,7 @@ if __name__ == "__main__":
                 else:
                     logger.info("All jobs in run_once() were successful")
                     if SCHEDULER.db.readonly:
-                        generate_caches(SCHEDULER.db.get_plugins())
+                        generate_caches()
 
             if CONFIG_NEED_GENERATION:
                 content = ""

src/ui/main.py

@@ -389,7 +389,7 @@ def inject_variables():
         plugins=app.config["CONFIG"].get_plugins(),
         pro_loading=ui_data.get("PRO_LOADING", False),
         bw_version=metadata["version"],
-        is_readonly=app.config["DB"].readonly,
+        is_readonly=ui_data.get("READONLY_MODE", False),
     )
 
 
@@ -442,11 +442,20 @@ def before_request():
     app.config["SCRIPT_NONCE"] = sha256(urandom(32)).hexdigest()
 
     if not request.path.startswith(("/css", "/images", "/js", "/json", "/webfonts")):
-        if app.config["DB"].database_uri and app.config["DB"].readonly:
+        ui_data = get_ui_data()
+
+        if (
+            app.config["DB"].database_uri
+            and app.config["DB"].readonly
+            and (
+                datetime.now(timezone.utc) - datetime.fromisoformat(ui_data.get("LAST_DATABASE_RETRY", "1970-01-01T00:00:00")).replace(tzinfo=timezone.utc)
+                > timedelta(minutes=1)
+            )
+        ):
             try:
                 app.config["DB"].retry_connection(pool_timeout=1)
                 app.config["DB"].retry_connection(log=False)
-                app.config["DB"].readonly = False
+                ui_data["READONLY_MODE"] = False
                 app.logger.info("The database is no longer read-only, defaulting to read-write mode")
             except BaseException:
                 try:
@@ -457,12 +466,22 @@ def before_request():
                         with suppress(BaseException):
                             app.config["DB"].retry_connection(fallback=True, pool_timeout=1)
                             app.config["DB"].retry_connection(fallback=True, log=False)
-                app.config["DB"].readonly = True
-        elif not app.config["DB"].readonly and request.method == "POST" and not ("/totp" in request.path or "/login" in request.path):
+                ui_data["READONLY_MODE"] = True
+            ui_data["LAST_DATABASE_RETRY"] = app.config["DB"].last_connection_retry.isoformat()
+        elif not ui_data.get("READONLY_MODE", False) and request.method == "POST" and not ("/totp" in request.path or "/login" in request.path):
             try:
                 app.config["DB"].test_write()
+                ui_data["READONLY_MODE"] = False
             except BaseException:
-                app.config["DB"].readonly = True
+                ui_data["READONLY_MODE"] = True
+                ui_data["LAST_DATABASE_RETRY"] = app.config["DB"].last_connection_retry.isoformat()
+
+        app.config["DB"].readonly = ui_data.get("READONLY_MODE", False)
+        with LOCK:
+            TMP_DATA_FILE.write_text(dumps(ui_data), encoding="utf-8")
+
+        if app.config["DB"].readonly:
+            flash("Database connection is in read-only mode : no modification possible.", "error")
 
         if current_user.is_authenticated:
             passed = True
@@ -478,11 +497,7 @@ def before_request():
                 passed = False
 
             if not passed:
-                logout_user()
-                session.clear()
-
-        if app.config["DB"].readonly:
-            flash("Database connection is in read-only mode : no modification possible.", "error")
+                return logout()
 
 
 @app.route("/", strict_slashes=False)
@@ -887,7 +902,7 @@ def instances():
         )
 
     # Display instances
-    config = app.config["CONFIG"].get_config()
+    config = app.config["CONFIG"].get_config(global_only=True)
     override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
     instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
     return render_template("instances.html", title="Instances", instances=instances, username=current_user.get_id())
@@ -938,21 +953,21 @@ def services():
                 custom_configs.append(variable)
                 del variables[variable]
 
-        # config variable format is custom_config_<type>_<filename>
+        # custom_config variable format is custom_config_<type>_<filename>
         # we want a list of dict with each dict containing type, filename, action and server name
         # after getting all configs, we want to save them after the end of current service action
         # to avoid create config for none existing service or in case editing server name
         format_configs = []
-        for config in custom_configs:
+        for custom_config in custom_configs:
             # first remove custom_config_ prefix
-            config = config.split("custom_config_")[1]
+            custom_config = custom_config.split("custom_config_")[1]
             # then split the config into type, filename, action
-            config = config.split("_")
+            custom_config = custom_config.split("_")
             # check if the config is valid
-            if len(config) == 2 and config[0] in config_types:
-                format_configs.append({"type": config[0], "filename": config[1], "action": operation, "server_name": server_name})
+            if len(custom_config) == 2 and custom_config[0] in config_types:
+                format_configs.append({"type": custom_config[0], "filename": custom_config[1], "action": operation, "server_name": server_name})
             else:
-                return redirect_flash_error("Invalid custom config {config}", "services", True)
+                return redirect_flash_error(f"Invalid custom config {custom_config}", "services", True)
 
         if request.form["operation"] in ("new", "edit"):
             del variables["operation"]
@@ -1108,7 +1123,7 @@ def global_config():
         del variables["csrf_token"]
 
         # Edit check fields and remove already existing ones
-        config = app.config["CONFIG"].get_config(methods=True, with_drafts=True)
+        config = app.config["CONFIG"].get_config(global_only=True, methods=True, with_drafts=True)
         services = config["SERVER_NAME"]["value"].split(" ")
         for variable, value in variables.copy().items():
             if variable in ("AUTOCONF_MODE", "SWARM_MODE", "KUBERNETES_MODE", "SERVER_NAME", "IS_LOADING", "IS_DRAFT") or variable.endswith("SCHEMA"):
@@ -1172,13 +1187,8 @@ def global_config():
             )
         )
 
-    global_config = app.config["CONFIG"].get_config()
-    for service in global_config["SERVER_NAME"]["value"].split(" "):
-        for key in global_config.copy():
-            if key.startswith(f"{service}_"):
-                global_config.pop(key)
-
     # Display global config
+    global_config = app.config["CONFIG"].get_config(global_only=True)
     return render_template("global_config.html", username=current_user.get_id(), global_config=global_config, dumped_global_config=dumps(global_config))
 
 

src/ui/src/Config.py

@@ -78,7 +78,7 @@ class Config:
     def get_settings(self) -> dict:
         return self.__settings
 
-    def get_config(self, methods: bool = True, with_drafts: bool = False) -> dict:
+    def get_config(self, global_only: bool = False, methods: bool = True, with_drafts: bool = False) -> dict:
         """Get the nginx variables env file and returns it as a dict
 
         Returns
@@ -86,7 +86,7 @@ class Config:
         dict
             The nginx variables env file as a dict
         """
-        return self.__db.get_config(methods=methods, with_drafts=with_drafts)
+        return self.__db.get_config(global_only=global_only, methods=methods, with_drafts=with_drafts)
 
     def get_services(self, methods: bool = True, with_drafts: bool = False) -> list[dict]:
         """Get nginx's services