Optimize web UI DB requests and avoid skip of pro-plugins download when the pro license key changes

src/common/core/pro/jobs/download-pro-plugins.py

@@ -96,7 +96,6 @@ def install_plugin(plugin_path: Path, db, preview: bool = True) -> bool:
 try:
     db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"))
     db_metadata = db.get_metadata()
-    db_config = db.get_config()
     current_date = datetime.now()
     pro_license_key = getenv("PRO_LICENSE_KEY", "").strip()
 
@@ -111,6 +110,7 @@ try:
     headers = {"User-Agent": f"BunkerWeb/{data['version']}"}
     default_metadata = {
         "is_pro": False,
+        "pro_license": pro_license_key,
         "pro_expire": None,
         "pro_status": "invalid",
         "pro_overlapped": False,
@@ -158,7 +158,7 @@ try:
 
     # ? If we already checked today, skip the check and if the metadata is the same, skip the check
     if (
-        pro_license_key == db_config["PRO_LICENSE_KEY"]
+        pro_license_key == db_metadata.get("pro_license", "")
         and metadata.get("is_pro", False) == db_metadata["is_pro"]
         and db_metadata["last_pro_check"]
         and current_date.replace(hour=0, minute=0, second=0, microsecond=0) == db_metadata["last_pro_check"].replace(hour=0, minute=0, second=0, microsecond=0)

src/common/db/Database.py

@@ -6,9 +6,9 @@ from datetime import datetime
 from io import BytesIO
 from logging import Logger
 from os import _exit, getenv, listdir, sep
-from os.path import join
+from os.path import join as os_join
 from pathlib import Path
-from re import compile as re_compile
+from re import compile as re_compile, escape, search
 from sys import argv, path as sys_path
 from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union
 from time import sleep
@@ -33,14 +33,14 @@ from model import (
     Metadata,
 )
 
-for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",))]:
+for deps_path in [os_join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",))]:
     if deps_path not in sys_path:
         sys_path.append(deps_path)
 
 from common_utils import bytes_hash  # type: ignore
 
 from pymysql import install_as_MySQLdb
-from sqlalchemy import create_engine, event, MetaData as sql_metadata, text, inspect
+from sqlalchemy import create_engine, event, MetaData as sql_metadata, join, select as db_select, text, inspect
 from sqlalchemy.engine import Engine
 from sqlalchemy.exc import (
     ArgumentError,
@@ -343,7 +343,7 @@ class Database:
 
         return ""
 
-    def set_pro_metadata(self, data: Dict[Literal["is_pro", "pro_expire", "pro_status", "pro_overlapped", "pro_services"], Any] = {}) -> str:
+    def set_pro_metadata(self, data: Dict[Literal["is_pro", "pro_license", "pro_expire", "pro_status", "pro_overlapped", "pro_services"], Any] = {}) -> str:
         """Set the pro metadata values"""
         with self.__db_session() as session:
             if self.readonly:
@@ -454,6 +454,7 @@ class Database:
             "integration": "unknown",
             "database_version": "Unknown",
             "is_pro": "no",
+            "pro_license": "",
             "pro_expire": None,
             "pro_services": 0,
             "pro_overlapped": False,
@@ -474,6 +475,7 @@ class Database:
                         Metadata.version,
                         Metadata.integration,
                         Metadata.is_pro,
+                        Metadata.pro_license,
                         Metadata.pro_expire,
                         Metadata.pro_services,
                         Metadata.pro_overlapped,
@@ -490,6 +492,7 @@ class Database:
                             "version": metadata.version,
                             "integration": metadata.integration,
                             "is_pro": metadata.is_pro,
+                            "pro_license": metadata.pro_license,
                             "pro_expire": metadata.pro_expire,
                             "pro_services": metadata.pro_services,
                             "pro_overlapped": metadata.pro_overlapped,
@@ -1620,6 +1623,111 @@ class Database:
 
             return config
 
+    def get_filtered_config(
+        self,
+        global_only: bool = False,
+        methods: bool = False,
+        with_drafts: bool = False,
+        filtered_settings: Optional[Union[List[str], Set[str], Tuple[str]]] = None,
+    ) -> Dict[str, Any]:
+        """Get the config from the database"""
+        filtered_settings = set(filtered_settings or [])
+
+        if filtered_settings and not global_only:
+            filtered_settings.update(("SERVER_NAME", "MULTISITE"))
+
+        with self.__db_session() as session:
+            config = {}
+
+            # Define the join operation
+            j = join(Settings, Global_values, Settings.id == Global_values.setting_id)
+
+            # Define the select statement
+            stmt = (
+                db_select(Settings.id, Settings.multiple, Global_values.value, Global_values.suffix, Global_values.method)
+                .select_from(j)
+                .order_by(Settings.order)
+            )
+
+            if filtered_settings:
+                stmt = stmt.where(Settings.id.in_(filtered_settings))
+
+            # Execute the query and fetch all results
+            results = session.execute(stmt).fetchall()
+
+            for setting in results:
+                key = setting.id + (f"_{setting.suffix}" if setting.multiple and setting.suffix else "")
+                config[key] = setting.value if not methods else {"value": setting.value, "global": True, "method": setting.method}
+
+            is_multisite = config.get("MULTISITE", {"value": "no"})["value"] == "yes" if methods else config.get("MULTISITE", "no") == "yes"
+
+            # Define the join operation
+            j = join(Services, Services_settings, Services.id == Services_settings.service_id)
+            j = j.join(Settings, Settings.id == Services_settings.setting_id)
+
+            # Define the select statement
+            stmt = db_select(
+                Services.id.label("service_id"),
+                Services.is_draft,
+                Settings.id.label("setting_id"),
+                Settings.context,
+                Settings.default,
+                Settings.multiple,
+                Services_settings.value,
+                Services_settings.suffix,
+                Services_settings.method,
+            ).select_from(j)
+
+            if not with_drafts:
+                stmt = stmt.where(Services.is_draft == False)  # noqa: E712
+
+            if filtered_settings:
+                stmt = stmt.where(Services_settings.setting_id.in_(filtered_settings))
+
+            # Execute the query and fetch all results
+            results = session.execute(stmt).fetchall()
+
+            if not global_only and is_multisite:
+                for result in results:
+                    if f"{result.service_id}_IS_DRAFT" not in config:
+                        config[f"{result.service_id}_IS_DRAFT"] = "yes" if result.is_draft else "no"
+                        if methods:
+                            config[f"{result.service_id}_IS_DRAFT"] = {"value": config[f"{result.service_id}_IS_DRAFT"], "global": False, "method": "default"}
+
+                    key = result.setting_id
+                    original_key = key
+                    if self.suffix_rx.search(key):
+                        key = key[: -len(str(key.split("_")[-1])) - 1]
+
+                    if result.context != "multisite":
+                        continue
+                    elif f"{result.service_id}_{original_key}" not in config:
+                        config[f"{result.service_id}_{original_key}"] = result.default
+
+                    value = result.value
+                    if key == "SERVER_NAME" and not search(r"(^| )" + escape(result.service_id) + r"($| )", value):
+                        value = f"{result.service_id} {value}".strip()
+
+                    config[f"{result.service_id}_{key}" + (f"_{result.suffix}" if result.multiple and result.suffix else "")] = (
+                        value
+                        if not methods
+                        else {
+                            "value": value,
+                            "global": False,
+                            "method": result.method,
+                        }
+                    )
+
+            services = session.query(Services).with_entities(Services.id)
+
+            if not with_drafts:
+                services = services.filter_by(is_draft=False)
+
+            servers = " ".join(service.id for service in services)
+            config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"}
+
+            return config
+
     def get_custom_configs(self) -> List[Dict[str, Any]]:
         """Get the custom configs from the database"""
         with self.__db_session() as session:

src/common/db/model.py

@@ -234,6 +234,7 @@ class Metadata(Base):
     id = Column(Integer, primary_key=True, default=1)
     is_initialized = Column(Boolean, nullable=False)
     is_pro = Column(Boolean, default=False, nullable=False)
+    pro_license = Column(String(128), default="", nullable=True)
     pro_expire = Column(DateTime, nullable=True)
     pro_status = Column(PRO_STATUS_ENUM, default="invalid", nullable=False)
     pro_services = Column(Integer, default=0, nullable=False)

src/ui/main.py

@@ -449,8 +449,7 @@ def handle_csrf_error(_):
     :param e: The exception object
     :return: A template with the error message and a 401 status code.
     """
-    session.clear()
-    logout_user()
+    logout()
     flash("Wrong CSRF token !", "error")
     if not current_user:
         return render_template("setup.html"), 403
@@ -548,7 +547,7 @@ def check():
 
 @app.route("/setup", methods=["GET", "POST"])
 def setup():
-    db_config = app.config["CONFIG"].get_config(methods=False)
+    db_config = app.config["CONFIG"].get_config(methods=False, filtered_settings=("SERVER_NAME", "USE_UI", "UI_HOST"))
 
     for server_name in db_config["SERVER_NAME"].split(" "):
         if db_config.get(f"{server_name}_USE_UI", "no") == "yes":
@@ -690,8 +689,8 @@ def home():
     if r and r.status_code == 200:
         remote_version = basename(r.url).strip().replace("v", "")
 
-    config = app.config["CONFIG"].get_config(with_drafts=True)
-    override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
+    config = app.config["CONFIG"].get_config(with_drafts=True, filtered_settings=("SERVER_NAME", "OVERRIDE_INSTANCES"))
+    override_instances = config.get("OVERRIDE_INSTANCES", {"value": ""})["value"] != ""
     instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
 
     instance_health_count = 0
@@ -812,8 +811,7 @@ def account():
 
             username = request.form["admin_username"]
 
-            session.clear()
-            logout_user()
+            logout()
 
         if request.form["operation"] == "password":
 
@@ -830,8 +828,7 @@ def account():
 
             password = request.form["admin_password"]
 
-            session.clear()
-            logout_user()
+            logout()
 
         if request.form["operation"] == "totp":
 
@@ -928,8 +925,8 @@ def instances():
         )
 
     # Display instances
-    config = app.config["CONFIG"].get_config(global_only=True)
-    override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
+    config = app.config["CONFIG"].get_config(global_only=True, methods=False, filtered_settings=("OVERRIDE_INSTANCES",))
+    override_instances = config.get("OVERRIDE_INSTANCES", "") != ""
     instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
     return render_template("instances.html", title="Instances", instances=instances, username=current_user.get_id())
 
@@ -966,7 +963,7 @@ def services():
         if "SERVER_NAME" not in variables:
             variables["SERVER_NAME"] = variables["OLD_SERVER_NAME"]
 
-        config = app.config["CONFIG"].get_config(methods=True, with_drafts=True)
+        config = app.config["CONFIG"].get_config(with_drafts=True, filtered_settings=variables.keys())
         server_name = variables["SERVER_NAME"].split(" ")[0]
         was_draft = config.get(f"{server_name}_IS_DRAFT", {"value": "no"})["value"] == "yes"
         operation = request.form["operation"]
@@ -1095,7 +1092,7 @@ def services():
 
     # Display services
     services = []
-    global_config = app.config["CONFIG"].get_config(with_drafts=True)
+    global_config = app.config["DB"].get_config(methods=True, with_drafts=True)
     service_names = global_config["SERVER_NAME"]["value"].split(" ")
     for service in service_names:
         service_settings = []
@@ -1155,7 +1152,7 @@ def global_config():
         del variables["csrf_token"]
 
         # Edit check fields and remove already existing ones
-        config = app.config["CONFIG"].get_config(methods=True, with_drafts=True)
+        config = app.config["CONFIG"].get_config(with_drafts=True, filtered_settings=variables.keys())
         services = config["SERVER_NAME"]["value"].split(" ")
         for variable, value in variables.copy().items():
             if variable in ("AUTOCONF_MODE", "SWARM_MODE", "KUBERNETES_MODE", "SERVER_NAME", "IS_LOADING", "IS_DRAFT") or variable.endswith("SCHEMA"):
@@ -1344,7 +1341,7 @@ def configs():
             path_to_dict(
                 join(sep, "etc", "bunkerweb", "configs"),
                 db_data=db_configs,
-                services=app.config["CONFIG"].get_config(methods=False).get("SERVER_NAME", "").split(" "),
+                services=app.config["CONFIG"].get_config(global_only=True, methods=False, filtered_settings=("SERVER_NAME",)).get("SERVER_NAME", "").split(" "),
             )
         ],
         username=current_user.get_id(),
@@ -1869,7 +1866,7 @@ def cache():
                 join(sep, "var", "cache", "bunkerweb"),
                 is_cache=True,
                 db_data=app.config["DB"].get_jobs_cache_files(),
-                services=app.config["CONFIG"].get_config(methods=False).get("SERVER_NAME", "").split(" "),
+                services=app.config["CONFIG"].get_config(global_only=True, methods=False, filtered_settings=("SERVER_NAME",)).get("SERVER_NAME", "").split(" "),
             )
         ],
         username=current_user.get_id(),
@@ -1879,8 +1876,8 @@ def cache():
 @app.route("/logs", methods=["GET"])
 @login_required
 def logs():
-    config = app.config["CONFIG"].get_config(with_drafts=True)
-    override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
+    config = app.config["CONFIG"].get_config(global_only=True, methods=False, filtered_settings=("OVERRIDE_INSTANCES",))
+    override_instances = config.get("OVERRIDE_INSTANCES", "") != ""
     instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
     return render_template("logs.html", instances=instances, username=current_user.get_id())
 
@@ -2158,7 +2155,25 @@ def bans():
         return redirect_flash_error("Database is in read-only mode", "bans")
 
     redis_client = None
-    db_config = app.config["CONFIG"].get_config(methods=False)
+    db_config = app.config["CONFIG"].get_config(
+        global_only=True,
+        methods=False,
+        filtered_settings=(
+            "USE_REDIS",
+            "REDIS_HOST",
+            "REDIS_PORT",
+            "REDIS_DB",
+            "REDIS_TIMEOUT",
+            "REDIS_KEEPALIVE_POOL",
+            "REDIS_SSL",
+            "REDIS_USERNAME",
+            "REDIS_PASSWORD",
+            "REDIS_SENTINEL_HOSTS",
+            "REDIS_SENTINEL_USERNAME",
+            "REDIS_SENTINEL_PASSWORD",
+            "REDIS_SENTINEL_MASTER",
+        ),
+    )
     use_redis = db_config.get("USE_REDIS", "no") == "yes"
     redis_host = db_config.get("REDIS_HOST")
     if use_redis and redis_host:

src/ui/src/Config.py

@@ -78,7 +78,13 @@ class Config:
     def get_settings(self) -> dict:
         return self.__settings
 
-    def get_config(self, global_only: bool = False, methods: bool = True, with_drafts: bool = False) -> dict:
+    def get_config(
+        self,
+        global_only: bool = False,
+        methods: bool = True,
+        with_drafts: bool = False,
+        filtered_settings: Optional[Union[List[str], Set[str], Tuple[str]]] = None,
+    ) -> dict:
         """Get the nginx variables env file and returns it as a dict
 
         Returns
@@ -86,7 +92,7 @@ class Config:
         dict
             The nginx variables env file as a dict
         """
-        return self.__db.get_config(global_only=global_only, methods=methods, with_drafts=with_drafts)
+        return self.__db.get_filtered_config(global_only=global_only, methods=methods, with_drafts=with_drafts, filtered_settings=filtered_settings)
 
     def get_services(self, methods: bool = True, with_drafts: bool = False) -> list[dict]:
         """Get nginx's services

src/ui/src/Instances.py

@@ -152,8 +152,8 @@ class Instances:
         instances = []
         # Override case : only return instances from DB
         if override_instances is None:
-            config = self.__db.get_config()
-            override_instances = config["OVERRIDE_INSTANCES"] != ""
+            config = self.__db.get_filtered_config(global_only=True, filtered_settings=("OVERRIDE_INSTANCES",))
+            override_instances = config.get("OVERRIDE_INSTANCES", "") != ""
         if override_instances:
             for instance in self.__db.get_instances():
                 instances.append(

src/ui/templates/setting_checkbox.html

@@ -6,6 +6,7 @@
     {% set inp_regex = setting_input['regex'] %}
     {% set inp_default = setting_input['default'] %}
     {% set inp_value = setting_input['value'] %}
+    {% set inp_method = setting_input['method'] %}
     {% set global_config_method = global_config.get(inp_name, {'method' : inp_method }).get('method') %}
     {% set global_config_value = global_config.get(inp_name, {'value' : inp_value }).get('value') %}
     {% set is_read_only = True if setting_input['is_pro_plugin'] and not is_pro_version else False %}

src/ui/templates/setting_input.html

@@ -5,6 +5,7 @@
     {% set inp_type = setting_input['type'] %}
     {% set inp_default = setting_input['default'] %}
     {% set inp_value = setting_input['value'] %}
+    {% set inp_method = setting_input['method'] %}
     {% set inp_regex = setting_input['regex'] %}
     {% set global_config_method = global_config.get(inp_name, {'method' : inp_method }).get('method') %}
     {% set global_config_value = global_config.get(inp_name, {'value' : inp_value }).get('value') %}

src/ui/templates/settings_plugins.html

@@ -76,7 +76,7 @@
         <div data-plugin-settings class="w-full grid grid-cols-12">
             <!-- plugin settings not multiple -->
             {% for setting, value in plugin["settings"].items() %}
-                {% set setting_input = { "is_pro_plugin" : True if plugin["type"] == "pro" else False, "name" : setting, "context" : value.get("context"), "method" : value.get("method"), "help" : value.get("help"), "label" : value.get("label"), "id" : value.get("id"), "type" : value.get("type"), "default" : value.get("default"), "select" : value.get("select"), "regex" : value.get("regex"), "value" : value.get("value"), "is_multiple" : False, "levels" : value.get('levels', {})} %}
+                {% set setting_input = { "is_pro_plugin" : True if plugin["type"] == "pro" else False, "name" : setting, "context" : value.get("context"), "method" : value.get("method", "default"), "help" : value.get("help"), "label" : value.get("label"), "id" : value.get("id"), "type" : value.get("type"), "default" : value.get("default", value.get("value")), "select" : value.get("select"), "regex" : value.get("regex"), "value" : value.get("value"), "is_multiple" : False, "levels" : value.get('levels', {})} %}
                 {% if setting != "IS_DRAFT" and (current_endpoint == "global-config" and setting not in ["SERVER_NAME", "IS_LOADING"] or current_endpoint == "services" and value['context'] == "multisite") %}
                     {% if value['multiple'] and value['multiple'] not in multList %}
                         {% if multList.append(value['multiple']) %}{% endif %}
@@ -118,7 +118,7 @@
             <!-- multiple settings -->
             <div data-{{ current_endpoint }}-settings-multiple="{{ multiple }}_SCHEMA" class="bg-gray-50 dark:bg-slate-900/30 hidden w-full my-4 grid-cols-12 border dark:border-gray-700 rounded">
                 {% for setting, value in plugin["settings"].items() %}
-                    {% set setting_input = { "is_pro_plugin" : True if plugin["type"] == "pro" else False, "name" : setting, "context" : value.get("context"), "method" : value.get("method"), "help" : value.get("help"), "label" : value.get("label"), "id" : value.get("id"), "type" : value.get("type"), "default" : value.get("default"), "select" : value.get("select"), "regex" : value.get("regex"), "value" : value.get("value"), "is_multiple" : True, "levels" : value.get('levels', {})} %}
+                    {% set setting_input = { "is_pro_plugin" : True if plugin["type"] == "pro" else False, "name" : setting, "context" : value.get("context"), "method" : value.get("method", "default"), "help" : value.get("help"), "label" : value.get("label"), "id" : value.get("id"), "type" : value.get("type"), "default" : value.get("default", value.get("value")), "select" : value.get("select"), "regex" : value.get("regex"), "value" : value.get("value"), "is_multiple" : True, "levels" : value.get('levels', {})} %}
                     {# render only setting that match the multiple id and context #}
                     {% if value['multiple'] == multiple and (
                         current_endpoint == "global-config"