From a7069bd60583a147a038fd2bb4e64cb9454dac7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Thu, 8 Jun 2023 18:09:44 -0400 Subject: [PATCH] Update UI to stop using env variables but werkzeug middleware + Send X-Forwarded-Prefix headers to UI service --- .../confs/server-http/reverse-proxy.conf | 6 + src/common/db/Database.py | 45 +++---- src/common/db/model.py | 1 - src/ui/main.py | 111 ++---------------- src/ui/src/ReverseProxied.py | 83 ++++++++++--- src/ui/utils.py | 17 --- 6 files changed, 96 insertions(+), 167 deletions(-) diff --git a/src/common/core/reverseproxy/confs/server-http/reverse-proxy.conf b/src/common/core/reverseproxy/confs/server-http/reverse-proxy.conf index 75e5302b4..c49f72d2c 100644 --- a/src/common/core/reverseproxy/confs/server-http/reverse-proxy.conf +++ b/src/common/core/reverseproxy/confs/server-http/reverse-proxy.conf @@ -46,6 +46,12 @@ location {{ url }} {% raw %}{{% endraw +%} proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Protocol $scheme; proxy_set_header X-Forwarded-Host $http_host; + {% if USE_UI == "yes" +%} + {% if url.endswith("/") +%} + {% set url = url[:-1] +%} + {% endif +%} + proxy_set_header X-Forwarded-Prefix {{ url }}; + {% endif +%} {% if buffering == "yes" +%} proxy_buffering on; {% else +%} diff --git a/src/common/db/Database.py b/src/common/db/Database.py index e0d50d9bc..310b0abb2 100644 --- a/src/common/db/Database.py +++ b/src/common/db/Database.py @@ -264,44 +264,32 @@ class Database: return "" - def check_changes( - self, _type: Union[Literal["scheduler"], Literal["ui"]] = "scheduler" - ) -> Union[Dict[str, bool], bool, str]: + def check_changes(self) -> Union[Dict[str, bool], bool, str]: """Check if either the config, the custom configs or plugins have changed inside the database""" with self.__db_session() as session: try: - if _type == "scheduler": - entities = ( + metadata = ( + session.query(Metadata) + .with_entities( Metadata.custom_configs_changed, Metadata.external_plugins_changed, Metadata.config_changed, ) - else: - entities = (Metadata.ui_config_changed,) - - metadata = ( - session.query(Metadata) - .with_entities(*entities) .filter_by(id=1) .first() ) - if _type == "scheduler": - return dict( - custom_configs_changed=metadata is not None - and metadata.custom_configs_changed, - external_plugins_changed=metadata is not None - and metadata.external_plugins_changed, - config_changed=metadata is not None and metadata.config_changed, - ) - else: - return metadata is not None and metadata.ui_config_changed + return dict( + custom_configs_changed=metadata is not None + and metadata.custom_configs_changed, + external_plugins_changed=metadata is not None + and metadata.external_plugins_changed, + config_changed=metadata is not None and metadata.config_changed, + ) except BaseException: return format_exc() - def checked_changes( - self, _type: Union[Literal["scheduler"], Literal["ui"]] = "scheduler" - ) -> str: + def checked_changes(self) -> str: """Set that the config, the custom configs and the plugins didn't change""" with self.__db_session() as session: try: @@ -310,12 +298,9 @@ class Database: if not metadata: return "The metadata are not set yet, try again" - if _type == "scheduler": - metadata.config_changed = False - metadata.custom_configs_changed = False - metadata.external_plugins_changed = False - else: - metadata.ui_config_changed = False + metadata.config_changed = False + metadata.custom_configs_changed = False + metadata.external_plugins_changed = False session.commit() except BaseException: return format_exc() diff --git a/src/common/db/model.py b/src/common/db/model.py index 1de9d2e5b..33100482e 100644 --- a/src/common/db/model.py +++ b/src/common/db/model.py @@ -282,6 +282,5 @@ class Metadata(Base): custom_configs_changed = Column(Boolean, default=False, nullable=True) external_plugins_changed = Column(Boolean, default=False, nullable=True) config_changed = Column(Boolean, default=False, nullable=True) - ui_config_changed = Column(Boolean, default=False, nullable=True) integration = Column(INTEGRATIONS_ENUM, default="Unknown", nullable=False) version = Column(String(32), default="1.5.0", nullable=False) diff --git a/src/ui/main.py b/src/ui/main.py index 24f7b633e..e92e7e1ff 100755 --- a/src/ui/main.py +++ b/src/ui/main.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -from os import _exit, environ, getenv, listdir, sep +from os import _exit, getenv, listdir, sep, urandom from os.path import basename, dirname, join from sys import path as sys_path, modules as sys_modules from pathlib import Path @@ -20,7 +20,7 @@ for deps_path in [ if deps_path not in sys_path: sys_path.append(deps_path) -from gevent import monkey, spawn +from gevent import monkey monkey.patch_all() @@ -124,21 +124,18 @@ app = Flask( static_folder="static", template_folder="templates", ) -app.wsgi_app = ReverseProxied(app.wsgi_app) +app.wsgi_app = ReverseProxied(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1) -# Set variables and instantiate objects -vars = get_variables() - -if "ADMIN_USERNAME" not in vars: +if not getenv("ADMIN_USERNAME"): logger.error("ADMIN_USERNAME is not set") stop(1) -elif "ADMIN_PASSWORD" not in vars: +elif not getenv("ADMIN_PASSWORD"): logger.error("ADMIN_PASSWORD is not set") stop(1) -if not vars.get("FLASK_DEBUG", False) and not regex_match( +if not getenv("FLASK_DEBUG", False) and not regex_match( r"^(?=.*?\p{Lowercase_Letter})(?=.*?\p{Uppercase_Letter})(?=.*?\d)(?=.*?[ !\"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]).{8,}$", - vars["ADMIN_PASSWORD"], + getenv("ADMIN_PASSWORD", "changeme"), ): logger.error( "The admin password is not strong enough. It must contain at least 8 characters, including at least 1 uppercase letter, 1 lowercase letter, 1 number and 1 special character (#@?!$%^&*-)." @@ -148,7 +145,7 @@ if not vars.get("FLASK_DEBUG", False) and not regex_match( login_manager = LoginManager() login_manager.init_app(app) login_manager.login_view = "login" -user = User(vars["ADMIN_USERNAME"], vars["ADMIN_PASSWORD"]) +user = User(getenv("ADMIN_USERNAME", "admin"), getenv("ADMIN_PASSWORD", "changeme")) PLUGIN_KEYS = [ "id", "name", @@ -176,7 +173,7 @@ kubernetes_client = None if INTEGRATION in ("Docker", "Swarm", "Autoconf"): try: docker_client: DockerClient = DockerClient( - base_url=vars.get("DOCKER_HOST", "unix:///var/run/docker.sock") + base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock") ) except (docker_APIError, DockerException): logger.warning("No docker host found") @@ -221,99 +218,13 @@ bw_version = ( .strip() ) -ABSOLUTE_URI = vars.get("ABSOLUTE_URI") -CONFIG = Config(db) - - -def update_config(): - global ABSOLUTE_URI - - ret = db.checked_changes("ui") - - if ret: - logger.error( - f"An error occurred when setting the changes to checked in the database : {ret}" - ) - stop(1) - - ssl = False - server_name = None - endpoint = None - - for service in CONFIG.get_services(): - if service.get("USE_UI", "no") == "no": - continue - - server_name = service.get("SERVER_NAME", {"value": None})["value"] - endpoint = service.get("REVERSE_PROXY_URL", {"value": "/"})["value"] - - if any( - [ - service.get("AUTO_LETS_ENCRYPT", {"value": "no"})["value"] == "yes", - service.get("GENERATE_SELF_SIGNED_SSL", {"value": "no"})["value"] - == "yes", - service.get("USE_CUSTOM_SSL", {"value": "no"})["value"] == "yes", - ] - ): - ssl = True - break - - if not server_name: - logger.error("No service found with USE_UI=yes") - stop(1) - - ABSOLUTE_URI = f"http{'s' if ssl else ''}://{server_name}{endpoint}" - SCRIPT_NAME = f"/{basename(ABSOLUTE_URI[:-1] if ABSOLUTE_URI.endswith('/') and ABSOLUTE_URI != '/' else ABSOLUTE_URI)}" - - if not ABSOLUTE_URI.endswith("/"): - ABSOLUTE_URI += "/" - - if ABSOLUTE_URI != app.config.get("ABSOLUTE_URI"): - app.config["ABSOLUTE_URI"] = ABSOLUTE_URI - app.config["SESSION_COOKIE_DOMAIN"] = server_name - - logger.info(f"The ABSOLUTE_URI is now {ABSOLUTE_URI}") - else: - logger.info(f"The ABSOLUTE_URI is still {ABSOLUTE_URI}") - - if SCRIPT_NAME != getenv("SCRIPT_NAME"): - environ["SCRIPT_NAME"] = f"/{basename(ABSOLUTE_URI[:-1])}" - logger.info(f"The SCRIPT_NAME is now {environ['SCRIPT_NAME']}") - else: - logger.info(f"The SCRIPT_NAME is still {environ['SCRIPT_NAME']}") - - -def check_config_changes(): - while True: - changes = db.check_changes("ui") - - if isinstance(changes, str): - continue - - if changes: - logger.info( - "Config changed in the database, updating ABSOLUTE_URI and SCRIPT_NAME ..." - ) - - update_config() - - sleep(1) - - -update_config() - -spawn(check_config_changes) - try: app.config.update( DEBUG=True, - SECRET_KEY=vars["FLASK_SECRET"], + SECRET_KEY=getenv("FLASK_SECRET", urandom(32)), INSTANCES=Instances(docker_client, kubernetes_client, INTEGRATION), CONFIG=Config(db), CONFIGFILES=ConfigFiles(logger, db), - SESSION_COOKIE_DOMAIN=ABSOLUTE_URI.replace("http://", "") - .replace("https://", "") - .split("/")[0], WTF_CSRF_SSL_STRICT=False, USER=user, SEND_FILE_MAX_AGE_DEFAULT=86400, @@ -394,7 +305,7 @@ def set_csp_header(response): @login_manager.user_loader def load_user(user_id): - return User(user_id, vars["ADMIN_PASSWORD"]) + return User(user_id, getenv("ADMIN_PASSWORD", "changeme")) @app.errorhandler(CSRFError) diff --git a/src/ui/src/ReverseProxied.py b/src/ui/src/ReverseProxied.py index 5af73cbe3..7d880d7bc 100644 --- a/src/ui/src/ReverseProxied.py +++ b/src/ui/src/ReverseProxied.py @@ -1,27 +1,72 @@ #!/usr/bin/python3 +from typing import Iterable +from wsgiref.types import StartResponse, WSGIEnvironment +from werkzeug.middleware.proxy_fix import ProxyFix -class ReverseProxied(object): - def __init__(self, app): - self.app = app - def __call__(self, environ, start_response): +class ReverseProxied(ProxyFix): + def __call__( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> Iterable[bytes]: + """Modify the WSGI environ based on the various ``Forwarded`` + headers before calling the wrapped application. Store the + original environ values in ``werkzeug.proxy_fix.orig_{key}``. """ - If the app is behind a reverse proxy, it will modify the - environ object to make it look like the request was received on the app directly + environ_get = environ.get + orig_remote_addr = environ_get("REMOTE_ADDR") + orig_wsgi_url_scheme = environ_get("wsgi.url_scheme") + orig_http_host = environ_get("HTTP_HOST") + environ.update( + { + "werkzeug.proxy_fix.orig": { + "REMOTE_ADDR": orig_remote_addr, + "wsgi.url_scheme": orig_wsgi_url_scheme, + "HTTP_HOST": orig_http_host, + "SERVER_NAME": environ_get("SERVER_NAME"), + "SERVER_PORT": environ_get("SERVER_PORT"), + "SCRIPT_NAME": environ_get("SCRIPT_NAME"), + } + } + ) - :param environ: The WSGI environment dict - :param start_response: This is the WSGI-compatible start_response function that the - :return: A WSGI application. - """ - script_name = environ.get("HTTP_X_SCRIPT_NAME", "") - if script_name: - environ["SCRIPT_NAME"] = script_name - path_info = environ["PATH_INFO"] - if path_info.startswith(script_name): - environ["PATH_INFO"] = path_info[len(script_name) :] + x_for = self._get_real_value(self.x_for, environ_get("HTTP_X_FORWARDED_FOR")) + if x_for: + environ["REMOTE_ADDR"] = x_for + + x_proto = self._get_real_value( + self.x_proto, environ_get("HTTP_X_FORWARDED_PROTO") + ) + if x_proto: + environ["wsgi.url_scheme"] = x_proto + + x_host = self._get_real_value(self.x_host, environ_get("HTTP_X_FORWARDED_HOST")) + if x_host: + environ["HTTP_HOST"] = environ["SERVER_NAME"] = x_host + # "]" to check for IPv6 address without port + if ":" in x_host and not x_host.endswith("]"): + environ["SERVER_NAME"], environ["SERVER_PORT"] = x_host.rsplit(":", 1) + + x_port = self._get_real_value(self.x_port, environ_get("HTTP_X_FORWARDED_PORT")) + if x_port: + host = environ.get("HTTP_HOST") + if host: + # "]" to check for IPv6 address without port + if ":" in host and not host.endswith("]"): + host = host.rsplit(":", 1)[0] + environ["HTTP_HOST"] = f"{host}:{x_port}" + environ["SERVER_PORT"] = x_port + + x_prefix = self._get_real_value( + self.x_prefix, environ_get("HTTP_X_FORWARDED_PREFIX") + ) + if x_prefix: + environ["SCRIPT_NAME"] = x_prefix + + environ["PATH_INFO"] = environ["PATH_INFO"][len(environ["SCRIPT_NAME"]) :] + environ[ + "ABSOLUTE_URI" + ] = f"{environ['wsgi.url_scheme']}://{environ['HTTP_HOST']}{environ['SCRIPT_NAME']}/" + environ["SESSION_COOKIE_DOMAIN"] = environ["HTTP_HOST"] - scheme = environ.get("HTTP_X_FORWARDED_PROTO", "") - if scheme: - environ["wsgi.url_scheme"] = scheme return self.app(environ, start_response) diff --git a/src/ui/utils.py b/src/ui/utils.py index 5958eb55e..a3d1beeaf 100644 --- a/src/ui/utils.py +++ b/src/ui/utils.py @@ -1,26 +1,9 @@ #!/usr/bin/python3 -from os import environ, urandom from os.path import join from typing import List, Optional -def get_variables(): - vars = {} - vars["DOCKER_HOST"] = "unix:///var/run/docker.sock" - vars["ABSOLUTE_URI"] = "" - vars["FLASK_SECRET"] = urandom(32) - vars["FLASK_ENV"] = "development" - vars["ADMIN_USERNAME"] = "admin" - vars["ADMIN_PASSWORD"] = "changeme" - - for k in vars: - if k in environ: - vars[k] = environ[k] - - return vars - - def path_to_dict( path: str, *,