From e25948ecaf0b2c2a78b714c80082277a85daffaf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 15:41:42 +0000 Subject: [PATCH 01/42] deps/terraform: bump scaleway/scaleway in /tests/terraform Bumps [scaleway/scaleway](https://github.com/scaleway/terraform-provider-scaleway) from 2.38.2 to 2.40.0. - [Release notes](https://github.com/scaleway/terraform-provider-scaleway/releases) - [Changelog](https://github.com/scaleway/terraform-provider-scaleway/blob/master/CHANGELOG.md) - [Commits](https://github.com/scaleway/terraform-provider-scaleway/compare/v2.38.2...v2.40.0) --- updated-dependencies: - dependency-name: scaleway/scaleway dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- tests/terraform/providers.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/terraform/providers.tf b/tests/terraform/providers.tf index e623037d2..919db7e10 100644 --- a/tests/terraform/providers.tf +++ b/tests/terraform/providers.tf @@ -2,7 +2,7 @@ terraform { required_providers { scaleway = { source = "scaleway/scaleway" - version = "2.38.2" + version = "2.40.0" } kubectl = { source = "gavinbunney/kubectl" From 31a8fed04cf8e24214f5efd8ca859daa0d661667 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 15:41:53 +0000 Subject: [PATCH 02/42] deps/terraform: bump hashicorp/kubernetes in /tests/terraform Bumps [hashicorp/kubernetes](https://github.com/hashicorp/terraform-provider-kubernetes) from 2.27.0 to 2.30.0. - [Release notes](https://github.com/hashicorp/terraform-provider-kubernetes/releases) - [Changelog](https://github.com/hashicorp/terraform-provider-kubernetes/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/terraform-provider-kubernetes/compare/v2.27.0...v2.30.0) --- updated-dependencies: - dependency-name: hashicorp/kubernetes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- tests/terraform/providers.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/terraform/providers.tf b/tests/terraform/providers.tf index e623037d2..67f27e70b 100644 --- a/tests/terraform/providers.tf +++ b/tests/terraform/providers.tf @@ -10,7 +10,7 @@ terraform { } kubernetes = { source = "hashicorp/kubernetes" - version = "2.27.0" + version = "2.30.0" } } } \ No newline at end of file From 5d5bdde8fa2367c9091754fa62a7be1f738fda0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 07:24:57 +0000 Subject: [PATCH 03/42] deps/linux: bump redhat/ubi8 from 8.9 to 8.10 in /src/linux Bumps redhat/ubi8 from 8.9 to 8.10. --- updated-dependencies: - dependency-name: redhat/ubi8 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- src/linux/Dockerfile-rhel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/linux/Dockerfile-rhel b/src/linux/Dockerfile-rhel index da493575c..e43c8aa99 100644 --- a/src/linux/Dockerfile-rhel +++ b/src/linux/Dockerfile-rhel @@ -1,4 +1,4 @@ -FROM redhat/ubi8:8.9@sha256:627867e53ad6846afba2dfbf5cef1d54c868a9025633ef0afd546278d4654eac as builder +FROM redhat/ubi8:8.10@sha256:a424544997de1960a93466b57d12f1f3fac62be0f4cd35482435bae305a6ca27 as builder ENV OS=rhel ENV NGINX_VERSION 1.24.0 @@ -68,7 +68,7 @@ COPY src/scheduler scheduler COPY src/ui ui COPY src/VERSION VERSION -FROM redhat/ubi8:8.9@sha256:627867e53ad6846afba2dfbf5cef1d54c868a9025633ef0afd546278d4654eac +FROM redhat/ubi8:8.10@sha256:a424544997de1960a93466b57d12f1f3fac62be0f4cd35482435bae305a6ca27 # Set default umask to prevent huge recursive chmod increasing the final image size RUN umask 027 From ef62453712149151d930054dfc402f4abf05943b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 07:30:45 +0000 Subject: [PATCH 04/42] deps/tests/linux: bump redhat/ubi8-init in /tests/linux Bumps redhat/ubi8-init from 8.9-7.1715071668 to 8.10-2. --- updated-dependencies: - dependency-name: redhat/ubi8-init dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/linux/Dockerfile-rhel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/linux/Dockerfile-rhel b/tests/linux/Dockerfile-rhel index 6f3a7281c..2b091030e 100644 --- a/tests/linux/Dockerfile-rhel +++ b/tests/linux/Dockerfile-rhel @@ -1,4 +1,4 @@ -FROM redhat/ubi8-init:8.9-7.1715071668@sha256:3bcb1434ddc595236a1e45b9c4d4722ab8f1348a371c6239973bbf4d67b24c96 +FROM redhat/ubi8-init:8.10-2@sha256:26aec3f78f127e39cb45e7eebd1dafc17071246d78dc51be4cfcb205ffc89caa ENV NGINX_VERSION 1.24.0 From 68ea285a618527f8a21235faf481beb9c01f3f47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 07:32:17 +0000 Subject: [PATCH 05/42] deps/gha: bump aquasecurity/trivy-action from 0.20.0 to 0.21.0 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.20.0 to 0.21.0. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/b2933f565dbc598b29947660e66259e3c7bc8561...fd25fed6972e341ff0007ddb61f77e88103953c2) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/container-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-build.yml b/.github/workflows/container-build.yml index e1b766bc5..951d4c4c5 100644 --- a/.github/workflows/container-build.yml +++ b/.github/workflows/container-build.yml @@ -117,7 +117,7 @@ jobs: # Check OS vulnerabilities - name: Check OS vulnerabilities if: ${{ inputs.CACHE_SUFFIX != 'arm' }} - uses: aquasecurity/trivy-action@b2933f565dbc598b29947660e66259e3c7bc8561 # v0.20.0 + uses: aquasecurity/trivy-action@fd25fed6972e341ff0007ddb61f77e88103953c2 # v0.21.0 with: vuln-type: os skip-dirs: /root/.cargo From 47857925a782495ce5cb5bac890178fce0817789 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 07:32:22 +0000 Subject: [PATCH 06/42] deps/gha: bump ruby/setup-ruby from 1.177.0 to 1.177.1 Bumps [ruby/setup-ruby](https://github.com/ruby/setup-ruby) from 1.177.0 to 1.177.1. - [Release notes](https://github.com/ruby/setup-ruby/releases) - [Commits](https://github.com/ruby/setup-ruby/compare/7dc18ff0ca6e3630d3f29d2a85ebf6cc27ae9d6c...943103cae7d3f1bb1e4951d5fcc7928b40e4b742) --- updated-dependencies: - dependency-name: ruby/setup-ruby dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/push-packagecloud.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-packagecloud.yml b/.github/workflows/push-packagecloud.yml index ae863fe81..6a2548432 100644 --- a/.github/workflows/push-packagecloud.yml +++ b/.github/workflows/push-packagecloud.yml @@ -42,7 +42,7 @@ jobs: - name: Check out repository code uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - name: Install ruby - uses: ruby/setup-ruby@7dc18ff0ca6e3630d3f29d2a85ebf6cc27ae9d6c # v1.177.0 + uses: ruby/setup-ruby@943103cae7d3f1bb1e4951d5fcc7928b40e4b742 # v1.177.1 with: ruby-version: "3.0" - name: Install packagecloud From f4b25af7ed85f1a8ed2cf23f1f9d76904025dc6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Thu, 23 May 2024 10:36:27 +0100 Subject: [PATCH 07/42] chore: Fix changed_service value being wrong when editing a service with multiple SERVER_NAME --- src/ui/src/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ui/src/Config.py b/src/ui/src/Config.py index f5a5b2b33..8fed54421 100644 --- a/src/ui/src/Config.py +++ b/src/ui/src/Config.py @@ -204,7 +204,7 @@ class Config: if k.startswith(old_server_name_splitted[0]): config.pop(k) - ret = self.__gen_conf(config, services, check_changes=check_changes, changed_service=variables["SERVER_NAME"]) + ret = self.__gen_conf(config, services, check_changes=check_changes, changed_service=server_name_splitted[0]) if ret: return ret, 1 return f"Configuration for {old_server_name_splitted[0]} has been edited.", 0 From 4361199513affadadc4e38471c808c9d869e1e46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Thu, 23 May 2024 11:25:46 +0100 Subject: [PATCH 08/42] [#1202] Add force renewal option to certbot_new function when the domains changed This commit adds a new optional parameter, `force`, to the `certbot_new` function in the `certbot-new.py` file. The `force` parameter allows for forcing the renewal of certificates even if they are not due for renewal. This can be useful in certain scenarios where certificate updates are required regardless of their expiration date. --- src/common/core/letsencrypt/jobs/certbot-new.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index 6956fb7fa..382b94afe 100755 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -29,7 +29,7 @@ LETS_ENCRYPT_WORK_DIR = join(sep, "var", "lib", "bunkerweb", "letsencrypt") LETS_ENCRYPT_LOGS_DIR = join(sep, "var", "log", "bunkerweb") -def certbot_new(domains: str, email: str, use_letsencrypt_staging: bool = False) -> int: +def certbot_new(domains: str, email: str, use_letsencrypt_staging: bool = False, *, force: bool = False) -> int: process = Popen( [ CERTBOT_BIN, @@ -54,7 +54,8 @@ def certbot_new(domains: str, email: str, use_letsencrypt_staging: bool = False) "--agree-tos", "--expand", ] - + (["--staging"] if use_letsencrypt_staging else []), + + (["--staging"] if use_letsencrypt_staging else []) + + (["--force-renewal"] if force else []), stdin=DEVNULL, stderr=PIPE, universal_newlines=True, @@ -96,7 +97,7 @@ try: # Restore Let's Encrypt data from db cache JOB.restore_cache(job_name="certbot-renew") - domains_to_ask = [] + domains_to_ask = {} # Multisite case if is_multisite: domains_server_names = {} @@ -133,18 +134,18 @@ try: if proc.returncode != 0: LOGGER.error(f"Error while checking certificates :\n{proc.stdout}") - domains_to_ask = server_names + domains_to_ask = {domain: True for domain in server_names} else: for first_server, domains in domains_server_names.items(): generated_domains.update(domains.split(" ")) current_domains = search(rf"Domains: {first_server}(?P.*)$", stdout, MULTILINE) if not current_domains: - domains_to_ask.append(first_server) + domains_to_ask[first_server] = False continue elif set(f"{first_server}{current_domains.groupdict()['domains']}".strip().split(" ")) != set(domains.split(" ")): LOGGER.warning(f"Domains for {first_server} are not the same as in the certificate, asking new certificate...") - domains_to_ask.append(first_server) + domains_to_ask[first_server] = True continue LOGGER.info(f"Certificates already exists for domain(s) {domains}") @@ -159,7 +160,7 @@ try: use_letsencrypt_staging = getenv(f"{first_server}_USE_LETS_ENCRYPT_STAGING", getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes" LOGGER.info(f"Asking certificates for domain(s) : {domains} (email = {real_email}) to Let's Encrypt {'staging ' if use_letsencrypt_staging else ''}...") - if certbot_new(domains.replace(" ", ","), real_email, use_letsencrypt_staging) != 0: + if certbot_new(domains.replace(" ", ","), real_email, use_letsencrypt_staging, force=domains_to_ask[first_server]) != 0: status = 2 LOGGER.error(f"Certificate generation failed for domain(s) {domains} ...") continue From 133552321bdf4eb402afedb464f69af883610a60 Mon Sep 17 00:00:00 2001 From: Jordan Blasenhauer Date: Thu, 23 May 2024 14:40:20 +0200 Subject: [PATCH 09/42] add clipboard await catch handler --- src/ui/static/js/global.js | 55 ++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/ui/static/js/global.js b/src/ui/static/js/global.js index d334d74bf..10251881b 100644 --- a/src/ui/static/js/global.js +++ b/src/ui/static/js/global.js @@ -85,7 +85,7 @@ class News { ) { sessionStorage.setItem( "lastRefetch", - Math.round(new Date().getTime() / 1000) + 3600, + Math.round(new Date().getTime() / 1000) + 3600 ); sessionStorage.setItem("lastNews", JSON.stringify(lastNews)); } @@ -102,7 +102,7 @@ class News { news.photo.url, news.excerpt, news.tags, - news.date, + news.date ); const BASE_URL = this.BASE_URL; let cleanHTML = DOMPurify.sanitize(cardHTML); @@ -114,7 +114,7 @@ class News { slug.addEventListener("click", function () { window.open( `${BASE_URL}blog/post/${news.slug}?utm_campaign=self&utm_source=ui`, - "_blank", + "_blank" ); }); }); @@ -268,7 +268,7 @@ class FlashMsg { if (Number(this.flashCount.textContent) > 0) this.animeBtn(); // display only one fixed flash message const flashFixedEls = document.querySelectorAll( - "[data-flash-message-fixed]", + "[data-flash-message-fixed]" ); if (flashFixedEls.length > 1) { flashFixedEls.forEach((el, i) => { @@ -298,7 +298,7 @@ class FlashMsg { flashEl.remove(); //update count this.flashCount.textContent = document.querySelectorAll( - "[data-flash-message]", + "[data-flash-message]" ).length; } } catch (err) {} @@ -401,7 +401,7 @@ class Banner { // Try to get data from api if (sessionStorage.getItem("bannerNews") !== null) { return this.updateBanner( - JSON.parse(sessionStorage.getItem("bannerNews")), + JSON.parse(sessionStorage.getItem("bannerNews")) ); } fetch("https://www.bunkerweb.io/api/bw-ui-news") @@ -413,7 +413,7 @@ class Banner { // Refetch after one hour sessionStorage.setItem( "bannerRefetch", - Math.round(new Date().getTime() / 1000) + 3600, + Math.round(new Date().getTime() / 1000) + 3600 ); return this.updateBanner(res.data[0].data); }) @@ -446,7 +446,7 @@ class Banner { this.bannerEl.querySelector( `[role="listitem"][data-id="${ +visibleEl.getAttribute("data-id") + 1 - }"]`, + }"]` ) || this.bannerEl.querySelector(`[role="listitem"][data-id="0"]`); // Hide current one @@ -534,30 +534,33 @@ class Clipboard { navigator.permissions .query({ name: "clipboard-write" }) .then((result) => { - if (result.state === "granted" || result.state === "prompt") { - /* write to the clipboard now */ - const copyEl = document.querySelector( - e.target.getAttribute("data-clipboard-target"), - ); + try { + if (result.state === "granted" || result.state === "prompt") { + /* write to the clipboard now */ + const copyEl = document.querySelector( + e.target.getAttribute("data-clipboard-target") + ); - copyEl.select(); - copyEl.setSelectionRange(0, 99999); // For mobile devices + copyEl.select(); + copyEl.setSelectionRange(0, 99999); // For mobile devices - // Copy the text inside the text field + // Copy the text inside the text field - navigator.clipboard.writeText(copyEl.value); - // Stop selecting - copyEl.blur(); - this.isCopy = true; - } - }); + navigator.clipboard.writeText(copyEl.value); + // Stop selecting + copyEl.blur(); + this.isCopy = true; + } + } catch (e) {} + }) + .catch((e) => {}); } catch (e) {} // With Firefox try { if (this.isCopy) return; /* write to the clipboard now */ const copyEl = document.querySelector( - e.target.getAttribute("data-clipboard-target"), + e.target.getAttribute("data-clipboard-target") ); copyEl.select(); @@ -575,7 +578,7 @@ class Clipboard { if (this.isCopy) return; /* write to the clipboard now */ const copyEl = document.querySelector( - e.target.getAttribute("data-clipboard-target"), + e.target.getAttribute("data-clipboard-target") ); copyEl.select(); @@ -609,13 +612,13 @@ const setMenu = new Menu(); const setNewsSidebar = new Sidebar( "[data-sidebar-info]", "[data-sidebar-info-open]", - "[data-sidebar-info-close]", + "[data-sidebar-info-close]" ); const setFlashSidebar = new Sidebar( "[data-flash-sidebar]", "[data-flash-sidebar-open]", - "[data-flash-sidebar-close]", + "[data-flash-sidebar-close]" ); const setClipboard = new Clipboard(); From 2587e9ee6ec766073f7236a9a39744765c7e9e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Thu, 23 May 2024 19:07:01 +0100 Subject: [PATCH 10/42] Stop waiting for the autoconf in scheduler's start (can break things) --- src/scheduler/main.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/scheduler/main.py b/src/scheduler/main.py index 3f5b193fb..fb4c742b4 100644 --- a/src/scheduler/main.py +++ b/src/scheduler/main.py @@ -374,15 +374,6 @@ if __name__ == "__main__": run_in_slave_mode() stop(1) - if INTEGRATION in ("Swarm", "Kubernetes", "Autoconf"): - while not SCHEDULER.db.is_initialized(): - logger.warning("Database is not initialized, retrying in 5s ...") - sleep(5) - - while not SCHEDULER.db.is_autoconf_loaded(): - logger.warning("Autoconf is not loaded yet in the database, retrying in 5s ...") - sleep(5) - if ( INTEGRATION in ("Swarm", "Kubernetes", "Autoconf") or not tmp_variables_path.exists() From 1e54e6adcda49d85b39c26f359be592d29e4c85c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Fri, 24 May 2024 11:11:17 +0100 Subject: [PATCH 11/42] chore: Refactor tar file extraction in Job class This commit refactors the tar file extraction logic in the `Job` class in the `jobs.py` file. Instead of using the `extractall` method, it now iterates over the members of the tar file and extracts each member individually. This change allows for better error handling and logging when extracting the tar file. --- src/common/utils/jobs.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/common/utils/jobs.py b/src/common/utils/jobs.py index 311b42552..cc8b108d7 100644 --- a/src/common/utils/jobs.py +++ b/src/common/utils/jobs.py @@ -9,7 +9,7 @@ from os.path import sep from pathlib import Path from shutil import rmtree from sys import argv -from tarfile import open as tar_open +from tarfile import TarFile, open as tar_open from threading import Lock from traceback import format_exc from typing import Any, Dict, Literal, Optional, Tuple, Union @@ -80,15 +80,22 @@ class Job: rmtree(extract_path, ignore_errors=True) extract_path.mkdir(parents=True, exist_ok=True) with tar_open(fileobj=BytesIO(job_cache_file["data"]), mode="r:gz") as tar: + assert isinstance(tar, TarFile) try: - tar.extractall(extract_path, filter="fully_trusted") - except TypeError: - tar.extractall(extract_path) + for member in tar.getmembers(): + try: + tar.extract(member, path=extract_path) + except Exception as e: + self.logger.error(f"Error extracting {member.name}: {e}") + except Exception as e: + self.logger.error(f"Error extracting tar file: {e}") + self.logger.debug(f"Restored cache directory {extract_path}") continue elif job_cache_file["job_name"] != job_name: continue cache_path.parent.mkdir(parents=True, exist_ok=True) cache_path.write_bytes(job_cache_file["data"]) + self.logger.debug(f"Restored cache file {job_cache_file['file_name']}") except BaseException as e: self.logger.error(f"Exception while restoring cache file {job_cache_file['file_name']} :\n{e}") ret = False @@ -207,7 +214,7 @@ class Job: tgz.add(dir_path, arcname=".") content.seek(0, 0) - return self.cache_file(file_name, content.read(), job_name=job_name, service_id=service_id) + return self.cache_file(file_name, content.getvalue(), job_name=job_name, service_id=service_id) def del_cache(self, name: str, *, job_name: str = "", service_id: str = "") -> Tuple[bool, str]: """Delete cache file from database and local cache file.""" From 6179d6f5ff051575bd427179d9206aca8e7a7444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Fri, 24 May 2024 11:12:24 +0100 Subject: [PATCH 12/42] Implement threading startup tasks in scheduler to speed up the starting process --- src/scheduler/main.py | 92 ++++++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/src/scheduler/main.py b/src/scheduler/main.py index fb4c742b4..b19e9aac8 100644 --- a/src/scheduler/main.py +++ b/src/scheduler/main.py @@ -167,11 +167,16 @@ def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Uni logger.error("Sending custom configs failed, configuration will not work as expected...") -def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: Union[Path, str] = EXTERNAL_PLUGINS_PATH): +def generate_external_plugins(plugins: Optional[List[Dict[str, Any]]], *, original_path: Union[Path, str] = EXTERNAL_PLUGINS_PATH): if not isinstance(original_path, Path): original_path = Path(original_path) pro = "pro" in original_path.parts + if not plugins: + assert SCHEDULER is not None + plugins = SCHEDULER.db.get_plugins(_type="pro" if pro else "external", with_data=True) + assert plugins is not None, "Couldn't get plugins from database" + # Remove old external/pro plugins files logger.info(f"Removing old/changed {'pro ' if pro else ''}external plugins files ...") ignored_plugins = set() @@ -423,6 +428,8 @@ if __name__ == "__main__": # Instantiate scheduler SCHEDULER.env = env | environ + threads = [] + if INTEGRATION in ("Docker", "Swarm", "Kubernetes", "Autoconf"): # Automatically setup the scheduler apis while not SCHEDULER.apis: @@ -431,49 +438,49 @@ if __name__ == "__main__": if not SCHEDULER.apis: logger.warning("No BunkerWeb API found, retrying in 5s ...") sleep(5) - SCHEDULER.db.update_instances([api_to_instance(api) for api in SCHEDULER.apis]) + threads.append(Thread(target=SCHEDULER.db.update_instances, args=([api_to_instance(api) for api in SCHEDULER.apis],))) scheduler_first_start = SCHEDULER.db.is_scheduler_first_start() logger.info("Scheduler started ...") - # Checking if any custom config has been created by the user - logger.info("Checking if there are any changes in custom configs ...") - custom_configs = [] - db_configs = SCHEDULER.db.get_custom_configs() - changes = False - for file in CUSTOM_CONFIGS_PATH.rglob("*.conf"): - if len(file.parts) > len(CUSTOM_CONFIGS_PATH.parts) + 3: - logger.warning(f"Custom config file {file} is not in the correct path, skipping ...") + def check_configs_changes(): + # Checking if any custom config has been created by the user + logger.info("Checking if there are any changes in custom configs ...") + custom_configs = [] + db_configs = SCHEDULER.db.get_custom_configs() + changes = False + for file in CUSTOM_CONFIGS_PATH.rglob("*.conf"): + if len(file.parts) > len(CUSTOM_CONFIGS_PATH.parts) + 3: + logger.warning(f"Custom config file {file} is not in the correct path, skipping ...") - content = file.read_text(encoding="utf-8") - service_id = file.parent.name if file.parent.name not in CUSTOM_CONFIGS_DIRS else None - config_type = file.parent.parent.name if service_id else file.parent.name + content = file.read_text(encoding="utf-8") + service_id = file.parent.name if file.parent.name not in CUSTOM_CONFIGS_DIRS else None + config_type = file.parent.parent.name if service_id else file.parent.name - saving = True - in_db = False - for db_conf in db_configs: - if db_conf["service_id"] == service_id and db_conf["name"] == file.stem: - in_db = True + saving = True + in_db = False + for db_conf in db_configs: + if db_conf["service_id"] == service_id and db_conf["name"] == file.stem: + in_db = True - if not in_db and content.startswith("# CREATED BY ENV"): - saving = False - changes = True + if not in_db and content.startswith("# CREATED BY ENV"): + saving = False + changes = True - if saving: - custom_configs.append({"value": content, "exploded": (service_id, config_type, file.stem)}) + if saving: + custom_configs.append({"value": content, "exploded": (service_id, config_type, file.stem)}) - changes = changes or {hash(dict_to_frozenset(d)) for d in custom_configs} != {hash(dict_to_frozenset(d)) for d in db_configs} + changes = changes or {hash(dict_to_frozenset(d)) for d in custom_configs} != {hash(dict_to_frozenset(d)) for d in db_configs} - if changes: - err = SCHEDULER.db.save_custom_configs(custom_configs, "manual") - if err: - logger.error(f"Couldn't save some manually created custom configs to database: {err}") + if changes: + err = SCHEDULER.db.save_custom_configs(custom_configs, "manual") + if err: + logger.error(f"Couldn't save some manually created custom configs to database: {err}") - if (scheduler_first_start and db_configs) or changes: generate_custom_configs(SCHEDULER.db.get_custom_configs()) - del custom_configs, db_configs + threads.append(Thread(target=check_configs_changes)) def check_plugin_changes(_type: Literal["external", "pro"] = "external"): # Check if any external or pro plugin has been added by the user @@ -524,11 +531,15 @@ if __name__ == "__main__": if err: logger.error(f"Couldn't save some manually added {_type} plugins to database: {err}") - if (scheduler_first_start and db_plugins) or changes: - generate_external_plugins(SCHEDULER.db.get_plugins(_type=_type, with_data=True), original_path=plugin_path) + generate_external_plugins(SCHEDULER.db.get_plugins(_type=_type, with_data=True), original_path=plugin_path) - check_plugin_changes("external") - check_plugin_changes("pro") + threads.extend([Thread(target=check_plugin_changes, args=("external",)), Thread(target=check_plugin_changes, args=("pro",))]) + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() logger.info("Running plugins download jobs ...") @@ -541,10 +552,18 @@ if __name__ == "__main__": changes = SCHEDULER.db.check_changes() if INTEGRATION not in ("Swarm", "Kubernetes", "Autoconf") and (changes["pro_plugins_changed"] or changes["external_plugins_changed"]): + threads.clear() + if changes["pro_plugins_changed"]: - generate_external_plugins(SCHEDULER.db.get_plugins(_type="pro", with_data=True), original_path=PRO_PLUGINS_PATH) + threads.append(Thread(target=generate_external_plugins, args=(None,), kwargs={"original_path": PRO_PLUGINS_PATH})) if changes["external_plugins_changed"]: - generate_external_plugins(SCHEDULER.db.get_plugins(_type="external", with_data=True)) + threads.append(Thread(target=generate_external_plugins, args=(None,))) + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() # run the config saver to save potential ignored external plugins settings logger.info("Running config saver to save potential ignored external plugins settings ...") @@ -575,7 +594,6 @@ if __name__ == "__main__": CONFIG_NEED_GENERATION = True RUN_JOBS_ONCE = True CHANGES = [] - threads = [] def send_nginx_configs(): logger.info(f"Sending {join(sep, 'etc', 'nginx')} folder ...") From 67a08031e1144edc71af90e6f41f4f9b46c49df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Fri, 24 May 2024 11:33:37 +0100 Subject: [PATCH 13/42] chore: Add possibility to override admin credentials from environment variables in web UI This commit adds the ability to override the admin credentials from environment variables in the web UI. By setting the `OVERRIDE_ADMIN_CREDS` variable to `yes`, the admin credentials can be changed even if they are already set. The `ADMIN_USERNAME` and `ADMIN_PASSWORD` variables can be used to specify the new username and password. The web UI will authenticate users using these variables. --- docs/web-ui.md | 10 ++++++++++ src/ui/gunicorn.conf.py | 16 ++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/web-ui.md b/docs/web-ui.md index 98ccff31a..508809a45 100644 --- a/docs/web-ui.md +++ b/docs/web-ui.md @@ -752,6 +752,16 @@ When your BunkerWeb instance has upgraded to the PRO version, you will see your ### Username / Password +!!! tip "Overriding admin credentials from environment variables" + + If you want to override the admin credentials from environment variables, you can set the following variables : + + - `OVERRIDE_ADMIN_CREDS` : set it to `yes` to enable the override even if the admin credentials are already set (default is `no`) + - `ADMIN_USERNAME` : username to access the web UI + - `ADMIN_PASSWORD` : password to access the web UI + + The web UI will use these variables to authenticate you. + !!! warning "Lost password/username" In case you forgot your UI credentials, you can reset them from the CLI following [the steps described in the troubleshooting section](troubleshooting.md#web-ui). diff --git a/src/ui/gunicorn.conf.py b/src/ui/gunicorn.conf.py index e4cd0c779..787d111a0 100644 --- a/src/ui/gunicorn.conf.py +++ b/src/ui/gunicorn.conf.py @@ -78,17 +78,25 @@ def on_starting(server): USER = User(**USER) if getenv("ADMIN_USERNAME") or getenv("ADMIN_PASSWORD"): - if USER.method == "manual": + override_admin_creds = getenv("OVERRIDE_ADMIN_CREDS", "no").lower() == "yes" + if USER.method == "manual" or override_admin_creds: updated = False if getenv("ADMIN_USERNAME", "") and USER.get_id() != getenv("ADMIN_USERNAME", ""): USER.id = getenv("ADMIN_USERNAME", "") updated = True if getenv("ADMIN_PASSWORD", "") and not USER.check_password(getenv("ADMIN_PASSWORD", "")): - USER.update_password(getenv("ADMIN_PASSWORD", "")) - updated = True + if not USER_PASSWORD_RX.match(getenv("ADMIN_PASSWORD", "")): + LOGGER.warning( + "The admin password is not strong enough. It must contain at least 8 characters, including at least 1 uppercase letter, 1 lowercase letter, 1 number and 1 special character (#@?!$%^&*-). It will not be updated." + ) + else: + USER.update_password(getenv("ADMIN_PASSWORD", "")) + updated = True if updated: - ret = db.update_ui_user(USER.get_id(), USER.password_hash, USER.is_two_factor_enabled, USER.secret_token) + if override_admin_creds: + LOGGER.warning("Overriding the admin user credentials, as the OVERRIDE_ADMIN_CREDS environment variable is set to 'yes'.") + ret = db.update_ui_user(USER.get_id(), USER.password_hash, USER.is_two_factor_enabled, USER.secret_token, method="manual") if ret: LOGGER.error(f"Couldn't update the admin user in the database: {ret}") exit(1) From 798c1b79d52bc32c16074cdab228faccb8aa306d Mon Sep 17 00:00:00 2001 From: fl0ppy-d1sk Date: Fri, 24 May 2024 12:34:58 +0200 Subject: [PATCH 14/42] ui - redirect setup wizard to https and configure fallback self-signed cert if LE is not used --- .../core/ui/confs/default-server-http/ui.conf | 9 +++++++++ src/ui/main.py | 1 + src/ui/templates/setup.html | 14 +++++--------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/common/core/ui/confs/default-server-http/ui.conf b/src/common/core/ui/confs/default-server-http/ui.conf index 67083e72f..99336022d 100644 --- a/src/common/core/ui/confs/default-server-http/ui.conf +++ b/src/common/core/ui/confs/default-server-http/ui.conf @@ -1,4 +1,13 @@ {% if UI_HOST != "" and not has_variable(all, "USE_UI", "yes") +%} +access_by_lua_block { + local ngx_var = ngx.var + local scheme = ngx_var.scheme + local http_host = ngx_var.http_host + local request_uri = ngx_var.request_uri + if scheme == "http" and http_host ~= nil and http_host ~= "" and request_uri and request_uri ~= "" then + return ngx.redirect("https://" .. http_host .. request_uri, ngx.HTTP_MOVED_PERMANENTLY) + end +} location /setup { etag off; add_header Last-Modified ""; diff --git a/src/ui/main.py b/src/ui/main.py index c2cb44286..77054091a 100755 --- a/src/ui/main.py +++ b/src/ui/main.py @@ -576,6 +576,7 @@ def setup(): "REVERSE_PROXY_HOST": request.form["ui_host"], "REVERSE_PROXY_URL": request.form["ui_url"] or "/", "AUTO_LETS_ENCRYPT": request.form.get("auto_lets_encrypt", "no"), + "GENERATE_SELF_SIGNED_SSL": "yes" if request.form.get("auto_lets_encrypt", "no") == "no" else "no", "INTERCEPTED_ERROR_CODES": "400 404 405 413 429 500 501 502 503 504", "MAX_CLIENT_SIZE": "50m", }, diff --git a/src/ui/templates/setup.html b/src/ui/templates/setup.html index 664cbce1a..cd535277f 100644 --- a/src/ui/templates/setup.html +++ b/src/ui/templates/setup.html @@ -320,7 +320,7 @@ Your BunkerWeb UI final URL will be

http://

+ data-resume>https://