From 7a8a75901f7b0a86b165af4f92386ce03ec03cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Wed, 22 Feb 2023 10:13:34 +0100 Subject: [PATCH 1/2] Fix multiple CVEs (see comment) (finally) --- .github/workflows/dev.yml | 8 ++++---- .trivyignore | 4 ---- src/autoconf/Dockerfile | 2 +- src/scheduler/Dockerfile | 2 +- src/ui/Dockerfile | 2 +- 5 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 .trivyignore diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index e9f65d7b3..73aa2f302 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -47,7 +47,7 @@ jobs: exit-code: 1 ignore-unfixed: false severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - trivyignores: .trivyignore + # trivyignores: .trivyignore # BW scheduler tests scheduler: @@ -91,7 +91,7 @@ jobs: exit-code: 1 ignore-unfixed: false severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - trivyignores: .trivyignore + # trivyignores: .trivyignore # BW autoconf tests autoconf: @@ -135,7 +135,7 @@ jobs: exit-code: 1 ignore-unfixed: false severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - trivyignores: .trivyignore + # trivyignores: .trivyignore # BW UI tests ui: @@ -180,7 +180,7 @@ jobs: exit-code: 1 ignore-unfixed: false severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL - trivyignores: .trivyignore + # trivyignores: .trivyignore # Python code security code-security: diff --git a/.trivyignore b/.trivyignore deleted file mode 100644 index 4be368d62..000000000 --- a/.trivyignore +++ /dev/null @@ -1,4 +0,0 @@ -# libcurl 7.87.0-r2 and curl 7.87.0-r2 are not yet available in python:3.11-alpine -CVE-2023-23916 -CVE-2023-23914 -CVE-2023-23915 \ No newline at end of file diff --git a/src/autoconf/Dockerfile b/src/autoconf/Dockerfile index dd3a139cb..1d0db2727 100644 --- a/src/autoconf/Dockerfile +++ b/src/autoconf/Dockerfile @@ -54,7 +54,7 @@ RUN apk add --no-cache bash && \ chmod 770 /var/log/letsencrypt /var/lib/letsencrypt # Fix CVEs -RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0" "git>=2.36.5-r0" +RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0" "git>=2.36.5-r0" "curl>=7.87.0-r2" "libcurl>=7.87.0-r2" VOLUME /data /etc/nginx diff --git a/src/scheduler/Dockerfile b/src/scheduler/Dockerfile index ba7e7852c..d8b375bab 100644 --- a/src/scheduler/Dockerfile +++ b/src/scheduler/Dockerfile @@ -64,7 +64,7 @@ RUN apk add --no-cache bash libgcc libstdc++ openssl && \ chmod 660 /usr/share/bunkerweb/INTEGRATION # Fix CVEs -RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0" "git>=2.36.5-r0" +RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0" "git>=2.36.5-r0" "curl>=7.87.0-r2" "libcurl>=7.87.0-r2" VOLUME /data /etc/nginx diff --git a/src/ui/Dockerfile b/src/ui/Dockerfile index d7bdbae61..66d995109 100755 --- a/src/ui/Dockerfile +++ b/src/ui/Dockerfile @@ -50,7 +50,7 @@ RUN apk add --no-cache bash && \ chmod 660 /usr/share/bunkerweb/INTEGRATION # Fix CVEs -RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0" "git>=2.36.5-r0" +RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0" "git>=2.36.5-r0" "curl>=7.87.0-r2" "libcurl>=7.87.0-r2" VOLUME /data /etc/nginx From 6bf59b59a25691c6c6dbafc4ffb3051bc1519f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Wed, 22 Feb 2023 10:46:14 +0100 Subject: [PATCH 2/2] Refactor the plugins jobs --- .../core/blacklist/jobs/blacklist-download.py | 14 +++---- .../core/bunkernet/jobs/bunkernet-data.py | 20 ++++++--- .../core/bunkernet/jobs/bunkernet-register.py | 42 ++++++++++++++----- src/common/core/bunkernet/jobs/bunkernet.py | 12 +++--- .../core/customcert/jobs/custom-cert.py | 27 +++++++----- .../core/greylist/jobs/greylist-download.py | 14 +++---- src/common/core/jobs/jobs/download-plugins.py | 5 ++- .../core/letsencrypt/jobs/certbot-auth.py | 3 +- .../core/letsencrypt/jobs/certbot-cleanup.py | 9 ++-- .../core/letsencrypt/jobs/certbot-deploy.py | 5 ++- .../core/letsencrypt/jobs/certbot-new.py | 13 +++--- .../core/letsencrypt/jobs/certbot-renew.py | 15 ++++--- .../core/misc/jobs/default-server-cert.py | 13 +++--- .../core/realip/jobs/realip-download.py | 13 ++++-- .../core/selfsigned/jobs/self-signed.py | 22 +++++----- .../core/whitelist/jobs/whitelist-download.py | 14 +++---- 16 files changed, 146 insertions(+), 95 deletions(-) diff --git a/src/common/core/blacklist/jobs/blacklist-download.py b/src/common/core/blacklist/jobs/blacklist-download.py index fdaf6ced2..a8d30b645 100755 --- a/src/common/core/blacklist/jobs/blacklist-download.py +++ b/src/common/core/blacklist/jobs/blacklist-download.py @@ -2,7 +2,7 @@ from contextlib import suppress from ipaddress import ip_address, ip_network -from os import _exit, getenv, makedirs +from os import _exit, getenv from pathlib import Path from re import IGNORECASE, compile as re_compile from sys import exit as sys_exit, path as sys_path @@ -80,13 +80,13 @@ try: elif getenv("USE_BLACKLIST", "yes") == "yes": blacklist_activated = True - if blacklist_activated is False: + if not blacklist_activated: logger.info("Blacklist is not activated, skipping downloads...") _exit(0) # Create directories if they don't exist - makedirs("/var/cache/bunkerweb/blacklist", exist_ok=True) - makedirs("/var/tmp/bunkerweb/blacklist", exist_ok=True) + Path("/var/cache/bunkerweb/blacklist").mkdir(parents=True, exist_ok=True) + Path("/var/tmp/bunkerweb/blacklist").mkdir(parents=True, exist_ok=True) # Our urls data urls = {"IP": [], "RDNS": [], "ASN": [], "USER_AGENT": [], "URI": []} @@ -116,7 +116,7 @@ try: logger.info( f"Blacklist for {kind} is already in cache, skipping downloads...", ) - if all_fresh is True: + if all_fresh: _exit(0) # Get URLs @@ -139,7 +139,7 @@ try: # Loop on kinds for kind, urls_list in urls.items(): - if kinds_fresh[kind] is True: + if kinds_fresh[kind]: continue # Write combined data of the kind to a single temp file for url in urls_list: @@ -161,7 +161,7 @@ try: line = line.split(b" ")[0] ok, data = check_line(kind, line) - if ok is True: + if ok: content += data + b"\n" i += 1 diff --git a/src/common/core/bunkernet/jobs/bunkernet-data.py b/src/common/core/bunkernet/jobs/bunkernet-data.py index 80a4aa3f2..22286aae2 100755 --- a/src/common/core/bunkernet/jobs/bunkernet-data.py +++ b/src/common/core/bunkernet/jobs/bunkernet-data.py @@ -1,7 +1,6 @@ #!/usr/bin/python3 -from os import _exit, getenv, makedirs -from os.path import isfile +from os import _exit, getenv from pathlib import Path from sys import exit as sys_exit, path as sys_path from traceback import format_exc @@ -44,15 +43,15 @@ try: elif getenv("USE_BUNKERNET", "yes") == "yes": bunkernet_activated = True - if bunkernet_activated is False: + if not bunkernet_activated: logger.info("BunkerNet is not activated, skipping download...") _exit(0) # Create directory if it doesn't exist - makedirs("/var/cache/bunkerweb/bunkernet", exist_ok=True) + Path("/var/cache/bunkerweb/bunkernet").mkdir(parents=True, exist_ok=True) # Check if ID is present - if not isfile("/var/cache/bunkerweb/bunkernet/instance.id"): + if not Path("/var/cache/bunkerweb/bunkernet/instance.id").is_file(): logger.error( "Not downloading BunkerNet data because instance is not registered", ) @@ -83,7 +82,16 @@ try: "BunkerNet has banned this instance, retrying a register later...", ) _exit(0) - elif data["result"] != "ok": + + try: + assert isinstance(data, dict) + except AssertionError: + logger.error( + f"Received invalid data from BunkerNet API while sending db request : {data}", + ) + _exit(1) + + if data["result"] != "ok": logger.error( f"Received error from BunkerNet API while sending db request : {data['data']}, removing instance ID", ) diff --git a/src/common/core/bunkernet/jobs/bunkernet-register.py b/src/common/core/bunkernet/jobs/bunkernet-register.py index 741068133..e8dd72199 100755 --- a/src/common/core/bunkernet/jobs/bunkernet-register.py +++ b/src/common/core/bunkernet/jobs/bunkernet-register.py @@ -1,7 +1,6 @@ #!/usr/bin/python3 -from os import _exit, getenv, makedirs, remove -from os.path import isfile +from os import _exit, getenv from pathlib import Path from sys import exit as sys_exit, path as sys_path from time import sleep @@ -33,7 +32,12 @@ try: bunkernet_activated = False # Multisite case if getenv("MULTISITE", "no") == "yes": - for first_server in getenv("SERVER_NAME").split(" "): + servers = getenv("SERVER_NAME", []) + + if isinstance(servers, str): + servers = servers.split(" ") + + for first_server in servers: if ( getenv(f"{first_server}_USE_BUNKERNET", getenv("USE_BUNKERNET", "yes")) == "yes" @@ -44,16 +48,16 @@ try: elif getenv("USE_BUNKERNET", "yes") == "yes": bunkernet_activated = True - if bunkernet_activated is False: + if not bunkernet_activated: logger.info("BunkerNet is not activated, skipping registration...") _exit(0) # Create directory if it doesn't exist - makedirs("/var/cache/bunkerweb/bunkernet", exist_ok=True) + Path("/var/cache/bunkerweb/bunkernet").mkdir(parents=True, exist_ok=True) # Ask an ID if needed bunkernet_id = None - if not isfile("/var/cache/bunkerweb/bunkernet/instance.id"): + if not Path("/var/cache/bunkerweb/bunkernet/instance.id").is_file(): logger.info("Registering instance on BunkerNet API ...") ok, status, data = register() if not ok: @@ -71,7 +75,16 @@ try: "BunkerNet has banned this instance, retrying a register later...", ) _exit(0) - elif status != 200: + + try: + assert isinstance(data, dict) + except AssertionError: + logger.error( + f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...", + ) + _exit(1) + + if status != 200: logger.error( f"Error {status} from BunkerNet API : {data['data']}", ) @@ -111,9 +124,18 @@ try: logger.warning( "Instance ID is not registered, removing it and retrying a register later...", ) - remove("/var/cache/bunkerweb/bunkernet/instance.id") + Path("/var/cache/bunkerweb/bunkernet/instance.id").unlink() _exit(2) - elif data.get("result", "ko") != "ok": + + try: + assert isinstance(data, dict) + except AssertionError: + logger.error( + f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...", + ) + _exit(1) + + if data.get("result", "ko") != "ok": logger.error( f"Received error from BunkerNet API while sending ping request : {data.get('data', {})}, removing instance ID", ) @@ -127,7 +149,7 @@ try: if bunkernet_ping and status != 403: logger.info("Connectivity with BunkerWeb is successful !") status = 1 - if not isfile("/var/cache/bunkerweb/bunkernet/instance.id"): + if not Path("/var/cache/bunkerweb/bunkernet/instance.id").is_file(): Path("/var/cache/bunkerweb/bunkernet/instance.id").write_text(bunkernet_id) # Update db diff --git a/src/common/core/bunkernet/jobs/bunkernet.py b/src/common/core/bunkernet/jobs/bunkernet.py index c8dc908d3..598dc14ab 100644 --- a/src/common/core/bunkernet/jobs/bunkernet.py +++ b/src/common/core/bunkernet/jobs/bunkernet.py @@ -1,7 +1,7 @@ -from typing import Literal, Optional, Tuple, Union -import requests from os import getenv -from os.path import exists +from pathlib import Path +from requests import request as requests_request, ReadTimeout +from typing import Literal, Optional, Tuple, Union def request( @@ -12,7 +12,7 @@ def request( if _id is not None: data["id"] = _id try: - resp = requests.request( + resp = requests_request( method, f"{getenv('BUNKERNET_SERVER', 'https://api.bunkerweb.io')}{url}", json=data, @@ -29,7 +29,7 @@ def request( assert "result" in raw_data assert "data" in raw_data - except requests.ReadTimeout: + except ReadTimeout: return False, None, "request timed out" except Exception as e: return False, None, f"request failed: {e}" @@ -66,7 +66,7 @@ def get_integration(): return "swarm" elif getenv("KUBERNETES_MODE") == "yes": return "kubernetes" - elif exists("/usr/share/bunkerweb/INTEGRATION"): + elif Path("/usr/share/bunkerweb/INTEGRATION").exists(): with open("/usr/share/bunkerweb/INTEGRATION", "r") as f: return f.read().strip().lower() diff --git a/src/common/core/customcert/jobs/custom-cert.py b/src/common/core/customcert/jobs/custom-cert.py index 8b0239c42..ce33df9bc 100644 --- a/src/common/core/customcert/jobs/custom-cert.py +++ b/src/common/core/customcert/jobs/custom-cert.py @@ -1,11 +1,11 @@ #!/usr/bin/python3 -from os import getenv, makedirs, remove -from os.path import isfile +from os import getenv, makedirs from pathlib import Path from shutil import copy from sys import exit as sys_exit, path as sys_path from traceback import format_exc +from typing import Optional sys_path.extend( ( @@ -26,19 +26,19 @@ db = Database( ) -def check_cert(cert_path, key_path, first_server: str = None) -> bool: +def check_cert(cert_path, key_path, first_server: Optional[str] = None) -> bool: try: if not cert_path or not key_path: logger.warning( "Both variables CUSTOM_HTTPS_CERT and CUSTOM_HTTPS_KEY have to be set to use custom certificates" ) return False - elif not isfile(cert_path): + elif not Path(cert_path).is_file(): logger.warning( f"Certificate file {cert_path} is not a valid file, ignoring the custom certificate" ) return False - elif not isfile(key_path): + elif not Path(key_path).is_file(): logger.warning( f"Key file {key_path} is not a valid file, ignoring the custom certificate" ) @@ -49,7 +49,7 @@ def check_cert(cert_path, key_path, first_server: str = None) -> bool: ) cert_hash = file_hash(cert_path) - if not isfile(cert_cache_path): + if not Path(cert_cache_path).is_file(): Path(cert_cache_path).write_text(cert_hash) old_hash = file_hash(cert_cache_path) @@ -59,12 +59,12 @@ def check_cert(cert_path, key_path, first_server: str = None) -> bool: Path(cert_cache_path).write_text(cert_hash) copy(cert_path, cert_cache_path.replace(".hash", "")) - if not isfile(key_path): + if not Path(key_path).is_file(): logger.warning( f"Key file {key_path} is not a valid file, removing the custom certificate ..." ) - remove(cert_path) - remove(cert_cache_path) + Path(cert_path).unlink() + Path(cert_cache_path).unlink() return False key_cache_path = ( @@ -72,7 +72,7 @@ def check_cert(cert_path, key_path, first_server: str = None) -> bool: ) key_hash = file_hash(key_path) - if not isfile(key_cache_path): + if not Path(key_cache_path).is_file(): Path(key_cache_path).write_text(key_hash) old_hash = file_hash(key_cache_path) @@ -122,7 +122,12 @@ try: # Multisite case if getenv("MULTISITE") == "yes": - for first_server in getenv("SERVER_NAME").split(" "): + servers = getenv("SERVER_NAME", []) + + if isinstance(servers, str): + servers = servers.split(" ") + + for first_server in servers: if not first_server or ( getenv( f"{first_server}_USE_CUSTOM_HTTPS", getenv("USE_CUSTOM_HTTPS", "no") diff --git a/src/common/core/greylist/jobs/greylist-download.py b/src/common/core/greylist/jobs/greylist-download.py index 8c10de358..54d627dc5 100755 --- a/src/common/core/greylist/jobs/greylist-download.py +++ b/src/common/core/greylist/jobs/greylist-download.py @@ -2,7 +2,7 @@ from contextlib import suppress from ipaddress import ip_address, ip_network -from os import _exit, getenv, makedirs +from os import _exit, getenv from pathlib import Path from re import IGNORECASE, compile as re_compile from sys import exit as sys_exit, path as sys_path @@ -80,13 +80,13 @@ try: elif getenv("USE_GREYLIST", "no") == "yes": greylist_activated = True - if greylist_activated is False: + if not greylist_activated: logger.info("Greylist is not activated, skipping downloads...") _exit(0) # Create directories if they don't exist - makedirs("/var/cache/bunkerweb/greylist", exist_ok=True) - makedirs("/var/tmp/bunkerweb/greylist", exist_ok=True) + Path("/var/cache/bunkerweb/greylist").mkdir(parents=True, exist_ok=True) + Path("/var/tmp/bunkerweb/greylist").mkdir(parents=True, exist_ok=True) # Our urls data urls = {"IP": [], "RDNS": [], "ASN": [], "USER_AGENT": [], "URI": []} @@ -111,7 +111,7 @@ try: logger.info( f"Greylist for {kind} is already in cache, skipping downloads...", ) - if all_fresh is True: + if all_fresh: _exit(0) # Get URLs @@ -123,7 +123,7 @@ try: # Loop on kinds for kind, urls_list in urls.items(): - if kinds_fresh[kind] is True: + if kinds_fresh[kind]: continue # Write combined data of the kind to a single temp file for url in urls_list: @@ -145,7 +145,7 @@ try: line = line.split(b" ")[0] ok, data = check_line(kind, line) - if ok is True: + if ok: content += data + b"\n" i += 1 diff --git a/src/common/core/jobs/jobs/download-plugins.py b/src/common/core/jobs/jobs/download-plugins.py index e15df4f59..54266b0e7 100644 --- a/src/common/core/jobs/jobs/download-plugins.py +++ b/src/common/core/jobs/jobs/download-plugins.py @@ -2,7 +2,8 @@ from io import BytesIO from os import getenv, listdir, makedirs, chmod, stat, _exit, walk -from os.path import join, isfile, dirname +from os.path import dirname, join +from pathlib import Path from stat import S_IEXEC from sys import exit as sys_exit, path as sys_path from uuid import uuid4 @@ -39,7 +40,7 @@ def install_plugin(plugin_dir): with open(f"{plugin_dir}plugin.json", "rb") as f: metadata = loads(f.read()) # Don't go further if plugin is already installed - if isfile(f"/data/plugins/{metadata['id']}/plugin.json"): + if Path(f"/data/plugins/{metadata['id']}/plugin.json").is_file(): logger.info( f"Skipping installation of plugin {metadata['id']} (already installed)", ) diff --git a/src/common/core/letsencrypt/jobs/certbot-auth.py b/src/common/core/letsencrypt/jobs/certbot-auth.py index 59be475ba..02065121d 100755 --- a/src/common/core/letsencrypt/jobs/certbot-auth.py +++ b/src/common/core/letsencrypt/jobs/certbot-auth.py @@ -1,7 +1,6 @@ #!/usr/bin/python3 from os import getenv, makedirs -from os.path import exists from pathlib import Path from sys import exit as sys_exit, path as sys_path from traceback import format_exc @@ -35,7 +34,7 @@ try: bw_integration = "Kubernetes" elif getenv("AUTOCONF_MODE") == "yes": bw_integration = "Autoconf" - elif exists("/usr/share/bunkerweb/INTEGRATION"): + elif Path("/usr/share/bunkerweb/INTEGRATION").exists(): with open("/usr/share/bunkerweb/INTEGRATION", "r") as f: bw_integration = f.read().strip() token = getenv("CERTBOT_TOKEN", "") diff --git a/src/common/core/letsencrypt/jobs/certbot-cleanup.py b/src/common/core/letsencrypt/jobs/certbot-cleanup.py index 152d3e489..10c5d099c 100755 --- a/src/common/core/letsencrypt/jobs/certbot-cleanup.py +++ b/src/common/core/letsencrypt/jobs/certbot-cleanup.py @@ -1,7 +1,8 @@ #!/usr/bin/python3 -from os import getenv, remove -from os.path import exists, isfile +from os import getenv +from os.path import isfile +from pathlib import Path from sys import exit as sys_exit, path as sys_path from traceback import format_exc @@ -34,7 +35,7 @@ try: bw_integration = "Kubernetes" elif getenv("AUTOCONF_MODE") == "yes": bw_integration = "Autoconf" - elif exists("/usr/share/bunkerweb/INTEGRATION"): + elif Path("/usr/share/bunkerweb/INTEGRATION").exists(): with open("/usr/share/bunkerweb/INTEGRATION", "r") as f: bw_integration = f.read().strip() token = getenv("CERTBOT_TOKEN", "") @@ -69,7 +70,7 @@ try: f"/var/tmp/bunkerweb/lets-encrypt/.well-known/acme-challenge/{token}" ) if isfile(challenge_path): - remove(challenge_path) + Path(challenge_path).unlink() except: status = 1 logger.error(f"Exception while running certbot-cleanup.py :\n{format_exc()}") diff --git a/src/common/core/letsencrypt/jobs/certbot-deploy.py b/src/common/core/letsencrypt/jobs/certbot-deploy.py index 268a9fd32..f90319b04 100755 --- a/src/common/core/letsencrypt/jobs/certbot-deploy.py +++ b/src/common/core/letsencrypt/jobs/certbot-deploy.py @@ -2,7 +2,8 @@ from io import BytesIO from os import chmod, getenv, walk -from os.path import exists, join +from os.path import join +from pathlib import Path from shutil import chown from subprocess import run, DEVNULL, STDOUT from sys import exit as sys_exit, path as sys_path @@ -38,7 +39,7 @@ try: bw_integration = "Kubernetes" elif getenv("AUTOCONF_MODE") == "yes": bw_integration = "Autoconf" - elif exists("/usr/share/bunkerweb/INTEGRATION"): + elif Path("/usr/share/bunkerweb/INTEGRATION").exists(): with open("/usr/share/bunkerweb/INTEGRATION", "r") as f: bw_integration = f.read().strip() token = getenv("CERTBOT_TOKEN", "") diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index 6e818d6fc..7878d86b3 100755 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -1,7 +1,6 @@ #!/usr/bin/python3 -from os import environ, getcwd, getenv -from os.path import exists +from os import environ, getenv from pathlib import Path from subprocess import DEVNULL, STDOUT, run from sys import exit as sys_exit, path as sys_path @@ -71,7 +70,7 @@ try: " ", "," ) - if exists(f"/etc/letsencrypt/live/{first_server}/cert.pem"): + if Path(f"/etc/letsencrypt/live/{first_server}/cert.pem").exists(): logger.info( f"Certificates already exists for domain(s) {domains}", ) @@ -97,7 +96,7 @@ try: f"Certificate generation succeeded for domain(s) : {domains}" ) - if exists(f"/etc/letsencrypt/live/{first_server}/cert.pem"): + if Path(f"/etc/letsencrypt/live/{first_server}/cert.pem").exists(): cert = Path( f"/etc/letsencrypt/live/{first_server}/cert.pem" ).read_bytes() @@ -113,11 +112,11 @@ try: logger.warning(f"Couldn't update db cache: {err}") # Singlesite case - elif getenv("AUTO_LETS_ENCRYPT", "no") == "yes" and getenv("SERVER_NAME", ""): + elif getenv("AUTO_LETS_ENCRYPT", "no") == "yes" and getenv("SERVER_NAME"): first_server = getenv("SERVER_NAME", "").split(" ")[0] domains = getenv("SERVER_NAME", "").replace(" ", ",") - if exists(f"/etc/letsencrypt/live/{first_server}/cert.pem"): + if Path(f"/etc/letsencrypt/live/{first_server}/cert.pem").exists(): logger.info(f"Certificates already exists for domain(s) {domains}") else: real_email = getenv("EMAIL_LETS_ENCRYPT", f"contact@{first_server}") @@ -135,7 +134,7 @@ try: f"Certificate generation succeeded for domain(s) : {domains}" ) - if exists(f"/etc/letsencrypt/live/{first_server}/cert.pem"): + if Path(f"/etc/letsencrypt/live/{first_server}/cert.pem").exists(): cert = Path( f"/etc/letsencrypt/live/{first_server}/cert.pem" ).read_bytes() diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index b8ad40bb0..5d046dedb 100755 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -from os import environ, getcwd, getenv -from os.path import exists +from os import environ, getenv +from pathlib import Path from subprocess import DEVNULL, STDOUT, run from sys import exit as sys_exit, path as sys_path from traceback import format_exc @@ -39,7 +39,12 @@ status = 0 try: if getenv("MULTISITE") == "yes": - for first_server in getenv("SERVER_NAME").split(" "): + servers = getenv("SERVER_NAME", []) + + if isinstance(servers, str): + servers = servers.split(" ") + + for first_server in servers: if ( not first_server or getenv( @@ -47,7 +52,7 @@ try: getenv("AUTO_LETS_ENCRYPT", "no"), ) != "yes" - or not exists(f"/etc/letsencrypt/live/{first_server}/cert.pem") + or not Path(f"/etc/letsencrypt/live/{first_server}/cert.pem").exists() ): continue @@ -59,7 +64,7 @@ try: ) elif getenv("AUTO_LETS_ENCRYPT", "no") == "yes" and not getenv("SERVER_NAME", ""): first_server = getenv("SERVER_NAME", "").split(" ")[0] - if exists(f"/etc/letsencrypt/live/{first_server}/cert.pem"): + if Path(f"/etc/letsencrypt/live/{first_server}/cert.pem").exists(): ret = renew(first_server) if ret != 0: status = 2 diff --git a/src/common/core/misc/jobs/default-server-cert.py b/src/common/core/misc/jobs/default-server-cert.py index abc4c51ce..f26ee3109 100644 --- a/src/common/core/misc/jobs/default-server-cert.py +++ b/src/common/core/misc/jobs/default-server-cert.py @@ -24,11 +24,11 @@ try: need_default_cert = False if getenv("MULTISITE", "no") == "yes": for first_server in getenv("SERVER_NAME", "").split(" "): - for check_var in [ + for check_var in ( "USE_CUSTOM_HTTPS", "AUTO_LETS_ENCRYPT", "GENERATE_SELF_SIGNED_SSL", - ]: + ): if ( getenv(f"{first_server}_{check_var}", getenv(check_var, "no")) == "yes" @@ -38,9 +38,12 @@ try: if need_default_cert: break elif getenv("DISABLE_DEFAULT_SERVER", "no") == "yes" and ( - getenv("USE_CUSTOM_HTTPS", "no") == "yes" - or getenv("AUTO_LETS_ENCRYPT", "no") == "yes" - or getenv("GENERATE_SELF_SIGNED_SSL", "no") == "yes" + "yes" + in ( + getenv("USE_CUSTOM_HTTPS", "no"), + getenv("AUTO_LETS_ENCRYPT", "no"), + getenv("GENERATE_SELF_SIGNED_SSL", "no"), + ) ): need_default_cert = True diff --git a/src/common/core/realip/jobs/realip-download.py b/src/common/core/realip/jobs/realip-download.py index 856262c9d..d2d885c3f 100755 --- a/src/common/core/realip/jobs/realip-download.py +++ b/src/common/core/realip/jobs/realip-download.py @@ -2,7 +2,7 @@ from contextlib import suppress from ipaddress import ip_address, ip_network -from os import _exit, getenv, makedirs +from os import _exit, getenv from pathlib import Path from sys import exit as sys_exit, path as sys_path from traceback import format_exc @@ -46,7 +46,12 @@ try: realip_activated = False # Multisite case if getenv("MULTISITE", "no") == "yes": - for first_server in getenv("SERVER_NAME").split(" "): + servers = getenv("SERVER_NAME", []) + + if isinstance(servers, str): + servers = servers.split(" ") + + for first_server in servers: if ( getenv(f"{first_server}_USE_REAL_IP", getenv("USE_REAL_IP", "no")) == "yes" @@ -58,12 +63,12 @@ try: elif getenv("USE_REAL_IP", "no") == "yes": realip_activated = True - if realip_activated is False: + if not realip_activated: logger.info("RealIP is not activated, skipping download...") _exit(0) # Create directory if it doesn't exist - makedirs("/var/cache/bunkerweb/realip", exist_ok=True) + Path("/var/cache/bunkerweb/realip").mkdir(parents=True, exist_ok=True) # Don't go further if the cache is fresh if is_cached_file("/var/cache/bunkerweb/realip/combined.list", "hour"): diff --git a/src/common/core/selfsigned/jobs/self-signed.py b/src/common/core/selfsigned/jobs/self-signed.py index 3cc9368f2..38b9f239d 100755 --- a/src/common/core/selfsigned/jobs/self-signed.py +++ b/src/common/core/selfsigned/jobs/self-signed.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -from os import getenv, makedirs -from os.path import isfile +from os import getenv +from pathlib import Path from subprocess import DEVNULL, STDOUT, run from sys import exit as sys_exit, path as sys_path from traceback import format_exc @@ -25,7 +25,7 @@ db = Database( def generate_cert(first_server, days, subj): - if isfile(f"/var/cache/bunkerweb/selfsigned/{first_server}.pem"): + if Path(f"/var/cache/bunkerweb/selfsigned/{first_server}.pem").is_file(): cmd = f"openssl x509 -checkend 86400 -noout -in /var/cache/bunkerweb/selfsigned/{first_server}.pem" proc = run(cmd.split(" "), stdin=DEVNULL, stderr=STDOUT) if proc.returncode == 0: @@ -67,12 +67,16 @@ def generate_cert(first_server, days, subj): status = 0 try: - - makedirs("/var/cache/bunkerweb/selfsigned/", exist_ok=True) + Path("/var/cache/bunkerweb/selfsigned/").mkdir(parents=True, exist_ok=True) # Multisite case if getenv("MULTISITE") == "yes": - for first_server in getenv("SERVER_NAME").split(" "): + servers = getenv("SERVER_NAME", []) + + if isinstance(servers, str): + servers = servers.split(" ") + + for first_server in servers: if ( not first_server or getenv( @@ -80,7 +84,7 @@ try: getenv("GENERATE_SELF_SIGNED_SSL", "no"), ) != "yes" - or isfile(f"/var/cache/bunkerweb/selfsigned/{first_server}.pem") + or Path(f"/var/cache/bunkerweb/selfsigned/{first_server}.pem").is_file() ): continue @@ -101,9 +105,7 @@ try: status = 1 # Singlesite case - elif getenv("GENERATE_SELF_SIGNED_SSL", "no") == "yes" and getenv( - "SERVER_NAME", "" - ): + elif getenv("GENERATE_SELF_SIGNED_SSL", "no") == "yes" and getenv("SERVER_NAME"): first_server = getenv("SERVER_NAME", "").split(" ")[0] ret, ret_status = generate_cert( first_server, diff --git a/src/common/core/whitelist/jobs/whitelist-download.py b/src/common/core/whitelist/jobs/whitelist-download.py index 4a8d6dec8..134943514 100755 --- a/src/common/core/whitelist/jobs/whitelist-download.py +++ b/src/common/core/whitelist/jobs/whitelist-download.py @@ -2,7 +2,7 @@ from contextlib import suppress from ipaddress import ip_address, ip_network -from os import _exit, getenv, makedirs +from os import _exit, getenv from pathlib import Path from re import IGNORECASE, compile as re_compile from sys import exit as sys_exit, path as sys_path @@ -80,13 +80,13 @@ try: elif getenv("USE_WHITELIST", "no") == "yes": whitelist_activated = True - if whitelist_activated is False: + if not whitelist_activated: logger.info("Whitelist is not activated, skipping downloads...") _exit(0) # Create directories if they don't exist - makedirs("/var/cache/bunkerweb/whitelist", exist_ok=True) - makedirs("/var/tmp/bunkerweb/whitelist", exist_ok=True) + Path("/var/cache/bunkerweb/whitelist").mkdir(parents=True, exist_ok=True) + Path("/var/tmp/bunkerweb/whitelist").mkdir(parents=True, exist_ok=True) # Our urls data urls = {"IP": [], "RDNS": [], "ASN": [], "USER_AGENT": [], "URI": []} @@ -111,7 +111,7 @@ try: logger.info( f"Whitelist for {kind} is already in cache, skipping downloads...", ) - if all_fresh is True: + if all_fresh: _exit(0) # Get URLs @@ -123,7 +123,7 @@ try: # Loop on kinds for kind, urls_list in urls.items(): - if kinds_fresh[kind] is True: + if kinds_fresh[kind]: continue # Write combined data of the kind to a single temp file for url in urls_list: @@ -145,7 +145,7 @@ try: line = line.split(b" ")[0] ok, data = check_line(kind, line) - if ok is True: + if ok: content += data + b"\n" i += 1