From a4aecabc33364db1264d302fa09d0390dda5ab64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Sun, 10 Mar 2024 15:48:15 +0000 Subject: [PATCH] Refactor all jobs * Made it more readable * Avoided duplication in the code as much as possible * Optimized a few things... --- src/common/confs/default-server-http.conf | 6 +- .../server-http/ssl-certificate-lua.conf | 4 +- .../ssl-certificate-stream-lua.conf | 4 +- .../core/blacklist/jobs/blacklist-download.py | 104 ++--- .../core/bunkernet/jobs/bunkernet-data.py | 115 ++--- .../core/bunkernet/jobs/bunkernet-register.py | 151 ++---- src/common/core/bunkernet/jobs/bunkernet.py | 7 +- src/common/core/customcert/customcert.lua | 8 +- .../core/customcert/jobs/custom-cert.py | 271 ++++------- .../core/greylist/jobs/greylist-download.py | 111 ++--- src/common/core/jobs/jobs/download-plugins.py | 40 +- src/common/core/jobs/jobs/mmdb-asn.py | 148 +++--- src/common/core/jobs/jobs/mmdb-country.py | 148 +++--- src/common/core/jobs/plugin.json | 4 +- .../core/letsencrypt/jobs/certbot-auth.py | 24 +- .../core/letsencrypt/jobs/certbot-cleanup.py | 24 +- .../core/letsencrypt/jobs/certbot-deploy.py | 36 +- .../core/letsencrypt/jobs/certbot-new.py | 101 ++-- .../core/letsencrypt/jobs/certbot-renew.py | 108 ++--- src/common/core/misc/jobs/anonymous-report.py | 61 +-- .../core/misc/jobs/default-server-cert.py | 65 +-- src/common/core/misc/jobs/update-check.py | 30 +- .../core/pro/jobs/download-pro-plugins.py | 70 ++- .../core/realip/jobs/realip-download.py | 89 ++-- .../core/selfsigned/jobs/self-signed.py | 116 ++--- src/common/core/selfsigned/selfsigned.lua | 8 +- .../core/whitelist/jobs/whitelist-download.py | 112 ++--- src/common/db/Database.py | 120 +++-- src/common/utils/common_utils.py | 80 ++++ src/common/utils/jobs.py | 439 +++++++++--------- 30 files changed, 1169 insertions(+), 1435 deletions(-) create mode 100644 src/common/utils/common_utils.py diff --git a/src/common/confs/default-server-http.conf b/src/common/confs/default-server-http.conf index d340d6651..e603b6873 100644 --- a/src/common/confs/default-server-http.conf +++ b/src/common/confs/default-server-http.conf @@ -17,7 +17,7 @@ server { # HTTPS listen {% set os = import("os") %} -{% if os.path.isfile("/var/cache/bunkerweb/default-server-cert/cert.pem") +%} +{% if os.path.isfile("/var/cache/bunkerweb/misc/default-server-cert.pem") +%} ssl_protocols {{ SSL_PROTOCOLS }}; ssl_prefer_server_ciphers on; ssl_session_tickets off; @@ -27,8 +27,8 @@ server { ssl_dhparam /etc/nginx/dhparam; ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; {% endif %} - ssl_certificate /var/cache/bunkerweb/default-server-cert/cert.pem; - ssl_certificate_key /var/cache/bunkerweb/default-server-cert/cert.key; + ssl_certificate /var/cache/bunkerweb/misc/default-server-cert.pem; + ssl_certificate_key /var/cache/bunkerweb/misc/default-server-cert.key; listen 0.0.0.0:{{ HTTPS_PORT }} ssl {% if HTTP2 == "yes" %}http2{% endif %} default_server {% if USE_PROXY_PROTOCOL == "yes" %}proxy_protocol{% endif %}; {% if USE_IPV6 == "yes" +%} listen [::]:{{ HTTPS_PORT }} ssl {% if HTTP2 == "yes" %}http2{% endif %} default_server {% if USE_PROXY_PROTOCOL == "yes" %}proxy_protocol{% endif %}; diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index 3e6ffc94d..79e1c4d9e 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -1,5 +1,5 @@ -ssl_certificate /var/cache/bunkerweb/default-server-cert/cert.pem; -ssl_certificate_key /var/cache/bunkerweb/default-server-cert/cert.key; +ssl_certificate /var/cache/bunkerweb/misc/default-server-cert.pem; +ssl_certificate_key /var/cache/bunkerweb/misc/default-server-cert.key; ssl_protocols {{ SSL_PROTOCOLS }}; ssl_prefer_server_ciphers on; ssl_session_tickets off; diff --git a/src/common/confs/server-stream/ssl-certificate-stream-lua.conf b/src/common/confs/server-stream/ssl-certificate-stream-lua.conf index 0ae726279..5009aa897 100644 --- a/src/common/confs/server-stream/ssl-certificate-stream-lua.conf +++ b/src/common/confs/server-stream/ssl-certificate-stream-lua.conf @@ -1,5 +1,5 @@ -ssl_certificate /var/cache/bunkerweb/default-server-cert/cert.pem; -ssl_certificate_key /var/cache/bunkerweb/default-server-cert/cert.key; +ssl_certificate /var/cache/bunkerweb/misc/default-server-cert.pem; +ssl_certificate_key /var/cache/bunkerweb/misc/default-server-cert.key; ssl_protocols {{ SSL_PROTOCOLS }}; ssl_prefer_server_ciphers on; ssl_session_tickets off; diff --git a/src/common/core/blacklist/jobs/blacklist-download.py b/src/common/core/blacklist/jobs/blacklist-download.py index 2bc5434b8..11ab47513 100755 --- a/src/common/core/blacklist/jobs/blacklist-download.py +++ b/src/common/core/blacklist/jobs/blacklist-download.py @@ -2,10 +2,9 @@ from contextlib import suppress from ipaddress import ip_address, ip_network -from os import _exit, getenv, sep +from os import getenv, sep from os.path import join, normpath -from pathlib import Path -from re import IGNORECASE, compile as re_compile +from re import compile as re_compile from sys import exit as sys_exit, path as sys_path from traceback import format_exc from typing import Tuple @@ -16,11 +15,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (( from requests import get -from Database import Database # type: ignore +from common_utils import bytes_hash # type: ignore from logger import setup_logger # type: ignore -from jobs import cache_file, cache_hash, del_file_in_db, is_cached_file, file_hash +from jobs import Job # type: ignore -rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE) +rdns_rx = re_compile(rb"^[^ ]+$") asn_rx = re_compile(rb"^\d+$") uri_rx = re_compile(rb"^/") @@ -51,7 +50,7 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]: return False, b"" -logger = setup_logger("BLACKLIST", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("BLACKLIST", getenv("LOG_LEVEL", "INFO")) status = 0 try: @@ -68,16 +67,10 @@ try: blacklist_activated = True if not blacklist_activated: - logger.info("Blacklist is not activated, skipping downloads...") - _exit(0) + LOGGER.info("Blacklist is not activated, skipping downloads...") + sys_exit(0) - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) - - # Create directories if they don't exist - blacklist_path = Path(sep, "var", "cache", "bunkerweb", "blacklist") - blacklist_path.mkdir(parents=True, exist_ok=True) - tmp_blacklist_path = Path(sep, "var", "tmp", "bunkerweb", "blacklist") - tmp_blacklist_path.mkdir(parents=True, exist_ok=True) + JOB = Job(LOGGER) # Get URLs urls = { @@ -110,37 +103,35 @@ try: "IGNORE_USER_AGENT": True, "IGNORE_URI": True, } - all_fresh = True for kind in kinds_fresh: - if not is_cached_file(blacklist_path.joinpath(f"{kind}.list"), "hour", db): - kinds_fresh[kind] = False - all_fresh = False - logger.info( - f"Blacklist for {kind} is not cached, processing downloads..", - ) - else: - logger.info( - f"Blacklist for {kind} is already in cache, skipping downloads...", - ) - if not urls[kind]: - logger.warning( - f"Blacklist for {kind} is cached but no URL is configured, removing from cache...", - ) - blacklist_path.joinpath(f"{kind}.list").unlink(missing_ok=True) - deleted, err = del_file_in_db(f"{kind}.list", db) - if not deleted: - logger.warning(f"Couldn't delete {kind}.list from cache : {err}") - if all_fresh: - _exit(0) + if not JOB.is_cached_file(f"{kind}.list", "hour"): + if urls[kind]: + kinds_fresh[kind] = False + LOGGER.info(f"Blacklist for {kind} is not cached, processing downloads..") + continue + + LOGGER.info(f"Blacklist for {kind} is already in cache, skipping downloads...") + + if not urls[kind]: + LOGGER.warning(f"Blacklist for {kind} is cached but no URL is configured, removing from cache...") + deleted, err = JOB.del_cache(f"{kind}.list") + if not deleted: + LOGGER.warning(f"Couldn't delete {kind}.list from cache : {err}") + + if all(kinds_fresh.values()): + if not any(urls.values()): + LOGGER.info("No blacklist URL is configured, nothing to do...") + sys_exit(0) # Loop on kinds for kind, urls_list in urls.items(): if kinds_fresh[kind]: continue - # Write combined data of the kind to a single temp file + + # Write combined data of the kind in memory and check if it has changed for url in urls_list: try: - logger.info(f"Downloading blacklist data from {url} ...") + LOGGER.info(f"Downloading blacklist data from {url} ...") if url.startswith("file://"): with open(normpath(url[7:]), "rb") as f: iterable = f.readlines() @@ -148,7 +139,7 @@ try: resp = get(url, stream=True, timeout=10) if resp.status_code != 200: - logger.warning(f"Got status code {resp.status_code}, skipping...") + LOGGER.warning(f"Got status code {resp.status_code}, skipping...") continue iterable = resp.iter_lines() @@ -168,39 +159,28 @@ try: content += data + b"\n" i += 1 - tmp_blacklist_path.joinpath(f"{kind}.list").write_bytes(content) - - logger.info(f"Downloaded {i} bad {kind}") + LOGGER.info(f"Downloaded {i} bad {kind}") # Check if file has changed - new_hash = file_hash(tmp_blacklist_path.joinpath(f"{kind}.list")) - old_hash = cache_hash(blacklist_path.joinpath(f"{kind}.list"), db) + new_hash = bytes_hash(content) + old_hash = JOB.cache_hash(f"{kind}.list") if new_hash == old_hash: - logger.info( - f"New file {kind}.list is identical to cache file, reload is not needed", - ) + LOGGER.info(f"New file {kind}.list is identical to cache file, reload is not needed") else: - logger.info( - f"New file {kind}.list is different than cache file, reload is needed", - ) + LOGGER.info(f"New file {kind}.list is different than cache file, reload is needed") # Put file in cache - cached, err = cache_file( - tmp_blacklist_path.joinpath(f"{kind}.list"), - blacklist_path.joinpath(f"{kind}.list"), - new_hash, - db, - ) - + cached, err = JOB.cache_file(f"{kind}.list", content, checksum=new_hash) if not cached: - logger.error(f"Error while caching blacklist : {err}") + LOGGER.error(f"Error while caching blacklist : {err}") status = 2 else: status = 1 except: status = 2 - logger.error(f"Exception while getting blacklist from {url} :\n{format_exc()}") - + LOGGER.error(f"Exception while getting blacklist from {url} :\n{format_exc()}") +except SystemExit as e: + status = e.code except: status = 2 - logger.error(f"Exception while running blacklist-download.py :\n{format_exc()}") + LOGGER.error(f"Exception while running blacklist-download.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/bunkernet/jobs/bunkernet-data.py b/src/common/core/bunkernet/jobs/bunkernet-data.py index d79b3377e..ed0f67692 100755 --- a/src/common/core/bunkernet/jobs/bunkernet-data.py +++ b/src/common/core/bunkernet/jobs/bunkernet-data.py @@ -1,29 +1,21 @@ #!/usr/bin/env python3 -from os import _exit, getenv, sep +from os import getenv, sep from os.path import join from pathlib import Path from sys import exit as sys_exit, path as sys_path from traceback import format_exc -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("db",), - ("core", "bunkernet", "jobs"), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) from bunkernet import data -from Database import Database # type: ignore from logger import setup_logger # type: ignore -from jobs import cache_file, cache_hash, file_hash, is_cached_file, get_file_in_db +from jobs import Job # type: ignore +from common_utils import bytes_hash # type: ignore -logger = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO")) exit_status = 0 try: @@ -40,109 +32,90 @@ try: bunkernet_activated = True if not bunkernet_activated: - logger.info("BunkerNet is not activated, skipping download...") - _exit(0) + LOGGER.info("BunkerNet is not activated, skipping download...") + sys_exit(0) # Create directory if it doesn't exist bunkernet_path = Path(sep, "var", "cache", "bunkerweb", "bunkernet") bunkernet_path.mkdir(parents=True, exist_ok=True) - bunkernet_tmp_path = Path(sep, "var", "tmp", "bunkerweb", "bunkernet") - bunkernet_tmp_path.mkdir(parents=True, exist_ok=True) + + JOB = Job(LOGGER) # Create empty file in case it doesn't exist - bunkernet_path.joinpath("ip.list").touch(exist_ok=True) + ip_list_path = bunkernet_path.joinpath("ip.list") + if not ip_list_path.is_file(): + ip_list_path.touch(exist_ok=True) # Get ID from cache bunkernet_id = None - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) - bunkernet_id = get_file_in_db("instance.id", db) + bunkernet_id = JOB.get_cache("instance.id") if bunkernet_id: bunkernet_path.joinpath("instance.id").write_bytes(bunkernet_id) - logger.info("Successfully retrieved BunkerNet ID from db cache") + LOGGER.info("Successfully retrieved BunkerNet ID from db cache") else: - logger.info("No BunkerNet ID found in db cache") + LOGGER.info("No BunkerNet ID found in db cache") # Check if ID is present if not bunkernet_path.joinpath("instance.id").is_file(): - logger.error( - "Not downloading BunkerNet data because instance is not registered", - ) - _exit(2) + LOGGER.error("Not downloading BunkerNet data because instance is not registered") + sys_exit(2) # Don't go further if the cache is fresh - if is_cached_file(bunkernet_path.joinpath("ip.list"), "day", db): - logger.info( - "BunkerNet list is already in cache, skipping download...", - ) - _exit(0) + if JOB.is_cached_file("ip.list", "day"): + LOGGER.info("BunkerNet list is already in cache, skipping download...") + sys_exit(0) exit_status = 1 # Download data - logger.info("Downloading BunkerNet data ...") + LOGGER.info("Downloading BunkerNet data ...") ok, status, data = data() if not ok: - logger.error( - f"Error while sending data request to BunkerNet API : {data}", - ) - _exit(2) + LOGGER.error(f"Error while sending data request to BunkerNet API : {data}") + sys_exit(2) elif status == 429: - logger.warning( - "BunkerNet API is rate limiting us, trying again later...", - ) - _exit(0) + LOGGER.warning("BunkerNet API is rate limiting us, trying again later...") + sys_exit(0) elif status == 403: - logger.warning( - "BunkerNet has banned this instance, retrying a register later...", - ) - _exit(0) + LOGGER.warning("BunkerNet has banned this instance, retrying a register later...") + sys_exit(0) try: assert isinstance(data, dict) except AssertionError: - logger.error( - f"Received invalid data from BunkerNet API while sending db request : {data}", - ) - _exit(2) + LOGGER.error(f"Received invalid data from BunkerNet API while sending db request : {data}") + sys_exit(2) if data["result"] != "ok": - logger.error( - f"Received error from BunkerNet API while sending db request : {data['data']}, removing instance ID", - ) - _exit(2) + LOGGER.error(f"Received error from BunkerNet API while sending db request : {data['data']}, removing instance ID") + sys_exit(2) - logger.info("Successfully downloaded data from BunkerNet API") + LOGGER.info("Successfully downloaded data from BunkerNet API") # Writing data to file - logger.info("Saving BunkerNet data ...") + LOGGER.info("Saving BunkerNet data ...") content = "\n".join(data["data"]).encode("utf-8") - bunkernet_tmp_path.joinpath("ip.list").write_bytes(content) # Check if file has changed - new_hash = file_hash(bunkernet_tmp_path.joinpath("ip.list")) - old_hash = cache_hash(bunkernet_path.joinpath("ip.list"), db) + new_hash = bytes_hash(content) + old_hash = JOB.cache_hash("ip.list") if new_hash == old_hash: - logger.info( - "New file is identical to cache file, reload is not needed", - ) - _exit(0) + LOGGER.info("New file is identical to cache file, reload is not needed") + sys_exit(0) # Put file in cache - cached, err = cache_file( - bunkernet_tmp_path.joinpath("ip.list"), - bunkernet_path.joinpath("ip.list"), - new_hash, - db, - ) + cached, err = JOB.cache_file("ip.list", content, checksum=new_hash) if not cached: - logger.error(f"Error while caching BunkerNet data : {err}") - _exit(2) + LOGGER.error(f"Error while caching BunkerNet data : {err}") + sys_exit(2) - logger.info("Successfully saved BunkerNet data") + LOGGER.info("Successfully saved BunkerNet data") exit_status = 1 +except SystemExit as e: + exit_status = e.code except: exit_status = 2 - logger.error(f"Exception while running bunkernet-data.py :\n{format_exc()}") + LOGGER.error(f"Exception while running bunkernet-data.py :\n{format_exc()}") sys_exit(exit_status) diff --git a/src/common/core/bunkernet/jobs/bunkernet-register.py b/src/common/core/bunkernet/jobs/bunkernet-register.py index 5e99ba237..e3ba1be8a 100755 --- a/src/common/core/bunkernet/jobs/bunkernet-register.py +++ b/src/common/core/bunkernet/jobs/bunkernet-register.py @@ -1,30 +1,19 @@ #!/usr/bin/env python3 -from os import _exit, getenv, sep +from os import getenv, sep from os.path import join -from pathlib import Path from sys import exit as sys_exit, path as sys_path -from time import sleep from traceback import format_exc -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("db",), - ("core", "bunkernet", "jobs"), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) -from bunkernet import register, ping -from Database import Database # type: ignore +from bunkernet import register from logger import setup_logger # type: ignore -from jobs import get_file_in_db, set_file_in_db, del_file_in_db # type: ignore +from jobs import Job # type: ignore -logger = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO")) exit_status = 0 try: @@ -41,132 +30,62 @@ try: bunkernet_activated = True if not bunkernet_activated: - logger.info("BunkerNet is not activated, skipping registration...") - _exit(0) - - # Create directory if it doesn't exist - bunkernet_path = Path(sep, "var", "cache", "bunkerweb", "bunkernet") - bunkernet_path.mkdir(parents=True, exist_ok=True) + LOGGER.info("BunkerNet is not activated, skipping registration...") + sys_exit(0) # Get ID from cache - bunkernet_id = None - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) - bunkernet_id = get_file_in_db("instance.id", db) - if bunkernet_id: - bunkernet_path.joinpath("instance.id").write_bytes(bunkernet_id) - logger.info("Successfully retrieved BunkerNet ID from db cache") - else: - logger.info("No BunkerNet ID found in db cache") + JOB = Job(LOGGER) + bunkernet_id = JOB.get_cache("instance.id") # Register instance registered = False - instance_id_path = bunkernet_path.joinpath("instance.id") - if not instance_id_path.is_file(): - logger.info("Registering instance on BunkerNet API ...") + if not bunkernet_id: + LOGGER.info("No BunkerNet ID found in db cache, Registering instance on BunkerNet API ...") ok, status, data = register() if not ok: - logger.error(f"Error while sending register request to BunkerNet API : {data}") - _exit(2) + LOGGER.error(f"Error while sending register request to BunkerNet API : {data}") + sys_exit(2) elif status == 429: - logger.warning( - "BunkerNet API is rate limiting us, trying again later...", - ) - _exit(0) + LOGGER.warning("BunkerNet API is rate limiting us, trying again later...") + sys_exit(0) elif status == 403: - logger.warning( - "BunkerNet has banned this instance, retrying a register later...", - ) - _exit(0) + LOGGER.warning("BunkerNet has banned this instance, retrying a register later...") + sys_exit(0) try: assert isinstance(data, dict) except AssertionError: - logger.error( - f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...", - ) - _exit(2) + LOGGER.error(f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...") + sys_exit(2) + bunkernet_id = data.get("data") if status != 200: - logger.error( - f"Error {status} from BunkerNet API : {data['data']}", - ) - _exit(2) + LOGGER.error(f"Error {status} from BunkerNet API : {bunkernet_id}") + sys_exit(2) elif data.get("result", "ko") != "ok": - logger.error(f"Received error from BunkerNet API while sending register request : {data.get('data', {})}") - _exit(2) - bunkernet_id = data["data"] - instance_id_path.write_text(bunkernet_id, encoding="utf-8") + LOGGER.error(f"Received error from BunkerNet API while sending register request : {bunkernet_id}") + sys_exit(2) + + assert isinstance(bunkernet_id, str), f"Received invalid bunkernet id : {bunkernet_id}" + registered = True exit_status = 1 - logger.info(f"Successfully registered on BunkerNet API with instance id {data['data']}") + LOGGER.info(f"Successfully registered on BunkerNet API with instance id {data['data']}") else: - bunkernet_id = bunkernet_id or instance_id_path.read_bytes() bunkernet_id = bunkernet_id.decode() - logger.info(f"Already registered on BunkerNet API with instance id {bunkernet_id}") - - sleep(1) + LOGGER.info(f"Already registered on BunkerNet API with instance id {bunkernet_id}") # Update cache with new bunkernet ID if registered: - cached, err = set_file_in_db("instance.id", bunkernet_id.encode(), db) + cached, err = JOB.cache_file("instance.id", bunkernet_id.encode()) if not cached: - logger.error(f"Error while saving BunkerNet data to db cache : {err}") + LOGGER.error(f"Error while saving BunkerNet data to db cache : {err}") else: - logger.info("Successfully saved BunkerNet data to db cache") - - # Ping - logger.info("Checking connectivity with BunkerNet API ...") - bunkernet_ping = False - for i in range(0, 5): - ok, status, data = ping(bunkernet_id) - retry = False - if not ok: - logger.error(f"Error while sending ping request to BunkerNet API : {data}") - retry = True - elif status == 429: - logger.warning( - "BunkerNet API is rate limiting us, trying again later...", - ) - retry = True - elif status == 403: - logger.warning( - "BunkerNet has banned this instance, retrying a register later...", - ) - _exit(2) - elif status == 401: - logger.warning( - "Instance ID is not registered, removing it and retrying a register later...", - ) - instance_id_path.unlink() - del_file_in_db("instance.id", db) - _exit(2) - - try: - assert isinstance(data, dict) - except AssertionError: - logger.error( - f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...", - ) - _exit(2) - - if data.get("result", "ko") != "ok": - logger.error( - f"Received error from BunkerNet API while sending ping request : {data.get('data', {})}", - ) - retry = True - if not retry: - bunkernet_ping = True - break - logger.warning("Waiting 1s and trying again ...") - sleep(1) - - if bunkernet_ping: - logger.info("Connectivity with BunkerNet is successful !") - else: - logger.error("Connectivity with BunkerNet failed ...") - exit_status = 2 + LOGGER.info("Successfully saved BunkerNet data to db cache") +except SystemExit as e: + exit_status = e.code except: exit_status = 2 - logger.error(f"Exception while running bunkernet-register.py :\n{format_exc()}") + LOGGER.error(f"Exception while running bunkernet-register.py :\n{format_exc()}") sys_exit(exit_status) diff --git a/src/common/core/bunkernet/jobs/bunkernet.py b/src/common/core/bunkernet/jobs/bunkernet.py index 004075b3f..96fccd7d6 100644 --- a/src/common/core/bunkernet/jobs/bunkernet.py +++ b/src/common/core/bunkernet/jobs/bunkernet.py @@ -5,10 +5,10 @@ from pathlib import Path from requests import request as requests_request, ReadTimeout from typing import Literal, Optional, Tuple, Union -from jobs import get_os_info, get_integration, get_version # type: ignore +from common_utils import get_os_info, get_integration, get_version # type: ignore -def request(method: Union[Literal["POST"], Literal["GET"]], url: str, _id: Optional[str] = None) -> Tuple[bool, Optional[int], Union[str, dict]]: +def request(method: Literal["POST", "GET"], url: str, _id: Optional[str] = None) -> Tuple[bool, Optional[int], Union[str, dict]]: data = { "integration": get_integration(), "version": get_version(), @@ -17,13 +17,12 @@ def request(method: Union[Literal["POST"], Literal["GET"]], url: str, _id: Optio if _id: data["id"] = _id - headers = {"User-Agent": f"BunkerWeb/{data['version']}"} try: resp = requests_request( method, f"{getenv('BUNKERNET_SERVER', 'https://api.bunkerweb.io')}{url}", json=data, - headers=headers, + headers={"User-Agent": f"BunkerWeb/{data['version']}"}, timeout=5, ) status = resp.status_code diff --git a/src/common/core/customcert/customcert.lua b/src/common/core/customcert/customcert.lua index 97aca3f54..0eeb788a9 100644 --- a/src/common/core/customcert/customcert.lua +++ b/src/common/core/customcert/customcert.lua @@ -44,8 +44,8 @@ function customcert:init() for server_name, multisite_vars in pairs(vars) do if multisite_vars["USE_CUSTOM_SSL"] == "yes" and server_name ~= "global" then local check, data = read_files({ - "/var/cache/bunkerweb/customcert/" .. server_name .. ".cert.pem", - "/var/cache/bunkerweb/customcert/" .. server_name .. ".key.pem", + "/var/cache/bunkerweb/customcert/" .. server_name .. "/cert.pem", + "/var/cache/bunkerweb/customcert/" .. server_name .. "/key.pem", }) if not check then self.logger:log(ERR, "error while reading files : " .. data) @@ -68,8 +68,8 @@ function customcert:init() return self:ret(false, "can't get SERVER_NAME variable : " .. err) end local check, data = read_files({ - "/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. ".cert.pem", - "/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. ".key.pem", + "/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/cert.pem", + "/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/key.pem", }) if not check then self.logger:log(ERR, "error while reading files : " .. data) diff --git a/src/common/core/customcert/jobs/custom-cert.py b/src/common/core/customcert/jobs/custom-cert.py index c808e902e..948dbf8e9 100644 --- a/src/common/core/customcert/jobs/custom-cert.py +++ b/src/common/core/customcert/jobs/custom-cert.py @@ -1,220 +1,141 @@ #!/usr/bin/env python3 +from contextlib import suppress from os import getenv, sep -from os.path import join, normpath +from os.path import join from pathlib import Path from sys import exit as sys_exit, path as sys_path from traceback import format_exc from base64 import b64decode +from typing import Tuple, Union -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("db",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) -from jobs import del_file_in_db, cache_file, cache_hash, file_hash -from Database import Database # type: ignore +from common_utils import bytes_hash # type: ignore +from jobs import Job # type: ignore from logger import setup_logger # type: ignore -logger = setup_logger("CUSTOM-CERT", getenv("LOG_LEVEL", "INFO")) -db = None +LOGGER = setup_logger("CUSTOM-CERT", getenv("LOG_LEVEL", "INFO")) +JOB = Job(LOGGER) -def check_cert(cert_path: str, key_path: str, first_server: str) -> bool: - try: +def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], first_server: str) -> Tuple[bool, str]: + with suppress(BaseException): ret = False - if not cert_path or not key_path: - logger.warning("Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates") - return False + if not cert_file or not key_file: + return False, "Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates" - cert_path: Path = Path(normpath(cert_path)) - key_path: Path = Path(normpath(key_path)) + if isinstance(cert_file, Path): + if not cert_file.is_file(): + return False, f"Certificate file {cert_file} is not a valid file, ignoring the custom certificate" + cert_file = cert_file.read_bytes() - if not cert_path.is_file(): - logger.warning(f"Certificate file {cert_path} is not a valid file, ignoring the custom certificate") - return False - elif not key_path.is_file(): - logger.warning(f"Key file {key_path} is not a valid file, ignoring the custom certificate") - return False + if isinstance(key_file, Path): + if not key_file.is_file(): + return False, f"Key file {key_file} is not a valid file, ignoring the custom certificate" + key_file = key_file.read_bytes() - cert_cache_path = Path( - sep, - "var", - "cache", - "bunkerweb", - "customcert", - f"{first_server}.cert.pem", - ) - cert_cache_path.parent.mkdir(parents=True, exist_ok=True) - - cert_hash = file_hash(cert_path) - old_hash = cache_hash(cert_cache_path, db) + cert_hash = bytes_hash(cert_file) + old_hash = JOB.cache_hash("cert.pem", service_id=first_server) if old_hash != cert_hash: ret = True - cached, err = cache_file(cert_path, cert_cache_path, cert_hash, db, delete_file=False) + cached, err = JOB.cache_file("cert.pem", cert_file, service_id=first_server, checksum=cert_hash, delete_file=False) if not cached: - logger.error(f"Error while caching custom-cert cert.pem file : {err}") - elif not cert_cache_path.is_file(): - cert_cache_path.write_bytes(cert_path.read_bytes()) - ret = True - key_cache_path = Path( - sep, - "var", - "cache", - "bunkerweb", - "customcert", - f"{first_server}.key.pem", - ) - key_cache_path.parent.mkdir(parents=True, exist_ok=True) + LOGGER.error(f"Error while caching custom-cert cert.pem file : {err}") - key_hash = file_hash(key_path) - old_hash = cache_hash(key_cache_path, db) + key_hash = bytes_hash(key_file) + old_hash = JOB.cache_hash("key.pem", service_id=first_server) if old_hash != key_hash: ret = True - cached, err = cache_file(key_path, key_cache_path, key_hash, db, delete_file=False) + cached, err = JOB.cache_file("key.pem", key_file, service_id=first_server, checksum=key_hash, delete_file=False) if not cached: - logger.error(f"Error while caching custom-cert key.pem file : {err}") - elif not key_cache_path.is_file(): - key_cache_path.write_bytes(key_path.read_bytes()) - ret = True + LOGGER.error(f"Error while caching custom-key key.pem file : {err}") - return ret - except: - logger.error( - f"Exception while running custom-cert.py (check_cert) :\n{format_exc()}", - ) - return False + return ret, "" + return False, "exception" status = 0 try: - Path(sep, "var", "cache", "bunkerweb", "customcert").mkdir(parents=True, exist_ok=True) + all_domains = getenv("SERVER_NAME") or [] - if getenv("MULTISITE", "no") == "no" and getenv("USE_CUSTOM_SSL", "no") == "yes" and getenv("SERVER_NAME", "") != "": - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) + if isinstance(all_domains, str): + all_domains = all_domains.split(" ") - cert_path = getenv("CUSTOM_SSL_CERT", "") - key_path = getenv("CUSTOM_SSL_KEY", "") - first_server = getenv("SERVER_NAME").split(" ")[0] + if not all_domains: + LOGGER.warning("No services found, exiting ...") + sys_exit(0) - cert_data = b64decode(getenv("CUSTOM_SSL_CERT_DATA", "")) - key_data = b64decode(getenv("CUSTOM_SSL_KEY_DATA", "")) - for file, data in (("cert.pem", cert_data), ("key.pem", key_data)): - if data: - file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", f"{first_server}.{file}") - file_path.write_bytes(data) - if file == "cert.pem": - cert_path = str(file_path) - else: - key_path = str(file_path) + skipped_servers = [] + if not getenv("MULTISITE", "no") == "yes": + all_domains = [all_domains[0]] + if getenv("USE_CUSTOM_SSL", "no") == "no": + LOGGER.info("Custom SSL is not enabled, skipping ...") + skipped_servers = all_domains - if cert_path and key_path: - logger.info(f"Checking certificate {cert_path} ...") - need_reload = check_cert(cert_path, key_path, first_server) - if need_reload: - logger.info(f"Detected change for certificate {cert_path}") - status = 1 - else: - logger.info(f"No change for certificate {cert_path}") - elif not cert_path or not key_path: - logger.warning( - "Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY (or CUSTOM_SSL_CERT_DATA and CUSTOM_SSL_KEY_DATA) have to be set to use custom certificates, clearing cache ..." - ) - cert_cache_path = Path( - sep, - "var", - "cache", - "bunkerweb", - "customcert", - f"{first_server}.cert.pem", - ) - cert_cache_path.unlink(missing_ok=True) - del_file_in_db(f"{first_server}.cert.pem", db, service_id=first_server) - key_cache_path = Path( - sep, - "var", - "cache", - "bunkerweb", - "customcert", - f"{first_server}.key.pem", - ) - key_cache_path.unlink(missing_ok=True) - del_file_in_db(f"{first_server}.key.pem", db, service_id=first_server) - elif getenv("MULTISITE", "no") == "yes": - servers = getenv("SERVER_NAME") or [] - - if isinstance(servers, str): - servers = servers.split(" ") - - for first_server in servers: - if not first_server or (getenv(f"{first_server}_USE_CUSTOM_SSL", getenv("USE_CUSTOM_SSL", "no")) != "yes"): + if not skipped_servers: + for first_server in all_domains: + if getenv(f"{first_server}_USE_CUSTOM_SSL", getenv("USE_CUSTOM_SSL", "no")) == "no": + LOGGER.info(f"Custom SSL is not enabled for {first_server}, skipping ...") + skipped_servers.append(first_server) continue - if not db: - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) + cert_file = getenv(f"{first_server}_CUSTOM_SSL_CERT", getenv("CUSTOM_SSL_CERT", "")) + key_file = getenv(f"{first_server}_CUSTOM_SSL_KEY", getenv("CUSTOM_SSL_KEY", "")) + cert_data = getenv(f"{first_server}_CUSTOM_SSL_CERT_DATA", getenv("CUSTOM_SSL_CERT_DATA", "")) + key_data = getenv(f"{first_server}_CUSTOM_SSL_KEY_DATA", getenv("CUSTOM_SSL_KEY_DATA", "")) - cert_path = getenv(f"{first_server}_CUSTOM_SSL_CERT", "") - key_path = getenv(f"{first_server}_CUSTOM_SSL_KEY", "") - - cert_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_CERT_DATA", "")) - key_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_KEY_DATA", "")) - for file, data in (("cert.pem", cert_data), ("key.pem", key_data)): - if data != b"": - file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", f"{first_server}.{file}") - file_path.parent.mkdir(parents=True, exist_ok=True) - file_path.write_bytes(data) - if file == "cert.pem": - cert_path = str(file_path) - else: - key_path = str(file_path) - - if cert_path and key_path: - logger.info( - f"Checking certificate {cert_path} ...", - ) - need_reload = check_cert(cert_path, key_path, first_server) - if need_reload: - logger.info( - f"Detected change for certificate {cert_path}", - ) - status = 1 + if cert_file or cert_data and key_file or key_data: + if isinstance(cert_file, str): + cert_file = Path(cert_file) else: - logger.info( - f"No change for certificate {cert_path}", - ) - elif not cert_path or not key_path: - logger.warning( - "Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY (or CUSTOM_SSL_CERT_DATA and CUSTOM_SSL_KEY_DATA) have to be set to use custom certificates, clearing cache ..." + try: + cert_file = b64decode(cert_data) + except BaseException: + LOGGER.exception(f"Error while decoding cert data, skipping server {first_server}...") + skipped_servers.append(first_server) + continue + + if isinstance(key_file, str): + key_file = Path(key_file) + else: + try: + key_file = b64decode(key_data) + except BaseException: + LOGGER.exception(f"Error while decoding key data, skipping server {first_server}...") + skipped_servers.append(first_server) + continue + + LOGGER.info(f"Checking certificate for {first_server} ...") + need_reload, err = check_cert(cert_file, key_file, first_server) + if err == "exception": + LOGGER.exception(f"Exception while checking {first_server}'s certificate, skipping ...") + skipped_servers.append(first_server) + continue + elif err: + LOGGER.warning(f"Error while checking {first_server}'s certificate : {err}") + skipped_servers.append(first_server) + continue + elif need_reload: + LOGGER.info(f"Detected change in {first_server}'s certificate") + status = 1 + continue + + LOGGER.info(f"No change in {first_server}'s certificate") + elif not cert_file or not key_file: + LOGGER.warning( + "Variables (CUSTOM_SSL_CERT or CUSTOM_SSL_CERT_DATA) and (CUSTOM_SSL_KEY or CUSTOM_SSL_KEY_DATA) have to be set to use custom certificates" ) - cert_cache_path = Path( - sep, - "var", - "cache", - "bunkerweb", - "customcert", - f"{first_server}.cert.pem", - ) - cert_cache_path.unlink(missing_ok=True) - del_file_in_db(f"{first_server}.cert.pem", db) - key_cache_path = Path( - sep, - "var", - "cache", - "bunkerweb", - "customcert", - f"{first_server}.key.pem", - ) - key_cache_path.unlink(missing_ok=True) - del_file_in_db(f"{first_server}.key.pem", db) + skipped_servers.append(first_server) + + for first_server in skipped_servers: + JOB.del_cache("cert.pem", service_id=first_server) + JOB.del_cache("key.pem", service_id=first_server) except: status = 2 - logger.error(f"Exception while running custom-cert.py :\n{format_exc()}") + LOGGER.error(f"Exception while running custom-cert.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/greylist/jobs/greylist-download.py b/src/common/core/greylist/jobs/greylist-download.py index 7684d47ae..f4f4fb016 100755 --- a/src/common/core/greylist/jobs/greylist-download.py +++ b/src/common/core/greylist/jobs/greylist-download.py @@ -2,10 +2,9 @@ from contextlib import suppress from ipaddress import ip_address, ip_network -from os import _exit, getenv, sep +from os import getenv, sep from os.path import join, normpath -from pathlib import Path -from re import IGNORECASE, compile as re_compile +from re import compile as re_compile from sys import exit as sys_exit, path as sys_path from traceback import format_exc from typing import Tuple @@ -16,11 +15,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (( from requests import get -from Database import Database # type: ignore +from common_utils import bytes_hash # type: ignore from logger import setup_logger # type: ignore -from jobs import cache_file, cache_hash, del_file_in_db, is_cached_file, file_hash +from jobs import Job # type: ignore -rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE) +rdns_rx = re_compile(rb"^[^ ]+$") asn_rx = re_compile(rb"^\d+$") uri_rx = re_compile(rb"^/") @@ -51,7 +50,7 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]: return False, b"" -logger = setup_logger("GREYLIST", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("GREYLIST", getenv("LOG_LEVEL", "INFO")) status = 0 try: @@ -68,16 +67,10 @@ try: greylist_activated = True if not greylist_activated: - logger.info("Greylist is not activated, skipping downloads...") - _exit(0) + LOGGER.info("Greylist is not activated, skipping downloads...") + sys_exit(0) - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) - - # Create directories if they don't exist - greylist_path = Path(sep, "var", "cache", "bunkerweb", "greylist") - greylist_path.mkdir(parents=True, exist_ok=True) - tmp_greylist_path = Path(sep, "var", "tmp", "bunkerweb", "greylist") - tmp_greylist_path.mkdir(parents=True, exist_ok=True) + JOB = Job(LOGGER) # Get URLs urls = {"IP": [], "RDNS": [], "ASN": [], "USER_AGENT": [], "URI": []} @@ -87,45 +80,36 @@ try: urls[kind].append(url) # Don't go further if the cache is fresh - kinds_fresh = { - "IP": True, - "RDNS": True, - "ASN": True, - "USER_AGENT": True, - "URI": True, - } - all_fresh = True + kinds_fresh = {"IP": True, "RDNS": True, "ASN": True, "USER_AGENT": True, "URI": True} for kind in kinds_fresh: - if not is_cached_file(greylist_path.joinpath(f"{kind}.list"), "hour", db): - kinds_fresh[kind] = False - all_fresh = False - logger.info( - f"Greylist for {kind} is not cached, processing downloads..", - ) - else: - logger.info( - f"Greylist for {kind} is already in cache, skipping downloads...", - ) - if not urls[kind]: - logger.warning( - f"Greylist for {kind} is cached but no URL is configured, removing from cache...", - ) - greylist_path.joinpath(f"{kind}.list").unlink(missing_ok=True) - deleted, err = del_file_in_db(f"{kind}.list", db) - if not deleted: - logger.warning(f"Couldn't delete {kind}.list from cache : {err}") + if not JOB.is_cached_file(f"{kind}.list", "hour"): + if urls[kind]: + kinds_fresh[kind] = False + LOGGER.info(f"Greylist for {kind} is not cached, processing downloads..") + continue - if all_fresh: - _exit(0) + LOGGER.info(f"Greylist for {kind} is already in cache, skipping downloads...") + + if not urls[kind]: + LOGGER.warning(f"Greylist for {kind} is cached but no URL is configured, removing from cache...") + deleted, err = JOB.del_cache(f"{kind}.list") + if not deleted: + LOGGER.warning(f"Couldn't delete {kind}.list from cache : {err}") + + if all(kinds_fresh.values()): + if not any(urls.values()): + LOGGER.info("No greylist URL is configured, nothing to do...") + sys_exit(0) # Loop on kinds for kind, urls_list in urls.items(): if kinds_fresh[kind]: continue - # Write combined data of the kind to a single temp file + + # Write combined data of the kind in memory and check if it has changed for url in urls_list: try: - logger.info(f"Downloading greylist data from {url} ...") + LOGGER.info(f"Downloading greylist data from {url} ...") if url.startswith("file://"): with open(normpath(url[7:]), "rb") as f: iterable = f.readlines() @@ -133,7 +117,7 @@ try: resp = get(url, stream=True, timeout=10) if resp.status_code != 200: - logger.warning(f"Got status code {resp.status_code}, skipping...") + LOGGER.warning(f"Got status code {resp.status_code}, skipping...") continue iterable = resp.iter_lines() @@ -153,39 +137,28 @@ try: content += data + b"\n" i += 1 - tmp_greylist_path.joinpath(f"{kind}.list").write_bytes(content) - - logger.info(f"Downloaded {i} bad {kind}") + LOGGER.info(f"Downloaded {i} bad {kind}") # Check if file has changed - new_hash = file_hash(tmp_greylist_path.joinpath(f"{kind}.list")) - old_hash = cache_hash(greylist_path.joinpath(f"{kind}.list"), db) + new_hash = bytes_hash(content) + old_hash = JOB.cache_hash(f"{kind}.list") if new_hash == old_hash: - logger.info( - f"New file {kind}.list is identical to cache file, reload is not needed", - ) + LOGGER.info(f"New file {kind}.list is identical to cache file, reload is not needed") else: - logger.info( - f"New file {kind}.list is different than cache file, reload is needed", - ) + LOGGER.info(f"New file {kind}.list is different than cache file, reload is needed") # Put file in cache - cached, err = cache_file( - tmp_greylist_path.joinpath(f"{kind}.list"), - greylist_path.joinpath(f"{kind}.list"), - new_hash, - db, - ) - + cached, err = JOB.cache_file(f"{kind}.list", content, checksum=new_hash) if not cached: - logger.error(f"Error while caching greylist : {err}") + LOGGER.error(f"Error while caching greylist : {err}") status = 2 else: status = 1 except: status = 2 - logger.error(f"Exception while getting greylist from {url} :\n{format_exc()}") - + LOGGER.error(f"Exception while getting greylist from {url} :\n{format_exc()}") +except SystemExit as e: + status = e.code except: status = 2 - logger.error(f"Exception while running greylist-download.py :\n{format_exc()}") + LOGGER.error(f"Exception while running greylist-download.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/jobs/jobs/download-plugins.py b/src/common/core/jobs/jobs/download-plugins.py index 5b6d5e7ec..c065fe8d1 100644 --- a/src/common/core/jobs/jobs/download-plugins.py +++ b/src/common/core/jobs/jobs/download-plugins.py @@ -36,7 +36,7 @@ from logger import setup_logger # type: ignore EXTERNAL_PLUGINS_DIR = Path(sep, "etc", "bunkerweb", "plugins") -logger = setup_logger("Jobs.download-plugins", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("Jobs.download-plugins", getenv("LOG_LEVEL", "INFO")) status = 0 @@ -45,14 +45,14 @@ def install_plugin(plugin_dir: str, db) -> bool: plugin_file = plugin_path.joinpath("plugin.json") if not plugin_file.is_file(): - logger.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json not found)") + LOGGER.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json not found)") return False # Load plugin.json try: metadata = loads(plugin_file.read_text(encoding="utf-8")) except JSONDecodeError: - logger.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json is not valid)") + LOGGER.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json is not valid)") return False # Don't go further if plugin is already installed @@ -65,12 +65,12 @@ def install_plugin(plugin_dir: str, db) -> bool: break if old_version == metadata["version"]: - logger.warning( + LOGGER.warning( f"Skipping installation of plugin {metadata['id']} (version {metadata['version']} already installed)", ) return False - logger.warning( + LOGGER.warning( f"Plugin {metadata['id']} is already installed but version {metadata['version']} is different from database ({old_version}), updating it...", ) rmtree(EXTERNAL_PLUGINS_DIR.joinpath(metadata["id"]), ignore_errors=True) @@ -81,7 +81,7 @@ def install_plugin(plugin_dir: str, db) -> bool: for job_file in glob(join(sep, "etc", "bunkerweb", "plugins", "jobs", "*")): st = Path(job_file).stat() chmod(job_file, st.st_mode | S_IEXEC) - logger.info(f"Plugin {metadata['id']} installed") + LOGGER.info(f"Plugin {metadata['id']} installed") return True @@ -89,14 +89,14 @@ try: # Check if we have plugins to download plugin_urls = getenv("EXTERNAL_PLUGIN_URLS") if not plugin_urls: - logger.info("No external plugins to download") + LOGGER.info("No external plugins to download") sys_exit(0) - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False) + db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"), pool=False) plugin_nbr = 0 # Loop on URLs - logger.info(f"Downloading external plugins from {plugin_urls}...") + LOGGER.info(f"Downloading external plugins from {plugin_urls}...") for plugin_url in plugin_urls.split(" "): # Download Plugin file try: @@ -112,7 +112,7 @@ try: ) if resp.status_code != 200: - logger.warning(f"Got status code {resp.status_code}, skipping...") + LOGGER.warning(f"Got status code {resp.status_code}, skipping...") continue # Iterate over the response content in chunks @@ -120,7 +120,7 @@ try: if chunk: content += chunk except: - logger.error( + LOGGER.error( f"Exception while downloading plugin(s) from {plugin_url} :\n{format_exc()}", ) status = 2 @@ -142,10 +142,10 @@ try: with tar_open(fileobj=BytesIO(content), mode="r") as tar: tar.extractall(path=temp_dir) else: - logger.error(f"Unknown file type for {plugin_url}, either zip or tar are supported, skipping...") + LOGGER.error(f"Unknown file type for {plugin_url}, either zip or tar are supported, skipping...") continue except: - logger.error(f"Exception while decompressing plugin(s) from {plugin_url} :\n{format_exc()}") + LOGGER.error(f"Exception while decompressing plugin(s) from {plugin_url} :\n{format_exc()}") status = 2 continue @@ -156,13 +156,13 @@ try: if install_plugin(plugin_dir, db): plugin_nbr += 1 except FileExistsError: - logger.warning(f"Skipping installation of plugin {basename(plugin_dir)} (already installed)") + LOGGER.warning(f"Skipping installation of plugin {basename(plugin_dir)} (already installed)") except: - logger.error(f"Exception while installing plugin(s) from {plugin_url} :\n{format_exc()}") + LOGGER.error(f"Exception while installing plugin(s) from {plugin_url} :\n{format_exc()}") status = 2 if not plugin_nbr: - logger.info("No external plugins to update to database") + LOGGER.info("No external plugins to update to database") sys_exit(0) external_plugins = [] @@ -170,7 +170,7 @@ try: for plugin in listdir(EXTERNAL_PLUGINS_DIR): path = EXTERNAL_PLUGINS_DIR.joinpath(plugin) if not path.joinpath("plugin.json").is_file(): - logger.warning(f"Plugin {plugin} is not valid, deleting it...") + LOGGER.warning(f"Plugin {plugin} is not valid, deleting it...") rmtree(path, ignore_errors=True) continue @@ -208,18 +208,18 @@ try: err = db.update_external_plugins(external_plugins) if err: - logger.error( + LOGGER.error( f"Couldn't update external plugins to database: {err}", ) status = 1 - logger.info("External plugins downloaded and installed") + LOGGER.info("External plugins downloaded and installed") except SystemExit as e: status = e.code except: status = 2 - logger.error(f"Exception while running download-plugins.py :\n{format_exc()}") + LOGGER.error(f"Exception while running download-plugins.py :\n{format_exc()}") for plugin_tmp in glob(join(sep, "var", "tmp", "bunkerweb", "plugins", "*")): rmtree(plugin_tmp, ignore_errors=True) diff --git a/src/common/core/jobs/jobs/mmdb-asn.py b/src/common/core/jobs/jobs/mmdb-asn.py index d11e705c2..68abfeadf 100755 --- a/src/common/core/jobs/jobs/mmdb-asn.py +++ b/src/common/core/jobs/jobs/mmdb-asn.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -from datetime import date +from datetime import date, datetime, timedelta from gzip import decompress from hashlib import sha1 -from os import _exit, getenv, sep +from io import BytesIO +from os import getenv, sep from os.path import join from pathlib import Path from sys import exit as sys_exit, path as sys_path from threading import Lock from traceback import format_exc +from typing import Optional, Union for deps_path in [ join(sep, "usr", "share", "bunkerweb", *paths) @@ -21,107 +23,141 @@ for deps_path in [ if deps_path not in sys_path: sys_path.append(deps_path) -from maxminddb import open_database -from requests import RequestException, get +from maxminddb import MODE_FD, open_database +from requests import RequestException, Response, get -from Database import Database # type: ignore from logger import setup_logger # type: ignore -from jobs import cache_file, cache_hash, file_hash, is_cached_file +from common_utils import bytes_hash # type: ignore +from jobs import Job # type: ignore -logger = setup_logger("JOBS.mmdb-asn", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("JOBS.mmdb-asn", getenv("LOG_LEVEL", "INFO")) status = 0 -lock = Lock() +LOCK = Lock() + + +def request_mmdb() -> Optional[Response]: + try: + response = get("https://db-ip.com/db/download/ip-to-asn-lite", timeout=5) + response.raise_for_status() + return response + except RequestException: + return None + + +def bytes_sha1(bio: Union[Path, bytes, BytesIO]) -> str: + if isinstance(bio, Path): + bio = bio.read_bytes() + if isinstance(bio, bytes): + bio = BytesIO(bio) + + assert isinstance(bio, BytesIO) + + _sha512 = sha1() + while True: + data = bio.read(1024) + if not data: + break + _sha512.update(data) + bio.seek(0) + return _sha512.hexdigest() + try: dl_mmdb = True tmp_path = Path(sep, "var", "tmp", "bunkerweb", "asn.mmdb") - cache_path = Path(sep, "var", "cache", "bunkerweb", "asn.mmdb") new_hash = None # Don't go further if the cache match the latest version - if tmp_path.exists(): - with lock: - response = None - try: - response = get("https://db-ip.com/db/download/ip-to-asn-lite", timeout=5) - except RequestException: - logger.warning("Unable to check if asn.mmdb is the latest version") + response = None + if tmp_path.is_file(): + response = request_mmdb() if response and response.status_code == 200: - _sha1 = sha1() - with tmp_path.open("rb") as f: - while True: - data = f.read(1024) - if not data: - break - _sha1.update(data) - - if response.content.decode().find(_sha1.hexdigest()) != -1: - logger.info("asn.mmdb is already the latest version, skipping download...") + if response.content.find(bytes_sha1(tmp_path).encode()) != -1: + LOGGER.info("asn.mmdb is already the latest version, skipping download...") dl_mmdb = False else: - logger.warning("Unable to check if asn.mmdb is the latest version, downloading it anyway...") + LOGGER.warning("Unable to check if the temporary mmdb file is the latest version, downloading it anyway...") - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) + JOB = Job(LOGGER) if dl_mmdb: - # Don't go further if the cache is fresh - if is_cached_file(cache_path, "month", db): - logger.info("asn.mmdb is already in cache, skipping download...") - _exit(0) + job_cache = JOB.get_cache("asn.mmdb", with_info=True, with_data=True) + if isinstance(job_cache, dict): + skip_dl = True + if response is None: + response = request_mmdb() + + if response and response.status_code == 200: + skip_dl = response.content.find(bytes_sha1(job_cache["data"]).encode()) != -1 + elif job_cache["last_update"] < (datetime.now() - timedelta(weeks=1)).timestamp(): + LOGGER.warning("Unable to check if the cache file is the latest version from db-ip.com and file is older than 1 week, checking anyway...") + skip_dl = False + + if skip_dl: + LOGGER.info("asn.mmdb is already the latest version and is cached, skipping...") + sys_exit(0) # Compute the mmdb URL mmdb_url = f"https://download.db-ip.com/free/dbip-asn-lite-{date.today().strftime('%Y-%m')}.mmdb.gz" # Download the mmdb file and save it to tmp - logger.info(f"Downloading mmdb file from url {mmdb_url} ...") - file_content = b"" + LOGGER.info(f"Downloading mmdb file from url {mmdb_url} ...") + file_content = BytesIO() try: with get(mmdb_url, stream=True, timeout=5) as resp: resp.raise_for_status() for chunk in resp.iter_content(chunk_size=4 * 1024): if chunk: - file_content += chunk + file_content.write(chunk) except RequestException: - logger.error(f"Error while downloading mmdb file from {mmdb_url}") - _exit(2) + LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}") + sys_exit(2) try: assert file_content except AssertionError: - logger.error(f"Error while downloading mmdb file from {mmdb_url}") - _exit(2) + LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}") + sys_exit(2) # Decompress it - logger.info("Decompressing mmdb file ...") - tmp_path.write_bytes(decompress(file_content)) + LOGGER.info("Decompressing mmdb file ...") + file_content.seek(0) + content = BytesIO(decompress(file_content.getvalue())) - # Check if file has changed - new_hash = file_hash(tmp_path) - old_hash = cache_hash(cache_path, db) - if new_hash == old_hash: - logger.info("New file is identical to cache file, reload is not needed") - _exit(0) + if job_cache: + # Check if file has changed + new_hash = bytes_hash(content) + if new_hash == job_cache["checksum"]: + LOGGER.info("New file is identical to cache file, reload is not needed") + sys_exit(0) # Try to load it - logger.info("Checking if mmdb file is valid ...") - with open_database(str(tmp_path)) as reader: - pass + LOGGER.info("Checking if mmdb file is valid ...") + if tmp_path.is_file(): + with open_database(tmp_path.as_posix()) as reader: + pass + else: + with open_database(content, mode=MODE_FD) as reader: + pass + tmp_path = None # Move it to cache folder - logger.info("Moving mmdb file to cache ...") - cached, err = cache_file(tmp_path, cache_path, new_hash, db) + LOGGER.info("Moving mmdb file to cache ...") + cached, err = JOB.cache_file("asn.mmdb", tmp_path or content, checksum=new_hash) if not cached: - logger.error(f"Error while caching mmdb file : {err}") - _exit(2) + LOGGER.error(f"Error while caching mmdb file : {err}") + sys_exit(2) # Success if dl_mmdb: - logger.info(f"Downloaded new mmdb from {mmdb_url}") + LOGGER.info(f"Downloaded new mmdb from {mmdb_url}") status = 1 +except SystemExit as e: + status = e.code except: status = 2 - logger.error(f"Exception while running mmdb-asn.py :\n{format_exc()}") + LOGGER.error(f"Exception while running mmdb-asn.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/jobs/jobs/mmdb-country.py b/src/common/core/jobs/jobs/mmdb-country.py index d305748b2..574a2d55a 100755 --- a/src/common/core/jobs/jobs/mmdb-country.py +++ b/src/common/core/jobs/jobs/mmdb-country.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -from datetime import date +from datetime import date, datetime, timedelta from gzip import decompress from hashlib import sha1 -from os import _exit, getenv, sep +from io import BytesIO +from os import getenv, sep from os.path import join from pathlib import Path from sys import exit as sys_exit, path as sys_path from threading import Lock from traceback import format_exc +from typing import Optional, Union for deps_path in [ join(sep, "usr", "share", "bunkerweb", *paths) @@ -21,107 +23,141 @@ for deps_path in [ if deps_path not in sys_path: sys_path.append(deps_path) -from maxminddb import open_database -from requests import RequestException, get +from maxminddb import MODE_FD, open_database +from requests import RequestException, Response, get -from Database import Database # type: ignore from logger import setup_logger # type: ignore -from jobs import cache_file, cache_hash, file_hash, is_cached_file +from common_utils import bytes_hash # type: ignore +from jobs import Job # type: ignore -logger = setup_logger("JOBS.mmdb-country", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("JOBS.mmdb-country", getenv("LOG_LEVEL", "INFO")) status = 0 -lock = Lock() +LOCK = Lock() + + +def request_mmdb() -> Optional[Response]: + try: + response = get("https://db-ip.com/db/download/ip-to-country-lite", timeout=5) + response.raise_for_status() + return response + except RequestException: + return None + + +def bytes_sha1(bio: Union[Path, bytes, BytesIO]) -> str: + if isinstance(bio, Path): + bio = bio.read_bytes() + if isinstance(bio, bytes): + bio = BytesIO(bio) + + assert isinstance(bio, BytesIO) + + _sha512 = sha1() + while True: + data = bio.read(1024) + if not data: + break + _sha512.update(data) + bio.seek(0) + return _sha512.hexdigest() + try: dl_mmdb = True tmp_path = Path(sep, "var", "tmp", "bunkerweb", "country.mmdb") - cache_path = Path(sep, "var", "cache", "bunkerweb", "country.mmdb") new_hash = None # Don't go further if the cache match the latest version - if tmp_path.exists(): - with lock: - response = None - try: - response = get("https://db-ip.com/db/download/ip-to-country-lite", timeout=5) - except RequestException: - logger.warning("Unable to check if country.mmdb is the latest version") + response = None + if tmp_path.is_file(): + response = request_mmdb() if response and response.status_code == 200: - _sha1 = sha1() - with tmp_path.open("rb") as f: - while True: - data = f.read(1024) - if not data: - break - _sha1.update(data) - - if response.content.decode().find(_sha1.hexdigest()) != -1: - logger.info("country.mmdb is already the latest version, skipping download...") + if response.content.find(bytes_sha1(tmp_path).encode()) != -1: + LOGGER.info("country.mmdb is already the latest version, skipping download...") dl_mmdb = False else: - logger.warning("Unable to check if country.mmdb is the latest version, downloading it anyway...") + LOGGER.warning("Unable to check if the temporary mmdb file is the latest version, downloading it anyway...") - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) + JOB = Job(LOGGER) if dl_mmdb: - # Don't go further if the cache is fresh - if is_cached_file(cache_path, "month", db): - logger.info("country.mmdb is already in cache, skipping download...") - _exit(0) + job_cache = JOB.get_cache("country.mmdb", with_info=True, with_data=True) + if isinstance(job_cache, dict): + skip_dl = True + if response is None: + response = request_mmdb() + + if response and response.status_code == 200: + skip_dl = response.content.find(bytes_sha1(job_cache["data"]).encode()) != -1 + elif job_cache["last_update"] < (datetime.now() - timedelta(weeks=1)).timestamp(): + LOGGER.warning("Unable to check if the cache file is the latest version from db-ip.com and file is older than 1 week, checking anyway...") + skip_dl = False + + if skip_dl: + LOGGER.info("country.mmdb is already the latest version and is cached, skipping...") + sys_exit(0) # Compute the mmdb URL mmdb_url = f"https://download.db-ip.com/free/dbip-country-lite-{date.today().strftime('%Y-%m')}.mmdb.gz" # Download the mmdb file and save it to tmp - logger.info(f"Downloading mmdb file from url {mmdb_url} ...") - file_content = b"" + LOGGER.info(f"Downloading mmdb file from url {mmdb_url} ...") + file_content = BytesIO() try: with get(mmdb_url, stream=True, timeout=5) as resp: resp.raise_for_status() for chunk in resp.iter_content(chunk_size=4 * 1024): if chunk: - file_content += chunk + file_content.write(chunk) except RequestException: - logger.error(f"Error while downloading mmdb file from {mmdb_url}") - _exit(2) + LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}") + sys_exit(2) try: assert file_content except AssertionError: - logger.error(f"Error while downloading mmdb file from {mmdb_url}") - _exit(2) + LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}") + sys_exit(2) # Decompress it - logger.info("Decompressing mmdb file ...") - tmp_path.write_bytes(decompress(file_content)) + LOGGER.info("Decompressing mmdb file ...") + file_content.seek(0) + content = BytesIO(decompress(file_content.getvalue())) - # Check if file has changed - new_hash = file_hash(tmp_path) - old_hash = cache_hash(cache_path, db) - if new_hash == old_hash: - logger.info("New file is identical to cache file, reload is not needed") - _exit(0) + if job_cache: + # Check if file has changed + new_hash = bytes_hash(content) + if new_hash == job_cache["checksum"]: + LOGGER.info("New file is identical to cache file, reload is not needed") + sys_exit(0) # Try to load it - logger.info("Checking if mmdb file is valid ...") - with open_database(str(tmp_path)) as reader: - pass + LOGGER.info("Checking if mmdb file is valid ...") + if tmp_path.is_file(): + with open_database(tmp_path.as_posix()) as reader: + pass + else: + with open_database(content, mode=MODE_FD) as reader: + pass + tmp_path = None # Move it to cache folder - logger.info("Moving mmdb file to cache ...") - cached, err = cache_file(tmp_path, cache_path, new_hash, db) + LOGGER.info("Moving mmdb file to cache ...") + cached, err = JOB.cache_file("country.mmdb", tmp_path or content, checksum=new_hash) if not cached: - logger.error(f"Error while caching mmdb file : {err}") - _exit(2) + LOGGER.error(f"Error while caching mmdb file : {err}") + sys_exit(2) # Success if dl_mmdb: - logger.info(f"Downloaded new mmdb from {mmdb_url}") + LOGGER.info(f"Downloaded new mmdb from {mmdb_url}") status = 1 +except SystemExit as e: + status = e.code except: status = 2 - logger.error(f"Exception while running mmdb-country.py :\n{format_exc()}") + LOGGER.error(f"Exception while running mmdb-country.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/jobs/plugin.json b/src/common/core/jobs/plugin.json index bd4c5c6c9..c88083464 100644 --- a/src/common/core/jobs/plugin.json +++ b/src/common/core/jobs/plugin.json @@ -9,13 +9,13 @@ { "name": "mmdb-country", "file": "mmdb-country.py", - "every": "week", + "every": "day", "reload": true }, { "name": "mmdb-asn", "file": "mmdb-asn.py", - "every": "week", + "every": "day", "reload": true }, { diff --git a/src/common/core/letsencrypt/jobs/certbot-auth.py b/src/common/core/letsencrypt/jobs/certbot-auth.py index 307d2edff..0e0564d63 100755 --- a/src/common/core/letsencrypt/jobs/certbot-auth.py +++ b/src/common/core/letsencrypt/jobs/certbot-auth.py @@ -7,24 +7,16 @@ from sys import exit as sys_exit, path as sys_path from threading import Lock from traceback import format_exc -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("api",), - ("db",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) from Database import Database # type: ignore -from jobs import get_integration # type: ignore +from common_utils import get_integration # type: ignore from logger import setup_logger # type: ignore from API import API # type: ignore -logger = setup_logger("Lets-encrypt.auth", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("Lets-encrypt.auth", getenv("LOG_LEVEL", "INFO")) status = 0 try: @@ -34,7 +26,7 @@ try: # Cluster case if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"): - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) + db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) lock = Lock() with lock: @@ -45,12 +37,12 @@ try: sent, err, status, resp = api.request("POST", "/lets-encrypt/challenge", data={"token": token, "validation": validation}) if not sent: status = 1 - logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}") + LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}") elif status != 200: status = 1 - logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}") + LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}") else: - logger.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge") + LOGGER.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge") # Linux case else: @@ -59,6 +51,6 @@ try: root_dir.joinpath(token).write_text(validation, encoding="utf-8") except: status = 1 - logger.error(f"Exception while running certbot-auth.py :\n{format_exc()}") + LOGGER.error(f"Exception while running certbot-auth.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/letsencrypt/jobs/certbot-cleanup.py b/src/common/core/letsencrypt/jobs/certbot-cleanup.py index 9a54dc044..479655500 100755 --- a/src/common/core/letsencrypt/jobs/certbot-cleanup.py +++ b/src/common/core/letsencrypt/jobs/certbot-cleanup.py @@ -7,24 +7,16 @@ from sys import exit as sys_exit, path as sys_path from threading import Lock from traceback import format_exc -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("api",), - ("db",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) from Database import Database # type: ignore -from jobs import get_integration # type: ignore +from common_utils import get_integration # type: ignore from logger import setup_logger # type: ignore from API import API # type: ignore -logger = setup_logger("Lets-encrypt.cleanup", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("Lets-encrypt.cleanup", getenv("LOG_LEVEL", "INFO")) status = 0 try: @@ -33,7 +25,7 @@ try: # Cluster case if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"): - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) + db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) lock = Lock() with lock: instances = db.get_instances() @@ -43,17 +35,17 @@ try: sent, err, status, resp = api.request("DELETE", "/lets-encrypt/challenge", data={"token": token}) if not sent: status = 1 - logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}") + LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}") elif status != 200: status = 1 - logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}") + LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}") else: - logger.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge") + LOGGER.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge") # Linux case else: Path(sep, "var", "tmp", "bunkerweb", "lets-encrypt", ".well-known", "acme-challenge", token).unlink(missing_ok=True) except: status = 1 - logger.error(f"Exception while running certbot-cleanup.py :\n{format_exc()}") + LOGGER.error(f"Exception while running certbot-cleanup.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/letsencrypt/jobs/certbot-deploy.py b/src/common/core/letsencrypt/jobs/certbot-deploy.py index 0c1b6fcd2..f65d559bf 100755 --- a/src/common/core/letsencrypt/jobs/certbot-deploy.py +++ b/src/common/core/letsencrypt/jobs/certbot-deploy.py @@ -9,31 +9,23 @@ from tarfile import open as tar_open from threading import Lock from traceback import format_exc -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("api",), - ("db",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) from Database import Database # type: ignore -from jobs import get_integration # type: ignore +from common_utils import get_integration # type: ignore from logger import setup_logger # type: ignore from API import API # type: ignore -logger = setup_logger("Lets-encrypt.deploy", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("Lets-encrypt.deploy", getenv("LOG_LEVEL", "INFO")) status = 0 try: # Get env vars token = getenv("CERTBOT_TOKEN", "") - logger.info(f"Certificates renewal for {getenv('RENEWED_DOMAINS')} successful") + LOGGER.info(f"Certificates renewal for {getenv('RENEWED_DOMAINS')} successful") # Cluster case if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"): @@ -45,7 +37,7 @@ try: tgz.seek(0, 0) files = {"archive.tar.gz": tgz} - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) + db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) lock = Lock() with lock: @@ -59,32 +51,32 @@ try: sent, err, status, resp = api.request("POST", "/lets-encrypt/certificates", files=files) if not sent: status = 1 - logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/certificates : {err}") + LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/certificates : {err}") elif status != 200: status = 1 - logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/certificates : status = {resp['status']}, msg = {resp['msg']}") + LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/certificates : status = {resp['status']}, msg = {resp['msg']}") else: - logger.info( + LOGGER.info( f"Successfully sent API request to {api.endpoint}/lets-encrypt/certificates", ) sent, err, status, resp = api.request("POST", "/reload") if not sent: status = 1 - logger.error(f"Can't send API request to {api.endpoint}/reload : {err}") + LOGGER.error(f"Can't send API request to {api.endpoint}/reload : {err}") elif status != 200: status = 1 - logger.error(f"Error while sending API request to {api.endpoint}/reload : status = {resp['status']}, msg = {resp['msg']}") + LOGGER.error(f"Error while sending API request to {api.endpoint}/reload : status = {resp['status']}, msg = {resp['msg']}") else: - logger.info(f"Successfully sent API request to {api.endpoint}/reload") + LOGGER.info(f"Successfully sent API request to {api.endpoint}/reload") # Linux case else: if run([join(sep, "usr", "sbin", "nginx"), "-s", "reload"], stdin=DEVNULL, stderr=STDOUT, check=False).returncode != 0: status = 1 - logger.error("Error while reloading nginx") + LOGGER.error("Error while reloading nginx") else: - logger.info("Successfully reloaded nginx") + LOGGER.info("Successfully reloaded nginx") except: status = 1 - logger.error(f"Exception while running certbot-deploy.py :\n{format_exc()}") + LOGGER.error(f"Exception while running certbot-deploy.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index 074a53be9..dc62e7d7f 100755 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -1,32 +1,22 @@ #!/usr/bin/env python3 -from os import _exit, environ, getenv, sep +from os import environ, getenv, sep from os.path import join from pathlib import Path -from subprocess import DEVNULL, STDOUT, run, PIPE +from subprocess import DEVNULL, STDOUT, Popen, run, PIPE from sys import exit as sys_exit, path as sys_path from traceback import format_exc -from tarfile import open as tar_open -from io import BytesIO -from shutil import rmtree from re import MULTILINE, search -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("db",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) -from Database import Database # type: ignore from logger import setup_logger # type: ignore -from jobs import get_file_in_db, set_file_in_db # type: ignore +from jobs import Job # type: ignore -logger = setup_logger("LETS-ENCRYPT.new", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("LETS-ENCRYPT.new", getenv("LOG_LEVEL", "INFO")) +LOGGER_CERTBOT = setup_logger("LETS-ENCRYPT.new.certbot", getenv("LOG_LEVEL", "INFO")) status = 0 CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot") @@ -38,7 +28,7 @@ LETS_ENCRYPT_LOGS_DIR = join(sep, "var", "log", "bunkerweb") def certbot_new(domains: str, email: str, use_letsencrypt_staging: bool = False) -> int: - return run( + with Popen( [ CERTBOT_BIN, "certonly", @@ -64,9 +54,14 @@ def certbot_new(domains: str, email: str, use_letsencrypt_staging: bool = False) ] + (["--staging"] if use_letsencrypt_staging else []), stdin=DEVNULL, - stderr=STDOUT, + stderr=PIPE, + universal_newlines=True, env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")}, - ).returncode + ) as process: + if process.stderr: + for line in process.stderr: + LOGGER_CERTBOT.info(line.strip()) + return process.returncode status = 0 @@ -82,36 +77,21 @@ try: use_letsencrypt = True elif is_multisite: for first_server in server_names: - if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes": + if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", getenv("AUTO_LETS_ENCRYPT", "no")) == "yes": use_letsencrypt = True break if not use_letsencrypt: - logger.info("Let's Encrypt is not activated, skipping generation...") - _exit(0) + LOGGER.info("Let's Encrypt is not activated, skipping generation...") + sys_exit(0) elif not getenv("SERVER_NAME"): - logger.warning("There are no server names, skipping generation...") - _exit(0) + LOGGER.warning("There are no server names, skipping generation...") + sys_exit(0) - # Create directories if they doesn't exist - LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True) - Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(parents=True, exist_ok=True) + JOB = Job(LOGGER) - # Extract letsencrypt folder if it exists in db - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False) - - tgz = get_file_in_db("folder.tgz", db, job_name="certbot-renew") - if tgz: - # Delete folder if needed - if LETS_ENCRYPT_PATH.exists(): - rmtree(LETS_ENCRYPT_PATH, ignore_errors=True) - LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True) - # Extract it - with tar_open(name="folder.tgz", mode="r:gz", fileobj=BytesIO(tgz)) as tf: - tf.extractall(LETS_ENCRYPT_PATH) - logger.info("Successfully retrieved Let's Encrypt data from db cache") - else: - logger.info("No Let's Encrypt data found in db cache") + # Restore Let's Encrypt data from db cache + JOB.restore_cache(job_name="certbot-renew") domains_to_ask = [] # Multisite case @@ -142,11 +122,12 @@ try: stderr=STDOUT, text=True, env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")}, + check=False, ) stdout = proc.stdout if proc.returncode != 0: - logger.error(f"Error while checking certificates :\n{proc.stdout}") + LOGGER.error(f"Error while checking certificates :\n{proc.stdout}") domains_to_ask = server_names else: for first_server, domains in domains_sever_names.items(): @@ -155,10 +136,10 @@ try: domains_to_ask.append(first_server) continue elif set(f"{first_server}{current_domains.groupdict()['domains']}".strip().split(" ")) != set(domains.split(" ")): - logger.warning(f"Domains for {first_server} are not the same as in the certificate, asking new certificate...") + LOGGER.warning(f"Domains for {first_server} are not the same as in the certificate, asking new certificate...") domains_to_ask.append(first_server) continue - logger.info(f"Certificates already exists for domain(s) {domains}") + LOGGER.info(f"Certificates already exists for domain(s) {domains}") for first_server, domains in domains_sever_names.items(): if first_server not in domains_to_ask: @@ -170,30 +151,26 @@ try: use_letsencrypt_staging = getenv(f"{first_server}_USE_LETS_ENCRYPT_STAGING", getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes" - logger.info(f"Asking certificates for domain(s) : {domains} (email = {real_email}) to Let's Encrypt {'staging ' if use_letsencrypt_staging else ''}...") + LOGGER.info(f"Asking certificates for domain(s) : {domains} (email = {real_email}) to Let's Encrypt {'staging ' if use_letsencrypt_staging else ''}...") if certbot_new(domains.replace(" ", ","), real_email, use_letsencrypt_staging) != 0: status = 2 - logger.error(f"Certificate generation failed for domain(s) {domains} ...") + LOGGER.error(f"Certificate generation failed for domain(s) {domains} ...") continue else: status = 1 if status == 0 else status - logger.info(f"Certificate generation succeeded for domain(s) : {domains}") + LOGGER.info(f"Certificate generation succeeded for domain(s) : {domains}") - # Put new folder in cache - bio = BytesIO() - with tar_open("folder.tgz", mode="w:gz", fileobj=bio, compresslevel=9) as tgz: - tgz.add(LETS_ENCRYPT_PATH, arcname=".") - bio.seek(0, 0) - - # Put tgz in cache - cached, err = set_file_in_db("folder.tgz", bio.read(), db, job_name="certbot-renew") - - if not cached: - logger.error(f"Error while saving Let's Encrypt data to db cache : {err}") - else: - logger.info("Successfully saved Let's Encrypt data to db cache") + # Save Let's Encrypt data to db cache + if LETS_ENCRYPT_PATH.is_dir() and list(LETS_ENCRYPT_PATH.iterdir()): + cached, err = JOB.cache_dir(LETS_ENCRYPT_PATH, job_name="certbot-renew") + if not cached: + LOGGER.error(f"Error while saving Let's Encrypt data to db cache : {err}") + else: + LOGGER.info("Successfully saved Let's Encrypt data to db cache") +except SystemExit as e: + status = e.code except: status = 3 - logger.error(f"Exception while running certbot-new.py :\n{format_exc()}") + LOGGER.error(f"Exception while running certbot-new.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index 13205ee37..59c22da5a 100755 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -1,14 +1,11 @@ #!/usr/bin/env python3 -from os import _exit, environ, getenv, sep +from os import environ, getenv, sep from os.path import join from pathlib import Path -from subprocess import DEVNULL, STDOUT, run +from subprocess import DEVNULL, PIPE, Popen from sys import exit as sys_exit, path as sys_path from traceback import format_exc -from tarfile import open as tar_open -from io import BytesIO -from shutil import rmtree for deps_path in [ join(sep, "usr", "share", "bunkerweb", *paths) @@ -21,14 +18,18 @@ for deps_path in [ if deps_path not in sys_path: sys_path.append(deps_path) -from Database import Database # type: ignore from logger import setup_logger # type: ignore -from jobs import get_file_in_db, set_file_in_db # type: ignore +from jobs import Job # type: ignore -logger = setup_logger("LETS-ENCRYPT.renew", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("LETS-ENCRYPT.renew", getenv("LOG_LEVEL", "INFO")) +LOGGER_CERTBOT = setup_logger("LETS-ENCRYPT.renew.certbot", getenv("LOG_LEVEL", "INFO")) status = 0 +CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot") + LETS_ENCRYPT_PATH = Path(sep, "var", "cache", "bunkerweb", "letsencrypt") +LETS_ENCRYPT_WORK_DIR = join(sep, "var", "lib", "bunkerweb", "letsencrypt") +LETS_ENCRYPT_LOGS_DIR = join(sep, "var", "log", "bunkerweb") try: # Check if we're using let's encrypt @@ -42,66 +43,47 @@ try: break if not use_letsencrypt: - logger.info("Let's Encrypt is not activated, skipping renew...") - _exit(0) + LOGGER.info("Let's Encrypt is not activated, skipping renew...") + sys_exit(0) - # Create directory if it doesn't exist - LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True) - Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(parents=True, exist_ok=True) + JOB = Job(LOGGER) - # Extract letsencrypt folder if it exists in db - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False) + with Popen( + [ + CERTBOT_BIN, + "renew", + "--no-random-sleep-on-renew", + "--config-dir", + LETS_ENCRYPT_PATH.joinpath("etc").as_posix(), + "--work-dir", + LETS_ENCRYPT_WORK_DIR, + "--logs-dir", + LETS_ENCRYPT_LOGS_DIR, + ], + stdin=DEVNULL, + stderr=PIPE, + universal_newlines=True, + env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")}, + ) as process: + if process.stderr: + for line in process.stderr: + LOGGER_CERTBOT.info(line.strip()) - tgz = get_file_in_db("folder.tgz", db) - if tgz: - # Delete folder if needed - if LETS_ENCRYPT_PATH.exists(): - rmtree(LETS_ENCRYPT_PATH, ignore_errors=True) - LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True) - # Extract it - with tar_open(name="folder.tgz", mode="r:gz", fileobj=BytesIO(tgz)) as tf: - tf.extractall(LETS_ENCRYPT_PATH) - logger.info("Successfully retrieved Let's Encrypt data from db cache") - else: - logger.info("No Let's Encrypt data found in db cache") + if process.returncode == 0: + status = 2 + LOGGER.error("Certificates renewal failed") - if ( - run( - [ - join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot"), - "renew", - "--no-random-sleep-on-renew", - "--config-dir", - LETS_ENCRYPT_PATH.joinpath("etc").as_posix(), - "--work-dir", - join(sep, "var", "lib", "bunkerweb", "letsencrypt"), - "--logs-dir", - join(sep, "var", "log", "bunkerweb"), - ], - stdin=DEVNULL, - stderr=STDOUT, - env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")}, - check=False, - ).returncode - != 0 - ): - status = 2 - logger.error("Certificates renewal failed") - - # Put new folder in cache - bio = BytesIO() - with tar_open("folder.tgz", mode="w:gz", fileobj=bio, compresslevel=9) as tgz: - tgz.add(LETS_ENCRYPT_PATH, arcname=".") - bio.seek(0, 0) - - # Put tgz in cache - cached, err = set_file_in_db("folder.tgz", bio.read(), db) - if not cached: - logger.error(f"Error while saving Let's Encrypt data to db cache : {err}") - else: - logger.info("Successfully saved Let's Encrypt data to db cache") + # Save Let's Encrypt data to db cache + if LETS_ENCRYPT_PATH.is_dir() and list(LETS_ENCRYPT_PATH.iterdir()): + cached, err = JOB.cache_dir(LETS_ENCRYPT_PATH) + if not cached: + LOGGER.error(f"Error while saving Let's Encrypt data to db cache : {err}") + else: + LOGGER.info("Successfully saved Let's Encrypt data to db cache") +except SystemExit as e: + status = e.code except: status = 2 - logger.error(f"Exception while running certbot-renew.py :\n{format_exc()}") + LOGGER.error(f"Exception while running certbot-renew.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/misc/jobs/anonymous-report.py b/src/common/core/misc/jobs/anonymous-report.py index ecb0cc1a3..2b95c2208 100644 --- a/src/common/core/misc/jobs/anonymous-report.py +++ b/src/common/core/misc/jobs/anonymous-report.py @@ -3,8 +3,6 @@ from json import dumps from os import getenv, sep from os.path import join -from pathlib import Path -from platform import machine from re import compile as re_compile from sys import exit as sys_exit, path as sys_path, version from traceback import format_exc @@ -14,33 +12,28 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (( if deps_path not in sys_path: sys_path.append(deps_path) -from Database import Database # type: ignore +from common_utils import get_os_info # type: ignore from logger import setup_logger # type: ignore -from jobs import cache_file, is_cached_file # type: ignore +from jobs import Job # type: ignore from requests import post -logger = setup_logger("ANONYMOUS-REPORT", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("ANONYMOUS-REPORT", getenv("LOG_LEVEL", "INFO")) status = 0 -if getenv("SEND_ANONYMOUS_REPORT", "yes") != "yes": - logger.info("Skipping the sending of anonymous report (disabled)") - sys_exit(status) - -anonymous_report_path = Path(sep, "var", "cache", "bunkerweb", "anonymous_report") -anonymous_report_path.mkdir(parents=True, exist_ok=True) -tmp_anonymous_report_path = Path(sep, "var", "tmp", "bunkerweb", "anonymous_report") -tmp_anonymous_report_path.mkdir(parents=True, exist_ok=True) - try: - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) - if is_cached_file(anonymous_report_path.joinpath("last_report.json"), "day", db): - logger.info("Skipping the sending of anonymous report (already sent today)") + if getenv("SEND_ANONYMOUS_REPORT", "yes") != "yes": + LOGGER.info("Skipping the sending of anonymous report (disabled)") + sys_exit(status) + + JOB = Job(LOGGER) + if JOB.is_cached_file("last_report.json", "day"): + LOGGER.info("Skipping the sending of anonymous report (already sent today)") sys_exit(0) # ? Get version and integration of BunkerWeb - data: Dict[str, Any] = db.get_metadata() + data: Dict[str, Any] = JOB.db.get_metadata() data["is_pro"] = "yes" if data["is_pro"] else "no" @@ -48,7 +41,7 @@ try: if key not in ("version", "integration", "database_version", "is_pro"): data.pop(key, None) - db_config = db.get_config(methods=True, with_drafts=True) + db_config = JOB.db.get_config(methods=True, with_drafts=True) services = db_config.get("SERVER_NAME", {"value": ""})["value"].split(" ") multisite = db_config.get("MULTISITE", {"value": "no"})["value"] == "yes" @@ -58,7 +51,7 @@ try: database_version = database_version.group(1) data["integration"] = data["integration"].lower() - data["database"] = f"{db.database_uri.split(':')[0].split('+')[0]}/{database_version}" + data["database"] = f"{JOB.db.database_uri.split(':')[0].split('+')[0]}/{database_version}" data["service_number"] = str(len(services)) data["draft_service_number"] = 0 data["python_version"] = version.split(" ")[0] @@ -82,26 +75,13 @@ try: data["external_plugins"] = [] data["pro_plugins"] = [] - for plugin in db.get_plugins(): + for plugin in JOB.db.get_plugins(): if plugin["type"] == "external": data["external_plugins"].append(f"{plugin['id']}/{plugin['version']}") elif plugin["type"] == "pro": data["pro_plugins"].append(f"{plugin['id']}/{plugin['version']}") - data["os"] = { - "name": "Linux", - "version": "Unknown", - "version_id": "Unknown", - "version_codename": "Unknown", - "id": "Unknown", - "arch": machine(), - } - os_release = Path("/etc/os-release") - if os_release.exists(): - for line in os_release.read_text().splitlines(): - if "=" not in line or line.split("=")[0].strip().lower() not in data["os"]: - continue - data["os"][line.split("=")[0].lower()] = line.split("=")[1].strip('"') + data["os"] = get_os_info() data["non_default_settings"] = {} for setting, setting_data in db_config.items(): @@ -121,18 +101,19 @@ try: for key in data["non_default_settings"].copy(): data["non_default_settings"][key] = str(data["non_default_settings"][key]) - data["bw_instances_number"] = str(len(db.get_instances())) - - tmp_anonymous_report_path.joinpath("last_report.json").write_text(dumps(data, indent=4), encoding="utf-8") + data["bw_instances_number"] = str(len(JOB.db.get_instances())) response = post("https://api.bunkerweb.io/data", json=data, headers={"User-Agent": f"BunkerWeb/{data['version']}"}, allow_redirects=True, timeout=10) response.raise_for_status() - cached, err = cache_file(tmp_anonymous_report_path.joinpath("last_report.json"), anonymous_report_path.joinpath("last_report.json"), None, db) + cached, err = JOB.cache_file("last_report.json", dumps(data, indent=4).encode()) + if not cached: + LOGGER.error(f"Failed to cache last_report.json :\n{err}") + status = 2 except SystemExit as e: status = e.code except: status = 2 - logger.error(f"Exception while running anonymous-report.py :\n{format_exc()}") + LOGGER.error(f"Exception while running anonymous-report.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/misc/jobs/default-server-cert.py b/src/common/core/misc/jobs/default-server-cert.py index 9435cc968..43336e8f0 100644 --- a/src/common/core/misc/jobs/default-server-cert.py +++ b/src/common/core/misc/jobs/default-server-cert.py @@ -7,29 +7,24 @@ from subprocess import DEVNULL, run from sys import exit as sys_exit, path as sys_path from traceback import format_exc -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("db",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) -from Database import Database # type: ignore from logger import setup_logger # type: ignore -from jobs import set_file_in_db +from jobs import Job # type: ignore -logger = setup_logger("DEFAULT-SERVER-CERT", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("DEFAULT-SERVER-CERT", getenv("LOG_LEVEL", "INFO")) +LOGGER_OPENSSL = setup_logger("DEFAULT-SERVER-CERT.openssl", getenv("LOG_LEVEL", "INFO")) status = 0 try: - cert_path = Path(sep, "var", "cache", "bunkerweb", "default-server-cert") - cert_path.mkdir(parents=True, exist_ok=True) - if not cert_path.joinpath("cert.pem").is_file(): - logger.info("Generating self-signed certificate for default server") + JOB = Job(LOGGER) + + cert_path = Path(sep, "var", "cache", "bunkerweb", "misc") + if not JOB.is_cached_file("default-server-cert.pem", "month") or not JOB.is_cached_file("default-server-cert.key", "month"): + LOGGER.info("Generating self-signed certificate for default server") + cert_path.mkdir(parents=True, exist_ok=True) if ( run( @@ -41,9 +36,9 @@ try: "-newkey", "ed25519", "-keyout", - str(cert_path.joinpath("cert.key")), + str(cert_path.joinpath("default-server-cert.key")), "-out", - str(cert_path.joinpath("cert.pem")), + str(cert_path.joinpath("default-server-cert.pem")), "-days", "3650", "-subj", @@ -55,43 +50,27 @@ try: ).returncode != 0 ): - logger.error( - "Self-signed certificate generation failed for default server", - ) + LOGGER.error("Self-signed certificate generation failed for default server") status = 2 else: + LOGGER.info("Successfully generated self-signed certificate for default server") status = 1 - logger.info( - "Successfully generated self-signed certificate for default server", - ) - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) - - cached, err = set_file_in_db( - "cert.pem", - cert_path.joinpath("cert.pem").read_bytes(), - db, - ) + cached, err = JOB.cache_file("default-server-cert.pem", cert_path.joinpath("default-server-cert.pem"), overwrite_file=False) if not cached: - logger.error(f"Error while saving default-server-cert cert.pem file to db cache : {err}") + LOGGER.error(f"Error while saving default-server-cert default-server-cert.pem file to db cache : {err}") else: - logger.info("Successfully saved default-server-cert cert.pem file to db cache") + LOGGER.info("Successfully saved default-server-cert default-server-cert.pem file to db cache") - cached, err = set_file_in_db( - "cert.key", - cert_path.joinpath("cert.key").read_bytes(), - db, - ) + cached, err = JOB.cache_file("default-server-cert.key", cert_path.joinpath("default-server-cert.key"), overwrite_file=False) if not cached: - logger.error(f"Error while saving default-server-cert cert.key file to db cache : {err}") + LOGGER.error(f"Error while saving default-server-cert default-server-cert.key file to db cache : {err}") else: - logger.info("Successfully saved default-server-cert cert.key file to db cache") + LOGGER.info("Successfully saved default-server-cert default-server-cert.key file to db cache") else: - logger.info( - "Skipping generation of self-signed certificate for default server (already present)", - ) + LOGGER.info("Skipping generation of self-signed certificate for default server (already present)") except: status = 2 - logger.error(f"Exception while running default-server-cert.py :\n{format_exc()}") + LOGGER.error(f"Exception while running default-server-cert.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/misc/jobs/update-check.py b/src/common/core/misc/jobs/update-check.py index 3c60f4927..1ebbfedcf 100644 --- a/src/common/core/misc/jobs/update-check.py +++ b/src/common/core/misc/jobs/update-check.py @@ -2,46 +2,34 @@ from os import getenv, sep from os.path import basename, join -from pathlib import Path from sys import exit as sys_exit, path as sys_path from traceback import format_exc -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",))]: if deps_path not in sys_path: sys_path.append(deps_path) from requests import get + +from common_utils import get_version # type: ignore from logger import setup_logger # type: ignore -logger = setup_logger("UPDATE-CHECK", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("UPDATE-CHECK", getenv("LOG_LEVEL", "INFO")) status = 0 try: - current_version = f"v{Path('/usr/share/bunkerweb/VERSION').read_text(encoding='utf-8').strip()}" + current_version = f"v{get_version().strip()}" - response = get( - "https://github.com/bunkerity/bunkerweb/releases/latest", - headers={"User-Agent": "BunkerWeb"}, - allow_redirects=True, - timeout=10, - ) + response = get("https://github.com/bunkerity/bunkerweb/releases/latest", headers={"User-Agent": "BunkerWeb"}, allow_redirects=True, timeout=10) response.raise_for_status() latest_version = basename(response.url) if current_version != latest_version: - logger.warning( - f"* \n* \n* 🚨 A new version of BunkerWeb is available: {latest_version} (current: {current_version}) 🚨\n* \n* ", - ) + LOGGER.warning(f"* \n* \n* 🚨 A new version of BunkerWeb is available: {latest_version} (current: {current_version}) 🚨\n* \n* ") else: - logger.info(f"Latest version is already installed: {current_version}") + LOGGER.info(f"Latest version is already installed: {current_version}") except: status = 2 - logger.error(f"Exception while running update-check.py :\n{format_exc()}") + LOGGER.error(f"Exception while running update-check.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/pro/jobs/download-pro-plugins.py b/src/common/core/pro/jobs/download-pro-plugins.py index f1be13c00..3b1d8ac02 100644 --- a/src/common/core/pro/jobs/download-pro-plugins.py +++ b/src/common/core/pro/jobs/download-pro-plugins.py @@ -17,15 +17,7 @@ from tarfile import open as tar_open from traceback import format_exc from zipfile import ZipFile -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("api",), - ("db",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) @@ -33,7 +25,7 @@ from requests import get from Database import Database # type: ignore from logger import setup_logger # type: ignore -from jobs import get_os_info, get_integration, get_version # type: ignore +from common_utils import get_os_info, get_integration, get_version # type: ignore API_ENDPOINT = "https://api.bunkerweb.io" PREVIEW_ENDPOINT = "https://assets.bunkerity.com/bw-pro/preview" @@ -44,12 +36,12 @@ STATUS_MESSAGES = { "expired": "has expired", "suspended": "has been suspended", } -logger = setup_logger("Jobs.download-pro-plugins", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("Jobs.download-pro-plugins", getenv("LOG_LEVEL", "INFO")) status = 0 def clean_pro_plugins(db) -> None: - logger.debug("Cleaning up Pro plugins...") + LOGGER.debug("Cleaning up Pro plugins...") # Clean Pro plugins rmtree(PRO_PLUGINS_DIR.joinpath("*"), ignore_errors=True) # Update database @@ -61,14 +53,14 @@ def install_plugin(plugin_dir: str, db, preview: bool = True) -> bool: plugin_file = plugin_path.joinpath("plugin.json") if not plugin_file.is_file(): - logger.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json not found)") + LOGGER.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json not found)") return False # Load plugin.json try: metadata = loads(plugin_file.read_text(encoding="utf-8")) except JSONDecodeError: - logger.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json is not valid)") + LOGGER.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json is not valid)") return False # Don't go further if plugin is already installed @@ -81,12 +73,12 @@ def install_plugin(plugin_dir: str, db, preview: bool = True) -> bool: break if old_version == metadata["version"]: - logger.warning( + LOGGER.warning( f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {metadata['id']} (version {metadata['version']} already installed)" ) return False - logger.warning( + LOGGER.warning( f"{'Preview version of ' if preview else ''}Pro plugin {metadata['id']} is already installed but version {metadata['version']} is different from database ({old_version}), updating it..." ) rmtree(PRO_PLUGINS_DIR.joinpath(metadata["id"]), ignore_errors=True) @@ -97,21 +89,21 @@ def install_plugin(plugin_dir: str, db, preview: bool = True) -> bool: for job_file in glob(PRO_PLUGINS_DIR.joinpath(metadata["id"], "jobs", "*").as_posix()): st = Path(job_file).stat() chmod(job_file, st.st_mode | S_IEXEC) - logger.info(f"✅ {'Preview version of ' if preview else ''}Pro plugin {metadata['id']} (version {metadata['version']}) installed successfully!") + LOGGER.info(f"✅ {'Preview version of ' if preview else ''}Pro plugin {metadata['id']} (version {metadata['version']}) installed successfully!") return True try: - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False) + db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"), pool=False) db_metadata = db.get_metadata() current_date = datetime.now() # If we already checked in the last 10 minutes, skip the check if db_metadata["last_pro_check"] and (current_date - db_metadata["last_pro_check"]).seconds < 600: - logger.info("Skipping the check for BunkerWeb Pro license (already checked in the last 10 minutes)") + LOGGER.info("Skipping the check for BunkerWeb Pro license (already checked in the last 10 minutes)") sys_exit(0) - logger.info("Checking BunkerWeb Pro license key...") + LOGGER.info("Checking BunkerWeb Pro license key...") data = { "integration": get_integration(), @@ -135,24 +127,24 @@ try: temp_dir.mkdir(parents=True, exist_ok=True) if pro_license_key: - logger.info("BunkerWeb Pro license provided, checking if it's valid...") + LOGGER.info("BunkerWeb Pro license provided, checking if it's valid...") headers["Authorization"] = f"Bearer {pro_license_key.strip()}" resp = get(f"{API_ENDPOINT}/pro-status", headers=headers, json=data, timeout=5, allow_redirects=True) if resp.status_code == 403: - logger.error(f"Access denied to {API_ENDPOINT}/pro-status - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/") + LOGGER.error(f"Access denied to {API_ENDPOINT}/pro-status - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/") error = True if db_metadata["is_pro"]: clean_pro_plugins(db) elif resp.status_code == 500: - logger.error("An error occurred with the remote server while checking BunkerWeb Pro license, please try again later") + LOGGER.error("An error occurred with the remote server while checking BunkerWeb Pro license, please try again later") status = 2 sys_exit(status) else: resp.raise_for_status() metadata = resp.json()["data"] - logger.debug(f"Got BunkerWeb Pro license metadata: {metadata}") + LOGGER.debug(f"Got BunkerWeb Pro license metadata: {metadata}") metadata["pro_expire"] = datetime.strptime(metadata["pro_expire"], "%Y-%m-%d") if metadata["pro_expire"] else None if metadata["pro_expire"] and metadata["pro_expire"] < datetime.now(): metadata["pro_status"] = "expired" @@ -164,7 +156,7 @@ try: db.set_pro_metadata(metadata | {"last_pro_check": current_date}) if metadata["is_pro"]: - logger.info("🚀 Your BunkerWeb Pro license is valid, checking if there are new or updated Pro plugins...") + LOGGER.info("🚀 Your BunkerWeb Pro license is valid, checking if there are new or updated Pro plugins...") if not db_metadata["is_pro"]: clean_pro_plugins(db) @@ -172,13 +164,13 @@ try: resp = get(f"{API_ENDPOINT}/pro", headers=headers, json=data, timeout=5, allow_redirects=True) if resp.status_code == 403: - logger.error(f"Access denied to {API_ENDPOINT}/pro - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/") + LOGGER.error(f"Access denied to {API_ENDPOINT}/pro - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/") error = True metadata = default_metadata db.set_pro_metadata(metadata | {"last_pro_check": current_date}) clean_pro_plugins(db) elif resp.headers.get("Content-Type", "") != "application/octet-stream": - logger.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {API_ENDPOINT}/pro") + LOGGER.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {API_ENDPOINT}/pro") status = 2 sys_exit(status) @@ -192,9 +184,9 @@ try: STATUS_MESSAGES.get(metadata["pro_status"], "is not valid or has expired") if not error else "is not valid or has expired" ) else: - logger.info("If you wish to purchase a BunkerWeb Pro license, please visit https://panel.bunkerweb.io/") + LOGGER.info("If you wish to purchase a BunkerWeb Pro license, please visit https://panel.bunkerweb.io/") message = "No BunkerWeb Pro license key provided" - logger.warning(f"{message}, only checking if there are new or updated preview versions of Pro plugins...") + LOGGER.warning(f"{message}, only checking if there are new or updated preview versions of Pro plugins...") if metadata["is_pro"]: clean_pro_plugins(db) @@ -202,16 +194,16 @@ try: resp = get(f"{PREVIEW_ENDPOINT}/v{data['version']}.zip", timeout=5, allow_redirects=True) if resp.status_code == 404: - logger.error(f"Couldn't find Pro plugins for BunkerWeb version {data['version']} at {PREVIEW_ENDPOINT}/v{data['version']}.zip") + LOGGER.error(f"Couldn't find Pro plugins for BunkerWeb version {data['version']} at {PREVIEW_ENDPOINT}/v{data['version']}.zip") status = 2 sys_exit(status) elif resp.headers.get("Content-Type", "") != "application/zip": - logger.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {PREVIEW_ENDPOINT}/v{data['version']}.zip") + LOGGER.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {PREVIEW_ENDPOINT}/v{data['version']}.zip") status = 2 sys_exit(status) if resp.status_code == 500: - logger.error("An error occurred with the remote server, please try again later") + LOGGER.error("An error occurred with the remote server, please try again later") status = 2 sys_exit(status) resp.raise_for_status() @@ -229,14 +221,14 @@ try: if install_plugin(plugin_dir, db, not metadata["is_pro"]): plugin_nbr += 1 except FileExistsError: - logger.warning(f"Skipping installation of pro plugin {basename(plugin_dir)} (already installed)") + LOGGER.warning(f"Skipping installation of pro plugin {basename(plugin_dir)} (already installed)") except: - logger.exception("Exception while installing pro plugin(s)") + LOGGER.exception("Exception while installing pro plugin(s)") status = 2 sys_exit(status) if not plugin_nbr: - logger.info("All Pro plugins are up to date") + LOGGER.info("All Pro plugins are up to date") sys_exit(0) pro_plugins = [] @@ -244,7 +236,7 @@ try: for plugin in listdir(PRO_PLUGINS_DIR): path = PRO_PLUGINS_DIR.joinpath(plugin) if not path.joinpath("plugin.json").is_file(): - logger.warning(f"Plugin {plugin} is not valid, deleting it...") + LOGGER.warning(f"Plugin {plugin} is not valid, deleting it...") rmtree(path, ignore_errors=True) continue @@ -282,15 +274,15 @@ try: err = db.update_external_plugins(pro_plugins, _type="pro") if err: - logger.error(f"Couldn't update Pro plugins to database: {err}") + LOGGER.error(f"Couldn't update Pro plugins to database: {err}") status = 1 - logger.info("🚀 Pro plugins downloaded and installed successfully!") + LOGGER.info("🚀 Pro plugins downloaded and installed successfully!") except SystemExit as e: status = e.code except: status = 2 - logger.error(f"Exception while running download-pro-plugins.py :\n{format_exc()}") + LOGGER.error(f"Exception while running download-pro-plugins.py :\n{format_exc()}") for plugin_tmp in glob(TMP_DIR.joinpath("*").as_posix()): rmtree(plugin_tmp, ignore_errors=True) diff --git a/src/common/core/realip/jobs/realip-download.py b/src/common/core/realip/jobs/realip-download.py index 9423510eb..0c36bd558 100755 --- a/src/common/core/realip/jobs/realip-download.py +++ b/src/common/core/realip/jobs/realip-download.py @@ -2,43 +2,35 @@ from contextlib import suppress from ipaddress import ip_address, ip_network -from os import _exit, getenv, sep +from os import getenv, sep from os.path import join, normpath -from pathlib import Path from sys import exit as sys_exit, path as sys_path from traceback import format_exc -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("db",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) from requests import get -from Database import Database # type: ignore from logger import setup_logger # type: ignore -from jobs import cache_file, cache_hash, del_file_in_db, file_hash, is_cached_file +from common_utils import bytes_hash # type: ignore +from jobs import Job # type: ignore def check_line(line): - if "/" in line: - with suppress(ValueError): + with suppress(ValueError): + if "/" in line: ip_network(line) return True, line - else: - with suppress(ValueError): + else: ip_address(line) return True, line return False, b"" -logger = setup_logger("REALIP", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("REALIP", getenv("LOG_LEVEL", "INFO")) +REALIP_CACHE_PATH = join(sep, "var", "cache", "bunkerweb", "realip") status = 0 try: @@ -61,37 +53,34 @@ try: realip_activated = True if not realip_activated: - logger.info("RealIP is not activated, skipping download...") - _exit(0) + LOGGER.info("RealIP is not activated, skipping download...") + sys_exit(0) - # Create directories if they don't exist - realip_path = Path(sep, "var", "cache", "bunkerweb", "realip") - realip_path.mkdir(parents=True, exist_ok=True) - tmp_realip_path = Path(sep, "var", "tmp", "bunkerweb", "realip") - tmp_realip_path.mkdir(parents=True, exist_ok=True) - - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) + JOB = Job(LOGGER) # Get URLs urls = [url for url in getenv("REAL_IP_FROM_URLS", "").split(" ") if url] # Don't go further if the cache is fresh - if is_cached_file(realip_path.joinpath("combined.list"), "hour", db): - logger.info("RealIP list is already in cache, skipping download...") + if JOB.is_cached_file("combined.list", "hour"): + LOGGER.info("RealIP list is already in cache, skipping download...") if not urls: - logger.warning("No URL found, deleting combined.list from cache...") - tmp_realip_path.joinpath("combined.list").unlink(missing_ok=True) - deleted, err = del_file_in_db("combined.list", db) + LOGGER.warning("No URL found, deleting combined.list from cache...") + deleted, err = JOB.del_cache("combined.list") if not deleted: - logger.warning(f"Couldn't delete combined.list from cache : {err}") - _exit(0) + LOGGER.warning(f"Couldn't delete combined.list from cache : {err}") + sys_exit(0) + + if not urls: + LOGGER.error("No URL found, skipping download...") + sys_exit(0) # Download and write data to temp file i = 0 content = b"" for url in urls: try: - logger.info(f"Downloading RealIP list from {url} ...") + LOGGER.info(f"Downloading RealIP list from {url} ...") if url.startswith("file://"): with open(normpath(url[7:]), "rb") as f: iterable = f.readlines() @@ -99,7 +88,7 @@ try: resp = get(url, stream=True, timeout=10) if resp.status_code != 200: - logger.warning(f"Got status code {resp.status_code}, skipping...") + LOGGER.warning(f"Got status code {resp.status_code}, skipping...") continue iterable = resp.iter_lines() @@ -116,34 +105,28 @@ try: i += 1 except: status = 2 - logger.error(f"Exception while getting RealIP list from {url} :\n{format_exc()}") - - tmp_realip_path.joinpath("combined.list").write_bytes(content) + LOGGER.error(f"Exception while getting RealIP list from {url} :\n{format_exc()}") # Check if file has changed - new_hash = file_hash(tmp_realip_path.joinpath("combined.list")) - old_hash = cache_hash(realip_path.joinpath("combined.list"), db) + new_hash = bytes_hash(content) + old_hash = JOB.cache_hash("combined.list") if new_hash == old_hash: - logger.info("New file is identical to cache file, reload is not needed") - _exit(0) + LOGGER.info("New file is identical to cache file, reload is not needed") + sys_exit(0) # Put file in cache - cached, err = cache_file( - tmp_realip_path.joinpath("combined.list"), - realip_path.joinpath("combined.list"), - new_hash, - db, - ) + cached, err = JOB.cache_file("combined.list", content, checksum=new_hash) if not cached: - logger.error(f"Error while caching list : {err}") - _exit(2) + LOGGER.error(f"Error while caching list : {err}") + sys_exit(2) - logger.info(f"Downloaded {i} trusted IP/net") + LOGGER.info(f"Downloaded {i} trusted IP/net") status = 1 - +except SystemExit as e: + status = e.code except: status = 2 - logger.error(f"Exception while running realip-download.py :\n{format_exc()}") + LOGGER.error(f"Exception while running realip-download.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/selfsigned/jobs/self-signed.py b/src/common/core/selfsigned/jobs/self-signed.py index 9ba78419d..a82e8edd5 100755 --- a/src/common/core/selfsigned/jobs/self-signed.py +++ b/src/common/core/selfsigned/jobs/self-signed.py @@ -6,31 +6,21 @@ from os.path import join from pathlib import Path from subprocess import DEVNULL, STDOUT, run from sys import exit as sys_exit, path as sys_path -from threading import Lock from traceback import format_exc from typing import Tuple -for deps_path in [ - join(sep, "usr", "share", "bunkerweb", *paths) - for paths in ( - ("deps", "python"), - ("utils",), - ("db",), - ) -]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) from cryptography import x509 from cryptography.hazmat.backends import default_backend -from Database import Database # type: ignore from logger import setup_logger # type: ignore -from jobs import set_file_in_db +from jobs import Job # type: ignore -logger = setup_logger("self-signed", getenv("LOG_LEVEL", "INFO")) -db = None -lock = Lock() +LOGGER = setup_logger("self-signed", getenv("LOG_LEVEL", "INFO")) +JOB = Job(LOGGER) status = 0 @@ -53,22 +43,24 @@ def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Pat ).returncode == 0 ): - logger.info(f"Self-signed certificate already present for {first_server}") + LOGGER.info(f"Self-signed certificate already present for {first_server}") certificate = x509.load_pem_x509_certificate( self_signed_path.joinpath(f"{first_server}.pem").read_bytes(), default_backend(), ) if sorted(attribute.rfc4514_string() for attribute in certificate.subject) != sorted(v for v in subj.split("/") if v): - logger.warning(f"Subject of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ...") + LOGGER.warning(f"Subject of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ...") elif certificate.not_valid_after - certificate.not_valid_before != timedelta(days=int(days)): - logger.warning( + LOGGER.warning( f"Expiration date of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ..." ) else: return True, 0 - logger.info(f"Generating self-signed certificate for {first_server}") + LOGGER.info(f"Generating self-signed certificate for {first_server}") + server_path = self_signed_path.joinpath(first_server) + server_path.mkdir(parents=True, exist_ok=True) if ( run( [ @@ -79,9 +71,9 @@ def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Pat "-newkey", "ed25519", "-keyout", - str(self_signed_path.joinpath(f"{first_server}.key")), + str(self_signed_path.joinpath(first_server, "key.pem")), "-out", - str(self_signed_path.joinpath(f"{first_server}.pem")), + str(self_signed_path.joinpath(first_server, "cert.pem")), "-days", days, "-subj", @@ -93,29 +85,19 @@ def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Pat ).returncode != 0 ): - logger.error(f"Self-signed certificate generation failed for {first_server}") + LOGGER.error(f"Self-signed certificate generation failed for {first_server}") return False, 2 # Update db - cached, err = set_file_in_db( - f"{first_server}.pem", - self_signed_path.joinpath(f"{first_server}.pem").read_bytes(), - db, - service_id=first_server, - ) + cached, err = JOB.cache_file("cert.pem", self_signed_path.joinpath(first_server, "cert.pem"), service_id=first_server, overwrite_file=False) if not cached: - logger.error(f"Error while caching self-signed {first_server}.pem file : {err}") + LOGGER.error(f"Error while caching self-signed cert.pem file for {first_server} : {err}") - cached, err = set_file_in_db( - f"{first_server}.key", - self_signed_path.joinpath(f"{first_server}.key").read_bytes(), - db, - service_id=first_server, - ) + cached, err = JOB.cache_file("key.pem", self_signed_path.joinpath(first_server, "key.pem"), service_id=first_server, overwrite_file=False) if not cached: - logger.error(f"Error while caching self-signed {first_server}.key file : {err}") + LOGGER.error(f"Error while caching self-signed {first_server}.key file : {err}") - logger.info(f"Successfully generated self-signed certificate for {first_server}") + LOGGER.info(f"Successfully generated self-signed certificate for {first_server}") return True, 1 @@ -123,58 +105,42 @@ status = 0 try: self_signed_path = Path(sep, "var", "cache", "bunkerweb", "selfsigned") - self_signed_path.mkdir(parents=True, exist_ok=True) + servers = getenv("SERVER_NAME") or [] - # Multisite case - if getenv("MULTISITE") == "yes": - servers = getenv("SERVER_NAME") or [] + if isinstance(servers, str): + servers = servers.split(" ") - if isinstance(servers, str): - servers = servers.split(" ") + if not servers: + LOGGER.info("No server found, skipping self-signed certificate generation ...") + sys_exit(0) + skipped_servers = [] + if not getenv("MULTISITE", "no") == "yes": + servers = [servers[0]] + if getenv("GENERATE_SELF_SIGNED_SSL", "no") == "no": + LOGGER.info("Generate self-signed SSL is not enabled, skipping certificate generation ...") + skipped_servers = servers + + if not skipped_servers: for first_server in servers: - if ( - not first_server - or getenv( - f"{first_server}_GENERATE_SELF_SIGNED_SSL", - getenv("GENERATE_SELF_SIGNED_SSL", "no"), - ) - != "yes" - or self_signed_path.joinpath(f"{first_server}.pem").is_file() - ): + if getenv(f"{first_server}_GENERATE_SELF_SIGNED_SSL", getenv("GENERATE_SELF_SIGNED_SSL", "no")) != "yes": + LOGGER.info(f"Self-signed SSL is not enabled for {first_server}, skipping certificate generation ...") + skipped_servers.append(first_server) continue - if not db: - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) - ret, ret_status = generate_cert( first_server, - getenv( - f"{first_server}_SELF_SIGNED_SSL_EXPIRY", - getenv("SELF_SIGNED_SSL_EXPIRY", "365"), - ), - getenv( - f"{first_server}_SELF_SIGNED_SSL_SUBJ", - getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/"), - ), + getenv(f"{first_server}_SELF_SIGNED_SSL_EXPIRY", getenv("SELF_SIGNED_SSL_EXPIRY", "365")), + getenv(f"{first_server}_SELF_SIGNED_SSL_SUBJ", getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/")), self_signed_path, ) status = ret_status - # Singlesite case - elif getenv("GENERATE_SELF_SIGNED_SSL", "no") == "yes" and getenv("SERVER_NAME"): - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) - - first_server = getenv("SERVER_NAME", "").split(" ")[0] - ret, ret_status = generate_cert( - first_server, - getenv("SELF_SIGNED_SSL_EXPIRY", "365"), - getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/"), - self_signed_path, - ) - status = ret_status + for first_server in skipped_servers: + JOB.del_cache("cert.pem", service_id=first_server) + JOB.del_cache("key.pem", service_id=first_server) except: status = 2 - logger.error(f"Exception while running self-signed.py :\n{format_exc()}") + LOGGER.error(f"Exception while running self-signed.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/core/selfsigned/selfsigned.lua b/src/common/core/selfsigned/selfsigned.lua index 08ab1f851..b48caac00 100644 --- a/src/common/core/selfsigned/selfsigned.lua +++ b/src/common/core/selfsigned/selfsigned.lua @@ -44,8 +44,8 @@ function selfsigned:init() for server_name, multisite_vars in pairs(vars) do if multisite_vars["GENERATE_SELF_SIGNED_SSL"] == "yes" and server_name ~= "global" then local check, data = read_files({ - "/var/cache/bunkerweb/selfsigned/" .. server_name .. ".pem", - "/var/cache/bunkerweb/selfsigned/" .. server_name .. ".key", + "/var/cache/bunkerweb/selfsigned/" .. server_name .. "/cert.pem", + "/var/cache/bunkerweb/selfsigned/" .. server_name .. "/key.pem", }) if not check then self.logger:log(ERR, "error while reading files : " .. data) @@ -68,8 +68,8 @@ function selfsigned:init() return self:ret(false, "can't get SERVER_NAME variable : " .. err) end local check, data = read_files({ - "/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. ".pem", - "/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. ".key", + "/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. "/cert.pem", + "/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. "/key.pem", }) if not check then self.logger:log(ERR, "error while reading files : " .. data) diff --git a/src/common/core/whitelist/jobs/whitelist-download.py b/src/common/core/whitelist/jobs/whitelist-download.py index dbb67ee10..8f6a0ec12 100755 --- a/src/common/core/whitelist/jobs/whitelist-download.py +++ b/src/common/core/whitelist/jobs/whitelist-download.py @@ -2,10 +2,9 @@ from contextlib import suppress from ipaddress import ip_address, ip_network -from os import _exit, getenv, sep +from os import getenv, sep from os.path import join, normpath -from pathlib import Path -from re import IGNORECASE, compile as re_compile +from re import compile as re_compile from sys import exit as sys_exit, path as sys_path from traceback import format_exc from typing import Tuple @@ -16,11 +15,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (( from requests import get -from Database import Database # type: ignore +from common_utils import bytes_hash # type: ignore from logger import setup_logger # type: ignore -from jobs import cache_file, cache_hash, del_file_in_db, is_cached_file, file_hash +from jobs import Job # type: ignore -rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE) +rdns_rx = re_compile(rb"^[^ ]+$") asn_rx = re_compile(rb"^\d+$") uri_rx = re_compile(rb"^/") @@ -51,7 +50,7 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]: return False, b"" -logger = setup_logger("WHITELIST", getenv("LOG_LEVEL", "INFO")) +LOGGER = setup_logger("WHITELIST", getenv("LOG_LEVEL", "INFO")) status = 0 try: @@ -68,16 +67,10 @@ try: whitelist_activated = True if not whitelist_activated: - logger.info("Whitelist is not activated, skipping downloads...") - _exit(0) + LOGGER.info("Whitelist is not activated, skipping downloads...") + sys_exit(0) - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False) - - # Create directories if they don't exist - whitelist_path = Path(sep, "var", "cache", "bunkerweb", "whitelist") - whitelist_path.mkdir(parents=True, exist_ok=True) - tmp_whitelist_path = Path(sep, "var", "tmp", "bunkerweb", "whitelist") - tmp_whitelist_path.mkdir(parents=True, exist_ok=True) + JOB = Job(LOGGER) # Get URLs urls = {"IP": [], "RDNS": [], "ASN": [], "USER_AGENT": [], "URI": []} @@ -87,44 +80,36 @@ try: urls[kind].append(url) # Don't go further if the cache is fresh - kinds_fresh = { - "IP": True, - "RDNS": True, - "ASN": True, - "USER_AGENT": True, - "URI": True, - } - all_fresh = True + kinds_fresh = {"IP": True, "RDNS": True, "ASN": True, "USER_AGENT": True, "URI": True} for kind in kinds_fresh: - if not is_cached_file(whitelist_path.joinpath(f"{kind}.list"), "hour", db): - kinds_fresh[kind] = False - all_fresh = False - logger.info( - f"Whitelist for {kind} is not cached, processing downloads..", - ) - else: - logger.info( - f"Whitelist for {kind} is already in cache, skipping downloads...", - ) - if not urls[kind]: - logger.info( - f"Whitelist for {kind} is already in cache, skipping downloads...", - ) - whitelist_path.joinpath(f"{kind}.list").unlink(missing_ok=True) - deleted, err = del_file_in_db(f"{kind}.list", db) - if not deleted: - logger.warning(f"Couldn't delete {kind}.list from cache : {err}") - if all_fresh: - _exit(0) + if not JOB.is_cached_file(f"{kind}.list", "hour"): + if urls[kind]: + kinds_fresh[kind] = False + LOGGER.info(f"Whitelist for {kind} is not cached, processing downloads..") + continue + + LOGGER.info(f"Whitelist for {kind} is already in cache, skipping downloads...") + + if not urls[kind]: + LOGGER.warning(f"Whitelist for {kind} is cached but no URL is configured, removing from cache...") + deleted, err = JOB.del_cache(f"{kind}.list") + if not deleted: + LOGGER.warning(f"Couldn't delete {kind}.list from cache : {err}") + + if all(kinds_fresh.values()): + if not any(urls.values()): + LOGGER.info("No whitelist URL is configured, nothing to do...") + sys_exit(0) # Loop on kinds for kind, urls_list in urls.items(): if kinds_fresh[kind]: continue - # Write combined data of the kind to a single temp file + + # Write combined data of the kind in memory and check if it has changed for url in urls_list: try: - logger.info(f"Downloading whitelist data from {url} ...") + LOGGER.info(f"Downloading whitelist data from {url} ...") if url.startswith("file://"): with open(normpath(url[7:]), "rb") as f: iterable = f.readlines() @@ -132,7 +117,7 @@ try: resp = get(url, stream=True, timeout=10) if resp.status_code != 200: - logger.warning(f"Got status code {resp.status_code}, skipping...") + LOGGER.warning(f"Got status code {resp.status_code}, skipping...") continue iterable = resp.iter_lines() @@ -152,39 +137,28 @@ try: content += data + b"\n" i += 1 - tmp_whitelist_path.joinpath(f"{kind}.list").write_bytes(content) - - logger.info(f"Downloaded {i} bad {kind}") + LOGGER.info(f"Downloaded {i} bad {kind}") # Check if file has changed - new_hash = file_hash(tmp_whitelist_path.joinpath(f"{kind}.list")) - old_hash = cache_hash(whitelist_path.joinpath(f"{kind}.list"), db) + new_hash = bytes_hash(content) + old_hash = JOB.cache_hash(f"{kind}.list") if new_hash == old_hash: - logger.info( - f"New file {kind}.list is identical to cache file, reload is not needed", - ) + LOGGER.info(f"New file {kind}.list is identical to cache file, reload is not needed") else: - logger.info( - f"New file {kind}.list is different than cache file, reload is needed", - ) + LOGGER.info(f"New file {kind}.list is different than cache file, reload is needed") # Put file in cache - cached, err = cache_file( - tmp_whitelist_path.joinpath(f"{kind}.list"), - whitelist_path.joinpath(f"{kind}.list"), - new_hash, - db, - ) - + cached, err = JOB.cache_file(f"{kind}.list", content, checksum=new_hash) if not cached: - logger.error(f"Error while caching whitelist : {err}") + LOGGER.error(f"Error while caching whitelist : {err}") status = 2 else: status = 1 except: status = 2 - logger.error(f"Exception while getting whitelist from {url} :\n{format_exc()}") - + LOGGER.error(f"Exception while getting whitelist from {url} :\n{format_exc()}") +except SystemExit as e: + status = e.code except: status = 2 - logger.error(f"Exception while running whitelist-download.py :\n{format_exc()}") + LOGGER.error(f"Exception while running whitelist-download.py :\n{format_exc()}") sys_exit(status) diff --git a/src/common/db/Database.py b/src/common/db/Database.py index a25fa6f31..55403af79 100644 --- a/src/common/db/Database.py +++ b/src/common/db/Database.py @@ -36,7 +36,7 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (( if deps_path not in sys_path: sys_path.append(deps_path) -from jobs import file_hash # type: ignore +from common_utils import file_hash # type: ignore from pymysql import install_as_MySQLdb from sqlalchemy import create_engine, MetaData as sql_metadata, text, inspect @@ -65,7 +65,7 @@ class Database: pool: bool = True, ) -> None: """Initialize the database""" - self.__logger = logger + self.logger = logger self.__session_factory = None self.__sql_engine = None @@ -74,14 +74,14 @@ class Database: match = self.DB_STRING_RX.search(sqlalchemy_string) if not match: - self.__logger.error(f"Invalid database string provided: {sqlalchemy_string}, exiting...") + self.logger.error(f"Invalid database string provided: {sqlalchemy_string}, exiting...") _exit(1) if match.group("database").startswith("sqlite"): db_path = Path(normpath(match.group("path"))) if ui: while not db_path.is_file(): - self.__logger.warning(f"Waiting for the database file to be created: {db_path}") + self.logger.warning(f"Waiting for the database file to be created: {db_path}") sleep(1) else: db_path.parent.mkdir(parents=True, exist_ok=True) @@ -102,10 +102,10 @@ class Database: try: self.__sql_engine = create_engine(sqlalchemy_string, **engine_kwargs) except ArgumentError: - self.__logger.error(f"Invalid database URI: {sqlalchemy_string}") + self.logger.error(f"Invalid database URI: {sqlalchemy_string}") error = True except SQLAlchemyError: - self.__logger.error(f"Error when trying to create the engine: {format_exc()}") + self.logger.error(f"Error when trying to create the engine: {format_exc()}") error = True finally: if error: @@ -114,7 +114,7 @@ class Database: try: assert self.__sql_engine is not None except AssertionError: - self.__logger.error("The database engine is not initialized") + self.logger.error("The database engine is not initialized") _exit(1) not_connected = True @@ -128,29 +128,29 @@ class Database: not_connected = False except (OperationalError, DatabaseError) as e: if retries <= 0: - self.__logger.error( + self.logger.error( f"Can't connect to database : {format_exc()}", ) _exit(1) if "attempt to write a readonly database" in str(e): - self.__logger.warning("The database is read-only, waiting for it to become writable. Retrying in 5 seconds ...") + self.logger.warning("The database is read-only, waiting for it to become writable. Retrying in 5 seconds ...") self.__sql_engine.dispose(close=True) self.__sql_engine = create_engine(sqlalchemy_string, **engine_kwargs) if "Unknown table" in str(e): not_connected = False continue else: - self.__logger.warning( + self.logger.warning( "Can't connect to database, retrying in 5 seconds ...", ) retries -= 1 sleep(5) except BaseException: - self.__logger.error(f"Error when trying to connect to the database: {format_exc()}") + self.logger.error(f"Error when trying to connect to the database: {format_exc()}") exit(1) - self.__logger.info("✅ Database connection established") + self.logger.info("✅ Database connection established") self.__session_factory = sessionmaker(bind=self.__sql_engine, autoflush=True, expire_on_commit=False) self.suffix_rx = re_compile(r"_\d+$") @@ -173,7 +173,7 @@ class Database: try: assert self.__session_factory is not None except AssertionError: - self.__logger.error("The database session is not initialized") + self.logger.error("The database session is not initialized") _exit(1) session = scoped_session(self.__session_factory) @@ -420,13 +420,13 @@ class Database: db_version = self.get_metadata()["version"] if db_version != bunkerweb_version: - self.__logger.warning(f"Database version ({db_version}) is different from Bunkerweb version ({bunkerweb_version}), migrating ...") + self.logger.warning(f"Database version ({db_version}) is different from Bunkerweb version ({bunkerweb_version}), migrating ...") metadata = sql_metadata() metadata.reflect(self.__sql_engine) for table_name in Base.metadata.tables.keys(): if not inspector.has_table(table_name): - self.__logger.warning(f'Table "{table_name}" is missing') + self.logger.warning(f'Table "{table_name}" is missing') has_all_tables = False continue @@ -519,7 +519,7 @@ class Database: updates[Plugins.checksum] = plugin.get("checksum") if updates: - self.__logger.warning(f'Plugin "{plugin["id"]}" already exists, updating it with the new values') + self.logger.warning(f'Plugin "{plugin["id"]}" already exists, updating it with the new values') session.query(Plugins).filter(Plugins.id == plugin["id"]).update(updates) else: to_put.append( @@ -578,11 +578,11 @@ class Database: updates[Settings.multiple] = value.get("multiple") if updates: - self.__logger.warning(f'Setting "{setting}" already exists, updating it with the new values') + self.logger.warning(f'Setting "{setting}" already exists, updating it with the new values') session.query(Settings).filter(Settings.id == setting).update(updates) else: if db_plugin: - self.__logger.warning(f'Setting "{setting}" does not exist, creating it') + self.logger.warning(f'Setting "{setting}" does not exist, creating it') to_put.append(Settings(**value)) db_values = [select.value for select in session.query(Selects).with_entities(Selects.value).filter_by(setting_id=value["id"])] @@ -591,7 +591,7 @@ class Database: if select_values: if missing_values: # Remove selects that are no longer in the list - self.__logger.warning(f'Removing {len(missing_values)} selects from setting "{setting}" as they are no longer in the list') + self.logger.warning(f'Removing {len(missing_values)} selects from setting "{setting}" as they are no longer in the list') session.query(Selects).filter(Selects.value.in_(missing_values)).delete() for select in select_values: @@ -599,7 +599,7 @@ class Database: to_put.append(Selects(setting_id=value["id"], value=select)) else: if missing_values: - self.__logger.warning(f'Removing all selects from setting "{setting}" as there are no longer any in the list') + self.logger.warning(f'Removing all selects from setting "{setting}" as there are no longer any in the list') session.query(Selects).filter_by(setting_id=value["id"]).delete() db_names = [job.name for job in session.query(Jobs).with_entities(Jobs.name).filter_by(plugin_id=plugin["id"])] @@ -608,7 +608,7 @@ class Database: if missing_names: # Remove jobs that are no longer in the list - self.__logger.warning(f'Removing {len(missing_names)} jobs from plugin "{plugin["id"]}" as they are no longer in the list') + self.logger.warning(f'Removing {len(missing_names)} jobs from plugin "{plugin["id"]}" as they are no longer in the list') session.query(Jobs).filter(Jobs.name.in_(missing_names)).delete() for job in jobs: @@ -623,7 +623,7 @@ class Database: job["file_name"] = job.pop("file") job["reload"] = job.get("reload", False) if db_plugin: - self.__logger.warning(f'Job "{job["name"]}" does not exist, creating it') + self.logger.warning(f'Job "{job["name"]}" does not exist, creating it') to_put.append(Jobs(plugin_id=plugin["id"], **job)) else: updates = {} @@ -638,7 +638,7 @@ class Database: updates[Jobs.reload] = job.get("reload", False) if updates: - self.__logger.warning(f'Job "{job["name"]}" already exists, updating it with the new values') + self.logger.warning(f'Job "{job["name"]}" already exists, updating it with the new values') updates[Jobs.last_run] = None session.query(Jobs_cache).filter(Jobs_cache.job_name == job["name"]).delete() session.query(Jobs).filter(Jobs.name == job["name"]).update(updates) @@ -681,12 +681,12 @@ class Database: ) if updates: - self.__logger.warning(f'Page for plugin "{plugin["id"]}" already exists, updating it with the new values') + self.logger.warning(f'Page for plugin "{plugin["id"]}" already exists, updating it with the new values') session.query(Plugin_pages).filter(Plugin_pages.plugin_id == plugin["id"]).update(updates) continue if db_plugin: - self.__logger.warning(f'Page for plugin "{plugin["id"]}" does not exist, creating it') + self.logger.warning(f'Page for plugin "{plugin["id"]}" does not exist, creating it') to_put.append( Plugin_pages( @@ -1162,10 +1162,16 @@ class Database: def delete_job_cache(self, file_name: str, *, job_name: Optional[str] = None, service_id: Optional[str] = None): job_name = job_name or basename(getsourcefile(_getframe(1))).replace(".py", "") - with self.__db_session() as session: - session.query(Jobs_cache).filter_by(job_name=job_name, file_name=file_name, service_id=service_id).delete() + filters = {"file_name": file_name} + if job_name: + filters["job_name"] = job_name + if service_id: + filters["service_id"] = service_id - def update_job_cache( + with self.__db_session() as session: + session.query(Jobs_cache).filter_by(**filters).delete() + + def upsert_job_cache( self, service_id: Optional[str], file_name: str, @@ -1254,7 +1260,7 @@ class Database: if db_plugin: if db_plugin.type not in ("external", "pro"): - self.__logger.warning( + self.logger.warning( f"Plugin \"{plugin['id']}\" is not {_type}, skipping update (updating a non-external or non-pro plugin is forbidden for security reasons)", # noqa: E501 ) continue @@ -1484,7 +1490,7 @@ class Database: db_setting = session.query(Settings).filter_by(id=setting).first() if db_setting is not None: - self.__logger.warning(f"A setting with id {setting} already exists, therefore it will not be added.") + self.logger.warning(f"A setting with id {setting} already exists, therefore it will not be added.") continue value.update({"plugin_id": plugin["id"], "name": value["id"], "id": setting}) @@ -1500,7 +1506,7 @@ class Database: ) if db_job is not None: - self.__logger.warning(f"A job with the name {job['name']} already exists in the database, therefore it will not be added.") + self.logger.warning(f"A job with the name {job['name']} already exists in the database, therefore it will not be added.") continue job["file_name"] = job.pop("file") @@ -1681,13 +1687,8 @@ class Database: } def get_job_cache_file( - self, - job_name: str, - file_name: str, - *, - with_info: bool = False, - with_data: bool = True, - ) -> Optional[Any]: + self, job_name: str, file_name: str, *, service_id: str = "", with_info: bool = False, with_data: bool = True + ) -> Optional[Union[Dict[str, Any], bytes]]: """Get job cache file.""" entities = [] if with_info: @@ -1695,26 +1696,47 @@ class Database: if with_data: entities.append(Jobs_cache.data) - with self.__db_session() as session: - return session.query(Jobs_cache).with_entities(*entities).filter_by(job_name=job_name, file_name=file_name).first() + filters = {"job_name": job_name, "file_name": file_name} + if service_id: + filters["service_id"] = service_id - def get_jobs_cache_files(self) -> List[Dict[str, Any]]: + with self.__db_session() as session: + data = session.query(Jobs_cache).with_entities(*entities).filter_by(**filters).first() + + if not data: + return None + + if with_data and not with_info: + return data.data + + ret_data = {} + if with_info: + ret_data["last_update"] = data.last_update.timestamp() if data.last_update is not None else "Never" + ret_data["checksum"] = data.checksum + if with_data: + ret_data["data"] = data.data + return ret_data + + def get_jobs_cache_files(self, *, job_name: str = "", with_data: bool = False) -> List[Dict[str, Any]]: """Get jobs cache files.""" with self.__db_session() as session: + entities = [Jobs_cache.job_name, Jobs_cache.service_id, Jobs_cache.file_name] + if with_data: + entities.append(Jobs_cache.data) + + query = session.query(Jobs_cache).with_entities(*entities) + + if job_name: + query = query.filter_by(job_name=job_name) + return [ { "job_name": cache.job_name, "service_id": cache.service_id, "file_name": cache.file_name, - "data": "Download file to view content", + "data": "Download file to view content" if not with_data else cache.data, } - for cache in ( - session.query(Jobs_cache).with_entities( - Jobs_cache.job_name, - Jobs_cache.service_id, - Jobs_cache.file_name, - ) - ) + for cache in query ] def add_instance(self, hostname: str, port: int, server_name: str, changed: Optional[bool] = True) -> str: diff --git a/src/common/utils/common_utils.py b/src/common/utils/common_utils.py new file mode 100644 index 000000000..5fdfdc654 --- /dev/null +++ b/src/common/utils/common_utils.py @@ -0,0 +1,80 @@ +from hashlib import sha512 +from io import BytesIO +from os import getenv, sep +from pathlib import Path +from platform import machine +from typing import Dict, Union + + +def get_version() -> str: + return Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text(encoding="utf-8").strip() + + +def get_integration() -> str: + try: + integration_path = Path(sep, "usr", "share", "bunkerweb", "INTEGRATION") + os_release_path = Path(sep, "etc", "os-release") + if getenv("KUBERNETES_MODE", "no").lower() == "yes": + return "Kubernetes" + elif getenv("SWARM_MODE", "no").lower() == "yes": + return "Swarm" + elif getenv("AUTOCONF_MODE", "no").lower() == "yes": + return "Autoconf" + elif integration_path.is_file(): + return integration_path.read_text(encoding="utf-8").strip().lower() + elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"): + return "Docker" + + return "Linux" + except: + return "Unknown" + + +def get_os_info() -> Dict[str, str]: + os_data = { + "name": "Linux", + "version": "Unknown", + "version_id": "Unknown", + "version_codename": "Unknown", + "id": "Unknown", + "arch": machine(), + } + + os_release = Path("/etc/os-release") + if os_release.exists(): + for line in os_release.read_text().splitlines(): + if "=" not in line or line.split("=")[0].strip().lower() not in os_data: + continue + os_data[line.split("=")[0].lower()] = line.split("=")[1].strip('"') + + return os_data + + +def file_hash(file: Union[str, Path]) -> str: + _sha512 = sha512() + if not isinstance(file, Path): + file = Path(file) + + with file.open("rb") as f: + while True: + data = f.read(1024) + if not data: + break + _sha512.update(data) + return _sha512.hexdigest() + + +def bytes_hash(bio: Union[bytes, BytesIO]) -> str: + if isinstance(bio, bytes): + bio = BytesIO(bio) + + assert isinstance(bio, BytesIO) + + _sha512 = sha512() + while True: + data = bio.read(1024) + if not data: + break + _sha512.update(data) + bio.seek(0) + return _sha512.hexdigest() diff --git a/src/common/utils/jobs.py b/src/common/utils/jobs.py index f7ef72520..bdf5a2887 100644 --- a/src/common/utils/jobs.py +++ b/src/common/utils/jobs.py @@ -1,252 +1,249 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- -from datetime import datetime -from hashlib import sha512 +from datetime import datetime, timedelta from inspect import getsourcefile -from io import BufferedReader -from json import dumps, loads -from os import getenv, sep -from os.path import basename +from io import BytesIO +from logging import Logger +from os import getenv +from os.path import sep from pathlib import Path -from platform import machine +from shutil import rmtree from sys import _getframe +from tarfile import open as tar_open from threading import Lock from traceback import format_exc -from typing import Dict, Literal, Optional, Tuple, Union +from typing import Any, Dict, Literal, Optional, Tuple, Union -lock = Lock() +from common_utils import bytes_hash, file_hash -""" -{ - "date": timestamp, - "checksum": sha512 +LOCK = Lock() +EXPIRE_TIME = { + "hour": timedelta(hours=1).total_seconds(), + "day": timedelta(days=1).total_seconds(), + "week": timedelta(weeks=1).total_seconds(), + "month": timedelta(days=30).total_seconds(), } -""" -def is_cached_file( - file: Union[str, Path], - expire: Union[Literal["hour"], Literal["day"], Literal["week"], Literal["month"]], - db=None, -) -> bool: - if not isinstance(file, Path): - file = Path(file) +class Job: + def __init__(self, logger: Optional[Logger] = None, db=None, *, job_name: str = "", deprecated: bool = False): + source_file = getsourcefile(_getframe(1)) + if source_file is None: + raise ValueError("source_file could not be determined.") + elif not logger and not db: + raise ValueError("Either logger or db must be provided.") + source_path = Path(source_file) + self.job_path = Path(sep, "var", "cache", "bunkerweb", source_path.parent.parent.name) + self.job_name = job_name or source_path.name.replace(".py", "") - is_cached = False - cached_file = None - try: - file_path = Path(f"{file}.md") - if not file_path.is_file(): - if not db: - return False - cached_file = db.get_job_cache_file( - basename(getsourcefile(_getframe(1))).replace(".py", ""), - file.name, - with_info=True, - ) + self.db = db + if not self.db: + from Database import Database # type: ignore - if not cached_file: - return False - cached_time = cached_file.last_update.timestamp() - else: - cached_time = loads(file_path.read_text())["date"] + self.db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False) + self.logger = logger or self.db.logger - current_time = datetime.now().timestamp() - if current_time < cached_time: - is_cached = False - else: - diff_time = current_time - cached_time - if expire == "hour": - is_cached = diff_time < 3600 - elif expire == "day": - is_cached = diff_time < 86400 - elif expire == "week": - is_cached = diff_time < 604800 - elif expire == "month": - is_cached = diff_time < 2592000 - except: - is_cached = False + if not deprecated: + self.restore_cache() - if is_cached and cached_file: - file.write_bytes(cached_file.data) + def restore_cache(self, *, job_name: str = "") -> bool: + """Restore job cache files from database.""" + ret = True + with LOCK: + job_cache_files = self.db.get_jobs_cache_files(job_name=job_name or self.job_name, with_data=True) # type: ignore - return is_cached and cached_file - - -def get_file_in_db(file: Union[str, Path], db, *, job_name: Optional[str] = None) -> Optional[bytes]: - cached_file = db.get_job_cache_file(job_name or basename(getsourcefile(_getframe(1))).replace(".py", ""), file) - if not cached_file: - return None - return cached_file.data - - -def set_file_in_db( - name: str, - content: bytes, - db, - *, - job_name: Optional[str] = None, - service_id: Optional[str] = None, - checksum: Optional[str] = None, -) -> Tuple[bool, str]: - ret, err = True, "success" - try: - with lock: - err = db.update_job_cache( - service_id, - name, - content, - job_name=job_name or basename(getsourcefile(_getframe(1))).replace(".py", ""), - checksum=checksum, - ) - - if err: + for job_cache_file in job_cache_files: + try: + cache_path = self.job_path.joinpath(job_cache_file["service_id"] or "", job_cache_file["file_name"]) + if job_cache_file["file_name"].endswith(".tgz"): + rmtree(cache_path.parent, ignore_errors=True) + cache_path.parent.mkdir(parents=True, exist_ok=True) + with tar_open(fileobj=BytesIO(job_cache_file["data"]), mode="r:gz") as tar: + tar.extractall(cache_path.parent) + else: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_bytes(job_cache_file["data"]) + except BaseException as e: + self.logger.error(f"Exception while restoring cache file {job_cache_file['file_name']} :\n{e}") ret = False - except: - return False, f"exception :\n{format_exc()}" - return ret, err + + return ret + + def get_cache( + self, name: str, *, job_name: str = "", service_id: str = "", with_info: bool = False, with_data: bool = True + ) -> Optional[Union[Dict[str, Any], bytes]]: + """Get cache file from database or from local cache file.""" + cache_path = self.job_path.joinpath(service_id, name) + if cache_path.is_file(): + if with_data and not with_info: + return cache_path.read_bytes() + + ret_data = {} + if with_info: + ret_data = { + "last_update": cache_path.stat().st_mtime, + "checksum": file_hash(cache_path), + } + if with_data: + ret_data["data"] = cache_path.read_bytes() + return ret_data + + with LOCK: + return self.db.get_job_cache_file(job_name or self.job_name, name, service_id=service_id, with_info=with_info, with_data=with_data) # type: ignore + + def is_cached_file(self, name: str, expire: Literal["hour", "day", "week", "month"], *, job_name: str = "", service_id: str = "") -> bool: + """Check if cache file is cached and if it's still fresh.""" + is_cached = False + try: + cache_info = self.get_cache(name, job_name=job_name, service_id=service_id, with_info=True, with_data=False) + if isinstance(cache_info, dict): + current_time = datetime.now().timestamp() + if current_time < cache_info["last_update"]: + is_cached = False + else: + is_cached = current_time - cache_info["last_update"] < EXPIRE_TIME[expire] + except: + is_cached = False + return is_cached + + def cache_file( + self, + name: str, + file_cache: Union[bytes, str, Path], + *, + job_name: str = "", + service_id: str = "", + checksum: Optional[str] = None, + delete_file: bool = True, + overwrite_file: bool = True, + ) -> Tuple[bool, str]: + """Cache file in database and in local cache file.""" + ret, err = True, "success" + cache_path = self.job_path.joinpath(service_id, name) + cache_path.parent.mkdir(parents=True, exist_ok=True) + + if isinstance(file_cache, bytes): + content = file_cache + else: + if isinstance(file_cache, str): + file_cache = Path(file_cache) + assert isinstance(file_cache, Path) + content = file_cache.read_bytes() + + if overwrite_file or not cache_path.is_file(): + cache_path.write_bytes(content) + + if not checksum: + checksum = bytes_hash(content) + + try: + with LOCK: + if self.db.upsert_job_cache(service_id, name, content, job_name=job_name or self.job_name, checksum=checksum): # type: ignore + ret = False + + if ret and isinstance(file_cache, Path) and delete_file and file_cache != cache_path: + file_cache.unlink(missing_ok=True) + except: + return False, f"exception :\n{format_exc()}" + return ret, err + + def cache_dir(self, dir_path: Union[str, Path], *, job_name: str = "", service_id: str = "") -> Tuple[bool, str]: + """Cache directory in database and in local cache file.""" + if isinstance(dir_path, str): + dir_path = Path(dir_path) + assert isinstance(dir_path, Path) + + file_name = f"{dir_path.name}.tgz" + content = BytesIO() + with tar_open(file_name, mode="w:gz", fileobj=content, compresslevel=9) as tgz: + tgz.add(dir_path, arcname=".") + content.seek(0, 0) + + return self.cache_file(file_name, content.read(), job_name=job_name, service_id=service_id) + + def del_cache(self, name: str, *, job_name: str = "", service_id: str = "") -> Tuple[bool, str]: + """Delete cache file from database and local cache file.""" + ret, err = True, "success" + job_name = job_name or self.job_name + job_path = self.job_path.joinpath(service_id) + cache_path = job_path.joinpath(name) + + if cache_path.is_file(): + cache_path.unlink(missing_ok=True) + + if job_path.is_dir() and not list(job_path.iterdir()): + rmtree(job_path, ignore_errors=True) + + try: + with LOCK: + self.db.delete_job_cache(name, job_name=job_name, service_id=service_id) # type: ignore + except: + return False, f"exception :\n{format_exc()}" + return ret, err + + def cache_hash(self, name: str, *, job_name: str = "", service_id: str = "") -> Optional[str]: + """Get cache file hash from database or from local cache file.""" + cache_path = self.job_path.joinpath(service_id, name) + if cache_path.is_file(): + return file_hash(cache_path) + + cache_info = self.get_cache(name, with_info=True, with_data=False, job_name=job_name, service_id=service_id) + + if isinstance(cache_info, dict): + return cache_info.get("checksum") + return None -def del_file_in_db(name: str, db, *, service_id: Optional[str] = None) -> Tuple[bool, str]: - ret, err = True, "success" - try: - db.delete_job_cache(name, job_name=basename(getsourcefile(_getframe(1))).replace(".py", ""), service_id=service_id) - except: - return False, f"exception :\n{format_exc()}" - return ret, err +# ? Backward compatibility functions -def file_hash(file: Union[str, Path]) -> str: - _sha512 = sha512() +def is_cached_file(file: Union[str, Path], expire: Literal["hour", "day", "week", "month"], db) -> bool: + job = Job(None, db, deprecated=True) + job.logger.warning("is_cached_file is deprecated, use the Job.is_cached_file method instead.") if not isinstance(file, Path): file = Path(file) - - with file.open("rb") as f: - while True: - data = f.read(1024) - if not data: - break - _sha512.update(data) - return _sha512.hexdigest() + return job.is_cached_file(file.name, expire) -def bytes_hash(bio: BufferedReader) -> str: - _sha512 = sha512() - while True: - data = bio.read(1024) - if not data: - break - _sha512.update(data) - bio.seek(0) - return _sha512.hexdigest() - - -def cache_hash(cache: Union[str, Path], db=None) -> Optional[str]: - checksum = None - cache_file = Path(f"{cache}.md") - if cache_file.is_file(): - checksum = loads(cache_file.read_text(encoding="utf-8")).get("checksum", None) - - if not checksum and db: - if not isinstance(cache, Path): - cache = Path(cache) - - cached_file = db.get_job_cache_file( - basename(getsourcefile(_getframe(1))).replace(".py", ""), - cache.name, - with_info=True, - with_data=False, - ) - checksum = cached_file.checksum if cached_file else None - - if checksum: - return checksum +def get_file_in_db(file: Union[str, Path], db, *, job_name: str = "") -> Optional[bytes]: + job = Job(None, db, deprecated=True) + job.logger.warning("get_file_in_db is deprecated, use the Job.get_cache method instead.") + if not isinstance(file, Path): + file = Path(file) + cache = job.get_cache(file.name, job_name=job_name, with_data=True) + if isinstance(cache, dict): + return cache["data"] return None +def set_file_in_db(name: str, content: bytes, db, *, job_name: str = "", service_id: str = "", checksum: Optional[str] = None) -> Tuple[bool, str]: + job = Job(None, db, deprecated=True) + job.logger.warning("set_file_in_db is deprecated, use the Job.cache_file method instead.") + return job.cache_file(name, content, job_name=job_name, service_id=service_id, checksum=checksum) + + +def del_file_in_db(name: str, db, *, service_id: str = "") -> Tuple[bool, str]: + job = Job(None, db, deprecated=True) + job.logger.warning("del_file_in_db is deprecated, use the Job.del_cache method instead.") + return job.del_cache(name, service_id=service_id) + + +def cache_hash(cache: Union[str, Path], db) -> Optional[str]: + job = Job(None, db, deprecated=True) + job.logger.warning("cache_hash is deprecated, use the Job.cache_hash method instead.") + if not isinstance(cache, Path): + cache = Path(cache) + return job.cache_hash(cache.name) + + def cache_file( - file: Union[str, Path], - cache: Union[str, Path], - _hash: Optional[str], - db=None, - *, - delete_file: bool = True, - service_id: Optional[str] = None, + file: Union[str, Path], cache: Union[str, Path], _hash: Optional[str], db, *, delete_file: bool = True, service_id: str = "" ) -> Tuple[bool, str]: - ret, err = True, "success" - try: - if not isinstance(file, Path): - file = Path(file) - if not isinstance(cache, Path): - cache = Path(cache) - - content = file.read_bytes() - cache.write_bytes(content) - - if delete_file: - file.unlink() - - if not _hash: - _hash = file_hash(str(cache)) - - if db: - return set_file_in_db( - cache.name, - content, - db, - job_name=basename(getsourcefile(_getframe(1))).replace(".py", ""), - service_id=service_id, - checksum=_hash, - ) - else: - Path(f"{cache}.md").write_text( - dumps(dict(date=datetime.now().timestamp(), checksum=_hash)), - encoding="utf-8", - ) - except: - return False, f"exception :\n{format_exc()}" - return ret, err - - -def get_version() -> str: - return Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text(encoding="utf-8").strip() - - -def get_integration() -> str: - try: - integration_path = Path(sep, "usr", "share", "bunkerweb", "INTEGRATION") - os_release_path = Path(sep, "etc", "os-release") - if getenv("KUBERNETES_MODE", "no").lower() == "yes": - return "Kubernetes" - elif getenv("SWARM_MODE", "no").lower() == "yes": - return "Swarm" - elif getenv("AUTOCONF_MODE", "no").lower() == "yes": - return "Autoconf" - elif integration_path.is_file(): - return integration_path.read_text(encoding="utf-8").strip().lower() - elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"): - return "Docker" - - return "Linux" - except: - return "Unknown" - - -def get_os_info() -> Dict[str, str]: - os_data = { - "name": "Linux", - "version": "Unknown", - "version_id": "Unknown", - "version_codename": "Unknown", - "id": "Unknown", - "arch": machine(), - } - - os_release = Path("/etc/os-release") - if os_release.exists(): - for line in os_release.read_text().splitlines(): - if "=" not in line or line.split("=")[0].strip().lower() not in os_data: - continue - os_data[line.split("=")[0].lower()] = line.split("=")[1].strip('"') - - return os_data + job = Job(None, db, deprecated=True) + job.logger.warning("cache_file is deprecated, use the Job.cache_file method instead.") + if not isinstance(file, Path): + file = Path(file) + if not isinstance(cache, Path): + cache = Path(cache) + return job.cache_file(cache.name, file, job_name=cache.name, service_id=service_id, checksum=_hash, delete_file=delete_file)