Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refactor all jobs

Browse source 
* Made it more readable
* Avoided duplication in the code as much as possible
* Optimized a few things...
This commit is contained in:
Théophile Diot 2024-03-10 15:48:15 +00:00
parent 9b10dd3030
commit a4aecabc33
No known key found for this signature in database
GPG key ID: 248FEA4BAE400D06
30 changed files with 1169 additions and 1435 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/common/confs/default-server-http.conf
View file

@ -17,7 +17,7 @@ server {
# HTTPS listen
{% set os = import("os") %}
{% if os.path.isfile("/var/cache/bunkerweb/default-server-cert/cert.pem") +%}
{% if os.path.isfile("/var/cache/bunkerweb/misc/default-server-cert.pem") +%}
ssl_protocols {{ SSL_PROTOCOLS }};
ssl_prefer_server_ciphers on;
ssl_session_tickets off;
@ -27,8 +27,8 @@ server {
ssl_dhparam /etc/nginx/dhparam;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
{% endif %}
ssl_certificate /var/cache/bunkerweb/default-server-cert/cert.pem;
ssl_certificate_key /var/cache/bunkerweb/default-server-cert/cert.key;
ssl_certificate /var/cache/bunkerweb/misc/default-server-cert.pem;
ssl_certificate_key /var/cache/bunkerweb/misc/default-server-cert.key;
listen 0.0.0.0:{{ HTTPS_PORT }} ssl {% if HTTP2 == "yes" %}http2{% endif %} default_server {% if USE_PROXY_PROTOCOL == "yes" %}proxy_protocol{% endif %};
{% if USE_IPV6 == "yes" +%}
listen [::]:{{ HTTPS_PORT }} ssl {% if HTTP2 == "yes" %}http2{% endif %} default_server {% if USE_PROXY_PROTOCOL == "yes" %}proxy_protocol{% endif %};

4
src/common/confs/server-http/ssl-certificate-lua.conf
View file

@ -1,5 +1,5 @@
ssl_certificate /var/cache/bunkerweb/default-server-cert/cert.pem;
ssl_certificate_key /var/cache/bunkerweb/default-server-cert/cert.key;
ssl_certificate /var/cache/bunkerweb/misc/default-server-cert.pem;
ssl_certificate_key /var/cache/bunkerweb/misc/default-server-cert.key;
ssl_protocols {{ SSL_PROTOCOLS }};
ssl_prefer_server_ciphers on;
ssl_session_tickets off;

4
src/common/confs/server-stream/ssl-certificate-stream-lua.conf
View file

@ -1,5 +1,5 @@
ssl_certificate /var/cache/bunkerweb/default-server-cert/cert.pem;
ssl_certificate_key /var/cache/bunkerweb/default-server-cert/cert.key;
ssl_certificate /var/cache/bunkerweb/misc/default-server-cert.pem;
ssl_certificate_key /var/cache/bunkerweb/misc/default-server-cert.key;
ssl_protocols {{ SSL_PROTOCOLS }};
ssl_prefer_server_ciphers on;
ssl_session_tickets off;

104
src/common/core/blacklist/jobs/blacklist-download.py
View file

@ -2,10 +2,9 @@
from contextlib import suppress
from ipaddress import ip_address, ip_network
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join, normpath
from pathlib import Path
from re import IGNORECASE, compile as re_compile
from re import compile as re_compile
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from typing import Tuple
@ -16,11 +15,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from requests import get
from Database import Database # type: ignore
from common_utils import bytes_hash # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, del_file_in_db, is_cached_file, file_hash
from jobs import Job # type: ignore
rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE)
rdns_rx = re_compile(rb"^[^ ]+$")
asn_rx = re_compile(rb"^\d+$")
uri_rx = re_compile(rb"^/")
@ -51,7 +50,7 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]:
return False, b""
logger = setup_logger("BLACKLIST", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("BLACKLIST", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
@ -68,16 +67,10 @@ try:
blacklist_activated = True
if not blacklist_activated:
logger.info("Blacklist is not activated, skipping downloads...")
_exit(0)
LOGGER.info("Blacklist is not activated, skipping downloads...")
sys_exit(0)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
# Create directories if they don't exist
blacklist_path = Path(sep, "var", "cache", "bunkerweb", "blacklist")
blacklist_path.mkdir(parents=True, exist_ok=True)
tmp_blacklist_path = Path(sep, "var", "tmp", "bunkerweb", "blacklist")
tmp_blacklist_path.mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Get URLs
urls = {
@ -110,37 +103,35 @@ try:
"IGNORE_USER_AGENT": True,
"IGNORE_URI": True,
}
all_fresh = True
for kind in kinds_fresh:
if not is_cached_file(blacklist_path.joinpath(f"{kind}.list"), "hour", db):
kinds_fresh[kind] = False
all_fresh = False
logger.info(
f"Blacklist for {kind} is not cached, processing downloads..",
)
else:
logger.info(
f"Blacklist for {kind} is already in cache, skipping downloads...",
)
if not urls[kind]:
logger.warning(
f"Blacklist for {kind} is cached but no URL is configured, removing from cache...",
)
blacklist_path.joinpath(f"{kind}.list").unlink(missing_ok=True)
deleted, err = del_file_in_db(f"{kind}.list", db)
if not deleted:
logger.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all_fresh:
_exit(0)
if not JOB.is_cached_file(f"{kind}.list", "hour"):
if urls[kind]:
kinds_fresh[kind] = False
LOGGER.info(f"Blacklist for {kind} is not cached, processing downloads..")
continue
LOGGER.info(f"Blacklist for {kind} is already in cache, skipping downloads...")
if not urls[kind]:
LOGGER.warning(f"Blacklist for {kind} is cached but no URL is configured, removing from cache...")
deleted, err = JOB.del_cache(f"{kind}.list")
if not deleted:
LOGGER.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all(kinds_fresh.values()):
if not any(urls.values()):
LOGGER.info("No blacklist URL is configured, nothing to do...")
sys_exit(0)
# Loop on kinds
for kind, urls_list in urls.items():
if kinds_fresh[kind]:
continue
# Write combined data of the kind to a single temp file
# Write combined data of the kind in memory and check if it has changed
for url in urls_list:
try:
logger.info(f"Downloading blacklist data from {url} ...")
LOGGER.info(f"Downloading blacklist data from {url} ...")
if url.startswith("file://"):
with open(normpath(url[7:]), "rb") as f:
iterable = f.readlines()
@ -148,7 +139,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -168,39 +159,28 @@ try:
content += data + b"\n"
i += 1
tmp_blacklist_path.joinpath(f"{kind}.list").write_bytes(content)
logger.info(f"Downloaded {i} bad {kind}")
LOGGER.info(f"Downloaded {i} bad {kind}")
# Check if file has changed
new_hash = file_hash(tmp_blacklist_path.joinpath(f"{kind}.list"))
old_hash = cache_hash(blacklist_path.joinpath(f"{kind}.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash(f"{kind}.list")
if new_hash == old_hash:
logger.info(
f"New file {kind}.list is identical to cache file, reload is not needed",
)
LOGGER.info(f"New file {kind}.list is identical to cache file, reload is not needed")
else:
logger.info(
f"New file {kind}.list is different than cache file, reload is needed",
)
LOGGER.info(f"New file {kind}.list is different than cache file, reload is needed")
# Put file in cache
cached, err = cache_file(
tmp_blacklist_path.joinpath(f"{kind}.list"),
blacklist_path.joinpath(f"{kind}.list"),
new_hash,
db,
)
cached, err = JOB.cache_file(f"{kind}.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching blacklist : {err}")
LOGGER.error(f"Error while caching blacklist : {err}")
status = 2
else:
status = 1
except:
status = 2
logger.error(f"Exception while getting blacklist from {url} :\n{format_exc()}")
LOGGER.error(f"Exception while getting blacklist from {url} :\n{format_exc()}")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running blacklist-download.py :\n{format_exc()}")
LOGGER.error(f"Exception while running blacklist-download.py :\n{format_exc()}")
sys_exit(status)

115
src/common/core/bunkernet/jobs/bunkernet-data.py
View file

@ -1,29 +1,21 @@
#!/usr/bin/env python3
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
("core", "bunkernet", "jobs"),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from bunkernet import data
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, file_hash, is_cached_file, get_file_in_db
from jobs import Job # type: ignore
from common_utils import bytes_hash # type: ignore
logger = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO"))
exit_status = 0
try:
@ -40,109 +32,90 @@ try:
bunkernet_activated = True
if not bunkernet_activated:
logger.info("BunkerNet is not activated, skipping download...")
_exit(0)
LOGGER.info("BunkerNet is not activated, skipping download...")
sys_exit(0)
# Create directory if it doesn't exist
bunkernet_path = Path(sep, "var", "cache", "bunkerweb", "bunkernet")
bunkernet_path.mkdir(parents=True, exist_ok=True)
bunkernet_tmp_path = Path(sep, "var", "tmp", "bunkerweb", "bunkernet")
bunkernet_tmp_path.mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Create empty file in case it doesn't exist
bunkernet_path.joinpath("ip.list").touch(exist_ok=True)
ip_list_path = bunkernet_path.joinpath("ip.list")
if not ip_list_path.is_file():
ip_list_path.touch(exist_ok=True)
# Get ID from cache
bunkernet_id = None
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
bunkernet_id = get_file_in_db("instance.id", db)
bunkernet_id = JOB.get_cache("instance.id")
if bunkernet_id:
bunkernet_path.joinpath("instance.id").write_bytes(bunkernet_id)
logger.info("Successfully retrieved BunkerNet ID from db cache")
LOGGER.info("Successfully retrieved BunkerNet ID from db cache")
else:
logger.info("No BunkerNet ID found in db cache")
LOGGER.info("No BunkerNet ID found in db cache")
# Check if ID is present
if not bunkernet_path.joinpath("instance.id").is_file():
logger.error(
"Not downloading BunkerNet data because instance is not registered",
)
_exit(2)
LOGGER.error("Not downloading BunkerNet data because instance is not registered")
sys_exit(2)
# Don't go further if the cache is fresh
if is_cached_file(bunkernet_path.joinpath("ip.list"), "day", db):
logger.info(
"BunkerNet list is already in cache, skipping download...",
)
_exit(0)
if JOB.is_cached_file("ip.list", "day"):
LOGGER.info("BunkerNet list is already in cache, skipping download...")
sys_exit(0)
exit_status = 1
# Download data
logger.info("Downloading BunkerNet data ...")
LOGGER.info("Downloading BunkerNet data ...")
ok, status, data = data()
if not ok:
logger.error(
f"Error while sending data request to BunkerNet API : {data}",
)
_exit(2)
LOGGER.error(f"Error while sending data request to BunkerNet API : {data}")
sys_exit(2)
elif status == 429:
logger.warning(
"BunkerNet API is rate limiting us, trying again later...",
)
_exit(0)
LOGGER.warning("BunkerNet API is rate limiting us, trying again later...")
sys_exit(0)
elif status == 403:
logger.warning(
"BunkerNet has banned this instance, retrying a register later...",
)
_exit(0)
LOGGER.warning("BunkerNet has banned this instance, retrying a register later...")
sys_exit(0)
try:
assert isinstance(data, dict)
except AssertionError:
logger.error(
f"Received invalid data from BunkerNet API while sending db request : {data}",
)
_exit(2)
LOGGER.error(f"Received invalid data from BunkerNet API while sending db request : {data}")
sys_exit(2)
if data["result"] != "ok":
logger.error(
f"Received error from BunkerNet API while sending db request : {data['data']}, removing instance ID",
)
_exit(2)
LOGGER.error(f"Received error from BunkerNet API while sending db request : {data['data']}, removing instance ID")
sys_exit(2)
logger.info("Successfully downloaded data from BunkerNet API")
LOGGER.info("Successfully downloaded data from BunkerNet API")
# Writing data to file
logger.info("Saving BunkerNet data ...")
LOGGER.info("Saving BunkerNet data ...")
content = "\n".join(data["data"]).encode("utf-8")
bunkernet_tmp_path.joinpath("ip.list").write_bytes(content)
# Check if file has changed
new_hash = file_hash(bunkernet_tmp_path.joinpath("ip.list"))
old_hash = cache_hash(bunkernet_path.joinpath("ip.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash("ip.list")
if new_hash == old_hash:
logger.info(
"New file is identical to cache file, reload is not needed",
)
_exit(0)
LOGGER.info("New file is identical to cache file, reload is not needed")
sys_exit(0)
# Put file in cache
cached, err = cache_file(
bunkernet_tmp_path.joinpath("ip.list"),
bunkernet_path.joinpath("ip.list"),
new_hash,
db,
)
cached, err = JOB.cache_file("ip.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching BunkerNet data : {err}")
_exit(2)
LOGGER.error(f"Error while caching BunkerNet data : {err}")
sys_exit(2)
logger.info("Successfully saved BunkerNet data")
LOGGER.info("Successfully saved BunkerNet data")
exit_status = 1
except SystemExit as e:
exit_status = e.code
except:
exit_status = 2
logger.error(f"Exception while running bunkernet-data.py :\n{format_exc()}")
LOGGER.error(f"Exception while running bunkernet-data.py :\n{format_exc()}")
sys_exit(exit_status)

151
src/common/core/bunkernet/jobs/bunkernet-register.py
View file

@ -1,30 +1,19 @@
#!/usr/bin/env python3
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from time import sleep
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
("core", "bunkernet", "jobs"),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from bunkernet import register, ping
from Database import Database # type: ignore
from bunkernet import register
from logger import setup_logger # type: ignore
from jobs import get_file_in_db, set_file_in_db, del_file_in_db # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO"))
exit_status = 0
try:
@ -41,132 +30,62 @@ try:
bunkernet_activated = True
if not bunkernet_activated:
logger.info("BunkerNet is not activated, skipping registration...")
_exit(0)
# Create directory if it doesn't exist
bunkernet_path = Path(sep, "var", "cache", "bunkerweb", "bunkernet")
bunkernet_path.mkdir(parents=True, exist_ok=True)
LOGGER.info("BunkerNet is not activated, skipping registration...")
sys_exit(0)
# Get ID from cache
bunkernet_id = None
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
bunkernet_id = get_file_in_db("instance.id", db)
if bunkernet_id:
bunkernet_path.joinpath("instance.id").write_bytes(bunkernet_id)
logger.info("Successfully retrieved BunkerNet ID from db cache")
else:
logger.info("No BunkerNet ID found in db cache")
JOB = Job(LOGGER)
bunkernet_id = JOB.get_cache("instance.id")
# Register instance
registered = False
instance_id_path = bunkernet_path.joinpath("instance.id")
if not instance_id_path.is_file():
logger.info("Registering instance on BunkerNet API ...")
if not bunkernet_id:
LOGGER.info("No BunkerNet ID found in db cache, Registering instance on BunkerNet API ...")
ok, status, data = register()
if not ok:
logger.error(f"Error while sending register request to BunkerNet API : {data}")
_exit(2)
LOGGER.error(f"Error while sending register request to BunkerNet API : {data}")
sys_exit(2)
elif status == 429:
logger.warning(
"BunkerNet API is rate limiting us, trying again later...",
)
_exit(0)
LOGGER.warning("BunkerNet API is rate limiting us, trying again later...")
sys_exit(0)
elif status == 403:
logger.warning(
"BunkerNet has banned this instance, retrying a register later...",
)
_exit(0)
LOGGER.warning("BunkerNet has banned this instance, retrying a register later...")
sys_exit(0)
try:
assert isinstance(data, dict)
except AssertionError:
logger.error(
f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...",
)
_exit(2)
LOGGER.error(f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...")
sys_exit(2)
bunkernet_id = data.get("data")
if status != 200:
logger.error(
f"Error {status} from BunkerNet API : {data['data']}",
)
_exit(2)
LOGGER.error(f"Error {status} from BunkerNet API : {bunkernet_id}")
sys_exit(2)
elif data.get("result", "ko") != "ok":
logger.error(f"Received error from BunkerNet API while sending register request : {data.get('data', {})}")
_exit(2)
bunkernet_id = data["data"]
instance_id_path.write_text(bunkernet_id, encoding="utf-8")
LOGGER.error(f"Received error from BunkerNet API while sending register request : {bunkernet_id}")
sys_exit(2)
assert isinstance(bunkernet_id, str), f"Received invalid bunkernet id : {bunkernet_id}"
registered = True
exit_status = 1
logger.info(f"Successfully registered on BunkerNet API with instance id {data['data']}")
LOGGER.info(f"Successfully registered on BunkerNet API with instance id {data['data']}")
else:
bunkernet_id = bunkernet_id or instance_id_path.read_bytes()
bunkernet_id = bunkernet_id.decode()
logger.info(f"Already registered on BunkerNet API with instance id {bunkernet_id}")
sleep(1)
LOGGER.info(f"Already registered on BunkerNet API with instance id {bunkernet_id}")
# Update cache with new bunkernet ID
if registered:
cached, err = set_file_in_db("instance.id", bunkernet_id.encode(), db)
cached, err = JOB.cache_file("instance.id", bunkernet_id.encode())
if not cached:
logger.error(f"Error while saving BunkerNet data to db cache : {err}")
LOGGER.error(f"Error while saving BunkerNet data to db cache : {err}")
else:
logger.info("Successfully saved BunkerNet data to db cache")
# Ping
logger.info("Checking connectivity with BunkerNet API ...")
bunkernet_ping = False
for i in range(0, 5):
ok, status, data = ping(bunkernet_id)
retry = False
if not ok:
logger.error(f"Error while sending ping request to BunkerNet API : {data}")
retry = True
elif status == 429:
logger.warning(
"BunkerNet API is rate limiting us, trying again later...",
)
retry = True
elif status == 403:
logger.warning(
"BunkerNet has banned this instance, retrying a register later...",
)
_exit(2)
elif status == 401:
logger.warning(
"Instance ID is not registered, removing it and retrying a register later...",
)
instance_id_path.unlink()
del_file_in_db("instance.id", db)
_exit(2)
try:
assert isinstance(data, dict)
except AssertionError:
logger.error(
f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...",
)
_exit(2)
if data.get("result", "ko") != "ok":
logger.error(
f"Received error from BunkerNet API while sending ping request : {data.get('data', {})}",
)
retry = True
if not retry:
bunkernet_ping = True
break
logger.warning("Waiting 1s and trying again ...")
sleep(1)
if bunkernet_ping:
logger.info("Connectivity with BunkerNet is successful !")
else:
logger.error("Connectivity with BunkerNet failed ...")
exit_status = 2
LOGGER.info("Successfully saved BunkerNet data to db cache")
except SystemExit as e:
exit_status = e.code
except:
exit_status = 2
logger.error(f"Exception while running bunkernet-register.py :\n{format_exc()}")
LOGGER.error(f"Exception while running bunkernet-register.py :\n{format_exc()}")
sys_exit(exit_status)

7
src/common/core/bunkernet/jobs/bunkernet.py
View file

@ -5,10 +5,10 @@ from pathlib import Path
from requests import request as requests_request, ReadTimeout
from typing import Literal, Optional, Tuple, Union
from jobs import get_os_info, get_integration, get_version # type: ignore
from common_utils import get_os_info, get_integration, get_version # type: ignore
def request(method: Union[Literal["POST"], Literal["GET"]], url: str, _id: Optional[str] = None) -> Tuple[bool, Optional[int], Union[str, dict]]:
def request(method: Literal["POST", "GET"], url: str, _id: Optional[str] = None) -> Tuple[bool, Optional[int], Union[str, dict]]:
data = {
"integration": get_integration(),
"version": get_version(),
@ -17,13 +17,12 @@ def request(method: Union[Literal["POST"], Literal["GET"]], url: str, _id: Optio
if _id:
data["id"] = _id
headers = {"User-Agent": f"BunkerWeb/{data['version']}"}
try:
resp = requests_request(
method,
f"{getenv('BUNKERNET_SERVER', 'https://api.bunkerweb.io')}{url}",
json=data,
headers=headers,
headers={"User-Agent": f"BunkerWeb/{data['version']}"},
timeout=5,
)
status = resp.status_code

8
src/common/core/customcert/customcert.lua
View file

@ -44,8 +44,8 @@ function customcert:init()
for server_name, multisite_vars in pairs(vars) do
if multisite_vars["USE_CUSTOM_SSL"] == "yes" and server_name ~= "global" then
local check, data = read_files({
"/var/cache/bunkerweb/customcert/" .. server_name .. ".cert.pem",
"/var/cache/bunkerweb/customcert/" .. server_name .. ".key.pem",
"/var/cache/bunkerweb/customcert/" .. server_name .. "/cert.pem",
"/var/cache/bunkerweb/customcert/" .. server_name .. "/key.pem",
})
if not check then
self.logger:log(ERR, "error while reading files : " .. data)
@ -68,8 +68,8 @@ function customcert:init()
return self:ret(false, "can't get SERVER_NAME variable : " .. err)
end
local check, data = read_files({
"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. ".cert.pem",
"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. ".key.pem",
"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/cert.pem",
"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/key.pem",
})
if not check then
self.logger:log(ERR, "error while reading files : " .. data)

271
src/common/core/customcert/jobs/custom-cert.py
View file

@ -1,220 +1,141 @@
#!/usr/bin/env python3
from contextlib import suppress
from os import getenv, sep
from os.path import join, normpath
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from base64 import b64decode
from typing import Tuple, Union
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from jobs import del_file_in_db, cache_file, cache_hash, file_hash
from Database import Database # type: ignore
from common_utils import bytes_hash # type: ignore
from jobs import Job # type: ignore
from logger import setup_logger # type: ignore
logger = setup_logger("CUSTOM-CERT", getenv("LOG_LEVEL", "INFO"))
db = None
LOGGER = setup_logger("CUSTOM-CERT", getenv("LOG_LEVEL", "INFO"))
JOB = Job(LOGGER)
def check_cert(cert_path: str, key_path: str, first_server: str) -> bool:
try:
def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], first_server: str) -> Tuple[bool, str]:
with suppress(BaseException):
ret = False
if not cert_path or not key_path:
logger.warning("Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates")
return False
if not cert_file or not key_file:
return False, "Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates"
cert_path: Path = Path(normpath(cert_path))
key_path: Path = Path(normpath(key_path))
if isinstance(cert_file, Path):
if not cert_file.is_file():
return False, f"Certificate file {cert_file} is not a valid file, ignoring the custom certificate"
cert_file = cert_file.read_bytes()
if not cert_path.is_file():
logger.warning(f"Certificate file {cert_path} is not a valid file, ignoring the custom certificate")
return False
elif not key_path.is_file():
logger.warning(f"Key file {key_path} is not a valid file, ignoring the custom certificate")
return False
if isinstance(key_file, Path):
if not key_file.is_file():
return False, f"Key file {key_file} is not a valid file, ignoring the custom certificate"
key_file = key_file.read_bytes()
cert_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.cert.pem",
)
cert_cache_path.parent.mkdir(parents=True, exist_ok=True)
cert_hash = file_hash(cert_path)
old_hash = cache_hash(cert_cache_path, db)
cert_hash = bytes_hash(cert_file)
old_hash = JOB.cache_hash("cert.pem", service_id=first_server)
if old_hash != cert_hash:
ret = True
cached, err = cache_file(cert_path, cert_cache_path, cert_hash, db, delete_file=False)
cached, err = JOB.cache_file("cert.pem", cert_file, service_id=first_server, checksum=cert_hash, delete_file=False)
if not cached:
logger.error(f"Error while caching custom-cert cert.pem file : {err}")
elif not cert_cache_path.is_file():
cert_cache_path.write_bytes(cert_path.read_bytes())
ret = True
key_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.key.pem",
)
key_cache_path.parent.mkdir(parents=True, exist_ok=True)
LOGGER.error(f"Error while caching custom-cert cert.pem file : {err}")
key_hash = file_hash(key_path)
old_hash = cache_hash(key_cache_path, db)
key_hash = bytes_hash(key_file)
old_hash = JOB.cache_hash("key.pem", service_id=first_server)
if old_hash != key_hash:
ret = True
cached, err = cache_file(key_path, key_cache_path, key_hash, db, delete_file=False)
cached, err = JOB.cache_file("key.pem", key_file, service_id=first_server, checksum=key_hash, delete_file=False)
if not cached:
logger.error(f"Error while caching custom-cert key.pem file : {err}")
elif not key_cache_path.is_file():
key_cache_path.write_bytes(key_path.read_bytes())
ret = True
LOGGER.error(f"Error while caching custom-key key.pem file : {err}")
return ret
except:
logger.error(
f"Exception while running custom-cert.py (check_cert) :\n{format_exc()}",
)
return False
return ret, ""
return False, "exception"
status = 0
try:
Path(sep, "var", "cache", "bunkerweb", "customcert").mkdir(parents=True, exist_ok=True)
all_domains = getenv("SERVER_NAME") or []
if getenv("MULTISITE", "no") == "no" and getenv("USE_CUSTOM_SSL", "no") == "yes" and getenv("SERVER_NAME", "") != "":
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
if isinstance(all_domains, str):
all_domains = all_domains.split(" ")
cert_path = getenv("CUSTOM_SSL_CERT", "")
key_path = getenv("CUSTOM_SSL_KEY", "")
first_server = getenv("SERVER_NAME").split(" ")[0]
if not all_domains:
LOGGER.warning("No services found, exiting ...")
sys_exit(0)
cert_data = b64decode(getenv("CUSTOM_SSL_CERT_DATA", ""))
key_data = b64decode(getenv("CUSTOM_SSL_KEY_DATA", ""))
for file, data in (("cert.pem", cert_data), ("key.pem", key_data)):
if data:
file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", f"{first_server}.{file}")
file_path.write_bytes(data)
if file == "cert.pem":
cert_path = str(file_path)
else:
key_path = str(file_path)
skipped_servers = []
if not getenv("MULTISITE", "no") == "yes":
all_domains = [all_domains[0]]
if getenv("USE_CUSTOM_SSL", "no") == "no":
LOGGER.info("Custom SSL is not enabled, skipping ...")
skipped_servers = all_domains
if cert_path and key_path:
logger.info(f"Checking certificate {cert_path} ...")
need_reload = check_cert(cert_path, key_path, first_server)
if need_reload:
logger.info(f"Detected change for certificate {cert_path}")
status = 1
else:
logger.info(f"No change for certificate {cert_path}")
elif not cert_path or not key_path:
logger.warning(
"Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY (or CUSTOM_SSL_CERT_DATA and CUSTOM_SSL_KEY_DATA) have to be set to use custom certificates, clearing cache ..."
)
cert_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.cert.pem",
)
cert_cache_path.unlink(missing_ok=True)
del_file_in_db(f"{first_server}.cert.pem", db, service_id=first_server)
key_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.key.pem",
)
key_cache_path.unlink(missing_ok=True)
del_file_in_db(f"{first_server}.key.pem", db, service_id=first_server)
elif getenv("MULTISITE", "no") == "yes":
servers = getenv("SERVER_NAME") or []
if isinstance(servers, str):
servers = servers.split(" ")
for first_server in servers:
if not first_server or (getenv(f"{first_server}_USE_CUSTOM_SSL", getenv("USE_CUSTOM_SSL", "no")) != "yes"):
if not skipped_servers:
for first_server in all_domains:
if getenv(f"{first_server}_USE_CUSTOM_SSL", getenv("USE_CUSTOM_SSL", "no")) == "no":
LOGGER.info(f"Custom SSL is not enabled for {first_server}, skipping ...")
skipped_servers.append(first_server)
continue
if not db:
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
cert_file = getenv(f"{first_server}_CUSTOM_SSL_CERT", getenv("CUSTOM_SSL_CERT", ""))
key_file = getenv(f"{first_server}_CUSTOM_SSL_KEY", getenv("CUSTOM_SSL_KEY", ""))
cert_data = getenv(f"{first_server}_CUSTOM_SSL_CERT_DATA", getenv("CUSTOM_SSL_CERT_DATA", ""))
key_data = getenv(f"{first_server}_CUSTOM_SSL_KEY_DATA", getenv("CUSTOM_SSL_KEY_DATA", ""))
cert_path = getenv(f"{first_server}_CUSTOM_SSL_CERT", "")
key_path = getenv(f"{first_server}_CUSTOM_SSL_KEY", "")
cert_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_CERT_DATA", ""))
key_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_KEY_DATA", ""))
for file, data in (("cert.pem", cert_data), ("key.pem", key_data)):
if data != b"":
file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", f"{first_server}.{file}")
file_path.parent.mkdir(parents=True, exist_ok=True)
file_path.write_bytes(data)
if file == "cert.pem":
cert_path = str(file_path)
else:
key_path = str(file_path)
if cert_path and key_path:
logger.info(
f"Checking certificate {cert_path} ...",
)
need_reload = check_cert(cert_path, key_path, first_server)
if need_reload:
logger.info(
f"Detected change for certificate {cert_path}",
)
status = 1
if cert_file or cert_data and key_file or key_data:
if isinstance(cert_file, str):
cert_file = Path(cert_file)
else:
logger.info(
f"No change for certificate {cert_path}",
)
elif not cert_path or not key_path:
logger.warning(
"Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY (or CUSTOM_SSL_CERT_DATA and CUSTOM_SSL_KEY_DATA) have to be set to use custom certificates, clearing cache ..."
try:
cert_file = b64decode(cert_data)
except BaseException:
LOGGER.exception(f"Error while decoding cert data, skipping server {first_server}...")
skipped_servers.append(first_server)
continue
if isinstance(key_file, str):
key_file = Path(key_file)
else:
try:
key_file = b64decode(key_data)
except BaseException:
LOGGER.exception(f"Error while decoding key data, skipping server {first_server}...")
skipped_servers.append(first_server)
continue
LOGGER.info(f"Checking certificate for {first_server} ...")
need_reload, err = check_cert(cert_file, key_file, first_server)
if err == "exception":
LOGGER.exception(f"Exception while checking {first_server}'s certificate, skipping ...")
skipped_servers.append(first_server)
continue
elif err:
LOGGER.warning(f"Error while checking {first_server}'s certificate : {err}")
skipped_servers.append(first_server)
continue
elif need_reload:
LOGGER.info(f"Detected change in {first_server}'s certificate")
status = 1
continue
LOGGER.info(f"No change in {first_server}'s certificate")
elif not cert_file or not key_file:
LOGGER.warning(
"Variables (CUSTOM_SSL_CERT or CUSTOM_SSL_CERT_DATA) and (CUSTOM_SSL_KEY or CUSTOM_SSL_KEY_DATA) have to be set to use custom certificates"
)
cert_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.cert.pem",
)
cert_cache_path.unlink(missing_ok=True)
del_file_in_db(f"{first_server}.cert.pem", db)
key_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.key.pem",
)
key_cache_path.unlink(missing_ok=True)
del_file_in_db(f"{first_server}.key.pem", db)
skipped_servers.append(first_server)
for first_server in skipped_servers:
JOB.del_cache("cert.pem", service_id=first_server)
JOB.del_cache("key.pem", service_id=first_server)
except:
status = 2
logger.error(f"Exception while running custom-cert.py :\n{format_exc()}")
LOGGER.error(f"Exception while running custom-cert.py :\n{format_exc()}")
sys_exit(status)

111
src/common/core/greylist/jobs/greylist-download.py
View file

@ -2,10 +2,9 @@
from contextlib import suppress
from ipaddress import ip_address, ip_network
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join, normpath
from pathlib import Path
from re import IGNORECASE, compile as re_compile
from re import compile as re_compile
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from typing import Tuple
@ -16,11 +15,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from requests import get
from Database import Database # type: ignore
from common_utils import bytes_hash # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, del_file_in_db, is_cached_file, file_hash
from jobs import Job # type: ignore
rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE)
rdns_rx = re_compile(rb"^[^ ]+$")
asn_rx = re_compile(rb"^\d+$")
uri_rx = re_compile(rb"^/")
@ -51,7 +50,7 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]:
return False, b""
logger = setup_logger("GREYLIST", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("GREYLIST", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
@ -68,16 +67,10 @@ try:
greylist_activated = True
if not greylist_activated:
logger.info("Greylist is not activated, skipping downloads...")
_exit(0)
LOGGER.info("Greylist is not activated, skipping downloads...")
sys_exit(0)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
# Create directories if they don't exist
greylist_path = Path(sep, "var", "cache", "bunkerweb", "greylist")
greylist_path.mkdir(parents=True, exist_ok=True)
tmp_greylist_path = Path(sep, "var", "tmp", "bunkerweb", "greylist")
tmp_greylist_path.mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Get URLs
urls = {"IP": [], "RDNS": [], "ASN": [], "USER_AGENT": [], "URI": []}
@ -87,45 +80,36 @@ try:
urls[kind].append(url)
# Don't go further if the cache is fresh
kinds_fresh = {
"IP": True,
"RDNS": True,
"ASN": True,
"USER_AGENT": True,
"URI": True,
}
all_fresh = True
kinds_fresh = {"IP": True, "RDNS": True, "ASN": True, "USER_AGENT": True, "URI": True}
for kind in kinds_fresh:
if not is_cached_file(greylist_path.joinpath(f"{kind}.list"), "hour", db):
kinds_fresh[kind] = False
all_fresh = False
logger.info(
f"Greylist for {kind} is not cached, processing downloads..",
)
else:
logger.info(
f"Greylist for {kind} is already in cache, skipping downloads...",
)
if not urls[kind]:
logger.warning(
f"Greylist for {kind} is cached but no URL is configured, removing from cache...",
)
greylist_path.joinpath(f"{kind}.list").unlink(missing_ok=True)
deleted, err = del_file_in_db(f"{kind}.list", db)
if not deleted:
logger.warning(f"Couldn't delete {kind}.list from cache : {err}")
if not JOB.is_cached_file(f"{kind}.list", "hour"):
if urls[kind]:
kinds_fresh[kind] = False
LOGGER.info(f"Greylist for {kind} is not cached, processing downloads..")
continue
if all_fresh:
_exit(0)
LOGGER.info(f"Greylist for {kind} is already in cache, skipping downloads...")
if not urls[kind]:
LOGGER.warning(f"Greylist for {kind} is cached but no URL is configured, removing from cache...")
deleted, err = JOB.del_cache(f"{kind}.list")
if not deleted:
LOGGER.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all(kinds_fresh.values()):
if not any(urls.values()):
LOGGER.info("No greylist URL is configured, nothing to do...")
sys_exit(0)
# Loop on kinds
for kind, urls_list in urls.items():
if kinds_fresh[kind]:
continue
# Write combined data of the kind to a single temp file
# Write combined data of the kind in memory and check if it has changed
for url in urls_list:
try:
logger.info(f"Downloading greylist data from {url} ...")
LOGGER.info(f"Downloading greylist data from {url} ...")
if url.startswith("file://"):
with open(normpath(url[7:]), "rb") as f:
iterable = f.readlines()
@ -133,7 +117,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -153,39 +137,28 @@ try:
content += data + b"\n"
i += 1
tmp_greylist_path.joinpath(f"{kind}.list").write_bytes(content)
logger.info(f"Downloaded {i} bad {kind}")
LOGGER.info(f"Downloaded {i} bad {kind}")
# Check if file has changed
new_hash = file_hash(tmp_greylist_path.joinpath(f"{kind}.list"))
old_hash = cache_hash(greylist_path.joinpath(f"{kind}.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash(f"{kind}.list")
if new_hash == old_hash:
logger.info(
f"New file {kind}.list is identical to cache file, reload is not needed",
)
LOGGER.info(f"New file {kind}.list is identical to cache file, reload is not needed")
else:
logger.info(
f"New file {kind}.list is different than cache file, reload is needed",
)
LOGGER.info(f"New file {kind}.list is different than cache file, reload is needed")
# Put file in cache
cached, err = cache_file(
tmp_greylist_path.joinpath(f"{kind}.list"),
greylist_path.joinpath(f"{kind}.list"),
new_hash,
db,
)
cached, err = JOB.cache_file(f"{kind}.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching greylist : {err}")
LOGGER.error(f"Error while caching greylist : {err}")
status = 2
else:
status = 1
except:
status = 2
logger.error(f"Exception while getting greylist from {url} :\n{format_exc()}")
LOGGER.error(f"Exception while getting greylist from {url} :\n{format_exc()}")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running greylist-download.py :\n{format_exc()}")
LOGGER.error(f"Exception while running greylist-download.py :\n{format_exc()}")
sys_exit(status)

40
src/common/core/jobs/jobs/download-plugins.py
View file

@ -36,7 +36,7 @@ from logger import setup_logger # type: ignore
EXTERNAL_PLUGINS_DIR = Path(sep, "etc", "bunkerweb", "plugins")
logger = setup_logger("Jobs.download-plugins", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Jobs.download-plugins", getenv("LOG_LEVEL", "INFO"))
status = 0
@ -45,14 +45,14 @@ def install_plugin(plugin_dir: str, db) -> bool:
plugin_file = plugin_path.joinpath("plugin.json")
if not plugin_file.is_file():
logger.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json not found)")
LOGGER.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json not found)")
return False
# Load plugin.json
try:
metadata = loads(plugin_file.read_text(encoding="utf-8"))
except JSONDecodeError:
logger.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json is not valid)")
LOGGER.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json is not valid)")
return False
# Don't go further if plugin is already installed
@ -65,12 +65,12 @@ def install_plugin(plugin_dir: str, db) -> bool:
break
if old_version == metadata["version"]:
logger.warning(
LOGGER.warning(
f"Skipping installation of plugin {metadata['id']} (version {metadata['version']} already installed)",
)
return False
logger.warning(
LOGGER.warning(
f"Plugin {metadata['id']} is already installed but version {metadata['version']} is different from database ({old_version}), updating it...",
)
rmtree(EXTERNAL_PLUGINS_DIR.joinpath(metadata["id"]), ignore_errors=True)
@ -81,7 +81,7 @@ def install_plugin(plugin_dir: str, db) -> bool:
for job_file in glob(join(sep, "etc", "bunkerweb", "plugins", "jobs", "*")):
st = Path(job_file).stat()
chmod(job_file, st.st_mode | S_IEXEC)
logger.info(f"Plugin {metadata['id']} installed")
LOGGER.info(f"Plugin {metadata['id']} installed")
return True
@ -89,14 +89,14 @@ try:
# Check if we have plugins to download
plugin_urls = getenv("EXTERNAL_PLUGIN_URLS")
if not plugin_urls:
logger.info("No external plugins to download")
LOGGER.info("No external plugins to download")
sys_exit(0)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
plugin_nbr = 0
# Loop on URLs
logger.info(f"Downloading external plugins from {plugin_urls}...")
LOGGER.info(f"Downloading external plugins from {plugin_urls}...")
for plugin_url in plugin_urls.split(" "):
# Download Plugin file
try:
@ -112,7 +112,7 @@ try:
)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
# Iterate over the response content in chunks
@ -120,7 +120,7 @@ try:
if chunk:
content += chunk
except:
logger.error(
LOGGER.error(
f"Exception while downloading plugin(s) from {plugin_url} :\n{format_exc()}",
)
status = 2
@ -142,10 +142,10 @@ try:
with tar_open(fileobj=BytesIO(content), mode="r") as tar:
tar.extractall(path=temp_dir)
else:
logger.error(f"Unknown file type for {plugin_url}, either zip or tar are supported, skipping...")
LOGGER.error(f"Unknown file type for {plugin_url}, either zip or tar are supported, skipping...")
continue
except:
logger.error(f"Exception while decompressing plugin(s) from {plugin_url} :\n{format_exc()}")
LOGGER.error(f"Exception while decompressing plugin(s) from {plugin_url} :\n{format_exc()}")
status = 2
continue
@ -156,13 +156,13 @@ try:
if install_plugin(plugin_dir, db):
plugin_nbr += 1
except FileExistsError:
logger.warning(f"Skipping installation of plugin {basename(plugin_dir)} (already installed)")
LOGGER.warning(f"Skipping installation of plugin {basename(plugin_dir)} (already installed)")
except:
logger.error(f"Exception while installing plugin(s) from {plugin_url} :\n{format_exc()}")
LOGGER.error(f"Exception while installing plugin(s) from {plugin_url} :\n{format_exc()}")
status = 2
if not plugin_nbr:
logger.info("No external plugins to update to database")
LOGGER.info("No external plugins to update to database")
sys_exit(0)
external_plugins = []
@ -170,7 +170,7 @@ try:
for plugin in listdir(EXTERNAL_PLUGINS_DIR):
path = EXTERNAL_PLUGINS_DIR.joinpath(plugin)
if not path.joinpath("plugin.json").is_file():
logger.warning(f"Plugin {plugin} is not valid, deleting it...")
LOGGER.warning(f"Plugin {plugin} is not valid, deleting it...")
rmtree(path, ignore_errors=True)
continue
@ -208,18 +208,18 @@ try:
err = db.update_external_plugins(external_plugins)
if err:
logger.error(
LOGGER.error(
f"Couldn't update external plugins to database: {err}",
)
status = 1
logger.info("External plugins downloaded and installed")
LOGGER.info("External plugins downloaded and installed")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running download-plugins.py :\n{format_exc()}")
LOGGER.error(f"Exception while running download-plugins.py :\n{format_exc()}")
for plugin_tmp in glob(join(sep, "var", "tmp", "bunkerweb", "plugins", "*")):
rmtree(plugin_tmp, ignore_errors=True)

148
src/common/core/jobs/jobs/mmdb-asn.py
View file

@ -1,14 +1,16 @@
#!/usr/bin/env python3
from datetime import date
from datetime import date, datetime, timedelta
from gzip import decompress
from hashlib import sha1
from os import _exit, getenv, sep
from io import BytesIO
from os import getenv, sep
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
from typing import Optional, Union
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
@ -21,107 +23,141 @@ for deps_path in [
if deps_path not in sys_path:
sys_path.append(deps_path)
from maxminddb import open_database
from requests import RequestException, get
from maxminddb import MODE_FD, open_database
from requests import RequestException, Response, get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, file_hash, is_cached_file
from common_utils import bytes_hash # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("JOBS.mmdb-asn", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("JOBS.mmdb-asn", getenv("LOG_LEVEL", "INFO"))
status = 0
lock = Lock()
LOCK = Lock()
def request_mmdb() -> Optional[Response]:
try:
response = get("https://db-ip.com/db/download/ip-to-asn-lite", timeout=5)
response.raise_for_status()
return response
except RequestException:
return None
def bytes_sha1(bio: Union[Path, bytes, BytesIO]) -> str:
if isinstance(bio, Path):
bio = bio.read_bytes()
if isinstance(bio, bytes):
bio = BytesIO(bio)
assert isinstance(bio, BytesIO)
_sha512 = sha1()
while True:
data = bio.read(1024)
if not data:
break
_sha512.update(data)
bio.seek(0)
return _sha512.hexdigest()
try:
dl_mmdb = True
tmp_path = Path(sep, "var", "tmp", "bunkerweb", "asn.mmdb")
cache_path = Path(sep, "var", "cache", "bunkerweb", "asn.mmdb")
new_hash = None
# Don't go further if the cache match the latest version
if tmp_path.exists():
with lock:
response = None
try:
response = get("https://db-ip.com/db/download/ip-to-asn-lite", timeout=5)
except RequestException:
logger.warning("Unable to check if asn.mmdb is the latest version")
response = None
if tmp_path.is_file():
response = request_mmdb()
if response and response.status_code == 200:
_sha1 = sha1()
with tmp_path.open("rb") as f:
while True:
data = f.read(1024)
if not data:
break
_sha1.update(data)
if response.content.decode().find(_sha1.hexdigest()) != -1:
logger.info("asn.mmdb is already the latest version, skipping download...")
if response.content.find(bytes_sha1(tmp_path).encode()) != -1:
LOGGER.info("asn.mmdb is already the latest version, skipping download...")
dl_mmdb = False
else:
logger.warning("Unable to check if asn.mmdb is the latest version, downloading it anyway...")
LOGGER.warning("Unable to check if the temporary mmdb file is the latest version, downloading it anyway...")
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
JOB = Job(LOGGER)
if dl_mmdb:
# Don't go further if the cache is fresh
if is_cached_file(cache_path, "month", db):
logger.info("asn.mmdb is already in cache, skipping download...")
_exit(0)
job_cache = JOB.get_cache("asn.mmdb", with_info=True, with_data=True)
if isinstance(job_cache, dict):
skip_dl = True
if response is None:
response = request_mmdb()
if response and response.status_code == 200:
skip_dl = response.content.find(bytes_sha1(job_cache["data"]).encode()) != -1
elif job_cache["last_update"] < (datetime.now() - timedelta(weeks=1)).timestamp():
LOGGER.warning("Unable to check if the cache file is the latest version from db-ip.com and file is older than 1 week, checking anyway...")
skip_dl = False
if skip_dl:
LOGGER.info("asn.mmdb is already the latest version and is cached, skipping...")
sys_exit(0)
# Compute the mmdb URL
mmdb_url = f"https://download.db-ip.com/free/dbip-asn-lite-{date.today().strftime('%Y-%m')}.mmdb.gz"
# Download the mmdb file and save it to tmp
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = b""
LOGGER.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = BytesIO()
try:
with get(mmdb_url, stream=True, timeout=5) as resp:
resp.raise_for_status()
for chunk in resp.iter_content(chunk_size=4 * 1024):
if chunk:
file_content += chunk
file_content.write(chunk)
except RequestException:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}")
sys_exit(2)
try:
assert file_content
except AssertionError:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}")
sys_exit(2)
# Decompress it
logger.info("Decompressing mmdb file ...")
tmp_path.write_bytes(decompress(file_content))
LOGGER.info("Decompressing mmdb file ...")
file_content.seek(0)
content = BytesIO(decompress(file_content.getvalue()))
# Check if file has changed
new_hash = file_hash(tmp_path)
old_hash = cache_hash(cache_path, db)
if new_hash == old_hash:
logger.info("New file is identical to cache file, reload is not needed")
_exit(0)
if job_cache:
# Check if file has changed
new_hash = bytes_hash(content)
if new_hash == job_cache["checksum"]:
LOGGER.info("New file is identical to cache file, reload is not needed")
sys_exit(0)
# Try to load it
logger.info("Checking if mmdb file is valid ...")
with open_database(str(tmp_path)) as reader:
pass
LOGGER.info("Checking if mmdb file is valid ...")
if tmp_path.is_file():
with open_database(tmp_path.as_posix()) as reader:
pass
else:
with open_database(content, mode=MODE_FD) as reader:
pass
tmp_path = None
# Move it to cache folder
logger.info("Moving mmdb file to cache ...")
cached, err = cache_file(tmp_path, cache_path, new_hash, db)
LOGGER.info("Moving mmdb file to cache ...")
cached, err = JOB.cache_file("asn.mmdb", tmp_path or content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching mmdb file : {err}")
_exit(2)
LOGGER.error(f"Error while caching mmdb file : {err}")
sys_exit(2)
# Success
if dl_mmdb:
logger.info(f"Downloaded new mmdb from {mmdb_url}")
LOGGER.info(f"Downloaded new mmdb from {mmdb_url}")
status = 1
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running mmdb-asn.py :\n{format_exc()}")
LOGGER.error(f"Exception while running mmdb-asn.py :\n{format_exc()}")
sys_exit(status)

148
src/common/core/jobs/jobs/mmdb-country.py
View file

@ -1,14 +1,16 @@
#!/usr/bin/env python3
from datetime import date
from datetime import date, datetime, timedelta
from gzip import decompress
from hashlib import sha1
from os import _exit, getenv, sep
from io import BytesIO
from os import getenv, sep
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
from typing import Optional, Union
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
@ -21,107 +23,141 @@ for deps_path in [
if deps_path not in sys_path:
sys_path.append(deps_path)
from maxminddb import open_database
from requests import RequestException, get
from maxminddb import MODE_FD, open_database
from requests import RequestException, Response, get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, file_hash, is_cached_file
from common_utils import bytes_hash # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("JOBS.mmdb-country", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("JOBS.mmdb-country", getenv("LOG_LEVEL", "INFO"))
status = 0
lock = Lock()
LOCK = Lock()
def request_mmdb() -> Optional[Response]:
try:
response = get("https://db-ip.com/db/download/ip-to-country-lite", timeout=5)
response.raise_for_status()
return response
except RequestException:
return None
def bytes_sha1(bio: Union[Path, bytes, BytesIO]) -> str:
if isinstance(bio, Path):
bio = bio.read_bytes()
if isinstance(bio, bytes):
bio = BytesIO(bio)
assert isinstance(bio, BytesIO)
_sha512 = sha1()
while True:
data = bio.read(1024)
if not data:
break
_sha512.update(data)
bio.seek(0)
return _sha512.hexdigest()
try:
dl_mmdb = True
tmp_path = Path(sep, "var", "tmp", "bunkerweb", "country.mmdb")
cache_path = Path(sep, "var", "cache", "bunkerweb", "country.mmdb")
new_hash = None
# Don't go further if the cache match the latest version
if tmp_path.exists():
with lock:
response = None
try:
response = get("https://db-ip.com/db/download/ip-to-country-lite", timeout=5)
except RequestException:
logger.warning("Unable to check if country.mmdb is the latest version")
response = None
if tmp_path.is_file():
response = request_mmdb()
if response and response.status_code == 200:
_sha1 = sha1()
with tmp_path.open("rb") as f:
while True:
data = f.read(1024)
if not data:
break
_sha1.update(data)
if response.content.decode().find(_sha1.hexdigest()) != -1:
logger.info("country.mmdb is already the latest version, skipping download...")
if response.content.find(bytes_sha1(tmp_path).encode()) != -1:
LOGGER.info("country.mmdb is already the latest version, skipping download...")
dl_mmdb = False
else:
logger.warning("Unable to check if country.mmdb is the latest version, downloading it anyway...")
LOGGER.warning("Unable to check if the temporary mmdb file is the latest version, downloading it anyway...")
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
JOB = Job(LOGGER)
if dl_mmdb:
# Don't go further if the cache is fresh
if is_cached_file(cache_path, "month", db):
logger.info("country.mmdb is already in cache, skipping download...")
_exit(0)
job_cache = JOB.get_cache("country.mmdb", with_info=True, with_data=True)
if isinstance(job_cache, dict):
skip_dl = True
if response is None:
response = request_mmdb()
if response and response.status_code == 200:
skip_dl = response.content.find(bytes_sha1(job_cache["data"]).encode()) != -1
elif job_cache["last_update"] < (datetime.now() - timedelta(weeks=1)).timestamp():
LOGGER.warning("Unable to check if the cache file is the latest version from db-ip.com and file is older than 1 week, checking anyway...")
skip_dl = False
if skip_dl:
LOGGER.info("country.mmdb is already the latest version and is cached, skipping...")
sys_exit(0)
# Compute the mmdb URL
mmdb_url = f"https://download.db-ip.com/free/dbip-country-lite-{date.today().strftime('%Y-%m')}.mmdb.gz"
# Download the mmdb file and save it to tmp
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = b""
LOGGER.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = BytesIO()
try:
with get(mmdb_url, stream=True, timeout=5) as resp:
resp.raise_for_status()
for chunk in resp.iter_content(chunk_size=4 * 1024):
if chunk:
file_content += chunk
file_content.write(chunk)
except RequestException:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}")
sys_exit(2)
try:
assert file_content
except AssertionError:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}")
sys_exit(2)
# Decompress it
logger.info("Decompressing mmdb file ...")
tmp_path.write_bytes(decompress(file_content))
LOGGER.info("Decompressing mmdb file ...")
file_content.seek(0)
content = BytesIO(decompress(file_content.getvalue()))
# Check if file has changed
new_hash = file_hash(tmp_path)
old_hash = cache_hash(cache_path, db)
if new_hash == old_hash:
logger.info("New file is identical to cache file, reload is not needed")
_exit(0)
if job_cache:
# Check if file has changed
new_hash = bytes_hash(content)
if new_hash == job_cache["checksum"]:
LOGGER.info("New file is identical to cache file, reload is not needed")
sys_exit(0)
# Try to load it
logger.info("Checking if mmdb file is valid ...")
with open_database(str(tmp_path)) as reader:
pass
LOGGER.info("Checking if mmdb file is valid ...")
if tmp_path.is_file():
with open_database(tmp_path.as_posix()) as reader:
pass
else:
with open_database(content, mode=MODE_FD) as reader:
pass
tmp_path = None
# Move it to cache folder
logger.info("Moving mmdb file to cache ...")
cached, err = cache_file(tmp_path, cache_path, new_hash, db)
LOGGER.info("Moving mmdb file to cache ...")
cached, err = JOB.cache_file("country.mmdb", tmp_path or content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching mmdb file : {err}")
_exit(2)
LOGGER.error(f"Error while caching mmdb file : {err}")
sys_exit(2)
# Success
if dl_mmdb:
logger.info(f"Downloaded new mmdb from {mmdb_url}")
LOGGER.info(f"Downloaded new mmdb from {mmdb_url}")
status = 1
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running mmdb-country.py :\n{format_exc()}")
LOGGER.error(f"Exception while running mmdb-country.py :\n{format_exc()}")
sys_exit(status)

4
src/common/core/jobs/plugin.json
View file

@ -9,13 +9,13 @@
{
"name": "mmdb-country",
"file": "mmdb-country.py",
"every": "week",
"every": "day",
"reload": true
},
{
"name": "mmdb-asn",
"file": "mmdb-asn.py",
"every": "week",
"every": "day",
"reload": true
},
{

24
src/common/core/letsencrypt/jobs/certbot-auth.py
View file

@ -7,24 +7,16 @@ from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("api",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from jobs import get_integration # type: ignore
from common_utils import get_integration # type: ignore
from logger import setup_logger # type: ignore
from API import API # type: ignore
logger = setup_logger("Lets-encrypt.auth", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Lets-encrypt.auth", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
@ -34,7 +26,7 @@ try:
# Cluster case
if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
lock = Lock()
with lock:
@ -45,12 +37,12 @@ try:
sent, err, status, resp = api.request("POST", "/lets-encrypt/challenge", data={"token": token, "validation": validation})
if not sent:
status = 1
logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
elif status != 200:
status = 1
logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}")
LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge")
LOGGER.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge")
# Linux case
else:
@ -59,6 +51,6 @@ try:
root_dir.joinpath(token).write_text(validation, encoding="utf-8")
except:
status = 1
logger.error(f"Exception while running certbot-auth.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-auth.py :\n{format_exc()}")
sys_exit(status)

24
src/common/core/letsencrypt/jobs/certbot-cleanup.py
View file

@ -7,24 +7,16 @@ from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("api",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from jobs import get_integration # type: ignore
from common_utils import get_integration # type: ignore
from logger import setup_logger # type: ignore
from API import API # type: ignore
logger = setup_logger("Lets-encrypt.cleanup", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Lets-encrypt.cleanup", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
@ -33,7 +25,7 @@ try:
# Cluster case
if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
lock = Lock()
with lock:
instances = db.get_instances()
@ -43,17 +35,17 @@ try:
sent, err, status, resp = api.request("DELETE", "/lets-encrypt/challenge", data={"token": token})
if not sent:
status = 1
logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
elif status != 200:
status = 1
logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}")
LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge")
LOGGER.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge")
# Linux case
else:
Path(sep, "var", "tmp", "bunkerweb", "lets-encrypt", ".well-known", "acme-challenge", token).unlink(missing_ok=True)
except:
status = 1
logger.error(f"Exception while running certbot-cleanup.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-cleanup.py :\n{format_exc()}")
sys_exit(status)

36
src/common/core/letsencrypt/jobs/certbot-deploy.py
View file

@ -9,31 +9,23 @@ from tarfile import open as tar_open
from threading import Lock
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("api",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from jobs import get_integration # type: ignore
from common_utils import get_integration # type: ignore
from logger import setup_logger # type: ignore
from API import API # type: ignore
logger = setup_logger("Lets-encrypt.deploy", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Lets-encrypt.deploy", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
# Get env vars
token = getenv("CERTBOT_TOKEN", "")
logger.info(f"Certificates renewal for {getenv('RENEWED_DOMAINS')} successful")
LOGGER.info(f"Certificates renewal for {getenv('RENEWED_DOMAINS')} successful")
# Cluster case
if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
@ -45,7 +37,7 @@ try:
tgz.seek(0, 0)
files = {"archive.tar.gz": tgz}
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
lock = Lock()
with lock:
@ -59,32 +51,32 @@ try:
sent, err, status, resp = api.request("POST", "/lets-encrypt/certificates", files=files)
if not sent:
status = 1
logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/certificates : {err}")
LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/certificates : {err}")
elif status != 200:
status = 1
logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/certificates : status = {resp['status']}, msg = {resp['msg']}")
LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/certificates : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(
LOGGER.info(
f"Successfully sent API request to {api.endpoint}/lets-encrypt/certificates",
)
sent, err, status, resp = api.request("POST", "/reload")
if not sent:
status = 1
logger.error(f"Can't send API request to {api.endpoint}/reload : {err}")
LOGGER.error(f"Can't send API request to {api.endpoint}/reload : {err}")
elif status != 200:
status = 1
logger.error(f"Error while sending API request to {api.endpoint}/reload : status = {resp['status']}, msg = {resp['msg']}")
LOGGER.error(f"Error while sending API request to {api.endpoint}/reload : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(f"Successfully sent API request to {api.endpoint}/reload")
LOGGER.info(f"Successfully sent API request to {api.endpoint}/reload")
# Linux case
else:
if run([join(sep, "usr", "sbin", "nginx"), "-s", "reload"], stdin=DEVNULL, stderr=STDOUT, check=False).returncode != 0:
status = 1
logger.error("Error while reloading nginx")
LOGGER.error("Error while reloading nginx")
else:
logger.info("Successfully reloaded nginx")
LOGGER.info("Successfully reloaded nginx")
except:
status = 1
logger.error(f"Exception while running certbot-deploy.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-deploy.py :\n{format_exc()}")
sys_exit(status)

101
src/common/core/letsencrypt/jobs/certbot-new.py
View file

@ -1,32 +1,22 @@
#!/usr/bin/env python3
from os import _exit, environ, getenv, sep
from os import environ, getenv, sep
from os.path import join
from pathlib import Path
from subprocess import DEVNULL, STDOUT, run, PIPE
from subprocess import DEVNULL, STDOUT, Popen, run, PIPE
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from tarfile import open as tar_open
from io import BytesIO
from shutil import rmtree
from re import MULTILINE, search
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import get_file_in_db, set_file_in_db # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("LETS-ENCRYPT.new", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("LETS-ENCRYPT.new", getenv("LOG_LEVEL", "INFO"))
LOGGER_CERTBOT = setup_logger("LETS-ENCRYPT.new.certbot", getenv("LOG_LEVEL", "INFO"))
status = 0
CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot")
@ -38,7 +28,7 @@ LETS_ENCRYPT_LOGS_DIR = join(sep, "var", "log", "bunkerweb")
def certbot_new(domains: str, email: str, use_letsencrypt_staging: bool = False) -> int:
return run(
with Popen(
[
CERTBOT_BIN,
"certonly",
@ -64,9 +54,14 @@ def certbot_new(domains: str, email: str, use_letsencrypt_staging: bool = False)
]
+ (["--staging"] if use_letsencrypt_staging else []),
stdin=DEVNULL,
stderr=STDOUT,
stderr=PIPE,
universal_newlines=True,
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
).returncode
) as process:
if process.stderr:
for line in process.stderr:
LOGGER_CERTBOT.info(line.strip())
return process.returncode
status = 0
@ -82,36 +77,21 @@ try:
use_letsencrypt = True
elif is_multisite:
for first_server in server_names:
if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes":
if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", getenv("AUTO_LETS_ENCRYPT", "no")) == "yes":
use_letsencrypt = True
break
if not use_letsencrypt:
logger.info("Let's Encrypt is not activated, skipping generation...")
_exit(0)
LOGGER.info("Let's Encrypt is not activated, skipping generation...")
sys_exit(0)
elif not getenv("SERVER_NAME"):
logger.warning("There are no server names, skipping generation...")
_exit(0)
LOGGER.warning("There are no server names, skipping generation...")
sys_exit(0)
# Create directories if they doesn't exist
LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True)
Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Extract letsencrypt folder if it exists in db
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
tgz = get_file_in_db("folder.tgz", db, job_name="certbot-renew")
if tgz:
# Delete folder if needed
if LETS_ENCRYPT_PATH.exists():
rmtree(LETS_ENCRYPT_PATH, ignore_errors=True)
LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True)
# Extract it
with tar_open(name="folder.tgz", mode="r:gz", fileobj=BytesIO(tgz)) as tf:
tf.extractall(LETS_ENCRYPT_PATH)
logger.info("Successfully retrieved Let's Encrypt data from db cache")
else:
logger.info("No Let's Encrypt data found in db cache")
# Restore Let's Encrypt data from db cache
JOB.restore_cache(job_name="certbot-renew")
domains_to_ask = []
# Multisite case
@ -142,11 +122,12 @@ try:
stderr=STDOUT,
text=True,
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
check=False,
)
stdout = proc.stdout
if proc.returncode != 0:
logger.error(f"Error while checking certificates :\n{proc.stdout}")
LOGGER.error(f"Error while checking certificates :\n{proc.stdout}")
domains_to_ask = server_names
else:
for first_server, domains in domains_sever_names.items():
@ -155,10 +136,10 @@ try:
domains_to_ask.append(first_server)
continue
elif set(f"{first_server}{current_domains.groupdict()['domains']}".strip().split(" ")) != set(domains.split(" ")):
logger.warning(f"Domains for {first_server} are not the same as in the certificate, asking new certificate...")
LOGGER.warning(f"Domains for {first_server} are not the same as in the certificate, asking new certificate...")
domains_to_ask.append(first_server)
continue
logger.info(f"Certificates already exists for domain(s) {domains}")
LOGGER.info(f"Certificates already exists for domain(s) {domains}")
for first_server, domains in domains_sever_names.items():
if first_server not in domains_to_ask:
@ -170,30 +151,26 @@ try:
use_letsencrypt_staging = getenv(f"{first_server}_USE_LETS_ENCRYPT_STAGING", getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes"
logger.info(f"Asking certificates for domain(s) : {domains} (email = {real_email}) to Let's Encrypt {'staging ' if use_letsencrypt_staging else ''}...")
LOGGER.info(f"Asking certificates for domain(s) : {domains} (email = {real_email}) to Let's Encrypt {'staging ' if use_letsencrypt_staging else ''}...")
if certbot_new(domains.replace(" ", ","), real_email, use_letsencrypt_staging) != 0:
status = 2
logger.error(f"Certificate generation failed for domain(s) {domains} ...")
LOGGER.error(f"Certificate generation failed for domain(s) {domains} ...")
continue
else:
status = 1 if status == 0 else status
logger.info(f"Certificate generation succeeded for domain(s) : {domains}")
LOGGER.info(f"Certificate generation succeeded for domain(s) : {domains}")
# Put new folder in cache
bio = BytesIO()
with tar_open("folder.tgz", mode="w:gz", fileobj=bio, compresslevel=9) as tgz:
tgz.add(LETS_ENCRYPT_PATH, arcname=".")
bio.seek(0, 0)
# Put tgz in cache
cached, err = set_file_in_db("folder.tgz", bio.read(), db, job_name="certbot-renew")
if not cached:
logger.error(f"Error while saving Let's Encrypt data to db cache : {err}")
else:
logger.info("Successfully saved Let's Encrypt data to db cache")
# Save Let's Encrypt data to db cache
if LETS_ENCRYPT_PATH.is_dir() and list(LETS_ENCRYPT_PATH.iterdir()):
cached, err = JOB.cache_dir(LETS_ENCRYPT_PATH, job_name="certbot-renew")
if not cached:
LOGGER.error(f"Error while saving Let's Encrypt data to db cache : {err}")
else:
LOGGER.info("Successfully saved Let's Encrypt data to db cache")
except SystemExit as e:
status = e.code
except:
status = 3
logger.error(f"Exception while running certbot-new.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-new.py :\n{format_exc()}")
sys_exit(status)

108
src/common/core/letsencrypt/jobs/certbot-renew.py
View file

@ -1,14 +1,11 @@
#!/usr/bin/env python3
from os import _exit, environ, getenv, sep
from os import environ, getenv, sep
from os.path import join
from pathlib import Path
from subprocess import DEVNULL, STDOUT, run
from subprocess import DEVNULL, PIPE, Popen
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from tarfile import open as tar_open
from io import BytesIO
from shutil import rmtree
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
@ -21,14 +18,18 @@ for deps_path in [
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import get_file_in_db, set_file_in_db # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("LETS-ENCRYPT.renew", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("LETS-ENCRYPT.renew", getenv("LOG_LEVEL", "INFO"))
LOGGER_CERTBOT = setup_logger("LETS-ENCRYPT.renew.certbot", getenv("LOG_LEVEL", "INFO"))
status = 0
CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot")
LETS_ENCRYPT_PATH = Path(sep, "var", "cache", "bunkerweb", "letsencrypt")
LETS_ENCRYPT_WORK_DIR = join(sep, "var", "lib", "bunkerweb", "letsencrypt")
LETS_ENCRYPT_LOGS_DIR = join(sep, "var", "log", "bunkerweb")
try:
# Check if we're using let's encrypt
@ -42,66 +43,47 @@ try:
break
if not use_letsencrypt:
logger.info("Let's Encrypt is not activated, skipping renew...")
_exit(0)
LOGGER.info("Let's Encrypt is not activated, skipping renew...")
sys_exit(0)
# Create directory if it doesn't exist
LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True)
Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Extract letsencrypt folder if it exists in db
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
with Popen(
[
CERTBOT_BIN,
"renew",
"--no-random-sleep-on-renew",
"--config-dir",
LETS_ENCRYPT_PATH.joinpath("etc").as_posix(),
"--work-dir",
LETS_ENCRYPT_WORK_DIR,
"--logs-dir",
LETS_ENCRYPT_LOGS_DIR,
],
stdin=DEVNULL,
stderr=PIPE,
universal_newlines=True,
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
) as process:
if process.stderr:
for line in process.stderr:
LOGGER_CERTBOT.info(line.strip())
tgz = get_file_in_db("folder.tgz", db)
if tgz:
# Delete folder if needed
if LETS_ENCRYPT_PATH.exists():
rmtree(LETS_ENCRYPT_PATH, ignore_errors=True)
LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True)
# Extract it
with tar_open(name="folder.tgz", mode="r:gz", fileobj=BytesIO(tgz)) as tf:
tf.extractall(LETS_ENCRYPT_PATH)
logger.info("Successfully retrieved Let's Encrypt data from db cache")
else:
logger.info("No Let's Encrypt data found in db cache")
if process.returncode == 0:
status = 2
LOGGER.error("Certificates renewal failed")
if (
run(
[
join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot"),
"renew",
"--no-random-sleep-on-renew",
"--config-dir",
LETS_ENCRYPT_PATH.joinpath("etc").as_posix(),
"--work-dir",
join(sep, "var", "lib", "bunkerweb", "letsencrypt"),
"--logs-dir",
join(sep, "var", "log", "bunkerweb"),
],
stdin=DEVNULL,
stderr=STDOUT,
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
check=False,
).returncode
!= 0
):
status = 2
logger.error("Certificates renewal failed")
# Put new folder in cache
bio = BytesIO()
with tar_open("folder.tgz", mode="w:gz", fileobj=bio, compresslevel=9) as tgz:
tgz.add(LETS_ENCRYPT_PATH, arcname=".")
bio.seek(0, 0)
# Put tgz in cache
cached, err = set_file_in_db("folder.tgz", bio.read(), db)
if not cached:
logger.error(f"Error while saving Let's Encrypt data to db cache : {err}")
else:
logger.info("Successfully saved Let's Encrypt data to db cache")
# Save Let's Encrypt data to db cache
if LETS_ENCRYPT_PATH.is_dir() and list(LETS_ENCRYPT_PATH.iterdir()):
cached, err = JOB.cache_dir(LETS_ENCRYPT_PATH)
if not cached:
LOGGER.error(f"Error while saving Let's Encrypt data to db cache : {err}")
else:
LOGGER.info("Successfully saved Let's Encrypt data to db cache")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running certbot-renew.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-renew.py :\n{format_exc()}")
sys_exit(status)

61
src/common/core/misc/jobs/anonymous-report.py
View file

@ -3,8 +3,6 @@
from json import dumps
from os import getenv, sep
from os.path import join
from pathlib import Path
from platform import machine
from re import compile as re_compile
from sys import exit as sys_exit, path as sys_path, version
from traceback import format_exc
@ -14,33 +12,28 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from common_utils import get_os_info # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, is_cached_file # type: ignore
from jobs import Job # type: ignore
from requests import post
logger = setup_logger("ANONYMOUS-REPORT", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("ANONYMOUS-REPORT", getenv("LOG_LEVEL", "INFO"))
status = 0
if getenv("SEND_ANONYMOUS_REPORT", "yes") != "yes":
logger.info("Skipping the sending of anonymous report (disabled)")
sys_exit(status)
anonymous_report_path = Path(sep, "var", "cache", "bunkerweb", "anonymous_report")
anonymous_report_path.mkdir(parents=True, exist_ok=True)
tmp_anonymous_report_path = Path(sep, "var", "tmp", "bunkerweb", "anonymous_report")
tmp_anonymous_report_path.mkdir(parents=True, exist_ok=True)
try:
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
if is_cached_file(anonymous_report_path.joinpath("last_report.json"), "day", db):
logger.info("Skipping the sending of anonymous report (already sent today)")
if getenv("SEND_ANONYMOUS_REPORT", "yes") != "yes":
LOGGER.info("Skipping the sending of anonymous report (disabled)")
sys_exit(status)
JOB = Job(LOGGER)
if JOB.is_cached_file("last_report.json", "day"):
LOGGER.info("Skipping the sending of anonymous report (already sent today)")
sys_exit(0)
# ? Get version and integration of BunkerWeb
data: Dict[str, Any] = db.get_metadata()
data: Dict[str, Any] = JOB.db.get_metadata()
data["is_pro"] = "yes" if data["is_pro"] else "no"
@ -48,7 +41,7 @@ try:
if key not in ("version", "integration", "database_version", "is_pro"):
data.pop(key, None)
db_config = db.get_config(methods=True, with_drafts=True)
db_config = JOB.db.get_config(methods=True, with_drafts=True)
services = db_config.get("SERVER_NAME", {"value": ""})["value"].split(" ")
multisite = db_config.get("MULTISITE", {"value": "no"})["value"] == "yes"
@ -58,7 +51,7 @@ try:
database_version = database_version.group(1)
data["integration"] = data["integration"].lower()
data["database"] = f"{db.database_uri.split(':')[0].split('+')[0]}/{database_version}"
data["database"] = f"{JOB.db.database_uri.split(':')[0].split('+')[0]}/{database_version}"
data["service_number"] = str(len(services))
data["draft_service_number"] = 0
data["python_version"] = version.split(" ")[0]
@ -82,26 +75,13 @@ try:
data["external_plugins"] = []
data["pro_plugins"] = []
for plugin in db.get_plugins():
for plugin in JOB.db.get_plugins():
if plugin["type"] == "external":
data["external_plugins"].append(f"{plugin['id']}/{plugin['version']}")
elif plugin["type"] == "pro":
data["pro_plugins"].append(f"{plugin['id']}/{plugin['version']}")
data["os"] = {
"name": "Linux",
"version": "Unknown",
"version_id": "Unknown",
"version_codename": "Unknown",
"id": "Unknown",
"arch": machine(),
}
os_release = Path("/etc/os-release")
if os_release.exists():
for line in os_release.read_text().splitlines():
if "=" not in line or line.split("=")[0].strip().lower() not in data["os"]:
continue
data["os"][line.split("=")[0].lower()] = line.split("=")[1].strip('"')
data["os"] = get_os_info()
data["non_default_settings"] = {}
for setting, setting_data in db_config.items():
@ -121,18 +101,19 @@ try:
for key in data["non_default_settings"].copy():
data["non_default_settings"][key] = str(data["non_default_settings"][key])
data["bw_instances_number"] = str(len(db.get_instances()))
tmp_anonymous_report_path.joinpath("last_report.json").write_text(dumps(data, indent=4), encoding="utf-8")
data["bw_instances_number"] = str(len(JOB.db.get_instances()))
response = post("https://api.bunkerweb.io/data", json=data, headers={"User-Agent": f"BunkerWeb/{data['version']}"}, allow_redirects=True, timeout=10)
response.raise_for_status()
cached, err = cache_file(tmp_anonymous_report_path.joinpath("last_report.json"), anonymous_report_path.joinpath("last_report.json"), None, db)
cached, err = JOB.cache_file("last_report.json", dumps(data, indent=4).encode())
if not cached:
LOGGER.error(f"Failed to cache last_report.json :\n{err}")
status = 2
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running anonymous-report.py :\n{format_exc()}")
LOGGER.error(f"Exception while running anonymous-report.py :\n{format_exc()}")
sys_exit(status)

65
src/common/core/misc/jobs/default-server-cert.py
View file

@ -7,29 +7,24 @@ from subprocess import DEVNULL, run
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import set_file_in_db
from jobs import Job # type: ignore
logger = setup_logger("DEFAULT-SERVER-CERT", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("DEFAULT-SERVER-CERT", getenv("LOG_LEVEL", "INFO"))
LOGGER_OPENSSL = setup_logger("DEFAULT-SERVER-CERT.openssl", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
cert_path = Path(sep, "var", "cache", "bunkerweb", "default-server-cert")
cert_path.mkdir(parents=True, exist_ok=True)
if not cert_path.joinpath("cert.pem").is_file():
logger.info("Generating self-signed certificate for default server")
JOB = Job(LOGGER)
cert_path = Path(sep, "var", "cache", "bunkerweb", "misc")
if not JOB.is_cached_file("default-server-cert.pem", "month") or not JOB.is_cached_file("default-server-cert.key", "month"):
LOGGER.info("Generating self-signed certificate for default server")
cert_path.mkdir(parents=True, exist_ok=True)
if (
run(
@ -41,9 +36,9 @@ try:
"-newkey",
"ed25519",
"-keyout",
str(cert_path.joinpath("cert.key")),
str(cert_path.joinpath("default-server-cert.key")),
"-out",
str(cert_path.joinpath("cert.pem")),
str(cert_path.joinpath("default-server-cert.pem")),
"-days",
"3650",
"-subj",
@ -55,43 +50,27 @@ try:
).returncode
!= 0
):
logger.error(
"Self-signed certificate generation failed for default server",
)
LOGGER.error("Self-signed certificate generation failed for default server")
status = 2
else:
LOGGER.info("Successfully generated self-signed certificate for default server")
status = 1
logger.info(
"Successfully generated self-signed certificate for default server",
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
cached, err = set_file_in_db(
"cert.pem",
cert_path.joinpath("cert.pem").read_bytes(),
db,
)
cached, err = JOB.cache_file("default-server-cert.pem", cert_path.joinpath("default-server-cert.pem"), overwrite_file=False)
if not cached:
logger.error(f"Error while saving default-server-cert cert.pem file to db cache : {err}")
LOGGER.error(f"Error while saving default-server-cert default-server-cert.pem file to db cache : {err}")
else:
logger.info("Successfully saved default-server-cert cert.pem file to db cache")
LOGGER.info("Successfully saved default-server-cert default-server-cert.pem file to db cache")
cached, err = set_file_in_db(
"cert.key",
cert_path.joinpath("cert.key").read_bytes(),
db,
)
cached, err = JOB.cache_file("default-server-cert.key", cert_path.joinpath("default-server-cert.key"), overwrite_file=False)
if not cached:
logger.error(f"Error while saving default-server-cert cert.key file to db cache : {err}")
LOGGER.error(f"Error while saving default-server-cert default-server-cert.key file to db cache : {err}")
else:
logger.info("Successfully saved default-server-cert cert.key file to db cache")
LOGGER.info("Successfully saved default-server-cert default-server-cert.key file to db cache")
else:
logger.info(
"Skipping generation of self-signed certificate for default server (already present)",
)
LOGGER.info("Skipping generation of self-signed certificate for default server (already present)")
except:
status = 2
logger.error(f"Exception while running default-server-cert.py :\n{format_exc()}")
LOGGER.error(f"Exception while running default-server-cert.py :\n{format_exc()}")
sys_exit(status)

30
src/common/core/misc/jobs/update-check.py
View file

@ -2,46 +2,34 @@
from os import getenv, sep
from os.path import basename, join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from requests import get
from common_utils import get_version # type: ignore
from logger import setup_logger # type: ignore
logger = setup_logger("UPDATE-CHECK", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("UPDATE-CHECK", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
current_version = f"v{Path('/usr/share/bunkerweb/VERSION').read_text(encoding='utf-8').strip()}"
current_version = f"v{get_version().strip()}"
response = get(
"https://github.com/bunkerity/bunkerweb/releases/latest",
headers={"User-Agent": "BunkerWeb"},
allow_redirects=True,
timeout=10,
)
response = get("https://github.com/bunkerity/bunkerweb/releases/latest", headers={"User-Agent": "BunkerWeb"}, allow_redirects=True, timeout=10)
response.raise_for_status()
latest_version = basename(response.url)
if current_version != latest_version:
logger.warning(
f"* \n* \n* 🚨 A new version of BunkerWeb is available: {latest_version} (current: {current_version}) 🚨\n* \n* ",
)
LOGGER.warning(f"* \n* \n* 🚨 A new version of BunkerWeb is available: {latest_version} (current: {current_version}) 🚨\n* \n* ")
else:
logger.info(f"Latest version is already installed: {current_version}")
LOGGER.info(f"Latest version is already installed: {current_version}")
except:
status = 2
logger.error(f"Exception while running update-check.py :\n{format_exc()}")
LOGGER.error(f"Exception while running update-check.py :\n{format_exc()}")
sys_exit(status)

70
src/common/core/pro/jobs/download-pro-plugins.py
View file

@ -17,15 +17,7 @@ from tarfile import open as tar_open
from traceback import format_exc
from zipfile import ZipFile
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("api",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -33,7 +25,7 @@ from requests import get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import get_os_info, get_integration, get_version # type: ignore
from common_utils import get_os_info, get_integration, get_version # type: ignore
API_ENDPOINT = "https://api.bunkerweb.io"
PREVIEW_ENDPOINT = "https://assets.bunkerity.com/bw-pro/preview"
@ -44,12 +36,12 @@ STATUS_MESSAGES = {
"expired": "has expired",
"suspended": "has been suspended",
}
logger = setup_logger("Jobs.download-pro-plugins", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Jobs.download-pro-plugins", getenv("LOG_LEVEL", "INFO"))
status = 0
def clean_pro_plugins(db) -> None:
logger.debug("Cleaning up Pro plugins...")
LOGGER.debug("Cleaning up Pro plugins...")
# Clean Pro plugins
rmtree(PRO_PLUGINS_DIR.joinpath("*"), ignore_errors=True)
# Update database
@ -61,14 +53,14 @@ def install_plugin(plugin_dir: str, db, preview: bool = True) -> bool:
plugin_file = plugin_path.joinpath("plugin.json")
if not plugin_file.is_file():
logger.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json not found)")
LOGGER.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json not found)")
return False
# Load plugin.json
try:
metadata = loads(plugin_file.read_text(encoding="utf-8"))
except JSONDecodeError:
logger.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json is not valid)")
LOGGER.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json is not valid)")
return False
# Don't go further if plugin is already installed
@ -81,12 +73,12 @@ def install_plugin(plugin_dir: str, db, preview: bool = True) -> bool:
break
if old_version == metadata["version"]:
logger.warning(
LOGGER.warning(
f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {metadata['id']} (version {metadata['version']} already installed)"
)
return False
logger.warning(
LOGGER.warning(
f"{'Preview version of ' if preview else ''}Pro plugin {metadata['id']} is already installed but version {metadata['version']} is different from database ({old_version}), updating it..."
)
rmtree(PRO_PLUGINS_DIR.joinpath(metadata["id"]), ignore_errors=True)
@ -97,21 +89,21 @@ def install_plugin(plugin_dir: str, db, preview: bool = True) -> bool:
for job_file in glob(PRO_PLUGINS_DIR.joinpath(metadata["id"], "jobs", "*").as_posix()):
st = Path(job_file).stat()
chmod(job_file, st.st_mode | S_IEXEC)
logger.info(f"✅ {'Preview version of ' if preview else ''}Pro plugin {metadata['id']} (version {metadata['version']}) installed successfully!")
LOGGER.info(f"✅ {'Preview version of ' if preview else ''}Pro plugin {metadata['id']} (version {metadata['version']}) installed successfully!")
return True
try:
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
db_metadata = db.get_metadata()
current_date = datetime.now()
# If we already checked in the last 10 minutes, skip the check
if db_metadata["last_pro_check"] and (current_date - db_metadata["last_pro_check"]).seconds < 600:
logger.info("Skipping the check for BunkerWeb Pro license (already checked in the last 10 minutes)")
LOGGER.info("Skipping the check for BunkerWeb Pro license (already checked in the last 10 minutes)")
sys_exit(0)
logger.info("Checking BunkerWeb Pro license key...")
LOGGER.info("Checking BunkerWeb Pro license key...")
data = {
"integration": get_integration(),
@ -135,24 +127,24 @@ try:
temp_dir.mkdir(parents=True, exist_ok=True)
if pro_license_key:
logger.info("BunkerWeb Pro license provided, checking if it's valid...")
LOGGER.info("BunkerWeb Pro license provided, checking if it's valid...")
headers["Authorization"] = f"Bearer {pro_license_key.strip()}"
resp = get(f"{API_ENDPOINT}/pro-status", headers=headers, json=data, timeout=5, allow_redirects=True)
if resp.status_code == 403:
logger.error(f"Access denied to {API_ENDPOINT}/pro-status - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/")
LOGGER.error(f"Access denied to {API_ENDPOINT}/pro-status - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/")
error = True
if db_metadata["is_pro"]:
clean_pro_plugins(db)
elif resp.status_code == 500:
logger.error("An error occurred with the remote server while checking BunkerWeb Pro license, please try again later")
LOGGER.error("An error occurred with the remote server while checking BunkerWeb Pro license, please try again later")
status = 2
sys_exit(status)
else:
resp.raise_for_status()
metadata = resp.json()["data"]
logger.debug(f"Got BunkerWeb Pro license metadata: {metadata}")
LOGGER.debug(f"Got BunkerWeb Pro license metadata: {metadata}")
metadata["pro_expire"] = datetime.strptime(metadata["pro_expire"], "%Y-%m-%d") if metadata["pro_expire"] else None
if metadata["pro_expire"] and metadata["pro_expire"] < datetime.now():
metadata["pro_status"] = "expired"
@ -164,7 +156,7 @@ try:
db.set_pro_metadata(metadata | {"last_pro_check": current_date})
if metadata["is_pro"]:
logger.info("🚀 Your BunkerWeb Pro license is valid, checking if there are new or updated Pro plugins...")
LOGGER.info("🚀 Your BunkerWeb Pro license is valid, checking if there are new or updated Pro plugins...")
if not db_metadata["is_pro"]:
clean_pro_plugins(db)
@ -172,13 +164,13 @@ try:
resp = get(f"{API_ENDPOINT}/pro", headers=headers, json=data, timeout=5, allow_redirects=True)
if resp.status_code == 403:
logger.error(f"Access denied to {API_ENDPOINT}/pro - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/")
LOGGER.error(f"Access denied to {API_ENDPOINT}/pro - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/")
error = True
metadata = default_metadata
db.set_pro_metadata(metadata | {"last_pro_check": current_date})
clean_pro_plugins(db)
elif resp.headers.get("Content-Type", "") != "application/octet-stream":
logger.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {API_ENDPOINT}/pro")
LOGGER.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {API_ENDPOINT}/pro")
status = 2
sys_exit(status)
@ -192,9 +184,9 @@ try:
STATUS_MESSAGES.get(metadata["pro_status"], "is not valid or has expired") if not error else "is not valid or has expired"
)
else:
logger.info("If you wish to purchase a BunkerWeb Pro license, please visit https://panel.bunkerweb.io/")
LOGGER.info("If you wish to purchase a BunkerWeb Pro license, please visit https://panel.bunkerweb.io/")
message = "No BunkerWeb Pro license key provided"
logger.warning(f"{message}, only checking if there are new or updated preview versions of Pro plugins...")
LOGGER.warning(f"{message}, only checking if there are new or updated preview versions of Pro plugins...")
if metadata["is_pro"]:
clean_pro_plugins(db)
@ -202,16 +194,16 @@ try:
resp = get(f"{PREVIEW_ENDPOINT}/v{data['version']}.zip", timeout=5, allow_redirects=True)
if resp.status_code == 404:
logger.error(f"Couldn't find Pro plugins for BunkerWeb version {data['version']} at {PREVIEW_ENDPOINT}/v{data['version']}.zip")
LOGGER.error(f"Couldn't find Pro plugins for BunkerWeb version {data['version']} at {PREVIEW_ENDPOINT}/v{data['version']}.zip")
status = 2
sys_exit(status)
elif resp.headers.get("Content-Type", "") != "application/zip":
logger.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {PREVIEW_ENDPOINT}/v{data['version']}.zip")
LOGGER.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {PREVIEW_ENDPOINT}/v{data['version']}.zip")
status = 2
sys_exit(status)
if resp.status_code == 500:
logger.error("An error occurred with the remote server, please try again later")
LOGGER.error("An error occurred with the remote server, please try again later")
status = 2
sys_exit(status)
resp.raise_for_status()
@ -229,14 +221,14 @@ try:
if install_plugin(plugin_dir, db, not metadata["is_pro"]):
plugin_nbr += 1
except FileExistsError:
logger.warning(f"Skipping installation of pro plugin {basename(plugin_dir)} (already installed)")
LOGGER.warning(f"Skipping installation of pro plugin {basename(plugin_dir)} (already installed)")
except:
logger.exception("Exception while installing pro plugin(s)")
LOGGER.exception("Exception while installing pro plugin(s)")
status = 2
sys_exit(status)
if not plugin_nbr:
logger.info("All Pro plugins are up to date")
LOGGER.info("All Pro plugins are up to date")
sys_exit(0)
pro_plugins = []
@ -244,7 +236,7 @@ try:
for plugin in listdir(PRO_PLUGINS_DIR):
path = PRO_PLUGINS_DIR.joinpath(plugin)
if not path.joinpath("plugin.json").is_file():
logger.warning(f"Plugin {plugin} is not valid, deleting it...")
LOGGER.warning(f"Plugin {plugin} is not valid, deleting it...")
rmtree(path, ignore_errors=True)
continue
@ -282,15 +274,15 @@ try:
err = db.update_external_plugins(pro_plugins, _type="pro")
if err:
logger.error(f"Couldn't update Pro plugins to database: {err}")
LOGGER.error(f"Couldn't update Pro plugins to database: {err}")
status = 1
logger.info("🚀 Pro plugins downloaded and installed successfully!")
LOGGER.info("🚀 Pro plugins downloaded and installed successfully!")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running download-pro-plugins.py :\n{format_exc()}")
LOGGER.error(f"Exception while running download-pro-plugins.py :\n{format_exc()}")
for plugin_tmp in glob(TMP_DIR.joinpath("*").as_posix()):
rmtree(plugin_tmp, ignore_errors=True)

89
src/common/core/realip/jobs/realip-download.py
View file

@ -2,43 +2,35 @@
from contextlib import suppress
from ipaddress import ip_address, ip_network
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join, normpath
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from requests import get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, del_file_in_db, file_hash, is_cached_file
from common_utils import bytes_hash # type: ignore
from jobs import Job # type: ignore
def check_line(line):
if "/" in line:
with suppress(ValueError):
with suppress(ValueError):
if "/" in line:
ip_network(line)
return True, line
else:
with suppress(ValueError):
else:
ip_address(line)
return True, line
return False, b""
logger = setup_logger("REALIP", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("REALIP", getenv("LOG_LEVEL", "INFO"))
REALIP_CACHE_PATH = join(sep, "var", "cache", "bunkerweb", "realip")
status = 0
try:
@ -61,37 +53,34 @@ try:
realip_activated = True
if not realip_activated:
logger.info("RealIP is not activated, skipping download...")
_exit(0)
LOGGER.info("RealIP is not activated, skipping download...")
sys_exit(0)
# Create directories if they don't exist
realip_path = Path(sep, "var", "cache", "bunkerweb", "realip")
realip_path.mkdir(parents=True, exist_ok=True)
tmp_realip_path = Path(sep, "var", "tmp", "bunkerweb", "realip")
tmp_realip_path.mkdir(parents=True, exist_ok=True)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
JOB = Job(LOGGER)
# Get URLs
urls = [url for url in getenv("REAL_IP_FROM_URLS", "").split(" ") if url]
# Don't go further if the cache is fresh
if is_cached_file(realip_path.joinpath("combined.list"), "hour", db):
logger.info("RealIP list is already in cache, skipping download...")
if JOB.is_cached_file("combined.list", "hour"):
LOGGER.info("RealIP list is already in cache, skipping download...")
if not urls:
logger.warning("No URL found, deleting combined.list from cache...")
tmp_realip_path.joinpath("combined.list").unlink(missing_ok=True)
deleted, err = del_file_in_db("combined.list", db)
LOGGER.warning("No URL found, deleting combined.list from cache...")
deleted, err = JOB.del_cache("combined.list")
if not deleted:
logger.warning(f"Couldn't delete combined.list from cache : {err}")
_exit(0)
LOGGER.warning(f"Couldn't delete combined.list from cache : {err}")
sys_exit(0)
if not urls:
LOGGER.error("No URL found, skipping download...")
sys_exit(0)
# Download and write data to temp file
i = 0
content = b""
for url in urls:
try:
logger.info(f"Downloading RealIP list from {url} ...")
LOGGER.info(f"Downloading RealIP list from {url} ...")
if url.startswith("file://"):
with open(normpath(url[7:]), "rb") as f:
iterable = f.readlines()
@ -99,7 +88,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -116,34 +105,28 @@ try:
i += 1
except:
status = 2
logger.error(f"Exception while getting RealIP list from {url} :\n{format_exc()}")
tmp_realip_path.joinpath("combined.list").write_bytes(content)
LOGGER.error(f"Exception while getting RealIP list from {url} :\n{format_exc()}")
# Check if file has changed
new_hash = file_hash(tmp_realip_path.joinpath("combined.list"))
old_hash = cache_hash(realip_path.joinpath("combined.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash("combined.list")
if new_hash == old_hash:
logger.info("New file is identical to cache file, reload is not needed")
_exit(0)
LOGGER.info("New file is identical to cache file, reload is not needed")
sys_exit(0)
# Put file in cache
cached, err = cache_file(
tmp_realip_path.joinpath("combined.list"),
realip_path.joinpath("combined.list"),
new_hash,
db,
)
cached, err = JOB.cache_file("combined.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching list : {err}")
_exit(2)
LOGGER.error(f"Error while caching list : {err}")
sys_exit(2)
logger.info(f"Downloaded {i} trusted IP/net")
LOGGER.info(f"Downloaded {i} trusted IP/net")
status = 1
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running realip-download.py :\n{format_exc()}")
LOGGER.error(f"Exception while running realip-download.py :\n{format_exc()}")
sys_exit(status)

116
src/common/core/selfsigned/jobs/self-signed.py
View file

@ -6,31 +6,21 @@ from os.path import join
from pathlib import Path
from subprocess import DEVNULL, STDOUT, run
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
from typing import Tuple
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import set_file_in_db
from jobs import Job # type: ignore
logger = setup_logger("self-signed", getenv("LOG_LEVEL", "INFO"))
db = None
lock = Lock()
LOGGER = setup_logger("self-signed", getenv("LOG_LEVEL", "INFO"))
JOB = Job(LOGGER)
status = 0
@ -53,22 +43,24 @@ def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Pat
).returncode
== 0
):
logger.info(f"Self-signed certificate already present for {first_server}")
LOGGER.info(f"Self-signed certificate already present for {first_server}")
certificate = x509.load_pem_x509_certificate(
self_signed_path.joinpath(f"{first_server}.pem").read_bytes(),
default_backend(),
)
if sorted(attribute.rfc4514_string() for attribute in certificate.subject) != sorted(v for v in subj.split("/") if v):
logger.warning(f"Subject of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ...")
LOGGER.warning(f"Subject of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ...")
elif certificate.not_valid_after - certificate.not_valid_before != timedelta(days=int(days)):
logger.warning(
LOGGER.warning(
f"Expiration date of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ..."
)
else:
return True, 0
logger.info(f"Generating self-signed certificate for {first_server}")
LOGGER.info(f"Generating self-signed certificate for {first_server}")
server_path = self_signed_path.joinpath(first_server)
server_path.mkdir(parents=True, exist_ok=True)
if (
run(
[
@ -79,9 +71,9 @@ def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Pat
"-newkey",
"ed25519",
"-keyout",
str(self_signed_path.joinpath(f"{first_server}.key")),
str(self_signed_path.joinpath(first_server, "key.pem")),
"-out",
str(self_signed_path.joinpath(f"{first_server}.pem")),
str(self_signed_path.joinpath(first_server, "cert.pem")),
"-days",
days,
"-subj",
@ -93,29 +85,19 @@ def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Pat
).returncode
!= 0
):
logger.error(f"Self-signed certificate generation failed for {first_server}")
LOGGER.error(f"Self-signed certificate generation failed for {first_server}")
return False, 2
# Update db
cached, err = set_file_in_db(
f"{first_server}.pem",
self_signed_path.joinpath(f"{first_server}.pem").read_bytes(),
db,
service_id=first_server,
)
cached, err = JOB.cache_file("cert.pem", self_signed_path.joinpath(first_server, "cert.pem"), service_id=first_server, overwrite_file=False)
if not cached:
logger.error(f"Error while caching self-signed {first_server}.pem file : {err}")
LOGGER.error(f"Error while caching self-signed cert.pem file for {first_server} : {err}")
cached, err = set_file_in_db(
f"{first_server}.key",
self_signed_path.joinpath(f"{first_server}.key").read_bytes(),
db,
service_id=first_server,
)
cached, err = JOB.cache_file("key.pem", self_signed_path.joinpath(first_server, "key.pem"), service_id=first_server, overwrite_file=False)
if not cached:
logger.error(f"Error while caching self-signed {first_server}.key file : {err}")
LOGGER.error(f"Error while caching self-signed {first_server}.key file : {err}")
logger.info(f"Successfully generated self-signed certificate for {first_server}")
LOGGER.info(f"Successfully generated self-signed certificate for {first_server}")
return True, 1
@ -123,58 +105,42 @@ status = 0
try:
self_signed_path = Path(sep, "var", "cache", "bunkerweb", "selfsigned")
self_signed_path.mkdir(parents=True, exist_ok=True)
servers = getenv("SERVER_NAME") or []
# Multisite case
if getenv("MULTISITE") == "yes":
servers = getenv("SERVER_NAME") or []
if isinstance(servers, str):
servers = servers.split(" ")
if isinstance(servers, str):
servers = servers.split(" ")
if not servers:
LOGGER.info("No server found, skipping self-signed certificate generation ...")
sys_exit(0)
skipped_servers = []
if not getenv("MULTISITE", "no") == "yes":
servers = [servers[0]]
if getenv("GENERATE_SELF_SIGNED_SSL", "no") == "no":
LOGGER.info("Generate self-signed SSL is not enabled, skipping certificate generation ...")
skipped_servers = servers
if not skipped_servers:
for first_server in servers:
if (
not first_server
or getenv(
f"{first_server}_GENERATE_SELF_SIGNED_SSL",
getenv("GENERATE_SELF_SIGNED_SSL", "no"),
)
!= "yes"
or self_signed_path.joinpath(f"{first_server}.pem").is_file()
):
if getenv(f"{first_server}_GENERATE_SELF_SIGNED_SSL", getenv("GENERATE_SELF_SIGNED_SSL", "no")) != "yes":
LOGGER.info(f"Self-signed SSL is not enabled for {first_server}, skipping certificate generation ...")
skipped_servers.append(first_server)
continue
if not db:
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
ret, ret_status = generate_cert(
first_server,
getenv(
f"{first_server}_SELF_SIGNED_SSL_EXPIRY",
getenv("SELF_SIGNED_SSL_EXPIRY", "365"),
),
getenv(
f"{first_server}_SELF_SIGNED_SSL_SUBJ",
getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/"),
),
getenv(f"{first_server}_SELF_SIGNED_SSL_EXPIRY", getenv("SELF_SIGNED_SSL_EXPIRY", "365")),
getenv(f"{first_server}_SELF_SIGNED_SSL_SUBJ", getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/")),
self_signed_path,
)
status = ret_status
# Singlesite case
elif getenv("GENERATE_SELF_SIGNED_SSL", "no") == "yes" and getenv("SERVER_NAME"):
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
first_server = getenv("SERVER_NAME", "").split(" ")[0]
ret, ret_status = generate_cert(
first_server,
getenv("SELF_SIGNED_SSL_EXPIRY", "365"),
getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/"),
self_signed_path,
)
status = ret_status
for first_server in skipped_servers:
JOB.del_cache("cert.pem", service_id=first_server)
JOB.del_cache("key.pem", service_id=first_server)
except:
status = 2
logger.error(f"Exception while running self-signed.py :\n{format_exc()}")
LOGGER.error(f"Exception while running self-signed.py :\n{format_exc()}")
sys_exit(status)

8
src/common/core/selfsigned/selfsigned.lua
View file

@ -44,8 +44,8 @@ function selfsigned:init()
for server_name, multisite_vars in pairs(vars) do
if multisite_vars["GENERATE_SELF_SIGNED_SSL"] == "yes" and server_name ~= "global" then
local check, data = read_files({
"/var/cache/bunkerweb/selfsigned/" .. server_name .. ".pem",
"/var/cache/bunkerweb/selfsigned/" .. server_name .. ".key",
"/var/cache/bunkerweb/selfsigned/" .. server_name .. "/cert.pem",
"/var/cache/bunkerweb/selfsigned/" .. server_name .. "/key.pem",
})
if not check then
self.logger:log(ERR, "error while reading files : " .. data)
@ -68,8 +68,8 @@ function selfsigned:init()
return self:ret(false, "can't get SERVER_NAME variable : " .. err)
end
local check, data = read_files({
"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. ".pem",
"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. ".key",
"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. "/cert.pem",
"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. "/key.pem",
})
if not check then
self.logger:log(ERR, "error while reading files : " .. data)

112
src/common/core/whitelist/jobs/whitelist-download.py
View file

@ -2,10 +2,9 @@
from contextlib import suppress
from ipaddress import ip_address, ip_network
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join, normpath
from pathlib import Path
from re import IGNORECASE, compile as re_compile
from re import compile as re_compile
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from typing import Tuple
@ -16,11 +15,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from requests import get
from Database import Database # type: ignore
from common_utils import bytes_hash # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, del_file_in_db, is_cached_file, file_hash
from jobs import Job # type: ignore
rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE)
rdns_rx = re_compile(rb"^[^ ]+$")
asn_rx = re_compile(rb"^\d+$")
uri_rx = re_compile(rb"^/")
@ -51,7 +50,7 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]:
return False, b""
logger = setup_logger("WHITELIST", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("WHITELIST", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
@ -68,16 +67,10 @@ try:
whitelist_activated = True
if not whitelist_activated:
logger.info("Whitelist is not activated, skipping downloads...")
_exit(0)
LOGGER.info("Whitelist is not activated, skipping downloads...")
sys_exit(0)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
# Create directories if they don't exist
whitelist_path = Path(sep, "var", "cache", "bunkerweb", "whitelist")
whitelist_path.mkdir(parents=True, exist_ok=True)
tmp_whitelist_path = Path(sep, "var", "tmp", "bunkerweb", "whitelist")
tmp_whitelist_path.mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Get URLs
urls = {"IP": [], "RDNS": [], "ASN": [], "USER_AGENT": [], "URI": []}
@ -87,44 +80,36 @@ try:
urls[kind].append(url)
# Don't go further if the cache is fresh
kinds_fresh = {
"IP": True,
"RDNS": True,
"ASN": True,
"USER_AGENT": True,
"URI": True,
}
all_fresh = True
kinds_fresh = {"IP": True, "RDNS": True, "ASN": True, "USER_AGENT": True, "URI": True}
for kind in kinds_fresh:
if not is_cached_file(whitelist_path.joinpath(f"{kind}.list"), "hour", db):
kinds_fresh[kind] = False
all_fresh = False
logger.info(
f"Whitelist for {kind} is not cached, processing downloads..",
)
else:
logger.info(
f"Whitelist for {kind} is already in cache, skipping downloads...",
)
if not urls[kind]:
logger.info(
f"Whitelist for {kind} is already in cache, skipping downloads...",
)
whitelist_path.joinpath(f"{kind}.list").unlink(missing_ok=True)
deleted, err = del_file_in_db(f"{kind}.list", db)
if not deleted:
logger.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all_fresh:
_exit(0)
if not JOB.is_cached_file(f"{kind}.list", "hour"):
if urls[kind]:
kinds_fresh[kind] = False
LOGGER.info(f"Whitelist for {kind} is not cached, processing downloads..")
continue
LOGGER.info(f"Whitelist for {kind} is already in cache, skipping downloads...")
if not urls[kind]:
LOGGER.warning(f"Whitelist for {kind} is cached but no URL is configured, removing from cache...")
deleted, err = JOB.del_cache(f"{kind}.list")
if not deleted:
LOGGER.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all(kinds_fresh.values()):
if not any(urls.values()):
LOGGER.info("No whitelist URL is configured, nothing to do...")
sys_exit(0)
# Loop on kinds
for kind, urls_list in urls.items():
if kinds_fresh[kind]:
continue
# Write combined data of the kind to a single temp file
# Write combined data of the kind in memory and check if it has changed
for url in urls_list:
try:
logger.info(f"Downloading whitelist data from {url} ...")
LOGGER.info(f"Downloading whitelist data from {url} ...")
if url.startswith("file://"):
with open(normpath(url[7:]), "rb") as f:
iterable = f.readlines()
@ -132,7 +117,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -152,39 +137,28 @@ try:
content += data + b"\n"
i += 1
tmp_whitelist_path.joinpath(f"{kind}.list").write_bytes(content)
logger.info(f"Downloaded {i} bad {kind}")
LOGGER.info(f"Downloaded {i} bad {kind}")
# Check if file has changed
new_hash = file_hash(tmp_whitelist_path.joinpath(f"{kind}.list"))
old_hash = cache_hash(whitelist_path.joinpath(f"{kind}.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash(f"{kind}.list")
if new_hash == old_hash:
logger.info(
f"New file {kind}.list is identical to cache file, reload is not needed",
)
LOGGER.info(f"New file {kind}.list is identical to cache file, reload is not needed")
else:
logger.info(
f"New file {kind}.list is different than cache file, reload is needed",
)
LOGGER.info(f"New file {kind}.list is different than cache file, reload is needed")
# Put file in cache
cached, err = cache_file(
tmp_whitelist_path.joinpath(f"{kind}.list"),
whitelist_path.joinpath(f"{kind}.list"),
new_hash,
db,
)
cached, err = JOB.cache_file(f"{kind}.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching whitelist : {err}")
LOGGER.error(f"Error while caching whitelist : {err}")
status = 2
else:
status = 1
except:
status = 2
logger.error(f"Exception while getting whitelist from {url} :\n{format_exc()}")
LOGGER.error(f"Exception while getting whitelist from {url} :\n{format_exc()}")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running whitelist-download.py :\n{format_exc()}")
LOGGER.error(f"Exception while running whitelist-download.py :\n{format_exc()}")
sys_exit(status)

120
src/common/db/Database.py
View file

@ -36,7 +36,7 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
if deps_path not in sys_path:
sys_path.append(deps_path)
from jobs import file_hash # type: ignore
from common_utils import file_hash # type: ignore
from pymysql import install_as_MySQLdb
from sqlalchemy import create_engine, MetaData as sql_metadata, text, inspect
@ -65,7 +65,7 @@ class Database:
pool: bool = True,
) -> None:
"""Initialize the database"""
self.__logger = logger
self.logger = logger
self.__session_factory = None
self.__sql_engine = None
@ -74,14 +74,14 @@ class Database:
match = self.DB_STRING_RX.search(sqlalchemy_string)
if not match:
self.__logger.error(f"Invalid database string provided: {sqlalchemy_string}, exiting...")
self.logger.error(f"Invalid database string provided: {sqlalchemy_string}, exiting...")
_exit(1)
if match.group("database").startswith("sqlite"):
db_path = Path(normpath(match.group("path")))
if ui:
while not db_path.is_file():
self.__logger.warning(f"Waiting for the database file to be created: {db_path}")
self.logger.warning(f"Waiting for the database file to be created: {db_path}")
sleep(1)
else:
db_path.parent.mkdir(parents=True, exist_ok=True)
@ -102,10 +102,10 @@ class Database:
try:
self.__sql_engine = create_engine(sqlalchemy_string, **engine_kwargs)
except ArgumentError:
self.__logger.error(f"Invalid database URI: {sqlalchemy_string}")
self.logger.error(f"Invalid database URI: {sqlalchemy_string}")
error = True
except SQLAlchemyError:
self.__logger.error(f"Error when trying to create the engine: {format_exc()}")
self.logger.error(f"Error when trying to create the engine: {format_exc()}")
error = True
finally:
if error:
@ -114,7 +114,7 @@ class Database:
try:
assert self.__sql_engine is not None
except AssertionError:
self.__logger.error("The database engine is not initialized")
self.logger.error("The database engine is not initialized")
_exit(1)
not_connected = True
@ -128,29 +128,29 @@ class Database:
not_connected = False
except (OperationalError, DatabaseError) as e:
if retries <= 0:
self.__logger.error(
self.logger.error(
f"Can't connect to database : {format_exc()}",
)
_exit(1)
if "attempt to write a readonly database" in str(e):
self.__logger.warning("The database is read-only, waiting for it to become writable. Retrying in 5 seconds ...")
self.logger.warning("The database is read-only, waiting for it to become writable. Retrying in 5 seconds ...")
self.__sql_engine.dispose(close=True)
self.__sql_engine = create_engine(sqlalchemy_string, **engine_kwargs)
if "Unknown table" in str(e):
not_connected = False
continue
else:
self.__logger.warning(
self.logger.warning(
"Can't connect to database, retrying in 5 seconds ...",
)
retries -= 1
sleep(5)
except BaseException:
self.__logger.error(f"Error when trying to connect to the database: {format_exc()}")
self.logger.error(f"Error when trying to connect to the database: {format_exc()}")
exit(1)
self.__logger.info("✅ Database connection established")
self.logger.info("✅ Database connection established")
self.__session_factory = sessionmaker(bind=self.__sql_engine, autoflush=True, expire_on_commit=False)
self.suffix_rx = re_compile(r"_\d+$")
@ -173,7 +173,7 @@ class Database:
try:
assert self.__session_factory is not None
except AssertionError:
self.__logger.error("The database session is not initialized")
self.logger.error("The database session is not initialized")
_exit(1)
session = scoped_session(self.__session_factory)
@ -420,13 +420,13 @@ class Database:
db_version = self.get_metadata()["version"]
if db_version != bunkerweb_version:
self.__logger.warning(f"Database version ({db_version}) is different from Bunkerweb version ({bunkerweb_version}), migrating ...")
self.logger.warning(f"Database version ({db_version}) is different from Bunkerweb version ({bunkerweb_version}), migrating ...")
metadata = sql_metadata()
metadata.reflect(self.__sql_engine)
for table_name in Base.metadata.tables.keys():
if not inspector.has_table(table_name):
self.__logger.warning(f'Table "{table_name}" is missing')
self.logger.warning(f'Table "{table_name}" is missing')
has_all_tables = False
continue
@ -519,7 +519,7 @@ class Database:
updates[Plugins.checksum] = plugin.get("checksum")
if updates:
self.__logger.warning(f'Plugin "{plugin["id"]}" already exists, updating it with the new values')
self.logger.warning(f'Plugin "{plugin["id"]}" already exists, updating it with the new values')
session.query(Plugins).filter(Plugins.id == plugin["id"]).update(updates)
else:
to_put.append(
@ -578,11 +578,11 @@ class Database:
updates[Settings.multiple] = value.get("multiple")
if updates:
self.__logger.warning(f'Setting "{setting}" already exists, updating it with the new values')
self.logger.warning(f'Setting "{setting}" already exists, updating it with the new values')
session.query(Settings).filter(Settings.id == setting).update(updates)
else:
if db_plugin:
self.__logger.warning(f'Setting "{setting}" does not exist, creating it')
self.logger.warning(f'Setting "{setting}" does not exist, creating it')
to_put.append(Settings(**value))
db_values = [select.value for select in session.query(Selects).with_entities(Selects.value).filter_by(setting_id=value["id"])]
@ -591,7 +591,7 @@ class Database:
if select_values:
if missing_values:
# Remove selects that are no longer in the list
self.__logger.warning(f'Removing {len(missing_values)} selects from setting "{setting}" as they are no longer in the list')
self.logger.warning(f'Removing {len(missing_values)} selects from setting "{setting}" as they are no longer in the list')
session.query(Selects).filter(Selects.value.in_(missing_values)).delete()
for select in select_values:
@ -599,7 +599,7 @@ class Database:
to_put.append(Selects(setting_id=value["id"], value=select))
else:
if missing_values:
self.__logger.warning(f'Removing all selects from setting "{setting}" as there are no longer any in the list')
self.logger.warning(f'Removing all selects from setting "{setting}" as there are no longer any in the list')
session.query(Selects).filter_by(setting_id=value["id"]).delete()
db_names = [job.name for job in session.query(Jobs).with_entities(Jobs.name).filter_by(plugin_id=plugin["id"])]
@ -608,7 +608,7 @@ class Database:
if missing_names:
# Remove jobs that are no longer in the list
self.__logger.warning(f'Removing {len(missing_names)} jobs from plugin "{plugin["id"]}" as they are no longer in the list')
self.logger.warning(f'Removing {len(missing_names)} jobs from plugin "{plugin["id"]}" as they are no longer in the list')
session.query(Jobs).filter(Jobs.name.in_(missing_names)).delete()
for job in jobs:
@ -623,7 +623,7 @@ class Database:
job["file_name"] = job.pop("file")
job["reload"] = job.get("reload", False)
if db_plugin:
self.__logger.warning(f'Job "{job["name"]}" does not exist, creating it')
self.logger.warning(f'Job "{job["name"]}" does not exist, creating it')
to_put.append(Jobs(plugin_id=plugin["id"], **job))
else:
updates = {}
@ -638,7 +638,7 @@ class Database:
updates[Jobs.reload] = job.get("reload", False)
if updates:
self.__logger.warning(f'Job "{job["name"]}" already exists, updating it with the new values')
self.logger.warning(f'Job "{job["name"]}" already exists, updating it with the new values')
updates[Jobs.last_run] = None
session.query(Jobs_cache).filter(Jobs_cache.job_name == job["name"]).delete()
session.query(Jobs).filter(Jobs.name == job["name"]).update(updates)
@ -681,12 +681,12 @@ class Database:
)
if updates:
self.__logger.warning(f'Page for plugin "{plugin["id"]}" already exists, updating it with the new values')
self.logger.warning(f'Page for plugin "{plugin["id"]}" already exists, updating it with the new values')
session.query(Plugin_pages).filter(Plugin_pages.plugin_id == plugin["id"]).update(updates)
continue
if db_plugin:
self.__logger.warning(f'Page for plugin "{plugin["id"]}" does not exist, creating it')
self.logger.warning(f'Page for plugin "{plugin["id"]}" does not exist, creating it')
to_put.append(
Plugin_pages(
@ -1162,10 +1162,16 @@ class Database:
def delete_job_cache(self, file_name: str, *, job_name: Optional[str] = None, service_id: Optional[str] = None):
job_name = job_name or basename(getsourcefile(_getframe(1))).replace(".py", "")
with self.__db_session() as session:
session.query(Jobs_cache).filter_by(job_name=job_name, file_name=file_name, service_id=service_id).delete()
filters = {"file_name": file_name}
if job_name:
filters["job_name"] = job_name
if service_id:
filters["service_id"] = service_id
def update_job_cache(
with self.__db_session() as session:
session.query(Jobs_cache).filter_by(**filters).delete()
def upsert_job_cache(
self,
service_id: Optional[str],
file_name: str,
@ -1254,7 +1260,7 @@ class Database:
if db_plugin:
if db_plugin.type not in ("external", "pro"):
self.__logger.warning(
self.logger.warning(
f"Plugin \"{plugin['id']}\" is not {_type}, skipping update (updating a non-external or non-pro plugin is forbidden for security reasons)", # noqa: E501
)
continue
@ -1484,7 +1490,7 @@ class Database:
db_setting = session.query(Settings).filter_by(id=setting).first()
if db_setting is not None:
self.__logger.warning(f"A setting with id {setting} already exists, therefore it will not be added.")
self.logger.warning(f"A setting with id {setting} already exists, therefore it will not be added.")
continue
value.update({"plugin_id": plugin["id"], "name": value["id"], "id": setting})
@ -1500,7 +1506,7 @@ class Database:
)
if db_job is not None:
self.__logger.warning(f"A job with the name {job['name']} already exists in the database, therefore it will not be added.")
self.logger.warning(f"A job with the name {job['name']} already exists in the database, therefore it will not be added.")
continue
job["file_name"] = job.pop("file")
@ -1681,13 +1687,8 @@ class Database:
}
def get_job_cache_file(
self,
job_name: str,
file_name: str,
*,
with_info: bool = False,
with_data: bool = True,
) -> Optional[Any]:
self, job_name: str, file_name: str, *, service_id: str = "", with_info: bool = False, with_data: bool = True
) -> Optional[Union[Dict[str, Any], bytes]]:
"""Get job cache file."""
entities = []
if with_info:
@ -1695,26 +1696,47 @@ class Database:
if with_data:
entities.append(Jobs_cache.data)
with self.__db_session() as session:
return session.query(Jobs_cache).with_entities(*entities).filter_by(job_name=job_name, file_name=file_name).first()
filters = {"job_name": job_name, "file_name": file_name}
if service_id:
filters["service_id"] = service_id
def get_jobs_cache_files(self) -> List[Dict[str, Any]]:
with self.__db_session() as session:
data = session.query(Jobs_cache).with_entities(*entities).filter_by(**filters).first()
if not data:
return None
if with_data and not with_info:
return data.data
ret_data = {}
if with_info:
ret_data["last_update"] = data.last_update.timestamp() if data.last_update is not None else "Never"
ret_data["checksum"] = data.checksum
if with_data:
ret_data["data"] = data.data
return ret_data
def get_jobs_cache_files(self, *, job_name: str = "", with_data: bool = False) -> List[Dict[str, Any]]:
"""Get jobs cache files."""
with self.__db_session() as session:
entities = [Jobs_cache.job_name, Jobs_cache.service_id, Jobs_cache.file_name]
if with_data:
entities.append(Jobs_cache.data)
query = session.query(Jobs_cache).with_entities(*entities)
if job_name:
query = query.filter_by(job_name=job_name)
return [
{
"job_name": cache.job_name,
"service_id": cache.service_id,
"file_name": cache.file_name,
"data": "Download file to view content",
"data": "Download file to view content" if not with_data else cache.data,
}
for cache in (
session.query(Jobs_cache).with_entities(
Jobs_cache.job_name,
Jobs_cache.service_id,
Jobs_cache.file_name,
)
)
for cache in query
]
def add_instance(self, hostname: str, port: int, server_name: str, changed: Optional[bool] = True) -> str:

80
src/common/utils/common_utils.py Normal file
View file

@ -0,0 +1,80 @@
from hashlib import sha512
from io import BytesIO
from os import getenv, sep
from pathlib import Path
from platform import machine
from typing import Dict, Union
def get_version() -> str:
return Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text(encoding="utf-8").strip()
def get_integration() -> str:
try:
integration_path = Path(sep, "usr", "share", "bunkerweb", "INTEGRATION")
os_release_path = Path(sep, "etc", "os-release")
if getenv("KUBERNETES_MODE", "no").lower() == "yes":
return "Kubernetes"
elif getenv("SWARM_MODE", "no").lower() == "yes":
return "Swarm"
elif getenv("AUTOCONF_MODE", "no").lower() == "yes":
return "Autoconf"
elif integration_path.is_file():
return integration_path.read_text(encoding="utf-8").strip().lower()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
return "Docker"
return "Linux"
except:
return "Unknown"
def get_os_info() -> Dict[str, str]:
os_data = {
"name": "Linux",
"version": "Unknown",
"version_id": "Unknown",
"version_codename": "Unknown",
"id": "Unknown",
"arch": machine(),
}
os_release = Path("/etc/os-release")
if os_release.exists():
for line in os_release.read_text().splitlines():
if "=" not in line or line.split("=")[0].strip().lower() not in os_data:
continue
os_data[line.split("=")[0].lower()] = line.split("=")[1].strip('"')
return os_data
def file_hash(file: Union[str, Path]) -> str:
_sha512 = sha512()
if not isinstance(file, Path):
file = Path(file)
with file.open("rb") as f:
while True:
data = f.read(1024)
if not data:
break
_sha512.update(data)
return _sha512.hexdigest()
def bytes_hash(bio: Union[bytes, BytesIO]) -> str:
if isinstance(bio, bytes):
bio = BytesIO(bio)
assert isinstance(bio, BytesIO)
_sha512 = sha512()
while True:
data = bio.read(1024)
if not data:
break
_sha512.update(data)
bio.seek(0)
return _sha512.hexdigest()

439
src/common/utils/jobs.py
View file

@ -1,252 +1,249 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from datetime import datetime
from hashlib import sha512
from datetime import datetime, timedelta
from inspect import getsourcefile
from io import BufferedReader
from json import dumps, loads
from os import getenv, sep
from os.path import basename
from io import BytesIO
from logging import Logger
from os import getenv
from os.path import sep
from pathlib import Path
from platform import machine
from shutil import rmtree
from sys import _getframe
from tarfile import open as tar_open
from threading import Lock
from traceback import format_exc
from typing import Dict, Literal, Optional, Tuple, Union
from typing import Any, Dict, Literal, Optional, Tuple, Union
lock = Lock()
from common_utils import bytes_hash, file_hash
"""
{
"date": timestamp,
"checksum": sha512
LOCK = Lock()
EXPIRE_TIME = {
"hour": timedelta(hours=1).total_seconds(),
"day": timedelta(days=1).total_seconds(),
"week": timedelta(weeks=1).total_seconds(),
"month": timedelta(days=30).total_seconds(),
}
"""
def is_cached_file(
file: Union[str, Path],
expire: Union[Literal["hour"], Literal["day"], Literal["week"], Literal["month"]],
db=None,
) -> bool:
if not isinstance(file, Path):
file = Path(file)
class Job:
def __init__(self, logger: Optional[Logger] = None, db=None, *, job_name: str = "", deprecated: bool = False):
source_file = getsourcefile(_getframe(1))
if source_file is None:
raise ValueError("source_file could not be determined.")
elif not logger and not db:
raise ValueError("Either logger or db must be provided.")
source_path = Path(source_file)
self.job_path = Path(sep, "var", "cache", "bunkerweb", source_path.parent.parent.name)
self.job_name = job_name or source_path.name.replace(".py", "")
is_cached = False
cached_file = None
try:
file_path = Path(f"{file}.md")
if not file_path.is_file():
if not db:
return False
cached_file = db.get_job_cache_file(
basename(getsourcefile(_getframe(1))).replace(".py", ""),
file.name,
with_info=True,
)
self.db = db
if not self.db:
from Database import Database # type: ignore
if not cached_file:
return False
cached_time = cached_file.last_update.timestamp()
else:
cached_time = loads(file_path.read_text())["date"]
self.db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
self.logger = logger or self.db.logger
current_time = datetime.now().timestamp()
if current_time < cached_time:
is_cached = False
else:
diff_time = current_time - cached_time
if expire == "hour":
is_cached = diff_time < 3600
elif expire == "day":
is_cached = diff_time < 86400
elif expire == "week":
is_cached = diff_time < 604800
elif expire == "month":
is_cached = diff_time < 2592000
except:
is_cached = False
if not deprecated:
self.restore_cache()
if is_cached and cached_file:
file.write_bytes(cached_file.data)
def restore_cache(self, *, job_name: str = "") -> bool:
"""Restore job cache files from database."""
ret = True
with LOCK:
job_cache_files = self.db.get_jobs_cache_files(job_name=job_name or self.job_name, with_data=True) # type: ignore
return is_cached and cached_file
def get_file_in_db(file: Union[str, Path], db, *, job_name: Optional[str] = None) -> Optional[bytes]:
cached_file = db.get_job_cache_file(job_name or basename(getsourcefile(_getframe(1))).replace(".py", ""), file)
if not cached_file:
return None
return cached_file.data
def set_file_in_db(
name: str,
content: bytes,
db,
*,
job_name: Optional[str] = None,
service_id: Optional[str] = None,
checksum: Optional[str] = None,
) -> Tuple[bool, str]:
ret, err = True, "success"
try:
with lock:
err = db.update_job_cache(
service_id,
name,
content,
job_name=job_name or basename(getsourcefile(_getframe(1))).replace(".py", ""),
checksum=checksum,
)
if err:
for job_cache_file in job_cache_files:
try:
cache_path = self.job_path.joinpath(job_cache_file["service_id"] or "", job_cache_file["file_name"])
if job_cache_file["file_name"].endswith(".tgz"):
rmtree(cache_path.parent, ignore_errors=True)
cache_path.parent.mkdir(parents=True, exist_ok=True)
with tar_open(fileobj=BytesIO(job_cache_file["data"]), mode="r:gz") as tar:
tar.extractall(cache_path.parent)
else:
cache_path.parent.mkdir(parents=True, exist_ok=True)
cache_path.write_bytes(job_cache_file["data"])
except BaseException as e:
self.logger.error(f"Exception while restoring cache file {job_cache_file['file_name']} :\n{e}")
ret = False
except:
return False, f"exception :\n{format_exc()}"
return ret, err
return ret
def get_cache(
self, name: str, *, job_name: str = "", service_id: str = "", with_info: bool = False, with_data: bool = True
) -> Optional[Union[Dict[str, Any], bytes]]:
"""Get cache file from database or from local cache file."""
cache_path = self.job_path.joinpath(service_id, name)
if cache_path.is_file():
if with_data and not with_info:
return cache_path.read_bytes()
ret_data = {}
if with_info:
ret_data = {
"last_update": cache_path.stat().st_mtime,
"checksum": file_hash(cache_path),
}
if with_data:
ret_data["data"] = cache_path.read_bytes()
return ret_data
with LOCK:
return self.db.get_job_cache_file(job_name or self.job_name, name, service_id=service_id, with_info=with_info, with_data=with_data) # type: ignore
def is_cached_file(self, name: str, expire: Literal["hour", "day", "week", "month"], *, job_name: str = "", service_id: str = "") -> bool:
"""Check if cache file is cached and if it's still fresh."""
is_cached = False
try:
cache_info = self.get_cache(name, job_name=job_name, service_id=service_id, with_info=True, with_data=False)
if isinstance(cache_info, dict):
current_time = datetime.now().timestamp()
if current_time < cache_info["last_update"]:
is_cached = False
else:
is_cached = current_time - cache_info["last_update"] < EXPIRE_TIME[expire]
except:
is_cached = False
return is_cached
def cache_file(
self,
name: str,
file_cache: Union[bytes, str, Path],
*,
job_name: str = "",
service_id: str = "",
checksum: Optional[str] = None,
delete_file: bool = True,
overwrite_file: bool = True,
) -> Tuple[bool, str]:
"""Cache file in database and in local cache file."""
ret, err = True, "success"
cache_path = self.job_path.joinpath(service_id, name)
cache_path.parent.mkdir(parents=True, exist_ok=True)
if isinstance(file_cache, bytes):
content = file_cache
else:
if isinstance(file_cache, str):
file_cache = Path(file_cache)
assert isinstance(file_cache, Path)
content = file_cache.read_bytes()
if overwrite_file or not cache_path.is_file():
cache_path.write_bytes(content)
if not checksum:
checksum = bytes_hash(content)
try:
with LOCK:
if self.db.upsert_job_cache(service_id, name, content, job_name=job_name or self.job_name, checksum=checksum): # type: ignore
ret = False
if ret and isinstance(file_cache, Path) and delete_file and file_cache != cache_path:
file_cache.unlink(missing_ok=True)
except:
return False, f"exception :\n{format_exc()}"
return ret, err
def cache_dir(self, dir_path: Union[str, Path], *, job_name: str = "", service_id: str = "") -> Tuple[bool, str]:
"""Cache directory in database and in local cache file."""
if isinstance(dir_path, str):
dir_path = Path(dir_path)
assert isinstance(dir_path, Path)
file_name = f"{dir_path.name}.tgz"
content = BytesIO()
with tar_open(file_name, mode="w:gz", fileobj=content, compresslevel=9) as tgz:
tgz.add(dir_path, arcname=".")
content.seek(0, 0)
return self.cache_file(file_name, content.read(), job_name=job_name, service_id=service_id)
def del_cache(self, name: str, *, job_name: str = "", service_id: str = "") -> Tuple[bool, str]:
"""Delete cache file from database and local cache file."""
ret, err = True, "success"
job_name = job_name or self.job_name
job_path = self.job_path.joinpath(service_id)
cache_path = job_path.joinpath(name)
if cache_path.is_file():
cache_path.unlink(missing_ok=True)
if job_path.is_dir() and not list(job_path.iterdir()):
rmtree(job_path, ignore_errors=True)
try:
with LOCK:
self.db.delete_job_cache(name, job_name=job_name, service_id=service_id) # type: ignore
except:
return False, f"exception :\n{format_exc()}"
return ret, err
def cache_hash(self, name: str, *, job_name: str = "", service_id: str = "") -> Optional[str]:
"""Get cache file hash from database or from local cache file."""
cache_path = self.job_path.joinpath(service_id, name)
if cache_path.is_file():
return file_hash(cache_path)
cache_info = self.get_cache(name, with_info=True, with_data=False, job_name=job_name, service_id=service_id)
if isinstance(cache_info, dict):
return cache_info.get("checksum")
return None
def del_file_in_db(name: str, db, *, service_id: Optional[str] = None) -> Tuple[bool, str]:
ret, err = True, "success"
try:
db.delete_job_cache(name, job_name=basename(getsourcefile(_getframe(1))).replace(".py", ""), service_id=service_id)
except:
return False, f"exception :\n{format_exc()}"
return ret, err
# ? Backward compatibility functions
def file_hash(file: Union[str, Path]) -> str:
_sha512 = sha512()
def is_cached_file(file: Union[str, Path], expire: Literal["hour", "day", "week", "month"], db) -> bool:
job = Job(None, db, deprecated=True)
job.logger.warning("is_cached_file is deprecated, use the Job.is_cached_file method instead.")
if not isinstance(file, Path):
file = Path(file)
with file.open("rb") as f:
while True:
data = f.read(1024)
if not data:
break
_sha512.update(data)
return _sha512.hexdigest()
return job.is_cached_file(file.name, expire)
def bytes_hash(bio: BufferedReader) -> str:
_sha512 = sha512()
while True:
data = bio.read(1024)
if not data:
break
_sha512.update(data)
bio.seek(0)
return _sha512.hexdigest()
def cache_hash(cache: Union[str, Path], db=None) -> Optional[str]:
checksum = None
cache_file = Path(f"{cache}.md")
if cache_file.is_file():
checksum = loads(cache_file.read_text(encoding="utf-8")).get("checksum", None)
if not checksum and db:
if not isinstance(cache, Path):
cache = Path(cache)
cached_file = db.get_job_cache_file(
basename(getsourcefile(_getframe(1))).replace(".py", ""),
cache.name,
with_info=True,
with_data=False,
)
checksum = cached_file.checksum if cached_file else None
if checksum:
return checksum
def get_file_in_db(file: Union[str, Path], db, *, job_name: str = "") -> Optional[bytes]:
job = Job(None, db, deprecated=True)
job.logger.warning("get_file_in_db is deprecated, use the Job.get_cache method instead.")
if not isinstance(file, Path):
file = Path(file)
cache = job.get_cache(file.name, job_name=job_name, with_data=True)
if isinstance(cache, dict):
return cache["data"]
return None
def set_file_in_db(name: str, content: bytes, db, *, job_name: str = "", service_id: str = "", checksum: Optional[str] = None) -> Tuple[bool, str]:
job = Job(None, db, deprecated=True)
job.logger.warning("set_file_in_db is deprecated, use the Job.cache_file method instead.")
return job.cache_file(name, content, job_name=job_name, service_id=service_id, checksum=checksum)
def del_file_in_db(name: str, db, *, service_id: str = "") -> Tuple[bool, str]:
job = Job(None, db, deprecated=True)
job.logger.warning("del_file_in_db is deprecated, use the Job.del_cache method instead.")
return job.del_cache(name, service_id=service_id)
def cache_hash(cache: Union[str, Path], db) -> Optional[str]:
job = Job(None, db, deprecated=True)
job.logger.warning("cache_hash is deprecated, use the Job.cache_hash method instead.")
if not isinstance(cache, Path):
cache = Path(cache)
return job.cache_hash(cache.name)
def cache_file(
file: Union[str, Path],
cache: Union[str, Path],
_hash: Optional[str],
db=None,
*,
delete_file: bool = True,
service_id: Optional[str] = None,
file: Union[str, Path], cache: Union[str, Path], _hash: Optional[str], db, *, delete_file: bool = True, service_id: str = ""
) -> Tuple[bool, str]:
ret, err = True, "success"
try:
if not isinstance(file, Path):
file = Path(file)
if not isinstance(cache, Path):
cache = Path(cache)
content = file.read_bytes()
cache.write_bytes(content)
if delete_file:
file.unlink()
if not _hash:
_hash = file_hash(str(cache))
if db:
return set_file_in_db(
cache.name,
content,
db,
job_name=basename(getsourcefile(_getframe(1))).replace(".py", ""),
service_id=service_id,
checksum=_hash,
)
else:
Path(f"{cache}.md").write_text(
dumps(dict(date=datetime.now().timestamp(), checksum=_hash)),
encoding="utf-8",
)
except:
return False, f"exception :\n{format_exc()}"
return ret, err
def get_version() -> str:
return Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text(encoding="utf-8").strip()
def get_integration() -> str:
try:
integration_path = Path(sep, "usr", "share", "bunkerweb", "INTEGRATION")
os_release_path = Path(sep, "etc", "os-release")
if getenv("KUBERNETES_MODE", "no").lower() == "yes":
return "Kubernetes"
elif getenv("SWARM_MODE", "no").lower() == "yes":
return "Swarm"
elif getenv("AUTOCONF_MODE", "no").lower() == "yes":
return "Autoconf"
elif integration_path.is_file():
return integration_path.read_text(encoding="utf-8").strip().lower()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
return "Docker"
return "Linux"
except:
return "Unknown"
def get_os_info() -> Dict[str, str]:
os_data = {
"name": "Linux",
"version": "Unknown",
"version_id": "Unknown",
"version_codename": "Unknown",
"id": "Unknown",
"arch": machine(),
}
os_release = Path("/etc/os-release")
if os_release.exists():
for line in os_release.read_text().splitlines():
if "=" not in line or line.split("=")[0].strip().lower() not in os_data:
continue
os_data[line.split("=")[0].lower()] = line.split("=")[1].strip('"')
return os_data
job = Job(None, db, deprecated=True)
job.logger.warning("cache_file is deprecated, use the Job.cache_file method instead.")
if not isinstance(file, Path):
file = Path(file)
if not isinstance(cache, Path):
cache = Path(cache)
return job.cache_file(cache.name, file, job_name=cache.name, service_id=service_id, checksum=_hash, delete_file=delete_file)