Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add dict_to_frozenset function to common_utils.py and optimize + made more secure custom configs and plugins handling via the scheduler

Browse source
This commit is contained in:
Théophile Diot 2024-04-05 13:49:42 +01:00
parent 391868b040
commit e1ecb6b324
No known key found for this signature in database
GPG key ID: 248FEA4BAE400D06
2 changed files with 126 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/common/utils/common_utils.py
View file

@ -6,6 +6,14 @@ from platform import machine
from typing import Dict, Union
def dict_to_frozenset(d):
if isinstance(d, list):
return tuple(sorted(d))
elif isinstance(d, dict):
return frozenset((k, dict_to_frozenset(v)) for k, v in d.items())
return d
def get_version() -> str:
return Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text(encoding="utf-8").strip()
@ -78,5 +86,5 @@ def bytes_hash(bio: Union[str, bytes, BytesIO], *, algorithm: str = "sha512") ->
if not data:
break
_hash.update(data)
bio.seek(0)
bio.seek(0, 0)
return _hash.hexdigest()

218
src/scheduler/main.py
View file

@ -1,15 +1,13 @@
#!/usr/bin/env python3
from argparse import ArgumentParser
from copy import deepcopy
from contextlib import suppress
from datetime import datetime
from glob import glob
from hashlib import sha256
from io import BytesIO
from itertools import chain
from json import load as json_load
from os import _exit, environ, getenv, getpid, listdir, sep, walk
from os.path import basename, dirname, join, normpath
from os import _exit, environ, getenv, getpid, sep
from os.path import join
from pathlib import Path
from shutil import copy, rmtree
from signal import SIGINT, SIGTERM, signal, SIGHUP
@ -28,24 +26,47 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from dotenv import dotenv_values
from common_utils import get_integration # type: ignore
from common_utils import bytes_hash, dict_to_frozenset, get_integration # type: ignore
from logger import setup_logger # type: ignore
from Database import Database # type: ignore
from JobScheduler import JobScheduler
RUN = True
SCHEDULER: Optional[JobScheduler] = None
CACHE_PATH = join(sep, "var", "cache", "bunkerweb")
Path(CACHE_PATH).mkdir(parents=True, exist_ok=True)
CUSTOM_CONFIGS_PATH = Path(sep, "etc", "bunkerweb", "configs")
CUSTOM_CONFIGS_PATH.mkdir(parents=True, exist_ok=True)
CUSTOM_CONFIGS_DIRS = (
"http",
"stream",
"server-http",
"server-stream",
"default-server-http",
"default-server-stream",
"modsec",
"modsec-crs",
)
for custom_config_dir in CUSTOM_CONFIGS_DIRS:
CUSTOM_CONFIGS_PATH.joinpath(custom_config_dir).mkdir(parents=True, exist_ok=True)
EXTERNAL_PLUGINS_PATH = Path(sep, "etc", "bunkerweb", "plugins")
EXTERNAL_PLUGINS_PATH.mkdir(parents=True, exist_ok=True)
PRO_PLUGINS_PATH = Path(sep, "etc", "bunkerweb", "pro", "plugins")
PRO_PLUGINS_PATH.mkdir(parents=True, exist_ok=True)
TMP_PATH = Path(sep, "var", "tmp", "bunkerweb")
TMP_PATH.mkdir(parents=True, exist_ok=True)
HEALTHY_PATH = TMP_PATH.joinpath("scheduler.healthy")
SCHEDULER_TMP_ENV_PATH = TMP_PATH.joinpath("scheduler.env")
SCHEDULER_TMP_ENV_PATH.touch()
DB_LOCK_FILE = Path(sep, "var", "lib", "bunkerweb", "db.lock")
logger = setup_logger("Scheduler", getenv("LOG_LEVEL", "INFO"))
@ -85,18 +106,19 @@ def stop(status):
_exit(status)
def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Union[Path, str] = join(sep, "etc", "bunkerweb", "configs")):
def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Union[Path, str] = CUSTOM_CONFIGS_PATH):
if not isinstance(original_path, Path):
original_path = Path(original_path)
# Remove old custom configs files
logger.info("Removing old custom configs files ...")
for file in glob(str(original_path.joinpath("*", "*"))):
file = Path(file)
if file.is_symlink() or file.is_file():
file.unlink()
elif file.is_dir():
rmtree(file, ignore_errors=True)
if original_path.is_dir():
for file in original_path.glob("*/*"):
if file.is_symlink() or file.is_file():
with suppress(OSError):
file.unlink()
elif file.is_dir():
rmtree(file, ignore_errors=True)
if configs:
logger.info("Generating new custom configs ...")
@ -107,11 +129,17 @@ def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Uni
tmp_path = original_path.joinpath(
custom_config["type"].replace("_", "-"),
custom_config["service_id"] or "",
f"{custom_config['name']}.conf",
f"{Path(custom_config['name']).stem}.conf",
)
tmp_path.parent.mkdir(parents=True, exist_ok=True)
tmp_path.write_bytes(custom_config["data"])
except OSError as e:
if custom_config["method"] != "manual":
logger.error(
f"Error while generating custom configs \"{custom_config['name']}\"{' for service ' + custom_config['service_id'] if custom_config['service_id'] else ''}: {e}"
)
except BaseException as e:
logger.debug(format_exc())
logger.error(
f"Error while generating custom configs \"{custom_config['name']}\"{' for service ' + custom_config['service_id'] if custom_config['service_id'] else ''}: {e}"
)
@ -127,16 +155,17 @@ def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Uni
def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: Union[Path, str] = EXTERNAL_PLUGINS_PATH):
if not isinstance(original_path, Path):
original_path = Path(original_path)
pro = original_path.as_posix().endswith("/pro/plugins")
pro = "pro" in original_path.parts
# Remove old external/pro plugins files
logger.info(f"Removing old {'pro ' if pro else ''}external plugins files ...")
for file in glob(str(original_path.joinpath("*"))):
file = Path(file)
if file.is_symlink() or file.is_file():
file.unlink()
elif file.is_dir():
rmtree(file, ignore_errors=True)
if original_path.is_dir():
for file in original_path.glob("*"):
if file.is_symlink() or file.is_file():
with suppress(OSError):
file.unlink()
elif file.is_dir():
rmtree(file, ignore_errors=True)
if plugins:
logger.info(f"Generating new {'pro ' if pro else ''}external plugins ...")
@ -155,7 +184,11 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U
for job_file in chain(original_path.joinpath(plugin["id"], "jobs").glob("*"), original_path.joinpath(plugin["id"], "bwcli").glob("*")):
job_file.chmod(job_file.stat().st_mode | S_IEXEC)
except OSError as e:
if plugin["method"] != "manual":
logger.error(f"Error while generating {'pro ' if pro else ''}external plugins \"{plugin['name']}\": {e}")
except BaseException as e:
logger.debug(format_exc())
logger.error(f"Error while generating {'pro ' if pro else ''}external plugins \"{plugin['name']}\": {e}")
if SCHEDULER and SCHEDULER.apis:
@ -166,14 +199,6 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U
logger.error(f"Sending {'pro ' if pro else ''}external plugins failed, configuration will not work as expected...")
def dict_to_frozenset(d):
if isinstance(d, list):
return tuple(sorted(d))
elif isinstance(d, dict):
return frozenset((k, dict_to_frozenset(v)) for k, v in d.items())
return d
def api_to_instance(api):
hostname_port = api.endpoint.replace("http://", "").replace("https://", "").replace("/", "").split(":")
return {
@ -201,13 +226,11 @@ if __name__ == "__main__":
args = parser.parse_args()
INTEGRATION = get_integration()
tmp_variables_path = normpath(args.variables) if args.variables else join(sep, "var", "tmp", "bunkerweb", "variables.env")
tmp_variables_path = Path(tmp_variables_path)
tmp_variables_path = Path(args.variables or join(sep, "var", "tmp", "bunkerweb", "variables.env"))
nginx_variables_path = Path(sep, "etc", "nginx", "variables.env")
dotenv_env = dotenv_values(str(tmp_variables_path))
db = Database(logger, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None)))
env = {}
if INTEGRATION in ("Swarm", "Kubernetes", "Autoconf"):
while not db.is_initialized():
@ -218,9 +241,9 @@ if __name__ == "__main__":
logger.warning("Autoconf is not loaded yet in the database, retrying in 5s ...")
sleep(5)
env = db.get_config()
elif (
not tmp_variables_path.exists()
if (
INTEGRATION in ("Swarm", "Kubernetes", "Autoconf")
or not tmp_variables_path.exists()
or not nginx_variables_path.exists()
or (tmp_variables_path.read_text(encoding="utf-8") != nginx_variables_path.read_text(encoding="utf-8"))
or db.is_initialized()
@ -242,17 +265,18 @@ if __name__ == "__main__":
if proc.returncode != 0:
logger.error("Config saver failed, configuration will not work as expected...")
while not db.is_initialized():
logger.warning("Database is not initialized, retrying in 5s ...")
sleep(5)
if INTEGRATION not in ("Swarm", "Kubernetes", "Autoconf"):
while not db.is_initialized():
logger.warning("Database is not initialized, retrying in 5s ...")
sleep(5)
env = db.get_config()
while not db.is_first_config_saved() or not env:
logger.warning("Database doesn't have any config saved yet, retrying in 5s ...")
sleep(5)
env = db.get_config()
else:
env = db.get_config()
while not db.is_first_config_saved() or not env:
logger.warning("Database doesn't have any config saved yet, retrying in 5s ...")
sleep(5)
env = db.get_config()
env = db.get_config()
env["DATABASE_URI"] = db.database_uri
@ -277,34 +301,27 @@ if __name__ == "__main__":
logger.info("Checking if there are any changes in custom configs ...")
custom_configs = []
db_configs = db.get_custom_configs()
configs_path = Path(sep, "etc", "bunkerweb", "configs")
root_dirs = listdir(str(configs_path))
changes = False
for root, dirs, files in walk(str(configs_path)):
if files or (dirs and basename(root) not in root_dirs):
path_exploded = root.split("/")
for file in files:
content = Path(join(root, file)).read_text(encoding="utf-8")
custom_conf = {
"value": content,
"exploded": (path_exploded.pop() if path_exploded[-1] not in root_dirs else None, path_exploded[-1], file.replace(".conf", "")),
}
for file in CUSTOM_CONFIGS_PATH.rglob("*.conf"):
if len(file.parts) > len(CUSTOM_CONFIGS_PATH.parts) + 3:
logger.warning(f"Custom config file {file} is not in the correct path, skipping ...")
saving = True
in_db = False
for db_conf in db_configs:
if db_conf["service_id"] == custom_conf["exploded"][0] and db_conf["name"] == custom_conf["exploded"][2]:
in_db = True
if db_conf["method"] != "manual":
saving = False
break
content = file.read_text(encoding="utf-8")
service_id = file.parent.name if file.parent.name not in CUSTOM_CONFIGS_DIRS else None
config_type = file.parent.parent.name if service_id else file.parent.name
if not in_db and content.startswith("# CREATED BY ENV"):
saving = False
changes = True
saving = True
in_db = False
for db_conf in db_configs:
if db_conf["service_id"] == service_id and db_conf["name"] == file.stem:
in_db = True
if saving:
custom_configs.append(custom_conf)
if not in_db and content.startswith("# CREATED BY ENV"):
saving = False
changes = True
if saving:
custom_configs.append({"value": content, "exploded": (service_id, config_type, file.stem)})
changes = changes or {hash(dict_to_frozenset(d)) for d in custom_configs} != {hash(dict_to_frozenset(d)) for d in db_configs}
@ -314,43 +331,45 @@ if __name__ == "__main__":
logger.error(f"Couldn't save some manually created custom configs to database: {err}")
if (scheduler_first_start and db_configs) or changes:
Thread(target=generate_custom_configs, args=(db.get_custom_configs(),), kwargs={"original_path": configs_path}).start()
generate_custom_configs(db.get_custom_configs())
del custom_configs, db_configs
def check_plugin_changes(_type: Literal["external", "pro"] = "external"):
# Check if any external or pro plugin has been added by the user
logger.info(f"Checking if there are any changes in {_type} plugins ...")
plugin_path = EXTERNAL_PLUGINS_PATH if _type == "external" else PRO_PLUGINS_PATH
external_plugins = []
db_plugins = db.get_plugins(_type=_type)
for filename in glob(str((EXTERNAL_PLUGINS_PATH if _type == "external" else PRO_PLUGINS_PATH).joinpath("*", "plugin.json"))):
with open(filename, "r", encoding="utf-8") as f:
_dir = dirname(filename)
plugin_content = BytesIO()
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(_dir, arcname=basename(_dir), recursive=True)
plugin_content.seek(0, 0)
value = plugin_content.getvalue()
external_plugins.append(
json_load(f)
| {
"type": _type,
"page": Path(_dir, "ui").exists(),
"method": "manual",
"data": value,
"checksum": sha256(value).hexdigest(),
}
)
tmp_external_plugins = []
for external_plugin in deepcopy(external_plugins):
external_plugin.pop("data", None)
external_plugin.pop("checksum", None)
external_plugin.pop("jobs", None)
external_plugin.pop("method", None)
tmp_external_plugins.append(external_plugin)
for file in plugin_path.glob("*/plugin.json"):
plugin_content = BytesIO()
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(file.parent, arcname=file.parent.name, recursive=True)
plugin_content.seek(0, 0)
with file.open("r", encoding="utf-8") as f:
plugin_data = json_load(f)
common_data = plugin_data | {
"type": _type,
"page": file.parent.joinpath("ui").is_dir(),
}
jobs = common_data.pop("jobs", [])
tmp_external_plugins.append(common_data)
checksum = bytes_hash(plugin_content, algorithm="sha256")
external_plugins.append(
common_data
| {
"method": "manual",
"data": plugin_content.getvalue(),
"checksum": checksum,
}
| ({"jobs": jobs} if jobs else {})
)
db_plugins = db.get_plugins(_type=_type)
tmp_db_plugins = []
for db_plugin in db_plugins.copy():
db_plugin.pop("method", None)
@ -364,10 +383,7 @@ if __name__ == "__main__":
logger.error(f"Couldn't save some manually added {_type} plugins to database: {err}")
if (scheduler_first_start and db_plugins) or changes:
generate_external_plugins(
db.get_plugins(_type=_type, with_data=True),
original_path=EXTERNAL_PLUGINS_PATH if _type == "external" else PRO_PLUGINS_PATH,
)
generate_external_plugins(db.get_plugins(_type=_type, with_data=True), original_path=plugin_path)
check_plugin_changes("external")
check_plugin_changes("pro")
@ -625,7 +641,7 @@ if __name__ == "__main__":
if CONFIGS_NEED_GENERATION:
CHANGES.append("custom_configs")
generate_custom_configs(db.get_custom_configs(), original_path=configs_path)
generate_custom_configs(db.get_custom_configs())
if PLUGINS_NEED_GENERATION:
CHANGES.append("external_plugins")