mirror of
https://github.com/bunkerity/bunkerweb
synced 2026-05-24 09:28:37 +00:00
Merge pull request #366 from TheophileDiot/dev
Add a few fixes for the 1.5
This commit is contained in:
Théophile Diot
GitHub
commit
2889b2638f
10 changed files with 162 additions and 137 deletions
|
|
@ -9,4 +9,5 @@ CHANGELOG.md
|
|||
CONTRIBUTING.md
|
||||
LICENSE.md
|
||||
README.md
|
||||
SECURITY.md
|
||||
SECURITY.md
|
||||
ui
|
||||
|
|
@ -111,6 +111,6 @@ try:
|
|||
|
||||
except:
|
||||
status = 2
|
||||
logger.error(f"Exception while running certbot-new.py :\n{format_exc()}")
|
||||
logger.error(f"Exception while running self-signed.py :\n{format_exc()}")
|
||||
|
||||
sys_exit(status)
|
||||
|
|
|
|||
|
|
@ -330,18 +330,17 @@ class Database:
|
|||
global_values = []
|
||||
db_services = (
|
||||
session.query(Services)
|
||||
.with_entities(Services.id)
|
||||
.filter_by(method=method)
|
||||
.with_entities(Services.id, Services.method)
|
||||
.all()
|
||||
)
|
||||
db_ids = [service.id for service in db_services]
|
||||
services = config.get("SERVER_NAME", "").split(" ")
|
||||
|
||||
if db_services:
|
||||
db_services = [service.id for service in db_services]
|
||||
missing_ids = [
|
||||
service
|
||||
service.id
|
||||
for service in db_services
|
||||
if service not in services
|
||||
if (service.method == method) and service.id not in services
|
||||
]
|
||||
|
||||
if missing_ids:
|
||||
|
|
@ -373,9 +372,9 @@ class Database:
|
|||
except StopIteration:
|
||||
continue
|
||||
|
||||
if server_name not in db_services:
|
||||
if server_name not in db_ids:
|
||||
to_put.append(Services(id=server_name, method=method))
|
||||
db_services.append(server_name)
|
||||
db_ids.append(server_name)
|
||||
|
||||
key = key.replace(f"{server_name}_", "")
|
||||
setting = (
|
||||
|
|
@ -471,7 +470,16 @@ class Database:
|
|||
}
|
||||
)
|
||||
else:
|
||||
if "SERVER_NAME" in config and config["SERVER_NAME"] != "":
|
||||
if (
|
||||
"SERVER_NAME" in config
|
||||
and config["SERVER_NAME"] != ""
|
||||
and not (
|
||||
session.query(Services)
|
||||
.with_entities(Services.id)
|
||||
.filter_by(id=config["SERVER_NAME"].split(" ")[0])
|
||||
.first()
|
||||
)
|
||||
):
|
||||
to_put.append(
|
||||
Services(
|
||||
id=config["SERVER_NAME"].split(" ")[0], method=method
|
||||
|
|
@ -640,96 +648,80 @@ class Database:
|
|||
)
|
||||
.all()
|
||||
):
|
||||
suffix = 0
|
||||
while True:
|
||||
global_value = (
|
||||
session.query(Global_values)
|
||||
.with_entities(Global_values.value, Global_values.method)
|
||||
.filter_by(setting_id=setting.id, suffix=suffix)
|
||||
.first()
|
||||
default = setting.default or ""
|
||||
config[setting.id] = (
|
||||
default
|
||||
if methods is False
|
||||
else {"value": default, "method": "default"}
|
||||
)
|
||||
|
||||
global_values = (
|
||||
session.query(Global_values)
|
||||
.with_entities(
|
||||
Global_values.value, Global_values.suffix, Global_values.method
|
||||
)
|
||||
.filter_by(setting_id=setting.id)
|
||||
.all()
|
||||
)
|
||||
|
||||
for global_value in global_values:
|
||||
config[
|
||||
setting.id
|
||||
+ (
|
||||
f"_{global_value.suffix}"
|
||||
if setting.multiple and global_value.suffix > 0
|
||||
else ""
|
||||
)
|
||||
] = (
|
||||
global_value.value
|
||||
if methods is False
|
||||
else {
|
||||
"value": global_value.value,
|
||||
"method": global_value.method,
|
||||
}
|
||||
)
|
||||
|
||||
if global_value is None:
|
||||
if suffix == 0:
|
||||
default = setting.default or ""
|
||||
config[setting.id] = (
|
||||
default
|
||||
if methods is False
|
||||
else {"value": default, "method": "default"}
|
||||
if setting.context != "multisite":
|
||||
continue
|
||||
|
||||
for service in db_services:
|
||||
config[f"{service.id}_{setting.id}"] = (
|
||||
config[setting.id]
|
||||
if methods is False
|
||||
else {
|
||||
"value": config[setting.id]["value"],
|
||||
"method": "default",
|
||||
}
|
||||
)
|
||||
|
||||
service_settings = (
|
||||
session.query(Services_settings)
|
||||
.with_entities(
|
||||
Services_settings.value,
|
||||
Services_settings.suffix,
|
||||
Services_settings.method,
|
||||
)
|
||||
.filter_by(service_id=service.id, setting_id=setting.id)
|
||||
.all()
|
||||
)
|
||||
|
||||
for service_setting in service_settings:
|
||||
config[
|
||||
f"{service.id}_{setting.id}"
|
||||
+ (
|
||||
f"_{service_setting.suffix}"
|
||||
if service_setting.suffix > 0
|
||||
else ""
|
||||
)
|
||||
elif setting.context != "multisite":
|
||||
break
|
||||
else:
|
||||
config[setting.id + (f"_{suffix}" if suffix > 0 else "")] = (
|
||||
global_value.value
|
||||
] = (
|
||||
service_setting.value
|
||||
if methods is False
|
||||
else {
|
||||
"value": global_value.value,
|
||||
"method": global_value.method,
|
||||
"value": service_setting.value,
|
||||
"method": service_setting.method,
|
||||
}
|
||||
)
|
||||
|
||||
if setting.context == "multisite":
|
||||
changed = False
|
||||
for service in db_services:
|
||||
if suffix == 0:
|
||||
config[f"{service.id}_{setting.id}"] = (
|
||||
config[setting.id]
|
||||
if methods is False
|
||||
else {
|
||||
"value": config[setting.id]["value"],
|
||||
"method": "default",
|
||||
}
|
||||
)
|
||||
changed = True
|
||||
elif f"{setting.id}_{suffix}" in config:
|
||||
config[f"{service.id}_{setting.id}_{suffix}"] = (
|
||||
config[f"{setting.id}_{suffix}"]
|
||||
if methods is False
|
||||
else {
|
||||
"value": config[f"{setting.id}_{suffix}"][
|
||||
"value"
|
||||
],
|
||||
"method": "default",
|
||||
}
|
||||
)
|
||||
changed = True
|
||||
|
||||
service_setting = (
|
||||
session.query(Services_settings)
|
||||
.with_entities(
|
||||
Services_settings.value, Services_settings.method
|
||||
)
|
||||
.filter_by(
|
||||
service_id=service.id,
|
||||
setting_id=setting.id,
|
||||
suffix=suffix,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if service_setting is not None:
|
||||
config[
|
||||
f"{service.id}_{setting.id}"
|
||||
+ (f"_{suffix}" if suffix > 0 else "")
|
||||
] = (
|
||||
service_setting.value
|
||||
if methods is False
|
||||
else {
|
||||
"value": service_setting.value,
|
||||
"method": service_setting.method,
|
||||
}
|
||||
)
|
||||
changed = True
|
||||
|
||||
if global_value is None and changed is False:
|
||||
break
|
||||
|
||||
if not setting.multiple:
|
||||
break
|
||||
|
||||
suffix += 1
|
||||
|
||||
return config
|
||||
|
||||
def get_custom_configs(self) -> List[Dict[str, Any]]:
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ from argparse import ArgumentParser
|
|||
from glob import glob
|
||||
from itertools import chain
|
||||
from json import loads
|
||||
from os import R_OK, W_OK, X_OK, access, environ, getenv, path
|
||||
from os.path import exists
|
||||
from os import R_OK, X_OK, access, environ, getenv, listdir, path, walk
|
||||
from os.path import exists, join
|
||||
from re import compile as re_compile
|
||||
from sys import exit as sys_exit, path as sys_path
|
||||
from traceback import format_exc
|
||||
|
|
@ -108,6 +108,12 @@ if __name__ == "__main__":
|
|||
action="store_true",
|
||||
help="Only initialize the database",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--method",
|
||||
default="scheduler",
|
||||
type=str,
|
||||
help="The method that is used to save the config",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logger.info("Save config started ...")
|
||||
|
|
@ -127,9 +133,12 @@ if __name__ == "__main__":
|
|||
with open("/usr/share/bunkerweb/INTEGRATION", "r") as f:
|
||||
integration = f.read().strip()
|
||||
|
||||
logger.info(f"Detected {integration} integration")
|
||||
if args.init:
|
||||
logger.info(f"Detected {integration} integration")
|
||||
|
||||
config_files = None
|
||||
db = None
|
||||
apis = []
|
||||
|
||||
# Check existences and permissions
|
||||
logger.info("Checking arguments ...")
|
||||
|
|
@ -194,10 +203,31 @@ if __name__ == "__main__":
|
|||
for k, v in environ.items()
|
||||
if custom_confs_rx.match(k)
|
||||
]
|
||||
root_dirs = listdir("/etc/bunkerweb/configs")
|
||||
for (root, dirs, files) in walk("/etc/bunkerweb/configs", topdown=True):
|
||||
if (
|
||||
root != "configs"
|
||||
and (dirs and not root.split("/")[-1] in root_dirs)
|
||||
or files
|
||||
):
|
||||
path_exploded = root.split("/")
|
||||
for file in files:
|
||||
with open(join(root, file), "r") as f:
|
||||
custom_confs.append(
|
||||
{
|
||||
"value": f.read(),
|
||||
"exploded": (
|
||||
f"{path_exploded.pop()}"
|
||||
if path_exploded[-1] not in root_dirs
|
||||
else "",
|
||||
path_exploded[-1],
|
||||
file.replace(".conf", ""),
|
||||
),
|
||||
}
|
||||
)
|
||||
elif integration == "Kubernetes":
|
||||
corev1 = kube_client.CoreV1Api()
|
||||
tmp_config = {}
|
||||
apis = []
|
||||
|
||||
for pod in corev1.list_pod_for_all_namespaces(watch=False).items:
|
||||
if (
|
||||
|
|
@ -280,7 +310,6 @@ if __name__ == "__main__":
|
|||
|
||||
tmp_config = {}
|
||||
custom_confs = []
|
||||
apis = []
|
||||
|
||||
for instance in (
|
||||
docker_client.containers.list(filters={"label": "bunkerweb.INSTANCE"})
|
||||
|
|
@ -354,10 +383,10 @@ if __name__ == "__main__":
|
|||
if args.init:
|
||||
sys_exit(0)
|
||||
|
||||
err = db.save_config(config_files, "scheduler")
|
||||
err = db.save_config(config_files, args.method)
|
||||
|
||||
if not err:
|
||||
err1 = db.save_custom_configs(custom_confs, "scheduler")
|
||||
err1 = db.save_custom_configs(custom_confs, args.method)
|
||||
else:
|
||||
err = None
|
||||
err1 = None
|
||||
|
|
|
|||
|
|
@ -68,6 +68,9 @@ getLogger("sqlalchemy.orm.strategies.LazyLoader").setLevel(
|
|||
getLogger("sqlalchemy.pool.impl.QueuePool").setLevel(
|
||||
default_level if default_level != INFO else WARNING
|
||||
)
|
||||
getLogger("sqlalchemy.pool.impl.NullPool").setLevel(
|
||||
default_level if default_level != INFO else WARNING
|
||||
)
|
||||
getLogger("sqlalchemy.engine.Engine").setLevel(
|
||||
default_level if default_level != INFO else WARNING
|
||||
)
|
||||
|
|
|
|||
|
|
@ -197,10 +197,14 @@ class JobScheduler(ApiCaller):
|
|||
path = job["path"]
|
||||
name = job["name"]
|
||||
file = job["file"]
|
||||
thread = Thread(
|
||||
target=self.__job_wrapper, args=(path, plugin, name, file)
|
||||
)
|
||||
threads.append(thread)
|
||||
|
||||
if job["name"].startswith("bunkernet"):
|
||||
self.__job_wrapper(path, plugin, name, file)
|
||||
else:
|
||||
thread = Thread(
|
||||
target=self.__job_wrapper, args=(path, plugin, name, file)
|
||||
)
|
||||
threads.append(thread)
|
||||
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
|
|
|
|||
|
|
@ -27,10 +27,6 @@ if ! grep -q "Docker" /usr/share/bunkerweb/INTEGRATION ; then
|
|||
fi
|
||||
fi
|
||||
|
||||
if [ -f /var/lib/bunkerweb/db.sqlite3 ] ; then
|
||||
chown scheduler:scheduler /var/lib/bunkerweb/db.sqlite3
|
||||
fi
|
||||
|
||||
# execute jobs
|
||||
log "ENTRYPOINT" "ℹ️ " "Executing scheduler ..."
|
||||
/usr/share/bunkerweb/scheduler/main.py
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
from argparse import ArgumentParser
|
||||
from copy import deepcopy
|
||||
from glob import glob
|
||||
from json import load
|
||||
from os import (
|
||||
_exit,
|
||||
chmod,
|
||||
|
|
@ -17,10 +16,9 @@ from os import (
|
|||
walk,
|
||||
)
|
||||
from os.path import dirname, exists, isdir, isfile, islink, join
|
||||
from re import compile as re_compile
|
||||
from shutil import chown, copy, rmtree
|
||||
from signal import SIGINT, SIGTERM, signal
|
||||
from subprocess import PIPE, run as subprocess_run, DEVNULL, STDOUT
|
||||
from subprocess import run as subprocess_run, DEVNULL, STDOUT
|
||||
from sys import path as sys_path
|
||||
from time import sleep
|
||||
from traceback import format_exc
|
||||
|
|
@ -290,6 +288,9 @@ if __name__ == "__main__":
|
|||
"Looks like BunkerWeb configuration is already generated, will not generate it again ..."
|
||||
)
|
||||
|
||||
if exists("/var/lib/bunkerweb/db.sqlite3"):
|
||||
chmod("/var/lib/bunkerweb/db.sqlite3", 0o760)
|
||||
|
||||
while True:
|
||||
# Instantiate scheduler
|
||||
scheduler = JobScheduler(
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev
|
|||
# Copy files
|
||||
# can't exclude specific files/dir from . so we are copying everything by hand
|
||||
COPY src/common/api /usr/share/bunkerweb/api
|
||||
COPY src/common/confs /usr/share/bunkerweb/confs
|
||||
COPY src/common/db /usr/share/bunkerweb/db
|
||||
COPY src/common/core /usr/share/bunkerweb/core
|
||||
COPY src/common/gen /usr/share/bunkerweb/gen
|
||||
|
|
@ -46,7 +45,7 @@ RUN apk add --no-cache bash file && \
|
|||
for dir in $(echo "/usr/share/bunkerweb /etc/bunkerweb") ; do find ${dir} -type f -exec chmod 0740 {} \; ; done && \
|
||||
for dir in $(echo "/usr/share/bunkerweb /etc/bunkerweb") ; do find ${dir} -type d -exec chmod 0750 {} \; ; done && \
|
||||
chmod 770 /var/cache/bunkerweb /var/lib/bunkerweb /var/tmp/bunkerweb && \
|
||||
chmod 750 /usr/share/bunkerweb/gen/main.py /usr/share/bunkerweb/deps/python/bin/* && \
|
||||
chmod 750 /usr/share/bunkerweb/gen/*.py /usr/share/bunkerweb/deps/python/bin/* && \
|
||||
chmod 660 /usr/share/bunkerweb/INTEGRATION
|
||||
|
||||
# Fix CVEs
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from copy import deepcopy
|
||||
from os import listdir
|
||||
from os import listdir, remove
|
||||
from time import sleep
|
||||
from flask import flash
|
||||
from os.path import exists, isfile
|
||||
|
|
@ -147,13 +147,8 @@ class Config:
|
|||
self.__dict_to_env(env_file, conf)
|
||||
proc = run(
|
||||
[
|
||||
"/usr/share/bunkerweb/gen/main.py",
|
||||
"--settings",
|
||||
"/usr/share/bunkerweb/settings.json",
|
||||
"--templates",
|
||||
"/usr/share/bunkerweb/confs",
|
||||
"--output",
|
||||
"/etc/nginx",
|
||||
"python3",
|
||||
"/usr/share/bunkerweb/gen/save_config.py",
|
||||
"--variables",
|
||||
env_file,
|
||||
"--method",
|
||||
|
|
@ -166,11 +161,7 @@ class Config:
|
|||
if proc.returncode != 0:
|
||||
raise Exception(f"Error from generator (return code = {proc.returncode})")
|
||||
|
||||
err = self.__db.save_config(conf, "ui")
|
||||
if err:
|
||||
self.__logger.error(
|
||||
f"Can't save config in database: {err}",
|
||||
)
|
||||
remove(env_file)
|
||||
|
||||
def get_plugins_settings(self) -> dict:
|
||||
return self.__plugins_settings
|
||||
|
|
@ -181,7 +172,7 @@ class Config:
|
|||
def get_settings(self) -> dict:
|
||||
return self.__settings
|
||||
|
||||
def get_config(self) -> dict:
|
||||
def get_config(self, methods: bool = True) -> dict:
|
||||
"""Get the nginx variables env file and returns it as a dict
|
||||
|
||||
Returns
|
||||
|
|
@ -191,13 +182,13 @@ class Config:
|
|||
"""
|
||||
if exists("/usr/sbin/nginx"):
|
||||
return {
|
||||
k: {"value": v, "method": "ui"}
|
||||
k: ({"value": v, "method": "ui"} if methods is True else v)
|
||||
for k, v in self.__env_to_dict("/etc/nginx/variables.env").items()
|
||||
}
|
||||
|
||||
return self.__db.get_config(methods=True)
|
||||
return self.__db.get_config(methods=methods)
|
||||
|
||||
def get_services(self) -> list[dict]:
|
||||
def get_services(self, methods: bool = True) -> list[dict]:
|
||||
"""Get nginx's services
|
||||
|
||||
Returns
|
||||
|
|
@ -208,15 +199,20 @@ class Config:
|
|||
if exists("/usr/sbin/nginx"):
|
||||
services = []
|
||||
for filename in iglob("/etc/nginx/**/variables.env"):
|
||||
service = filename.split("/")[3]
|
||||
env = {
|
||||
k: {"value": v, "method": "ui"}
|
||||
k.replace(f"{service}_", ""): (
|
||||
{"value": v, "method": "ui"} if methods is True else v
|
||||
)
|
||||
for k, v in self.__env_to_dict(filename).items()
|
||||
if k.startswith(f"{service}_")
|
||||
or k in self.__plugins_settings.keys()
|
||||
}
|
||||
services.append(env)
|
||||
|
||||
return services
|
||||
|
||||
return self.__db.get_services_settings(methods=True)
|
||||
return self.__db.get_services_settings(methods=methods)
|
||||
|
||||
def check_variables(self, variables: dict, _global: bool = False) -> int:
|
||||
"""Testify that the variables passed are valid
|
||||
|
|
@ -265,7 +261,9 @@ class Config:
|
|||
return error
|
||||
|
||||
def reload_config(self) -> None:
|
||||
self.__gen_conf(self.get_config(), self.get_services())
|
||||
self.__gen_conf(
|
||||
self.get_config(methods=False), self.get_services(methods=False)
|
||||
)
|
||||
|
||||
def new_service(self, variables: dict) -> Tuple[str, int]:
|
||||
"""Creates a new service from the given variables
|
||||
|
|
@ -285,7 +283,7 @@ class Config:
|
|||
Exception
|
||||
raise this if the service already exists
|
||||
"""
|
||||
services = self.get_services()
|
||||
services = self.get_services(methods=False)
|
||||
for service in services:
|
||||
if service["SERVER_NAME"] == variables["SERVER_NAME"] or service[
|
||||
"SERVER_NAME"
|
||||
|
|
@ -293,7 +291,7 @@ class Config:
|
|||
return f"Service {service['SERVER_NAME']} already exists.", 1
|
||||
|
||||
services.append(variables)
|
||||
self.__gen_conf(self.get_config(), services)
|
||||
self.__gen_conf(self.get_config(methods=False), services)
|
||||
return f"Configuration for {variables['SERVER_NAME']} has been generated.", 0
|
||||
|
||||
def edit_service(self, old_server_name: str, variables: dict) -> Tuple[str, int]:
|
||||
|
|
@ -332,7 +330,9 @@ class Config:
|
|||
str
|
||||
the confirmation message
|
||||
"""
|
||||
self.__gen_conf(self.get_config() | variables, self.get_services())
|
||||
self.__gen_conf(
|
||||
self.get_config(methods=False) | variables, self.get_services(methods=False)
|
||||
)
|
||||
return f"The global configuration has been edited."
|
||||
|
||||
def delete_service(self, service_name: str) -> Tuple[str, int]:
|
||||
|
|
@ -353,8 +353,8 @@ class Config:
|
|||
Exception
|
||||
raises this if the service_name given isn't found
|
||||
"""
|
||||
full_env = self.get_config()
|
||||
services = self.get_services()
|
||||
full_env = self.get_config(methods=False)
|
||||
services = self.get_services(methods=False)
|
||||
new_services = []
|
||||
found = False
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue