Merge pull request #660 from bunkerity/dev

Merge branch "dev" into branch "staging"
This commit is contained in:
Théophile Diot 2023-09-28 15:05:21 +01:00 committed by GitHub
commit cd4d529d7e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 182 additions and 93 deletions

View file

@ -58,35 +58,25 @@ class Config(ConfigCaller):
def apply(self, instances, services, configs={}, first=False) -> bool:
success = True
# update types
updates = {
"instances": False,
"services": False,
"configs": False,
"config": False,
}
changes = []
if instances != self.__instances or first:
self.__instances = instances
updates["instances"] = True
changes.append("instances")
if services != self.__services or first:
self.__services = services
updates["services"] = True
changes.append("services")
if configs != self.__configs or first:
self.__configs = configs
updates["configs"] = True
changes.append("custom_configs")
if updates["instances"] or updates["services"]:
if "instances" in changes or "services" in changes:
old_env = deepcopy(self.__config)
new_env = self.__get_full_env()
if old_env != new_env or first:
self.__config = new_env
updates["config"] = True
changes.append("config")
custom_configs = []
if updates["configs"]:
if "custom_configs" in changes:
for config_type in self.__configs:
for file, data in self.__configs[config_type].items():
site = None
@ -126,21 +116,24 @@ class Config(ConfigCaller):
"Scheduler is already applying a configuration, retrying in 5 seconds ...",
)
sleep(5)
# update instances in database
if updates["instances"]:
if "instances" in changes:
err = self._db.update_instances(self.__instances, changed=False)
if err:
self.__logger.error(f"Failed to update instances: {err}")
# save config to database
if updates["config"]:
if "config" in changes:
err = self._db.save_config(self.__config, "autoconf", changed=False)
if err:
success = False
self.__logger.error(
f"Can't save config in database: {err}, config may not work as expected",
)
# save custom configs to database
if updates["configs"]:
if "custom_configs" in changes:
err = self._db.save_custom_configs(
custom_configs, "autoconf", changed=False
)
@ -149,6 +142,7 @@ class Config(ConfigCaller):
self.__logger.error(
f"Can't save autoconf custom configs in database: {err}, custom configs may not work as expected",
)
# update changes in db
ret = self._db.checked_changes(changes, value=True)
if ret:

View file

@ -352,6 +352,8 @@ class Database:
return "The metadata are not set yet, try again"
if "config" in changes:
if not metadata.first_config_saved:
metadata.first_config_saved = True
metadata.config_changed = value
if "custom_configs" in changes:
metadata.custom_configs_changed = value
@ -1733,7 +1735,9 @@ class Database:
)
]
def add_instance(self, hostname: str, port: int, server_name: str) -> str:
def add_instance(
self, hostname: str, port: int, server_name: str, changed: Optional[bool] = True
) -> str:
"""Add instance."""
with self.__db_session() as session:
db_instance = (
@ -1750,6 +1754,12 @@ class Database:
Instances(hostname=hostname, port=port, server_name=server_name)
)
if changed:
with suppress(ProgrammingError, OperationalError):
metadata = session.query(Metadata).get(1)
if metadata is not None:
metadata.instances_changed = True
try:
session.commit()
except BaseException:

View file

@ -340,43 +340,68 @@ if __name__ == "__main__":
if args.init:
sys_exit(0)
err = db.save_config(config_files, args.method)
changes = []
err = db.save_config(config_files, args.method, changed=False)
if not err:
err1 = db.save_custom_configs(custom_confs, args.method)
else:
err = None
err1 = None
if err or err1:
logger.error(
f"Can't save config to database : {err or err1}",
if err:
logger.warning(
f"Couldn't save config to database : {err}, config may not work as expected"
)
sys_exit(1)
else:
changes.append("config")
logger.info("Config successfully saved to database")
if args.method != "ui":
err1 = db.save_custom_configs(custom_confs, args.method, changed=False)
if err1:
logger.warning(
f"Couldn't save custom configs to database : {err1}, custom configs may not work as expected"
)
else:
changes.append("custom_configs")
logger.info("Custom configs successfully saved to database")
if apis:
for api in apis:
endpoint_data = api.endpoint.replace("http://", "").split(":")
err = db.add_instance(
endpoint_data[0], endpoint_data[1].replace("/", ""), api.host
endpoint_data[0],
endpoint_data[1].replace("/", ""),
api.host,
changed=False,
)
if err:
logger.warning(err)
else:
if "instances" not in changes:
changes.append("instances")
logger.info(
f"Instance {endpoint_data[0]} successfully saved to database"
)
else:
err = db.add_instance(
"127.0.0.1",
config_files.get("API_HTTP_PORT", 5000),
config_files.get("API_SERVER_NAME", "bwapi"),
changed=False,
)
if err:
logger.warning(err)
else:
changes.append("instances")
logger.info("Instance 127.0.0.1 successfully saved to database")
# update changes in db
ret = db.checked_changes(changes, value=True)
if ret:
logger.error(
f"An error occurred when setting the changes to checked in the database : {ret}"
)
except SystemExit as e:
raise e
sys_exit(e.code)
except:
logger.error(
f"Exception while executing config saver : {format_exc()}",

View file

@ -10,4 +10,4 @@
--before-install /usr/share/bunkerweb/scripts/beforeInstall.sh
--after-install /usr/share/bunkerweb/scripts/postinstall.sh
--after-remove /usr/share/bunkerweb/scripts/afterRemoveRPM.sh
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb=/var/lib/bunkerweb
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb/=/var/lib/bunkerweb

View file

@ -10,4 +10,4 @@
--before-install /usr/share/bunkerweb/scripts/beforeInstall.sh
--after-install /usr/share/bunkerweb/scripts/postinstall.sh
--after-remove /usr/share/bunkerweb/scripts/afterRemoveDEB.sh
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb=/var/lib/bunkerweb
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb/=/var/lib/bunkerweb

View file

@ -10,4 +10,4 @@
--before-install /usr/share/bunkerweb/scripts/beforeInstall.sh
--after-install /usr/share/bunkerweb/scripts/postinstall.sh
--after-remove /usr/share/bunkerweb/scripts/afterRemoveRPM.sh
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb=/var/lib/bunkerweb
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb/=/var/lib/bunkerweb

View file

@ -10,4 +10,4 @@
--before-install /usr/share/bunkerweb/scripts/beforeInstall.sh
--after-install /usr/share/bunkerweb/scripts/postinstall.sh
--after-remove /usr/share/bunkerweb/scripts/afterRemoveRPM.sh
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb=/var/lib/bunkerweb
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb/=/var/lib/bunkerweb

View file

@ -11,4 +11,4 @@
--after-install /usr/share/bunkerweb/scripts/postinstall.sh
--after-remove /usr/share/bunkerweb/scripts/afterRemoveDEB.sh
--deb-no-default-config-files
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb=/var/lib/bunkerweb
/usr/share/bunkerweb/=/usr/share/bunkerweb/ /usr/bin/bwcli=/usr/bin/bwcli /etc/bunkerweb/=/etc/bunkerweb /var/tmp/bunkerweb/=/var/tmp/bunkerweb /var/run/bunkerweb/=/var/run/bunkerweb /var/log/bunkerweb/=/var/log/bunkerweb /var/cache/bunkerweb/=/var/cache/bunkerweb /lib/systemd/system/bunkerweb.service=/lib/systemd/system/bunkerweb.service /lib/systemd/system/bunkerweb-ui.service=/lib/systemd/system/bunkerweb-ui.service /var/lib/bunkerweb/=/var/lib/bunkerweb

View file

@ -470,6 +470,7 @@ if __name__ == "__main__":
FIRST_RUN = True
CONFIG_NEED_GENERATION = True
RUN_JOBS_ONCE = True
CHANGES = []
threads = []
@ -498,15 +499,16 @@ if __name__ == "__main__":
)
stop(1)
# Update the environment variables of the scheduler
SCHEDULER.env = env.copy() | environ.copy()
SCHEDULER.setup()
if RUN_JOBS_ONCE:
# Update the environment variables of the scheduler
SCHEDULER.env = env.copy() | environ.copy()
SCHEDULER.setup()
# Only run jobs once
if not SCHEDULER.run_once():
logger.error("At least one job in run_once() failed")
else:
logger.info("All jobs in run_once() were successful")
# Only run jobs once
if not SCHEDULER.run_once():
logger.error("At least one job in run_once() failed")
else:
logger.info("All jobs in run_once() were successful")
if CONFIG_NEED_GENERATION:
content = ""
@ -613,6 +615,7 @@ if __name__ == "__main__":
)
NEED_RELOAD = False
RUN_JOBS_ONCE = False
CONFIG_NEED_GENERATION = False
CONFIGS_NEED_GENERATION = False
PLUGINS_NEED_GENERATION = False
@ -675,6 +678,7 @@ if __name__ == "__main__":
)
PLUGINS_NEED_GENERATION = True
CONFIG_NEED_GENERATION = True
NEED_RELOAD = True
# check if the custom configs have changed since last time
@ -693,6 +697,7 @@ if __name__ == "__main__":
if changes["instances_changed"]:
logger.info("Instances changed, generating ...")
INSTANCES_NEED_GENERATION = True
CONFIG_NEED_GENERATION = True
NEED_RELOAD = True
FIRST_RUN = False
@ -718,6 +723,7 @@ if __name__ == "__main__":
CHANGES.append("config")
env = db.get_config()
env["DATABASE_URI"] = db.database_uri
RUN_JOBS_ONCE = True
if INSTANCES_NEED_GENERATION:
CHANGES.append("instances")

View file

@ -245,25 +245,27 @@ LOG_RX = re_compile(
def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
# Do the operation
error = False
if method == "services":
error = False
editing = False
editing = operation == "edit"
service_custom_confs = glob(
join(sep, "etc", "bunkerweb", "configs", "*", args[1])
)
moved = False
if operation == "new":
operation, error = app.config["CONFIG"].new_service(args[0])
elif operation == "edit":
editing = True
if args[1] != args[2] and service_custom_confs:
for service_custom_conf in service_custom_confs:
move(
service_custom_conf,
service_custom_conf.replace(
f"{sep}{args[1]}", f"{sep}{args[2]}"
).replace(join(sep, "etc"), join(sep, "var", "tmp")),
)
if listdir(service_custom_conf):
move(
service_custom_conf,
service_custom_conf.replace(
f"{sep}{args[1]}", f"{sep}{args[2]}"
).replace(join(sep, "etc"), join(sep, "var", "tmp")),
)
moved = True
operation, error = app.config["CONFIG"].edit_service(args[1], args[0])
elif operation == "delete":
operation, error = app.config["CONFIG"].delete_service(args[2])
@ -273,7 +275,7 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
else:
app.config["TO_FLASH"].append({"content": operation, "type": "success"})
if editing and args[1] != args[2] and service_custom_confs:
if editing and moved and args[1] != args[2] and service_custom_confs:
for tmp_service_custom_conf in glob(
join(sep, "var", "tmp", "bunkerweb", "configs", "*", args[2])
):
@ -292,7 +294,6 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
)
if method == "global_config":
operation = app.config["CONFIG"].edit_global_conf(args[0])
app.config["TO_FLASH"].append({"content": operation, "type": "success"})
elif method == "plugins":
app.config["CONFIG"].reload_config()
@ -304,18 +305,21 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
operation = app.config["INSTANCES"].stop_instance(args[0])
elif operation == "restart":
operation = app.config["INSTANCES"].restart_instance(args[0])
else:
elif not error:
operation = "The scheduler will be in charge of reloading the instances."
if isinstance(operation, list):
for op in operation:
app.config["TO_FLASH"].append(
{"content": f"Reload failed for the instance {op}", "type": "error"}
)
elif operation.startswith("Can't"):
app.config["TO_FLASH"].append({"content": operation, "type": "error"})
else:
app.config["TO_FLASH"].append({"content": operation, "type": "success"})
operation = ""
if operation:
if isinstance(operation, list):
for op in operation:
app.config["TO_FLASH"].append(
{"content": f"Reload failed for the instance {op}", "type": "error"}
)
elif operation.startswith("Can't"):
app.config["TO_FLASH"].append({"content": operation, "type": "error"})
else:
app.config["TO_FLASH"].append({"content": operation, "type": "success"})
app.config["RELOADING"] = False
@ -530,6 +534,18 @@ def services():
):
del variables[variable]
if (
request.form["operation"] == "edit"
and len(variables) == 1
and "SERVER_NAME" in variables
and variables["SERVER_NAME"] == request.form.get("OLD_SERVER_NAME", "")
):
flash(
"The service was not edited because no values were changed.",
"error",
)
return redirect(url_for("loading", next=url_for("services")))
error = app.config["CONFIG"].check_variables(variables)
if error:

View file

@ -76,11 +76,10 @@ class Config:
check=False,
)
env_file.unlink()
if proc.returncode != 0:
raise Exception(f"Error from generator (return code = {proc.returncode})")
env_file.unlink()
def get_plugins_settings(self) -> dict:
return {
**{k: v for x in self.get_plugins() for k, v in x["settings"].items()},
@ -180,7 +179,7 @@ class Config:
self.get_config(methods=False), self.get_services(methods=False)
)
def new_service(self, variables: dict, edit: bool = False) -> Tuple[str, int]:
def new_service(self, variables: dict) -> Tuple[str, int]:
"""Creates a new service from the given variables
Parameters
@ -199,17 +198,16 @@ class Config:
raise this if the service already exists
"""
services = self.get_services(methods=False)
for i, service in enumerate(services):
if service["SERVER_NAME"] == variables["SERVER_NAME"] or service[
"SERVER_NAME"
] in variables["SERVER_NAME"].split(" "):
if not edit:
return (
f"Service {service['SERVER_NAME'].split(' ')[0]} already exists.",
1,
)
services.pop(i)
server_name_splitted = variables["SERVER_NAME"].split(" ")
for service in services:
if (
service["SERVER_NAME"] == variables["SERVER_NAME"]
or service["SERVER_NAME"] in server_name_splitted
):
return (
f"Service {service['SERVER_NAME'].split(' ')[0]} already exists.",
1,
)
services.append(variables)
self.__gen_conf(self.get_config(methods=False), services)
@ -233,19 +231,39 @@ class Config:
str
the confirmation message
"""
message, error = self.delete_service(old_server_name)
services = self.get_services(methods=False)
changed_server_name = old_server_name != variables["SERVER_NAME"]
server_name_splitted = variables["SERVER_NAME"].split(" ")
old_server_name_splitted = old_server_name.split(" ")
for i, service in enumerate(deepcopy(services)):
if (
service["SERVER_NAME"] == variables["SERVER_NAME"]
or service["SERVER_NAME"] in server_name_splitted
):
if changed_server_name:
return (
f"Service {service['SERVER_NAME'].split(' ')[0]} already exists.",
1,
)
services.pop(i)
elif changed_server_name and (
service["SERVER_NAME"] == old_server_name
or service["SERVER_NAME"] in old_server_name_splitted
):
services.pop(i)
if error:
return message, error
services.append(variables)
config = self.get_config(methods=False)
message, error = self.new_service(variables, edit=True)
if error:
return message, error
if changed_server_name:
for k in deepcopy(config):
if k.startswith(old_server_name):
config.pop(k)
self.__gen_conf(config, services)
return (
f"Configuration for {old_server_name.split(' ')[0]} has been edited.",
error,
0,
)
def edit_global_conf(self, variables: dict) -> str:

View file

@ -333,9 +333,10 @@ class Upload {
: (fileSize = (loaded / (1024 * 1024)).toFixed(2) + " MB");
const progressHTML = this.fileLoad(name, fileSize);
let cleanHTML = DOMPurify.sanitize(progressHTML);
this.uploadedArea.classList.add("onprogress");
this.progressArea.innerHTML = progressHTML;
this.progressArea.innerHTML = cleanHTML;
});
xhr.addEventListener("readystatechange", () => {

View file

@ -10,6 +10,29 @@ elif [ "$integration" != "docker" ] && [ "$integration" != "linux" ] ; then
exit 1
fi
echo "🌐 Building UI stack for integration \"$integration\" ..."
cleanup_stack () {
echo "🌐 Cleaning up current stack ..."
if [ "$integration" == "docker" ] ; then
docker compose down -v --remove-orphans
else
sudo systemctl stop bunkerweb
sudo truncate -s 0 /var/log/bunkerweb/error.log
fi
if [ $? -ne 0 ] ; then
echo "🌐 Cleanup failed ❌"
exit 1
fi
echo "🌐 Cleaning up current stack done ✅"
}
# Cleanup stack on exit
trap cleanup_stack EXIT
# Prepare environment
if [ "$integration" = "docker" ] ; then
sed -i "s@bunkerity/bunkerweb:.*@bunkerweb-tests@" docker-compose.yml
@ -27,7 +50,7 @@ if [ "$integration" = "docker" ] ; then
docker compose up -d
if [ $? -ne 0 ] ; then
echo "🌐 Up failed, retrying ... ⚠️"
docker compose down -v --remove-orphans
cleanup_stack
docker compose up -d
if [ $? -ne 0 ] ; then
echo "🌐 Up failed ❌"
@ -42,6 +65,7 @@ else
export TEST_TYPE="linux"
fi
echo "🌐 Waiting for stack to be healthy ..."
i=0
if [ "$integration" == "docker" ] ; then
while [ $i -lt 120 ] ; do
@ -96,9 +120,7 @@ else
echo "🌐 ⚠ Linux stack got an issue, restarting ..."
sudo journalctl --rotate
sudo journalctl --vacuum-time=1s
manual=1
cleanup_stack
manual=0
sudo systemctl start bunkerweb
retries=$((retries+1))
else
@ -142,6 +164,3 @@ if [ $? -ne 0 ] ; then
echo "❌ Tests failed"
exit 1
fi
# Exit
exit 0