Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: Update handling of plugins changes

Browse source
This commit is contained in:
Théophile Diot 2024-05-21 14:37:53 +01:00
parent 1584d571af
commit 3742157f7d
No known key found for this signature in database
GPG key ID: 248FEA4BAE400D06
4 changed files with 98 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
src/common/core/jobs/jobs/download-plugins.py
View file

@ -9,7 +9,7 @@ from stat import S_IEXEC
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from uuid import uuid4
from json import JSONDecodeError, loads
from json import JSONDecodeError, load as json_load, loads
from shutil import copytree, rmtree
from tarfile import open as tar_open
from traceback import format_exc
@ -176,26 +176,27 @@ try:
rmtree(plugin_path, ignore_errors=True)
continue
plugin_file = loads(plugin_path.joinpath("plugin.json").read_text(encoding="utf-8"))
with BytesIO() as plugin_content:
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(plugin_path, arcname=plugin_path.name)
plugin_content.seek(0)
value = plugin_content.getvalue()
tar.add(plugin_path, arcname=plugin_path.name, recursive=True)
plugin_content.seek(0, 0)
plugin_file.update(
{
"type": "external",
"page": plugin_path.joinpath("ui").is_dir(),
"method": "scheduler",
"data": value,
"checksum": bytes_hash(value, algorithm="sha256"),
}
)
with plugin_path.joinpath("plugin.json").open("r", encoding="utf-8") as f:
plugin_data = json_load(f)
external_plugins.append(plugin_file)
external_plugins_ids.append(plugin_file["id"])
checksum = bytes_hash(plugin_content, algorithm="sha256")
plugin_data.update(
{
"type": "external",
"page": plugin_path.joinpath("ui").is_dir(),
"method": "scheduler",
"data": plugin_content.getvalue(),
"checksum": checksum,
}
)
external_plugins.append(plugin_data)
external_plugins_ids.append(plugin_data["id"])
lock = Lock()

37
src/common/core/pro/jobs/download-pro-plugins.py
View file

@ -10,7 +10,7 @@ from stat import S_IEXEC
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from uuid import uuid4
from json import JSONDecodeError, load, loads
from json import JSONDecodeError, load as json_load, loads
from shutil import copytree, rmtree
from tarfile import open as tar_open
from traceback import format_exc
@ -188,7 +188,7 @@ try:
for chunk in resp.iter_content(chunk_size=8192):
resp_content.write(chunk)
resp_content.seek(0)
resp_data = load(resp_content)
resp_data = json_load(resp_content)
clean = resp_data.get("action") == "clean"
@ -280,26 +280,27 @@ try:
rmtree(plugin_path, ignore_errors=True)
continue
plugin_file = loads(plugin_path.joinpath("plugin.json").read_text(encoding="utf-8"))
with BytesIO() as plugin_content:
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(plugin_path, arcname=plugin_path.name)
plugin_content.seek(0)
value = plugin_content.getvalue()
tar.add(plugin_path, arcname=plugin_path.name, recursive=True)
plugin_content.seek(0, 0)
plugin_file.update(
{
"type": "pro",
"page": plugin_path.joinpath("ui").is_dir(),
"method": "scheduler",
"data": value,
"checksum": bytes_hash(value, algorithm="sha256"),
}
)
with plugin_path.joinpath("plugin.json").open("r", encoding="utf-8") as f:
plugin_data = json_load(f)
pro_plugins.append(plugin_file)
pro_plugins_ids.append(plugin_file["id"])
checksum = bytes_hash(plugin_content, algorithm="sha256")
plugin_data.update(
{
"type": "pro",
"page": plugin_path.joinpath("ui").is_dir(),
"method": "scheduler",
"data": plugin_content.getvalue(),
"checksum": checksum,
}
)
pro_plugins.append(plugin_data)
pro_plugins_ids.append(plugin_data["id"])
lock = Lock()

10
src/common/db/Database.py
View file

@ -92,7 +92,8 @@ class Database:
if sqlalchemy_string == sqlalchemy_string_readonly:
self.readonly = True
self.logger.warning("The database connection is set to read-only, the changes will not be saved")
if log:
self.logger.warning("The database connection is set to read-only, the changes will not be saved")
match = self.DB_STRING_RX.search(sqlalchemy_string)
if not match:
@ -1519,6 +1520,10 @@ class Database:
)
if db_plugin:
if plugin["method"] not in (db_plugin.method, "autoconf"):
self.logger.warning(f'Plugin "{plugin["id"]}" already exists, but the method is different, skipping update')
continue
if db_plugin.type not in ("external", "pro"):
self.logger.warning(
f"Plugin \"{plugin['id']}\" is not {_type}, skipping update (updating a non-external or non-pro plugin is forbidden for security reasons)", # noqa: E501
@ -1993,7 +1998,6 @@ class Database:
"method": plugin.method,
"page": page is not None,
"settings": {},
"bwcli": {},
"checksum": plugin.checksum,
} | ({"data": plugin.data} if with_data else {})
@ -2029,6 +2033,8 @@ class Database:
]
for command in session.query(BwcliCommands).with_entities(BwcliCommands.name, BwcliCommands.file_name).filter_by(plugin_id=plugin.id):
if "bwcli" not in data:
data["bwcli"] = {}
data["bwcli"][command.name] = command.file_name
plugins.append(data)

88
src/scheduler/main.py
View file

@ -173,9 +173,23 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U
pro = "pro" in original_path.parts
# Remove old external/pro plugins files
logger.info(f"Removing old {'pro ' if pro else ''}external plugins files ...")
logger.info(f"Removing old/changed {'pro ' if pro else ''}external plugins files ...")
if original_path.is_dir():
for file in original_path.glob("*"):
try:
index = next(i for i, plugin in enumerate(plugins) if plugin["id"] == file.name)
except StopIteration:
index = -1
if index > -1:
with BytesIO() as plugin_content:
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(file, arcname=file.name, recursive=True)
plugin_content.seek(0, 0)
if bytes_hash(plugin_content, algorithm="sha256") == plugins[index]["checksum"]:
continue
logger.debug(f"Checksum of {file} has changed, removing it ...")
if file.is_symlink() or file.is_file():
with suppress(OSError):
file.unlink()
@ -472,51 +486,55 @@ if __name__ == "__main__":
# Check if any external or pro plugin has been added by the user
logger.info(f"Checking if there are any changes in {_type} plugins ...")
plugin_path = EXTERNAL_PLUGINS_PATH if _type == "external" else PRO_PLUGINS_PATH
db_plugins = SCHEDULER.db.get_plugins(_type=_type)
external_plugins = []
tmp_external_plugins = []
for file in plugin_path.glob("*/plugin.json"):
plugin_content = BytesIO()
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(file.parent, arcname=file.parent.name, recursive=True)
plugin_content.seek(0, 0)
with BytesIO() as plugin_content:
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(file.parent, arcname=file.parent.name, recursive=True)
plugin_content.seek(0, 0)
with file.open("r", encoding="utf-8") as f:
plugin_data = json_load(f)
with file.open("r", encoding="utf-8") as f:
plugin_data = json_load(f)
common_data = plugin_data | {
"type": _type,
"page": file.parent.joinpath("ui").is_dir(),
}
jobs = common_data.pop("jobs", [])
tmp_external_plugins.append(common_data)
checksum = bytes_hash(plugin_content, algorithm="sha256")
external_plugins.append(
common_data
| {
"method": "manual",
"data": plugin_content.getvalue(),
checksum = bytes_hash(plugin_content, algorithm="sha256")
common_data = plugin_data | {
"type": _type,
"page": file.parent.joinpath("ui").is_dir(),
"checksum": checksum,
}
| ({"jobs": jobs} if jobs else {})
)
jobs = common_data.pop("jobs", [])
db_plugins = SCHEDULER.db.get_plugins(_type=_type)
tmp_db_plugins = []
for db_plugin in db_plugins.copy():
db_plugin.pop("method", None)
tmp_db_plugins.append(db_plugin)
try:
index = next(i for i, plugin in enumerate(db_plugins) if plugin["id"] == common_data["id"])
except StopIteration:
index = -1
changes = {hash(dict_to_frozenset(d)) for d in tmp_external_plugins} != {hash(dict_to_frozenset(d)) for d in tmp_db_plugins}
if index > -1 and checksum == db_plugins[index]["checksum"] or db_plugins[index]["method"] != "manual":
continue
if changes:
err = SCHEDULER.db.update_external_plugins(external_plugins, _type=_type, delete_missing=True)
if err:
logger.error(f"Couldn't save some manually added {_type} plugins to database: {err}")
tmp_external_plugins.append(common_data.copy())
if (scheduler_first_start and db_plugins) or changes:
generate_external_plugins(SCHEDULER.db.get_plugins(_type=_type, with_data=True), original_path=plugin_path)
external_plugins.append(
common_data
| {
"method": "manual",
"data": plugin_content.getvalue(),
}
| ({"jobs": jobs} if jobs else {})
)
if tmp_external_plugins:
changes = {hash(dict_to_frozenset(d)) for d in tmp_external_plugins} != {hash(dict_to_frozenset(d)) for d in db_plugins}
if changes:
err = SCHEDULER.db.update_external_plugins(external_plugins, _type=_type, delete_missing=True)
if err:
logger.error(f"Couldn't save some manually added {_type} plugins to database: {err}")
if (scheduler_first_start and db_plugins) or changes:
generate_external_plugins(SCHEDULER.db.get_plugins(_type=_type, with_data=True), original_path=plugin_path)
check_plugin_changes("external")
check_plugin_changes("pro")