diff --git a/src/common/core/jobs/jobs/download-plugins.py b/src/common/core/jobs/jobs/download-plugins.py index 2a66ac6e2..2608c1fd7 100644 --- a/src/common/core/jobs/jobs/download-plugins.py +++ b/src/common/core/jobs/jobs/download-plugins.py @@ -35,19 +35,35 @@ from Database import Database # type: ignore from logger import setup_logger # type: ignore +EXTERNAL_PLUGINS_DIR = Path(sep, "etc", "bunkerweb", "plugins") logger = setup_logger("Jobs.download-plugins", getenv("LOG_LEVEL", "INFO")) status = 0 -def install_plugin(plugin_dir) -> bool: +def install_plugin(plugin_dir, db) -> bool: + plugin_path = Path(plugin_dir) # Load plugin.json - metadata = loads(Path(plugin_dir, "plugin.json").read_text(encoding="utf-8")) + metadata = loads(plugin_path.joinpath("plugin.json").read_text(encoding="utf-8")) # Don't go further if plugin is already installed - if Path("etc", "bunkerweb", "plugins", metadata["id"], "plugin.json").is_file(): + if EXTERNAL_PLUGINS_DIR.joinpath(metadata["id"], "plugin.json").is_file(): + old_version = None + + for plugin in db.get_plugins(external=True): + if plugin["id"] == metadata["id"]: + old_version = plugin["version"] + break + + if old_version == metadata["version"]: + logger.warning( + f"Skipping installation of plugin {metadata['id']} (version {metadata['version']} already installed)", + ) + return False + logger.warning( - f"Skipping installation of plugin {metadata['id']} (already installed)", + f"Plugin {metadata['id']} is already installed but version {metadata['version']} is different from database ({old_version}), updating it...", ) - return False + rmtree(EXTERNAL_PLUGINS_DIR.joinpath(metadata["id"]), ignore_errors=True) + # Copy the plugin copytree(plugin_dir, join(sep, "etc", "bunkerweb", "plugins", metadata["id"])) # Add u+x permissions to jobs files @@ -65,6 +81,7 @@ try: logger.info("No external plugins to download") _exit(0) + db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False) plugin_nbr = 0 # Loop on URLs @@ -127,7 +144,7 @@ try: try: for plugin_dir in glob(join(temp_dir, "**", "plugin.json"), recursive=True): try: - if install_plugin(dirname(plugin_dir)): + if install_plugin(dirname(plugin_dir), db): plugin_nbr += 1 except FileExistsError: logger.warning( @@ -145,21 +162,20 @@ try: external_plugins = [] external_plugins_ids = [] - plugins_dir = join(sep, "etc", "bunkerweb", "plugins") - for plugin in listdir(plugins_dir): - path = join(plugins_dir, plugin) - if not Path(path, "plugin.json").is_file(): + for plugin in listdir(EXTERNAL_PLUGINS_DIR): + path = EXTERNAL_PLUGINS_DIR.joinpath(plugin) + if not path.joinpath("plugin.json").is_file(): logger.warning(f"Plugin {plugin} is not valid, deleting it...") rmtree(path, ignore_errors=True) continue - plugin_file = loads(Path(path, "plugin.json").read_text(encoding="utf-8")) + plugin_file = loads(path.joinpath("plugin.json").read_text(encoding="utf-8")) - plugin_content = BytesIO() - with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar: - tar.add(path, arcname=basename(path)) - plugin_content.seek(0) - value = plugin_content.getvalue() + with BytesIO() as plugin_content: + with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar: + tar.add(path, arcname=path.name) + plugin_content.seek(0) + value = plugin_content.getvalue() plugin_file.update( { @@ -177,7 +193,6 @@ try: external_plugins.append(plugin_file) external_plugins_ids.append(plugin_file["id"]) - db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False) lock = Lock() for plugin in db.get_plugins(external=True, with_data=True): diff --git a/src/common/db/Database.py b/src/common/db/Database.py index a04d96c49..b10477638 100644 --- a/src/common/db/Database.py +++ b/src/common/db/Database.py @@ -1455,33 +1455,10 @@ class Database: """Get all plugins from the database.""" plugins = [] with self.__db_session() as session: - for plugin in ( - session.query(Plugins) - .with_entities( - Plugins.id, - Plugins.stream, - Plugins.name, - Plugins.description, - Plugins.version, - Plugins.external, - Plugins.method, - Plugins.data, - Plugins.checksum, - ) - .all() - if with_data - else session.query(Plugins) - .with_entities( - Plugins.id, - Plugins.stream, - Plugins.name, - Plugins.description, - Plugins.version, - Plugins.external, - Plugins.method, - ) - .all() - ): + entities = [Plugins.id, Plugins.stream, Plugins.name, Plugins.description, Plugins.version, Plugins.external, Plugins.method, Plugins.checksum] + if with_data: + entities.append(Plugins.data) + for plugin in session.query(Plugins).with_entities(*entities).all(): if external and not plugin.external: continue @@ -1496,7 +1473,8 @@ class Database: "method": plugin.method, "page": page is not None, "settings": {}, - } | ({"data": plugin.data, "checksum": plugin.checksum} if with_data else {}) + "checksum": plugin.checksum, + } | ({"data": plugin.data} if with_data else {}) for setting in ( session.query(Settings) diff --git a/src/common/utils/jobs.py b/src/common/utils/jobs.py index 8bec6df72..096fda0cc 100644 --- a/src/common/utils/jobs.py +++ b/src/common/utils/jobs.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from contextlib import suppress from datetime import datetime from hashlib import sha512 from inspect import getsourcefile @@ -142,8 +141,9 @@ def bytes_hash(bio: BufferedReader) -> str: def cache_hash(cache: Union[str, Path], db=None) -> Optional[str]: checksum = None - with suppress(BaseException): - checksum = loads(Path(f"{cache}.md").read_text(encoding="utf-8")).get("checksum", None) + cache_file = Path(f"{cache}.md") + if cache_file.is_file(): + checksum = loads(cache_file.read_text(encoding="utf-8")).get("checksum", None) if not checksum and db: if not isinstance(cache, Path):