From 5e3dadbfe39f3052d8b5a98cc4c00fc2041e466d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Wed, 22 Feb 2023 19:04:59 +0100 Subject: [PATCH] Refactor ui --- src/ui/main.py | 107 ++++++++++++++++++++------------------ src/ui/src/Config.py | 35 ++++++------- src/ui/src/ConfigFiles.py | 16 +++--- 3 files changed, 80 insertions(+), 78 deletions(-) diff --git a/src/ui/main.py b/src/ui/main.py index 6ae43fb5c..2710347db 100755 --- a/src/ui/main.py +++ b/src/ui/main.py @@ -1,11 +1,5 @@ -from contextlib import suppress -from importlib.machinery import SourceFileLoader -from io import BytesIO -from pathlib import Path -from signal import SIGINT, signal, SIGTERM -from subprocess import PIPE, Popen, call -from tempfile import NamedTemporaryFile from bs4 import BeautifulSoup +from contextlib import suppress from copy import deepcopy from datetime import datetime, timedelta, timezone from dateutil.parser import parse as dateutil_parse @@ -27,18 +21,24 @@ from flask import ( ) from flask_login import LoginManager, login_required, login_user, logout_user from flask_wtf.csrf import CSRFProtect, CSRFError, generate_csrf +from importlib.machinery import SourceFileLoader +from io import BytesIO from json import JSONDecodeError, dumps, load as json_load from jinja2 import Template from kubernetes import client as kube_client from kubernetes.client.exceptions import ApiException as kube_ApiException -from os import _exit, chmod, getenv, getpid, listdir, mkdir, remove, walk -from os.path import exists, isdir, isfile, join +from os import _exit, chmod, getenv, getpid, listdir, walk +from os.path import join +from pathlib import Path from re import match as re_match from requests import get from shutil import rmtree, copytree, chown +from signal import SIGINT, signal, SIGTERM +from subprocess import PIPE, Popen, call from sys import path as sys_path, modules as sys_modules from tarfile import CompressionError, HeaderError, ReadError, TarError, open as tar_open from threading import Thread +from tempfile import NamedTemporaryFile from time import time from traceback import format_exc from typing import Optional @@ -66,7 +66,7 @@ from utils import ( from logger import setup_logger from Database import Database -if not exists("/var/log/nginx/ui.log"): +if not Path("/var/log/nginx/ui.log").exists(): Path("/var/log/nginx").mkdir(parents=True, exist_ok=True) Path("/var/log/nginx/ui.log").touch() @@ -81,8 +81,8 @@ def stop_gunicorn(): def stop(status, stop=True): - if exists("/var/tmp/bunkerweb/ui.pid"): - remove("/var/tmp/bunkerweb/ui.pid") + if Path("/var/tmp/bunkerweb/ui.pid").exists(): + Path("/var/tmp/bunkerweb/ui.pid").unlink() if stop is True: stop_gunicorn() _exit(status) @@ -154,7 +154,7 @@ elif getenv("SWARM_MODE", "no") == "yes": integration = "Swarm" elif getenv("AUTOCONF_MODE", "no") == "yes": integration = "Autoconf" -elif exists("/usr/share/bunkerweb/INTEGRATION"): +elif Path("/usr/share/bunkerweb/INTEGRATION").exists(): with open("/usr/share/bunkerweb/INTEGRATION", "r") as f: integration = f.read().strip() @@ -718,7 +718,7 @@ def plugins(): flash(f"Can't delete internal plugin {variables['name']}", "error") return redirect(url_for("loading", next=url_for("plugins"))), 500 - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").is_file(): plugins = app.config["CONFIG"].get_plugins() for plugin in deepcopy(plugins): if plugin["external"] is False or plugin["id"] == variables["name"]: @@ -749,7 +749,7 @@ def plugins(): flash(operation, "error") return redirect(url_for("loading", next=url_for("plugins"))) else: - if not exists("/var/tmp/bunkerweb/ui") or not listdir( + if not Path("/var/tmp/bunkerweb/ui").exists() or not listdir( "/var/tmp/bunkerweb/ui" ): flash("Please upload new plugins to reload plugins", "error") @@ -759,7 +759,7 @@ def plugins(): files_count = 0 for file in listdir("/var/tmp/bunkerweb/ui"): - if not isfile(f"/var/tmp/bunkerweb/ui/{file}"): + if not Path(f"/var/tmp/bunkerweb/ui/{file}").is_file(): continue files_count += 1 @@ -799,7 +799,7 @@ def plugins(): ) raise Exception - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").is_file(): plugins = app.config["CONFIG"].get_plugins() for plugin in deepcopy(plugins): if plugin["id"] == folder_name: @@ -817,9 +817,9 @@ def plugins(): ) raise Exception else: - if exists( + if Path( f"/etc/bunkerweb/plugins/{folder_name}" - ): + ).exists(): raise FileExistsError copytree( @@ -835,17 +835,17 @@ def plugins(): for d in listdir( f"/var/tmp/bunkerweb/ui/{temp_folder_name}" ) - if isdir( + if Path( f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{d}" - ) + ).is_dir() ] if ( not dirs or len(dirs) > 1 - or not exists( + or not Path( f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}/plugin.json" - ) + ).is_file() ): raise KeyError @@ -873,7 +873,7 @@ def plugins(): ) raise Exception - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").is_file(): plugins = app.config["CONFIG"].get_plugins() for plugin in deepcopy(plugins): if plugin["id"] == folder_name: @@ -891,9 +891,9 @@ def plugins(): ) raise Exception else: - if exists( + if Path( f"/etc/bunkerweb/plugins/{folder_name}" - ): + ).exists(): raise FileExistsError copytree( @@ -942,7 +942,7 @@ def plugins(): ) raise Exception - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").is_file(): plugins = app.config["CONFIG"].get_plugins() for plugin in deepcopy(plugins): if plugin["id"] == folder_name: @@ -960,9 +960,9 @@ def plugins(): ) raise Exception else: - if exists( + if Path( f"/etc/bunkerweb/plugins/{folder_name}" - ): + ).exists(): raise FileExistsError copytree( @@ -978,17 +978,17 @@ def plugins(): for d in listdir( f"/var/tmp/bunkerweb/ui/{temp_folder_name}" ) - if isdir( + if Path( f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{d}" - ) + ).is_dir() ] if ( not dirs or len(dirs) > 1 - or not exists( + or not Path( f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}/plugin.json" - ) + ).is_file() ): raise KeyError @@ -1016,7 +1016,7 @@ def plugins(): ) raise Exception - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").is_file(): plugins = app.config["CONFIG"].get_plugins() for plugin in deepcopy(plugins): if plugin["id"] == folder_name: @@ -1034,9 +1034,9 @@ def plugins(): ) raise Exception else: - if exists( + if Path( f"/etc/bunkerweb/plugins/{folder_name}" - ): + ).exists(): raise FileExistsError copytree( @@ -1129,7 +1129,7 @@ def plugins(): ).start() # Remove tmp folder - if exists("/var/tmp/bunkerweb/ui"): + if Path("/var/tmp/bunkerweb/ui").exists(): with suppress(OSError): rmtree("/var/tmp/bunkerweb/ui") @@ -1144,7 +1144,7 @@ def plugins(): plugin_id = request.args.get("plugin_id") template = None - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").is_file(): page = db.get_plugin_template(plugin_id) if page is not None: @@ -1152,9 +1152,11 @@ def plugins(): else: page_path = "" - if exists(f"/etc/bunkerweb/plugins/{plugin_id}/ui/template.html"): + if Path(f"/etc/bunkerweb/plugins/{plugin_id}/ui/template.html").exists(): page_path = f"/etc/bunkerweb/plugins/{plugin_id}/ui/template.html" - elif exists(f"/usr/share/bunkerweb/core/{plugin_id}/ui/template.html"): + elif Path( + f"/usr/share/bunkerweb/core/{plugin_id}/ui/template.html" + ).exists(): page_path = f"/usr/share/bunkerweb/core/{plugin_id}/ui/template.html" else: flash(f"Plugin {plugin_id} not found", "error") @@ -1201,8 +1203,7 @@ def upload_plugin(): if not request.files: return {"status": "ko"}, 400 - if not exists("/var/tmp/bunkerweb/ui"): - mkdir("/var/tmp/bunkerweb/ui") + Path("/var/tmp/bunkerweb/ui").mkdir(parents=True, exist_ok=True) for file in request.files.values(): if not file.filename.endswith((".zip", ".tar.gz", ".tar.xz")): @@ -1223,7 +1224,7 @@ def custom_plugin(plugin): ) return redirect(url_for("loading", next=url_for("plugins", plugin_id=plugin))) - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").is_file(): module = db.get_plugin_actions(plugin) if module is None: @@ -1252,8 +1253,9 @@ def custom_plugin(plugin): url_for("loading", next=url_for("plugins", plugin_id=plugin)) ) else: - if not exists(f"/etc/bunkerweb/plugins/{plugin}/ui/actions.py") and not exists( - f"/usr/share/bunkerweb/core/{plugin}/ui/actions.py" + if ( + not Path(f"/etc/bunkerweb/plugins/{plugin}/ui/actions.py").exists() + and not Path(f"/usr/share/bunkerweb/core/{plugin}/ui/actions.py").exists() ): flash( f"The actions.py file for the plugin {plugin} does not exist", @@ -1267,7 +1269,7 @@ def custom_plugin(plugin): sys_path.append( ( "/etc/bunkerweb/plugins" - if exists(f"/etc/bunkerweb/plugins/{plugin}/ui/actions.py") + if Path(f"/etc/bunkerweb/plugins/{plugin}/ui/actions.py").exists() else "/usr/share/bunkerweb/core" ) + f"/{plugin}/ui/" @@ -1305,7 +1307,7 @@ def custom_plugin(plugin): ) error = True finally: - if exists("/usr/sbin/nginx"): + if Path("/usr/sbin/nginx").is_file(): # Remove the custom plugin from the shared library sys_path.pop() sys_modules.pop("actions") @@ -1359,7 +1361,7 @@ def logs(): @app.route("/logs/local", methods=["GET"]) @login_required def logs_linux(): - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").is_file(): return ( jsonify( { @@ -1375,21 +1377,21 @@ def logs_linux(): raw_logs_error = [] if last_update: - if exists("/var/log/nginx/error.log"): + if Path("/var/log/nginx/error.log").exists(): with open("/var/log/nginx/error.log", "r") as f: raw_logs_error = f.read().splitlines()[int(last_update.split(".")[0]) :] - if exists("/var/log/nginx/access.log"): + if Path("/var/log/nginx/access.log").exists(): with open("/var/log/nginx/access.log", "r") as f: raw_logs_access = f.read().splitlines()[ int(last_update.split(".")[1]) : ] else: - if exists("/var/log/nginx/error.log"): + if Path("/var/log/nginx/error.log").exists(): with open("/var/log/nginx/error.log", "r") as f: raw_logs_error = f.read().splitlines() - if exists("/var/log/nginx/access.log"): + if Path("/var/log/nginx/access.log").exists(): with open("/var/log/nginx/access.log", "r") as f: raw_logs_access = f.read().splitlines() @@ -1515,6 +1517,7 @@ def logs_container(container_id): to_date = int(to_date) // 1000 if to_date else None logs = [] + tmp_logs = [] if docker_client: try: if getenv("SWARM_MODE", "no") == "no": diff --git a/src/ui/src/Config.py b/src/ui/src/Config.py index 67703de02..6cd7ba758 100644 --- a/src/ui/src/Config.py +++ b/src/ui/src/Config.py @@ -1,15 +1,14 @@ from copy import deepcopy -from os import listdir, remove -from pathlib import Path -from time import sleep from flask import flash -from os.path import exists, isfile -from typing import List, Tuple -from json import load as json_load -from uuid import uuid4 from glob import iglob +from json import load as json_load +from os import listdir +from pathlib import Path from re import search as re_search from subprocess import run, DEVNULL, STDOUT +from time import sleep +from typing import List, Tuple +from uuid import uuid4 class Config: @@ -20,7 +19,7 @@ class Config: self.__logger = logger self.__db = db - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").exists(): while not self.__db.is_initialized(): self.__logger.warning( "Database is not initialized, retrying in 5s ...", @@ -48,7 +47,7 @@ class Config: dict The values of the file converted to dict """ - if not isfile(filename): + if not Path(filename).is_file(): return {} data = {} @@ -94,7 +93,7 @@ class Config: plugins_settings = self.get_plugins_settings() for service in services_conf: server_name = service["SERVER_NAME"].split(" ")[0] - for k in service.keys(): + for k in service: key_without_server_name = k.replace(f"{server_name}_", "") if ( plugins_settings[key_without_server_name]["context"] != "global" @@ -127,7 +126,7 @@ class Config: if proc.returncode != 0: raise Exception(f"Error from generator (return code = {proc.returncode})") - remove(env_file) + Path(env_file).unlink() def get_plugins_settings(self) -> dict: return { @@ -136,7 +135,7 @@ class Config: } def get_plugins(self) -> List[dict]: - if not exists("/usr/sbin/nginx"): + if not Path("/usr/sbin/nginx").exists(): plugins = self.__db.get_plugins() plugins.sort(key=lambda x: x["name"]) @@ -205,9 +204,9 @@ class Config: dict The nginx variables env file as a dict """ - if exists("/usr/sbin/nginx"): + if Path("/usr/sbin/nginx").exists(): return { - k: ({"value": v, "method": "ui"} if methods is True else v) + k: ({"value": v, "method": "ui"} if methods else v) for k, v in self.__env_to_dict("/etc/nginx/variables.env").items() } @@ -221,17 +220,17 @@ class Config: list The services """ - if exists("/usr/sbin/nginx"): + if Path("/usr/sbin/nginx").exists(): services = [] plugins_settings = self.get_plugins_settings() for filename in iglob("/etc/nginx/**/variables.env"): service = filename.split("/")[3] env = { k.replace(f"{service}_", ""): ( - {"value": v, "method": "ui"} if methods is True else v + {"value": v, "method": "ui"} if methods else v ) for k, v in self.__env_to_dict(filename).items() - if k.startswith(f"{service}_") or k in plugins_settings.keys() + if k.startswith(f"{service}_") or k in plugins_settings } services.append(env) @@ -314,7 +313,7 @@ class Config: if service["SERVER_NAME"] == variables["SERVER_NAME"] or service[ "SERVER_NAME" ] in variables["SERVER_NAME"].split(" "): - if edit is False: + if not edit: return ( f"Service {service['SERVER_NAME'].split(' ')[0]} already exists.", 1, diff --git a/src/ui/src/ConfigFiles.py b/src/ui/src/ConfigFiles.py index 93446f15f..83a49fcb9 100644 --- a/src/ui/src/ConfigFiles.py +++ b/src/ui/src/ConfigFiles.py @@ -1,5 +1,5 @@ -from os import listdir, remove, replace, walk -from os.path import dirname, exists, join, isfile +from os import listdir, replace, walk +from os.path import dirname, join from pathlib import Path from re import compile as re_compile from shutil import rmtree, move as shutil_move @@ -52,7 +52,7 @@ class ConfigFiles: return "" def check_name(self, name: str) -> bool: - return self.__name_regex.match(name) + return self.__name_regex.match(name) is not None def check_path(self, path: str, root_path: str = "/etc/bunkerweb/configs/") -> str: root_dir: str = path.split("/")[4] @@ -75,17 +75,17 @@ class ConfigFiles: dirs = "/".join(dirs) if len(dirs) > 1: for x in range(nbr_children - 1): - if not exists( + if not Path( f"{root_path}{root_dir}/{'/'.join(dirs.split('/')[0:-x])}" - ): + ).exists(): return f"{root_path}{root_dir}/{'/'.join(dirs.split('/')[0:-x])} doesn't exist" return "" def delete_path(self, path: str) -> Tuple[str, int]: try: - if isfile(path) or isfile(f"{path}.conf"): - remove(f"{path}.conf") + if Path(path).is_file() or Path(f"{path}.conf").is_file(): + Path(f"{path}.conf").unlink() else: rmtree(path) except OSError: @@ -154,7 +154,7 @@ class ConfigFiles: new_path = old_path else: try: - remove(old_path) + Path(old_path).unlink() except OSError: return f"Could not remove {old_path}", 1