From e62b7c9d19829dec962c615addb094987d7879fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 5 Sep 2023 12:06:30 +0200 Subject: [PATCH 1/6] Remove unused js files in web-ui --- src/ui/static/js/utils/en.js | 84 ------------------------------------ src/ui/templates/head.html | 4 -- 2 files changed, 88 deletions(-) delete mode 100644 src/ui/static/js/utils/en.js diff --git a/src/ui/static/js/utils/en.js b/src/ui/static/js/utils/en.js deleted file mode 100644 index 33b16139e..000000000 --- a/src/ui/static/js/utils/en.js +++ /dev/null @@ -1,84 +0,0 @@ -(function (global, factory) { - typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : - typeof define === 'function' && define.amd ? define(['exports'], factory) : - (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.default = {})); -}(this, (function (exports) { 'use strict'; - - var english = { - weekdays: { - shorthand: ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"], - longhand: [ - "Sunday", - "Monday", - "Tuesday", - "Wednesday", - "Thursday", - "Friday", - "Saturday", - ], - }, - months: { - shorthand: [ - "Jan", - "Feb", - "Mar", - "Apr", - "May", - "Jun", - "Jul", - "Aug", - "Sep", - "Oct", - "Nov", - "Dec", - ], - longhand: [ - "January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December", - ], - }, - daysInMonth: [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], - firstDayOfWeek: 0, - ordinal: function (nth) { - var s = nth % 100; - if (s > 3 && s < 21) - return "th"; - switch (s % 10) { - case 1: - return "st"; - case 2: - return "nd"; - case 3: - return "rd"; - default: - return "th"; - } - }, - rangeSeparator: " to ", - weekAbbreviation: "Wk", - scrollTitle: "Scroll to increment", - toggleTitle: "Click to toggle", - amPM: ["AM", "PM"], - yearAriaLabel: "Year", - monthAriaLabel: "Month", - hourAriaLabel: "Hour", - minuteAriaLabel: "Minute", - time_24hr: false, - }; - - exports.default = english; - exports.english = english; - - Object.defineProperty(exports, '__esModule', { value: true }); - -}))); diff --git a/src/ui/templates/head.html b/src/ui/templates/head.html index b0bc88a26..c9a4ed862 100644 --- a/src/ui/templates/head.html +++ b/src/ui/templates/head.html @@ -33,12 +33,8 @@ {% elif current_endpoint == "logs" %} - - - - Date: Tue, 5 Sep 2023 17:09:35 +0200 Subject: [PATCH 2/6] Deactivate BunkerNet on first start with linux --- src/linux/scripts/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/linux/scripts/start.sh b/src/linux/scripts/start.sh index 73c242f44..e774ac9a2 100644 --- a/src/linux/scripts/start.sh +++ b/src/linux/scripts/start.sh @@ -90,7 +90,7 @@ function start() { # Create dummy variables.env if [ ! -f /etc/bunkerweb/variables.env ]; then - sudo -E -u nginx -g nginx /bin/bash -c "echo -ne '# remove IS_LOADING=yes when your config is ready\nIS_LOADING=yes\nDNS_RESOLVERS=8.8.8.8 8.8.4.4\nHTTP_PORT=80\nHTTPS_PORT=443\nAPI_LISTEN_IP=127.0.0.1\nSERVER_NAME=\n' > /etc/bunkerweb/variables.env" + sudo -E -u nginx -g nginx /bin/bash -c "echo -ne '# remove IS_LOADING=yes when your config is ready\nIS_LOADING=yes\nUSE_BUNKERNET=no\nDNS_RESOLVERS=8.8.8.8 8.8.4.4\nHTTP_PORT=80\nHTTPS_PORT=443\nAPI_LISTEN_IP=127.0.0.1\nSERVER_NAME=\n' > /etc/bunkerweb/variables.env" log "SYSTEMCTL" "ℹ️" "Created dummy variables.env file" fi From 1b84c620248ec81f4b1a3aa32dac19003f9ad1e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 5 Sep 2023 17:12:21 +0200 Subject: [PATCH 3/6] [#613] Fix logs with web-ui and Linux --- src/ui/main.py | 126 ++++++++++++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 55 deletions(-) diff --git a/src/ui/main.py b/src/ui/main.py index 95d87d870..b260dec81 100755 --- a/src/ui/main.py +++ b/src/ui/main.py @@ -242,6 +242,10 @@ app.jinja_env.globals.update(check_settings=check_settings) csrf = CSRFProtect() csrf.init_app(app) +LOG_RX = re_compile( + r"^(?P\d+/\d+/\d+\s\d+:\d+:\d+)\s\[(?P[a-z]+)\]\s\d+#\d+:\s(?P[^\n]+)$" +) + def manage_bunkerweb(method: str, *args, operation: str = "reloads"): # Do the operation @@ -528,8 +532,8 @@ def services(): args=( "services", variables, - request.form.get("OLD_SERVER_NAME", "").split(" ")[0], - variables.get("SERVER_NAME", "").split(" ")[0], + request.form.get("OLD_SERVER_NAME", "").split()[0], + variables.get("SERVER_NAME", "").split()[0], ), kwargs={"operation": request.form["operation"]}, ).start() @@ -537,11 +541,11 @@ def services(): message = "" if request.form["operation"] == "new": - message = f"Creating service {variables['SERVER_NAME'].split(' ')[0]}" + message = f"Creating service {variables['SERVER_NAME'].split()[0]}" elif request.form["operation"] == "edit": - message = f"Saving configuration for service {request.form['OLD_SERVER_NAME'].split(' ')[0]}" + message = f"Saving configuration for service {request.form['OLD_SERVER_NAME'].split()[0]}" elif request.form["operation"] == "delete": - message = f"Deleting service {request.form['SERVER_NAME'].split(' ')[0]}" + message = f"Deleting service {request.form['SERVER_NAME'].split()[0]}" return redirect(url_for("loading", next=url_for("services"), message=message)) @@ -552,7 +556,7 @@ def services(): services=[ { "SERVER_NAME": { - "value": service["SERVER_NAME"]["value"].split(" ")[0], + "value": service["SERVER_NAME"]["value"].split()[0], "method": service["SERVER_NAME"]["method"], }, "USE_REVERSE_PROXY": service["USE_REVERSE_PROXY"], @@ -725,7 +729,7 @@ def configs(): db_data=db.get_custom_configs(), services=app.config["CONFIG"] .get_config(methods=False)["SERVER_NAME"] - .split(" "), + .split(), ) ], dark_mode=app.config["DARK_MODE"], @@ -1171,7 +1175,7 @@ def cache(): db_data=db.get_jobs_cache_files(), services=app.config["CONFIG"] .get_config(methods=False)["SERVER_NAME"] - .split(" "), + .split(), ) ], dark_mode=app.config["DARK_MODE"], @@ -1202,22 +1206,9 @@ def logs_linux(): 404, ) - last_update = request.args.get("last_update") - raw_logs_access = [] - raw_logs_error = [] - - nginx_error_file = Path(sep, "var", "log", "nginx", "error.log") - if nginx_error_file.is_file(): - raw_logs_access = nginx_error_file.read_text(encoding="utf-8").splitlines()[ - int(last_update.split(".")[0]) if last_update else 0 : - ] - - nginx_access_file = Path(sep, "var", "log", "nginx", "access.log") - if nginx_access_file.is_file(): - raw_logs_error = nginx_access_file.read_text(encoding="utf-8").splitlines()[ - int(last_update.split(".")[1]) if last_update else 0 : - ] - + last_update = request.args.get("last_update", "0.0") + from_date = request.args.get("from_date", None) + to_date = request.args.get("to_date", None) logs_error = [] temp_multiple_lines = [] NGINX_LOG_LEVELS = [ @@ -1230,44 +1221,69 @@ def logs_linux(): "alert", "emerg", ] - for line in raw_logs_error: - line_lower = line.lower() - if ( - ("[info]" in line_lower or "ℹ️" in line_lower) - and line.endswith(":") - or ("[error]" in line_lower or "❌" in line_lower) - ): - if temp_multiple_lines: - logs_error.append("\n".join(temp_multiple_lines)) + nginx_error_file = Path(sep, "var", "log", "bunkerweb", "error.log") + if nginx_error_file.is_file(): + with open(nginx_error_file, encoding="utf-8") as f: + for line in f.readlines()[ + int(last_update.split(".")[0]) if last_update else 0 : + ]: + match = LOG_RX.search(line) + if not match: + continue + date = match.group("date") + level = match.group("level") - temp_multiple_lines = [ - f"{datetime.strptime(' '.join(line.strip().split(' ')[0:2]), '%Y/%m/%d %H:%M:%S').replace(tzinfo=timezone.utc).timestamp()} {line}" - ] - elif ( - all(f"[{log_level}]" not in line_lower for log_level in NGINX_LOG_LEVELS) - and temp_multiple_lines - ): - temp_multiple_lines.append(line) - else: - logs_error.append( - f"{datetime.strptime(' '.join(line.strip().split(' ')[0:2]), '%Y/%m/%d %H:%M:%S').replace(tzinfo=timezone.utc).timestamp()} {line}" - ) + if not date: + if logs_error: + logs_error[-1] += f"\n{line}" + continue + logs_error.append(line) + elif ( + all(f"[{log_level}]" != level for log_level in NGINX_LOG_LEVELS) + and temp_multiple_lines + ): + temp_multiple_lines.append(line) + else: + logs_error.append( + f"{datetime.strptime(date, '%Y/%m/%d %H:%M:%S').replace(tzinfo=timezone.utc).timestamp()} {line}" + ) if temp_multiple_lines: logs_error.append("\n".join(temp_multiple_lines)) - logs_access = [ - f"{datetime.strptime(line[line.find('[') + 1: line.find(']')], '%d/%b/%Y:%H:%M:%S %z').timestamp()} {line}" - for line in raw_logs_access - ] + logs_access = [] + nginx_access_file = Path(sep, "var", "log", "bunkerweb", "access.log") + if nginx_access_file.is_file(): + with open(nginx_access_file, encoding="utf-8") as f: + for line in f.readlines()[ + int(last_update.split(".")[1]) if last_update else 0 : + ]: + logs_access.append( + f"{datetime.strptime(line[line.find('[') + 1: line.find(']')], '%d/%b/%Y:%H:%M:%S %z').replace(tzinfo=timezone.utc).timestamp()} {line}" + ) + raw_logs = logs_error + logs_access - raw_logs.sort( - key=lambda x: float(x.split(" ")[0]) if x.split(" ")[0].isdigit() else 0 - ) + + if from_date and from_date.isdigit(): + from_date = int(from_date) // 1000 + else: + from_date = 0 + + if to_date and to_date.isdigit(): + to_date = int(to_date) // 1000 + else: + to_date = None + + def date_filter(log: str): + log_date = log.split()[0] + log_date = float(log_date) if regex_match(r"^\d+\.\d+$", log_date) else 0 + if to_date is not None and log_date > int(to_date): + return False + return log_date > from_date logs = [] - for log in raw_logs: + for log in filter(date_filter, raw_logs): if "[48;2" in log or not log.strip(): continue @@ -1289,7 +1305,7 @@ def logs_linux(): logs.append( { - "content": " ".join(log.strip().split(" ")[1:]), + "content": " ".join(log.strip().split()[1:]), "type": error_type, } ) @@ -1393,7 +1409,7 @@ def logs_container(container_id): ) for log in tmp_logs: - splitted = log.split(" ") + splitted = log.split() timestamp = splitted[0] if to_date is not None and dateutil_parse(timestamp).timestamp() > to_date: From d1dd1fbae75ea502ccc175b1618aaf34b0aba553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Wed, 6 Sep 2023 10:04:47 +0200 Subject: [PATCH 4/6] Fix shinanigans with the /data volume in the doc --- docs/integrations.md | 2 +- docs/quickstart-guide.md | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/integrations.md b/docs/integrations.md index 5dc0c78e8..136f72393 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -102,7 +102,7 @@ A volume is needed to store the SQLite database that will be used by the schedul ```yaml ... services: - mybunker: + bw-scheduler: image: bunkerity/bunkerweb-scheduler:1.5.1 volumes: - bw-data:/data diff --git a/docs/quickstart-guide.md b/docs/quickstart-guide.md index fb721ade2..cce84308c 100644 --- a/docs/quickstart-guide.md +++ b/docs/quickstart-guide.md @@ -1475,7 +1475,7 @@ Some integrations provide more convenient ways to apply configurations, such as default_type 'text/plain'; content_by_lua_block { ngx.say('world') - } + } } ... ``` @@ -1509,8 +1509,8 @@ Some integrations provide more convenient ways to apply configurations, such as When starting the scheduler container, you will need to mount the folder on /data : ```yaml - mybunker: - image: bunkerity/bunkerweb:1.5.1 + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.1 volumes: - ./bw-data:/data ... @@ -1545,7 +1545,8 @@ Some integrations provide more convenient ways to apply configurations, such as default_type 'text/plain'; content_by_lua_block { ngx.say('world') - } + } + } ... ``` @@ -1578,7 +1579,7 @@ Some integrations provide more convenient ways to apply configurations, such as When starting the scheduler container, you will need to mount the folder on /data : ```yaml - myautoconf: + bw-scheduler: image: bunkerity/bunkerweb-scheduler:1.5.1 volumes: - ./bw-data:/data From 947690af8f193d26e26f108930c6f249576770d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Wed, 6 Sep 2023 10:13:46 +0200 Subject: [PATCH 5/6] Fix UI workflow --- .github/workflows/ui.yml | 52 ++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 3d9b80c20..5850e6f7b 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -7,51 +7,35 @@ on: branches: [ui] jobs: - # Containers - build-bw: + build-containers: + permissions: + contents: read + packages: write + strategy: + matrix: + image: [bunkerweb, scheduler, ui] + include: + - image: bunkerweb + dockerfile: src/bw/Dockerfile + - image: scheduler + dockerfile: src/scheduler/Dockerfile + - image: ui + dockerfile: src/ui/Dockerfile uses: ./.github/workflows/container-build.yml with: RELEASE: ui + CACHE: true ARCH: linux/amd64 - IMAGE: bunkerweb - DOCKERFILE: src/bw/Dockerfile + IMAGE: ${{ matrix.image }} + DOCKERFILE: ${{ matrix.dockerfile }} secrets: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} - PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }} - PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - build-sc: - uses: ./.github/workflows/container-build.yml - with: - RELEASE: ui - ARCH: linux/amd64 - IMAGE: scheduler - DOCKERFILE: src/scheduler/Dockerfile - secrets: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} - PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }} - PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} - build-ui: - uses: ./.github/workflows/container-build.yml - with: - RELEASE: ui - ARCH: linux/amd64 - IMAGE: ui - DOCKERFILE: src/ui/Dockerfile - secrets: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} - PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }} - PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} # UI tests tests-ui: - needs: [build-bw, build-sc, build-ui] + needs: [build-containers] uses: ./.github/workflows/tests-ui.yml with: RELEASE: ui - secrets: - PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }} - PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} From 55ba29cd5408869eb72759063e5902bb0902fa16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Wed, 6 Sep 2023 10:25:56 +0200 Subject: [PATCH 6/6] Fix UI error when values are empty --- src/ui/main.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/ui/main.py b/src/ui/main.py index b260dec81..1dc9a6731 100755 --- a/src/ui/main.py +++ b/src/ui/main.py @@ -532,8 +532,8 @@ def services(): args=( "services", variables, - request.form.get("OLD_SERVER_NAME", "").split()[0], - variables.get("SERVER_NAME", "").split()[0], + request.form.get("OLD_SERVER_NAME", "").split(" ")[0], + variables.get("SERVER_NAME", "").split(" ")[0], ), kwargs={"operation": request.form["operation"]}, ).start() @@ -541,11 +541,11 @@ def services(): message = "" if request.form["operation"] == "new": - message = f"Creating service {variables['SERVER_NAME'].split()[0]}" + message = f"Creating service {variables.get('SERVER_NAME', '').split(' ')[0]}" elif request.form["operation"] == "edit": - message = f"Saving configuration for service {request.form['OLD_SERVER_NAME'].split()[0]}" + message = f"Saving configuration for service {request.form.get('OLD_SERVER_NAME', '').split(' ')[0]}" elif request.form["operation"] == "delete": - message = f"Deleting service {request.form['SERVER_NAME'].split()[0]}" + message = f"Deleting service {request.form.get('SERVER_NAME', '').split(' ')[0]}" return redirect(url_for("loading", next=url_for("services"), message=message)) @@ -556,7 +556,7 @@ def services(): services=[ { "SERVER_NAME": { - "value": service["SERVER_NAME"]["value"].split()[0], + "value": service["SERVER_NAME"]["value"].split(" ")[0], "method": service["SERVER_NAME"]["method"], }, "USE_REVERSE_PROXY": service["USE_REVERSE_PROXY"], @@ -728,8 +728,8 @@ def configs(): join(sep, "etc", "bunkerweb", "configs"), db_data=db.get_custom_configs(), services=app.config["CONFIG"] - .get_config(methods=False)["SERVER_NAME"] - .split(), + .get_config(methods=False).get("SERVER_NAME", "") + .split(" "), ) ], dark_mode=app.config["DARK_MODE"], @@ -1174,8 +1174,8 @@ def cache(): is_cache=True, db_data=db.get_jobs_cache_files(), services=app.config["CONFIG"] - .get_config(methods=False)["SERVER_NAME"] - .split(), + .get_config(methods=False).get("SERVER_NAME", "") + .split(" "), ) ], dark_mode=app.config["DARK_MODE"], @@ -1276,7 +1276,7 @@ def logs_linux(): to_date = None def date_filter(log: str): - log_date = log.split()[0] + log_date = log.split(" ")[0] log_date = float(log_date) if regex_match(r"^\d+\.\d+$", log_date) else 0 if to_date is not None and log_date > int(to_date): return False @@ -1305,7 +1305,7 @@ def logs_linux(): logs.append( { - "content": " ".join(log.strip().split()[1:]), + "content": " ".join(log.strip().split(" ")[1:]), "type": error_type, } ) @@ -1409,7 +1409,7 @@ def logs_container(container_id): ) for log in tmp_logs: - splitted = log.split() + splitted = log.split(" ") timestamp = splitted[0] if to_date is not None and dateutil_parse(timestamp).timestamp() > to_date: