Merge pull request #837 from bunkerity/dev

Merge branch "dev" into branch "ui"

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
-exclude: (^LICENSE.md$|^src/VERSION$|^src/(bw/misc/root-ca.pem$|deps/src/|common/core/modsecurity/files|ui/static/js/(editor/|utils/purify/|tsparticles\.bundle\.min\.js))|\.(svg|drawio|patch\d?|ascii|tf|tftpl)$)
+exclude: (^LICENSE.md$|^src/VERSION$|^env/|^src/(bw/misc/root-ca.pem$|deps/src/|common/core/modsecurity/files|ui/static/js/(editor/|utils/purify/|tsparticles\.bundle\.min\.js))|\.(svg|drawio|patch\d?|ascii|tf|tftpl|key)$)
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: c4a0b883114b00d8d76b479c820ce7950211c99b # frozen: v4.5.0
@@ -16,8 +16,8 @@ repos:
         args: ["--allow-multiple-documents"]
       - id: check-case-conflict
 
-  - repo: https://github.com/ambv/black
-    rev: 2a1c67e0b2f81df602ec1f6e7aeb030b9709dc7c # frozen: 23.11.0
+  - repo: https://github.com/psf/black
+    rev: ec91a2be3c44d88e1a3960a4937ad6ed3b63464e # frozen: 23.12.1
     hooks:
       - id: black
         name: Black Python Formatter
@@ -36,21 +36,21 @@ repos:
         exclude: ^src/(bw/lua/middleclass.lua|common/core/antibot/captcha.lua)$
 
   - repo: https://github.com/lunarmodules/luacheck
-    rev: ababb6d403d634eb74d2c541035e9ede966e710d # frozen: v1.1.1
+    rev: 418f48976c73be697fe64b0eba9ea9821ac9bca8 # frozen: v1.1.2
     hooks:
       - id: luacheck
         exclude: ^src/(bw/lua/middleclass.lua|common/core/antibot/captcha.lua)$
         args: ["--std", "min", "--codes", "--ranges", "--no-cache"]
 
   - repo: https://github.com/pycqa/flake8
-    rev: 10f4af6dbcf93456ba7df762278ae61ba3120dc6 # frozen: 6.1.0
+    rev: 7d37d9032d0d161634be4554273c30efd4dea0b3 # frozen: 7.0.0
     hooks:
       - id: flake8
         name: Flake8 Python Linter
         args: ["--max-line-length=250", "--ignore=E266,E402,E722,W503"]
 
   - repo: https://github.com/dosisod/refurb
-    rev: a25b5d6087bba1509f96654c583efcd7796452cd # frozen: v1.24.0
+    rev: a7c461fcfaa2ca3248d489cdf7fed8e2d4fd8520 # frozen: v1.26.0
     hooks:
       - id: refurb
         name: Refurb Python Refactoring Tool

docs/json2md.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from io import StringIO
 from json import loads

docs/settings.md

@@ -243,7 +243,7 @@ Manage default error pages
 |         Setting         |                     Default                     | Context |Multiple|                                                      Description                                                       |
 |-------------------------|-------------------------------------------------|---------|--------|------------------------------------------------------------------------------------------------------------------------|
 |`ERRORS`                 |                                                 |multisite|no      |List of HTTP error code and corresponding error pages, separated with spaces (404=/my404.html 403=/errors/403.html ...).|
-|`INTERCEPTED_ERROR_CODES`|`400 401 403 404 405 413 429 500 501 502 503 504`|multisite|no      |List of HTTP error code intercepted by Bunkerweb                                                                        |
+|`INTERCEPTED_ERROR_CODES`|`400 401 403 404 405 413 429 500 501 502 503 504`|multisite|no      |List of HTTP error code intercepted by BunkerWeb                                                                        |
 
 ### Greylist
 
@@ -550,4 +550,3 @@ Allow access based on internal and external IP/network/rDNS/ASN whitelists.
 |`WHITELIST_USER_AGENT_URLS`|                                                                                                                                                                            |global   |no      |List of URLs, separated with spaces, containing good User-Agent to whitelist.     |
 |`WHITELIST_URI`            |                                                                                                                                                                            |multisite|no      |List of URI (PCRE regex), separated with spaces, to whitelist.                    |
 |`WHITELIST_URI_URLS`       |                                                                                                                                                                            |global   |no      |List of URLs, separated with spaces, containing bad URI to whitelist.             |
-

docs/web-ui.md

@@ -44,7 +44,7 @@ The setup wizard is a feature that helps you to **configure** and **install the
   <figcaption>Account section of the setup wizard</figcaption>
 </figure>
 
-Choose your administator username and password. Please note that password must have at least 8 chars with 1 lower case letter, 1 upper case letter, 1 digit and 1 special char.
+Choose your administrator username and password. Please note that password must have at least 8 chars with 1 lower case letter, 1 upper case letter, 1 digit and 1 special char.
 
 <figure markdown>
   ![Overview](assets/img/ui-wizard-settings.webp){ align=center, width="350" }

examples/behind-reverse-proxy/setup-linux.sh

@@ -24,6 +24,7 @@ if [ -f /lib/systemd/system/haproxy.service ] ; then
 	systemctl daemon-reload
 fi
 systemctl start haproxy
+# shellcheck disable=SC2181
 if [ $? -ne 0 ] ; then
 	systemctl status haproxy
 	journalctl -u haproxy.service

src/autoconf/Config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import getenv
 from time import sleep

src/autoconf/Controller.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from abc import abstractmethod
 from os import getenv

src/autoconf/DockerController.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from typing import Any, Dict, List
 from docker import DockerClient

src/autoconf/IngressController.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from time import sleep
 from traceback import format_exc

src/autoconf/SwarmController.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from time import sleep
 from traceback import format_exc

src/autoconf/main.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import _exit, getenv, sep
 from os.path import join

src/bw/lua/bunkerweb/api.lua

@@ -1,14 +1,14 @@
 local ngx = ngx
 local ngx_req = ngx.req
+local cdatastore = require "bunkerweb.datastore"
 local cjson = require "cjson"
 local class = require "middleclass"
-local cdatastore = require "bunkerweb.datastore"
 local clogger = require "bunkerweb.logger"
+local helpers = require "bunkerweb.helpers"
 local process = require "ngx.process"
 local rsignal = require "resty.signal"
 local upload = require "resty.upload"
 local utils = require "bunkerweb.utils"
-local helpers = require "bunkerweb.helpers"
 
 local api = class("api")
 
@@ -287,13 +287,14 @@ function api:do_api_call()
 		return false, resp["msg"], HTTP_INTERNAL_SERVER_ERROR, encode(resp)
 	end
 	for _, plugin in ipairs(list) do
-		local plugin_lua, err = require_plugin(plugin.id)
+		local plugin_lua, _ = require_plugin(plugin.id)
 		if plugin_lua and plugin_lua.api ~= nil then
 			local ok, plugin_obj = new_plugin(plugin_lua, self.ctx)
 			if not ok then
 				logger:log(ERR, "can't instantiate " .. plugin.id .. " : " .. plugin_obj)
 			else
-				local ok, ret = call_plugin(plugin_obj, "api")
+				local ret
+				ok, ret = call_plugin(plugin_obj, "api")
 				if not ok then
 					logger:log(ERR, "error while executing " .. plugin.id .. ":api() : " .. ret)
 				else

src/bw/lua/bunkerweb/cachestore.lua

@@ -1,7 +1,7 @@
 local ngx = ngx
 local class = require "middleclass"
-local clusterstore = require "bunkerweb.clusterstore"
 local clogger = require "bunkerweb.logger"
+local clusterstore = require "bunkerweb.clusterstore"
 local mlcache = require "resty.mlcache"
 local utils = require "bunkerweb.utils"
 local cachestore = class("cachestore")
@@ -184,10 +184,12 @@ function cachestore:del_redis(key)
 	return true
 end
 
+-- luacheck: ignore 212
 function cachestore:purge()
 	return cache:purge(true)
 end
 
+-- luacheck: ignore 212
 function cachestore:update()
 	return cache:update()
 end

src/bw/lua/bunkerweb/clusterstore.lua

@@ -33,7 +33,7 @@ function clusterstore:initialize(pool)
 		["REDIS_SENTINEL_HOSTS"] = "",
 		["REDIS_SENTINEL_USERNAME"] = "",
 		["REDIS_SENTINEL_PASSWORD"] = "",
-		["REDIS_SENTINEL_MASTER"] = ""
+		["REDIS_SENTINEL_MASTER"] = "",
 	}
 	-- Set them for later use
 	self.variables = {}
@@ -57,7 +57,6 @@ function clusterstore:initialize(pool)
 		keepalive_poolsize = tonumber(self.variables["REDIS_KEEPALIVE_POOL"]),
 		connection_options = {
 			ssl = self.variables["REDIS_SSL"] == "yes",
-			
 		},
 		host = self.variables["REDIS_HOST"],
 		port = tonumber(self.variables["REDIS_PORT"]),
@@ -68,7 +67,7 @@ function clusterstore:initialize(pool)
 		sentinel_password = self.variables["REDIS_SENTINEL_PASSWORD"],
 		master_name = self.variables["REDIS_SENTINEL_MASTER"],
 		role = "master",
-		sentinels = {}
+		sentinels = {},
 	}
 	self.pool = pool == nil or pool
 	if self.pool then
@@ -83,7 +82,7 @@ function clusterstore:initialize(pool)
 			else
 				sport = tonumber(sport)
 			end
-			table.insert(options.sentinel, {host = shost, port = sport})
+			table.insert(options.sentinel, { host = shost, port = sport })
 		end
 	end
 	self.options = options
@@ -110,12 +109,13 @@ function clusterstore:connect(readonly)
 	-- Connect to sentinels if needed
 	local redis_client, err
 	if #self.options.sentinels > 0 then
-		local redis_sentinel, err = self.redis_connector:connect()
+		local redis_sentinel
+		redis_sentinel, err = self.redis_connector:connect()
 		if not redis_sentinel then
 			return false, "error while connecting to sentinels : " .. err
 		end
 		if readonly then
-			local redis_clients, err = rs.get_slaves(redis_sentinel, self.options.master_name)
+			local redis_clients, _ = rs.get_slaves(redis_sentinel, self.options.master_name)
 			if redis_clients then
 				redis_client = redis_clients[random(#redis_clients)]
 			else
@@ -124,7 +124,7 @@ function clusterstore:connect(readonly)
 		else
 			redis_client, err = rs.get_master(redis_sentinel, self.options.master_name)
 		end
-	-- Classic connection
+		-- Classic connection
 	else
 		redis_client, err = self.redis_connector:connect()
 	end
@@ -155,7 +155,7 @@ function clusterstore:close()
 	local ok, err
 	if self.pool then
 		ok, err = self.redis_connector:set_keepalive(self.redis_client)
-	-- No pool
+		-- No pool
 	else
 		ok, err = self.redis_client:close()
 	end

src/bw/lua/bunkerweb/ctx.lua

@@ -13,16 +13,13 @@
 --   * allow passing of context
 --   * updated to work with new 1.19.x apis
 
-local ffi = require "ffi"
 local base = require "resty.core.base"
+local ffi = require "ffi"
 require "resty.core.ctx"
 
-
 local C = ffi.C
 local ngx = ngx
 local var = ngx.var
-local ngx_log = ngx.log
-local ngx_WARN = ngx.WARN
 local tonumber = tonumber
 local registry = debug.getregistry()
 local subsystem = ngx.config.subsystem
@@ -34,75 +31,69 @@ local ngx_ERR = ngx.ERR
 
 local ngx_lua_ffi_get_ctx_ref
 if subsystem == "http" then
-  ngx_lua_ffi_get_ctx_ref = C.ngx_http_lua_ffi_get_ctx_ref
+	ngx_lua_ffi_get_ctx_ref = C.ngx_http_lua_ffi_get_ctx_ref
 elseif subsystem == "stream" then
-  ngx_lua_ffi_get_ctx_ref = C.ngx_stream_lua_ffi_get_ctx_ref
+	ngx_lua_ffi_get_ctx_ref = C.ngx_stream_lua_ffi_get_ctx_ref
 end
 
-
 local in_ssl_phase = ffi.new("int[1]")
 local ssl_ctx_ref = ffi.new("int[1]")
 
-
 local FFI_NO_REQ_CTX = base.FFI_NO_REQ_CTX
 
-
 local _M = {}
 
-
 function _M.stash_ref(ctx)
-  local r = get_request()
-  if not r then
-    logger:log(ngx_ERR, "could not stash ngx.ctx ref: no request found")
-    return
-  end
+	local r = get_request()
+	if not r then
+		logger:log(ngx_ERR, "could not stash ngx.ctx ref: no request found")
+		return
+	end
 
-  do
-    local ctx_ref = var.ctx_ref
-    if not ctx_ref or ctx_ref ~= "" then
-      return
-    end
+	do
+		local ctx_ref = var.ctx_ref
+		if not ctx_ref or ctx_ref ~= "" then
+			return
+		end
 
-    if not ctx then
-      local _ = ngx.ctx -- load context if not previously loaded
-    end
-  end
-  local ctx_ref = ngx_lua_ffi_get_ctx_ref(r, in_ssl_phase, ssl_ctx_ref)
-  if ctx_ref == FFI_NO_REQ_CTX then
-    logger:log(ngx_ERR, "could not stash ngx.ctx ref: no ctx found")
-    return
-  end
+		if not ctx then
+			local _ = ngx.ctx -- load context if not previously loaded
+		end
+	end
+	local ctx_ref = ngx_lua_ffi_get_ctx_ref(r, in_ssl_phase, ssl_ctx_ref)
+	if ctx_ref == FFI_NO_REQ_CTX then
+		logger:log(ngx_ERR, "could not stash ngx.ctx ref: no ctx found")
+		return
+	end
 
-  var.ctx_ref = ctx_ref
+	var.ctx_ref = ctx_ref
 end
 
-
 function _M.apply_ref()
-  local r = get_request()
-  if not r then
-    logger:log(ngx_ERR, "could not apply ngx.ctx: no request found")
-    return
-  end
+	local r = get_request()
+	if not r then
+		logger:log(ngx_ERR, "could not apply ngx.ctx: no request found")
+		return
+	end
 
-  local ctx_ref = var.ctx_ref
-  if not ctx_ref or ctx_ref == "" then
-    return
-  end
+	local ctx_ref = var.ctx_ref
+	if not ctx_ref or ctx_ref == "" then
+		return
+	end
 
-  ctx_ref = tonumber(ctx_ref)
-  if not ctx_ref then
-    return
-  end
+	ctx_ref = tonumber(ctx_ref)
+	if not ctx_ref then
+		return
+	end
 
-  local orig_ctx = registry.ngx_lua_ctx_tables[ctx_ref]
-  if not orig_ctx then
-    logger:log(ngx_ERR, "could not apply ngx.ctx: no ctx found")
-    return
-  end
+	local orig_ctx = registry.ngx_lua_ctx_tables[ctx_ref]
+	if not orig_ctx then
+		logger:log(ngx_ERR, "could not apply ngx.ctx: no ctx found")
+		return
+	end
 
-  ngx.ctx = orig_ctx
-  var.ctx_ref = ""
+	ngx.ctx = orig_ctx
+	var.ctx_ref = ""
 end
 
-
 return _M

src/bw/lua/bunkerweb/helpers.lua

@@ -1,8 +1,8 @@
-local ngx	= ngx
+local ngx = ngx
+local base = require "resty.core.base"
+local bwctx = require "bunkerweb.ctx"
 local cjson = require "cjson"
 local utils = require "bunkerweb.utils"
-local bwctx = require "bunkerweb.ctx"
-local base 	= require "resty.core.base"
 
 local open = io.open
 local decode = cjson.decode

src/bw/lua/bunkerweb/plugin.lua

@@ -26,7 +26,7 @@ function plugin:initialize(id, ctx)
 		"header_filter",
 		"body_filter",
 		"log",
-		"preread"
+		"preread",
 	} do
 		if current_phase == check_phase then
 			is_request = true
@@ -44,8 +44,7 @@ function plugin:initialize(id, ctx)
 	if self.is_request then
 		self.ctx = ctx or ngx.ctx
 		self.datastore = get_ctx_obj("datastore", self.ctx) or datastore:new()
-		self.cachestore = get_ctx_obj("cachestore", self.ctx)
-			or cachestore:new(use_redis == "yes", self.ctx)
+		self.cachestore = get_ctx_obj("cachestore", self.ctx) or cachestore:new(use_redis == "yes", self.ctx)
 		self.clusterstore = get_ctx_obj("clusterstore", self.ctx) or clusterstore:new()
 		self.cachestore_local = get_ctx_obj("cachestore_local", self.ctx) or cachestore:new(false, self.ctx)
 	else

src/bw/lua/bunkerweb/utils.lua

@@ -769,7 +769,7 @@ utils.get_phases = function()
 		"log",
 		"preread",
 		"log_stream",
-		"log_default"
+		"log_default",
 	}
 end
 
@@ -780,7 +780,7 @@ utils.is_cosocket_available = function()
 		"access",
 		"content",
 		"ssl_certificate",
-		"preread"
+		"preread",
 	}
 	local current_phase = get_phase()
 	for _, phase in ipairs(phases) do
@@ -808,4 +808,17 @@ utils.get_ctx_obj = function(obj, ctx)
 	return nil
 end
 
+utils.read_files = function(files)
+	local data = {}
+	for _, file in ipairs(files) do
+		local f, err = open(file, "r")
+		if not f then
+			return false, file .. " = " .. err
+		end
+		table.insert(data, f:read("*a"))
+		f:close()
+	end
+	return true, data
+end
+
 return utils

src/common/api/API.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from typing import Literal, Optional, Union
 from requests import request

src/common/cli/CLI.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from dotenv import dotenv_values
 from os import getenv, sep
@@ -6,7 +6,7 @@ from os.path import join
 from pathlib import Path
 from redis import StrictRedis
 from sys import path as sys_path
-from typing import Tuple
+from typing import Optional, Tuple
 
 
 for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("utils",), ("db",))]:
@@ -15,7 +15,6 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
 
 from API import API  # type: ignore
 from ApiCaller import ApiCaller  # type: ignore
-from Database import Database  # type: ignore
 from logger import setup_logger  # type: ignore
 
 
@@ -47,28 +46,35 @@ class CLI(ApiCaller):
         if variables_path.is_file():
             self.__variables = dotenv_values(variables_path)
 
-        db = Database(self.__logger, sqlalchemy_string=self.__variables.get("DATABASE_URI", None))
-        self.__variables = db.get_config()
+        if Path(sep, "usr", "share", "bunkerweb", "db").exists():
+            from Database import Database  # type: ignore
+
+            db = Database(self.__logger, sqlalchemy_string=self.__get_variable("DATABASE_URI", None))
+            self.__variables = db.get_config()
+
+        assert isinstance(self.__variables, dict), "Failed to get variables from database"
 
         self.__integration = self.__detect_integration()
-        self.__use_redis = self.__variables.get("USE_REDIS", "no") == "yes"
+        self.__use_redis = self.__get_variable("USE_REDIS", "no") == "yes"
         self.__redis = None
         if self.__use_redis:
-            redis_host = self.__variables.get("REDIS_HOST")
+            redis_host = self.__get_variable("REDIS_HOST")
             if redis_host:
-                redis_port = self.__variables.get("REDIS_PORT", "6379")
+                redis_port = self.__get_variable("REDIS_PORT", "6379")
+                assert isinstance(redis_port, str), "REDIS_PORT is not a string"
                 if not redis_port.isdigit():
                     self.__logger.error(f"REDIS_PORT is not a valid port number: {redis_port}, defaulting to 6379")
                     redis_port = "6379"
                 redis_port = int(redis_port)
 
-                redis_db = self.__variables.get("REDIS_DB", "0")
+                redis_db = self.__get_variable("REDIS_DB", "0")
+                assert isinstance(redis_db, str), "REDIS_DB is not a string"
                 if not redis_db.isdigit():
                     self.__logger.error(f"REDIS_DB is not a valid database number: {redis_db}, defaulting to 0")
                     redis_db = "0"
                 redis_db = int(redis_db)
 
-                redis_timeout = self.__variables.get("REDIS_TIMEOUT", "1000.0")
+                redis_timeout = self.__get_variable("REDIS_TIMEOUT", "1000.0")
                 if redis_timeout:
                     try:
                         redis_timeout = float(redis_timeout)
@@ -76,7 +82,8 @@ class CLI(ApiCaller):
                         self.__logger.error(f"REDIS_TIMEOUT is not a valid timeout: {redis_timeout}, defaulting to 1000 ms")
                         redis_timeout = 1000.0
 
-                redis_keepalive_pool = self.__variables.get("REDIS_KEEPALIVE_POOL", "10")
+                redis_keepalive_pool = self.__get_variable("REDIS_KEEPALIVE_POOL", "10")
+                assert isinstance(redis_keepalive_pool, str), "REDIS_KEEPALIVE_POOL is not a string"
                 if not redis_keepalive_pool.isdigit():
                     self.__logger.error(f"REDIS_KEEPALIVE_POOL is not a valid number of connections: {redis_keepalive_pool}, defaulting to 10")
                     redis_keepalive_pool = "10"
@@ -90,7 +97,7 @@ class CLI(ApiCaller):
                     socket_connect_timeout=redis_timeout,
                     socket_keepalive=True,
                     max_connections=redis_keepalive_pool,
-                    ssl=self.__variables.get("REDIS_SSL", "no") == "yes",
+                    ssl=self.__get_variable("REDIS_SSL", "no") == "yes",
                 )
             else:
                 self.__logger.error("USE_REDIS is set to yes but REDIS_HOST is not set, disabling redis")
@@ -100,8 +107,8 @@ class CLI(ApiCaller):
             super().__init__(
                 [
                     API(
-                        f"http://127.0.0.1:{self.__variables.get('API_HTTP_PORT', '5000')}",
-                        host=self.__variables.get("API_SERVER_NAME", "bwapi"),
+                        f"http://127.0.0.1:{self.__get_variable('API_HTTP_PORT', '5000')}",
+                        host=self.__get_variable("API_SERVER_NAME", "bwapi"),
                     )
                 ]
             )
@@ -109,14 +116,20 @@ class CLI(ApiCaller):
             super().__init__()
             self.auto_setup(self.__integration)
 
+    def __get_variable(self, variable: str, default: Optional[str] = None) -> Optional[str]:
+        return getenv(variable, self.__variables.get(variable, default))
+
     def __detect_integration(self) -> str:
+        if Path(sep, "usr", "sbin", "nginx").exists():
+            return "linux"
+
         integration_path = Path(sep, "usr", "share", "bunkerweb", "INTEGRATION")
         os_release_path = Path(sep, "etc", "os-release")
-        if self.__variables.get("KUBERNETES_MODE", "no").lower() == "yes":
+        if self.__get_variable("KUBERNETES_MODE", "no").lower() == "yes":  # type: ignore
             return "kubernetes"
-        elif self.__variables.get("SWARM_MODE", "no").lower() == "yes":
+        elif self.__get_variable("SWARM_MODE", "no").lower() == "yes":  # type: ignore
             return "swarm"
-        elif self.__variables.get("AUTOCONF_MODE", "no").lower() == "yes":
+        elif self.__get_variable("AUTOCONF_MODE", "no").lower() == "yes":  # type: ignore
             return "autoconf"
         elif integration_path.is_file():
             return integration_path.read_text(encoding="utf-8").strip().lower()
@@ -137,19 +150,12 @@ class CLI(ApiCaller):
 
     def ban(self, ip: str, exp: float) -> Tuple[bool, str]:
         if self.__redis:
-            ok = self.__redis.set(
-                f"bans_ip_{ip}",
-                "manual",
-                ex=exp,
-            )
+            ok = self.__redis.set(f"bans_ip_{ip}", "manual", ex=exp)
             if not ok:
                 self.__logger.error(f"Failed to ban {ip} in redis")
 
         if self.send_to_apis("POST", "/ban", data={"ip": ip, "exp": exp}):
-            return (
-                True,
-                f"IP {ip} has been banned for {format_remaining_time(exp)}",
-            )
+            return (True, f"IP {ip} has been banned for {format_remaining_time(exp)}")
         return False, "error"
 
     def bans(self) -> Tuple[bool, str]:
@@ -167,13 +173,7 @@ class CLI(ApiCaller):
             for key in self.__redis.scan_iter("bans_ip_*"):
                 ip = key.decode("utf-8").replace("bans_ip_", "")
                 exp = self.__redis.ttl(key)
-                servers["redis"].append(
-                    {
-                        "ip": ip,
-                        "exp": exp,
-                        "reason": "manual",
-                    }
-                )
+                servers["redis"].append({"ip": ip, "exp": exp, "reason": "manual"})
 
         cli_str = ""
         for server, bans in servers.items():

src/common/cli/main.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from argparse import ArgumentParser
 from os import _exit, getenv, sep

src/common/confs/api.conf

@@ -18,7 +18,7 @@ server {
 		-- Instantiate objects and import required modules
 		local logger	= require "bunkerweb.logger":new("API")
 		local helpers	= require "bunkerweb.helpers"
-		
+
 		local ngx = ngx
 		local INFO = ngx.INFO
 		local ERR = ngx.ERR

src/common/core/antibot/antibot.lua

@@ -26,7 +26,7 @@ local to_hex = str.to_hex
 local http_new = http.new
 local decode = cjson.decode
 
-local template = nil
+local template
 local render = nil
 if subsystem == "http" then
 	template = require "resty.template"

src/common/core/badbehavior/badbehavior.lua

@@ -65,7 +65,7 @@ function badbehavior.increase(premature, ip, count_time, ban_time, threshold, us
 	-- Instantiate objects
 	local logger = require "bunkerweb.logger":new("badbehavior")
 	local datastore = require "bunkerweb.datastore":new()
-	
+
 	-- Declare counter
 	local counter = false
 	-- Redis case

src/common/core/blacklist/blacklist.lua

@@ -348,6 +348,7 @@ function blacklist:is_blacklisted_ua()
 	return false, "ok"
 end
 
+-- luacheck: ignore 212
 function blacklist:get_data(blacklisted)
 	local data = {}
 	if blacklisted == "ip" then

src/common/core/blacklist/jobs/blacklist-download.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from contextlib import suppress
 from ipaddress import ip_address, ip_network

src/common/core/bunkernet/bunkernet.lua

@@ -199,9 +199,11 @@ function bunkernet:log(bypass_checks)
 	elseif data then
 		return self:ret(true, "already reported recently")
 	end
+	local ok, err
 	-- luacheck: ignore 212 431
 	local function report_callback(premature, obj, ip, reason, reason_data, method, url, headers, use_redis)
-		local ok, err, status, _ = obj:report(ip, reason, reason_data, method, url, headers)
+		local status, _
+		ok, err, status, _ = obj:report(ip, reason, reason_data, method, url, headers)
 		if status == 429 then
 			obj.logger:log(WARN, "bunkernet API is rate limiting us")
 		elseif not ok then

src/common/core/bunkernet/jobs/bunkernet-data.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import _exit, getenv, sep
 from os.path import join

src/common/core/bunkernet/jobs/bunkernet-register.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import _exit, getenv, sep
 from os.path import join

src/common/core/bunkernet/jobs/bunkernet.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import getenv, sep
 from pathlib import Path

src/common/core/cors/cors.lua

@@ -106,7 +106,7 @@ function cors:access()
 			nil,
 			{
 				id = "origin",
-				origin = self.ctx.bw.http_origin
+				origin = self.ctx.bw.http_origin,
 			}
 		)
 	end

src/common/core/country/country.lua

@@ -5,7 +5,6 @@ local utils = require "bunkerweb.utils"
 
 local country = class("country", plugin)
 
-local ngx = ngx
 local get_country = utils.get_country
 local get_deny_status = utils.get_deny_status
 local decode = cjson.decode
@@ -46,7 +45,7 @@ function country:access()
 			nil,
 			{
 				id = "country",
-				country = data.country
+				country = data.country,
 			}
 		)
 	end
@@ -93,7 +92,7 @@ function country:access()
 			nil,
 			{
 				id = "country",
-				country = country_data
+				country = country_data,
 			}
 		)
 	end
@@ -113,7 +112,7 @@ function country:access()
 					nil,
 					{
 						id = "country",
-						country = country_data
+						country = country_data,
 					}
 				)
 			end

src/common/core/customcert/customcert.lua

@@ -1,7 +1,7 @@
 local class = require "middleclass"
 local plugin = require "bunkerweb.plugin"
-local utils = require "bunkerweb.utils"
 local ssl = require "ngx.ssl"
+local utils = require "bunkerweb.utils"
 
 local customcert = class("customcert", plugin)
 
@@ -13,7 +13,7 @@ local ssl_server_name = ssl.server_name
 local get_variable = utils.get_variable
 local get_multiple_variables = utils.get_multiple_variables
 local has_variable = utils.has_variable
-local open = io.open
+local read_files = utils.read_files
 
 function customcert:initialize(ctx)
 	-- Call parent initialize
@@ -22,25 +22,29 @@ end
 
 function customcert:init()
 	local ret_ok, ret_err = true, "success"
-    if has_variable("USE_CUSTOM_SSL", "yes") then
+	if has_variable("USE_CUSTOM_SSL", "yes") then
 		local multisite, err = get_variable("MULTISITE", false)
 		if not multisite then
 			return self:ret(false, "can't get MULTISITE variable : " .. err)
 		end
 		if multisite == "yes" then
-			local vars, err = get_multiple_variables({"USE_CUSTOM_SSL", "SERVER_NAME"})
+			local vars
+			vars, err = get_multiple_variables({ "USE_CUSTOM_SSL", "SERVER_NAME" })
 			if not vars then
 				return self:ret(false, "can't get USE_CUSTOM_SSL variables : " .. err)
 			end
 			for server_name, multisite_vars in pairs(vars) do
 				if multisite_vars["USE_CUSTOM_SSL"] == "yes" and server_name ~= "global" then
-					local check, data = self:read_files(server_name)
+					local check, data = read_files({
+						"/var/cache/bunkerweb/customcert/" .. server_name .. "/cert.pem",
+						"/var/cache/bunkerweb/customcert/" .. server_name .. "/key.pem",
+					})
 					if not check then
 						self.logger:log(ERR, "error while reading files : " .. data)
 						ret_ok = false
 						ret_err = "error reading files"
 					else
-						local check, err = self:load_data(data, multisite_vars["SERVER_NAME"])
+						check, err = self:load_data(data, multisite_vars["SERVER_NAME"])
 						if not check then
 							self.logger:log(ERR, "error while loading data : " .. err)
 							ret_ok = false
@@ -50,17 +54,21 @@ function customcert:init()
 				end
 			end
 		else
-			local server_name, err = get_variable("SERVER_NAME", false)
+			local server_name
+			server_name, err = get_variable("SERVER_NAME", false)
 			if not server_name then
 				return self:ret(false, "can't get SERVER_NAME variable : " .. err)
 			end
-			local check, data = self:read_files(server_name:match("%S+"))
+			local check, data = read_files({
+				"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/cert.pem",
+				"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/key.pem",
+			})
 			if not check then
 				self.logger:log(ERR, "error while reading files : " .. data)
 				ret_ok = false
 				ret_err = "error reading files"
 			else
-				local check, err = self:load_data(data, server_name)
+				check, err = self:load_data(data, server_name)
 				if not check then
 					self.logger:log(ERR, "error while loading data : " .. err)
 					ret_ok = false
@@ -70,7 +78,7 @@ function customcert:init()
 		end
 	else
 		ret_err = "custom cert is not used"
-    end
+	end
 	return self:ret(ret_ok, ret_err)
 end
 
@@ -79,31 +87,18 @@ function customcert:ssl_certificate()
 	if not server_name then
 		return self:ret(false, "can't get server_name : " .. err)
 	end
-    if self.variables["USE_CUSTOM_SSL"] == "yes" then
-		local data, err = self.datastore:get("plugin_customcert_" .. server_name, true)
+	if self.variables["USE_CUSTOM_SSL"] == "yes" then
+		local data
+		data, err = self.datastore:get("plugin_customcert_" .. server_name, true)
 		if not data then
-			return self:ret(false, "error while getting plugin_customcert_" .. server_name .. " from datastore : " .. err)
+			return self:ret(
+				false,
+				"error while getting plugin_customcert_" .. server_name .. " from datastore : " .. err
+			)
 		end
-        return self:ret(true, "certificate/key data found", data)
-    end
-    return self:ret(true, "custom certificate is not used")
-end
-
-function customcert:read_files(server_name)
-	local files = {
-		"/var/cache/bunkerweb/customcert/" .. server_name .. "/cert.pem",
-		"/var/cache/bunkerweb/customcert/" .. server_name .. "/key.pem"
-	}
-	local data = {}
-	for i, file in ipairs(files) do
-		local f, err = open(file, "r")
-		if not f then
-			return false, file .. " = " .. err
-		end
-		table.insert(data, f:read("*a"))
-		f:close()
+		return self:ret(true, "certificate/key data found", data)
 	end
-	return true, data
+	return self:ret(true, "custom certificate is not used")
 end
 
 function customcert:load_data(data, server_name)
@@ -120,7 +115,8 @@ function customcert:load_data(data, server_name)
 	-- Cache data
 	for key in server_name:gmatch("%S+") do
 		local cache_key = "plugin_customcert_" .. key
-		local ok, err = self.datastore:set(cache_key, {cert_chain, priv_key}, nil, true)
+		local ok
+		ok, err = self.datastore:set(cache_key, { cert_chain, priv_key }, nil, true)
 		if not ok then
 			return false, "error while setting data into datastore : " .. err
 		end

src/common/core/customcert/jobs/custom-cert.py

@@ -1,11 +1,10 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import getenv, sep
 from os.path import join, normpath
 from pathlib import Path
 from sys import exit as sys_exit, path as sys_path
 from traceback import format_exc
-from typing import Optional
 from base64 import b64decode
 
 for deps_path in [
@@ -103,7 +102,7 @@ try:
 
         cert_data = b64decode(getenv("CUSTOM_SSL_CERT_DATA", ""))
         key_data = b64decode(getenv("CUSTOM_SSL_KEY_DATA", ""))
-        for file, data in [("cert.pem", cert_data), ("key.pem", key_data)]:
+        for file, data in (("cert.pem", cert_data), ("key.pem", key_data)):
             if data != b"":
                 file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", first_server, file)
                 file_path.parent.mkdir(parents=True, exist_ok=True)
@@ -140,7 +139,7 @@ try:
 
             cert_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_CERT_DATA", ""))
             key_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_KEY_DATA", ""))
-            for file, data in [("cert.pem", cert_data), ("key.pem", key_data)]:
+            for file, data in (("cert.pem", cert_data), ("key.pem", key_data)):
                 if data != b"":
                     file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", first_server, file)
                     file_path.parent.mkdir(parents=True, exist_ok=True)

src/common/core/dnsbl/dnsbl.lua

@@ -98,7 +98,7 @@ function dnsbl:access()
 			nil,
 			{
 				id = "dnsbl",
-				dnsbl = cached
+				dnsbl = cached,
 			}
 		)
 	end
@@ -161,7 +161,13 @@ function dnsbl:access()
 			if not ok then
 				return self:ret(false, "error while adding element to cache : " .. err)
 			end
-			return self:ret(true, "IP is blacklisted by " .. ret_server, get_deny_status(), nil, {id = "dnsbl", dnsbl = ret_server})
+			return self:ret(
+				true,
+				"IP is blacklisted by " .. ret_server,
+				get_deny_status(),
+				nil,
+				{ id = "dnsbl", dnsbl = ret_server }
+			)
 		end
 		-- Error case
 		return self:ret(false, ret_err)

src/common/core/errors/errors.lua

@@ -4,7 +4,7 @@ local plugin = require "bunkerweb.plugin"
 local ngx = ngx
 local subsystem = ngx.config.subsystem
 
-local template = nil
+local template
 local render = nil
 if subsystem == "http" then
 	template = require "resty.template"

src/common/core/errors/files/error.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />

src/common/core/greylist/jobs/greylist-download.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from contextlib import suppress
 from ipaddress import ip_address, ip_network

src/common/core/jobs/jobs/download-plugins.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from hashlib import sha256
 from io import BytesIO

src/common/core/jobs/jobs/mmdb-asn.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from datetime import date
 from gzip import decompress

src/common/core/jobs/jobs/mmdb-country.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from datetime import date
 from gzip import decompress

src/common/core/letsencrypt/jobs/certbot-auth.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import getenv, sep
 from os.path import join

src/common/core/letsencrypt/jobs/certbot-cleanup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import getenv, sep
 from os.path import join

src/common/core/letsencrypt/jobs/certbot-deploy.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from io import BytesIO
 from os import getenv, sep

src/common/core/letsencrypt/jobs/certbot-new.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import _exit, environ, getenv, sep
 from os.path import join

src/common/core/letsencrypt/jobs/certbot-renew.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import _exit, environ, getenv, sep
 from os.path import join

src/common/core/letsencrypt/letsencrypt.lua

@@ -1,8 +1,8 @@
 local cjson = require "cjson"
 local class = require "middleclass"
 local plugin = require "bunkerweb.plugin"
-local utils = require "bunkerweb.utils"
 local ssl = require "ngx.ssl"
+local utils = require "bunkerweb.utils"
 
 local letsencrypt = class("letsencrypt", plugin)
 
@@ -20,6 +20,7 @@ local ssl_server_name = ssl.server_name
 local get_variable = utils.get_variable
 local get_multiple_variables = utils.get_multiple_variables
 local has_variable = utils.has_variable
+local read_files = utils.read_files
 local open = io.open
 local sub = string.sub
 local match = string.match
@@ -34,25 +35,29 @@ end
 
 function letsencrypt:init()
 	local ret_ok, ret_err = true, "success"
-    if has_variable("AUTO_LETS_ENCRYPT", "yes") then
+	if has_variable("AUTO_LETS_ENCRYPT", "yes") then
 		local multisite, err = get_variable("MULTISITE", false)
 		if not multisite then
 			return self:ret(false, "can't get MULTISITE variable : " .. err)
 		end
 		if multisite == "yes" then
-			local vars, err = get_multiple_variables({"AUTO_LETS_ENCRYPT", "SERVER_NAME"})
+			local vars
+			vars, err = get_multiple_variables({ "AUTO_LETS_ENCRYPT", "SERVER_NAME" })
 			if not vars then
 				return self:ret(false, "can't get AUTO_LETS_ENCRYPT variables : " .. err)
 			end
 			for server_name, multisite_vars in pairs(vars) do
 				if multisite_vars["AUTO_LETS_ENCRYPT"] == "yes" and server_name ~= "global" then
-					local check, data = self:read_files(server_name)
+					local check, data = read_files({
+						"/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name .. "/fullchain.pem",
+						"/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name .. "/privkey.pem",
+					})
 					if not check then
 						self.logger:log(ERR, "error while reading files : " .. data)
 						ret_ok = false
 						ret_err = "error reading files"
 					else
-						local check, err = self:load_data(data, multisite_vars["SERVER_NAME"])
+						check, err = self:load_data(data, multisite_vars["SERVER_NAME"])
 						if not check then
 							self.logger:log(ERR, "error while loading data : " .. err)
 							ret_ok = false
@@ -62,17 +67,21 @@ function letsencrypt:init()
 				end
 			end
 		else
-			local server_name, err = get_variable("SERVER_NAME", false)
+			local server_name
+			server_name, err = get_variable("SERVER_NAME", false)
 			if not server_name then
 				return self:ret(false, "can't get SERVER_NAME variable : " .. err)
 			end
-			local check, data = self:read_files(server_name:match("%S+"))
+			local check, data = read_files({
+				"/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name:match("%S+") .. "/fullchain.pem",
+				"/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name:match("%S+") .. "/privkey.pem",
+			})
 			if not check then
 				self.logger:log(ERR, "error while reading files : " .. data)
 				ret_ok = false
 				ret_err = "error reading files"
 			else
-				local check, err = self:load_data(data, server_name)
+				check, err = self:load_data(data, server_name)
 				if not check then
 					self.logger:log(ERR, "error while loading data : " .. err)
 					ret_ok = false
@@ -82,7 +91,7 @@ function letsencrypt:init()
 		end
 	else
 		ret_err = "let's encrypt is not used"
-    end
+	end
 	return self:ret(ret_ok, ret_err)
 end
 
@@ -91,31 +100,18 @@ function letsencrypt:ssl_certificate()
 	if not server_name then
 		return self:ret(false, "can't get server_name : " .. err)
 	end
-    if self.variables["AUTO_LETS_ENCRYPT"] == "yes" then
-		local data, err = self.datastore:get("plugin_letsencrypt_" .. server_name, true)
+	if self.variables["AUTO_LETS_ENCRYPT"] == "yes" then
+		local data
+		data, err = self.datastore:get("plugin_letsencrypt_" .. server_name, true)
 		if not data then
-			return self:ret(false, "error while getting plugin_letsencrypt_" .. server_name .. " from datastore : " .. err)
+			return self:ret(
+				false,
+				"error while getting plugin_letsencrypt_" .. server_name .. " from datastore : " .. err
+			)
 		end
-        return self:ret(true, "certificate/key data found", data)
-    end
-    return self:ret(true, "let's encrypt is not used")
-end
-
-function letsencrypt:read_files(server_name)
-	local files = {
-		"/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name .. "/fullchain.pem",
-		"/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name .. "/privkey.pem"
-	}
-	local data = {}
-	for i, file in ipairs(files) do
-		local f, err = open(file, "r")
-		if not f then
-			return false, file .. " = " .. err
-		end
-		table.insert(data, f:read("*a"))
-		f:close()
+		return self:ret(true, "certificate/key data found", data)
 	end
-	return true, data
+	return self:ret(true, "let's encrypt is not used")
 end
 
 function letsencrypt:load_data(data, server_name)
@@ -125,14 +121,16 @@ function letsencrypt:load_data(data, server_name)
 		return false, "error while parsing pem cert : " .. err
 	end
 	-- Load key
-	local priv_key, err = parse_pem_priv_key(data[2])
+	local priv_key
+	priv_key, err = parse_pem_priv_key(data[2])
 	if not priv_key then
 		return false, "error while parsing pem priv key : " .. err
 	end
 	-- Cache data
 	for key in server_name:gmatch("%S+") do
 		local cache_key = "plugin_letsencrypt_" .. key
-		local ok, err = self.datastore:set(cache_key, {cert_chain, priv_key}, nil, true)
+		local ok
+		ok, err = self.datastore:set(cache_key, { cert_chain, priv_key }, nil, true)
 		if not ok then
 			return false, "error while setting data into datastore : " .. err
 		end
@@ -172,7 +170,7 @@ function letsencrypt:api()
 		file:write(data.validation)
 		file:close()
 		return self:ret(true, "validation token written", HTTP_OK)
-	elseif ctx.bw.request_method == "DELETE" then
+	elseif self.ctx.bw.request_method == "DELETE" then
 		local ok, err = remove(acme_folder .. data.token)
 		if not ok then
 			return self:ret(true, "can't remove validation token : " .. err, HTTP_INTERNAL_SERVER_ERROR)

src/common/core/misc/files/default.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />

src/common/core/misc/jobs/anonymous-report.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from json import dumps
 from os import getenv, sep

src/common/core/misc/jobs/default-server-cert.py

@@ -1,6 +1,6 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
-from os import _exit, getenv, sep
+from os import getenv, sep
 from os.path import join
 from pathlib import Path
 from subprocess import DEVNULL, run

src/common/core/misc/jobs/update-check.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import getenv, sep
 from os.path import basename, join

src/common/core/realip/jobs/realip-download.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from contextlib import suppress
 from ipaddress import ip_address, ip_network

src/common/core/selfsigned/jobs/self-signed.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from datetime import timedelta
 from os import getenv, sep

src/common/core/selfsigned/selfsigned.lua

@@ -1,7 +1,7 @@
 local class = require "middleclass"
 local plugin = require "bunkerweb.plugin"
-local utils = require "bunkerweb.utils"
 local ssl = require "ngx.ssl"
+local utils = require "bunkerweb.utils"
 
 local selfsigned = class("selfsigned", plugin)
 
@@ -13,7 +13,7 @@ local ssl_server_name = ssl.server_name
 local get_variable = utils.get_variable
 local get_multiple_variables = utils.get_multiple_variables
 local has_variable = utils.has_variable
-local open = io.open
+local read_files = utils.read_files
 
 function selfsigned:initialize(ctx)
 	-- Call parent initialize
@@ -22,25 +22,29 @@ end
 
 function selfsigned:init()
 	local ret_ok, ret_err = true, "success"
-    if has_variable("GENERATE_SELF_SIGNED_SSL", "yes") then
+	if has_variable("GENERATE_SELF_SIGNED_SSL", "yes") then
 		local multisite, err = get_variable("MULTISITE", false)
 		if not multisite then
 			return self:ret(false, "can't get MULTISITE variable : " .. err)
 		end
 		if multisite == "yes" then
-			local vars, err = get_multiple_variables({"GENERATE_SELF_SIGNED_SSL", "SERVER_NAME"})
+			local vars
+			vars, err = get_multiple_variables({ "GENERATE_SELF_SIGNED_SSL", "SERVER_NAME" })
 			if not vars then
 				return self:ret(false, "can't get GENERATE_SELF_SIGNED_SSL variables : " .. err)
 			end
 			for server_name, multisite_vars in pairs(vars) do
 				if multisite_vars["GENERATE_SELF_SIGNED_SSL"] == "yes" and server_name ~= "global" then
-					local check, data = self:read_files(server_name)
+					local check, data = read_files({
+						"/var/cache/bunkerweb/selfsigned/" .. server_name .. ".pem",
+						"/var/cache/bunkerweb/selfsigned/" .. server_name .. ".key",
+					})
 					if not check then
 						self.logger:log(ERR, "error while reading files : " .. data)
 						ret_ok = false
 						ret_err = "error reading files"
 					else
-						local check, err = self:load_data(data, multisite_vars["SERVER_NAME"])
+						check, err = self:load_data(data, multisite_vars["SERVER_NAME"])
 						if not check then
 							self.logger:log(ERR, "error while loading data : " .. err)
 							ret_ok = false
@@ -50,17 +54,21 @@ function selfsigned:init()
 				end
 			end
 		else
-			local server_name, err = get_variable("SERVER_NAME", false)
+			local server_name
+			server_name, err = get_variable("SERVER_NAME", false)
 			if not server_name then
 				return self:ret(false, "can't get SERVER_NAME variable : " .. err)
 			end
-			local check, data = self:read_files(server_name:match("%S+"))
+			local check, data = read_files({
+				"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. ".pem",
+				"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. ".key",
+			})
 			if not check then
 				self.logger:log(ERR, "error while reading files : " .. data)
 				ret_ok = false
 				ret_err = "error reading files"
 			else
-				local check, err = self:load_data(data, server_name)
+				check, err = self:load_data(data, server_name)
 				if not check then
 					self.logger:log(ERR, "error while loading data : " .. err)
 					ret_ok = false
@@ -70,7 +78,7 @@ function selfsigned:init()
 		end
 	else
 		ret_err = "self signed is not used"
-    end
+	end
 	return self:ret(ret_ok, ret_err)
 end
 
@@ -79,31 +87,18 @@ function selfsigned:ssl_certificate()
 	if not server_name then
 		return self:ret(false, "can't get server_name : " .. err)
 	end
-    if self.variables["GENERATE_SELF_SIGNED_SSL"] == "yes" then
-		local data, err = self.datastore:get("plugin_selfsigned_" .. server_name, true)
+	if self.variables["GENERATE_SELF_SIGNED_SSL"] == "yes" then
+		local data
+		data, err = self.datastore:get("plugin_selfsigned_" .. server_name, true)
 		if not data then
-			return self:ret(false, "error while getting plugin_selfsigned_" .. server_name .. " from datastore : " .. err)
+			return self:ret(
+				false,
+				"error while getting plugin_selfsigned_" .. server_name .. " from datastore : " .. err
+			)
 		end
-        return self:ret(true, "certificate/key data found", data)
-    end
-    return self:ret(true, "selfsigned is not used")
-end
-
-function selfsigned:read_files(server_name)
-	local files = {
-		"/var/cache/bunkerweb/selfsigned/" .. server_name .. ".pem",
-		"/var/cache/bunkerweb/selfsigned/" .. server_name .. ".key"
-	}
-	local data = {}
-	for i, file in ipairs(files) do
-		local f, err = open(file, "r")
-		if not f then
-			return false, file .. " = " .. err
-		end
-		table.insert(data, f:read("*a"))
-		f:close()
+		return self:ret(true, "certificate/key data found", data)
 	end
-	return true, data
+	return self:ret(true, "selfsigned is not used")
 end
 
 function selfsigned:load_data(data, server_name)
@@ -120,7 +115,8 @@ function selfsigned:load_data(data, server_name)
 	-- Cache data
 	for key in server_name:gmatch("%S+") do
 		local cache_key = "plugin_selfsigned_" .. key
-		local ok, err = self.datastore:set(cache_key, {cert_chain, priv_key}, nil, true)
+		local ok
+		ok, err = self.datastore:set(cache_key, { cert_chain, priv_key }, nil, true)
 		if not ok then
 			return false, "error while setting data into datastore : " .. err
 		end

src/common/core/sessions/sessions.lua

@@ -66,7 +66,7 @@ function sessions:init()
 		["REDIS_SENTINEL_HOSTS"] = "",
 		["REDIS_SENTINEL_USERNAME"] = "",
 		["REDIS_SENTINEL_PASSWORD"] = "",
-		["REDIS_SENTINEL_MASTER"] = ""
+		["REDIS_SENTINEL_MASTER"] = "",
 	}
 	for k, _ in pairs(redis_vars) do
 		local value, err = get_variable(k, false)
@@ -124,7 +124,7 @@ function sessions:init()
 			pool = "bw-redis",
 			pool_size = tonumber(redis_vars["REDIS_KEEPALIVE_POOL"]),
 			ssl = redis_vars["REDIS_SSL"] == "yes",
-			database = tonumber(redis_vars["REDIS_DATABASE"])
+			database = tonumber(redis_vars["REDIS_DATABASE"]),
 		}
 		if redis_vars["REDIS_SENTINEL_HOSTS"] ~= nil then
 			config.redis.master = redis_vars["REDIS_SENTINEL_MASTER"]
@@ -139,7 +139,7 @@ function sessions:init()
 				else
 					sport = tonumber(sport)
 				end
-				table.insert(config.redis.sentinels, {host = shost, port = sport})
+				table.insert(config.redis.sentinels, { host = shost, port = sport })
 			end
 		else
 			config.redis.host = redis_vars["REDIS_HOST"]

src/common/core/whitelist/jobs/whitelist-download.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from contextlib import suppress
 from ipaddress import ip_address, ip_network

src/common/db/Database.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from contextlib import contextmanager, suppress
 from copy import deepcopy

src/common/db/model.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from sqlalchemy import (
     Boolean,

src/common/gen/Configurator.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from glob import glob
 from hashlib import sha256

src/common/gen/Templator.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from glob import glob
 from importlib import import_module

src/common/gen/main.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from argparse import ArgumentParser
 from glob import glob

src/common/gen/save_config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from argparse import ArgumentParser
 from os import R_OK, X_OK, access, environ, getenv, sep

src/common/gen/utils.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import getegid, geteuid
 from pathlib import Path

src/common/utils/ApiCaller.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from io import BytesIO
 from os import getenv, sep

src/common/utils/ConfigCaller.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from glob import glob
 from json import JSONDecodeError, loads

src/common/utils/jobs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from contextlib import suppress
 from datetime import datetime

src/scheduler/Dockerfile

@@ -50,7 +50,6 @@ WORKDIR /usr/share/bunkerweb
 
 # Add scheduler user, drop bwcli, install runtime dependencies, create data folders and set permissions
 RUN apk add --no-cache bash libgcc libstdc++ libpq openssl libmagic && \
-  ln -s /usr/local/bin/python3 /usr/bin/python3 && \
   addgroup -g 101 scheduler && \
   adduser -h /var/cache/nginx -g scheduler -s /bin/sh -G scheduler -D -H -u 101 scheduler && \
   cp helpers/bwcli /usr/bin/ && \

src/scheduler/JobScheduler.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from copy import deepcopy
 from functools import partial

src/scheduler/main.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from argparse import ArgumentParser
 from copy import deepcopy
@@ -6,16 +6,7 @@ from glob import glob
 from hashlib import sha256
 from io import BytesIO
 from json import load as json_load
-from os import (
-    _exit,
-    chmod,
-    environ,
-    getenv,
-    getpid,
-    listdir,
-    sep,
-    walk,
-)
+from os import _exit, chmod, environ, getenv, getpid, listdir, sep, walk
 from os.path import basename, dirname, join, normpath
 from pathlib import Path
 from shutil import copy, rmtree
@@ -322,11 +313,7 @@ if __name__ == "__main__":
                     content = Path(join(root, file)).read_text(encoding="utf-8")
                     custom_conf = {
                         "value": content,
-                        "exploded": (
-                            f"{path_exploded.pop()}" if path_exploded[-1] not in root_dirs else None,
-                            path_exploded[-1],
-                            file.replace(".conf", ""),
-                        ),
+                        "exploded": (path_exploded.pop() if path_exploded[-1] not in root_dirs else None, path_exploded[-1], file.replace(".conf", "")),
                     }
 
                     saving = True
@@ -674,7 +661,6 @@ if __name__ == "__main__":
                     env = db.get_config()
                     env["DATABASE_URI"] = db.database_uri
 
-
     except:
         logger.error(
             f"Exception while executing scheduler : {format_exc()}",

src/ui/main.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from contextlib import suppress
 from os import _exit, getenv, listdir, sep, urandom
@@ -1132,11 +1132,11 @@ def plugins():
                 except (TarError, OSError) as e:
                     errors += 1
                     error = 1
-                    flash(f"{e}", "error")
+                    flash(str(e), "error")
                 except Exception as e:
                     errors += 1
                     error = 1
-                    flash(f"{e}", "error")
+                    flash(str(e), "error")
                 finally:
                     if error != 1:
                         flash(f"Successfully created plugin: <b><i>{folder_name}</i></b>")

src/ui/src/Config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from copy import deepcopy
 from os import sep

src/ui/src/ConfigFiles.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from glob import glob
 from os import listdir, replace, sep, walk
@@ -62,11 +62,7 @@ class ConfigFiles:
                         custom_configs.append(
                             {
                                 "value": f.read(),
-                                "exploded": (
-                                    f"{path_exploded.pop()}" if path_exploded[-1] not in root_dirs else None,
-                                    path_exploded[-1],
-                                    file.replace(".conf", ""),
-                                ),
+                                "exploded": (path_exploded.pop() if path_exploded[-1] not in root_dirs else None, path_exploded[-1], file.replace(".conf", "")),
                             }
                         )
 

src/ui/src/Instances.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from os import sep
 from os.path import join

src/ui/src/ReverseProxied.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from werkzeug.middleware.proxy_fix import ProxyFix
 

src/ui/src/User.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from typing import Optional
 

src/ui/static/css/datepicker-foundation.css

@@ -1,3 +1,3 @@
 .datepicker{display:none}.datepicker.active{display:block}.datepicker-dropdown{position:absolute;top:0;left:0;z-index:100;padding-top:4px}.datepicker-dropdown.datepicker-orient-top{padding-top:0;padding-bottom:4px}.datepicker-picker{display:inline-block;border-radius:0;background-color:#fefefe}.datepicker-dropdown .datepicker-picker{box-shadow:0 0 0 1px #cacaca}.datepicker-picker span{display:block;flex:1;border:0;border-radius:0;cursor:default;text-align:center;-webkit-touch-callout:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.datepicker-main{padding:2px}.datepicker-footer{box-shadow:inset 0 1px 1px rgba(10,10,10,.1);background-color:#e6e6e6}.datepicker-controls,.datepicker-grid,.datepicker-view,.datepicker-view .days-of-week{display:flex}.datepicker-grid{flex-wrap:wrap;width:15.75rem}.datepicker-view .days .datepicker-cell,.datepicker-view .dow{flex-basis:14.2857142857%}.datepicker-view.datepicker-grid .datepicker-cell{flex-basis:25%;height:4.5rem;line-height:4.5rem}.datepicker-cell,.datepicker-view .week{height:2.25rem;line-height:2.25rem}.datepicker-title{box-shadow:inset 0 -1px 1px rgba(10,10,10,.1);background-color:#e6e6e6;padding:.375rem .75rem;text-align:center;font-weight:700}.datepicker-header .datepicker-controls{padding:2px 2px 0}.datepicker-controls .button{margin:.4rem 0;background-color:#fefefe;color:#707070}.datepicker-controls .button:focus,.datepicker-controls .button:hover{background-color:#d8d8d8}.datepicker-controls .button:focus[disabled],.datepicker-controls .button:hover[disabled]{opacity:.25;background-color:#fefefe;margin-top:.4rem;margin-bottom:.4rem;color:#707070}.datepicker-header .datepicker-controls .button{border-color:transparent;font-weight:700}.datepicker-footer .datepicker-controls .button{margin:calc(.375rem - 1px) .375rem;border-radius:0;width:100%;font-size:.75rem}.datepicker-controls .view-switch{flex:auto}.datepicker-controls .next-btn,.datepicker-controls .prev-btn{padding-right:.375rem;padding-left:.375rem;width:2.25rem}.datepicker-controls .next-btn.disabled,.datepicker-controls .prev-btn.disabled{visibility:hidden}.datepicker-view .dow{height:1.5rem;line-height:1.5rem;font-size:.875rem;font-weight:700;color:#064059}.datepicker-view .week{width:2.25rem;color:#8a8a8a;font-size:.75rem}@media (max-width:22.5rem){.datepicker-view .week{width:1.96875rem}.calendar-weeks+.days .datepicker-grid{width:13.78125rem}}.datepicker-cell:not(.disabled):hover{background-color:#f8f8f8;cursor:pointer}.datepicker-cell.focused:not(.selected),.datepicker-cell.highlighted:not(.selected):not(.range):not(.today).focused,.datepicker-cell.highlighted:not(.selected):not(.range):not(.today):not(
 .disabled
-):hover{background-color:#f1f1f1}.datepicker-cell.selected,.datepicker-cell.selected:hover{background-color:#1779ba;color:#fefefe;font-weight:semibold}.datepicker-cell.disabled{color:#e6e6e6}.datepicker-cell.next:not(.disabled),.datepicker-cell.prev:not(.disabled){color:#cacaca}.datepicker-cell.next.selected,.datepicker-cell.prev.selected{color:#e5e5e5}.datepicker-cell.highlighted:not(.selected):not(.range):not(.today){border-radius:0;background-color:#f7f7f7}.datepicker-cell.today:not(.selected){background-color:#d7ecfa}.datepicker-cell.today:not(.selected):not(.disabled){color:#8a8a8a}.datepicker-cell.today.focused:not(.selected){background-color:#cbe7f9}.datepicker-cell.range-end:not(.selected),.datepicker-cell.range-start:not(.selected){background-color:#767676;color:#fefefe}.datepicker-cell.range-end.focused:not(.selected),.datepicker-cell.range-start.focused:not(.selected){background-color:#707070}.datepicker-cell.range-end,.datepicker-cell.range-start{border-radius:0}.datepicker-cell.range{border-radius:0;background-color:#e6e6e6}.datepicker-cell.range:not(.disabled):not(.focused):not(.today):hover{background-color:#e0e0e0}.datepicker-cell.range.disabled{color:#cdcdcd}.datepicker-cell.range.focused{background-color:#d9d9d9}.datepicker-cell.range.today{background-color:#b3dbf6}.datepicker-input.in-edit{border-color:#a4a4a4}.datepicker-input.in-edit:active,.datepicker-input.in-edit:focus{box-shadow:0 0 .25em .25em rgba(164,164,164,.2)}
+):hover{background-color:#f1f1f1}.datepicker-cell.selected,.datepicker-cell.selected:hover{background-color:#1779ba;color:#fefefe;font-weight:semibold}.datepicker-cell.disabled{color:#e6e6e6}.datepicker-cell.next:not(.disabled),.datepicker-cell.prev:not(.disabled){color:#cacaca}.datepicker-cell.next.selected,.datepicker-cell.prev.selected{color:#e5e5e5}.datepicker-cell.highlighted:not(.selected):not(.range):not(.today){border-radius:0;background-color:#f7f7f7}.datepicker-cell.today:not(.selected){background-color:#d7ecfa}.datepicker-cell.today:not(.selected):not(.disabled){color:#8a8a8a}.datepicker-cell.today.focused:not(.selected){background-color:#cbe7f9}.datepicker-cell.range-end:not(.selected),.datepicker-cell.range-start:not(.selected){background-color:#767676;color:#fefefe}.datepicker-cell.range-end.focused:not(.selected),.datepicker-cell.range-start.focused:not(.selected){background-color:#707070}.datepicker-cell.range-end,.datepicker-cell.range-start{border-radius:0}.datepicker-cell.range{border-radius:0;background-color:#e6e6e6}.datepicker-cell.range:not(.disabled):not(.focused):not(.today):hover{background-color:#e0e0e0}.datepicker-cell.range.disabled{color:#cdcdcd}.datepicker-cell.range.focused{background-color:#d9d9d9}.datepicker-cell.range.today{background-color:#b3dbf6}.datepicker-input.in-edit{border-color:#a4a4a4}.datepicker-input.in-edit:active,.datepicker-input.in-edit:focus{box-shadow:0 0 .25em .25em rgba(164,164,164,.2)}

src/ui/static/js/global.js

@@ -78,7 +78,7 @@ class News {
         news.photo.url,
         news.excerpt,
         news.tags,
-        news.date
+        news.date,
       );
       let cleanHTML = DOMPurify.sanitize(cardHTML);
       //add to DOM
@@ -205,7 +205,7 @@ class darkMode {
     };
     const send = await fetch(
       `${location.href.split("/").slice(0, -1).join("/")}/darkmode`,
-      data
+      data,
     );
   }
 }
@@ -245,7 +245,7 @@ class FlashMsg {
           flashEl.remove();
           //update count
           this.flashCount.textContent = document.querySelectorAll(
-            "[data-flash-message]"
+            "[data-flash-message]",
           ).length;
         }
       } catch (err) {}
@@ -338,7 +338,7 @@ class Banner {
         this.bannerEl.querySelector(
           `[role="listitem"][data-id="${
             +visibleEl.getAttribute("data-id") + 1
-          }"]`
+          }"]`,
         ) || this.bannerEl.querySelector(`[role="listitem"][data-id="0"]`);
 
       // Hide current one
@@ -426,7 +426,7 @@ class Clipboard {
           if (result.state === "granted" || result.state === "prompt") {
             /* write to the clipboard now */
             const copyEl = document.querySelector(
-              e.target.getAttribute("data-clipboard-target")
+              e.target.getAttribute("data-clipboard-target"),
             );
 
             copyEl.select();
@@ -454,13 +454,13 @@ const setMenu = new Menu();
 const setNewsSidebar = new Sidebar(
   "[data-sidebar-info]",
   "[data-sidebar-info-open]",
-  "[data-sidebar-info-close]"
+  "[data-sidebar-info-close]",
 );
 
 const setFlashSidebar = new Sidebar(
   "[data-flash-sidebar]",
   "[data-flash-sidebar-open]",
-  "[data-flash-sidebar-close]"
+  "[data-flash-sidebar-close]",
 );
 
 const setClipboard = new Clipboard();

src/ui/static/js/profile.js

@@ -45,7 +45,7 @@ class SubmitProfile {
       "focus:valid:!ring-red-500",
       "active:!border-red-500",
       "active:valid:!border-red-500",
-      "valid:!border-red-500"
+      "valid:!border-red-500",
     );
     this.pwAlertEl.classList.add("opacity-0");
     this.pwAlertEl.setAttribute("aria-hidden", "true");
@@ -59,7 +59,7 @@ class SubmitProfile {
       "focus:valid:!ring-red-500",
       "active:!border-red-500",
       "active:valid:!border-red-500",
-      "valid:!border-red-500"
+      "valid:!border-red-500",
     );
     this.pwAlertEl.classList.remove("opacity-0");
     this.pwAlertEl.setAttribute("aria-hidden", "false");
@@ -77,14 +77,14 @@ class PwBtn {
       const passwordContainer = e.target.closest("[data-input-group]");
       const inpEl = passwordContainer.querySelector("input");
       const invBtn = passwordContainer.querySelector(
-        '[data-setting-password="invisible"]'
+        '[data-setting-password="invisible"]',
       );
       const visBtn = passwordContainer.querySelector(
-        '[data-setting-password="visible"]'
+        '[data-setting-password="visible"]',
       );
       inpEl.setAttribute(
         "type",
-        inpEl.getAttribute("type") === "password" ? "text" : "password"
+        inpEl.getAttribute("type") === "password" ? "text" : "password",
       );
 
       if (inpEl.getAttribute("type") === "password") {

src/ui/utils.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from base64 import b64encode
 from io import BytesIO

tests/ui/main.py

@@ -1483,12 +1483,9 @@ location /hello {
         username_input.send_keys("admin2")
         password_input.send_keys("S$cr3tP@ssw0rd")
 
-        access_page(
-            driver,
-            driver_wait,
-            "//button[@value='login']",
-            "profile",
-        )
+        access_page(driver, driver_wait, "//button[@value='login']", "home")
+
+        access_page(driver, driver_wait, "/html/body/aside[1]/div[1]/div[2]/ul/li[10]/a", "profile")
 
         username_input = safe_get_element(driver, By.ID, "admin_username")
 
@@ -1543,12 +1540,9 @@ location /hello {
         username_input.send_keys("admin2")
         password_input.send_keys("P@ssw0rd")
 
-        access_page(
-            driver,
-            driver_wait,
-            "//button[@value='login']",
-            "profile",
-        )
+        access_page(driver, driver_wait, "//button[@value='login']", "home")
+
+        access_page(driver, driver_wait, "/html/body/aside[1]/div[1]/div[2]/ul/li[10]/a", "profile")
 
         print("Successfully logged in with new password, trying 2FA ...", flush=True)
 
@@ -1584,12 +1578,7 @@ location /hello {
 
         password_input.send_keys("P@ssw0rd")
 
-        access_page(
-            driver,
-            driver_wait,
-            "//button[@id='totp-button' and @class='valid-btn']",
-            "profile",
-        )
+        access_page(driver, driver_wait, "//button[@id='totp-button' and @class='valid-btn']", "profile")
 
         assert_button_click(driver, "//button[@data-tab-handler='totp']")