Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add new templating feature to allow to quickly override the default values of settings and custom configurations. You can also precise steps to follow in the UI to help the user configure services.

Browse source
This commit is contained in:
Théophile Diot 2024-08-06 14:09:07 +01:00
parent 9289864808
commit 57a1e223a9
No known key found for this signature in database
GPG key ID: FA995104A0BA376A
20 changed files with 1518 additions and 1111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -8,6 +8,7 @@
- [FEATURE] Add new `REVERSE_PROXY_PASS_REQUEST_BODY` setting to control if the request body should be passed to the upstream server (default is yes)
- [FEATURE] Jobs now have an history which the size can be controlled via the `DATABASE_MAX_JOBS_RUNS` setting (default is 10000) and it will be possible to see it in the web UI in a future release
- [FEATURE] Add support for HTTP/3 connections limiting via the `HTTP3_CONNECTIONS_LIMIT` setting (default is 100) in the `limit` plugin
- [FEATURE] Add new templating feature to allow to quickly override the default values of settings and custom configurations. You can also precise steps to follow in the UI to help the user configure services.
- [SCHEDULER] Refactor the scheduler to use the `BUNKERWEB_INSTANCES` (previously known as `OVERRIDE_INSTANCES`) environment variable instead of an integration specific system
- [AUTOCONF] Add new `NAMESPACES` environment variable to allow setting the namespaces to watch for the autoconf feature which makes it possible to use multiple autoconf instances in the same cluster while keeping the configuration separated
- [UI] Start refactoring the UI to make it more modular and easier to maintain with migration from Jinja to Vue.js

4
docs/assets/img/bunkerweb_db.svg
View file

File diff suppressed because one or more lines are too long
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 911 KiB

After

Width:  |  Height:  |  Size: 950 KiB

7
src/autoconf/Config.py
View file

@ -50,9 +50,10 @@ class Config:
for variable, value in service.items():
if variable == "NAMESPACE" or variable.startswith("CUSTOM_CONF") or not variable.isupper():
continue
if not self._db.is_setting(variable, multisite=True):
if variable in service:
self.__logger.warning(f"Variable {variable}: {value} is not a valid multisite setting, ignoring it")
success, err = self._db.is_valid_setting(variable, value=value, multisite=True)
if not success:
self.__logger.warning(f"Variable {variable}: {value} is not a valid autoconf setting ({err}), ignoring it")
continue
config[f"{server_name}_{variable}"] = value
config["SERVER_NAME"] += f" {server_name}"

2
src/common/core/securitytxt/confs/server-http/securitytxt.conf
View file

@ -1,7 +1,7 @@
{% if USE_SECURITYTXT == "yes" and SECURITYTXT_CONTACT != "" +%}
location = {{ SECURITYTXT_URI }} {
default_type 'text/plain; charset=utf-8';
root /usr/share/bunkerweb/core/securitytxt/templates;
root /usr/share/bunkerweb/core/securitytxt/files;
content_by_lua_block {
local logger = require "bunkerweb.logger":new("SECURITYTXT")
local helpers = require "bunkerweb.helpers"

0
src/common/core/securitytxt/templates/security.txt → src/common/core/securitytxt/files/security.txt
View file

8
src/common/core/templates/plugin.json Normal file
View file

@ -0,0 +1,8 @@
{
"id": "templates",
"name": "Templates",
"description": "Fake core plugin for internal templates.",
"version": "1.0",
"stream": "yes",
"settings": {}
}

1
src/common/core/templates/templates/high.json Normal file
View file

@ -0,0 +1 @@
{} // TODO

144
src/common/core/templates/templates/low.json Normal file
View file

@ -0,0 +1,144 @@
{
"name": "Basic security level",
"settings": {
"SERVER_NAME": "www.example.com",
"USE_REVERSE_PROXY": "yes",
"REVERSE_PROXY_HOST": "http://upstream-server:8080",
"REVERSE_PROXY_URL": "/",
"REVERSE_PROXY_CUSTOM_HOST": "",
"REVERSE_PROXY_SSL_SNI": "no",
"REVERSE_PROXY_SSL_SNI_NAME": "",
"REVERSE_PROXY_WS": "no",
"REVERSE_PROXY_KEEPALIVE": "no",
"AUTO_LETS_ENCRYPT": "yes",
"USE_LETS_ENCRYPT_STAGING": "no",
"ALLOWED_METHODS": "GET|POST|HEAD|OPTIONS|PUT|DELETE|PATCH",
"MAX_CLIENT_SIZE": "100m",
"HTTP2": "yes",
"HTTP3": "yes",
"SSL_PROTOCOLS": "TLSv1.2 TLSv1.3",
"COOKIE_FLAGS": "* SameSite=Lax",
"CONTENT_SECURITY_POLICY": "",
"PERMISSIONS_POLICY": "",
"KEEP_UPSTREAM_HEADERS": "*",
"REFERRER_POLICY": "no-referrer-when-downgrade",
"USE_CORS": "yes",
"CORS_ALLOW_ORIGIN": "*",
"USE_BAD_BEHAVIOR": "yes",
"BAD_BEHAVIOR_STATUS_CODES": "400 401 403 404 405 429 444",
"BAD_BEHAVIOR_BAN_TIME": "3600",
"BAD_BEHAVIOR_THRESHOLD": "30",
"BAD_BEHAVIOR_COUNT_TIME": "60",
"USE_ANTIBOT": "no",
"ANTIBOT_URI": "/challenge",
"ANTIBOT_RECAPTCHA_SCORE": "0.7",
"ANTIBOT_RECAPTCHA_SITEKEY": "",
"ANTIBOT_RECAPTCHA_SECRET": "",
"ANTIBOT_HCAPTCHA_SITEKEY": "",
"ANTIBOT_HCAPTCHA_SECRET": "",
"ANTIBOT_TURNSTILE_SITEKEY": "",
"ANTIBOT_TURNSTILE_SECRET": "",
"USE_BLACKLIST": "yes",
"USE_DNSBL": "no",
"USE_LIMIT_CONN": "yes",
"LIMIT_CONN_MAX_HTTP1": "25",
"LIMIT_CONN_MAX_HTTP2": "200",
"USE_LIMIT_REQ": "yes",
"LIMIT_REQ_URL": "/",
"LIMIT_REQ_RATE": "5r/s"
},
"configs": ["modsec/anomaly_score.conf"],
"steps": [
{
"title": "Web service - Front service",
"subtitle": "Configure your web service facing your clients",
"settings": [
"SERVER_NAME",
"AUTO_LETS_ENCRYPT",
"USE_LETS_ENCRYPT_STAGING"
]
},
{
"title": "Web service - Upstream server",
"subtitle": "Configure the upstream server to be protected by BunkerWeb",
"settings": [
"USE_REVERSE_PROXY",
"REVERSE_PROXY_HOST",
"REVERSE_PROXY_URL",
"REVERSE_PROXY_CUSTOM_HOST",
"REVERSE_PROXY_SSL_SNI",
"REVERSE_PROXY_SSL_SNI_NAME",
"REVERSE_PROXY_WS",
"REVERSE_PROXY_KEEPALIVE"
]
},
{
"title": "HTTP - General",
"subtitle": "Configure the settings related to the HTTP(S) protocol",
"settings": [
"MAX_CLIENT_SIZE",
"ALLOWED_METHODS",
"HTTP2",
"HTTP3",
"SSL_PROTOCOLS"
]
},
{
"title": "HTTP - Headers",
"subtitle": "Configure the settings related to the HTTP headers",
"settings": [
"COOKIE_FLAGS",
"CONTENT_SECURITY_POLICY",
"PERMISSIONS_POLICY",
"USE_CORS",
"CORS_ALLOW_ORIGIN",
"KEEP_UPSTREAM_HEADERS",
"REFERRER_POLICY"
]
},
{
"title": "Security - Bad behavior",
"subtitle": "Configure the settings related to the automatic ban when a bad behavior is detected.",
"settings": [
"USE_BAD_BEHAVIOR",
"BAD_BEHAVIOR_STATUS_CODES",
"BAD_BEHAVIOR_BAN_TIME",
"BAD_BEHAVIOR_THRESHOLD",
"BAD_BEHAVIOR_COUNT_TIME"
]
},
{
"title": "Security - Blacklisting",
"subtitle": "Configure the settings related to the external blacklists.",
"settings": ["USE_BLACKLIST", "USE_DNSBL"]
},
{
"title": "Security - Limiting",
"subtitle": "Configure the settings related to limiting requests and connections.",
"settings": [
"USE_LIMIT_CONN",
"LIMIT_CONN_MAX_HTTP1",
"LIMIT_CONN_MAX_HTTP2",
"LIMIT_CONN_MAX_HTTP3",
"USE_LIMIT_REQ",
"LIMIT_REQ_URL",
"LIMIT_REQ_RATE"
]
},
{
"title": "Security - Antibot",
"subtitle": "Configure the settings about bot detection",
"settings": [
"USE_ANTIBOT",
"ANTIBOT_URI",
"ANTIBOT_RECAPTCHA_SCORE",
"ANTIBOT_RECAPTCHA_SITEKEY",
"ANTIBOT_RECAPTCHA_SECRET",
"ANTIBOT_HCAPTCHA_SITEKEY",
"ANTIBOT_HCAPTCHA_SECRET",
"ANTIBOT_TURNSTILE_SITEKEY",
"ANTIBOT_TURNSTILE_SECRET"
]
}
]
}

9
src/common/core/templates/templates/low/configs/modsec/anomaly_score.conf Normal file
View file

@ -0,0 +1,9 @@
SecAction \
"id:900110,\
phase:1,\
pass,\
t:none,\
nolog,\
tag:'OWASP_CRS',\
setvar:tx.inbound_anomaly_score_threshold=6,\
setvar:tx.outbound_anomaly_score_threshold=5"

1
src/common/core/templates/templates/medium.json Normal file
View file

@ -0,0 +1 @@
{} // TODO

1743
src/common/db/Database.py
View file

File diff suppressed because it is too large Load diff

98
src/common/db/model.py
View file

@ -2,20 +2,7 @@
from datetime import datetime, timezone
from functools import partial
from sqlalchemy import (
TEXT,
Boolean,
Column,
DateTime,
Enum,
ForeignKey,
Identity,
Integer,
LargeBinary,
PrimaryKeyConstraint,
String,
func,
)
from sqlalchemy import TEXT, Boolean, Column, DateTime, Enum, ForeignKey, Identity, Integer, LargeBinary, String, func
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy.schema import UniqueConstraint
@ -71,21 +58,18 @@ class Plugins(Base):
settings = relationship("Settings", back_populates="plugin", cascade="all, delete-orphan")
jobs = relationship("Jobs", back_populates="plugin", cascade="all, delete-orphan")
pages = relationship("Plugin_pages", back_populates="plugin", cascade="all")
commands = relationship("BwcliCommands", back_populates="plugin", cascade="all")
commands = relationship("Bw_cli_commands", back_populates="plugin", cascade="all")
templates = relationship("Templates", back_populates="plugin", cascade="all")
class Settings(Base):
__tablename__ = "bw_settings"
__table_args__ = (
PrimaryKeyConstraint("id", "name"),
UniqueConstraint("id"),
)
id = Column(String(256), primary_key=True)
name = Column(String(256), primary_key=True)
name = Column(String(256), unique=True, nullable=False)
plugin_id = Column(String(64), ForeignKey("bw_plugins.id", onupdate="cascade", ondelete="cascade"), nullable=False)
context = Column(CONTEXTS_ENUM, nullable=False)
default = Column(String(4096), nullable=True, default="")
default = Column(TEXT, nullable=True, default="")
help = Column(String(512), nullable=False)
label = Column(String(256), nullable=True)
regex = Column(String(1024), nullable=False)
@ -96,6 +80,7 @@ class Settings(Base):
selects = relationship("Selects", back_populates="setting", cascade="all")
services = relationship("Services_settings", back_populates="setting", cascade="all")
global_value = relationship("Global_values", back_populates="setting", cascade="all")
templates = relationship("Template_settings", back_populates="setting", cascade="all")
plugin = relationship("Plugins", back_populates="settings")
@ -162,14 +147,9 @@ class Plugin_pages(Base):
__tablename__ = "bw_plugin_pages"
id = Column(Integer, Identity(start=1, increment=1), primary_key=True)
plugin_id = Column(String(64), ForeignKey("bw_plugins.id", onupdate="cascade", ondelete="cascade"), nullable=False)
# TODO: replace with a raw data that gets extracted by the plugin
template_file = Column(LargeBinary(length=(2**32) - 1), nullable=False)
template_checksum = Column(String(128), nullable=False)
actions_file = Column(LargeBinary(length=(2**32) - 1), nullable=False)
actions_checksum = Column(String(128), nullable=False)
obfuscation_file = Column(LargeBinary(length=(2**32) - 1), default=None, nullable=True)
obfuscation_checksum = Column(String(128), default=None, nullable=True)
plugin_id = Column(String(64), ForeignKey("bw_plugins.id", onupdate="cascade", ondelete="cascade"), unique=True, nullable=False)
data = Column(LargeBinary(length=(2**32) - 1), nullable=False)
checksum = Column(String(128), nullable=False)
plugin = relationship("Plugins", back_populates="pages")
@ -228,7 +208,7 @@ class Instances(Base):
last_seen = Column(DateTime, nullable=True, server_default=func.now(), onupdate=partial(datetime.now, timezone.utc))
class BwcliCommands(Base):
class Bw_cli_commands(Base):
__tablename__ = "bw_cli_commands"
__table_args__ = (UniqueConstraint("plugin_id", "name"),)
@ -240,6 +220,64 @@ class BwcliCommands(Base):
plugin = relationship("Plugins", back_populates="commands")
class Templates(Base):
__tablename__ = "bw_templates"
id = Column(String(256), primary_key=True)
name = Column(String(256), unique=True, nullable=False)
plugin_id = Column(String(64), ForeignKey("bw_plugins.id", onupdate="cascade", ondelete="cascade"), nullable=False)
plugin = relationship("Plugins", back_populates="templates")
steps = relationship("Template_steps", back_populates="template", cascade="all")
settings = relationship("Template_settings", back_populates="template", cascade="all")
custom_configs = relationship("Template_custom_configs", back_populates="template", cascade="all")
class Template_steps(Base):
__tablename__ = "bw_template_steps"
id = Column(Integer, primary_key=True)
template_id = Column(String(256), ForeignKey("bw_templates.id", onupdate="cascade", ondelete="cascade"), primary_key=True)
title = Column(TEXT, nullable=False)
subtitle = Column(TEXT, nullable=True)
template = relationship("Templates", back_populates="steps")
settings = relationship("Template_settings", back_populates="step", cascade="all")
custom_configs = relationship("Template_custom_configs", back_populates="step", cascade="all")
class Template_settings(Base):
__tablename__ = "bw_template_settings"
__table_args__ = (UniqueConstraint("template_id", "setting_id", "step_id", "suffix"),)
id = Column(Integer, Identity(start=1, increment=1), primary_key=True)
template_id = Column(String(256), ForeignKey("bw_templates.id", onupdate="cascade", ondelete="cascade"), nullable=False)
setting_id = Column(String(256), ForeignKey("bw_settings.id", onupdate="cascade", ondelete="cascade"), nullable=False)
step_id = Column(Integer, ForeignKey("bw_template_steps.id", onupdate="cascade", ondelete="cascade"), nullable=True)
default = Column(TEXT, nullable=False)
suffix = Column(Integer, nullable=True, default=0)
template = relationship("Templates", back_populates="settings")
step = relationship("Template_steps", back_populates="settings")
setting = relationship("Settings", back_populates="templates")
class Template_custom_configs(Base):
__tablename__ = "bw_template_custom_configs"
__table_args__ = (UniqueConstraint("template_id", "step_id", "type", "name"),)
id = Column(Integer, Identity(start=1, increment=1), primary_key=True)
template_id = Column(String(256), ForeignKey("bw_templates.id", onupdate="cascade", ondelete="cascade"), nullable=False)
step_id = Column(Integer, ForeignKey("bw_template_steps.id", onupdate="cascade", ondelete="cascade"), nullable=True)
type = Column(CUSTOM_CONFIGS_TYPES_ENUM, nullable=False)
name = Column(String(256), nullable=False)
data = Column(LargeBinary(length=(2**32) - 1), nullable=False)
checksum = Column(String(128), nullable=False)
template = relationship("Templates", back_populates="custom_configs")
step = relationship("Template_steps", back_populates="custom_configs")
class Metadata(Base):
__tablename__ = "bw_metadata"

177
src/common/gen/Configurator.py
View file

@ -1,103 +1,94 @@
#!/usr/bin/env python3
from glob import glob
from hashlib import sha256
from copy import deepcopy
from functools import cache
from io import BytesIO
from json import loads
from logging import Logger
from os import cpu_count, listdir, sep
from os.path import basename, dirname, join
from os import listdir, sep
from os.path import join
from pathlib import Path
from re import compile as re_compile, error as RegexError, search as re_search
from sys import path as sys_path
from tarfile import open as tar_open
from threading import Lock, Semaphore, Thread
from traceback import format_exc
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
from typing import Dict, List, Literal, Optional, Tuple, Union
if join(sep, "usr", "share", "bunkerweb", "utils") not in sys_path:
sys_path.append(join(sep, "usr", "share", "bunkerweb", "utils"))
from common_utils import bytes_hash # type: ignore
class Configurator:
def __init__(
self,
settings: str,
core: str,
external_plugins: Union[str, List[Dict[str, Any]]],
pro_plugins: Union[str, List[Dict[str, Any]]],
variables: Union[str, Dict[str, Any]],
external_plugins: Union[str, List[Dict[str, str]]],
pro_plugins: Union[str, List[Dict[str, str]]],
variables: Union[str, Dict[str, str]],
logger: Logger,
):
self.__logger = logger
self.__thread_lock = Lock()
self.__semaphore = Semaphore(cpu_count() or 1)
self.__plugin_id_rx = re_compile(r"^[\w.-]{1,64}$")
self.__plugin_version_rx = re_compile(r"^\d+\.\d+(\.\d+)?$")
self.__setting_id_rx = re_compile(r"^[A-Z0-9_]{1,256}$")
self.__name_rx = re_compile(r"^[\w.-]{1,128}$")
self.__job_file_rx = re_compile(r"^[\w./-]{1,256}$")
self.__settings = self.__load_settings(settings)
self.__settings = self.__load_settings(Path(settings))
self.__core_plugins = []
self.__load_plugins(core)
self.__load_plugins(Path(core))
if isinstance(external_plugins, str):
self.__external_plugins = []
self.__load_plugins(external_plugins, "external")
self.__load_plugins(Path(external_plugins), "external")
else:
self.__external_plugins = external_plugins
if isinstance(pro_plugins, str):
self.__pro_plugins = []
self.__load_plugins(pro_plugins, "pro")
self.__load_plugins(Path(pro_plugins), "pro")
else:
self.__pro_plugins = pro_plugins
if isinstance(variables, str):
self.__variables = self.__load_variables(variables)
self.__variables = self.__load_variables(Path(variables))
else:
self.__variables = variables
self.__multisite = self.__variables.get("MULTISITE", "no") == "yes"
self.__servers = self.__map_servers()
def get_settings(self) -> Dict[str, Any]:
return self.__settings
def get_settings(self) -> Dict[str, str]:
return self.__settings.copy()
def get_plugins(self, _type: Literal["core", "external", "pro"]) -> List[Dict[str, Any]]:
return {"core": self.__core_plugins, "external": self.__external_plugins, "pro": self.__pro_plugins}[_type]
def get_plugins(self, _type: Literal["core", "external", "pro"]) -> List[Dict[str, str]]:
return {"core": deepcopy(self.__core_plugins), "external": deepcopy(self.__external_plugins), "pro": deepcopy(self.__pro_plugins)}.get(_type, [])
def get_plugins_settings(self, _type: Literal["core", "external", "pro"]) -> Dict[str, Any]:
if _type == "core":
plugins = self.__core_plugins
elif _type == "pro":
plugins = self.__pro_plugins
else:
plugins = self.__external_plugins
@cache
def get_plugins_settings(self, _type: Literal["core", "external", "pro"]) -> Dict[str, str]:
plugins_settings = {}
for plugin in plugins:
plugins_settings.update(plugin["settings"])
for plugin in self.get_plugins(_type):
plugins_settings.update(plugin.get("settings", {}))
return plugins_settings
@cache
def __map_servers(self) -> Dict[str, List[str]]:
if not self.__multisite or "SERVER_NAME" not in self.__variables:
return {}
servers = {}
for server_name in self.__variables["SERVER_NAME"].strip().split(" "):
if not server_name:
continue
if not re_search(self.__settings["SERVER_NAME"]["regex"], server_name):
if re_search(self.__settings["SERVER_NAME"]["regex"], server_name) is None:
self.__logger.warning(f"Ignoring server name {server_name} because regex is not valid")
continue
names = [server_name]
if f"{server_name}_SERVER_NAME" in self.__variables:
if not re_search(
self.__settings["SERVER_NAME"]["regex"],
self.__variables[f"{server_name}_SERVER_NAME"],
):
if re_search(self.__settings["SERVER_NAME"]["regex"], self.__variables[f"{server_name}_SERVER_NAME"]) is None:
self.__logger.warning(f"Ignoring {server_name}_SERVER_NAME because regex is not valid")
else:
names = self.__variables[f"{server_name}_SERVER_NAME"].strip().split(" ")
@ -105,21 +96,19 @@ class Configurator:
servers[server_name] = names
return servers
def __load_settings(self, path: str) -> Dict[str, Any]:
return loads(Path(path).read_text())
def __load_settings(self, path: Path) -> Dict[str, str]:
return loads(path.read_text())
def __load_plugins(self, path: str, _type: Literal["core", "external", "pro"] = "core"):
threads = []
for file in glob(join(path, "*", "plugin.json")):
thread = Thread(target=self.__load_plugin, args=(file, _type))
thread.start()
threads.append(thread)
def __load_plugins(self, path: Path, _type: Literal["core", "external", "pro"] = "core"):
x = 0
for file in path.glob("*/plugin.json"):
self.__logger.debug(f"Loading {_type} plugin {file}")
self.__load_plugin(file, _type)
x += 1
for thread in threads:
thread.join()
self.__logger.info(f"Computed {x} {_type} plugin{'s' if x > 1 else ''}")
def __load_plugin(self, file: str, _type: Literal["core", "external", "pro"] = "core"):
self.__semaphore.acquire(timeout=60)
def __load_plugin(self, file: Path, _type: Literal["core", "external", "pro"] = "core"):
try:
data = self.__load_settings(file)
@ -128,39 +117,32 @@ class Configurator:
self.__logger.warning(f"Ignoring {_type} plugin {file} : {msg}")
return
data["page"] = "ui" in listdir(dirname(file))
data["page"] = "ui" in listdir(file.parent)
if _type != "core":
plugin_content = BytesIO()
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(dirname(file), arcname=basename(dirname(file)), recursive=True)
plugin_content.seek(0, 0)
value = plugin_content.getvalue()
with BytesIO() as plugin_content:
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(file.parent, arcname=file.parent.name, recursive=True)
plugin_content.seek(0)
checksum = bytes_hash(plugin_content, algorithm="sha256")
value = plugin_content.getvalue()
data.update(
{
"type": _type,
"method": "manual",
"data": value,
"checksum": sha256(value).hexdigest(),
}
)
data.update({"type": _type, "method": "manual", "data": value, "checksum": checksum})
with self.__thread_lock:
if _type == "pro":
self.__pro_plugins.append(data)
else:
self.__external_plugins.append(data)
else:
with self.__thread_lock:
self.__core_plugins.append(data)
except:
self.__logger.error(f"Exception while loading JSON from {file} : {format_exc()}")
self.__semaphore.release()
if _type == "pro":
self.__pro_plugins.append(data)
else:
self.__external_plugins.append(data)
self.__logger.debug(f"Loaded {_type} plugin {file} with {len(data.get('settings', {}))} setting(s)")
return
self.__core_plugins.append(data)
self.__logger.debug(f"Loaded core plugin {file} with {len(data.get('settings', {}))} setting(s)")
except BaseException as e:
self.__logger.error(f"Exception while loading JSON from {file} : {e}")
def __load_variables(self, path: str) -> Dict[str, Any]:
def __load_variables(self, path: Path) -> Dict[str, str]:
variables = {}
with open(path) as f:
with path.open("r", encoding="utf-8") as f:
lines = f.readlines()
for line in lines:
line = line.strip()
@ -170,18 +152,34 @@ class Configurator:
variables[split[0]] = split[1]
return variables
def get_config(self) -> Dict[str, Any]:
def get_config(self, db=None) -> Dict[str, str]:
config = {}
template = self.__variables.get("USE_TEMPLATE", "")
# Extract default settings
default_settings = [
self.__settings,
self.get_settings(),
self.get_plugins_settings("core"),
self.get_plugins_settings("external"),
self.get_plugins_settings("pro"),
]
if not default_settings[0]:
self.__logger.error("No settings found, exiting")
exit(1)
elif not default_settings[1]:
self.__logger.error("No core plugins found, exiting")
exit(1)
# Extract template overridden settings
template_settings = {}
if template and db:
self.__logger.info(f"Using template {template}")
template_settings = db.get_template_settings(template)
for settings in default_settings:
for setting, data in settings.items():
config[setting] = data["default"]
config[setting] = template_settings.get(setting, data["default"])
# Override with variables
for variable, value in self.__variables.items():
@ -213,14 +211,20 @@ class Configurator:
"NAMESPACE",
)
):
self.__logger.warning(f"Ignoring variable {variable} : {err}")
self.__logger.warning(f"Ignoring variable {variable} : {err} - {value = !r}")
# Expand variables to each sites if MULTISITE=yes and if not present
if config.get("MULTISITE", "no") == "yes":
for server_name in config["SERVER_NAME"].split(" "):
for server_name in config["SERVER_NAME"].strip().split(" "):
server_name = server_name.strip()
if not server_name:
continue
service_template = config.get(f"{server_name}_USE_TEMPLATE", template)
service_template_settings = {}
if service_template != template and db:
service_template_settings = db.get_template_settings(service_template)
for settings in default_settings:
for setting, data in settings.items():
if data["context"] == "global":
@ -231,7 +235,8 @@ class Configurator:
if setting == "SERVER_NAME":
config[key] = server_name
elif setting in config:
config[key] = config[setting]
config[key] = service_template_settings.get(setting, config[setting])
return config
def __check_var(self, variable: str) -> Tuple[bool, str]:
@ -243,7 +248,7 @@ class Configurator:
return False, f"variable name {variable} doesn't exist"
try:
if not re_search(where[real_var]["regex"], value):
if re_search(where[real_var]["regex"], value) is None:
return (False, f"value {value} doesn't match regex {where[real_var]['regex']}")
except RegexError:
self.__logger.warning(f"Invalid regex for {variable} : {where[real_var]['regex']}, ignoring regex check")
@ -265,9 +270,9 @@ class Configurator:
return True, "ok"
def __find_var(self, variable: str) -> Tuple[Optional[Dict[str, Any]], str]:
def __find_var(self, variable: str) -> Tuple[Optional[Dict[str, str]], str]:
targets = [
self.__settings,
self.get_settings(),
self.get_plugins_settings("core"),
self.get_plugins_settings("external"),
self.get_plugins_settings("pro"),
@ -301,7 +306,7 @@ class Configurator:
elif plugin["stream"] not in ("yes", "no", "partial"):
return (False, f"Invalid stream for plugin {plugin['id']} (Must be yes, no or partial)")
for setting, data in plugin["settings"].items():
for setting, data in plugin.get("settings", {}).items():
if not all(key in data.keys() for key in ("context", "default", "help", "id", "label", "regex", "type")):
return (False, f"missing keys for setting {setting} in plugin {plugin['id']}, must have context, default, help, id, label, regex and type")

21
src/common/gen/main.py
View file

@ -21,6 +21,7 @@ from logger import setup_logger # type: ignore
from Configurator import Configurator
from Templator import Templator
DB_PATH = Path(sep, "usr", "share", "bunkerweb", "db")
if __name__ == "__main__":
logger = setup_logger("Generator", getenv("LOG_LEVEL", "INFO"))
@ -68,6 +69,15 @@ if __name__ == "__main__":
integration = get_integration()
db = None
if DB_PATH.is_dir():
if DB_PATH.as_posix() not in sys_path:
sys_path.append(DB_PATH.as_posix())
from Database import Database # type: ignore
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None))
if args.variables:
variables_path = Path(args.variables)
variables_path.parent.mkdir(parents=True, exist_ok=True)
@ -105,17 +115,8 @@ if __name__ == "__main__":
logger.info("Computing config ...")
config: Dict[str, Any] = Configurator(
str(settings_path), str(core_path), str(plugins_path), str(pro_plugins_path), str(variables_path), logger
).get_config()
).get_config(db)
else:
if join(sep, "usr", "share", "bunkerweb", "db") not in sys_path:
sys_path.append(join(sep, "usr", "share", "bunkerweb", "db"))
from Database import Database # type: ignore
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
)
config: Dict[str, Any] = db.get_config()
# Remove old files

53
src/common/gen/save_config.py
View file

@ -101,31 +101,6 @@ if __name__ == "__main__":
config = Configurator(
str(settings_path), str(core_path), external_plugins, pro_plugins, str(variables_path) if args.variables else environ.copy(), LOGGER
)
settings = config.get_config()
# Parse BunkerWeb instances from environment
apis = []
hostnames = set()
for bw_instance in settings.get("BUNKERWEB_INSTANCES", "").split(" "):
if not bw_instance:
continue
match = BUNKERWEB_STATIC_INSTANCES_RX.search(bw_instance)
if match:
if match.group("hostname") in hostnames:
LOGGER.warning(f"Duplicate BunkerWeb instance hostname {match.group('hostname')}, skipping it")
hostnames.add(match.group("hostname"))
apis.append(
API(
f"http://{match.group('hostname')}:{match.group('port') or settings.get('API_HTTP_PORT', '5000')}",
host=settings.get("API_SERVER_NAME", "bwapi"),
)
)
else:
LOGGER.warning(
f"Invalid BunkerWeb instance {bw_instance}, it should match the following regex: (http://)<hostname>(:<port>) ({BUNKERWEB_STATIC_INSTANCES_RX.pattern}), skipping it"
)
custom_confs = []
for k, v in environ.items():
@ -150,7 +125,7 @@ if __name__ == "__main__":
bunkerweb_version = get_version()
db_metadata = db.get_metadata()
db_initialized = isinstance(db_metadata, str) or not db_metadata["is_initialized"]
db_initialized = not isinstance(db_metadata, str) and db_metadata["is_initialized"]
if not db_initialized:
LOGGER.info("Database not initialized, initializing ...")
@ -192,6 +167,32 @@ if __name__ == "__main__":
if args.init:
sys_exit(0)
settings = config.get_config(db)
# Parse BunkerWeb instances from environment
apis = []
hostnames = set()
for bw_instance in settings.get("BUNKERWEB_INSTANCES", "").split(" "):
if not bw_instance:
continue
match = BUNKERWEB_STATIC_INSTANCES_RX.search(bw_instance)
if match:
if match.group("hostname") in hostnames:
LOGGER.warning(f"Duplicate BunkerWeb instance hostname {match.group('hostname')}, skipping it")
hostnames.add(match.group("hostname"))
apis.append(
API(
f"http://{match.group('hostname')}:{match.group('port') or settings.get('API_HTTP_PORT', '5000')}",
host=settings.get("API_SERVER_NAME", "bwapi"),
)
)
else:
LOGGER.warning(
f"Invalid BunkerWeb instance {bw_instance}, it should match the following regex: (http://)<hostname>(:<port>) ({BUNKERWEB_STATIC_INSTANCES_RX.pattern}), skipping it"
)
changes = []
changed_plugins = set()
err = db.save_config(settings, args.method, changed=False)

9
src/common/settings.json
View file

@ -325,5 +325,14 @@
"label": "BunkerWeb instances",
"regex": "^.*$",
"type": "text"
},
"USE_TEMPLATE": {
"context": "multisite",
"default": "",
"help": "Config template to use that will override the default values of specific settings.",
"id": "use-template",
"label": "Use template",
"regex": "^.*$",
"type": "text"
}
}

129
src/common/templates/high.json
View file

@ -1,129 +0,0 @@
{
"name": "medium",
"description": "Generic settings template with high security level required for your web service. False positives will certainly appear without any custom edit.",
"steps": [
{
"name": "Server configuration",
"description": "Configure your server name and reverse proxy settings. Don't forget to add the corresponding DNS A entry pointing to your BunkerWeb IP.",
"settings": {
"SERVER_NAME": "www.example.com",
"USE_REVERSE_PROXY": "yes",
"REVERSE_PROXY_HOST": "http://my-upstream-server:8080",
"REVERSE_PROXY_URL": "/",
"REVERSE_PROXY_INTERCEPT_ERRORS": "yes",
"REVERSE_PROXY_WS": "no",
"REVERSE_PROXY_CUSTOM_HOST": "",
"REVERSE_PROXY_HEADERS": "Accept-Encoding ''",
"SERVE_FILES": "no"
}
},
{
"name": "HTTPS",
"description": "Enable/disable and configure HTTPS for your service.",
"settings": {
"AUTO_LETS_ENCRYPT": "yes",
"SSL_PROTOCOLS": "TLSv1.3"
}
},
{
"name": "HTTP configuration",
"description": "Miscellaneous settings related to HTTP protocol.",
"settings": {
"DENY_HTTP_STATUS": "444",
"USE_GZIP": "yes",
"USE_BROTLI": "yes",
"ALLOWED_METHODS": "GET|POST|HEAD",
"MAX_SIZES": "10m",
"COOKIE_FLAGS": "* HttpOnly SameSite=Lax",
"CONTENT_SECURITY_POLICY": "object-src 'none'; form-action 'self'; frame-ancestors 'self';",
"X_FRAME_OPTIONS": "SAMEORIGIN",
"PERMISSIONS_POLICY": "accelerometer=(), ambient-light-sensor=(), autoplay=(), battery=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), fullscreen=(), geolocation=(), gyroscope=(), hid=(), idle-detection=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), serial=(), usb=(), web-share=(), xr-spatial-tracking=()",
"FEATURE_POLICY": "accelerometer 'none'; ambient-light-sensor 'none'; autoplay 'none'; battery 'none'; camera 'none'; display-capture 'none'; document-domain 'none'; encrypted-media 'none'; execution-while-not-rendered 'none'; execution-while-out-of-viewport 'none'; fullscreen 'none'; geolocation 'none'; gyroscope 'none'; layout-animation 'none'; legacy-image-formats 'none'; magnetometer 'none'; microphone 'none'; midi 'none'; navigation-override 'none'; payment 'none'; picture-in-picture 'none'; publickey-credentials-get 'none'; speaker-selection 'none'; sync-xhr 'none'; unoptimized-images 'none'; unsized-media 'none'; usb 'none'; screen-wake-lock 'none'; web-share 'none'; xr-spatial-tracking 'none';"
}
},
{
"name": "Bad behavior",
"description": "Configure automatic bans when detecting bad behaviors on your web service.",
"settings": {
"USE_BAD_BEHAVIOR": "yes",
"BAD_BEHAVIOR_STATUS_CODES": "400 401 403 404 405 429 444",
"BAD_BEHAVIOR_BAN_TIME": "86400",
"BAD_BEHAVIOR_THRESHOLD": "5",
"BAD_BEHAVIOR_COUNT_TIME": "60"
}
},
{
"name": "Limit",
"description": "Configure requests and connections limits on your web service.",
"settings": {
"USE_LIMIT_CONN": "yes",
"LIMIT_CONN_MAX_HTTP1": "10",
"LIMIT_CONN_MAX_HTTP2": "100",
"USE_LIMIT_REQ": "yes",
"LIMIT_REQ_URL": "/",
"LIMIT_REQ_RATE": "2r/s"
}
},
{
"name": "DNSBL",
"description": "Enable/disable DNSBL protection. Might generate false positives especially if you have a worldwide audience.",
"settings": {
"USE_DNSBL": "yes"
}
},
{
"name": "Country",
"description": "Configure allowed countries to reach out your web service. Recommended if you protect a restricted area such as extranet or administration panel.",
"settings": {
"WHITELIST_COUNTRY": ""
}
},
{
"name": "Antibot",
"description": "Enable/disable and configure antibot protection globally on your web service.",
"settings": {
"USE_ANTIBOT": "captcha",
"ANTIBOT_TIME_RESOLVE": "120",
"ANTIBOT_TIME_VALID": "86400",
"ANTIBOT_RECAPTCHA_SCORE": "0.7",
"ANTIBOT_RECAPTCHA_SITEKEY": "",
"ANTIBOT_RECAPTCHA_SECRET": "",
"ANTIBOT_HCAPTCHA_SITEKEY": "",
"ANTIBOT_HCAPTCHA_SECRET": "",
"ANTIBOT_TURNSTILE_SITEKEY": "",
"ANTIBOT_TURNSTILE_SECRET": ""
}
},
{
"name": "CORS",
"description": "Configure Cross-Origin Resource Sharing (CORS) to allow/deny external requests to your web service.",
"settings": {
"USE_CORS": "yes",
"CORS_ALLOW_ORIGIN": ""
}
},
{
"name": "Reverse scan",
"description": "Configure reverse scan of client to detect open proxy or datacenter connections.",
"settings": {
"USE_REVERSE_SCAN": "yes",
"REVERSE_SCAN_PORTS": "22 80 443 3128 8000 8080"
}
},
{
"name": "ModSecurity",
"description": "Enable/disable and configure ModSecurity on your web service.",
"settings": {
"USE_MODSECURITY": "yes",
"MODSECURITY_CRS_VERSION": "4"
},
"configs": [
{
"name": "template-high",
"type": "modsec-crs",
"data": "SecAction \"id:900000,phase:1,pass,t:none,nolog,tag:'OWASP_CRS',ver:'OWASP_CRS/4.2.0',setvar:tx.blocking_paranoia_level=4\""
}
]
}
]
}

111
src/common/templates/low.json
View file

@ -1,111 +0,0 @@
{
"name": "low",
"description": "Generic settings template with low security level to avoid false positives and get started with BunkerWeb.",
"steps": [
{
"name": "Server configuration",
"description": "Configure your server name and reverse proxy settings. Don't forget to add the corresponding DNS A entry pointing to your BunkerWeb IP.",
"settings": {
"SERVER_NAME": "www.example.com",
"USE_REVERSE_PROXY": "yes",
"REVERSE_PROXY_HOST": "http://my-upstream-server:8080",
"REVERSE_PROXY_URL": "/",
"REVERSE_PROXY_INTERCEPT_ERRORS": "no",
"REVERSE_PROXY_WS": "yes",
"REVERSE_PROXY_CUSTOM_HOST": "",
"REVERSE_PROXY_HEADERS": "Accept-Encoding ''"
}
},
{
"name": "HTTPS",
"description": "Enable/disable HTTPS for your service.",
"settings": {
"AUTO_LETS_ENCRYPT": "yes"
}
},
{
"name": "HTTP configuration",
"description": "Miscellaneous settings related to HTTP protocol.",
"settings": {
"USE_GZIP": "yes",
"USE_BROTLI": "yes",
"ALLOWED_METHODS": "GET|POST|HEAD|PUT|PATCH|OPTIONS|DELETE",
"MAX_SIZES": "50m",
"COOKIE_FLAGS": "* SameSite=Lax",
"CONTENT_SECURITY_POLICY": "",
"X_FRAME_OPTIONS": "",
"PERMISSIONS_POLICY": "",
"FEATURE_POLICY": "",
"KEEP_UPSTREAM_HEADERS": "*"
}
},
{
"name": "Bad behavior",
"description": "Configure automatic bans when detecting bad behaviors on your web service.",
"settings": {
"USE_BAD_BEHAVIOR": "yes",
"BAD_BEHAVIOR_STATUS_CODES": "400 401 403 405 429 444",
"BAD_BEHAVIOR_BAN_TIME": "3600",
"BAD_BEHAVIOR_THRESHOLD": "20",
"BAD_BEHAVIOR_COUNT_TIME": "60"
}
},
{
"name": "Limit",
"description": "Configure requests and connections limits on your web service.",
"settings": {
"USE_LIMIT_CONN": "yes",
"LIMIT_CONN_MAX_HTTP1": 20,
"LIMIT_CONN_MAX_HTTP2": 200,
"USE_LIMIT_REQ": "yes",
"LIMIT_REQ_URL": "/",
"LIMIT_REQ_RATE": "5r/s"
}
},
{
"name": "DNSBL",
"description": "Enable/disable DNSBL protection. Might generate false positives especially if you have a worldwide audience.",
"settings": {
"USE_DNSBL": "no"
}
},
{
"name": "Country",
"description": "Configure allowed countries to reach out your web service. Recommended if you protect a restricted area such as extranet or administration panel.",
"settings": {
"WHITELIST_COUNTRY": ""
}
},
{
"name": "Antibot",
"description": "Enable/disable and configure antibot protection globally on your web service.",
"settings": {
"USE_ANTIBOT": "no",
"ANTIBOT_TIME_RESOLVE": "120",
"ANTIBOT_TIME_VALID": "86400",
"ANTIBOT_RECAPTCHA_SCORE": "0.7",
"ANTIBOT_RECAPTCHA_SITEKEY": "",
"ANTIBOT_RECAPTCHA_SECRET": "",
"ANTIBOT_HCAPTCHA_SITEKEY": "",
"ANTIBOT_HCAPTCHA_SECRET": "",
"ANTIBOT_TURNSTILE_SITEKEY": "",
"ANTIBOT_TURNSTILE_SECRET": ""
}
},
{
"name": "ModSecurity",
"description": "Enable/disable and configure ModSecurity on your web service.",
"settings": {
"USE_MODSECURITY": "yes"
},
"configs": [
{
"name": "template-low",
"type": "modsec-crs",
"description": "Override ModSecurity CRS settings.",
"data": "SecAction \"id:900110,phase:1,nolog,pass,t:none,setvar:tx.inbound_anomaly_score_threshold=7,setvar:tx.outbound_anomaly_score_threshold=4\""
}
]
}
]
}

110
src/common/templates/medium.json
View file

@ -1,110 +0,0 @@
{
"name": "medium",
"description": "Generic settings template with medium security level aimed for average web service in production. False positives may appear depending on your environment.",
"steps": [
{
"name": "Server configuration",
"description": "Configure your server name and reverse proxy settings. Don't forget to add the corresponding DNS A entry pointing to your BunkerWeb IP.",
"settings": {
"SERVER_NAME": "www.example.com",
"USE_REVERSE_PROXY": "yes",
"REVERSE_PROXY_HOST": "http://my-upstream-server:8080",
"REVERSE_PROXY_URL": "/",
"REVERSE_PROXY_INTERCEPT_ERRORS": "yes",
"REVERSE_PROXY_WS": "no",
"REVERSE_PROXY_CUSTOM_HOST": "",
"REVERSE_PROXY_HEADERS": "Accept-Encoding ''"
}
},
{
"name": "HTTPS",
"description": "Enable/disable HTTPS for your service.",
"settings": {
"AUTO_LETS_ENCRYPT": "yes"
}
},
{
"name": "HTTP configuration",
"description": "Miscellaneous settings related to HTTP protocol.",
"settings": {
"USE_GZIP": "yes",
"USE_BROTLI": "yes",
"ALLOWED_METHODS": "GET|POST|HEAD",
"MAX_SIZES": "10m",
"COOKIE_FLAGS": "* HttpOnly SameSite=Lax",
"CONTENT_SECURITY_POLICY": "object-src 'none'; form-action 'self'; frame-ancestors 'self';",
"X_FRAME_OPTIONS": "SAMEORIGIN",
"PERMISSIONS_POLICY": "accelerometer=(), ambient-light-sensor=(), autoplay=(), battery=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), fullscreen=(), geolocation=(), gyroscope=(), hid=(), idle-detection=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), serial=(), usb=(), web-share=(), xr-spatial-tracking=()",
"FEATURE_POLICY": "accelerometer 'none'; ambient-light-sensor 'none'; autoplay 'none'; battery 'none'; camera 'none'; display-capture 'none'; document-domain 'none'; encrypted-media 'none'; execution-while-not-rendered 'none'; execution-while-out-of-viewport 'none'; fullscreen 'none'; geolocation 'none'; gyroscope 'none'; layout-animation 'none'; legacy-image-formats 'none'; magnetometer 'none'; microphone 'none'; midi 'none'; navigation-override 'none'; payment 'none'; picture-in-picture 'none'; publickey-credentials-get 'none'; speaker-selection 'none'; sync-xhr 'none'; unoptimized-images 'none'; unsized-media 'none'; usb 'none'; screen-wake-lock 'none'; web-share 'none'; xr-spatial-tracking 'none';"
}
},
{
"name": "Bad behavior",
"description": "Configure automatic bans when detecting bad behaviors on your web service.",
"settings": {
"USE_BAD_BEHAVIOR": "yes",
"BAD_BEHAVIOR_STATUS_CODES": "400 401 403 404 405 429 444",
"BAD_BEHAVIOR_BAN_TIME": "86400",
"BAD_BEHAVIOR_THRESHOLD": "10",
"BAD_BEHAVIOR_COUNT_TIME": "60"
}
},
{
"name": "Limit",
"description": "Configure requests and connections limits on your web service.",
"settings": {
"USE_LIMIT_CONN": "yes",
"LIMIT_CONN_MAX_HTTP1": "10",
"LIMIT_CONN_MAX_HTTP2": "100",
"USE_LIMIT_REQ": "yes",
"LIMIT_REQ_URL": "/",
"LIMIT_REQ_RATE": "2r/s"
}
},
{
"name": "DNSBL",
"description": "Enable/disable DNSBL protection. Might generate false positives especially if you have a worldwide audience.",
"settings": {
"USE_DNSBL": "yes"
}
},
{
"name": "Country",
"description": "Configure allowed countries to reach out your web service. Recommended if you protect a restricted area such as extranet or administration panel.",
"settings": {
"WHITELIST_COUNTRY": ""
}
},
{
"name": "Antibot",
"description": "Enable/disable and configure antibot protection globally on your web service.",
"settings": {
"USE_ANTIBOT": "javascript",
"ANTIBOT_TIME_RESOLVE": "120",
"ANTIBOT_TIME_VALID": "86400",
"ANTIBOT_RECAPTCHA_SCORE": "0.7",
"ANTIBOT_RECAPTCHA_SITEKEY": "",
"ANTIBOT_RECAPTCHA_SECRET": "",
"ANTIBOT_HCAPTCHA_SITEKEY": "",
"ANTIBOT_HCAPTCHA_SECRET": "",
"ANTIBOT_TURNSTILE_SITEKEY": "",
"ANTIBOT_TURNSTILE_SECRET": ""
}
},
{
"name": "CORS",
"description": "Configure Cross-Origin Resource Sharing (CORS) to allow/deny external requests to your web service.",
"settings": {
"USE_CORS": "no",
"CORS_ALLOW_ORIGIN": "*"
}
},
{
"name": "ModSecurity",
"description": "Enable/disable and configure ModSecurity on your web service.",
"settings": {
"USE_MODSECURITY": "yes"
}
}
]
}

1
src/ui/Dockerfile
View file

@ -33,7 +33,6 @@ COPY src/common/gen gen
COPY src/common/settings.json settings.json
COPY src/common/utils utils
COPY src/common/helpers helpers
COPY src/common/templates templates
COPY src/VERSION VERSION
COPY src/ui/builder ui/builder