mirror of
https://github.com/bunkerity/bunkerweb
synced 2026-05-24 09:28:37 +00:00
984 lines
41 KiB
Python
984 lines
41 KiB
Python
from contextlib import contextmanager
|
||
from glob import iglob
|
||
from hashlib import sha512
|
||
from json import dumps, load
|
||
from os import environ, getenv
|
||
from os.path import dirname, join
|
||
from pathlib import Path
|
||
from re import compile as re_compile
|
||
from sqlalchemy import create_engine, text
|
||
from sqlalchemy.exc import (
|
||
ArgumentError,
|
||
DatabaseError,
|
||
OperationalError,
|
||
SQLAlchemyError,
|
||
)
|
||
from sqlalchemy.orm import scoped_session, sessionmaker
|
||
from traceback import format_exc
|
||
from time import sleep
|
||
|
||
from db.model import (
|
||
Custom_configs,
|
||
Global_values,
|
||
Jobs,
|
||
Metadata,
|
||
Plugins,
|
||
Plugin_pages,
|
||
Services,
|
||
Services_settings,
|
||
Settings,
|
||
)
|
||
|
||
try:
|
||
database_uri = getenv("DATABASE_URI", "sqlite:////var/lib/bunkerweb/db.sqlite3")
|
||
|
||
if database_uri == "sqlite:////var/lib/bunkerweb/db.sqlite3":
|
||
database_uri = "sqlite:////data/lib/db.sqlite3"
|
||
|
||
error = False
|
||
|
||
print(f"ℹ️ Connecting to database: {database_uri}", flush=True)
|
||
|
||
try:
|
||
sql_engine = create_engine(
|
||
database_uri,
|
||
future=True,
|
||
)
|
||
except ArgumentError:
|
||
print(f"❌ Invalid database URI: {database_uri}", flush=True)
|
||
error = True
|
||
except SQLAlchemyError:
|
||
print(f"❌ Error when trying to create the engine: {format_exc()}", flush=True)
|
||
error = True
|
||
finally:
|
||
if error:
|
||
exit(1)
|
||
|
||
try:
|
||
assert sql_engine is not None
|
||
except AssertionError:
|
||
print("❌ The database engine is not initialized", flush=True)
|
||
exit(1)
|
||
|
||
not_connected = True
|
||
retries = 15
|
||
|
||
while not_connected:
|
||
try:
|
||
with sql_engine.connect() as conn:
|
||
conn.execute(text("CREATE TABLE IF NOT EXISTS test (id INT)"))
|
||
conn.execute(text("DROP TABLE test"))
|
||
not_connected = False
|
||
except (OperationalError, DatabaseError) as e:
|
||
if retries <= 0:
|
||
print(f"❌ Can't connect to database : {format_exc()}", flush=True)
|
||
exit(1)
|
||
|
||
if "attempt to write a readonly database" in str(e):
|
||
print(
|
||
"⚠️ The database is read-only, waiting for it to become writable. Retrying in 5 seconds ...",
|
||
flush=True,
|
||
)
|
||
sql_engine.dispose(close=True)
|
||
sql_engine = create_engine(
|
||
database_uri,
|
||
future=True,
|
||
)
|
||
if "Unknown table" in str(e):
|
||
not_connected = False
|
||
continue
|
||
else:
|
||
print(
|
||
"⚠️ Can't connect to database, retrying in 5 seconds ...",
|
||
flush=True,
|
||
)
|
||
retries -= 1
|
||
sleep(5)
|
||
except BaseException:
|
||
print(
|
||
f"❌ Error when trying to connect to the database: {format_exc()}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
print("ℹ️ Database connection established, launching tests ...", flush=True)
|
||
|
||
session = sessionmaker()
|
||
sql_session = scoped_session(session)
|
||
sql_session.remove()
|
||
sql_session.configure(bind=sql_engine, autoflush=False, expire_on_commit=False)
|
||
|
||
@contextmanager
|
||
def db_session():
|
||
try:
|
||
assert sql_session is not None
|
||
except AssertionError:
|
||
print("❌ The database session is not initialized", flush=True)
|
||
exit(1)
|
||
|
||
session = sql_session()
|
||
session.expire_on_commit = False
|
||
|
||
try:
|
||
yield session
|
||
except BaseException:
|
||
session.rollback()
|
||
raise
|
||
finally:
|
||
session.close()
|
||
|
||
print("ℹ️ Checking if database is initialized ...", flush=True)
|
||
|
||
with db_session() as session:
|
||
metadata = (
|
||
session.query(Metadata)
|
||
.with_entities(Metadata.is_initialized)
|
||
.filter_by(id=1)
|
||
.first()
|
||
)
|
||
|
||
if metadata is None or not metadata.is_initialized:
|
||
print(
|
||
"❌ The database is not initialized, it should be, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
print("✅ Database is initialized", flush=True)
|
||
print(" ", flush=True)
|
||
print("ℹ️ Checking if service bwadm.example.com is in the database ...", flush=True)
|
||
|
||
with db_session() as session:
|
||
services = session.query(Services).all()
|
||
|
||
if not services:
|
||
print(
|
||
"❌ The bw_services database table is empty, it shouldn't be, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
if services[0].id != "bwadm.example.com":
|
||
print(
|
||
"❌ The service bwadm.example.com is not in the database, it should be, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
print("✅ Service bwadm.example.com is in the database", flush=True)
|
||
print(" ", flush=True)
|
||
print(
|
||
"ℹ️ Checking if global values are in the database and are correct ...",
|
||
flush=True,
|
||
)
|
||
|
||
global_settings = {}
|
||
service_settings = {}
|
||
multisite = getenv("GLOBAL_MULTISITE", "no") == "yes"
|
||
for env in environ:
|
||
if env.startswith("GLOBAL_"):
|
||
if env == "GLOBAL_MULTISITE" and environ[env] == "no":
|
||
continue
|
||
global_settings[env[7:]] = {"value": environ[env], "checked": False}
|
||
elif env.startswith("SERVICE_"):
|
||
service_settings[env[8:]] = {"value": environ[env], "checked": False}
|
||
|
||
with db_session() as session:
|
||
global_values = session.query(Global_values).all()
|
||
|
||
for global_value in global_values:
|
||
if global_value.setting_id in global_settings:
|
||
if (
|
||
global_value.value
|
||
!= global_settings[global_value.setting_id]["value"]
|
||
):
|
||
print(
|
||
f"❌ The global value {global_value.setting_id} is in the database but is not correct, exiting ...\n{global_value.value} (database) != {global_settings[global_value.setting_id]['value']} (env)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif global_value.suffix != 0:
|
||
print(
|
||
f"❌ The global value {global_value.setting_id} is in the database but has the wrong suffix, exiting ...\n{global_value.suffix} (database) != 0 (env)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif global_value.method != "scheduler":
|
||
print(
|
||
f"❌ The global value {global_value.setting_id} is in the database but has the wrong method, exiting ...\n{global_value.method} (database) != scheduler (env)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
global_settings[global_value.setting_id]["checked"] = True
|
||
else:
|
||
print(
|
||
f"❌ The global value {global_value.setting_id} is in the database but should not be, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
if not all(
|
||
[global_settings[global_value]["checked"] for global_value in global_settings]
|
||
):
|
||
print(
|
||
f"❌ Not all global values are in the database, exiting ...\nmissing values: {', '.join([global_value for global_value in global_settings if not global_settings[global_value]['checked']])}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
print("✅ Global values are in the database and are correct", flush=True)
|
||
print(" ", flush=True)
|
||
print(
|
||
"ℹ️ Checking if service values are in the database and are correct ...",
|
||
flush=True,
|
||
)
|
||
|
||
with db_session() as session:
|
||
services_settings = session.query(Services_settings).all()
|
||
|
||
if not multisite and service_settings:
|
||
print(
|
||
'❌ The bw_services_settings database table is not empty, it should be when multisite is set to "no", exiting ...',
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
else:
|
||
for service_setting in services_settings:
|
||
if service_setting.setting_id in service_settings:
|
||
if (
|
||
service_setting.value
|
||
!= service_settings[service_setting.setting_id]["value"]
|
||
):
|
||
print(
|
||
f"❌ The service value {service_setting.setting_id} is in the database but is not correct, exiting ...\n{service_setting.value} (database) != {service_settings[service_setting.setting_id]['value']} (env)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif service_setting.suffix != 0:
|
||
print(
|
||
f"❌ The service value {service_setting.setting_id} is in the database but has the wrong suffix, exiting ...\n{service_setting.suffix} (database) != 0 (env)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif service_setting.method != "scheduler":
|
||
print(
|
||
f"❌ The service value {service_setting.setting_id} is in the database but has the wrong method, exiting ...\n{service_setting.method} (database) != scheduler (env)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
service_settings[service_setting.setting_id]["checked"] = True
|
||
else:
|
||
print(
|
||
f"❌ The service value {service_setting.setting_id} is in the database but should not be, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
if not all(
|
||
[
|
||
service_settings[service_setting]["checked"]
|
||
for service_setting in service_settings
|
||
]
|
||
):
|
||
print(
|
||
f"❌ Not all service values are in the database, exiting ...\nmissing values: {', '.join([service_setting for service_setting in service_settings if not service_settings[service_setting]['checked']])}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
print("✅ Service values are correct", flush=True)
|
||
print(" ", flush=True)
|
||
print("ℹ️ Checking if the plugins are correct ...", flush=True)
|
||
|
||
core_plugins = {
|
||
"general": {
|
||
"order": 999,
|
||
"name": "General",
|
||
"description": "The general settings for the server",
|
||
"version": "0.1",
|
||
"stream": "partial",
|
||
"external": False,
|
||
"checked": False,
|
||
"page_checked": True,
|
||
"settings": {
|
||
"IS_LOADING": {
|
||
"context": "global",
|
||
"default": "no",
|
||
"help": "Internal use : set to yes when BW is loading.",
|
||
"id": "internal-use",
|
||
"label": "internal use",
|
||
"regex": "^(yes|no)$",
|
||
"type": "check",
|
||
},
|
||
"NGINX_PREFIX": {
|
||
"context": "global",
|
||
"default": "/etc/nginx/",
|
||
"help": "Where nginx will search for configurations.",
|
||
"id": "nginx-prefix",
|
||
"label": "nginx prefix",
|
||
"regex": "^(/[\\w. -]+)*/$",
|
||
"type": "text",
|
||
},
|
||
"HTTP_PORT": {
|
||
"context": "global",
|
||
"default": "8080",
|
||
"help": "HTTP port number which bunkerweb binds to.",
|
||
"id": "http-port",
|
||
"label": "HTTP port",
|
||
"regex": "^\\d+$",
|
||
"type": "text",
|
||
},
|
||
"HTTPS_PORT": {
|
||
"context": "global",
|
||
"default": "8443",
|
||
"help": "HTTPS port number which bunkerweb binds to.",
|
||
"id": "https-port",
|
||
"label": "HTTPS port",
|
||
"regex": "^\\d+$",
|
||
"type": "text",
|
||
},
|
||
"MULTISITE": {
|
||
"context": "global",
|
||
"default": "no",
|
||
"help": "Multi site activation.",
|
||
"id": "multisite",
|
||
"label": "Multisite",
|
||
"regex": "^(yes|no)$",
|
||
"type": "check",
|
||
},
|
||
"SERVER_NAME": {
|
||
"context": "multisite",
|
||
"default": "www.example.com",
|
||
"help": "List of the virtual hosts served by bunkerweb.",
|
||
"id": "server-name",
|
||
"label": "Server name",
|
||
"regex": "^(?! )( ?((?=.{1,255}$)[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?(?:\\.[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?)*\\.?)(?!.* \\2))*$",
|
||
"type": "text",
|
||
},
|
||
"WORKER_PROCESSES": {
|
||
"context": "global",
|
||
"default": "auto",
|
||
"help": "Number of worker processes.",
|
||
"id": "worker-processes",
|
||
"label": "Worker processes",
|
||
"regex": "^(auto|\\d+)$",
|
||
"type": "text",
|
||
},
|
||
"WORKER_RLIMIT_NOFILE": {
|
||
"context": "global",
|
||
"default": "2048",
|
||
"help": "Maximum number of open files for worker processes.",
|
||
"id": "worker-rlimit-nofile",
|
||
"label": "Open files per worker",
|
||
"regex": "^\\d+$",
|
||
"type": "text",
|
||
},
|
||
"WORKER_CONNECTIONS": {
|
||
"context": "global",
|
||
"default": "1024",
|
||
"help": "Maximum number of connections per worker.",
|
||
"id": "worker-connections",
|
||
"label": "Connections per worker",
|
||
"regex": "^\\d+$",
|
||
"type": "text",
|
||
},
|
||
"LOG_FORMAT": {
|
||
"context": "global",
|
||
"default": '$host $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"',
|
||
"help": "The format to use for access logs.",
|
||
"id": "log-format",
|
||
"label": "Log format",
|
||
"regex": "^.*$",
|
||
"type": "text",
|
||
},
|
||
"LOG_LEVEL": {
|
||
"context": "global",
|
||
"default": "notice",
|
||
"help": "The level to use for error logs.",
|
||
"id": "log-level",
|
||
"label": "Log level",
|
||
"regex": "^(debug|info|notice|warn|error|crit|alert|emerg)$",
|
||
"type": "select",
|
||
"select": [
|
||
"debug",
|
||
"info",
|
||
"notice",
|
||
"warn",
|
||
"error",
|
||
"crit",
|
||
"alert",
|
||
"emerg",
|
||
],
|
||
},
|
||
"DNS_RESOLVERS": {
|
||
"context": "global",
|
||
"default": "127.0.0.11",
|
||
"help": "DNS addresses of resolvers to use.",
|
||
"id": "dns-resolvers",
|
||
"label": "DNS resolvers",
|
||
"regex": "^(?! )( *(((\\b25[0-5]|\\b2[0-4]\\d|\\b[01]?\\d\\d?)(\\.(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)){3})(\\/([1-2][0-9]?|3[0-2]?|[04-9]))?|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]Z{0,4}){0,4}%[0-9a-zA-Z]+|::(ffff(:0{1,4})?:)?((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d)|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d))(\\/(12[0-8]|1[01][0-9]|[0-9][0-9]?))?)(?!.*\\D\\2([^\\d\\/]|$)) *)*$",
|
||
"type": "text",
|
||
},
|
||
"DATASTORE_MEMORY_SIZE": {
|
||
"context": "global",
|
||
"default": "64m",
|
||
"help": "Size of the internal datastore.",
|
||
"id": "datastore-memory-size",
|
||
"label": "Datastore memory size",
|
||
"regex": "^\\d+[kKmMgG]?$",
|
||
"type": "text",
|
||
},
|
||
"CACHESTORE_MEMORY_SIZE": {
|
||
"context": "global",
|
||
"default": "64m",
|
||
"help": "Size of the internal cachestore.",
|
||
"id": "cachestore-memory-size",
|
||
"label": "Cachestore memory size",
|
||
"regex": "^\\d+[kKmMgG]?$",
|
||
"type": "text",
|
||
},
|
||
"CACHESTORE_IPC_MEMORY_SIZE": {
|
||
"context": "global",
|
||
"default": "16m",
|
||
"help": "Size of the internal cachestore (ipc).",
|
||
"id": "cachestore-ipc-memory-size",
|
||
"label": "Cachestore ipc memory size",
|
||
"regex": "^\\d+[kKmMgG]?$",
|
||
"type": "text",
|
||
},
|
||
"CACHESTORE_MISS_MEMORY_SIZE": {
|
||
"context": "global",
|
||
"default": "16m",
|
||
"help": "Size of the internal cachestore (miss).",
|
||
"id": "cachestore-miss-memory-size",
|
||
"label": "Cachestore miss memory size",
|
||
"regex": "^\\d+[kKmMgG]?$",
|
||
"type": "text",
|
||
},
|
||
"CACHESTORE_LOCKS_MEMORY_SIZE": {
|
||
"context": "global",
|
||
"default": "16m",
|
||
"help": "Size of the internal cachestore (locks).",
|
||
"id": "cachestore-locks-memory-size",
|
||
"label": "Cachestore locks memory size",
|
||
"regex": "^\\d+[kKmMgG]?$",
|
||
"type": "text",
|
||
},
|
||
"USE_API": {
|
||
"context": "global",
|
||
"default": "yes",
|
||
"help": "Activate the API to control BunkerWeb.",
|
||
"id": "use-api",
|
||
"label": "Activate API",
|
||
"regex": "^(yes|no)$",
|
||
"type": "check",
|
||
},
|
||
"API_HTTP_PORT": {
|
||
"context": "global",
|
||
"default": "5000",
|
||
"help": "Listen port number for the API.",
|
||
"id": "api-http-listen",
|
||
"label": "API port number",
|
||
"regex": "^\\d+$",
|
||
"type": "text",
|
||
},
|
||
"API_LISTEN_IP": {
|
||
"context": "global",
|
||
"default": "0.0.0.0",
|
||
"help": "Listen IP address for the API.",
|
||
"id": "api-ip-listen",
|
||
"label": "API listen IP",
|
||
"regex": "^.*$",
|
||
"type": "text",
|
||
},
|
||
"API_SERVER_NAME": {
|
||
"context": "global",
|
||
"default": "bwapi",
|
||
"help": "Server name (virtual host) for the API.",
|
||
"id": "api-server-name",
|
||
"label": "API server name",
|
||
"regex": "^(?=.{1,255}$)[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?(?:\\.[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?)*\\.?$",
|
||
"type": "text",
|
||
},
|
||
"API_WHITELIST_IP": {
|
||
"context": "global",
|
||
"default": "127.0.0.0/8",
|
||
"help": "List of IP/network allowed to contact the API.",
|
||
"id": "api-whitelist-ip",
|
||
"label": "API whitelist IP",
|
||
"regex": "^(?! )( *(((\\b25[0-5]|\\b2[0-4]\\d|\\b[01]?\\d\\d?)(\\.(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)){3})(\\/([1-2][0-9]?|3[0-2]?|[04-9]))?|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]Z{0,4}){0,4}%[0-9a-zA-Z]+|::(ffff(:0{1,4})?:)?((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d)|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d))(\\/(12[0-8]|1[01][0-9]|[0-9][0-9]?))?)(?!.*\\D\\2([^\\d\\/]|$)) *)*$",
|
||
"type": "text",
|
||
},
|
||
"AUTOCONF_MODE": {
|
||
"context": "global",
|
||
"default": "no",
|
||
"help": "Enable Autoconf Docker integration.",
|
||
"id": "autoconf-mode",
|
||
"label": "Autoconf mode",
|
||
"regex": "^(yes|no)$",
|
||
"type": "check",
|
||
},
|
||
"SWARM_MODE": {
|
||
"context": "global",
|
||
"default": "no",
|
||
"help": "Enable Docker Swarm integration.",
|
||
"id": "swarm-mode",
|
||
"label": "Swarm mode",
|
||
"regex": "^(yes|no)$",
|
||
"type": "check",
|
||
},
|
||
"KUBERNETES_MODE": {
|
||
"context": "global",
|
||
"default": "no",
|
||
"help": "Enable Kubernetes integration.",
|
||
"id": "kubernetes-mode",
|
||
"label": "Kubernetes mode",
|
||
"regex": "^(yes|no)$",
|
||
"type": "check",
|
||
},
|
||
"SERVER_TYPE": {
|
||
"context": "multisite",
|
||
"default": "http",
|
||
"help": "Server type : http or stream.",
|
||
"id": "server-type",
|
||
"label": "Server type",
|
||
"regex": "^(http|stream)$",
|
||
"type": "select",
|
||
"select": ["http", "stream"],
|
||
},
|
||
"LISTEN_STREAM": {
|
||
"context": "multisite",
|
||
"default": "yes",
|
||
"help": "Enable listening for non-ssl (passthrough).",
|
||
"id": "listen-stream",
|
||
"label": "Listen stream",
|
||
"regex": "^(yes|no)$",
|
||
"type": "check",
|
||
},
|
||
"LISTEN_STREAM_PORT": {
|
||
"context": "multisite",
|
||
"default": "1337",
|
||
"help": "Listening port for non-ssl (passthrough).",
|
||
"id": "listen-stream-port",
|
||
"label": "Listen stream port",
|
||
"regex": "^[0-9]+$",
|
||
"type": "text",
|
||
},
|
||
"LISTEN_STREAM_PORT_SSL": {
|
||
"context": "multisite",
|
||
"default": "4242",
|
||
"help": "Listening port for ssl (passthrough).",
|
||
"id": "listen-stream-port-ssl",
|
||
"label": "Listen stream port ssl",
|
||
"regex": "^[0-9]+$",
|
||
"type": "text",
|
||
},
|
||
"USE_UDP": {
|
||
"context": "multisite",
|
||
"default": "no",
|
||
"help": "UDP listen instead of TCP (stream).",
|
||
"id": "use-udp",
|
||
"label": "Listen UDP",
|
||
"regex": "^(yes|no)$",
|
||
"type": "check",
|
||
},
|
||
},
|
||
}
|
||
}
|
||
for filename in iglob(join("core", "*", "plugin.json")):
|
||
with open(filename, "r") as f:
|
||
data = load(f)
|
||
data["checked"] = False
|
||
for x, job in enumerate(data.get("jobs", [])):
|
||
data["jobs"][x]["checked"] = False
|
||
data["page_checked"] = not Path(f"{dirname(filename)}/ui").exists() or False
|
||
core_plugins[data.pop("id")] = data
|
||
|
||
external_plugins = {}
|
||
for filename in iglob(join("external", "*", "plugin.json")):
|
||
with open(filename, "r") as f:
|
||
data = load(f)
|
||
data["checked"] = False
|
||
for x, job in enumerate(data.get("jobs", [])):
|
||
data["jobs"][x]["checked"] = False
|
||
data["page_checked"] = not Path(f"{dirname(filename)}/ui").exists() or False
|
||
external_plugins[data.pop("id")] = data
|
||
|
||
with db_session() as session:
|
||
plugins = (
|
||
session.query(Plugins)
|
||
.with_entities(
|
||
Plugins.id,
|
||
Plugins.order,
|
||
Plugins.name,
|
||
Plugins.description,
|
||
Plugins.version,
|
||
Plugins.stream,
|
||
Plugins.external,
|
||
Plugins.method,
|
||
)
|
||
.all()
|
||
)
|
||
|
||
for plugin in plugins:
|
||
if not plugin.external and plugin.id in core_plugins:
|
||
current_plugin = core_plugins
|
||
elif plugin.external and plugin.id in external_plugins:
|
||
current_plugin = external_plugins
|
||
else:
|
||
print(
|
||
f"❌ The {'external' if plugin.external else 'core'} plugin {plugin.name} (id: {plugin.id}) is in the database but should not be, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
if (
|
||
plugin.order != current_plugin[plugin.id]["order"]
|
||
or plugin.name != current_plugin[plugin.id]["name"]
|
||
or plugin.description != current_plugin[plugin.id]["description"]
|
||
or plugin.version != current_plugin[plugin.id]["version"]
|
||
or plugin.stream != current_plugin[plugin.id]["stream"]
|
||
):
|
||
print(
|
||
f"❌ The {'external' if plugin.external else 'core'} plugin {plugin.name} (id: {plugin.id}) is in the database but is not correct, exiting ...\n{dumps({'order': plugin.order, 'name': plugin.name, 'description': plugin.description, 'version': plugin.version, 'stream': plugin.stream})} (database) != {dumps({'order': current_plugin[plugin.id]['order'], 'name': current_plugin[plugin.id]['name'], 'description': current_plugin[plugin.id]['description'], 'version': current_plugin[plugin.id]['version'], 'stream': current_plugin[plugin.id]['stream']})} (file)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
else:
|
||
settings = session.query(Settings).filter_by(plugin_id=plugin.id).all()
|
||
|
||
for setting in settings:
|
||
if (
|
||
setting.name
|
||
!= current_plugin[plugin.id]["settings"][setting.id]["id"]
|
||
or setting.context
|
||
!= current_plugin[plugin.id]["settings"][setting.id]["context"]
|
||
or setting.default
|
||
!= current_plugin[plugin.id]["settings"][setting.id]["default"]
|
||
or setting.help
|
||
!= current_plugin[plugin.id]["settings"][setting.id]["help"]
|
||
or setting.label
|
||
!= current_plugin[plugin.id]["settings"][setting.id]["label"]
|
||
or setting.regex
|
||
!= current_plugin[plugin.id]["settings"][setting.id]["regex"]
|
||
or setting.type
|
||
!= current_plugin[plugin.id]["settings"][setting.id]["type"]
|
||
or setting.multiple
|
||
!= current_plugin[plugin.id]["settings"][setting.id].get(
|
||
"multiple", None
|
||
)
|
||
):
|
||
print(
|
||
f"❌ The {'external' if plugin.external else 'core'} plugin {plugin.name} (id: {plugin.id}) is in the database but is not correct, exiting ...\n{dumps({'default': setting.default, 'help': setting.help, 'label': setting.label, 'regex': setting.regex, 'type': setting.type})} (database) != {dumps({'default': current_plugin[plugin.id]['settings'][setting.id]['default'], 'help': current_plugin[plugin.id]['settings'][setting.id]['help'], 'label': current_plugin[plugin.id]['settings'][setting.id]['label'], 'regex': current_plugin[plugin.id]['settings'][setting.id]['regex'], 'type': current_plugin[plugin.id]['settings'][setting.id]['type']})} (file)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
current_plugin[plugin.id]["checked"] = True
|
||
|
||
if not all([core_plugins[plugin]["checked"] for plugin in core_plugins]):
|
||
print(
|
||
f"❌ Not all core plugins are in the database, exiting ...\nmissing plugins: {', '.join([plugin for plugin in core_plugins if not core_plugins[plugin]])}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif not all([external_plugins[plugin]["checked"] for plugin in external_plugins]):
|
||
print(
|
||
f"❌ Not all external plugins are in the database, exiting ...\nmissing plugins: {', '.join([plugin for plugin in external_plugins if not external_plugins[plugin]])}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
print("✅ The ClamAV plugin and all core plugins are in the database", flush=True)
|
||
print(" ", flush=True)
|
||
print("ℹ️ Checking if the jobs are in the database ...", flush=True)
|
||
|
||
with db_session() as session:
|
||
jobs = session.query(Jobs).all()
|
||
|
||
for job in jobs:
|
||
if not job.success:
|
||
print(
|
||
f"❌ The job {job.name} (plugin_id: {job.plugin_id}) is in the database but failed, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
if job.plugin_id in core_plugins:
|
||
current_plugin = core_plugins
|
||
elif job.plugin_id in external_plugins:
|
||
current_plugin = external_plugins
|
||
else:
|
||
print(
|
||
f"❌ The job {job.name} (plugin_id: {job.plugin_id}) is in the database but should not be, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
index = next(
|
||
index
|
||
for (index, d) in enumerate(
|
||
current_plugin[job.plugin_id].get("jobs", [])
|
||
)
|
||
if d["name"] == job.name
|
||
)
|
||
core_job = current_plugin[job.plugin_id]["jobs"][index]
|
||
|
||
if (
|
||
job.name != core_job["name"]
|
||
or job.file_name != core_job["file"]
|
||
or job.every != core_job["every"]
|
||
or job.reload != core_job["reload"]
|
||
):
|
||
print(
|
||
f"❌ The job {job.name} (plugin_id: {job.plugin_id}) is in the database but is not correct, exiting ...\n{dumps({'name': job.name, 'file': job.file_name, 'every': job.every, 'reload': job.reload})} (database) != {dumps({'name': core_job['name'], 'file': core_job['file'], 'every': core_job['every'], 'reload': core_job['reload']})} (file)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
current_plugin[job.plugin_id]["jobs"][index]["checked"] = True
|
||
|
||
if not all(
|
||
[
|
||
all([job["checked"] for job in core_plugins[plugin].get("jobs", [])])
|
||
for plugin in core_plugins
|
||
]
|
||
):
|
||
print(
|
||
f"❌ Not all jobs from core plugins are in the database, exiting ...\nmissing jobs: {dumps({plugin: [job['name'] for job in core_plugins[plugin]['jobs'] if not job['checked']] for plugin in core_plugins})}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif not all(
|
||
[
|
||
all([job["checked"] for job in external_plugins[plugin].get("jobs", [])])
|
||
for plugin in external_plugins
|
||
]
|
||
):
|
||
print(
|
||
f"❌ Not all jobs from external plugins are in the database, exiting ...\nmissing jobs: {dumps({plugin: [job['name'] for job in external_plugins[plugin]['jobs'] if not job['checked']] for plugin in external_plugins})}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
print("✅ All jobs are in the database and have successfully ran", flush=True)
|
||
print(" ", flush=True)
|
||
print("ℹ️ Checking if all plugin pages are in the database ...", flush=True)
|
||
|
||
def file_hash(file: str) -> str:
|
||
_sha512 = sha512()
|
||
with open(file, "rb") as f:
|
||
while True:
|
||
data = f.read(1024)
|
||
if not data:
|
||
break
|
||
_sha512.update(data)
|
||
return _sha512.hexdigest()
|
||
|
||
with db_session() as session:
|
||
plugin_pages = (
|
||
session.query(Plugin_pages)
|
||
.with_entities(
|
||
Plugin_pages.id,
|
||
Plugin_pages.plugin_id,
|
||
Plugin_pages.template_checksum,
|
||
Plugin_pages.actions_checksum,
|
||
)
|
||
.all()
|
||
)
|
||
|
||
for plugin_page in plugin_pages:
|
||
if plugin_page.plugin_id in core_plugins:
|
||
current_plugin = core_plugins
|
||
elif plugin_page.plugin_id in external_plugins:
|
||
current_plugin = external_plugins
|
||
else:
|
||
print(
|
||
f"❌ The plugin page from {plugin_page.plugin_id} is in the database but should not be, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
path_ui = (
|
||
Path(join("core", plugin_page.plugin_id, "ui"))
|
||
if Path(join("core", plugin_page.plugin_id, "ui")).exists()
|
||
else Path(join("external", plugin_page.plugin_id, "ui"))
|
||
)
|
||
|
||
if not path_ui.exists():
|
||
print(
|
||
f'❌ The plugin page from {plugin_page.plugin_id} is in the database but should not be because the "ui" folder is missing from the plugin, exiting ...',
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
template_checksum = file_hash(f"{path_ui}/template.html")
|
||
actions_checksum = file_hash(f"{path_ui}/actions.py")
|
||
|
||
if plugin_page.template_checksum != template_checksum:
|
||
print(
|
||
f"❌ The plugin page from {plugin_page.plugin_id} is in the database but the template file checksum differ, exiting ...\n{plugin_page.template_checksum} (database) != {template_checksum} (file)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif plugin_page.actions_checksum != actions_checksum:
|
||
print(
|
||
f"❌ The plugin page from {plugin_page.plugin_id} is in the database but the actions file checksum differ, exiting ...\n{plugin_page.actions_checksum} (database) != {actions_checksum} (file)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
current_plugin[plugin_page.plugin_id]["page_checked"] = True
|
||
|
||
if not all([core_plugins[plugin]["page_checked"] for plugin in core_plugins]):
|
||
print(
|
||
f"❌ Not all core plugins pages are in the database, exiting ...\nmissing plugins pages: {', '.join([plugin for plugin in core_plugins if not core_plugins[plugin]['page_checked']])}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif not all(
|
||
[external_plugins[plugin]["page_checked"] for plugin in external_plugins]
|
||
):
|
||
print(
|
||
f"❌ Not all external plugins pages are in the database, exiting ...\nmissing plugins pages: {', '.join([plugin for plugin in external_plugins if not external_plugins[plugin]['page_checked']])}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
print("✅ All plugin pages are in the database and have the right value", flush=True)
|
||
print(" ", flush=True)
|
||
print("ℹ️ Checking if all custom configs are in the database ...", flush=True)
|
||
|
||
custom_confs_rx = re_compile(
|
||
r"^([0-9a-z\.-]*)_?CUSTOM_CONF_(SERVICE_)?(HTTP|SERVER_STREAM|STREAM|DEFAULT_SERVER_HTTP|SERVER_HTTP|MODSEC_CRS|MODSEC)_(.+)$"
|
||
)
|
||
|
||
global_custom_configs = {}
|
||
service_custom_configs = {}
|
||
for env in environ:
|
||
if not custom_confs_rx.match(env):
|
||
continue
|
||
|
||
custom_conf = custom_confs_rx.search(env).groups()
|
||
if custom_conf[1]:
|
||
service_custom_configs[custom_conf[3]] = {
|
||
"value": environ[env].encode(),
|
||
"type": custom_conf[2].lower(),
|
||
"method": "scheduler",
|
||
"checked": False,
|
||
}
|
||
continue
|
||
|
||
global_custom_configs[custom_conf[3]] = {
|
||
"value": environ[env].encode(),
|
||
"type": custom_conf[2].lower(),
|
||
"method": "scheduler",
|
||
"checked": False,
|
||
}
|
||
|
||
with db_session() as session:
|
||
custom_configs = (
|
||
session.query(Custom_configs)
|
||
.with_entities(
|
||
Custom_configs.service_id,
|
||
Custom_configs.type,
|
||
Custom_configs.name,
|
||
Custom_configs.data,
|
||
Custom_configs.method,
|
||
)
|
||
.all()
|
||
)
|
||
|
||
for custom_config in custom_configs:
|
||
if (
|
||
not multisite
|
||
and custom_config.name in global_custom_configs
|
||
and custom_config.service_id
|
||
):
|
||
print(
|
||
f"❌ The custom config {custom_config.name} is in the database but should not be owned by the service {custom_config.service_id} because multisite is not enabled, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif (
|
||
multisite
|
||
and custom_config.name in service_custom_configs
|
||
and not custom_config.service_id
|
||
):
|
||
print(
|
||
f"❌ The custom config {custom_config.name} is in the database but should be owned by the service bwadm.example.com because it's a service config, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
if custom_config.name in global_custom_configs:
|
||
current_custom_configs = global_custom_configs
|
||
elif custom_config.name in service_custom_configs:
|
||
current_custom_configs = service_custom_configs
|
||
else:
|
||
print(
|
||
f"❌ The custom config {custom_config.name} is in the database but should not be, exiting ...",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
if custom_config.type != current_custom_configs[custom_config.name]["type"]:
|
||
print(
|
||
f"❌ The custom config {custom_config.name} is in the database but the type differ, exiting ...\n{custom_config.type} (database) != {current_custom_configs[custom_config.name]['type']} (env)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif (
|
||
custom_config.data
|
||
!= current_custom_configs[custom_config.name]["value"]
|
||
):
|
||
print(
|
||
f"❌ The custom config {custom_config.name} is in the database but the value differ, exiting ...\n{custom_config.data} (database) != {current_custom_configs[custom_config.name]['value']} (env)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif (
|
||
custom_config.method
|
||
!= current_custom_configs[custom_config.name]["method"]
|
||
):
|
||
print(
|
||
f"❌ The custom config {custom_config.name} is in the database but the method differ, exiting ...\n{custom_config.method} (database) != {current_custom_configs[custom_config.name]['method']} (env)",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
current_custom_configs[custom_config.name]["checked"] = True
|
||
|
||
if not all(
|
||
[
|
||
global_custom_configs[custom_config]["checked"]
|
||
for custom_config in global_custom_configs
|
||
]
|
||
):
|
||
print(
|
||
f"❌ Not all global custom configs are in the database, exiting ...\nmissing custom configs: {', '.join([custom_config for custom_config in global_custom_configs if not global_custom_configs[custom_config]['checked']])}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
elif not all(
|
||
[
|
||
service_custom_configs[custom_config]["checked"]
|
||
for custom_config in service_custom_configs
|
||
]
|
||
):
|
||
print(
|
||
f"❌ Not all service custom configs are in the database, exiting ...\nmissing custom configs: {', '.join([custom_config for custom_config in service_custom_configs if not service_custom_configs[custom_config]['checked']])}",
|
||
flush=True,
|
||
)
|
||
exit(1)
|
||
|
||
print(
|
||
"✅ All custom configs are in the database and have the right value", flush=True
|
||
)
|
||
except SystemExit:
|
||
exit(1)
|
||
except:
|
||
print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True)
|
||
exit(1)
|