Rename src files of web UI and optimize a few things with instances

This commit is contained in:
Théophile Diot 2024-08-01 13:15:14 +01:00
parent 64955cbe0e
commit 184c271aab
No known key found for this signature in database
GPG key ID: FA995104A0BA376A
12 changed files with 452 additions and 494 deletions

View file

@ -2483,11 +2483,36 @@ class Database:
"hostname": instance.hostname,
"port": instance.port,
"server_name": instance.server_name,
"status": instance.status,
"method": instance.method,
"creation_date": instance.creation_date,
"last_seen": instance.last_seen,
}
for instance in query
]
def get_instance(self, hostname: str, *, method: Optional[str] = None) -> Dict[str, Any]:
"""Get instance."""
with self._db_session() as session:
query = session.query(Instances).filter_by(hostname=hostname)
if method:
query = query.filter_by(method=method)
instance = query.first()
if not instance:
return {}
return {
"hostname": instance.hostname,
"port": instance.port,
"server_name": instance.server_name,
"status": instance.status,
"method": instance.method,
"creation_date": instance.creation_date,
"last_seen": instance.last_seen,
}
def get_plugin_actions(self, plugin: str) -> Optional[Any]:
"""get actions file for the plugin"""
with self._db_session() as session:

View file

@ -1,5 +1,7 @@
#!/usr/bin/env python3
from datetime import datetime, timezone
from functools import partial
from sqlalchemy import (
TEXT,
Boolean,
@ -12,6 +14,7 @@ from sqlalchemy import (
LargeBinary,
PrimaryKeyConstraint,
String,
func,
)
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy.schema import UniqueConstraint
@ -46,6 +49,7 @@ INTEGRATIONS_ENUM = Enum(
STREAM_TYPES_ENUM = Enum("no", "yes", "partial", name="stream_types_enum")
PLUGIN_TYPES_ENUM = Enum("core", "external", "pro", name="plugin_types_enum")
PRO_STATUS_ENUM = Enum("active", "invalid", "expired", "suspended", name="pro_status_enum")
INSTANCE_STATUS_ENUM = Enum("loading", "up", "down", name="instance_status_enum")
Base = declarative_base()
@ -206,7 +210,10 @@ class Instances(Base):
hostname = Column(String(256), primary_key=True)
port = Column(Integer, nullable=False)
server_name = Column(String(256), nullable=False)
status = Column(INSTANCE_STATUS_ENUM, nullable=True, default="up")
method = Column(METHODS_ENUM, nullable=False, default="manual")
creation_date = Column(DateTime, nullable=False, server_default=func.now())
last_seen = Column(DateTime, nullable=True, server_default=func.now(), onupdate=partial(datetime.now, timezone.utc))
class BwcliCommands(Base):

View file

View file

@ -3,15 +3,18 @@ import json
from os.path import join, sep
from sys import path as sys_path
from typing import List
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from src.instance import Instance
from builder.utils.widgets import instance_widget
def instances_builder(instances: list) -> str:
def instances_builder(instances: List[Instance]) -> str:
"""
It returns the needed format from data to render the instances page in JSON format for the Vue.js builder
"""
@ -21,18 +24,18 @@ def instances_builder(instances: list) -> str:
# setup actions buttons
actions = (
["restart", "stop"]
if instance._type == "local" and instance.health
if instance.hostname in ("localhost", "127.0.0.1") and instance.status == "up"
else (
["reload", "stop"]
if not instance._type == "local" and instance.health
else ["start"] if instance._type == "local" and not instance.health else []
if instance.hostname not in ("localhost", "127.0.0.1") and instance.status == "up"
else ["start"] if instance.hostname in ("localhost", "127.0.0.1") and instance.status != "up" else []
)
)
buttons = [
{
"attrs": {
"data-submit-form": f"""{{"INSTANCE_ID" : "{instance._id}", "operation" : "{action}" }}""",
"data-submit-form": f"""{{"INSTANCE_ID" : "{instance.hostname}", "operation" : "{action}" }}""",
},
"text": f"action_{action}",
"color": "success" if action == "start" else "error" if action == "stop" else "warning",
@ -43,15 +46,15 @@ def instances_builder(instances: list) -> str:
instance = instance_widget(
containerColumns={"pc": 6, "tablet": 6, "mobile": 12},
pairs=[
{"key": "instances_hostname", "value": instance.hostname},
{"key": "instances_type", "value": instance._type},
{"key": "instances_status", "value": "instances_active" if instance.health else "instances_inactive"},
{"key": "instances_method", "value": instance.method},
{"key": "instances_creation_date", "value": instance.creation_date.strftime("%d-%m-%Y %H:%M:%S")},
{"key": "instances_last_seen", "value": instance.last_seen.strftime("%d-%m-%Y %H:%M:%S")},
],
status="success" if instance.health else "error",
title=instance.name,
status="success" if instance.status == "up" else "error",
title=instance.hostname,
buttons=buttons,
)
builder.append(instance)
return base64.b64encode(bytes(json.dumps(builder), "utf-8")).decode("ascii")
return base64.b64encode(json.dumps(builder).encode("utf-8")).decode("ascii")

View file

@ -3,7 +3,7 @@ import json
from contextlib import suppress
from math import floor
from multiprocessing import Manager
from os import _exit, getenv, listdir, sep, urandom
from os import _exit, getenv, listdir, sep
from os.path import basename, dirname, isabs, join
from random import randint
from secrets import choice, token_urlsafe
@ -46,11 +46,10 @@ from time import sleep, time
from werkzeug.utils import secure_filename
from zipfile import BadZipFile, ZipFile
from src.Instances import Instances
from src.ConfigFiles import ConfigFiles
from src.Config import Config
from src.ReverseProxied import ReverseProxied
from src.Templates import get_ui_templates
from src.instance import Instance, InstancesUtils
from src.custom_config import CustomConfig
from src.config import Config
from src.reverse_proxied import ReverseProxied
from src.totp import Totp
from builder.home import home_builder
@ -170,6 +169,10 @@ with app.app_context():
app.config["SESSION_COOKIE_HTTPONLY"] = True # Recommended for security
app.config["SESSION_COOKIE_SAMESITE"] = "Lax"
app.config["WTF_CSRF_SSL_STRICT"] = False
app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 86400
app.config["SCRIPT_NONCE"] = ""
login_manager = LoginManager()
login_manager.session_protection = "strong"
login_manager.init_app(app)
@ -273,20 +276,6 @@ with app.app_context():
app.logger.error(f"Couldn't create the admin user in the database: {ret}")
exit(1)
try:
app.config.update(
INSTANCES=Instances(DB),
CONFIG=Config(DB),
CONFIGFILES=ConfigFiles(),
WTF_CSRF_SSL_STRICT=False,
SEND_FILE_MAX_AGE_DEFAULT=86400,
SCRIPT_NONCE=sha256(urandom(32)).hexdigest(),
UI_TEMPLATES=get_ui_templates(),
)
except FileNotFoundError as e:
app.logger.error(repr(e), e.filename)
stop(1)
# Declare functions for jinja2
app.jinja_env.globals.update(check_settings=check_settings)
@ -294,8 +283,11 @@ with app.app_context():
csrf = CSRFProtect()
csrf.init_app(app)
app.bw_instances_utils = InstancesUtils(DB)
app.bw_config = Config(DB)
app.bw_custom_configs = CustomConfig()
app.data = Manager().dict()
app.totp = Totp(app, TOTP_SECRETS, [key.encode() for key in MF_RECOVERY_CODES_KEYS])
app.totp = Totp(app, TOTP_SECRETS, [key.encode("utf-8") for key in MF_RECOVERY_CODES_KEYS])
LOG_RX = re_compile(r"^(?P<date>\d+/\d+/\d+\s\d+:\d+:\d+)\s\[(?P<level>[a-z]+)\]\s\d+#\d+:\s(?P<message>[^\n]+)$")
REVERSE_PROXY_PATH = re_compile(r"^(?P<host>https?://.{1,255}(:((6553[0-5])|(655[0-2]\d)|(65[0-4]\d{2})|(6[0-4]\d{3})|([1-5]\d{4})|([0-5]{0,5})|(\d{1,4})))?)$")
@ -334,22 +326,38 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads", is_draft: b
if method == "services":
if operation == "new":
operation, error = app.config["CONFIG"].new_service(args[0], is_draft=is_draft)
operation, error = app.bw_config.new_service(args[0], is_draft=is_draft)
elif operation == "edit":
operation, error = app.config["CONFIG"].edit_service(args[1], args[0], check_changes=(was_draft != is_draft or not is_draft), is_draft=is_draft)
operation, error = app.bw_config.edit_service(args[1], args[0], check_changes=(was_draft != is_draft or not is_draft), is_draft=is_draft)
elif operation == "delete":
operation, error = app.config["CONFIG"].delete_service(args[2], check_changes=(was_draft != is_draft or not is_draft))
operation, error = app.bw_config.delete_service(args[2], check_changes=(was_draft != is_draft or not is_draft))
elif method == "global_config":
operation, error = app.config["CONFIG"].edit_global_conf(args[0], check_changes=True)
operation, error = app.bw_config.edit_global_conf(args[0], check_changes=True)
if operation == "reload":
operation = app.config["INSTANCES"].reload_instance(args[0])
instance = Instance.from_hostname(args[0], DB)
if instance:
operation = instance.reload()
else:
operation = "The instance does not exist."
elif operation == "start":
operation = app.config["INSTANCES"].start_instance(args[0])
instance = Instance.from_hostname(args[0], DB)
if instance:
operation = instance.start()
else:
operation = "The instance does not exist."
elif operation == "stop":
operation = app.config["INSTANCES"].stop_instance(args[0])
instance = Instance.from_hostname(args[0], DB)
if instance:
operation = instance.stop()
else:
operation = "The instance does not exist."
elif operation == "restart":
operation = app.config["INSTANCES"].restart_instance(args[0])
instance = Instance.from_hostname(args[0], DB)
if instance:
operation = instance.restart()
else:
operation = "The instance does not exist."
elif not error:
operation = "The scheduler will be in charge of applying the changes."
@ -531,7 +539,7 @@ def inject_variables():
pro_services=metadata["pro_services"],
pro_expire=metadata["pro_expire"].strftime("%d-%m-%Y") if metadata["pro_expire"] else "Unknown",
pro_overlapped=metadata["pro_overlapped"],
plugins=app.config["CONFIG"].get_plugins(),
plugins=app.bw_config.get_plugins(),
pro_loading=app.data.get("PRO_LOADING", False),
bw_version=metadata["version"],
is_readonly=app.data.get("READONLY_MODE", False),
@ -617,7 +625,7 @@ def before_request():
response.headers["Retry-After"] = 30 # Clients should retry after 30 seconds # type: ignore
return response
app.config["SCRIPT_NONCE"] = sha256(urandom(32)).hexdigest()
app.config["SCRIPT_NONCE"] = token_urlsafe(32)
if not request.path.startswith(("/css", "/images", "/js", "/json", "/webfonts")):
if (
@ -702,7 +710,7 @@ def check():
@app.route("/setup", methods=["GET", "POST"])
def setup():
db_config = app.config["CONFIG"].get_config(methods=False, filtered_settings=("SERVER_NAME", "MULTISITE", "USE_UI", "UI_HOST", "AUTO_LETS_ENCRYPT"))
db_config = app.bw_config.get_config(methods=False, filtered_settings=("SERVER_NAME", "MULTISITE", "USE_UI", "UI_HOST", "AUTO_LETS_ENCRYPT"))
admin_user = DB.get_ui_user()
@ -781,7 +789,7 @@ def setup():
config["SELF_SIGNED_SSL_SUBJ"] = f"/CN={request.form['server_name']}/"
if not config.get("MULTISITE", "no") == "yes":
app.config["CONFIG"].edit_global_conf({"MULTISITE": "yes"}, check_changes=False)
app.bw_config.edit_global_conf({"MULTISITE": "yes"}, check_changes=False)
# deepcode ignore MissingAPI: We don't need to check to wait for the thread to finish
Thread(
@ -854,13 +862,13 @@ def home():
if r and r.status_code == 200:
remote_version = basename(r.url).strip().replace("v", "")
config = app.config["CONFIG"].get_config(with_drafts=True, filtered_settings=("SERVER_NAME",))
instances = app.config["INSTANCES"].get_instances()
config = app.bw_config.get_config(with_drafts=True, filtered_settings=("SERVER_NAME",))
instances = app.bw_instances_utils.get_instances()
instance_health_count = 0
for instance in instances:
if instance.health is True:
if instance.status == "up":
instance_health_count += 1
services = 0
@ -895,7 +903,7 @@ def home():
"pro_status": metadata["pro_status"],
"pro_services": metadata["pro_services"],
"pro_overlapped": metadata["pro_overlapped"],
"plugins_number": len(app.config["CONFIG"].get_plugins()),
"plugins_number": len(app.bw_config.get_plugins()),
"plugins_errors": DB.get_plugins_errors(),
}
@ -927,7 +935,7 @@ def account():
variable = {}
variable["PRO_LICENSE_KEY"] = request.form["license"]
variable = app.config["CONFIG"].check_variables(variable, {"PRO_LICENSE_KEY": request.form["license"]})
variable = app.bw_config.check_variables(variable, {"PRO_LICENSE_KEY": request.form["license"]})
if not variable:
return handle_error("The license key variable checks returned error", "account", True)
@ -1101,7 +1109,7 @@ def instances():
)
# Display instances
instances = app.config["INSTANCES"].get_instances()
instances = app.bw_instances_utils.get_instances()
data_server_builder = instances_builder(instances)
return render_template("instances.html", title="Instances", data_server_builder=json.dumps(data_server_builder))
@ -1179,7 +1187,7 @@ def get_service_data():
):
del variables[variable]
variables = app.config["CONFIG"].check_variables(variables, config)
variables = app.bw_config.check_variables(variables, config)
return config, variables, format_configs, server_name, old_server_name, operation, is_draft, was_draft, is_draft_unchanged
@ -1208,7 +1216,7 @@ def services():
# Delete
if request.form["operation"] == "delete":
is_service = app.config["CONFIG"].check_variables({"SERVER_NAME": request.form["SERVER_NAME"]}, config)
is_service = app.bw_config.check_variables({"SERVER_NAME": request.form["SERVER_NAME"]}, config)
if not is_service:
error_message(f"Error while deleting the service {request.form['SERVER_NAME']}")
@ -1319,7 +1327,7 @@ def services_raw(service_name: str):
# Delete
if request.form["operation"] == "delete":
is_service = app.config["CONFIG"].check_variables({"SERVER_NAME": request.form["SERVER_NAME"]}, config)
is_service = app.bw_config.check_variables({"SERVER_NAME": request.form["SERVER_NAME"]}, config)
if not is_service:
error_message(f"Error while deleting the service {request.form['SERVER_NAME']}")
@ -1436,7 +1444,7 @@ def global_config():
del variables[variable]
continue
variables = app.config["CONFIG"].check_variables(variables, config)
variables = app.bw_config.check_variables(variables, config)
if not variables:
return handle_error("The global configuration was not edited because no values were changed.", "global_config", True)
@ -1483,7 +1491,7 @@ def global_config():
)
global_config = DB.get_config(global_only=True, methods=True)
plugins = app.config["CONFIG"].get_plugins()
plugins = app.bw_config.get_plugins()
data_server_builder = global_config_builder(plugins, global_config)
return render_template("global-config.html", data_server_builder=data_server_builder)
@ -1513,7 +1521,9 @@ def configs():
if variables["type"] != "file":
return handle_error("Invalid type parameter on /configs.", "configs", True)
operation = app.config["CONFIGFILES"].check_path(variables["path"])
# TODO: revamp this to use a path but a form to edit the content
operation = app.bw_custom_configs.check_path(variables["path"])
if operation:
return handle_error(operation, "configs", True)
@ -1543,7 +1553,7 @@ def configs():
# New or edit a config
if request.form["operation"] in ("new", "edit"):
if not app.config["CONFIGFILES"].check_name(name):
if not app.bw_custom_configs.check_name(name):
return handle_error(
f"Invalid {variables['type']} name. (Can only contain numbers, letters, underscores, dots and hyphens (min 4 characters and max 64))",
"configs",
@ -1598,7 +1608,7 @@ def configs():
path_to_dict(
join(sep, "etc", "bunkerweb", "configs"),
db_data=db_configs,
services=app.config["CONFIG"].get_config(global_only=True, methods=False, filtered_settings=("SERVER_NAME",)).get("SERVER_NAME", "").split(" "),
services=app.bw_config.get_config(global_only=True, methods=False, filtered_settings=("SERVER_NAME",)).get("SERVER_NAME", "").split(" "),
)
],
)
@ -1636,7 +1646,7 @@ def plugins():
def update_plugins(threaded: bool = False): # type: ignore
wait_applying()
plugins = app.config["CONFIG"].get_plugins(_type="external", with_data=True)
plugins = app.bw_config.get_plugins(_type="external", with_data=True)
for x, plugin in enumerate(plugins):
if plugin["id"] == variables["name"]:
del plugins[x]
@ -1758,7 +1768,7 @@ def plugins():
folder_name = plugin_file["id"]
if not app.config["CONFIGFILES"].check_name(folder_name):
if not app.bw_custom_configs.check_name(folder_name):
errors += 1
error = 1
flash(
@ -1842,7 +1852,7 @@ def plugins():
def update_plugins(threaded: bool = False):
wait_applying()
plugins = app.config["CONFIG"].get_plugins(_type="external", with_data=True)
plugins = app.bw_config.get_plugins(_type="external", with_data=True)
for plugin in deepcopy(plugins):
if plugin["id"] in new_plugins_ids:
flash(f"Plugin {plugin['id']} already exists", "error")
@ -1882,7 +1892,7 @@ def plugins():
if tmp_ui_path.is_dir():
rmtree(tmp_ui_path, ignore_errors=True)
plugins = app.config["CONFIG"].get_plugins()
plugins = app.bw_config.get_plugins()
plugins_internal = 0
plugins_external = 0
plugins_pro = 0
@ -1992,7 +2002,7 @@ def custom_plugin(plugin: str):
return error_message("The plugin does not have a template"), 404
# Case template, prepare data
plugins = app.config["CONFIG"].get_plugins()
plugins = app.bw_config.get_plugins()
plugin_id = None
curr_plugin = {}
is_used = False
@ -2119,7 +2129,7 @@ def cache():
join(sep, "var", "cache", "bunkerweb"),
is_cache=True,
db_data=DB.get_jobs_cache_files(),
services=app.config["CONFIG"].get_config(global_only=True, methods=False, filtered_settings=("SERVER_NAME",)).get("SERVER_NAME", "").split(" "),
services=app.bw_config.get_config(global_only=True, methods=False, filtered_settings=("SERVER_NAME",)).get("SERVER_NAME", "").split(" "),
)
],
)
@ -2128,8 +2138,7 @@ def cache():
@app.route("/logs", methods=["GET"])
@login_required
def logs():
instances = app.config["INSTANCES"].get_instances()
return render_template("logs.html", instances=instances, username=current_user.get_id())
return render_template("logs.html", instances=app.bw_instances_utils.get_instances(), username=current_user.get_id())
@app.route("/logs/local", methods=["GET"])
@ -2370,7 +2379,7 @@ def logs_container(container_id):
@app.route("/reports", methods=["GET"])
@login_required
def reports():
reports = app.config["INSTANCES"].get_reports()
reports = app.bw_instances_utils.get_reports()
total_reports = len(reports)
reports = reports[:100]
@ -2413,7 +2422,7 @@ def bans():
verify_data_in_form(data={"data": None}, err_message="Missing data parameter on /bans.", redirect_url="bans")
redis_client = None
db_config = app.config["CONFIG"].get_config(
db_config = app.bw_config.get_config(
global_only=True,
methods=False,
filtered_settings=(
@ -2528,7 +2537,7 @@ def bans():
if not redis_client.delete(f"bans_ip_{unban['ip']}"):
flash(f"Couldn't unban {unban['ip']} on redis", "error")
resp = app.config["INSTANCES"].unban(unban["ip"])
resp = app.bw_instances_utils.unban(unban["ip"])
if resp:
flash(f"Couldn't unban {unban['ip']} on the following instances: {', '.join(resp)}", "error")
else:
@ -2560,7 +2569,7 @@ def bans():
flash(f"Couldn't ban {ban['ip']} on redis", "error")
redis_client.expire(f"bans_ip_{ban['ip']}", int(ban_end))
resp = app.config["INSTANCES"].ban(ban["ip"], ban_end, reason)
resp = app.bw_instances_utils.ban(ban["ip"], ban_end, reason)
if resp:
flash(f"Couldn't ban {ban['ip']} on the following instances: {', '.join(resp)}", "error")
else:
@ -2577,7 +2586,7 @@ def bans():
continue
exp = redis_client.ttl(key)
bans.append({"ip": ip, "exp": exp} | json_loads(data)) # type: ignore
instance_bans = app.config["INSTANCES"].get_bans()
instance_bans = app.bw_instances_utils.get_bans()
# Prepare data
reasons = {}

View file

@ -1,418 +0,0 @@
#!/usr/bin/env python3
from operator import itemgetter
from os import sep
from os.path import join
from subprocess import DEVNULL, STDOUT, run
from typing import Any, List, Optional, Tuple, Union
from API import API # type: ignore
from ApiCaller import ApiCaller # type: ignore
class Instance:
_id: str
name: str
hostname: str
_type: str
health: bool
env: Any
apiCaller: ApiCaller
def __init__(
self,
_id: str,
name: str,
hostname: str,
_type: str,
status: str,
data: Any = None,
apiCaller: Optional[ApiCaller] = None,
) -> None:
self._id = _id
self.name = name
self.hostname = hostname
self._type = _type
self.health = status == "up" and (
(data.attrs["State"]["Health"]["Status"] == "healthy" if "Health" in data.attrs["State"] else False) if _type == "container" and data else True
)
self.env = data
self.apiCaller = apiCaller or ApiCaller()
@property
def id(self) -> str:
return self._id
def reload(self) -> bool:
if self._type == "local":
return (
run(
[join(sep, "usr", "sbin", "nginx"), "-s", "reload"],
stdin=DEVNULL,
stderr=STDOUT,
check=False,
).returncode
== 0
)
return self.apiCaller.send_to_apis("POST", "/reload")
def start(self) -> bool:
if self._type == "local":
return (
run(
[join(sep, "usr", "sbin", "nginx"), "-e", "/var/log/bunkerweb/error.log"],
stdin=DEVNULL,
stderr=STDOUT,
check=False,
).returncode
== 0
)
return self.apiCaller.send_to_apis("POST", "/start")
def stop(self) -> bool:
if self._type == "local":
return (
run(
[join(sep, "usr", "sbin", "nginx"), "-s", "stop"],
stdin=DEVNULL,
stderr=STDOUT,
check=False,
).returncode
== 0
)
return self.apiCaller.send_to_apis("POST", "/stop")
def restart(self) -> bool:
if self._type == "local":
proc = run(
[join(sep, "usr", "sbin", "nginx"), "-s", "stop"],
stdin=DEVNULL,
stderr=STDOUT,
check=False,
)
if proc.returncode != 0:
return False
return (
run(
[join(sep, "usr", "sbin", "nginx"), "-e", "/var/log/bunkerweb/error.log"],
stdin=DEVNULL,
stderr=STDOUT,
check=False,
).returncode
== 0
)
return self.apiCaller.send_to_apis("POST", "/restart")
def bans(self) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("GET", "/bans", response=True)
def ban(self, ip: str, exp: float, reason: str) -> bool:
return self.apiCaller.send_to_apis("POST", "/ban", data={"ip": ip, "exp": exp, "reason": reason})
def unban(self, ip: str) -> bool:
return self.apiCaller.send_to_apis("POST", "/unban", data={"ip": ip})
def reports(self) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("GET", "/metrics/requests", response=True)
def metrics(self, plugin_id) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("GET", f"/metrics/{plugin_id}", response=True)
def metrics_redis(self) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("GET", "/redis/stats", response=True)
def ping(self, plugin_id) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("POST", f"/{plugin_id}/ping", response=True)
def data(self, plugin_endpoint) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("GET", f"/{plugin_endpoint}", response=True)
class Instances:
def __init__(self, db):
self.__db = db
def __instance_from_id(self, _id) -> Instance:
instances: list[Instance] = self.get_instances()
for instance in instances:
if instance.id == _id:
return instance
raise ValueError(f"Can't find instance with _id {_id}")
def get_instances(self) -> list[Instance]:
return [
Instance(
instance["hostname"],
instance["hostname"],
instance["hostname"],
instance["method"],
"up",
None,
ApiCaller(
[
API(
f"http://{instance['hostname']}:{instance['port']}",
instance["server_name"],
)
]
),
)
for instance in self.__db.get_instances()
]
def reload_instances(self) -> Union[list[str], str]:
not_reloaded: list[str] = []
for instance in self.get_instances():
if not instance.health:
not_reloaded.append(instance.name)
continue
if self.reload_instance(instance=instance).startswith("Can't reload"):
not_reloaded.append(instance.name)
return not_reloaded or "Successfully reloaded instances"
def reload_instance(self, _id: Optional[int] = None, instance: Optional[Instance] = None) -> str:
if not instance:
instance = self.__instance_from_id(_id)
try:
result = instance.reload()
except BaseException as e:
return f"Can't reload {instance.name}: {e}"
if result:
return f"Instance {instance.name} has been reloaded."
return f"Can't reload {instance.name}"
def start_instance(self, _id) -> str:
instance = self.__instance_from_id(_id)
try:
result = instance.start()
except BaseException as e:
return f"Can't start {instance.name}: {e}"
if result:
return f"Instance {instance.name} has been started."
return f"Can't start {instance.name}"
def stop_instance(self, _id) -> str:
instance = self.__instance_from_id(_id)
try:
result = instance.stop()
except BaseException as e:
return f"Can't stop {instance.name}: {e}"
if result:
return f"Instance {instance.name} has been stopped."
return f"Can't stop {instance.name}"
def restart_instance(self, _id) -> str:
instance = self.__instance_from_id(_id)
try:
result = instance.restart()
except BaseException as e:
return f"Can't restart {instance.name}: {e}"
if result:
return f"Instance {instance.name} has been restarted."
return f"Can't restart {instance.name}"
def get_bans(self, _id: Optional[int] = None) -> List[dict[str, Any]]:
if _id:
instance = self.__instance_from_id(_id)
try:
resp, instance_bans = instance.bans()
except:
return []
if not resp:
return []
return instance_bans[instance.name if instance.name != "local" else "127.0.0.1"].get("data", [])
bans: List[dict[str, Any]] = []
for instance in self.get_instances():
try:
resp, instance_bans = instance.bans()
except:
continue
if not resp:
continue
bans.extend(instance_bans[instance.name if instance.name != "local" else "127.0.0.1"].get("data", []))
bans.sort(key=itemgetter("exp"))
unique_bans = {}
return [unique_bans.setdefault(item["ip"], item) for item in bans if item["ip"] not in unique_bans]
def ban(self, ip: str, exp: float, reason: str, _id: Optional[int] = None) -> Union[str, list[str]]:
if _id:
instance = self.__instance_from_id(_id)
try:
if instance.ban(ip, exp, reason):
return ""
except BaseException as e:
return f"Can't ban {ip} on {instance.name}: {e}"
return f"Can't ban {ip} on {instance.name}"
try:
return [instance.name for instance in self.get_instances() if not instance.ban(ip, exp, reason)]
except BaseException as e:
return f"Can't ban {ip}: {e}"
def unban(self, ip: str, _id: Optional[int] = None) -> Union[str, list[str]]:
if _id:
instance = self.__instance_from_id(_id)
try:
if instance.unban(ip):
return ""
except BaseException as e:
return f"Can't unban {ip} on {instance.name}: {e}"
return f"Can't unban {ip} on {instance.name}"
try:
return [instance.name for instance in self.get_instances() if not instance.unban(ip)]
except BaseException as e:
return f"Can't unban {ip}: {e}"
def get_reports(self, _id: Optional[int] = None) -> List[dict[str, Any]]:
if _id:
instance = self.__instance_from_id(_id)
try:
resp, instance_reports = instance.reports()
except:
return []
if not resp:
return []
return (instance_reports[instance.name if instance.name != "local" else "127.0.0.1"].get("msg") or {"requests": []})["requests"]
reports: List[dict[str, Any]] = []
for instance in self.get_instances():
try:
resp, instance_reports = instance.reports()
except:
continue
if not resp:
continue
reports.extend((instance_reports[instance.name if instance.name != "local" else "127.0.0.1"].get("msg") or {"requests": []})["requests"])
reports.sort(key=itemgetter("date"), reverse=True)
return reports
def get_metrics(self, plugin_id: str):
# Get metrics from all instances
metrics = {}
for instance in self.get_instances():
instance_name = instance.name if instance.name != "local" else "127.0.0.1"
try:
if plugin_id == "redis":
resp, instance_metrics = instance.metrics_redis()
else:
resp, instance_metrics = instance.metrics(plugin_id)
except:
continue
# filters
if not resp:
continue
if (
not isinstance(instance_metrics.get(instance_name, {"msg": None}).get("msg"), dict)
or instance_metrics[instance_name].get("status", "error") != "success"
):
continue
metric_data = instance_metrics[instance_name]["msg"]
# Update metrics looking for value type
for key, value in metric_data.items():
if key not in metrics:
metrics[key] = value
continue
# Some value are the same for all instances, we don't need to update them
# Example redis_nb_keys count
if key == "redis_nb_keys":
continue
# Case value is number, add it to the existing value
if isinstance(value, (int, float)):
metrics[key] += value
# Case value is string, replace the existing value
elif isinstance(value, str):
metrics[key] = value
# Case value is list, extend it to the existing value
elif isinstance(value, list):
metrics[key].extend(value)
# Case value is a dict, loop on it and update the existing value
elif isinstance(value, dict):
for k, v in value.items():
if k not in metrics[key]:
metrics[key][k] = v
elif isinstance(v, (int, float)):
metrics[key][k] += v
elif isinstance(v, list):
metrics[key][k].extend(v)
elif isinstance(v, str):
metrics[key][k] = v
return metrics
def get_ping(self, plugin_id: str):
# Need at least one instance to get a success ping to return success
ping = {"status": "error"}
for instance in self.get_instances():
instance_name = instance.name if instance.name != "local" else "127.0.0.1"
try:
resp, ping_data = instance.ping(plugin_id)
except:
continue
if not resp:
continue
ping["status"] = ping_data[instance_name].get("status", "error")
if ping["status"] == "success":
break
return ping
def get_data(self, plugin_endpoint: str):
# Need at least one instance to get a success ping to return success
data = []
for instance in self.get_instances():
instance_name = instance.name if instance.name != "local" else "127.0.0.1"
try:
resp, instance_data = instance.data(plugin_endpoint)
except:
data.append({instance_name: {"status": "error"}})
continue
if not resp:
data.append({instance_name: {"status": "error"}})
continue
if instance_data[instance_name].get("status", "error") == "error":
data.append({instance_name: {"status": "error"}})
continue
data.append({instance_name: instance_data[instance_name].get("msg", {})})
return data

View file

@ -8,7 +8,7 @@ from re import compile as re_compile
from utils import path_to_dict
class ConfigFiles:
class CustomConfig:
def __init__(self):
self.__name_regex = re_compile(r"^[\w.-]{4,64}$")
self.__root_dirs = [child["name"] for child in path_to_dict(join(sep, "etc", "bunkerweb", "configs"))["children"]]

332
src/ui/src/instance.py Normal file
View file

@ -0,0 +1,332 @@
#!/usr/bin/env python3
from datetime import datetime
from operator import itemgetter
from typing import Any, List, Literal, Optional, Tuple, Union
from API import API # type: ignore
from ApiCaller import ApiCaller # type: ignore
class Instance:
hostname: str
method: Literal["ui", "scheduler", "autoconf", "manual"]
status: Literal["loading", "up", "down"]
creation_date: datetime
last_seen: datetime
apiCaller: ApiCaller
def __init__(
self,
hostname: str,
method: Literal["ui", "scheduler", "autoconf", "manual"],
status: Literal["loading", "up", "down"],
creation_date: datetime,
last_seen: datetime,
apiCaller: ApiCaller,
) -> None:
self.hostname = hostname
self.method = method
self.status = status
self.creation_date = creation_date
self.last_seen = last_seen
self.apiCaller = apiCaller or ApiCaller()
@staticmethod
def from_hostname(hostname: str, db) -> Optional["Instance"]:
instance = db.get_instance(hostname)
if not instance:
return None
return Instance(
instance["hostname"],
instance["method"],
instance["status"],
instance["creation_date"],
instance["last_seen"],
ApiCaller(
[
API(
f"http://{instance['hostname']}:{instance['port']}",
instance["server_name"],
)
]
),
)
@property
def id(self) -> str:
return self.hostname
def reload(self) -> str:
try:
result = self.apiCaller.send_to_apis("POST", "/reload")
except BaseException as e:
return f"Can't reload {self.hostname}: {e}"
if result:
return f"Instance {self.hostname} has been reloaded."
return f"Can't reload {self.hostname}"
def start(self) -> str:
raise NotImplementedError("Method not implemented yet")
try:
result = self.apiCaller.send_to_apis("POST", "/start")
except BaseException as e:
return f"Can't start {self.hostname}: {e}"
if result:
return f"Instance {self.hostname} has been started."
return f"Can't start {self.hostname}"
def stop(self) -> str:
try:
result = self.apiCaller.send_to_apis("POST", "/stop")
except BaseException as e:
return f"Can't stop {self.hostname}: {e}"
if result:
return f"Instance {self.hostname} has been stopped."
return f"Can't stop {self.hostname}"
def restart(self) -> str:
try:
result = self.apiCaller.send_to_apis("POST", "/restart")
except BaseException as e:
return f"Can't restart {self.hostname}: {e}"
if result:
return f"Instance {self.hostname} has been restarted."
return f"Can't restart {self.hostname}"
def ban(self, ip: str, exp: float, reason: str) -> str:
try:
result = self.apiCaller.send_to_apis("POST", "/ban", data={"ip": ip, "exp": exp, "reason": reason})
except BaseException as e:
return f"Can't ban {ip} on {self.hostname}: {e}"
if result:
return f"IP {ip} has been banned on {self.hostname} for {exp} seconds{f' with reason: {reason}' if reason else ''}."
return f"Can't ban {ip} on {self.hostname}"
def unban(self, ip: str) -> str:
try:
result = self.apiCaller.send_to_apis("POST", "/unban", data={"ip": ip})
except BaseException as e:
return f"Can't unban {ip} on {self.hostname}: {e}"
if result:
return f"IP {ip} has been unbanned on {self.hostname}."
return f"Can't unban {ip} on {self.hostname}"
def bans(self) -> Tuple[str, dict[str, Any]]:
try:
result = self.apiCaller.send_to_apis("GET", "/bans", response=True)
except BaseException as e:
return f"Can't get bans from {self.hostname}: {e}", result[1]
if result[0]:
return "", result[1]
return f"Can't get bans from {self.hostname}", result[1]
def reports(self) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("GET", "/metrics/requests", response=True)
def metrics(self, plugin_id) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("GET", f"/metrics/{plugin_id}", response=True)
def metrics_redis(self) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("GET", "/redis/stats", response=True)
def ping(self, plugin_id) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("POST", f"/{plugin_id}/ping", response=True)
def data(self, plugin_endpoint) -> Tuple[bool, dict[str, Any]]:
return self.apiCaller.send_to_apis("GET", f"/{plugin_endpoint}", response=True)
class InstancesUtils:
def __init__(self, db):
self.__db = db
def get_instances(self) -> list[Instance]:
return [
Instance(
instance["hostname"],
instance["method"],
instance["status"],
instance["creation_date"],
instance["last_seen"],
ApiCaller(
[
API(
f"http://{instance['hostname']}:{instance['port']}",
instance["server_name"],
)
]
),
)
for instance in self.__db.get_instances()
]
def reload_instances(self, *, instances: Optional[List[Instance]] = None) -> Union[list[str], str]:
return [
instance.name for instance in instances or self.get_instances() if instance.status == "down" or instance.reload().startswith("Can't reload")
] or "Successfully reloaded instances"
def get_bans(self, hostname: Optional[str] = None, *, instances: Optional[List[Instance]] = None) -> List[dict[str, Any]]:
"""Get unique bans from all instances or a specific instance and sort them by expiration date"""
def get_instance_bans(instance: Instance) -> List[dict[str, Any]]:
resp, instance_bans = instance.bans()
if resp:
self.__db.logger.warning(resp)
return []
return instance_bans[instance.hostname].get("data", [])
bans: List[dict[str, Any]] = []
if hostname:
instance = Instance.from_hostname(hostname, self.__db)
if not instance:
return []
bans = get_instance_bans(instance)
else:
for instance in instances or self.get_instances():
bans.extend(get_instance_bans(instance))
unique_bans = {}
return [unique_bans.setdefault(item["ip"], item) for item in sorted(bans, key=itemgetter("exp")) if item["ip"] not in unique_bans]
def get_reports(self, hostname: Optional[str] = None, *, instances: Optional[List[Instance]] = None) -> List[dict[str, Any]]:
"""Get reports from all instances or a specific instance and sort them by date"""
def get_instance_reports(instance: Instance) -> Tuple[bool, dict[str, Any]]:
resp, instance_reports = instance.reports()
if resp:
self.__db.logger.warning(resp)
return []
return (instance_reports[instance.hostname].get("msg") or {"requests": []}).get("requests", [])
reports: List[dict[str, Any]] = []
if hostname:
instance = Instance.from_hostname(hostname, self.__db)
if not instance:
return []
reports = get_instance_reports(instance)
else:
for instance in instances or self.get_instances():
reports.extend(get_instance_reports(instance))
return sorted(reports, key=itemgetter("date"), reverse=True)
def get_metrics(self, plugin_id: str, hostname: Optional[str] = None, *, instances: Optional[List[Instance]] = None):
"""Get metrics from all instances or a specific instance"""
def update_metrics_from_instance(instance: Instance, metrics: dict) -> dict[str, Any]:
try:
if plugin_id == "redis":
resp, instance_metrics = instance.metrics_redis()
else:
resp, instance_metrics = instance.metrics(plugin_id)
except BaseException as e:
self.__db.logger.warning(f"Can't get metrics from {instance.hostname}: {e}")
return metrics
# filters
if not resp:
self.__db.logger.warning(f"Can't get metrics from {instance.hostname}")
return metrics
if (
not isinstance(instance_metrics.get(instance.hostname, {"msg": None}).get("msg"), dict)
or instance_metrics[instance.hostname].get("status", "error") != "success"
):
self.__db.logger.warning(
f"Can't get metrics from {instance.hostname}: {instance_metrics[instance.hostname].get('msg')} - {instance_metrics[instance.hostname].get('status')}"
)
return metrics
# Update metrics looking for value type
for key, value in instance_metrics[instance.hostname]["msg"].items():
if key not in metrics:
metrics[key] = value
continue
# Some value are the same for all instances, we don't need to update them
# Example redis_nb_keys count
if key == "redis_nb_keys":
continue
# Case value is number, add it to the existing value
if isinstance(value, (int, float)):
metrics[key] += value
# Case value is string, replace the existing value
elif isinstance(value, str):
metrics[key] = value
# Case value is list, extend it to the existing value
elif isinstance(value, list):
metrics[key].extend(value)
# Case value is a dict, loop on it and update the existing value
elif isinstance(value, dict):
for k, v in value.items():
if k not in metrics[key]:
metrics[key][k] = v
continue
elif isinstance(v, (int, float)):
metrics[key][k] += v
continue
elif isinstance(v, list):
metrics[key][k].extend(v)
continue
metrics[key][k] = v
return metrics
metrics = {}
if hostname:
instance = Instance.from_hostname(hostname, self.__db)
if not instance:
return {}
return update_metrics_from_instance(instance, metrics.copy())
for instance in instances or self.get_instances():
metrics = update_metrics_from_instance(instance, metrics.copy())
return metrics
def get_ping(self, plugin_id: str, *, instances: Optional[List[Instance]] = None):
"""Get ping from all instances and return the first success"""
ping = {"status": "error"}
for instance in instances or self.get_instances():
try:
resp, ping_data = instance.ping(plugin_id)
except:
continue
if not resp:
continue
ping["status"] = ping_data[instance.hostname].get("status", "error")
if ping["status"] == "success":
return ping
return ping
def get_data(self, plugin_endpoint: str, *, instances: Optional[List[Instance]] = None):
"""Get data from all instances and return the first success"""
data = []
for instance in instances or self.get_instances():
try:
resp, instance_data = instance.data(plugin_endpoint)
except:
data.append({instance.hostname: {"status": "error"}})
continue
if not resp:
data.append({instance.hostname: {"status": "error"}})
continue
if instance_data[instance.hostname].get("status", "error") == "error":
data.append({instance.hostname: {"status": "error"}})
continue
data.append({instance.hostname: instance_data[instance.hostname].get("msg", {})})
return data

View file

@ -15,8 +15,8 @@
{% for instance in instances %}
{% if loop.first %}
{% if
instance.name %}
{{ instance.name }}
instance.hostname %}
{{ instance.hostname }}
{% else %}
no instance
{% endif %}
@ -32,8 +32,8 @@
<!-- dropdown-->
<div data-{{ attribute_name }}-setting-select-dropdown="instances" class="mt-1 hidden z-100 absolute flex-col w-full translate-y-16 max-h-[350px] overflow-hidden overflow-y-auto">
{% for instance in instances %}
<button data-{{ attribute_name }}-setting-select-dropdown-btn="instances" value="{{ instance.name }}" data-_type="{{ instance._type }}" class="{% if loop.first %}dark:bg-primary bg-primary text-gray-300 border-t rounded-t {% else %} bg-white dark:bg-slate-700 {% endif %} {% if loop.last %}rounded-b{% endif %} border-b border-l border-r border-gray-300 dark:hover:brightness-90 hover:brightness-90 my-0 relative py-2 px-3 text-left align-middle transition-all rounded-none cursor-pointer leading-normal text-sm ease-in tracking-tight-rem dark:border-slate-600 dark:text-gray-300">
{{ instance.name }}
<button data-{{ attribute_name }}-setting-select-dropdown-btn="instances" value="{{ instance.hostname }}" data-_type="{{ instance._type }}" class="{% if loop.first %}dark:bg-primary bg-primary text-gray-300 border-t rounded-t {% else %} bg-white dark:bg-slate-700 {% endif %} {% if loop.last %}rounded-b{% endif %} border-b border-l border-r border-gray-300 dark:hover:brightness-90 hover:brightness-90 my-0 relative py-2 px-3 text-left align-middle transition-all rounded-none cursor-pointer leading-normal text-sm ease-in tracking-tight-rem dark:border-slate-600 dark:text-gray-300">
{{ instance.hostname }}
</button>
{% endfor %}
</div>