Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add backup command line interface scripts and everything around it

Browse source
This commit is contained in:
Théophile Diot 2024-04-02 19:26:57 +01:00
parent 2a7936bfb8
commit 9a8e0a38ff
No known key found for this signature in database
GPG key ID: 248FEA4BAE400D06
8 changed files with 369 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
src/common/core/backup/bwcli/list.py Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env python3
from datetime import datetime
from os.path import join, sep
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
deps_path = join(sep, "usr", "share", "bunkerweb", "core", "backup")
if deps_path not in sys_path:
sys_path.append(deps_path)
from utils import BACKUP_DIR, LOGGER
try:
backups = sorted(BACKUP_DIR.glob("*.zip"), reverse=True)
message = ""
if backups:
message = f"Found {len(backups)} backup{'s' if len(backups) > 1 else ''} in {BACKUP_DIR} :"
# Show a table with the backups details
message += "\n+------------+---------------------+"
message += "\n| Database | Date |"
message += "\n+------------+---------------------+"
for backup in backups:
database = backup.name.split("-")[1]
date = datetime.strptime("-".join(backup.stem.split("-")[2:]), "%Y-%m-%d_%H-%M-%S")
message += f"\n| {database:<10} | {date.strftime('%d/%m/%Y %H:%M:%S')} |"
message += "\n+------------+---------------------+"
else:
message = f"No backup found in {BACKUP_DIR}"
LOGGER.info(message)
except SystemExit as se:
sys_exit(se.code)
except:
LOGGER.error(f"Error while executing backup list command :\n{format_exc()}")
sys_exit(1)

70
src/common/core/backup/bwcli/restore.py Executable file
View file

@ -0,0 +1,70 @@
#!/usr/bin/env python3
from argparse import ArgumentParser
from datetime import datetime
from os.path import join, sep
from pathlib import Path
from shutil import rmtree
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
deps_path = join(sep, "usr", "share", "bunkerweb", "core", "backup")
if deps_path not in sys_path:
sys_path.append(deps_path)
from utils import acquire_db_lock, backup_database, BACKUP_DIR, DB_LOCK_FILE, LOGGER, restore_database
status = 0
try:
acquire_db_lock()
# Global parser
parser = ArgumentParser(description="BunkerWeb's backup plugin restore command line interface")
backups = sorted(BACKUP_DIR.glob("*.zip"), reverse=True)
backup_file = backups[0] if backups else None
# Optional backup file argument
parser.add_argument("backup_file", nargs="?", const="default_value", default=backup_file, type=str, help="backup file to restore (default : latest backup)")
# Parse args
args = parser.parse_args()
backup_file = Path(args.backup_file) if args.backup_file else None
if not backup_file:
if backup_file == BACKUP_DIR and not BACKUP_DIR.is_dir():
LOGGER.error(f"Backup directory {BACKUP_DIR} does not exist, aborting restore")
sys_exit(1)
if not backups:
LOGGER.error(f"No backup found in {BACKUP_DIR}, aborting restore")
sys_exit(1)
if not backup_file:
LOGGER.error("No backup file to restore, aborting restore")
sys_exit(1)
if not backup_file.is_file():
LOGGER.error(f"Backup file {backup_file} does not exist, aborting restore")
sys_exit(1)
LOGGER.info("Backing up the current database before restoring the backup ...")
current_time = datetime.now()
tmp_backup_dir = Path(sep, "tmp", "bunkerweb", "backup", current_time.strftime("%Y-%m-%d_%H-%M-%S"))
tmp_backup_dir.mkdir(parents=True, exist_ok=True)
backup_database(current_time, tmp_backup_dir)
LOGGER.info(f"Restoring backup {backup_file} ...")
restore_database(backup_file)
rmtree(tmp_backup_dir.parent, ignore_errors=True)
except SystemExit as se:
status = se.code
except:
LOGGER.error(f"Error while executing backup restore command :\n{format_exc()}")
status = 1
finally:
DB_LOCK_FILE.unlink(missing_ok=True)
sys_exit(status)

51
src/common/core/backup/bwcli/save.py Executable file
View file

@ -0,0 +1,51 @@
#!/usr/bin/env python3
from argparse import ArgumentParser
from datetime import datetime
from os.path import join, sep
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
deps_path = join(sep, "usr", "share", "bunkerweb", "core", "backup")
if deps_path not in sys_path:
sys_path.append(deps_path)
from utils import acquire_db_lock, backup_database, BACKUP_DIR, DB_LOCK_FILE, LOGGER
status = 0
try:
acquire_db_lock()
# Global parser
parser = ArgumentParser(description="BunkerWeb's backup plugin save command line interface")
# Optional directory argument
parser.add_argument("-d", "--directory", default=BACKUP_DIR, type=str, help="directory where to save the backup")
# Parse args
args = parser.parse_args()
directory = Path(args.directory)
LOGGER.debug(f"Backup directory: {directory}")
if not directory.is_dir():
if directory == BACKUP_DIR:
LOGGER.error(f"Backup directory {directory} does not exist")
sys_exit(1)
LOGGER.info(f"Creating directory {directory} as it does not exist")
directory.mkdir(parents=True, exist_ok=True)
backup_database(datetime.now(), directory)
except SystemExit as se:
status = se.code
except:
LOGGER.error(f"Error while executing backup save command :\n{format_exc()}")
status = 1
finally:
DB_LOCK_FILE.unlink(missing_ok=True)
sys_exit(status)

60
src/common/core/backup/jobs/backup-data.py Normal file → Executable file
View file

@ -1,35 +1,33 @@
#!/usr/bin/env python3
from datetime import datetime, timedelta
from os import environ, getenv, sep
from os import getenv, sep
from os.path import join
from pathlib import Path
from subprocess import PIPE, run
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
from typing import Literal
from zipfile import ZIP_DEFLATED, ZipFile
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",), ("core", "backup"))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from logger import setup_logger # type: ignore
from jobs import Job # type: ignore
from utils import backup_database
LOGGER = setup_logger("BACKUP", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
backup_dir = Path(getenv("BACKUP_DIRECTORY", "/var/lib/bunkerweb/backups"))
backup_dir.mkdir(parents=True, exist_ok=True)
# Check if backup is activated
if getenv("USE_BACKUP", "yes") == "no":
LOGGER.info("Backup feature is disabled, skipping backup ...")
sys_exit(0)
backup_dir = Path(getenv("BACKUP_DIRECTORY", "/var/lib/bunkerweb/backups"))
JOB = Job(LOGGER)
last_backup = JOB.get_cache("last_backup.txt")
@ -55,51 +53,7 @@ try:
LOGGER.info("First start of the scheduler, skipping backup ...")
sys_exit(0)
database: Literal["sqlite", "mariadb", "mysql", "postgresql"] = JOB.db.database_uri.split(":")[0].split("+")[0]
backup_file = backup_dir.joinpath(f"backup-{database}-{current_time.strftime('%Y-%m-%d')}.zip")
backup_file.parent.mkdir(parents=True, exist_ok=True)
LOGGER.debug(f"Backup file path: {backup_file}")
if database == "sqlite":
match = JOB.db.DB_STRING_RX.search(JOB.db.database_uri)
if not match:
LOGGER.error(f"Invalid database string provided: {JOB.db.database_uri}, skipping backup ...")
sys_exit(1)
db_path = Path(match.group("path"))
LOGGER.info("Creating a backup for the SQLite database ...")
proc = run(["sqlite3", db_path.as_posix(), ".dump"], stdout=PIPE, stderr=PIPE)
else:
db_host = JOB.db.database_uri.rsplit("@", 1)[1].split("/")[0].split(":")
db_port = None
if len(db_host) == 1:
db_host = db_host[0]
else:
db_host, db_port = db_host
db_user = JOB.db.database_uri.split("://")[1].split(":")[0]
db_password = JOB.db.database_uri.split("://")[1].split(":")[1].rsplit("@", 1)[0]
db_database_name = JOB.db.database_uri.split("/")[-1]
if database in ("mariadb", "mysql"):
LOGGER.info("Creating a backup for the MariaDB/MySQL database ...")
proc = run(["mysqldump", "-h", db_host, "-u", db_user, db_database_name], stdout=PIPE, stderr=PIPE, env=environ | {"MYSQL_PWD": db_password})
elif database == "postgresql":
LOGGER.info("Creating a backup for the PostgreSQL database ...")
proc = run(
["pg_dump", "-h", db_host, "-U", db_user, "-d", db_database_name, "-w"], stdout=PIPE, stderr=PIPE, env=environ | {"PGPASSWORD": db_password}
)
if proc.returncode != 0:
LOGGER.error(f"Failed to dump the database: {proc.stderr.decode()}")
sys_exit(1)
with ZipFile(backup_file, "w", compression=ZIP_DEFLATED) as zipf:
zipf.writestr(backup_file.with_suffix(".sql").name, proc.stdout)
backup_database(current_time)
backup_rotation = int(getenv("BACKUP_ROTATION", "7"))

7
src/common/core/backup/plugin.json
View file

@ -50,5 +50,10 @@
"every": "day",
"reload": false
}
]
],
"bwcli": {
"list": "list.py",
"save": "save.py",
"restore": "restore.py"
}
}

175
src/common/core/backup/utils.py Executable file
View file

@ -0,0 +1,175 @@
#!/usr/bin/env python3
from datetime import datetime
from os import environ, getenv
from os.path import join, sep
from pathlib import Path
from re import compile as re_compile
from subprocess import PIPE, run
from sys import exit as sys_exit, path as sys_path
from time import sleep
from typing import Literal
from zipfile import ZIP_DEFLATED, ZipFile
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from model import Base # type: ignore
LOGGER = setup_logger("BACKUP", "INFO")
DB_STRING_RX = re_compile(r"^(?P<database>(mariadb|mysql)(\+pymysql)?|sqlite(\+pysqlite)?|postgresql(\+psycopg)?):/+(?P<path>/[^\s]+)")
BACKUP_DIR = Path(getenv("BACKUP_DIRECTORY", "/var/lib/bunkerweb/backups"))
DB_LOCK_FILE = Path(sep, "var", "lib", "bunkerweb", "db.lock")
def acquire_db_lock():
"""Acquire the database lock to prevent concurrent access to the database."""
current_time = datetime.now()
while DB_LOCK_FILE.is_file() and DB_LOCK_FILE.stat().st_ctime + 30 > current_time.timestamp():
LOGGER.warning("Database is locked, waiting for it to be unlocked (timeout: 30s) ...")
sleep(1)
DB_LOCK_FILE.unlink(missing_ok=True)
DB_LOCK_FILE.touch()
def backup_database(current_time: datetime, backup_dir: Path = BACKUP_DIR):
"""Backup the database."""
database_uri = getenv("DATABASE_URI", "sqlite:////var/lib/bunkerweb/db.sqlite3")
database: Literal["sqlite", "mariadb", "mysql", "postgresql"] = database_uri.split(":")[0].split("+")[0] # type: ignore
backup_file = backup_dir.joinpath(f"backup-{database}-{current_time.strftime('%Y-%m-%d_%H-%M-%S')}.zip")
LOGGER.debug(f"Backup file path: {backup_file}")
if database == "sqlite":
match = DB_STRING_RX.search(database_uri)
if not match:
LOGGER.error(f"Invalid database string provided: {database_uri}, skipping backup ...")
sys_exit(1)
db_path = Path(match.group("path"))
LOGGER.info("Creating a backup for the SQLite database ...")
proc = run(["sqlite3", db_path.as_posix(), ".dump"], stdout=PIPE, stderr=PIPE)
else:
db_host = database_uri.rsplit("@", 1)[1].split("/")[0].split(":")
db_port = None
if len(db_host) == 1:
db_host = db_host[0]
else:
db_host, db_port = db_host
db_user = database_uri.split("://")[1].split(":")[0]
db_password = database_uri.split("://")[1].split(":")[1].rsplit("@", 1)[0]
db_database_name = database_uri.split("/")[-1]
if database in ("mariadb", "mysql"):
LOGGER.info("Creating a backup for the MariaDB/MySQL database ...")
cmd = ["mysqldump", "-h", db_host, "-u", db_user, db_database_name]
if db_port:
cmd.extend(["-P", db_port])
proc = run(cmd, stdout=PIPE, stderr=PIPE, env=environ | {"MYSQL_PWD": db_password})
elif database == "postgresql":
LOGGER.info("Creating a backup for the PostgreSQL database ...")
cmd = ["pg_dump", "-h", db_host, "-U", db_user, db_database_name, "-w"]
if db_port:
cmd.extend(["-p", db_port])
proc = run(cmd, stdout=PIPE, stderr=PIPE, env=environ | {"PGPASSWORD": db_password})
if proc.returncode != 0:
LOGGER.error(f"Failed to dump the database: {proc.stderr.decode()}")
sys_exit(1)
with ZipFile(backup_file, "w", compression=ZIP_DEFLATED) as zipf:
zipf.writestr(backup_file.with_suffix(".sql").name, proc.stdout)
backup_file.chmod(0o600)
LOGGER.info(f"💾 Backup {backup_file.name} created successfully in {backup_dir}")
def restore_database(backup_file: Path):
"""Restore the database from a backup."""
db = Database(LOGGER)
Base.metadata.drop_all(db.sql_engine)
database: Literal["sqlite", "mariadb", "mysql", "postgresql"] = db.database_uri.split(":")[0].split("+")[0] # type: ignore
if database == "sqlite":
match = DB_STRING_RX.search(db.database_uri)
if not match:
LOGGER.error(f"Invalid database string provided: {db.database_uri}, skipping restore ...")
sys_exit(1)
db_path = Path(match.group("path"))
# Clear the database
proc = run(["sqlite3", db_path.as_posix(), ".read", "/dev/null"], stdout=PIPE, stderr=PIPE)
LOGGER.info("Restoring the SQLite database ...")
tmp_file = Path(sep, "var", "tmp", "bunkerweb", backup_file.with_suffix(".sql").name)
with ZipFile(backup_file, "r") as zipf:
zipf.extractall(path=tmp_file.parent)
proc = run(["sqlite3", db_path.as_posix(), f".read {tmp_file.as_posix()}"], stdout=PIPE, stderr=PIPE)
tmp_file.unlink(missing_ok=True)
else:
db_host = db.database_uri.rsplit("@", 1)[1].split("/")[0].split(":")
db_port = None
if len(db_host) == 1:
db_host = db_host[0]
else:
db_host, db_port = db_host
db_user = db.database_uri.split("://")[1].split(":")[0]
db_password = db.database_uri.split("://")[1].split(":")[1].rsplit("@", 1)[0]
db_database_name = db.database_uri.split("/")[-1]
if database in ("mariadb", "mysql"):
LOGGER.info("Restoring the MariaDB/MySQL database ...")
cmd = ["mysql", "-h", db_host, "-u", db_user, db_database_name]
if db_port:
cmd.extend(["-P", db_port])
with ZipFile(backup_file, "r") as zipf:
proc = run(
cmd,
stdout=PIPE,
stderr=PIPE,
env=environ | {"MYSQL_PWD": db_password},
input=zipf.read(backup_file.with_suffix(".sql").name),
)
elif database == "postgresql":
LOGGER.info("Restoring the PostgreSQL database ...")
cmd = ["psql", "-h", db_host, "-U", db_user, db_database_name]
if db_port:
cmd.extend(["-p", db_port])
with ZipFile(backup_file, "r") as zipf:
proc = run(
cmd,
stdout=PIPE,
stderr=PIPE,
env=environ | {"PGPASSWORD": db_password},
input=zipf.read(backup_file.with_suffix(".sql").name),
)
if proc.returncode != 0:
LOGGER.error(f"Failed to restore the database: {proc.stderr.decode()}")
sys_exit(1)
err = db.checked_changes(value=True)
if err:
LOGGER.error(f"Error while applying changes to the database: {err}, you may need to reload the application")
LOGGER.info(f"💾 Database restored successfully from {backup_file}")

30
src/common/db/Database.py
View file

@ -76,7 +76,7 @@ class Database:
self.logger.warning("The pool parameter is deprecated, it will be removed in the next version")
self.__session_factory = None
self.__sql_engine = None
self.sql_engine = None
if not sqlalchemy_string:
sqlalchemy_string = getenv("DATABASE_URI", "sqlite:////var/lib/bunkerweb/db.sqlite3")
@ -116,7 +116,7 @@ class Database:
}
try:
self.__sql_engine = create_engine(sqlalchemy_string, **engine_kwargs)
self.sql_engine = create_engine(sqlalchemy_string, **engine_kwargs)
except ArgumentError:
self.logger.error(f"Invalid database URI: {sqlalchemy_string}")
error = True
@ -128,7 +128,7 @@ class Database:
_exit(1)
try:
assert self.__sql_engine is not None
assert self.sql_engine is not None
except AssertionError:
self.logger.error("The database engine is not initialized")
_exit(1)
@ -138,7 +138,7 @@ class Database:
while not_connected:
try:
with self.__sql_engine.connect() as conn:
with self.sql_engine.connect() as conn:
conn.execute(text("CREATE TABLE IF NOT EXISTS test (id INT)"))
conn.execute(text("DROP TABLE test"))
not_connected = False
@ -151,8 +151,8 @@ class Database:
if "attempt to write a readonly database" in str(e):
self.logger.warning("The database is read-only, waiting for it to become writable. Retrying in 5 seconds ...")
self.__sql_engine.dispose(close=True)
self.__sql_engine = create_engine(sqlalchemy_string, **engine_kwargs)
self.sql_engine.dispose(close=True)
self.sql_engine = create_engine(sqlalchemy_string, **engine_kwargs)
if "Unknown table" in str(e):
not_connected = False
continue
@ -174,18 +174,18 @@ class Database:
if self.__session_factory:
self.__session_factory.close_all()
if self.__sql_engine:
self.__sql_engine.dispose()
if self.sql_engine:
self.sql_engine.dispose()
@contextmanager
def __db_session(self, raise_error: bool = False) -> Any:
try:
assert self.__sql_engine is not None
assert self.sql_engine is not None
except AssertionError:
self.logger.error("The database engine is not initialized")
_exit(1)
with self.__sql_engine.connect() as conn:
with self.sql_engine.connect() as conn:
session_factory = sessionmaker(bind=conn, autoflush=True, expire_on_commit=False)
session = scoped_session(session_factory)
try:
@ -423,9 +423,9 @@ class Database:
def init_tables(self, default_plugins: List[dict], bunkerweb_version: str) -> Tuple[bool, str]:
"""Initialize the database tables and return the result"""
assert self.__sql_engine is not None, "The database engine is not initialized"
assert self.sql_engine is not None, "The database engine is not initialized"
inspector = inspect(self.__sql_engine)
inspector = inspect(self.sql_engine)
db_version = None
has_all_tables = True
old_data = {}
@ -439,7 +439,7 @@ class Database:
if db_version != bunkerweb_version:
self.logger.warning(f"Database version ({db_version}) is different from Bunkerweb version ({bunkerweb_version}), migrating ...")
metadata = sql_metadata()
metadata.reflect(self.__sql_engine)
metadata.reflect(self.sql_engine)
for table_name in Base.metadata.tables.keys():
if not inspector.has_table(table_name):
@ -460,13 +460,13 @@ class Database:
session.execute(text(f"DROP TABLE {table_name}_{db_version_id}"))
session.execute(text(f"ALTER TABLE {table_name} RENAME TO {table_name}_{db_version_id}"))
Base.metadata.drop_all(self.__sql_engine)
Base.metadata.drop_all(self.sql_engine)
if has_all_tables and db_version and db_version == bunkerweb_version:
return False, ""
try:
Base.metadata.create_all(self.__sql_engine, checkfirst=True)
Base.metadata.create_all(self.sql_engine, checkfirst=True)
except BaseException:
return False, format_exc()

11
src/scheduler/main.py
View file

@ -6,6 +6,7 @@ from datetime import datetime
from glob import glob
from hashlib import sha256
from io import BytesIO
from itertools import chain
from json import load as json_load
from os import _exit, environ, getenv, getpid, listdir, sep, walk
from os.path import basename, dirname, join, normpath
@ -45,6 +46,7 @@ TMP_PATH.mkdir(parents=True, exist_ok=True)
HEALTHY_PATH = TMP_PATH.joinpath("scheduler.healthy")
SCHEDULER_TMP_ENV_PATH = TMP_PATH.joinpath("scheduler.env")
SCHEDULER_TMP_ENV_PATH.touch()
DB_LOCK_FILE = Path(sep, "var", "lib", "bunkerweb", "db.lock")
logger = setup_logger("Scheduler", getenv("LOG_LEVEL", "INFO"))
@ -151,7 +153,7 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U
tar.extractall(original_path)
tmp_path.unlink(missing_ok=True)
for job_file in original_path.joinpath(plugin["id"], "jobs").glob("*"):
for job_file in chain(original_path.joinpath(plugin["id"], "jobs").glob("*"), original_path.joinpath(plugin["id"], "bwcli").glob("*")):
job_file.chmod(job_file.stat().st_mode | S_IEXEC)
except BaseException as e:
logger.error(f"Error while generating {'pro ' if pro else ''}external plugins \"{plugin['name']}\": {e}")
@ -563,6 +565,13 @@ if __name__ == "__main__":
while RUN and not NEED_RELOAD:
SCHEDULER.run_pending()
sleep(1)
current_time = datetime.now()
while DB_LOCK_FILE.is_file() and DB_LOCK_FILE.stat().st_ctime + 30 > current_time.timestamp():
logger.debug("Database is locked, waiting for it to be unlocked (timeout: 30s) ...")
sleep(1)
DB_LOCK_FILE.unlink(missing_ok=True)
changes = db.check_changes()