From aed1c912b5372c44aa7d0935b845fc05ab01b55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 2 Apr 2024 15:55:20 +0100 Subject: [PATCH] Add new backup plugin and update required dependencies accordingly --- src/common/core/backup/jobs/backup-data.py | 132 +++++++++++++++++++++ src/common/core/backup/plugin.json | 54 +++++++++ src/linux/fpm-centos | 2 +- src/linux/fpm-debian | 2 +- src/linux/fpm-fedora | 2 +- src/linux/fpm-rhel | 2 +- src/linux/fpm-rhel9 | 2 +- src/linux/fpm-ubuntu | 2 +- src/scheduler/Dockerfile | 2 +- 9 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 src/common/core/backup/jobs/backup-data.py create mode 100644 src/common/core/backup/plugin.json diff --git a/src/common/core/backup/jobs/backup-data.py b/src/common/core/backup/jobs/backup-data.py new file mode 100644 index 000000000..7ea746855 --- /dev/null +++ b/src/common/core/backup/jobs/backup-data.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +from datetime import datetime, timedelta +from os import environ, getenv, sep +from os.path import join +from pathlib import Path +from subprocess import PIPE, run +from sys import exit as sys_exit, path as sys_path +from threading import Lock +from traceback import format_exc +from typing import Literal +from zipfile import ZIP_DEFLATED, ZipFile + +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: + if deps_path not in sys_path: + sys_path.append(deps_path) + +from logger import setup_logger # type: ignore +from jobs import Job # type: ignore + + +LOGGER = setup_logger("BACKUP", getenv("LOG_LEVEL", "INFO")) +status = 0 + +try: + # Check if backup is activated + if getenv("USE_BACKUP", "yes") == "no": + LOGGER.info("Backup feature is disabled, skipping backup ...") + sys_exit(0) + + backup_dir = Path(getenv("BACKUP_DIRECTORY", "/var/lib/bunkerweb/backups")) + + JOB = Job(LOGGER) + + last_backup = JOB.get_cache("last_backup.txt") + if last_backup: + last_backup = datetime.fromisoformat(last_backup.decode()) + + current_time = datetime.now() + backup_period = getenv("BACKUP_SCHEDULE", "daily") + PERIOD_STAMPS = { + "daily": timedelta(days=1).total_seconds(), + "weekly": timedelta(weeks=1).total_seconds(), + "monthly": timedelta(weeks=4).total_seconds(), + } + + if last_backup and last_backup.timestamp() + PERIOD_STAMPS[backup_period] > current_time.timestamp(): + LOGGER.info(f"Backup already done within the last {backup_period} period, skipping backup ...") + sys_exit(0) + + with Lock(): + is_scheduler_first_start = JOB.db.is_scheduler_first_start() + + if is_scheduler_first_start: + LOGGER.info("First start of the scheduler, skipping backup ...") + sys_exit(0) + + database: Literal["sqlite", "mariadb", "mysql", "postgresql"] = JOB.db.database_uri.split(":")[0].split("+")[0] + backup_file = backup_dir.joinpath(f"backup-{database}-{current_time.strftime('%Y-%m-%d')}.zip") + backup_file.parent.mkdir(parents=True, exist_ok=True) + LOGGER.debug(f"Backup file path: {backup_file}") + + if database == "sqlite": + match = JOB.db.DB_STRING_RX.search(JOB.db.database_uri) + if not match: + LOGGER.error(f"Invalid database string provided: {JOB.db.database_uri}, skipping backup ...") + sys_exit(1) + + db_path = Path(match.group("path")) + + LOGGER.info("Creating a backup for the SQLite database ...") + + proc = run(["sqlite3", db_path.as_posix(), ".dump"], stdout=PIPE, stderr=PIPE) + else: + db_host = JOB.db.database_uri.rsplit("@", 1)[1].split("/")[0].split(":") + db_port = None + if len(db_host) == 1: + db_host = db_host[0] + else: + db_host, db_port = db_host + + db_user = JOB.db.database_uri.split("://")[1].split(":")[0] + db_password = JOB.db.database_uri.split("://")[1].split(":")[1].rsplit("@", 1)[0] + db_database_name = JOB.db.database_uri.split("/")[-1] + + if database in ("mariadb", "mysql"): + LOGGER.info("Creating a backup for the MariaDB/MySQL database ...") + + proc = run(["mysqldump", "-h", db_host, "-u", db_user, db_database_name], stdout=PIPE, stderr=PIPE, env=environ | {"MYSQL_PWD": db_password}) + elif database == "postgresql": + LOGGER.info("Creating a backup for the PostgreSQL database ...") + + proc = run( + ["pg_dump", "-h", db_host, "-U", db_user, "-d", db_database_name, "-w"], stdout=PIPE, stderr=PIPE, env=environ | {"PGPASSWORD": db_password} + ) + + if proc.returncode != 0: + LOGGER.error(f"Failed to dump the database: {proc.stderr.decode()}") + sys_exit(1) + + with ZipFile(backup_file, "w", compression=ZIP_DEFLATED) as zipf: + zipf.writestr(backup_file.with_suffix(".sql").name, proc.stdout) + + backup_rotation = int(getenv("BACKUP_ROTATION", "7")) + + # Get all backup files in the directory + backup_files = backup_dir.glob("backup-*.zip") + + # Sort the backup files by name + sorted_files = sorted(backup_files) + + # Check if the number of backup files exceeds the rotation limit + if len(sorted_files) > backup_rotation: + # Calculate the number of files to remove + num_files_to_remove = len(sorted_files) - backup_rotation + + # Remove the oldest backup files + for file in sorted_files[:num_files_to_remove]: + LOGGER.warning(f"Removing old backup file: {file}, as the rotation limit has been reached ...") + file.unlink() + + cached, err = JOB.cache_file("last_backup.txt", current_time.isoformat().encode()) + if not cached: + LOGGER.error(f"Failed to cache last_backup.txt :\n{err}") + status = 2 +except SystemExit as e: + status = e.code +except: + status = 2 + LOGGER.error(f"Exception while running backup-data.py :\n{format_exc()}") + +sys_exit(status) diff --git a/src/common/core/backup/plugin.json b/src/common/core/backup/plugin.json new file mode 100644 index 000000000..85775b803 --- /dev/null +++ b/src/common/core/backup/plugin.json @@ -0,0 +1,54 @@ +{ + "id": "backup", + "name": "Backup", + "description": "Backup your data to a custom location. Ensure the safety and availability of your important files by creating regular backups.", + "version": "1.0", + "stream": "no", + "settings": { + "USE_BACKUP": { + "context": "global", + "default": "yes", + "help": "Enable or disable the backup feature", + "id": "use-backup", + "label": "Activate automatic backup", + "regex": "^(yes|no)$", + "type": "check" + }, + "BACKUP_DIRECTORY": { + "context": "global", + "default": "/var/lib/bunkerweb/backups", + "help": "The directory where the backup will be stored", + "id": "backup-directory", + "label": "Backup directory", + "regex": "^.*$", + "type": "text" + }, + "BACKUP_SCHEDULE": { + "context": "global", + "default": "daily", + "help": "The frequency of the backup", + "id": "backup-schedule", + "label": "Backup schedule", + "regex": "^(daily|weekly|monthly)$", + "type": "select", + "select": ["daily", "weekly", "monthly"] + }, + "BACKUP_ROTATION": { + "context": "global", + "default": "7", + "help": "The number of backups to keep", + "id": "backup-rotation", + "label": "Backup rotation", + "regex": "^[1-9][0-9]*$", + "type": "text" + } + }, + "jobs": [ + { + "name": "backup-data", + "file": "backup-data.py", + "every": "day", + "reload": false + } + ] +} diff --git a/src/linux/fpm-centos b/src/linux/fpm-centos index d9d12cdfc..3e4182063 100644 --- a/src/linux/fpm-centos +++ b/src/linux/fpm-centos @@ -3,7 +3,7 @@ --license agpl3 --version %VERSION% --architecture x86_64 ---depends bash --depends epel-release --depends python39 --depends 'nginx = 1:1.24.0-1.el8.ngx' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends GeoIP-devel --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends brotli --depends openssl --depends libpq +--depends bash --depends epel-release --depends python39 --depends 'nginx = 1:1.24.0-1.el8.ngx' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends GeoIP-devel --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends brotli --depends openssl --depends libpq --depends mysql --depends postgresql --depends sqlite --description "BunkerWeb %VERSION% for CentOS Stream 8" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " diff --git a/src/linux/fpm-debian b/src/linux/fpm-debian index f1d68b8de..59f506017 100644 --- a/src/linux/fpm-debian +++ b/src/linux/fpm-debian @@ -3,7 +3,7 @@ --license agpl3 --version %VERSION% --architecture %ARCH% ---depends bash --depends python3 --depends procps --depends python3-pip --depends 'nginx = 1.24.0-1~bookworm' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends lsof --depends libpq5 --depends libpcre3 --depends libcap2-bin --depends logrotate +--depends bash --depends python3 --depends procps --depends python3-pip --depends 'nginx = 1.24.0-1~bookworm' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends lsof --depends libpq5 --depends libpcre3 --depends libcap2-bin --depends logrotate --depends mariadb-client --depends postgresql --depends sqlite3 --description "BunkerWeb %VERSION% for Debian 12" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " diff --git a/src/linux/fpm-fedora b/src/linux/fpm-fedora index 80ca393ed..a5b8a35c1 100644 --- a/src/linux/fpm-fedora +++ b/src/linux/fpm-fedora @@ -3,7 +3,7 @@ --license agpl3 --version %VERSION% --architecture %ARCH% ---depends bash --depends python3 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream --depends pcre --depends libpq --depends libcap --depends openssl --depends logrotate +--depends bash --depends python3 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream --depends pcre --depends libpq --depends libcap --depends openssl --depends logrotate --depends mysql --depends postgresql --depends sqlite3 --description "BunkerWeb %VERSION% for Fedora 39" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " diff --git a/src/linux/fpm-rhel b/src/linux/fpm-rhel index b68d89474..ed37f996b 100644 --- a/src/linux/fpm-rhel +++ b/src/linux/fpm-rhel @@ -3,7 +3,7 @@ --license agpl3 --version %VERSION% --architecture %ARCH% ---depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends geoip --depends libpq --depends libcap --depends openssl +--depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends geoip --depends libpq --depends libcap --depends openssl --depends mysql --depends postgresql --depends sqlite --description "BunkerWeb %VERSION% for RHEL 8" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " diff --git a/src/linux/fpm-rhel9 b/src/linux/fpm-rhel9 index 0164b8818..ca00ecce1 100644 --- a/src/linux/fpm-rhel9 +++ b/src/linux/fpm-rhel9 @@ -3,7 +3,7 @@ --license agpl3 --version %VERSION% --architecture %ARCH% ---depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends libmaxminddb --depends libpq --depends libcap --depends openssl +--depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends libmaxminddb --depends libpq --depends libcap --depends openssl --depends mysql --depends postgresql --depends sqlite --description "BunkerWeb %VERSION% for RHEL 9" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " diff --git a/src/linux/fpm-ubuntu b/src/linux/fpm-ubuntu index 6f365f875..9c0a03a1d 100644 --- a/src/linux/fpm-ubuntu +++ b/src/linux/fpm-ubuntu @@ -3,7 +3,7 @@ --license agpl3 --version %VERSION% --architecture %ARCH% ---depends bash --depends python3 --depends python3-pip --depends 'nginx = 1.24.0-1~jammy' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends procps --depends lsof --depends libpq5 --depends libcap2-bin --depends logrotate +--depends bash --depends python3 --depends python3-pip --depends 'nginx = 1.24.0-1~jammy' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends procps --depends lsof --depends libpq5 --depends libcap2-bin --depends logrotate --depends mariadb-client --depends postgresql --depends sqlite3 --description "BunkerWeb %VERSION% for Ubuntu 22.04" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " diff --git a/src/scheduler/Dockerfile b/src/scheduler/Dockerfile index bbf43757a..4f2dcf4df 100644 --- a/src/scheduler/Dockerfile +++ b/src/scheduler/Dockerfile @@ -46,7 +46,7 @@ COPY --from=builder --chown=0:101 /usr/share/bunkerweb /usr/share/bunkerweb WORKDIR /usr/share/bunkerweb # Add scheduler user, drop bwcli, install runtime dependencies, create data folders and set permissions -RUN apk add --no-cache bash libgcc libstdc++ libpq openssl libmagic && \ +RUN apk add --no-cache bash libgcc libstdc++ libpq openssl libmagic mariadb-client postgresql sqlite && \ addgroup -g 101 scheduler && \ adduser -h /var/cache/nginx -g scheduler -s /bin/sh -G scheduler -D -H -u 101 scheduler && \ cp helpers/bwcli /usr/bin/ && \