Add new backup plugin and update required dependencies accordingly

src/common/core/backup/jobs/backup-data.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+
+from datetime import datetime, timedelta
+from os import environ, getenv, sep
+from os.path import join
+from pathlib import Path
+from subprocess import PIPE, run
+from sys import exit as sys_exit, path as sys_path
+from threading import Lock
+from traceback import format_exc
+from typing import Literal
+from zipfile import ZIP_DEFLATED, ZipFile
+
+for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
+    if deps_path not in sys_path:
+        sys_path.append(deps_path)
+
+from logger import setup_logger  # type: ignore
+from jobs import Job  # type: ignore
+
+
+LOGGER = setup_logger("BACKUP", getenv("LOG_LEVEL", "INFO"))
+status = 0
+
+try:
+    # Check if backup is activated
+    if getenv("USE_BACKUP", "yes") == "no":
+        LOGGER.info("Backup feature is disabled, skipping backup ...")
+        sys_exit(0)
+
+    backup_dir = Path(getenv("BACKUP_DIRECTORY", "/var/lib/bunkerweb/backups"))
+
+    JOB = Job(LOGGER)
+
+    last_backup = JOB.get_cache("last_backup.txt")
+    if last_backup:
+        last_backup = datetime.fromisoformat(last_backup.decode())
+
+    current_time = datetime.now()
+    backup_period = getenv("BACKUP_SCHEDULE", "daily")
+    PERIOD_STAMPS = {
+        "daily": timedelta(days=1).total_seconds(),
+        "weekly": timedelta(weeks=1).total_seconds(),
+        "monthly": timedelta(weeks=4).total_seconds(),
+    }
+
+    if last_backup and last_backup.timestamp() + PERIOD_STAMPS[backup_period] > current_time.timestamp():
+        LOGGER.info(f"Backup already done within the last {backup_period} period, skipping backup ...")
+        sys_exit(0)
+
+    with Lock():
+        is_scheduler_first_start = JOB.db.is_scheduler_first_start()
+
+    if is_scheduler_first_start:
+        LOGGER.info("First start of the scheduler, skipping backup ...")
+        sys_exit(0)
+
+    database: Literal["sqlite", "mariadb", "mysql", "postgresql"] = JOB.db.database_uri.split(":")[0].split("+")[0]
+    backup_file = backup_dir.joinpath(f"backup-{database}-{current_time.strftime('%Y-%m-%d')}.zip")
+    backup_file.parent.mkdir(parents=True, exist_ok=True)
+    LOGGER.debug(f"Backup file path: {backup_file}")
+
+    if database == "sqlite":
+        match = JOB.db.DB_STRING_RX.search(JOB.db.database_uri)
+        if not match:
+            LOGGER.error(f"Invalid database string provided: {JOB.db.database_uri}, skipping backup ...")
+            sys_exit(1)
+
+        db_path = Path(match.group("path"))
+
+        LOGGER.info("Creating a backup for the SQLite database ...")
+
+        proc = run(["sqlite3", db_path.as_posix(), ".dump"], stdout=PIPE, stderr=PIPE)
+    else:
+        db_host = JOB.db.database_uri.rsplit("@", 1)[1].split("/")[0].split(":")
+        db_port = None
+        if len(db_host) == 1:
+            db_host = db_host[0]
+        else:
+            db_host, db_port = db_host
+
+        db_user = JOB.db.database_uri.split("://")[1].split(":")[0]
+        db_password = JOB.db.database_uri.split("://")[1].split(":")[1].rsplit("@", 1)[0]
+        db_database_name = JOB.db.database_uri.split("/")[-1]
+
+        if database in ("mariadb", "mysql"):
+            LOGGER.info("Creating a backup for the MariaDB/MySQL database ...")
+
+            proc = run(["mysqldump", "-h", db_host, "-u", db_user, db_database_name], stdout=PIPE, stderr=PIPE, env=environ | {"MYSQL_PWD": db_password})
+        elif database == "postgresql":
+            LOGGER.info("Creating a backup for the PostgreSQL database ...")
+
+            proc = run(
+                ["pg_dump", "-h", db_host, "-U", db_user, "-d", db_database_name, "-w"], stdout=PIPE, stderr=PIPE, env=environ | {"PGPASSWORD": db_password}
+            )
+
+    if proc.returncode != 0:
+        LOGGER.error(f"Failed to dump the database: {proc.stderr.decode()}")
+        sys_exit(1)
+
+    with ZipFile(backup_file, "w", compression=ZIP_DEFLATED) as zipf:
+        zipf.writestr(backup_file.with_suffix(".sql").name, proc.stdout)
+
+    backup_rotation = int(getenv("BACKUP_ROTATION", "7"))
+
+    # Get all backup files in the directory
+    backup_files = backup_dir.glob("backup-*.zip")
+
+    # Sort the backup files by name
+    sorted_files = sorted(backup_files)
+
+    # Check if the number of backup files exceeds the rotation limit
+    if len(sorted_files) > backup_rotation:
+        # Calculate the number of files to remove
+        num_files_to_remove = len(sorted_files) - backup_rotation
+
+        # Remove the oldest backup files
+        for file in sorted_files[:num_files_to_remove]:
+            LOGGER.warning(f"Removing old backup file: {file}, as the rotation limit has been reached ...")
+            file.unlink()
+
+    cached, err = JOB.cache_file("last_backup.txt", current_time.isoformat().encode())
+    if not cached:
+        LOGGER.error(f"Failed to cache last_backup.txt :\n{err}")
+        status = 2
+except SystemExit as e:
+    status = e.code
+except:
+    status = 2
+    LOGGER.error(f"Exception while running backup-data.py :\n{format_exc()}")
+
+sys_exit(status)

src/common/core/backup/plugin.json

@@ -0,0 +1,54 @@
+{
+  "id": "backup",
+  "name": "Backup",
+  "description": "Backup your data to a custom location. Ensure the safety and availability of your important files by creating regular backups.",
+  "version": "1.0",
+  "stream": "no",
+  "settings": {
+    "USE_BACKUP": {
+      "context": "global",
+      "default": "yes",
+      "help": "Enable or disable the backup feature",
+      "id": "use-backup",
+      "label": "Activate automatic backup",
+      "regex": "^(yes|no)$",
+      "type": "check"
+    },
+    "BACKUP_DIRECTORY": {
+      "context": "global",
+      "default": "/var/lib/bunkerweb/backups",
+      "help": "The directory where the backup will be stored",
+      "id": "backup-directory",
+      "label": "Backup directory",
+      "regex": "^.*$",
+      "type": "text"
+    },
+    "BACKUP_SCHEDULE": {
+      "context": "global",
+      "default": "daily",
+      "help": "The frequency of the backup",
+      "id": "backup-schedule",
+      "label": "Backup schedule",
+      "regex": "^(daily|weekly|monthly)$",
+      "type": "select",
+      "select": ["daily", "weekly", "monthly"]
+    },
+    "BACKUP_ROTATION": {
+      "context": "global",
+      "default": "7",
+      "help": "The number of backups to keep",
+      "id": "backup-rotation",
+      "label": "Backup rotation",
+      "regex": "^[1-9][0-9]*$",
+      "type": "text"
+    }
+  },
+  "jobs": [
+    {
+      "name": "backup-data",
+      "file": "backup-data.py",
+      "every": "day",
+      "reload": false
+    }
+  ]
+}

src/linux/fpm-centos

@@ -3,7 +3,7 @@
 --license agpl3
 --version %VERSION%
 --architecture x86_64
---depends bash --depends epel-release --depends python39 --depends 'nginx = 1:1.24.0-1.el8.ngx' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends GeoIP-devel --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends brotli --depends openssl --depends libpq
+--depends bash --depends epel-release --depends python39 --depends 'nginx = 1:1.24.0-1.el8.ngx' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends GeoIP-devel --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends brotli --depends openssl --depends libpq --depends mysql --depends postgresql --depends sqlite
 --description "BunkerWeb %VERSION% for CentOS Stream 8"
 --url "https://www.bunkerweb.io"
 --maintainer "Bunkerity <contact at bunkerity dot com>"

src/linux/fpm-debian

@@ -3,7 +3,7 @@
 --license agpl3
 --version %VERSION%
 --architecture %ARCH%
---depends bash --depends python3 --depends procps --depends python3-pip --depends 'nginx = 1.24.0-1~bookworm' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends lsof --depends libpq5 --depends libpcre3 --depends libcap2-bin --depends logrotate
+--depends bash --depends python3 --depends procps --depends python3-pip --depends 'nginx = 1.24.0-1~bookworm' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends lsof --depends libpq5 --depends libpcre3 --depends libcap2-bin --depends logrotate --depends mariadb-client --depends postgresql --depends sqlite3
 --description "BunkerWeb %VERSION% for Debian 12"
 --url "https://www.bunkerweb.io"
 --maintainer "Bunkerity <contact at bunkerity dot com>"

src/linux/fpm-fedora

@@ -3,7 +3,7 @@
 --license agpl3
 --version %VERSION%
 --architecture %ARCH%
---depends bash --depends python3 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream --depends pcre --depends libpq --depends libcap --depends openssl --depends logrotate
+--depends bash --depends python3 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream --depends pcre --depends libpq --depends libcap --depends openssl --depends logrotate --depends mysql --depends postgresql --depends sqlite3
 --description "BunkerWeb %VERSION% for Fedora 39"
 --url "https://www.bunkerweb.io"
 --maintainer "Bunkerity <contact at bunkerity dot com>"

src/linux/fpm-rhel

@@ -3,7 +3,7 @@
 --license agpl3
 --version %VERSION%
 --architecture %ARCH%
---depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends geoip --depends libpq --depends libcap --depends openssl
+--depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends geoip --depends libpq --depends libcap --depends openssl --depends mysql --depends postgresql --depends sqlite
 --description "BunkerWeb %VERSION% for RHEL 8"
 --url "https://www.bunkerweb.io"
 --maintainer "Bunkerity <contact at bunkerity dot com>"

src/linux/fpm-rhel9

@@ -3,7 +3,7 @@
 --license agpl3
 --version %VERSION%
 --architecture %ARCH%
---depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends libmaxminddb --depends libpq --depends libcap --depends openssl
+--depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends libmaxminddb --depends libpq --depends libcap --depends openssl --depends mysql --depends postgresql --depends sqlite
 --description "BunkerWeb %VERSION% for RHEL 9"
 --url "https://www.bunkerweb.io"
 --maintainer "Bunkerity <contact at bunkerity dot com>"

src/linux/fpm-ubuntu

@@ -3,7 +3,7 @@
 --license agpl3
 --version %VERSION%
 --architecture %ARCH%
---depends bash --depends python3 --depends python3-pip --depends 'nginx = 1.24.0-1~jammy' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends procps --depends lsof --depends libpq5 --depends libcap2-bin --depends logrotate
+--depends bash --depends python3 --depends python3-pip --depends 'nginx = 1.24.0-1~jammy' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends procps --depends lsof --depends libpq5 --depends libcap2-bin --depends logrotate --depends mariadb-client --depends postgresql --depends sqlite3
 --description "BunkerWeb %VERSION% for Ubuntu 22.04"
 --url "https://www.bunkerweb.io"
 --maintainer "Bunkerity <contact at bunkerity dot com>"

src/scheduler/Dockerfile

@@ -46,7 +46,7 @@ COPY --from=builder --chown=0:101 /usr/share/bunkerweb /usr/share/bunkerweb
 WORKDIR /usr/share/bunkerweb
 
 # Add scheduler user, drop bwcli, install runtime dependencies, create data folders and set permissions
-RUN apk add --no-cache bash libgcc libstdc++ libpq openssl libmagic && \
+RUN apk add --no-cache bash libgcc libstdc++ libpq openssl libmagic mariadb-client postgresql sqlite && \
   addgroup -g 101 scheduler && \
   adduser -h /var/cache/nginx -g scheduler -s /bin/sh -G scheduler -D -H -u 101 scheduler && \
   cp helpers/bwcli /usr/bin/ && \