Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enhance backup functionality with forced backup option and version change handling

Browse source
This commit is contained in:
Théophile Diot 2024-11-05 10:10:46 +01:00
parent c03c1b5406
commit 7c7a67ab65
No known key found for this signature in database
GPG key ID: FA995104A0BA376A
3 changed files with 95 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

117
src/common/core/backup/jobs/backup-data.py
View file

@ -11,6 +11,7 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import Job # type: ignore
from utils import backup_database, update_cache_file
@ -22,68 +23,78 @@ try:
backup_dir = Path(getenv("BACKUP_DIRECTORY", "/var/lib/bunkerweb/backups"))
backup_dir.mkdir(parents=True, exist_ok=True)
# Check if backup is activated
if getenv("USE_BACKUP", "yes") == "no":
LOGGER.info("Backup feature is disabled, skipping backup ...")
sys_exit(0)
JOB = Job(LOGGER)
last_backup = loads(JOB.get_cache("backup.json") or "{}")
last_backup_date = last_backup.get("date", None)
if last_backup_date:
last_backup_date = datetime.fromisoformat(last_backup_date).astimezone()
force_backup = getenv("FORCE_BACKUP", "no") == "yes"
current_time = datetime.now().astimezone()
backup_period = getenv("BACKUP_SCHEDULE", "daily")
PERIOD_STAMPS = {
"daily": timedelta(days=1).total_seconds(),
"weekly": timedelta(weeks=1).total_seconds(),
"monthly": timedelta(weeks=4).total_seconds(),
}
already_done = last_backup_date and last_backup_date.timestamp() + PERIOD_STAMPS[backup_period] > current_time.timestamp()
backup_rotation = int(getenv("BACKUP_ROTATION", "7"))
sorted_files = []
if already_done:
# Get all backup files in the directory
backup_files = backup_dir.glob("backup-*.zip")
# Sort the backup files by name
sorted_files = sorted(backup_files)
if len(sorted_files) <= backup_rotation and already_done:
LOGGER.info(f"Backup already done within the last {backup_period} period, skipping backup ...")
sys_exit(0)
if not already_done:
db_metadata = JOB.db.get_metadata()
if isinstance(db_metadata, str) or db_metadata["scheduler_first_start"]:
LOGGER.info("First start of the scheduler, skipping backup ...")
if not force_backup:
# Check if backup is activated
if getenv("USE_BACKUP", "yes") == "no":
LOGGER.info("Backup feature is disabled, skipping backup ...")
sys_exit(0)
backup_database(current_time, JOB.db, backup_dir)
JOB = Job(LOGGER)
# Get all backup files in the directory
backup_files = backup_dir.glob("backup-*.zip")
last_backup = loads(JOB.get_cache("backup.json") or "{}")
last_backup_date = last_backup.get("date", None)
if last_backup_date:
last_backup_date = datetime.fromisoformat(last_backup_date).astimezone()
# Sort the backup files by name
sorted_files = sorted(backup_files)
backup_period = getenv("BACKUP_SCHEDULE", "daily")
PERIOD_STAMPS = {
"daily": timedelta(days=1).total_seconds(),
"weekly": timedelta(weeks=1).total_seconds(),
"monthly": timedelta(weeks=4).total_seconds(),
}
# Check if the number of backup files exceeds the rotation limit
if len(sorted_files) > backup_rotation:
# Calculate the number of files to remove
num_files_to_remove = len(sorted_files) - backup_rotation
already_done = last_backup_date and last_backup_date.timestamp() + PERIOD_STAMPS[backup_period] > current_time.timestamp()
backup_rotation = int(getenv("BACKUP_ROTATION", "7"))
# Remove the oldest backup files
for file in sorted_files[:num_files_to_remove]:
LOGGER.warning(f"Removing old backup file: {file}, as the rotation limit has been reached ...")
file.unlink()
sorted_files = []
if already_done:
update_cache_file(JOB.db, backup_dir)
# Get all backup files in the directory
backup_files = backup_dir.glob("backup-*.zip")
# Sort the backup files by name
sorted_files = sorted(backup_files)
if len(sorted_files) <= backup_rotation and already_done:
LOGGER.info(f"Backup already done within the last {backup_period} period, skipping backup ...")
sys_exit(0)
db = JOB.db
else:
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"))
if force_backup or not already_done:
if not force_backup:
db_metadata = db.get_metadata()
if isinstance(db_metadata, str) or db_metadata["scheduler_first_start"]:
LOGGER.info("First start of the scheduler, skipping backup ...")
sys_exit(0)
backup_database(current_time, db, backup_dir)
if not force_backup:
# Get all backup files in the directory
backup_files = backup_dir.glob("backup-*.zip")
# Sort the backup files by name
sorted_files = sorted(backup_files)
if not force_backup:
# Check if the number of backup files exceeds the rotation limit
if len(sorted_files) > backup_rotation:
# Calculate the number of files to remove
num_files_to_remove = len(sorted_files) - backup_rotation
# Remove the oldest backup files
for file in sorted_files[:num_files_to_remove]:
LOGGER.warning(f"Removing old backup file: {file}, as the rotation limit has been reached ...")
file.unlink()
update_cache_file(db, backup_dir)
except SystemExit as e:
status = e.code
except BaseException as e:

22
src/common/db/Database.py
View file

@ -382,6 +382,17 @@ class Database:
return ""
def get_version(self) -> str:
"""Get the database version"""
with self._db_session() as session:
try:
metadata = session.query(Metadata).with_entities(Metadata.version).filter_by(id=1).first()
if metadata:
return metadata.version
return "1.6.0-beta"
except BaseException as e:
return f"Error: {e}"
def get_metadata(self) -> Dict[str, Any]:
"""Get the metadata from the database"""
data = {
@ -558,17 +569,6 @@ class Database:
with self._db_session() as session:
old_data[table_name] = session.query(metadata.tables[table_name]).all()
# ? Rename the old tables to keep the data in case of rollback
db_version_id = db_version.replace(".", "_")
for table_name in metadata.tables.keys():
if table_name in Base.metadata.tables:
with self._db_session() as session:
if inspector.has_table(f"{table_name}_{db_version_id}"):
self.logger.warning(f'Table "{table_name}" already exists, dropping it to make room for the new one')
session.execute(text(f"DROP TABLE {table_name}_{db_version_id}"))
session.execute(text(f"ALTER TABLE {table_name} RENAME TO {table_name}_{db_version_id}"))
session.commit()
Base.metadata.drop_all(self.sql_engine)
if has_all_tables and db_version and db_version == bunkerweb_version:

23
src/scheduler/main.py
View file

@ -27,7 +27,7 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from dotenv import dotenv_values
from schedule import every as schedule_every, run_pending
from common_utils import bytes_hash, dict_to_frozenset # type: ignore
from common_utils import bytes_hash, dict_to_frozenset, get_version # type: ignore
from logger import setup_logger # type: ignore
from Database import Database # type: ignore
from JobScheduler import JobScheduler
@ -557,10 +557,27 @@ if __name__ == "__main__":
run_in_slave_mode()
stop(1)
db_metadata = SCHEDULER.db.get_metadata()
APPLYING_CHANGES.set()
db_version = SCHEDULER.db.get_version()
if not db_version.startswith("Error") and db_version != get_version():
LOGGER.warning("BunkerWeb version changed, creating a backup of the database and proceeding with the upgrade ...")
SCHEDULER.env = {
"DATABASE_URI": SCHEDULER.db.database_uri,
"USE_BACKUP": "yes",
"FORCE_BACKUP": "yes",
"BACKUP_SCHEDULE": "daily",
"BACKUP_ROTATION": "7",
"BACKUP_DIRECTORY": "/var/lib/bunkerweb/upgrade_backups",
"LOG_LEVEL": getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "notice")),
"RELOAD_MIN_TIMEOUT": str(RELOAD_MIN_TIMEOUT),
}
if not SCHEDULER.run_single("backup-data"):
LOGGER.error("backup-data job failed, stopping ...")
stop(1)
LOGGER.info("Backup completed successfully, if you want to restore the backup, you can find it in /var/lib/bunkerweb/upgrade_backups")
if SCHEDULER.db.readonly:
LOGGER.warning("The database is read-only, no need to save the changes in the configuration as they will not be saved")
else: