mirror of
https://github.com/bunkerity/bunkerweb
synced 2026-05-24 09:28:37 +00:00
Refactor save_config.py and main.py to utilize dotenv for environment variable management and improve path handling
This commit is contained in:
Théophile Diot
parent
3117ab5a8b
commit
6094d779bd
3 changed files with 75 additions and 64 deletions
|
|
@ -14,16 +14,17 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
|
|||
if deps_path not in sys_path:
|
||||
sys_path.append(deps_path)
|
||||
|
||||
from dotenv import dotenv_values
|
||||
|
||||
from logger import setup_logger # type: ignore
|
||||
from Configurator import Configurator
|
||||
from Templator import Templator
|
||||
|
||||
DB_PATH = Path(sep, "usr", "share", "bunkerweb", "db")
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger = setup_logger("Generator", getenv("LOG_LEVEL", "INFO"))
|
||||
wait_retry_interval = int(getenv("WAIT_RETRY_INTERVAL", "5"))
|
||||
LOGGER = setup_logger("Generator", getenv("LOG_LEVEL", "INFO"))
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
# Parse arguments
|
||||
parser = ArgumentParser(description="BunkerWeb config generator")
|
||||
|
|
@ -42,27 +43,39 @@ if __name__ == "__main__":
|
|||
|
||||
settings_path = Path(args.settings)
|
||||
settings_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
templates_path = Path(args.templates)
|
||||
templates_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
core_path = Path(args.core)
|
||||
core_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
plugins_path = Path(args.plugins)
|
||||
plugins_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pro_plugins_path = Path(args.pro_plugins)
|
||||
pro_plugins_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
output_path = Path(args.output)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
target_path = Path(args.target)
|
||||
target_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info("Generator started ...")
|
||||
logger.info(f"Settings : {settings_path}")
|
||||
logger.info(f"Templates : {templates_path}")
|
||||
logger.info(f"Core : {core_path}")
|
||||
logger.info(f"Plugins : {plugins_path}")
|
||||
logger.info(f"Pro plugins : {pro_plugins_path}")
|
||||
logger.info(f"Output : {output_path}")
|
||||
logger.info(f"Target : {target_path}")
|
||||
LOGGER.info("Generator started ...")
|
||||
LOGGER.info(f"Settings : {settings_path}")
|
||||
LOGGER.info(f"Templates : {templates_path}")
|
||||
LOGGER.info(f"Core : {core_path}")
|
||||
LOGGER.info(f"Plugins : {plugins_path}")
|
||||
LOGGER.info(f"Pro plugins : {pro_plugins_path}")
|
||||
LOGGER.info(f"Output : {output_path}")
|
||||
LOGGER.info(f"Target : {target_path}")
|
||||
|
||||
dotenv_env = {}
|
||||
if args.variables:
|
||||
variables_path = Path(args.variables)
|
||||
LOGGER.info(f"Variables : {variables_path}")
|
||||
dotenv_env = dotenv_values(variables_path.as_posix())
|
||||
|
||||
db = None
|
||||
if DB_PATH.is_dir():
|
||||
|
|
@ -71,71 +84,73 @@ if __name__ == "__main__":
|
|||
|
||||
from Database import Database # type: ignore
|
||||
|
||||
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None))
|
||||
db = Database(LOGGER, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None)))
|
||||
|
||||
if args.variables:
|
||||
variables_path = Path(args.variables)
|
||||
variables_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Variables : {variables_path}")
|
||||
|
||||
# Check existences and permissions
|
||||
logger.info("Checking arguments ...")
|
||||
LOGGER.info("Checking arguments ...")
|
||||
files = [settings_path, variables_path]
|
||||
paths_rx = [core_path, plugins_path, pro_plugins_path, templates_path]
|
||||
paths_rwx = [output_path]
|
||||
for file in files:
|
||||
if not file.is_file():
|
||||
logger.error(f"Missing file : {file}")
|
||||
LOGGER.error(f"Missing file : {file}")
|
||||
sys_exit(1)
|
||||
elif not access(file, R_OK):
|
||||
logger.error(f"Can't read file : {file}")
|
||||
LOGGER.error(f"Can't read file : {file}")
|
||||
sys_exit(1)
|
||||
for path in paths_rx + paths_rwx:
|
||||
if not path.is_dir():
|
||||
logger.error(f"Missing directory : {path}")
|
||||
LOGGER.error(f"Missing directory : {path}")
|
||||
sys_exit(1)
|
||||
elif not access(path, R_OK | X_OK):
|
||||
logger.error(
|
||||
f"Missing RX rights on directory : {path}",
|
||||
)
|
||||
LOGGER.error(f"Missing RX rights on directory : {path}")
|
||||
sys_exit(1)
|
||||
for path in paths_rwx:
|
||||
if not access(path, W_OK):
|
||||
logger.error(
|
||||
f"Missing W rights on directory : {path}",
|
||||
)
|
||||
LOGGER.error(f"Missing W rights on directory : {path}")
|
||||
sys_exit(1)
|
||||
|
||||
# Compute the config
|
||||
logger.info("Computing config ...")
|
||||
LOGGER.info("Computing config ...")
|
||||
config: Dict[str, Any] = Configurator(
|
||||
str(settings_path), str(core_path), str(plugins_path), str(pro_plugins_path), str(variables_path), logger
|
||||
settings_path.as_posix(),
|
||||
core_path.as_posix(),
|
||||
plugins_path.as_posix(),
|
||||
pro_plugins_path.as_posix(),
|
||||
variables_path.as_posix(),
|
||||
LOGGER,
|
||||
).get_config(db)
|
||||
else:
|
||||
config: Dict[str, Any] = db.get_config()
|
||||
|
||||
# Remove old files
|
||||
logger.info("Removing old files ...")
|
||||
LOGGER.info("Removing old files ...")
|
||||
files = glob(join(args.output, "*"))
|
||||
for file in files:
|
||||
file = Path(file)
|
||||
if file.is_symlink() or file.is_file():
|
||||
file.unlink()
|
||||
elif file.is_dir():
|
||||
rmtree(str(file), ignore_errors=True)
|
||||
rmtree(file.as_posix(), ignore_errors=True)
|
||||
|
||||
# Render the templates
|
||||
logger.info("Rendering templates ...")
|
||||
templator = Templator(str(templates_path), str(core_path), str(plugins_path), str(pro_plugins_path), str(output_path), str(target_path), config)
|
||||
LOGGER.info("Rendering templates ...")
|
||||
templator = Templator(
|
||||
templates_path.as_posix(),
|
||||
core_path.as_posix(),
|
||||
plugins_path.as_posix(),
|
||||
pro_plugins_path.as_posix(),
|
||||
output_path.as_posix(),
|
||||
target_path.as_posix(),
|
||||
config,
|
||||
)
|
||||
templator.render()
|
||||
|
||||
except SystemExit as e:
|
||||
raise e
|
||||
except:
|
||||
logger.error(
|
||||
f"Exception while executing generator : {format_exc()}",
|
||||
)
|
||||
LOGGER.error(f"Exception while executing generator : {format_exc()}")
|
||||
sys_exit(1)
|
||||
|
||||
# We're done
|
||||
logger.info("Generator successfully executed !")
|
||||
LOGGER.info("Generator successfully executed !")
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
|
|||
if deps_path not in sys_path:
|
||||
sys_path.append(deps_path)
|
||||
|
||||
from dotenv import dotenv_values
|
||||
|
||||
from common_utils import get_integration, get_version # type: ignore
|
||||
from logger import setup_logger # type: ignore
|
||||
from Database import Database # type: ignore
|
||||
|
|
@ -27,14 +29,6 @@ LOGGER = setup_logger("Generator.save_config", getenv("CUSTOM_LOG_LEVEL", getenv
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
wait_retry_interval = getenv("WAIT_RETRY_INTERVAL", "5")
|
||||
|
||||
if not wait_retry_interval.isdigit():
|
||||
LOGGER.error("Invalid WAIT_RETRY_INTERVAL value, must be an integer")
|
||||
sys_exit(1)
|
||||
|
||||
wait_retry_interval = int(wait_retry_interval)
|
||||
|
||||
try:
|
||||
# Parse arguments
|
||||
parser = ArgumentParser(description="BunkerWeb config saver")
|
||||
|
|
@ -69,9 +63,15 @@ if __name__ == "__main__":
|
|||
external_plugins = args.plugins
|
||||
pro_plugins = args.pro_plugins
|
||||
|
||||
dotenv_env = {}
|
||||
if args.variables:
|
||||
variables_path = Path(args.variables)
|
||||
LOGGER.info(f"Variables : {variables_path}")
|
||||
dotenv_env = dotenv_values(variables_path.as_posix())
|
||||
|
||||
# Check existences and permissions
|
||||
LOGGER.info("Checking arguments ...")
|
||||
files = [settings_path] + ([Path(args.variables)] if args.variables else [])
|
||||
files = [settings_path] + ([variables_path] if args.variables else [])
|
||||
paths_rx = [core_path, plugins_path, pro_plugins_path]
|
||||
for file in files:
|
||||
if not file.is_file():
|
||||
|
|
@ -88,16 +88,15 @@ if __name__ == "__main__":
|
|||
LOGGER.error(f"Missing RX rights on directory : {path}")
|
||||
sys_exit(1)
|
||||
|
||||
tmp_config = {}
|
||||
|
||||
if args.variables:
|
||||
variables_path = Path(args.variables)
|
||||
LOGGER.info(f"Variables : {variables_path}")
|
||||
|
||||
# Compute the config
|
||||
LOGGER.info("Computing config ...")
|
||||
config = Configurator(
|
||||
str(settings_path), str(core_path), external_plugins, pro_plugins, str(variables_path) if args.variables else environ.copy(), LOGGER
|
||||
settings_path.as_posix(),
|
||||
core_path.as_posix(),
|
||||
external_plugins,
|
||||
pro_plugins,
|
||||
variables_path.as_posix() if args.variables else environ.copy(),
|
||||
LOGGER,
|
||||
)
|
||||
|
||||
custom_confs = []
|
||||
|
|
@ -119,7 +118,7 @@ if __name__ == "__main__":
|
|||
)
|
||||
continue
|
||||
|
||||
db = Database(LOGGER)
|
||||
db = Database(LOGGER, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None)))
|
||||
|
||||
bunkerweb_version = get_version()
|
||||
db_metadata = db.get_metadata()
|
||||
|
|
|
|||
|
|
@ -291,14 +291,11 @@ def generate_external_plugins(original_path: Union[Path, str] = EXTERNAL_PLUGINS
|
|||
|
||||
try:
|
||||
if plugin["data"]:
|
||||
tmp_path = TMP_PATH.joinpath(f"{plugin['id']}_{plugin['name']}.tar.gz")
|
||||
tmp_path.write_bytes(plugin["data"])
|
||||
with tar_open(str(tmp_path), "r:gz") as tar:
|
||||
with tar_open(fileobj=BytesIO(plugin["data"]), mode="r:gz") as tar:
|
||||
try:
|
||||
tar.extractall(original_path, filter="fully_trusted")
|
||||
except TypeError:
|
||||
tar.extractall(original_path)
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
for job_file in chain(original_path.joinpath(plugin["id"], "jobs").glob("*"), original_path.joinpath(plugin["id"], "bwcli").glob("*")):
|
||||
job_file.chmod(job_file.stat().st_mode | S_IEXEC)
|
||||
|
|
@ -424,7 +421,7 @@ def run_in_slave_mode(): # TODO: Refactor this feature
|
|||
"--output",
|
||||
CONFIG_PATH.as_posix(),
|
||||
"--variables",
|
||||
str(SCHEDULER_TMP_ENV_PATH),
|
||||
SCHEDULER_TMP_ENV_PATH.as_posix(),
|
||||
],
|
||||
stdin=DEVNULL,
|
||||
stderr=STDOUT,
|
||||
|
|
@ -550,7 +547,7 @@ if __name__ == "__main__":
|
|||
|
||||
tmp_variables_path = Path(args.variables or join(sep, "var", "tmp", "bunkerweb", "variables.env"))
|
||||
nginx_variables_path = CONFIG_PATH.joinpath("variables.env")
|
||||
dotenv_env = dotenv_values(str(tmp_variables_path))
|
||||
dotenv_env = dotenv_values(tmp_variables_path.as_posix())
|
||||
|
||||
SCHEDULER = JobScheduler(environ, LOGGER, db=Database(LOGGER, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None)))) # type: ignore
|
||||
|
||||
|
|
@ -576,7 +573,7 @@ if __name__ == "__main__":
|
|||
join(sep, "usr", "share", "bunkerweb", "settings.json"),
|
||||
"--first-run",
|
||||
]
|
||||
+ (["--variables", str(tmp_variables_path)] if args.variables else []),
|
||||
+ (["--variables", tmp_variables_path.as_posix()] if args.variables else []),
|
||||
stdin=DEVNULL,
|
||||
stderr=STDOUT,
|
||||
check=False,
|
||||
|
|
@ -757,7 +754,7 @@ if __name__ == "__main__":
|
|||
"--settings",
|
||||
join(sep, "usr", "share", "bunkerweb", "settings.json"),
|
||||
]
|
||||
+ (["--variables", str(tmp_variables_path)] if args.variables else []),
|
||||
+ (["--variables", tmp_variables_path.as_posix()] if args.variables else []),
|
||||
stdin=DEVNULL,
|
||||
stderr=STDOUT,
|
||||
check=False,
|
||||
|
|
@ -820,7 +817,7 @@ if __name__ == "__main__":
|
|||
"--output",
|
||||
CONFIG_PATH.as_posix(),
|
||||
"--variables",
|
||||
str(SCHEDULER_TMP_ENV_PATH),
|
||||
SCHEDULER_TMP_ENV_PATH.as_posix(),
|
||||
]
|
||||
+ (["--no-linux-reload"] if MASTER_MODE else []),
|
||||
stdin=DEVNULL,
|
||||
|
|
@ -831,7 +828,7 @@ if __name__ == "__main__":
|
|||
if proc.returncode != 0:
|
||||
LOGGER.error("Config generator failed, configuration will not work as expected...")
|
||||
else:
|
||||
copy(str(nginx_variables_path), join(sep, "var", "tmp", "bunkerweb", "variables.env"))
|
||||
copy(nginx_variables_path.as_posix(), join(sep, "var", "tmp", "bunkerweb", "variables.env"))
|
||||
|
||||
if SCHEDULER.apis:
|
||||
# send nginx configs
|
||||
|
|
|
|||
Loading…
Reference in a new issue