Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #355 from TheophileDiot/1.5

Browse source 
Fix database with autoconf
This commit is contained in:
Théophile Diot 2022-11-15 17:04:14 +01:00 committed by GitHub
commit 7db7aee7c2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 292 additions and 264 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/autoconf/Config.py
View file

@ -18,11 +18,6 @@ class Config(ConfigCaller):
self.__config = {} self.__config = {}
self._db = Database(self.__logger) self._db = Database(self.__logger)
while not self._db.is_initialized():
self.__logger.warning(
"Database is not initialized, retrying in 5 seconds ...",
)
sleep(5)
def __get_full_env(self) -> dict: def __get_full_env(self) -> dict:
env_instances = {} env_instances = {}
@ -76,6 +71,12 @@ class Config(ConfigCaller):
} }
) )
while not self._db.is_initialized():
self.__logger.warning(
"Database is not initialized, retrying in 5 seconds ...",
)
sleep(5)
# save config to database # save config to database
ret = self._db.save_config(self.__config, "autoconf") ret = self._db.save_config(self.__config, "autoconf")
if ret: if ret:

2
src/autoconf/IngressController.py
View file

@ -274,7 +274,7 @@ class IngressController(Controller, ConfigCaller):
self.__logger.error( self.__logger.error(
f"Unknown exception while reading k8s event (type = {watch_type}) :\n{format_exc()}", f"Unknown exception while reading k8s event (type = {watch_type}) :\n{format_exc()}",
) )
finally : finally:
if locked: if locked:
self.__internal_lock.release() self.__internal_lock.release()
locked = False locked = False

539
src/common/db/Database.py
View file

@ -244,7 +244,7 @@ class Database:
"order": 999, "order": 999,
"name": "Default", "name": "Default",
"description": "Default settings", "description": "Default settings",
"version": "1.0.0", "version": "0.1",
} }
else: else:
settings = plugin.pop("settings", {}) settings = plugin.pop("settings", {})
@ -318,134 +318,156 @@ class Database:
).delete() ).delete()
if config: if config:
if config["MULTISITE"] == "yes": if config.get("MULTISITE", "no") == "yes":
global_values = [] global_values = []
for server_name in config["SERVER_NAME"].split(" "): db_services = (
if ( session.query(Services)
server_name .with_entities(Services.id)
and session.query(Services) .filter_by(method=method)
.with_entities(Services.id) .all()
.filter_by(id=server_name) )
services = config.get("SERVER_NAME", "").split(" ")
if db_services:
db_services = [service.id for service in db_services]
missing_ids = [
service
for service in db_services
if service not in services
]
# Remove plugins that are no longer in the list
session.query(Services).filter(
Services.id.in_(missing_ids)
).delete()
for key, value in deepcopy(config).items():
suffix = 0
if search(r"_\d+$", key):
suffix = int(key.split("_")[-1])
key = key[: -len(str(suffix)) - 1]
setting = (
session.query(Settings)
.with_entities(Settings.default)
.filter_by(id=key)
.first() .first()
is None )
):
to_put.append(Services(id=server_name))
for key, value in deepcopy(config).items(): if not setting and services:
suffix = 0 try:
if search(r"_\d+$", key): server_name = next(
suffix = int(key.split("_")[-1]) service
key = key[: -len(str(suffix)) - 1] for service in services
if key.startswith(f"{service}_")
)
except StopIteration:
continue
if server_name not in db_services:
to_put.append(Services(id=server_name, method=method))
db_services.append(server_name)
key = key.replace(f"{server_name}_", "")
setting = ( setting = (
session.query(Settings) session.query(Settings)
.with_entities(Settings.default) .with_entities(Settings.default)
.filter_by(id=key.replace(f"{server_name}_", "")) .filter_by(id=key)
.first()
)
service_setting = (
session.query(Services_settings)
.with_entities(Services_settings.value)
.filter_by(
service_id=server_name,
setting_id=key,
suffix=suffix,
)
.first() .first()
) )
if not setting: if service_setting is None:
continue if key != "SERVER_NAME" and (
value == setting.default
or (value == "" and setting.default is None)
or (key in config and value == config[key])
):
continue
if server_name and key.startswith(server_name): to_put.append(
key = key.replace(f"{server_name}_", "") Services_settings(
service_setting = (
session.query(Services_settings)
.with_entities(Services_settings.value)
.filter_by(
service_id=server_name, service_id=server_name,
setting_id=key, setting_id=key,
value=value,
suffix=suffix, suffix=suffix,
method=method,
) )
.first()
) )
elif (
method == "autoconf"
and value != setting.default
and service_setting.value != value
):
session.query(Services_settings).filter(
Services_settings.service_id == server_name,
Services_settings.setting_id == key,
Services_settings.suffix == suffix,
).update(
{
Services_settings.value: value,
Services_settings.method: method,
}
)
elif key not in global_values:
global_values.append(key)
global_value = (
session.query(Global_values)
.with_entities(Global_values.value)
.filter_by(
setting_id=key,
suffix=suffix,
)
.first()
)
if service_setting is None: if global_value is None:
if key != "SERVER_NAME" and ( if (
value == setting.default not setting
or (value == "" and setting.default is None) or value == setting.default
or (key in config and value == config[key]) or (value == "" and setting.default is None)
): ):
continue continue
to_put.append( to_put.append(
Services_settings( Global_values(
service_id=server_name,
setting_id=key,
value=value,
suffix=suffix,
method=method,
)
)
elif method == "autoconf":
if key != "SERVER_NAME" and (
value == setting.default
or (value == "" and setting.default is None)
or (key in config and value == config[key])
):
session.query(Services_settings).filter(
Services_settings.service_id == server_name,
Services_settings.setting_id == key,
Services_settings.suffix == suffix,
).delete()
elif global_value.value != value:
session.query(Services_settings).filter(
Services_settings.service_id == server_name,
Services_settings.setting_id == key,
Services_settings.suffix == suffix,
).update(
{
Services_settings.value: value,
Services_settings.method: method,
}
)
elif key not in global_values:
global_values.append(key)
global_value = (
session.query(Global_values)
.with_entities(Global_values.value)
.filter_by(
setting_id=key, setting_id=key,
value=value,
suffix=suffix, suffix=suffix,
method=method,
) )
.first()
) )
elif (
if global_value is None: setting
if value == setting.default or ( and method == "autoconf"
value == "" and setting.default is None and value != setting.default
): and global_value.value != value
continue ):
session.query(Global_values).filter(
to_put.append( Global_values.setting_id == key,
Global_values( Global_values.suffix == suffix,
setting_id=key, ).update(
value=value, {
suffix=suffix, Global_values.value: value,
method=method, Global_values.method: method,
) }
) )
elif method == "autoconf":
if value == setting.default or (
value == "" and setting.default is None
):
session.query(Global_values).filter(
Global_values.setting_id == key,
Global_values.suffix == suffix,
).delete()
elif global_value.value != value:
session.query(Global_values).filter(
Global_values.setting_id == key,
Global_values.suffix == suffix,
).update(
{
Global_values.value: value,
Global_values.method: method,
}
)
else: else:
primary_server_name = config["SERVER_NAME"].split(" ")[0] if "SERVER_NAME" in config and config["SERVER_NAME"] != "":
to_put.append(Services(id=primary_server_name)) to_put.append(
Services(
id=config["SERVER_NAME"].split(" ")[0], method=method
)
)
for key, value in config.items(): for key, value in config.items():
suffix = 0 suffix = 0
@ -461,8 +483,8 @@ class Database:
) )
if ( if (
setting not setting
and value == setting.default or value == setting.default
or (value == "" and setting.default is None) or (value == "" and setting.default is None)
): ):
continue continue
@ -517,77 +539,74 @@ class Database:
to_put = [] to_put = []
endl = "\n" endl = "\n"
if custom_configs: for custom_config in custom_configs:
for custom_config in custom_configs: config = {
config = { "data": custom_config["value"].replace("\\\n", "\n").encode("utf-8")
"data": custom_config["value"] if isinstance(custom_config["value"], str)
.replace("\\\n", "\n") else custom_config["value"].replace(b"\\\n", b"\n"),
.encode("utf-8") "method": method,
if isinstance(custom_config["value"], str) }
else custom_config["value"].replace(b"\\\n", b"\n"), config["checksum"] = sha256(config["data"]).hexdigest()
"method": method,
}
config["checksum"] = sha256(config["data"]).hexdigest()
if custom_config["exploded"][0]: if custom_config["exploded"][0]:
if ( if (
not session.query(Services) not session.query(Services)
.with_entities(Services.id) .with_entities(Services.id)
.filter_by(id=custom_config["exploded"][0]) .filter_by(id=custom_config["exploded"][0])
.first()
):
message += f"{endl if message else ''}Service {custom_config['exploded'][0]} not found, please check your config"
config.update(
{
"service_id": custom_config["exploded"][0],
"type": custom_config["exploded"][1]
.replace("-", "_")
.lower(),
"name": custom_config["exploded"][2],
}
)
else:
config.update(
{
"type": custom_config["exploded"][1]
.replace("-", "_")
.lower(),
"name": custom_config["exploded"][2],
}
)
custom_conf = (
session.query(Custom_configs)
.with_entities(Custom_configs.checksum, Custom_configs.method)
.filter_by(
service_id=config.get("service_id", None),
type=config["type"],
name=config["name"],
)
.first() .first()
):
message += f"{endl if message else ''}Service {custom_config['exploded'][0]} not found, please check your config"
config.update(
{
"service_id": custom_config["exploded"][0],
"type": custom_config["exploded"][1]
.replace("-", "_")
.lower(),
"name": custom_config["exploded"][2],
}
)
else:
config.update(
{
"type": custom_config["exploded"][1]
.replace("-", "_")
.lower(),
"name": custom_config["exploded"][2],
}
) )
if custom_conf is None: custom_conf = (
to_put.append(Custom_configs(**config)) session.query(Custom_configs)
elif config["checksum"] != custom_conf.checksum and ( .with_entities(Custom_configs.checksum, Custom_configs.method)
method == custom_conf.method or method == "autoconf" .filter_by(
): service_id=config.get("service_id", None),
session.query(Custom_configs).filter( type=config["type"],
Custom_configs.service_id == config.get("service_id", None), name=config["name"],
Custom_configs.type == config["type"], )
Custom_configs.name == config["name"], .first()
).update( )
{
Custom_configs.data: config["data"], if custom_conf is None:
Custom_configs.checksum: config["checksum"], to_put.append(Custom_configs(**config))
} elif config["checksum"] != custom_conf.checksum and (
| ( method == custom_conf.method or method == "autoconf"
{Custom_configs.method: "autoconf"} ):
if method == "autoconf" session.query(Custom_configs).filter(
else {} Custom_configs.service_id == config.get("service_id", None),
) Custom_configs.type == config["type"],
Custom_configs.name == config["name"],
).update(
{
Custom_configs.data: config["data"],
Custom_configs.checksum: config["checksum"],
}
| (
{Custom_configs.method: "autoconf"}
if method == "autoconf"
else {}
) )
)
try: try:
session.add_all(to_put) session.add_all(to_put)
@ -601,100 +620,106 @@ class Database:
"""Get the config from the database""" """Get the config from the database"""
with self.__db_session() as session: with self.__db_session() as session:
config = {} config = {}
for service in session.query(Services).with_entities(Services.id).all(): db_services = session.query(Services).with_entities(Services.id).all()
for setting in ( for setting in (
session.query(Settings) session.query(Settings)
.with_entities( .with_entities(
Settings.id, Settings.id,
Settings.context, Settings.context,
Settings.default, Settings.default,
Settings.multiple, Settings.multiple,
)
.all()
):
suffix = 0
while True:
global_value = (
session.query(Global_values)
.with_entities(Global_values.value, Global_values.method)
.filter_by(setting_id=setting.id, suffix=suffix)
.first()
) )
.all()
):
suffix = 0
while True:
global_value = (
session.query(Global_values)
.with_entities(Global_values.value, Global_values.method)
.filter_by(setting_id=setting.id, suffix=suffix)
.first()
)
if global_value is None:
if suffix == 0:
default = setting.default or ""
config[setting.id] = (
default
if methods is False
else {"value": default, "method": "default"}
)
else:
config[
setting.id + (f"_{suffix}" if suffix > 0 else "")
] = (
global_value.value
if methods is False
else {
"value": global_value.value,
"method": global_value.method,
}
)
if setting.context != "multisite":
break
if global_value is None:
if suffix == 0: if suffix == 0:
config[f"{service.id}_{setting.id}"] = ( default = setting.default or ""
config[setting.id] config[setting.id] = (
default
if methods is False if methods is False
else { else {"value": default, "method": "default"}
"value": config[setting.id]["value"],
"method": "default",
}
) )
elif f"{setting.id}_{suffix}" in config: elif setting.context != "multisite":
config[f"{service.id}_{setting.id}_{suffix}"] = ( break
config[f"{setting.id}_{suffix}"] else:
if methods is False config[setting.id + (f"_{suffix}" if suffix > 0 else "")] = (
else { global_value.value
"value": config[f"{setting.id}_{suffix}"]["value"], if methods is False
"method": "default", else {
} "value": global_value.value,
) "method": global_value.method,
}
service_setting = (
session.query(Services_settings)
.with_entities(
Services_settings.value, Services_settings.method
)
.filter_by(
service_id=service.id,
setting_id=setting.id,
suffix=suffix,
)
.first()
) )
if service_setting is not None: if setting.context == "multisite":
config[ changed = False
f"{service.id}_{setting.id}" for service in db_services:
+ (f"_{suffix}" if suffix > 0 else "") if suffix == 0:
] = ( config[f"{service.id}_{setting.id}"] = (
service_setting.value config[setting.id]
if methods is False if methods is False
else { else {
"value": service_setting.value, "value": config[setting.id]["value"],
"method": service_setting.method, "method": "default",
} }
)
changed = True
elif f"{setting.id}_{suffix}" in config:
config[f"{service.id}_{setting.id}_{suffix}"] = (
config[f"{setting.id}_{suffix}"]
if methods is False
else {
"value": config[f"{setting.id}_{suffix}"][
"value"
],
"method": "default",
}
)
changed = True
service_setting = (
session.query(Services_settings)
.with_entities(
Services_settings.value, Services_settings.method
)
.filter_by(
service_id=service.id,
setting_id=setting.id,
suffix=suffix,
)
.first()
) )
elif suffix > 0:
if service_setting is not None:
config[
f"{service.id}_{setting.id}"
+ (f"_{suffix}" if suffix > 0 else "")
] = (
service_setting.value
if methods is False
else {
"value": service_setting.value,
"method": service_setting.method,
}
)
changed = True
if global_value is None and changed is False:
break break
if not setting.multiple: if not setting.multiple:
break break
suffix += 1 suffix += 1
return config return config

1
src/common/db/model.py
View file

@ -115,6 +115,7 @@ class Services(Base):
__tablename__ = "services" __tablename__ = "services"
id = Column(String(64), primary_key=True) id = Column(String(64), primary_key=True)
method = Column(METHODS_ENUM, nullable=False)
settings = relationship( settings = relationship(
"Services_settings", back_populates="service", cascade="all, delete" "Services_settings", back_populates="service", cascade="all, delete"

3
src/common/gen/main.py
View file

@ -77,7 +77,6 @@ if __name__ == "__main__":
logger.info(f"Plugins : {args.plugins}") logger.info(f"Plugins : {args.plugins}")
logger.info(f"Output : {args.output}") logger.info(f"Output : {args.output}")
logger.info(f"Target : {args.target}") logger.info(f"Target : {args.target}")
logger.info(f"Variables : {args.variables}")
integration = "Linux" integration = "Linux"
if getenv("KUBERNETES_MODE", "no") == "yes": if getenv("KUBERNETES_MODE", "no") == "yes":
@ -91,6 +90,8 @@ if __name__ == "__main__":
integration = f.read().strip() integration = f.read().strip()
if args.variables: if args.variables:
logger.info(f"Variables : {args.variables}")
# Check existences and permissions # Check existences and permissions
logger.info("Checking arguments ...") logger.info("Checking arguments ...")
files = [args.settings, args.variables] files = [args.settings, args.variables]