From 70844ca6040c7db40a4a10925816c48698335376 Mon Sep 17 00:00:00 2001 From: TheophileDiot Date: Tue, 15 Nov 2022 17:02:47 +0100 Subject: [PATCH] Fix database with autoconf --- src/autoconf/Config.py | 11 +- src/autoconf/IngressController.py | 2 +- src/common/db/Database.py | 539 ++++++++++++++++-------------- src/common/db/model.py | 1 + src/common/gen/main.py | 3 +- 5 files changed, 292 insertions(+), 264 deletions(-) diff --git a/src/autoconf/Config.py b/src/autoconf/Config.py index 5f050f6e7..630216137 100644 --- a/src/autoconf/Config.py +++ b/src/autoconf/Config.py @@ -18,11 +18,6 @@ class Config(ConfigCaller): self.__config = {} self._db = Database(self.__logger) - while not self._db.is_initialized(): - self.__logger.warning( - "Database is not initialized, retrying in 5 seconds ...", - ) - sleep(5) def __get_full_env(self) -> dict: env_instances = {} @@ -76,6 +71,12 @@ class Config(ConfigCaller): } ) + while not self._db.is_initialized(): + self.__logger.warning( + "Database is not initialized, retrying in 5 seconds ...", + ) + sleep(5) + # save config to database ret = self._db.save_config(self.__config, "autoconf") if ret: diff --git a/src/autoconf/IngressController.py b/src/autoconf/IngressController.py index a11003f05..df438c0da 100644 --- a/src/autoconf/IngressController.py +++ b/src/autoconf/IngressController.py @@ -274,7 +274,7 @@ class IngressController(Controller, ConfigCaller): self.__logger.error( f"Unknown exception while reading k8s event (type = {watch_type}) :\n{format_exc()}", ) - finally : + finally: if locked: self.__internal_lock.release() locked = False diff --git a/src/common/db/Database.py b/src/common/db/Database.py index e371f9de9..8ea905daf 100644 --- a/src/common/db/Database.py +++ b/src/common/db/Database.py @@ -244,7 +244,7 @@ class Database: "order": 999, "name": "Default", "description": "Default settings", - "version": "1.0.0", + "version": "0.1", } else: settings = plugin.pop("settings", {}) @@ -318,134 +318,156 @@ class Database: ).delete() if config: - if config["MULTISITE"] == "yes": + if config.get("MULTISITE", "no") == "yes": global_values = [] - for server_name in config["SERVER_NAME"].split(" "): - if ( - server_name - and session.query(Services) - .with_entities(Services.id) - .filter_by(id=server_name) + db_services = ( + session.query(Services) + .with_entities(Services.id) + .filter_by(method=method) + .all() + ) + services = config.get("SERVER_NAME", "").split(" ") + + if db_services: + db_services = [service.id for service in db_services] + missing_ids = [ + service + for service in db_services + if service not in services + ] + + # Remove plugins that are no longer in the list + session.query(Services).filter( + Services.id.in_(missing_ids) + ).delete() + + for key, value in deepcopy(config).items(): + suffix = 0 + if search(r"_\d+$", key): + suffix = int(key.split("_")[-1]) + key = key[: -len(str(suffix)) - 1] + + setting = ( + session.query(Settings) + .with_entities(Settings.default) + .filter_by(id=key) .first() - is None - ): - to_put.append(Services(id=server_name)) + ) - for key, value in deepcopy(config).items(): - suffix = 0 - if search(r"_\d+$", key): - suffix = int(key.split("_")[-1]) - key = key[: -len(str(suffix)) - 1] + if not setting and services: + try: + server_name = next( + service + for service in services + if key.startswith(f"{service}_") + ) + except StopIteration: + continue + if server_name not in db_services: + to_put.append(Services(id=server_name, method=method)) + db_services.append(server_name) + + key = key.replace(f"{server_name}_", "") setting = ( session.query(Settings) .with_entities(Settings.default) - .filter_by(id=key.replace(f"{server_name}_", "")) + .filter_by(id=key) + .first() + ) + service_setting = ( + session.query(Services_settings) + .with_entities(Services_settings.value) + .filter_by( + service_id=server_name, + setting_id=key, + suffix=suffix, + ) .first() ) - if not setting: - continue + if service_setting is None: + if key != "SERVER_NAME" and ( + value == setting.default + or (value == "" and setting.default is None) + or (key in config and value == config[key]) + ): + continue - if server_name and key.startswith(server_name): - key = key.replace(f"{server_name}_", "") - service_setting = ( - session.query(Services_settings) - .with_entities(Services_settings.value) - .filter_by( + to_put.append( + Services_settings( service_id=server_name, setting_id=key, + value=value, suffix=suffix, + method=method, ) - .first() ) + elif ( + method == "autoconf" + and value != setting.default + and service_setting.value != value + ): + session.query(Services_settings).filter( + Services_settings.service_id == server_name, + Services_settings.setting_id == key, + Services_settings.suffix == suffix, + ).update( + { + Services_settings.value: value, + Services_settings.method: method, + } + ) + elif key not in global_values: + global_values.append(key) + global_value = ( + session.query(Global_values) + .with_entities(Global_values.value) + .filter_by( + setting_id=key, + suffix=suffix, + ) + .first() + ) - if service_setting is None: - if key != "SERVER_NAME" and ( - value == setting.default - or (value == "" and setting.default is None) - or (key in config and value == config[key]) - ): - continue + if global_value is None: + if ( + not setting + or value == setting.default + or (value == "" and setting.default is None) + ): + continue - to_put.append( - Services_settings( - service_id=server_name, - setting_id=key, - value=value, - suffix=suffix, - method=method, - ) - ) - elif method == "autoconf": - if key != "SERVER_NAME" and ( - value == setting.default - or (value == "" and setting.default is None) - or (key in config and value == config[key]) - ): - session.query(Services_settings).filter( - Services_settings.service_id == server_name, - Services_settings.setting_id == key, - Services_settings.suffix == suffix, - ).delete() - elif global_value.value != value: - session.query(Services_settings).filter( - Services_settings.service_id == server_name, - Services_settings.setting_id == key, - Services_settings.suffix == suffix, - ).update( - { - Services_settings.value: value, - Services_settings.method: method, - } - ) - elif key not in global_values: - global_values.append(key) - global_value = ( - session.query(Global_values) - .with_entities(Global_values.value) - .filter_by( + to_put.append( + Global_values( setting_id=key, + value=value, suffix=suffix, + method=method, ) - .first() ) - - if global_value is None: - if value == setting.default or ( - value == "" and setting.default is None - ): - continue - - to_put.append( - Global_values( - setting_id=key, - value=value, - suffix=suffix, - method=method, - ) - ) - elif method == "autoconf": - if value == setting.default or ( - value == "" and setting.default is None - ): - session.query(Global_values).filter( - Global_values.setting_id == key, - Global_values.suffix == suffix, - ).delete() - elif global_value.value != value: - session.query(Global_values).filter( - Global_values.setting_id == key, - Global_values.suffix == suffix, - ).update( - { - Global_values.value: value, - Global_values.method: method, - } - ) + elif ( + setting + and method == "autoconf" + and value != setting.default + and global_value.value != value + ): + session.query(Global_values).filter( + Global_values.setting_id == key, + Global_values.suffix == suffix, + ).update( + { + Global_values.value: value, + Global_values.method: method, + } + ) else: - primary_server_name = config["SERVER_NAME"].split(" ")[0] - to_put.append(Services(id=primary_server_name)) + if "SERVER_NAME" in config and config["SERVER_NAME"] != "": + to_put.append( + Services( + id=config["SERVER_NAME"].split(" ")[0], method=method + ) + ) for key, value in config.items(): suffix = 0 @@ -461,8 +483,8 @@ class Database: ) if ( - setting - and value == setting.default + not setting + or value == setting.default or (value == "" and setting.default is None) ): continue @@ -517,77 +539,74 @@ class Database: to_put = [] endl = "\n" - if custom_configs: - for custom_config in custom_configs: - config = { - "data": custom_config["value"] - .replace("\\\n", "\n") - .encode("utf-8") - if isinstance(custom_config["value"], str) - else custom_config["value"].replace(b"\\\n", b"\n"), - "method": method, - } - config["checksum"] = sha256(config["data"]).hexdigest() + for custom_config in custom_configs: + config = { + "data": custom_config["value"].replace("\\\n", "\n").encode("utf-8") + if isinstance(custom_config["value"], str) + else custom_config["value"].replace(b"\\\n", b"\n"), + "method": method, + } + config["checksum"] = sha256(config["data"]).hexdigest() - if custom_config["exploded"][0]: - if ( - not session.query(Services) - .with_entities(Services.id) - .filter_by(id=custom_config["exploded"][0]) - .first() - ): - message += f"{endl if message else ''}Service {custom_config['exploded'][0]} not found, please check your config" - - config.update( - { - "service_id": custom_config["exploded"][0], - "type": custom_config["exploded"][1] - .replace("-", "_") - .lower(), - "name": custom_config["exploded"][2], - } - ) - else: - config.update( - { - "type": custom_config["exploded"][1] - .replace("-", "_") - .lower(), - "name": custom_config["exploded"][2], - } - ) - - custom_conf = ( - session.query(Custom_configs) - .with_entities(Custom_configs.checksum, Custom_configs.method) - .filter_by( - service_id=config.get("service_id", None), - type=config["type"], - name=config["name"], - ) + if custom_config["exploded"][0]: + if ( + not session.query(Services) + .with_entities(Services.id) + .filter_by(id=custom_config["exploded"][0]) .first() + ): + message += f"{endl if message else ''}Service {custom_config['exploded'][0]} not found, please check your config" + + config.update( + { + "service_id": custom_config["exploded"][0], + "type": custom_config["exploded"][1] + .replace("-", "_") + .lower(), + "name": custom_config["exploded"][2], + } + ) + else: + config.update( + { + "type": custom_config["exploded"][1] + .replace("-", "_") + .lower(), + "name": custom_config["exploded"][2], + } ) - if custom_conf is None: - to_put.append(Custom_configs(**config)) - elif config["checksum"] != custom_conf.checksum and ( - method == custom_conf.method or method == "autoconf" - ): - session.query(Custom_configs).filter( - Custom_configs.service_id == config.get("service_id", None), - Custom_configs.type == config["type"], - Custom_configs.name == config["name"], - ).update( - { - Custom_configs.data: config["data"], - Custom_configs.checksum: config["checksum"], - } - | ( - {Custom_configs.method: "autoconf"} - if method == "autoconf" - else {} - ) + custom_conf = ( + session.query(Custom_configs) + .with_entities(Custom_configs.checksum, Custom_configs.method) + .filter_by( + service_id=config.get("service_id", None), + type=config["type"], + name=config["name"], + ) + .first() + ) + + if custom_conf is None: + to_put.append(Custom_configs(**config)) + elif config["checksum"] != custom_conf.checksum and ( + method == custom_conf.method or method == "autoconf" + ): + session.query(Custom_configs).filter( + Custom_configs.service_id == config.get("service_id", None), + Custom_configs.type == config["type"], + Custom_configs.name == config["name"], + ).update( + { + Custom_configs.data: config["data"], + Custom_configs.checksum: config["checksum"], + } + | ( + {Custom_configs.method: "autoconf"} + if method == "autoconf" + else {} ) + ) try: session.add_all(to_put) @@ -601,100 +620,106 @@ class Database: """Get the config from the database""" with self.__db_session() as session: config = {} - for service in session.query(Services).with_entities(Services.id).all(): - for setting in ( - session.query(Settings) - .with_entities( - Settings.id, - Settings.context, - Settings.default, - Settings.multiple, + db_services = session.query(Services).with_entities(Services.id).all() + for setting in ( + session.query(Settings) + .with_entities( + Settings.id, + Settings.context, + Settings.default, + Settings.multiple, + ) + .all() + ): + suffix = 0 + while True: + global_value = ( + session.query(Global_values) + .with_entities(Global_values.value, Global_values.method) + .filter_by(setting_id=setting.id, suffix=suffix) + .first() ) - .all() - ): - suffix = 0 - while True: - global_value = ( - session.query(Global_values) - .with_entities(Global_values.value, Global_values.method) - .filter_by(setting_id=setting.id, suffix=suffix) - .first() - ) - - if global_value is None: - if suffix == 0: - default = setting.default or "" - config[setting.id] = ( - default - if methods is False - else {"value": default, "method": "default"} - ) - else: - config[ - setting.id + (f"_{suffix}" if suffix > 0 else "") - ] = ( - global_value.value - if methods is False - else { - "value": global_value.value, - "method": global_value.method, - } - ) - - if setting.context != "multisite": - break + if global_value is None: if suffix == 0: - config[f"{service.id}_{setting.id}"] = ( - config[setting.id] + default = setting.default or "" + config[setting.id] = ( + default if methods is False - else { - "value": config[setting.id]["value"], - "method": "default", - } + else {"value": default, "method": "default"} ) - elif f"{setting.id}_{suffix}" in config: - config[f"{service.id}_{setting.id}_{suffix}"] = ( - config[f"{setting.id}_{suffix}"] - if methods is False - else { - "value": config[f"{setting.id}_{suffix}"]["value"], - "method": "default", - } - ) - - service_setting = ( - session.query(Services_settings) - .with_entities( - Services_settings.value, Services_settings.method - ) - .filter_by( - service_id=service.id, - setting_id=setting.id, - suffix=suffix, - ) - .first() + elif setting.context != "multisite": + break + else: + config[setting.id + (f"_{suffix}" if suffix > 0 else "")] = ( + global_value.value + if methods is False + else { + "value": global_value.value, + "method": global_value.method, + } ) - if service_setting is not None: - config[ - f"{service.id}_{setting.id}" - + (f"_{suffix}" if suffix > 0 else "") - ] = ( - service_setting.value - if methods is False - else { - "value": service_setting.value, - "method": service_setting.method, - } + if setting.context == "multisite": + changed = False + for service in db_services: + if suffix == 0: + config[f"{service.id}_{setting.id}"] = ( + config[setting.id] + if methods is False + else { + "value": config[setting.id]["value"], + "method": "default", + } + ) + changed = True + elif f"{setting.id}_{suffix}" in config: + config[f"{service.id}_{setting.id}_{suffix}"] = ( + config[f"{setting.id}_{suffix}"] + if methods is False + else { + "value": config[f"{setting.id}_{suffix}"][ + "value" + ], + "method": "default", + } + ) + changed = True + + service_setting = ( + session.query(Services_settings) + .with_entities( + Services_settings.value, Services_settings.method + ) + .filter_by( + service_id=service.id, + setting_id=setting.id, + suffix=suffix, + ) + .first() ) - elif suffix > 0: + + if service_setting is not None: + config[ + f"{service.id}_{setting.id}" + + (f"_{suffix}" if suffix > 0 else "") + ] = ( + service_setting.value + if methods is False + else { + "value": service_setting.value, + "method": service_setting.method, + } + ) + changed = True + + if global_value is None and changed is False: break - if not setting.multiple: - break + if not setting.multiple: + break - suffix += 1 + suffix += 1 return config diff --git a/src/common/db/model.py b/src/common/db/model.py index 71e6eebb5..d07841560 100644 --- a/src/common/db/model.py +++ b/src/common/db/model.py @@ -115,6 +115,7 @@ class Services(Base): __tablename__ = "services" id = Column(String(64), primary_key=True) + method = Column(METHODS_ENUM, nullable=False) settings = relationship( "Services_settings", back_populates="service", cascade="all, delete" diff --git a/src/common/gen/main.py b/src/common/gen/main.py index 3d59189a5..08909931a 100644 --- a/src/common/gen/main.py +++ b/src/common/gen/main.py @@ -77,7 +77,6 @@ if __name__ == "__main__": logger.info(f"Plugins : {args.plugins}") logger.info(f"Output : {args.output}") logger.info(f"Target : {args.target}") - logger.info(f"Variables : {args.variables}") integration = "Linux" if getenv("KUBERNETES_MODE", "no") == "yes": @@ -91,6 +90,8 @@ if __name__ == "__main__": integration = f.read().strip() if args.variables: + logger.info(f"Variables : {args.variables}") + # Check existences and permissions logger.info("Checking arguments ...") files = [args.settings, args.variables]