diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e43791ec..115951f9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,9 +15,11 @@ - [BUGFIX] Remove certbot renew delay causing errors on k8s - [BUGFIX] Fix missing custom modsec files when BW instances change - [BUGFIX] Fix inconsistency on config changes when using Redis +- [BUGFIX] Fix web UI not working when using / URL - [FEATURE] Add Anonymous reporting feature - [FEATURE] Add support for fallback Referrer-Policies -- [FEATURE] Add profile page to web ui and the possibility to activate the 2FA +- [FEATURE] Add 2FA support to web UI +- [FEATURE] Add username and password management to web UI - [FEATURE] Add setting REVERSE_PROXY_INCLUDES to manually add "include" directives in the reverse proxies - [FEATURE] Add support for Redis Sentinel - [FEATURE] Add support for tls in Ingress definition diff --git a/examples/autoconf-configs/tests.json b/examples/autoconf-configs/tests.json index dc92c1d69..e58915d3a 100644 --- a/examples/autoconf-configs/tests.json +++ b/examples/autoconf-configs/tests.json @@ -1,7 +1,7 @@ { "name": "autoconf-configs", "kinds": ["autoconf"], - "delay": 60, + "delay": 180, "timeout": 60, "tests": [ { diff --git a/examples/docker-configs/tests.json b/examples/docker-configs/tests.json index c9bb815c2..ab3805887 100644 --- a/examples/docker-configs/tests.json +++ b/examples/docker-configs/tests.json @@ -1,7 +1,7 @@ { "name": "docker-configs", "kinds": ["docker"], - "delay": 30, + "delay": 120, "timeout": 60, "tests": [ { diff --git a/examples/kubernetes-ingress/tests.json b/examples/kubernetes-ingress/tests.json index 5049e3bee..caa4f01dc 100644 --- a/examples/kubernetes-ingress/tests.json +++ b/examples/kubernetes-ingress/tests.json @@ -2,7 +2,7 @@ "name": "kubernetes-ingress", "kinds": ["kubernetes"], "timeout": 60, - "delay": 60, + "delay": 120, "tests": [ { "type": "string", diff --git a/examples/kubernetes-tls/tests.json b/examples/kubernetes-tls/tests.json index 031368a2a..00c2c9aa6 100644 --- a/examples/kubernetes-tls/tests.json +++ b/examples/kubernetes-tls/tests.json @@ -1,26 +1,29 @@ { - "name": "kubernetes-ingress", + "name": "kubernetes-tls", "kinds": ["kubernetes"], "timeout": 60, - "delay": 60, + "delay": 300, "tests": [ { "type": "string", "url": "https://app1.example.com", "string": "hello", - "tls": "app1.example.com,app2.example.com" + "tls": "app1.example.com,app2.example.com", + "tls_edit": false }, { "type": "string", "url": "https://app2.example.com", "string": "hello", - "tls": "app1.example.com,app2.example.com" + "tls": "app1.example.com,app2.example.com", + "tls_edit": false }, { "type": "string", "url": "https://app3.example.com", "string": "hello", - "tls": "app3.example.com" + "tls": "app3.example.com", + "tls_edit": false } ] } diff --git a/examples/swarm-configs/tests.json b/examples/swarm-configs/tests.json index 53c474dae..da9979f12 100644 --- a/examples/swarm-configs/tests.json +++ b/examples/swarm-configs/tests.json @@ -1,7 +1,8 @@ { "name": "swarm-configs", "kinds": ["swarm"], - "timeout": 120, + "timeout": 60, + "delay": 120, "tests": [ { "type": "string", diff --git a/misc/integrations/k8s.mariadb.yml b/misc/integrations/k8s.mariadb.yml index 565a576f8..ba0aedd33 100644 --- a/misc/integrations/k8s.mariadb.yml +++ b/misc/integrations/k8s.mariadb.yml @@ -1,3 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-bunkerweb +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -247,14 +258,3 @@ spec: protocol: TCP port: 6379 targetPort: 6379 ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: pvc-bunkerweb -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 5Gi diff --git a/src/autoconf/DockerController.py b/src/autoconf/DockerController.py index 4529e176d..0c5e5aa56 100644 --- a/src/autoconf/DockerController.py +++ b/src/autoconf/DockerController.py @@ -100,10 +100,15 @@ class DockerController(Controller): first=not self._loaded, ) + def __process_event(self, event): + return "Actor" in event and "Attributes" in event["Actor"] and ("bunkerweb.INSTANCE" in event["Actor"]["Attributes"] or "bunkerweb.SERVER_NAME" in event["Actor"]["Attributes"]) + def process_events(self): self._set_autoconf_load_db() - for _ in self.__client.events(decode=True, filters={"type": "container"}): + for event in self.__client.events(decode=True, filters={"type": "container"}): try: + if not self.__process_event(event): + continue self._update_settings() self._instances = self.get_instances() self._services = self.get_services() diff --git a/src/autoconf/IngressController.py b/src/autoconf/IngressController.py index 2e9db4a77..c49f5039c 100644 --- a/src/autoconf/IngressController.py +++ b/src/autoconf/IngressController.py @@ -146,15 +146,16 @@ class IngressController(Controller): for host in tls.hosts: for service in services: if host in service["SERVER_NAME"].split(" "): - secret_tls = self.__corev1.list_secret_for_all_namespaces( + secrets_tls = self.__corev1.list_secret_for_all_namespaces( watch=False, field_selector=f"metadata.name={tls.secret_name},metadata.namespace={namespace}", ).items - if not secret_tls: + if len(secrets_tls) == 0: self._logger.warning( f"Ignoring tls setting for {host} : secret {tls.secret_name} not found.", ) break + secret_tls = secrets_tls[0] if not secret_tls.data: self._logger.warning( f"Ignoring tls setting for {host} : secret {tls.secret_name} contains no data.", @@ -227,6 +228,22 @@ class IngressController(Controller): configs[config_type][f"{config_site}{config_name}"] = config_data return configs + def __process_event(self, event): + obj = event["object"] + metadata = obj.metadata if obj else None + annotations = metadata.annotations if metadata else None + if not obj: + return False + if obj.kind == "Pod": + return annotations and "bunkerweb.io/INSTANCE" in annotations + if obj.kind == "Ingress": + return True + if obj.kind == "ConfigMap": + return annotations and "bunkerweb.io/CONFIG_TYPE" in annotations + if obj.kind == "Service": + return True + return False + def __watch(self, watch_type): w = watch.Watch() what = None @@ -245,9 +262,13 @@ class IngressController(Controller): locked = False error = False try: - for _ in w.stream(what): + for event in w.stream(what): self.__internal_lock.acquire() locked = True + if not self.__process_event(event): + self.__internal_lock.release() + locked = False + continue self._update_settings() self._instances = self.get_instances() self._services = self.get_services() diff --git a/src/autoconf/SwarmController.py b/src/autoconf/SwarmController.py index d0976620b..dd415f364 100644 --- a/src/autoconf/SwarmController.py +++ b/src/autoconf/SwarmController.py @@ -16,14 +16,20 @@ class SwarmController(Controller): super().__init__("swarm") self.__client = DockerClient(base_url=docker_host) self.__internal_lock = Lock() + self.__swarm_instances = [] + self.__swarm_services = [] + self.__swarm_configs = [] def _get_controller_instances(self) -> List[Service]: + self.__swarm_instances = [] return self.__client.services.list(filters={"label": "bunkerweb.INSTANCE"}) def _get_controller_services(self) -> List[Service]: + self.__swarm_services = [] return self.__client.services.list(filters={"label": "bunkerweb.SERVER_NAME"}) def _to_instances(self, controller_instance) -> List[dict]: + self.__swarm_instances.append(controller_instance.id) instances = [] instance_env = {} for env in controller_instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]: @@ -46,6 +52,7 @@ class SwarmController(Controller): return instances def _to_services(self, controller_service) -> List[dict]: + self.__swarm_services.append(controller_service.id) service = {} for variable, value in controller_service.attrs["Spec"]["Labels"].items(): if not variable.startswith("bunkerweb."): @@ -80,6 +87,7 @@ class SwarmController(Controller): return services def get_configs(self) -> Dict[str, Dict[str, Any]]: + self.__swarm_configs = [] configs = {} for config_type in self._supported_config_types: configs[config_type] = {} @@ -103,6 +111,7 @@ class SwarmController(Controller): continue config_site = f"{config.attrs['Spec']['Labels']['bunkerweb.CONFIG_SITE']}/" configs[config_type][f"{config_site}{config_name}"] = b64decode(config.attrs["Spec"]["Data"]) + self.__swarm_configs.append(config.id) return configs def apply_config(self) -> bool: @@ -113,14 +122,40 @@ class SwarmController(Controller): first=not self._loaded, ) + def __process_event(self, event): + if "Actor" not in event or "ID" not in event["Actor"] or "Type" not in event: + return False + if event["Type"] not in ["service", "config"]: + return False + if event["Type"] == "service": + if event["Actor"]["ID"] in self.__swarm_instances or event["Actor"]["ID"] in self.__swarm_services: + return True + try: + labels = self.__client.services.get(event["Actor"]["ID"]).attrs["Spec"]["Labels"] + return "bunkerweb.INSTANCE" in labels or "bunkerweb.SERVER_NAME" in labels + except: + return False + if event["Type"] == "config": + if event["Actor"]["ID"] in self.__swarm_configs: + return True + try: + return "bunkerweb.CONFIG_TYPE" in self.__client.configs.get(event["Actor"]["ID"]).attrs["Spec"]["Labels"] + except: + return False + return False + def __event(self, event_type): while True: locked = False error = False try: - for _ in self.__client.events(decode=True, filters={"type": event_type}): + for event in self.__client.events(decode=True, filters={"type": event_type}): self.__internal_lock.acquire() locked = True + if not self.__process_event(event): + self.__internal_lock.release() + locked = False + continue try: self._update_settings() self._instances = self.get_instances() @@ -137,6 +172,7 @@ class SwarmController(Controller): self._logger.info( "Successfully deployed new configuration 🚀", ) + self._set_autoconf_load_db() except: self._logger.error(f"Exception while processing Swarm event ({event_type}) :\n{format_exc()}") self.__internal_lock.release() diff --git a/src/common/confs/api.conf b/src/common/confs/api.conf index 61efdd377..f804ffa46 100644 --- a/src/common/confs/api.conf +++ b/src/common/confs/api.conf @@ -13,6 +13,11 @@ server { # default mime type is JSON default_type 'application/json'; + # variables + set $reason ''; + set $reason_data ''; + set $ctx_ref ''; + # check IP and do the API call access_by_lua_block { -- Instantiate objects and import required modules diff --git a/src/common/confs/default-server-http.conf b/src/common/confs/default-server-http.conf index 84bc51da4..0afea7423 100644 --- a/src/common/confs/default-server-http.conf +++ b/src/common/confs/default-server-http.conf @@ -1,7 +1,9 @@ server { - # reason variable + # variables set $reason ''; + set $reason_data ''; + set $ctx_ref ''; server_name _; @@ -99,6 +101,8 @@ server { local ok, ret = call_plugin(plugin_obj, "log_default") if not ok then logger:log(ERR, ret) + elseif not ret.ret then + logger:log(ERR, plugin_id .. ":log_default() call failed : " .. ret.msg) else logger:log(INFO, plugin_id .. ":log_default() call successful : " .. ret.msg) end diff --git a/src/common/confs/server-http/ssl-certificate-lua.conf b/src/common/confs/server-http/ssl-certificate-lua.conf index b4c7ebfef..a8cd9b7f6 100644 --- a/src/common/confs/server-http/ssl-certificate-lua.conf +++ b/src/common/confs/server-http/ssl-certificate-lua.conf @@ -85,6 +85,7 @@ ssl_certificate_by_lua_block { if not ok then logger:log(ERR, "error while setting private key : " .. err) else + logger:log(INFO, "certificate set by " .. plugin_id) return true end end diff --git a/src/common/core/bunkernet/bunkernet.lua b/src/common/core/bunkernet/bunkernet.lua index fb030b0af..accf7de56 100644 --- a/src/common/core/bunkernet/bunkernet.lua +++ b/src/common/core/bunkernet/bunkernet.lua @@ -137,10 +137,6 @@ function bunkernet:access() if not self:is_needed() then return self:ret(true, "service doesn't use BunkerNet, skipping access") end - -- Check id - if not self.bunkernet_id then - return self:ret(false, "missing instance ID") - end -- Check if IP is global if not self.ctx.bw.ip_is_global then return self:ret(true, "IP is not global") @@ -149,6 +145,10 @@ function bunkernet:access() if is_whitelisted(self.ctx) then return self:ret(true, "client is whitelisted") end + -- Check id + if not self.bunkernet_id then + return self:ret(false, "missing instance ID") + end -- Extract DB local db, err = self.datastore:get("plugin_bunkernet_db", true) if db then @@ -175,10 +175,6 @@ function bunkernet:log(bypass_checks) if not self:is_needed() then return self:ret(true, "service doesn't use BunkerNet, skipping log") end - -- Check id - if not self.bunkernet_id then - return self:ret(false, "missing instance ID") - end end -- Check if IP has been blocked local reason, reason_data = get_reason(self.ctx) @@ -192,6 +188,10 @@ function bunkernet:log(bypass_checks) if not self.ctx.bw.ip_is_global then return self:ret(true, "IP is not global") end + -- Check id + if not self.bunkernet_id then + return self:ret(false, "missing instance ID") + end -- Check if IP has been reported recently local ok, data = self.cachestore:get("plugin_bunkernet_" .. self.ctx.bw.remote_addr .. "_" .. reason) if not ok then @@ -239,10 +239,6 @@ function bunkernet:log_default() if not self:is_needed() then return self:ret(true, "no service uses BunkerNet, skipping log_default") end - -- Check id - if not self.bunkernet_id then - return self:ret(false, "missing instance ID") - end -- Check if default server is disabled local check, err = get_variable("DISABLE_DEFAULT_SERVER", false) if check == nil then diff --git a/src/common/core/bunkernet/jobs/bunkernet-register.py b/src/common/core/bunkernet/jobs/bunkernet-register.py index b9803e84f..5e99ba237 100755 --- a/src/common/core/bunkernet/jobs/bunkernet-register.py +++ b/src/common/core/bunkernet/jobs/bunkernet-register.py @@ -32,12 +32,7 @@ try: bunkernet_activated = False # Multisite case if getenv("MULTISITE", "no") == "yes": - servers = getenv("SERVER_NAME") or [] - - if isinstance(servers, str): - servers = servers.split(" ") - - for first_server in servers: + for first_server in getenv("SERVER_NAME", "").split(" "): if getenv(f"{first_server}_USE_BUNKERNET", getenv("USE_BUNKERNET", "yes")) == "yes": bunkernet_activated = True break diff --git a/src/common/core/customcert/customcert.lua b/src/common/core/customcert/customcert.lua index 76655c1f9..0eb9e097a 100644 --- a/src/common/core/customcert/customcert.lua +++ b/src/common/core/customcert/customcert.lua @@ -36,8 +36,8 @@ function customcert:init() for server_name, multisite_vars in pairs(vars) do if multisite_vars["USE_CUSTOM_SSL"] == "yes" and server_name ~= "global" then local check, data = read_files({ - "/var/cache/bunkerweb/customcert/" .. server_name .. "/cert.pem", - "/var/cache/bunkerweb/customcert/" .. server_name .. "/key.pem", + "/var/cache/bunkerweb/customcert/" .. server_name .. ".cert.pem", + "/var/cache/bunkerweb/customcert/" .. server_name .. ".key.pem", }) if not check then self.logger:log(ERR, "error while reading files : " .. data) @@ -60,8 +60,8 @@ function customcert:init() return self:ret(false, "can't get SERVER_NAME variable : " .. err) end local check, data = read_files({ - "/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/cert.pem", - "/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/key.pem", + "/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. ".cert.pem", + "/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. ".key.pem", }) if not check then self.logger:log(ERR, "error while reading files : " .. data) @@ -87,15 +87,14 @@ function customcert:ssl_certificate() if not server_name then return self:ret(false, "can't get server_name : " .. err) end - if self.variables["USE_CUSTOM_SSL"] == "yes" then - local data - data, err = self.datastore:get("plugin_customcert_" .. server_name, true) - if not data then - return self:ret( - false, - "error while getting plugin_customcert_" .. server_name .. " from datastore : " .. err - ) - end + local data + data, err = self.datastore:get("plugin_customcert_" .. server_name, true) + if not data and err ~= "not found" then + return self:ret( + false, + "error while getting plugin_customcert_" .. server_name .. " from datastore : " .. err + ) + elseif data then return self:ret(true, "certificate/key data found", data) end return self:ret(true, "custom certificate is not used") diff --git a/src/common/core/customcert/jobs/custom-cert.py b/src/common/core/customcert/jobs/custom-cert.py index 7377305b9..974f52f86 100644 --- a/src/common/core/customcert/jobs/custom-cert.py +++ b/src/common/core/customcert/jobs/custom-cert.py @@ -28,6 +28,7 @@ db = None def check_cert(cert_path: str, key_path: str, first_server: str) -> bool: try: + ret = False if not cert_path or not key_path: logger.warning("Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates") return False @@ -48,19 +49,17 @@ def check_cert(cert_path: str, key_path: str, first_server: str) -> bool: "cache", "bunkerweb", "customcert", - first_server, - "cert.pem", + f"{first_server}.cert.pem", ) cert_cache_path.parent.mkdir(parents=True, exist_ok=True) cert_hash = file_hash(cert_path) old_hash = cache_hash(cert_cache_path, db) - if old_hash == cert_hash: - return False - - cached, err = cache_file(cert_path, cert_cache_path, cert_hash, db, delete_file=False) - if not cached: - logger.error(f"Error while caching custom-cert cert.pem file : {err}") + if old_hash != cert_hash: + ret = True + cached, err = cache_file(cert_path, cert_cache_path, cert_hash, db, delete_file=False) + if not cached: + logger.error(f"Error while caching custom-cert cert.pem file : {err}") key_cache_path = Path( sep, @@ -68,19 +67,19 @@ def check_cert(cert_path: str, key_path: str, first_server: str) -> bool: "cache", "bunkerweb", "customcert", - first_server, - "key.pem", + f"{first_server}.key.pem", ) key_cache_path.parent.mkdir(parents=True, exist_ok=True) key_hash = file_hash(key_path) old_hash = cache_hash(key_cache_path, db) if old_hash != key_hash: + ret = True cached, err = cache_file(key_path, key_cache_path, key_hash, db, delete_file=False) if not cached: logger.error(f"Error while caching custom-cert key.pem file : {err}") - return True + return ret except: logger.error( f"Exception while running custom-cert.py (check_cert) :\n{format_exc()}", @@ -104,7 +103,7 @@ try: key_data = b64decode(getenv("CUSTOM_SSL_KEY_DATA", "")) for file, data in (("cert.pem", cert_data), ("key.pem", key_data)): if data != b"": - file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", first_server, file) + file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", f"{first_server}.{file}") file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_bytes(data) if file == "cert.pem": @@ -141,7 +140,7 @@ try: key_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_KEY_DATA", "")) for file, data in (("cert.pem", cert_data), ("key.pem", key_data)): if data != b"": - file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", first_server, file) + file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", f"{first_server}.{file}") file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_bytes(data) if file == "cert.pem": diff --git a/src/common/core/letsencrypt/letsencrypt.lua b/src/common/core/letsencrypt/letsencrypt.lua index dcd1fcc78..342321e8b 100644 --- a/src/common/core/letsencrypt/letsencrypt.lua +++ b/src/common/core/letsencrypt/letsencrypt.lua @@ -100,15 +100,14 @@ function letsencrypt:ssl_certificate() if not server_name then return self:ret(false, "can't get server_name : " .. err) end - if self.variables["AUTO_LETS_ENCRYPT"] == "yes" then - local data - data, err = self.datastore:get("plugin_letsencrypt_" .. server_name, true) - if not data then - return self:ret( - false, - "error while getting plugin_letsencrypt_" .. server_name .. " from datastore : " .. err - ) - end + local data + data, err = self.datastore:get("plugin_letsencrypt_" .. server_name, true) + if not data and err ~= "not found" then + return self:ret( + false, + "error while getting plugin_letsencrypt_" .. server_name .. " from datastore : " .. err + ) + elseif data then return self:ret(true, "certificate/key data found", data) end return self:ret(true, "let's encrypt is not used") @@ -175,9 +174,9 @@ function letsencrypt:api() if not ok then return self:ret(true, "can't remove validation token : " .. err, HTTP_INTERNAL_SERVER_ERROR) end - return true, HTTP_OK, { status = "success", msg = "validation token removed" } + return self:ret(true, "validation token removed", HTTP_OK) end - return true, HTTP_NOT_FOUND, { status = "error", msg = "unknown request" } + return self:ret(true, "unknown request", HTTP_NOT_FOUND) end return letsencrypt diff --git a/src/common/core/selfsigned/selfsigned.lua b/src/common/core/selfsigned/selfsigned.lua index fa0deb411..32c413bd0 100644 --- a/src/common/core/selfsigned/selfsigned.lua +++ b/src/common/core/selfsigned/selfsigned.lua @@ -87,18 +87,17 @@ function selfsigned:ssl_certificate() if not server_name then return self:ret(false, "can't get server_name : " .. err) end - if self.variables["GENERATE_SELF_SIGNED_SSL"] == "yes" then - local data - data, err = self.datastore:get("plugin_selfsigned_" .. server_name, true) - if not data then - return self:ret( - false, - "error while getting plugin_selfsigned_" .. server_name .. " from datastore : " .. err - ) - end + local data + data, err = self.datastore:get("plugin_selfsigned_" .. server_name, true) + if not data and err ~= "not found" then + return self:ret( + false, + "error while getting plugin_selfsigned_" .. server_name .. " from datastore : " .. err + ) + elseif data then return self:ret(true, "certificate/key data found", data) end - return self:ret(true, "selfsigned is not used") + return self:ret(true, "self signed is not used") end function selfsigned:load_data(data, server_name) diff --git a/src/common/db/model.py b/src/common/db/model.py index 36a481f45..7d2cd00c7 100644 --- a/src/common/db/model.py +++ b/src/common/db/model.py @@ -97,7 +97,7 @@ class Global_values(Base): ForeignKey("bw_settings.id", onupdate="cascade", ondelete="cascade"), primary_key=True, ) - value = Column(String(4096), nullable=False) + value = Column(String(8192), nullable=False) suffix = Column(Integer, primary_key=True, nullable=True, default=0) method = Column(METHODS_ENUM, nullable=False) @@ -128,7 +128,7 @@ class Services_settings(Base): ForeignKey("bw_settings.id", onupdate="cascade", ondelete="cascade"), primary_key=True, ) - value = Column(String(4096), nullable=False) + value = Column(String(8192), nullable=False) suffix = Column(Integer, primary_key=True, nullable=True, default=0) method = Column(METHODS_ENUM, nullable=False) diff --git a/src/linux/fpm-fedora b/src/linux/fpm-fedora index b68651ac7..a81af1efb 100644 --- a/src/linux/fpm-fedora +++ b/src/linux/fpm-fedora @@ -3,7 +3,7 @@ --license agpl3 --version %VERSION% --architecture %ARCH% ---depends bash --depends python3 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream --depends pcre --depends libpq --depends libcap +--depends bash --depends python3 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream --depends pcre --depends libpq --depends libcap --depends openssl --description "BunkerWeb %VERSION% for Fedora 39" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " diff --git a/src/linux/fpm-rhel b/src/linux/fpm-rhel index 5b0d56e26..a920d7337 100644 --- a/src/linux/fpm-rhel +++ b/src/linux/fpm-rhel @@ -3,7 +3,7 @@ --license agpl3 --version %VERSION% --architecture %ARCH% ---depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends geoip --depends libpq --depends libcap +--depends bash --depends python39 --depends 'nginx >= 1:1.24.0' --depends 'nginx < 1:1.25.0' --depends libcurl-devel --depends libxml2 --depends yajl --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends geoip --depends libpq --depends libcap --depends openssl --description "BunkerWeb %VERSION% for RHEL 8" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " diff --git a/tests/AutoconfTest.py b/tests/AutoconfTest.py index 25e02d1f4..9ea94540f 100644 --- a/tests/AutoconfTest.py +++ b/tests/AutoconfTest.py @@ -10,7 +10,7 @@ from yaml import safe_load, dump class AutoconfTest(Test): - def __init__(self, name, timeout, tests, no_copy_container=False, delay=0): + def __init__(self, name, timeout, tests, no_copy_container=False, delay=0, domains={}): super().__init__( name, "autoconf", @@ -19,13 +19,7 @@ class AutoconfTest(Test): no_copy_container=no_copy_container, delay=delay, ) - self._domains = { - r"www\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1')}", - r"auth\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1')}", - r"app1\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_1')}", - r"app2\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_2')}", - r"app3\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_3')}", - } + self._domains = domains self._check_domains() @staticmethod diff --git a/tests/DockerTest.py b/tests/DockerTest.py index 4903fbfcb..33d13ed01 100644 --- a/tests/DockerTest.py +++ b/tests/DockerTest.py @@ -7,7 +7,7 @@ from logger import log class DockerTest(Test): - def __init__(self, name, timeout, tests, no_copy_container=False, delay=0): + def __init__(self, name, timeout, tests, no_copy_container=False, delay=0, domains={}): super().__init__( name, "docker", @@ -16,13 +16,7 @@ class DockerTest(Test): no_copy_container=no_copy_container, delay=delay, ) - self._domains = { - r"www\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1"), - r"auth\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1"), - r"app1\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1_1"), - r"app2\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1_2"), - r"app3\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1_3"), - } + self._domains = domains self._check_domains() @staticmethod @@ -60,7 +54,7 @@ class DockerTest(Test): Test.replace_in_file( compose, r"AUTO_LETS_ENCRYPT=yes", - "AUTO_LETS_ENCRYPT=yes\n - USE_LETS_ENCRYPT_STAGING=yes", + "AUTO_LETS_ENCRYPT=yes\n - USE_LETS_ENCRYPT_STAGING=yes\n - LOG_LEVEL=info", ) Test.replace_in_file(compose, r"DISABLE_DEFAULT_SERVER=yes", "DISABLE_DEFAULT_SERVER=no") for ex_domain, test_domain in self._domains.items(): diff --git a/tests/KubernetesTest.py b/tests/KubernetesTest.py index 966b0f934..afbfffe36 100644 --- a/tests/KubernetesTest.py +++ b/tests/KubernetesTest.py @@ -10,15 +10,9 @@ from yaml import safe_load_all, dump_all class KubernetesTest(Test): - def __init__(self, name, timeout, tests, delay=0): + def __init__(self, name, timeout, tests, delay=0, domains={}): super().__init__(name, "kubernetes", timeout, tests, delay=delay) - self._domains = { - r"www\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_2')}", - r"auth\.example\.com": f"{Test.random_string(1)}.{getenv('TEST_DOMAIN1_2')}", - r"app1\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1')}", - r"app2\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN2')}", - r"app3\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN3')}", - } + self._domains = domains @staticmethod def init(): @@ -38,6 +32,7 @@ class KubernetesTest(Test): "USE_PROXY_PROTOCOL": "yes", "REAL_IP_FROM": "100.64.0.0/10 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8", "REAL_IP_HEADER": "proxy_protocol", + "LOG_LEVEL": "info" } replace_env = {"API_WHITELIST_IP": "127.0.0.1/8 100.64.0.0/10 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8"} for yaml in data: diff --git a/tests/LinuxTest.py b/tests/LinuxTest.py index 78b977a21..c2ef8700c 100644 --- a/tests/LinuxTest.py +++ b/tests/LinuxTest.py @@ -8,15 +8,9 @@ from logger import log class LinuxTest(Test): - def __init__(self, name, timeout, tests, distro): + def __init__(self, name, timeout, tests, distro, domains={}): super().__init__(name, "linux", timeout, tests, delay=20) - self._domains = { - r"www\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1')}", - r"auth\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1')}", - r"app1\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_1')}", - r"app2\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_2')}", - r"app3\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_3')}", - } + self._domains = domains if distro not in ("ubuntu", "debian", "fedora", "centos", "rhel"): raise Exception(f"unknown distro {distro}") self.__distro = distro @@ -114,7 +108,7 @@ class LinuxTest(Test): raise Exception("docker exec cp variables.env failed (test)") proc = self.docker_exec( self.__distro, - "echo '' >> /etc/bunkerweb/variables.env ; echo 'USE_LETS_ENCRYPT_STAGING=yes' >> /etc/bunkerweb/variables.env", + "echo '' >> /etc/bunkerweb/variables.env ; echo 'USE_LETS_ENCRYPT_STAGING=yes' >> /etc/bunkerweb/variables.env ; echo 'LOG_LEVEL=info' >> /etc/bunkerweb/variables.env", ) if proc.returncode != 0: raise (Exception("docker exec append variables.env failed (test)")) diff --git a/tests/SwarmTest.py b/tests/SwarmTest.py index 38666b98c..d3c44ffb6 100644 --- a/tests/SwarmTest.py +++ b/tests/SwarmTest.py @@ -10,15 +10,9 @@ from yaml import safe_load, dump class SwarmTest(Test): - def __init__(self, name, timeout, tests, delay=0): + def __init__(self, name, timeout, tests, delay=0, domains={}): super().__init__(name, "swarm", timeout, tests, delay=delay) - self._domains = { - r"www\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_1')}", - r"auth\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_2')}", - r"app1\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1')}", - r"app2\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN2')}", - r"app3\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN3')}", - } + self._domains = domains @staticmethod def init(): @@ -38,6 +32,7 @@ class SwarmTest(Test): if "AUTO_LETS_ENCRYPT=yes" not in data["services"]["bunkerweb"]["environment"]: data["services"]["bunkerweb"]["environment"].append("AUTO_LETS_ENCRYPT=yes") data["services"]["bunkerweb"]["environment"].append("USE_LETS_ENCRYPT_STAGING=yes") + data["services"]["bunkerweb"]["environment"].append("LOG_LEVEL=info") del data["services"]["bunkerweb"]["deploy"]["placement"] with open(compose, "w") as f: f.write(dump(data)) diff --git a/tests/Test.py b/tests/Test.py index 811938271..46f8a705f 100644 --- a/tests/Test.py +++ b/tests/Test.py @@ -142,15 +142,24 @@ class Test(ABC): r = get(ex_url, timeout=10, verify=False) ok = test["status"] == r.status_code if ok and "tls" in test: + ex_tls = test["tls"] + tls_edit = True + if "tls_edit" in test: + tls_edit = test["tls_edit"] + if tls_edit: + for ex_domain, test_domain in self._domains.items(): + if search(ex_domain, ex_tls): + ex_tls = sub(ex_domain, test_domain, ex_tls) connection = create_connection((urlparse(ex_url).netloc, 443)) context = SSLContext() sock = context.wrap_socket(connection, server_hostname=urlparse(ex_url).netloc) cert = sock.getpeercert(True) sock.close() x509 = crypto.load_certificate(crypto.FILETYPE_ASN1, cert) - if x509.get_subject().CN != test["tls"]: + if x509.get_subject().CN != ex_tls: ok = False - log("TEST", "⚠ī¸", f"wrong cert CN : {x509.get_subject().CN}") + log("TEST", "⚠ī¸", f"wrong cert CN : {x509.get_subject().CN} != {ex_tls}") + return ok except: return False raise (Exception(f"unknown test type {test['type']}")) diff --git a/tests/main.py b/tests/main.py index 8678c9c83..42245bc15 100755 --- a/tests/main.py +++ b/tests/main.py @@ -3,10 +3,10 @@ from pathlib import Path from sys import path, argv, exit from glob import glob -from os import _exit +from os import _exit, getenv from os.path import isfile from traceback import format_exc -from json import loads +from json import loads, dumps from subprocess import run path.extend((f"{Path.cwd()}/utils", f"{Path.cwd()}/tests")) @@ -16,6 +16,7 @@ from AutoconfTest import AutoconfTest from SwarmTest import SwarmTest from KubernetesTest import KubernetesTest from LinuxTest import LinuxTest +from Test import Test from logger import log if len(argv) <= 1: @@ -52,6 +53,32 @@ if not ret: log("TESTS", "❌", "Test.init() failed") exit(1) +domains = {} +if test_type in ["docker", "autoconf", "linux"]: + domains = { + r"www\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1"), + r"auth\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1"), + r"app1\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1_1"), + r"app2\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1_2"), + r"app3\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1_3"), + } +elif test_type == "kubernetes": + domains = { + r"www\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1_2"), + r"auth\.example\.com": Test.random_string(1) + "." + getenv("TEST_DOMAIN1_2"), + r"app1\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1"), + r"app2\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN2"), + r"app3\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN3"), + } +elif test_type == "swarm": + domains = { + r"www\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1_1"), + r"auth\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1_2"), + r"app1\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN1"), + r"app2\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN2"), + r"app3\.example\.com": Test.random_string(6) + "." + getenv("TEST_DOMAIN3"), + } + for example in glob("./examples/*"): if isfile(f"{example}/tests.json"): try: @@ -64,6 +91,7 @@ for example in glob("./examples/*"): "Skipping tests for " + tests["name"] + " (not in kinds)", ) continue + log("TESTS", "ℹī¸", f"JSON test = {dumps(tests)}") test_obj = None no_copy_container = False delay = 0 @@ -78,6 +106,7 @@ for example in glob("./examples/*"): tests["tests"], no_copy_container=no_copy_container, delay=delay, + domains=domains, ) elif test_type == "autoconf": test_obj = AutoconfTest( @@ -86,13 +115,14 @@ for example in glob("./examples/*"): tests["tests"], no_copy_container=no_copy_container, delay=delay, + domains=domains, ) elif test_type == "swarm": - test_obj = SwarmTest(tests["name"], tests["timeout"], tests["tests"], delay=delay) + test_obj = SwarmTest(tests["name"], tests["timeout"], tests["tests"], delay=delay, domains=domains) elif test_type == "kubernetes": - test_obj = KubernetesTest(tests["name"], tests["timeout"], tests["tests"], delay=delay) + test_obj = KubernetesTest(tests["name"], tests["timeout"], tests["tests"], delay=delay, domains=domains) elif test_type == "linux": - test_obj = LinuxTest(tests["name"], tests["timeout"], tests["tests"], distro) + test_obj = LinuxTest(tests["name"], tests["timeout"], tests["tests"], distro, domains=domains) if not test_obj.run_tests(): log("TESTS", "❌", "Tests failed for " + tests["name"]) if test_type == "linux": diff --git a/tests/terraform/k8s.tf b/tests/terraform/k8s.tf index 30ba56ad0..b0e5e64cb 100644 --- a/tests/terraform/k8s.tf +++ b/tests/terraform/k8s.tf @@ -12,7 +12,7 @@ resource "scaleway_vpc_private_network" "pn" { resource "scaleway_k8s_cluster" "cluster" { type = "kapsule" name = "bw_k8s" - version = "1.24.7" + version = "1.28.2" cni = "cilium" private_network_id = scaleway_vpc_private_network.pn.id delete_additional_resources = true