diff --git a/src/common/core/blacklist/jobs/blacklist-download.py b/src/common/core/blacklist/jobs/blacklist-download.py index d9ef1605b..8517d0a59 100644 --- a/src/common/core/blacklist/jobs/blacklist-download.py +++ b/src/common/core/blacklist/jobs/blacklist-download.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta from ipaddress import ip_address, ip_network from os import getenv, sep from os.path import join, normpath -from pathlib import Path from re import compile as re_compile from sys import exit as sys_exit, path as sys_path from typing import Tuple @@ -102,11 +101,16 @@ try: if not any(url for urls in services_blacklist_urls.values() for url in urls.values()): LOGGER.warning("No blacklist URL is configured, nothing to do...") - if Path(JOB.job_path.joinpath("urls.json")).exists(): - LOGGER.warning("Blacklist URLs are cached but no URL is configured, removing from cache...") - deleted, err = JOB.del_cache("urls.json") + for file in JOB.job_path.rglob("*.list"): + if file.parent == JOB.job_path: + LOGGER.warning(f"Removing no longer used url file {file} ...") + deleted, err = JOB.del_cache(file) + else: + LOGGER.warning(f"Removing no longer used service file {file} ...") + deleted, err = JOB.del_cache(file, service_id=file.parent.name) + if not deleted: - LOGGER.warning(f"Couldn't delete blacklist URLs from cache : {err}") + LOGGER.warning(f"Couldn't delete file {file} from cache : {err}") sys_exit(0) urls = set() @@ -116,7 +120,7 @@ try: for service, kinds in services_blacklist_urls.items(): for kind, urls_list in kinds.items(): if not urls_list: - if JOB.job_path.joinpath(service, f"{kind}.list").exists(): + if JOB.job_path.joinpath(service, f"{kind}.list").is_file(): LOGGER.warning(f"{service} blacklist for {kind} is cached but no URL is configured, removing from cache...") deleted, err = JOB.del_cache(f"{kind}.list", service_id=service) if not deleted: diff --git a/src/common/core/greylist/jobs/greylist-download.py b/src/common/core/greylist/jobs/greylist-download.py index f31d6490f..f3dc3c3c7 100644 --- a/src/common/core/greylist/jobs/greylist-download.py +++ b/src/common/core/greylist/jobs/greylist-download.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta from ipaddress import ip_address, ip_network from os import getenv, sep from os.path import join, normpath -from pathlib import Path from re import compile as re_compile from sys import exit as sys_exit, path as sys_path from typing import Tuple @@ -102,11 +101,16 @@ try: if not any(url for urls in services_greylist_urls.values() for url in urls.values()): LOGGER.warning("No greylist URL is configured, nothing to do...") - if Path(JOB.job_path.joinpath("urls.json")).exists(): - LOGGER.warning("Greylist URLs are cached but no URL is configured, removing from cache...") - deleted, err = JOB.del_cache("urls.json") + for file in JOB.job_path.rglob("*.list"): + if file.parent == JOB.job_path: + LOGGER.warning(f"Removing no longer used url file {file} ...") + deleted, err = JOB.del_cache(file) + else: + LOGGER.warning(f"Removing no longer used service file {file} ...") + deleted, err = JOB.del_cache(file, service_id=file.parent.name) + if not deleted: - LOGGER.warning(f"Couldn't delete greylist URLs from cache : {err}") + LOGGER.warning(f"Couldn't delete file {file} from cache : {err}") sys_exit(0) urls = set() @@ -116,7 +120,7 @@ try: for service, kinds in services_greylist_urls.items(): for kind, urls_list in kinds.items(): if not urls_list: - if JOB.job_path.joinpath(service, f"{kind}.list").exists(): + if JOB.job_path.joinpath(service, f"{kind}.list").is_file(): LOGGER.warning(f"{service} greylist for {kind} is cached but no URL is configured, removing from cache...") deleted, err = JOB.del_cache(f"{kind}.list", service_id=service) if not deleted: diff --git a/src/common/core/realip/jobs/realip-download.py b/src/common/core/realip/jobs/realip-download.py index 34f98664f..f681fb1e3 100644 --- a/src/common/core/realip/jobs/realip-download.py +++ b/src/common/core/realip/jobs/realip-download.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta from ipaddress import ip_address, ip_network from os import getenv, sep from os.path import join, normpath -from pathlib import Path from sys import exit as sys_exit, path as sys_path for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: @@ -79,11 +78,16 @@ try: if not any(services_realip_urls.values()): LOGGER.warning("No URL configured, nothing to do...") - if Path(JOB.job_path.joinpath("urls.json")).exists(): - LOGGER.warning("RealIP URLs are cached but no URL is configured, removing from cache...") - deleted, err = JOB.del_cache("urls.json") + for file in JOB.job_path.rglob("*.list"): + if file.parent == JOB.job_path: + LOGGER.warning(f"Removing no longer used url file {file} ...") + deleted, err = JOB.del_cache(file) + else: + LOGGER.warning(f"Removing no longer used service file {file} ...") + deleted, err = JOB.del_cache(file, service_id=file.parent.name) + if not deleted: - LOGGER.warning(f"Couldn't delete realip URLs from cache : {err}") + LOGGER.warning(f"Couldn't delete file {file} from cache : {err}") sys_exit(0) urls = set() @@ -91,7 +95,7 @@ try: for service, urls in services_realip_urls.items(): if not urls: - if JOB.job_path.joinpath(service, "combined.list").exists(): + if JOB.job_path.joinpath(service, "combined.list").is_file(): LOGGER.warning(f"{service} realip combined.list is cached but no URL is configured, removing from cache...") deleted, err = JOB.del_cache("combined.list", service_id=service) if not deleted: diff --git a/src/common/core/whitelist/jobs/whitelist-download.py b/src/common/core/whitelist/jobs/whitelist-download.py index b84bd93ad..a0386ca0c 100644 --- a/src/common/core/whitelist/jobs/whitelist-download.py +++ b/src/common/core/whitelist/jobs/whitelist-download.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta from ipaddress import ip_address, ip_network from os import getenv, sep from os.path import join, normpath -from pathlib import Path from re import compile as re_compile from sys import exit as sys_exit, path as sys_path from typing import Tuple @@ -102,11 +101,16 @@ try: if not any(url for urls in services_whitelist_urls.values() for url in urls.values()): LOGGER.warning("No whitelist URL is configured, nothing to do...") - if Path(JOB.job_path.joinpath("urls.json")).exists(): - LOGGER.warning("Whitelist URLs are cached but no URL is configured, removing from cache...") - deleted, err = JOB.del_cache("urls.json") + for file in JOB.job_path.rglob("*.list"): + if file.parent == JOB.job_path: + LOGGER.warning(f"Removing no longer used url file {file} ...") + deleted, err = JOB.del_cache(file) + else: + LOGGER.warning(f"Removing no longer used service file {file} ...") + deleted, err = JOB.del_cache(file, service_id=file.parent.name) + if not deleted: - LOGGER.warning(f"Couldn't delete whitelist URLs from cache : {err}") + LOGGER.warning(f"Couldn't delete file {file} from cache : {err}") sys_exit(0) urls = set() @@ -116,7 +120,7 @@ try: for service, kinds in services_whitelist_urls.items(): for kind, urls_list in kinds.items(): if not urls_list: - if JOB.job_path.joinpath(service, f"{kind}.list").exists(): + if JOB.job_path.joinpath(service, f"{kind}.list").is_file(): LOGGER.warning(f"{service} whitelist for {kind} is cached but no URL is configured, removing from cache...") deleted, err = JOB.del_cache(f"{kind}.list", service_id=service) if not deleted: