Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Small tweaks on core jobs

Browse source
This commit is contained in:
Théophile Diot 2023-06-01 12:17:29 -04:00
parent 4f324231d2
commit 95d4f0f87c
No known key found for this signature in database
GPG key ID: E752C80DB72BB014
19 changed files with 76 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/common/core/blacklist/jobs/blacklist-download.py
View file

@ -145,7 +145,7 @@ try:
for url in urls_list:
try:
logger.info(f"Downloading blacklist data from {url} ...")
resp = get(url, stream=True)
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
continue

3
src/common/core/bunkernet/jobs/bunkernet-data.py
View file

@ -53,8 +53,7 @@ try:
bunkernet_tmp_path.mkdir(parents=True, exist_ok=True)
# Create empty file in case it doesn't exist
if not bunkernet_path.joinpath("ip.list").is_file():
bunkernet_path.joinpath("ip.list").write_text("")
bunkernet_path.joinpath("ip.list").touch(exist_ok=True)
# Get ID from cache
bunkernet_id = None

4
src/common/core/bunkernet/jobs/bunkernet-register.py
View file

@ -32,7 +32,7 @@ try:
bunkernet_activated = False
# Multisite case
if getenv("MULTISITE", "no") == "yes":
servers = getenv("SERVER_NAME", [])
servers = getenv("SERVER_NAME") or []
if isinstance(servers, str):
servers = servers.split(" ")
@ -110,7 +110,7 @@ try:
)
_exit(2)
bunkernet_id = data["data"]
instance_id_path.write_text(bunkernet_id)
instance_id_path.write_text(bunkernet_id, encoding="utf-8")
registered = True
exit_status = 1
logger.info(

14
src/common/core/bunkernet/jobs/bunkernet.py
View file

@ -53,13 +53,17 @@ def data() -> Tuple[bool, Optional[int], Union[str, dict]]:
def get_id() -> str:
return (
Path(sep, "var", "cache", "bunkerweb", "bunkernet", "instance.id")
.read_text()
.read_text(encoding="utf-8")
.strip()
)
def get_version() -> str:
return Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text().strip()
return (
Path(sep, "usr", "share", "bunkerweb", "VERSION")
.read_text(encoding="utf-8")
.strip()
)
def get_integration() -> str:
@ -73,8 +77,10 @@ def get_integration() -> str:
elif getenv("AUTOCONF_MODE", "no").lower() == "yes":
return "autoconf"
elif integration_path.is_file():
return integration_path.read_text().strip().lower()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text():
return integration_path.read_text(encoding="utf-8").strip().lower()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(
encoding="utf-8"
):
return "docker"
return "linux"

4
src/common/core/customcert/jobs/custom-cert.py
View file

@ -36,8 +36,8 @@ def check_cert(
)
return False
cert_path = Path(normpath(cert_path))
key_path = Path(normpath(key_path))
cert_path: Path = Path(normpath(cert_path))
key_path: Path = Path(normpath(key_path))
if not cert_path.is_file():
logger.warning(

2
src/common/core/greylist/jobs/greylist-download.py
View file

@ -129,7 +129,7 @@ try:
for url in urls_list:
try:
logger.info(f"Downloading greylist data from {url} ...")
resp = get(url, stream=True)
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
continue

6
src/common/core/jobs/jobs/download-plugins.py
View file

@ -40,7 +40,7 @@ status = 0
def install_plugin(plugin_dir) -> bool:
# Load plugin.json
metadata = loads(Path(plugin_dir, "plugin.json").read_text())
metadata = loads(Path(plugin_dir, "plugin.json").read_text(encoding="utf-8"))
# Don't go further if plugin is already installed
if Path("etc", "bunkerweb", "plugins", metadata["id"], "plugin.json").is_file():
logger.warning(
@ -71,7 +71,7 @@ try:
for plugin_url in plugin_urls.split(" "):
# Download ZIP file
try:
req = get(plugin_url)
req = get(plugin_url, timeout=10)
except:
logger.error(
f"Exception while downloading plugin(s) from {plugin_url} :\n{format_exc()}",
@ -122,7 +122,7 @@ try:
rmtree(path, ignore_errors=True)
continue
plugin_file = loads(Path(path, "plugin.json").read_text())
plugin_file = loads(Path(path, "plugin.json").read_text(encoding="utf-8"))
plugin_content = BytesIO()
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:

26
src/common/core/jobs/jobs/mmdb-asn.py
View file

@ -22,7 +22,7 @@ for deps_path in [
sys_path.append(deps_path)
from maxminddb import open_database
from requests import get
from requests import RequestException, get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
@ -41,9 +41,15 @@ try:
# Don't go further if the cache match the latest version
if tmp_path.exists():
with lock:
response = get("https://db-ip.com/db/download/ip-to-asn-lite")
response = None
try:
response = get(
"https://db-ip.com/db/download/ip-to-asn-lite", timeout=5
)
except RequestException:
logger.warning("Unable to check if asn.mmdb is the latest version")
if response.status_code == 200:
if response and response.status_code == 200:
_sha1 = sha1()
with open(str(tmp_path), "rb") as f:
while True:
@ -79,11 +85,15 @@ try:
# Download the mmdb file and save it to tmp
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = b""
with get(mmdb_url, stream=True) as resp:
resp.raise_for_status()
for chunk in resp.iter_content(chunk_size=4 * 1024):
if chunk:
file_content += chunk
try:
with get(mmdb_url, stream=True, timeout=5) as resp:
resp.raise_for_status()
for chunk in resp.iter_content(chunk_size=4 * 1024):
if chunk:
file_content += chunk
except RequestException:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
try:
assert file_content

26
src/common/core/jobs/jobs/mmdb-country.py
View file

@ -22,7 +22,7 @@ for deps_path in [
sys_path.append(deps_path)
from maxminddb import open_database
from requests import get
from requests import RequestException, get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
@ -41,9 +41,15 @@ try:
# Don't go further if the cache match the latest version
if tmp_path.exists():
with lock:
response = get("https://db-ip.com/db/download/ip-to-country-lite")
response = None
try:
response = get(
"https://db-ip.com/db/download/ip-to-country-lite", timeout=5
)
except RequestException:
logger.warning("Unable to check if country.mmdb is the latest version")
if response.status_code == 200:
if response and response.status_code == 200:
_sha1 = sha1()
with open(str(tmp_path), "rb") as f:
while True:
@ -79,11 +85,15 @@ try:
# Download the mmdb file and save it to tmp
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = b""
with get(mmdb_url, stream=True) as resp:
resp.raise_for_status()
for chunk in resp.iter_content(chunk_size=4 * 1024):
if chunk:
file_content += chunk
try:
with get(mmdb_url, stream=True, timeout=5) as resp:
resp.raise_for_status()
for chunk in resp.iter_content(chunk_size=4 * 1024):
if chunk:
file_content += chunk
except RequestException:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
try:
assert file_content

4
src/common/core/letsencrypt/jobs/certbot-auth.py
View file

@ -37,7 +37,7 @@ try:
elif getenv("AUTOCONF_MODE") == "yes":
bw_integration = "Autoconf"
elif integration_path.is_file():
integration = integration_path.read_text().strip()
integration = integration_path.read_text(encoding="utf-8").strip()
token = getenv("CERTBOT_TOKEN", "")
validation = getenv("CERTBOT_VALIDATION", "")
@ -89,7 +89,7 @@ try:
"acme-challenge",
)
root_dir.mkdir(parents=True, exist_ok=True)
root_dir.joinpath(token).write_text(validation)
root_dir.joinpath(token).write_text(validation, encoding="utf-8")
except:
status = 1
logger.error(f"Exception while running certbot-auth.py :\n{format_exc()}")

2
src/common/core/letsencrypt/jobs/certbot-cleanup.py
View file

@ -37,7 +37,7 @@ try:
elif getenv("AUTOCONF_MODE") == "yes":
bw_integration = "Autoconf"
elif integration_path.is_file():
integration = integration_path.read_text().strip()
integration = integration_path.read_text(encoding="utf-8").strip()
token = getenv("CERTBOT_TOKEN", "")
# Cluster case

3
src/common/core/letsencrypt/jobs/certbot-deploy.py
View file

@ -40,7 +40,7 @@ try:
elif getenv("AUTOCONF_MODE") == "yes":
bw_integration = "Autoconf"
elif integration_path.is_file():
integration = integration_path.read_text().strip()
integration = integration_path.read_text(encoding="utf-8").strip()
token = getenv("CERTBOT_TOKEN", "")
logger.info(f"Certificates renewal for {getenv('RENEWED_DOMAINS')} successful")
@ -111,6 +111,7 @@ try:
["sudo", join(sep, "usr", "sbin", "nginx"), "-s", "reload"],
stdin=DEVNULL,
stderr=STDOUT,
check=False,
).returncode
!= 0
):

3
src/common/core/letsencrypt/jobs/certbot-new.py
View file

@ -60,6 +60,7 @@ def certbot_new(
stderr=STDOUT,
env=environ.copy()
| {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
check=True,
).returncode
@ -190,7 +191,7 @@ try:
bio.seek(0, 0)
# Put tgz in cache
cached, err = set_file_in_db(f"folder.tgz", bio.read(), db)
cached, err = set_file_in_db("folder.tgz", bio.read(), db)
if not cached:
logger.error(f"Error while saving Let's Encrypt data to db cache : {err}")

5
src/common/core/letsencrypt/jobs/certbot-renew.py
View file

@ -54,6 +54,7 @@ def renew(domain: str, letsencrypt_path: Path) -> int:
stdin=DEVNULL,
stderr=STDOUT,
env=environ,
check=False,
).returncode
@ -101,8 +102,8 @@ try:
else:
logger.info("No Let's Encrypt data found in db cache")
if getenv("MULTISITE") == "yes":
servers = getenv("SERVER_NAME", [])
if getenv("MULTISITE", "no") == "yes":
servers = getenv("SERVER_NAME") or []
if isinstance(servers, str):
servers = servers.split(" ")

1
src/common/core/misc/jobs/default-server-cert.py
View file

@ -85,6 +85,7 @@ try:
],
stdin=DEVNULL,
stderr=DEVNULL,
check=False,
).returncode
!= 0
):

5
src/common/core/misc/jobs/update-check.py
View file

@ -23,11 +23,14 @@ logger = setup_logger("UPDATE-CHECK", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
current_version = f"v{Path('/usr/share/bunkerweb/VERSION').read_text().strip()}"
current_version = (
f"v{Path('/usr/share/bunkerweb/VERSION').read_text(encoding='utf-8').strip()}"
)
response = get(
"https://github.com/bunkerity/bunkerweb/releases/latest",
allow_redirects=True,
timeout=5,
)
response.raise_for_status()

2
src/common/core/realip/jobs/realip-download.py
View file

@ -92,7 +92,7 @@ try:
for url in urls:
try:
logger.info(f"Downloading RealIP list from {url} ...")
resp = get(url, stream=True)
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
continue

4
src/common/core/selfsigned/jobs/self-signed.py
View file

@ -47,6 +47,7 @@ def generate_cert(
],
stdin=DEVNULL,
stderr=STDOUT,
check=False,
).returncode
== 0
):
@ -74,6 +75,7 @@ def generate_cert(
],
stdin=DEVNULL,
stderr=DEVNULL,
check=False,
).returncode
!= 0
):
@ -111,7 +113,7 @@ try:
# Multisite case
if getenv("MULTISITE") == "yes":
servers = getenv("SERVER_NAME", [])
servers = getenv("SERVER_NAME") or []
if isinstance(servers, str):
servers = servers.split(" ")

2
src/common/core/whitelist/jobs/whitelist-download.py
View file

@ -129,7 +129,7 @@ try:
for url in urls_list:
try:
logger.info(f"Downloading whitelist data from {url} ...")
resp = get(url, stream=True)
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
continue