Merge branch 'dev' into staging

This commit is contained in:
florian 2024-04-09 11:50:08 +02:00
commit dba057f010
No known key found for this signature in database
GPG key ID: 93EE47CC3D061500
11 changed files with 330 additions and 159 deletions

View file

@ -2,17 +2,16 @@
## v1.5.7 - ????/??/??
- [LINUX] Fix potential issues when removing the bunkerweb package
- [BUGFIX] Fix rare error when the cache is not properly initialized and jobs are executed
- [FEATURE] Add an automatic renaming of old database tables when upgrading to a new version in order to avoid data loss
- [FEATURE] Add the possibility to add custom bwcli commands in plugins
- [BUGFIX] Fix bug when downloading new mmdb files
- [BUGFIX] Remove potential false positives with ModSecurity on the jobs page of the web UI
- [BUGFIX] Fix bwcli not working with Redis sentinel
- [BUGFIX] Fix potential issues when removing the bunkerweb Linux package
- [FEATURE] Add backup plugin to backup and restore easily the database
- [FEATURE] Add LETS_ENCRYPT_CLEAR_OLD_CERTS setting to control if old certificates should be removed when generating Let's Encrypt certificates (default is no)
- [FEATURE] Add DISABLE_DEFAULT_SERVER_STRICT_SNI setting to allow/block requests when SNI is unknown or unset (default is no)
- [MISC] Remove potential false positives with ModSecurity on the jobs page of the web UI
- [MISC] Fix rare bug when downloading new mmdb files
- [DOCUMENTATION] Add procedure to follow when upgrading from 1.5.7+
- [DOCUMENTATION] Add documentation about the procedure to follow when upgrading from a version prior to 1.5.0
- [DOCUMENTATION] Add upgrade procedure for 1.5.7+
- [MISC] Support custom bwcli commands using plugins
- [DEPS] Updated LuaJIT version to v2.1-20240314
## v1.5.6 - 2024/03/25

View file

@ -8,9 +8,9 @@ ansible==9.2.0 \
--hash=sha256:39b19c252800aeed531413a626ccd07473b79615a3cea77568a1624c1aefaf7c \
--hash=sha256:a207a4a00a45e5cd178a7f94ca42afe26f23c9d27be49901ea8c45d18a07b7c6
# via -r requirements-ansible.in
ansible-core==2.16.3 \
--hash=sha256:50c9f33a5b2ee645470a77f4bf99cf35d1ffdefef60388910020b0c58534bec1 \
--hash=sha256:76a8765a8586064ef073a299562e308fa2c180a75b5f7569bbd0f61d4171cdb3
ansible-core==2.16.5 \
--hash=sha256:371b0bb11d109a58982684307c18cc44ff8d408b1b3350c0c5c78d9f096ee1f1 \
--hash=sha256:cdd29b0ec3f20c35657355a2f6a9c1d0cf1131da99cc9a4a3401801b0ab36d6d
# via ansible
cffi==1.16.0 \
--hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \
@ -66,39 +66,39 @@ cffi==1.16.0 \
--hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \
--hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357
# via cryptography
cryptography==42.0.2 \
--hash=sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380 \
--hash=sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589 \
--hash=sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea \
--hash=sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65 \
--hash=sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a \
--hash=sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3 \
--hash=sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008 \
--hash=sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1 \
--hash=sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2 \
--hash=sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635 \
--hash=sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2 \
--hash=sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90 \
--hash=sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee \
--hash=sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a \
--hash=sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242 \
--hash=sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12 \
--hash=sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2 \
--hash=sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d \
--hash=sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be \
--hash=sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee \
--hash=sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6 \
--hash=sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529 \
--hash=sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929 \
--hash=sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1 \
--hash=sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6 \
--hash=sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a \
--hash=sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446 \
--hash=sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9 \
--hash=sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888 \
--hash=sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4 \
--hash=sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33 \
--hash=sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f
cryptography==42.0.5 \
--hash=sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee \
--hash=sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576 \
--hash=sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d \
--hash=sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30 \
--hash=sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413 \
--hash=sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb \
--hash=sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da \
--hash=sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4 \
--hash=sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd \
--hash=sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc \
--hash=sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8 \
--hash=sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1 \
--hash=sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc \
--hash=sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e \
--hash=sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8 \
--hash=sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940 \
--hash=sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400 \
--hash=sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7 \
--hash=sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16 \
--hash=sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278 \
--hash=sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74 \
--hash=sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec \
--hash=sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1 \
--hash=sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2 \
--hash=sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c \
--hash=sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922 \
--hash=sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a \
--hash=sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6 \
--hash=sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1 \
--hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \
--hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \
--hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7
# via ansible-core
jinja2==3.1.3 \
--hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \
@ -166,13 +166,13 @@ markupsafe==2.1.5 \
--hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \
--hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68
# via jinja2
packaging==23.2 \
--hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \
--hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7
packaging==24.0 \
--hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \
--hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9
# via ansible-core
pycparser==2.21 \
--hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
--hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
pycparser==2.22 \
--hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
--hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc
# via cffi
pyyaml==6.0.1 \
--hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \

View file

@ -77,7 +77,8 @@ class CLI(ApiCaller):
if self.__use_redis:
self.__logger.info("Fetching redis configuration")
redis_host = self.__get_variable("REDIS_HOST")
if redis_host:
sentinel_hosts = self.__get_variable("REDIS_SENTINEL_HOSTS")
if redis_host or sentinel_hosts:
redis_port = self.__get_variable("REDIS_PORT", "6379")
assert isinstance(redis_port, str), "REDIS_PORT is not a string"
if not redis_port.isdigit():
@ -107,8 +108,6 @@ class CLI(ApiCaller):
redis_keepalive_pool = "10"
redis_keepalive_pool = int(redis_keepalive_pool)
self.__logger.info("Redis configuration is valid")
redis_ssl = self.__get_variable("REDIS_SSL", "no") == "yes"
username = self.__get_variable("REDIS_USERNAME", None) or None
password = self.__get_variable("REDIS_PASSWORD", None) or None
@ -177,7 +176,7 @@ class CLI(ApiCaller):
self.__use_redis = False
self.__logger.info("Connected to redis")
else:
self.__logger.error("USE_REDIS is set to yes but REDIS_HOST is not set, disabling redis")
self.__logger.error("USE_REDIS is set to yes but REDIS_HOST or REDIS_SENTINEL_HOSTS is not set, disabling redis")
self.__use_redis = False
if self.__integration == "linux":

View file

@ -197,6 +197,8 @@ class Configurator:
"NJS_VERSION",
"PKG_RELEASE",
"DOCKER_HOST",
"SLAVE_MODE",
"MASTER_MODE",
)
):
self.__logger.warning(f"Ignoring variable {variable} : {err}")

View file

@ -316,5 +316,14 @@
"alert",
"emerg"
]
},
"OVERRIDE_INSTANCES": {
"context": "global",
"default": "",
"help": "List of BunkerWeb instances separated with spaces (format : fqdn-or-ip:5000 fqdn-or-ip:5000)",
"id": "override-instances",
"label": "Override instances",
"regex": "^.*$",
"type": "text"
}
}

View file

@ -138,7 +138,7 @@ class ApiCaller:
else:
responses[instance] = resp.json()
if response and responses:
if response:
return ret, responses
return ret

View file

@ -100,98 +100,104 @@ function start() {
log "SYSTEMCTL" "" "Created dummy variables.env file"
fi
# Create PID folder
if [ ! -f /var/run/bunkerweb ] ; then
mkdir -p /var/run/bunkerweb
chown nginx:nginx /var/run/bunkerweb
fi
# Stop scheduler if it's running
stop_scheduler
# Stop nginx if it's running
stop_nginx
# Generate temp conf for jobs and start nginx
DNS_RESOLVERS="$(grep "^DNS_RESOLVERS=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$DNS_RESOLVERS" = "" ] ; then
DNS_RESOLVERS="8.8.8.8 8.8.4.4"
fi
API_LISTEN_IP="$(grep "^API_LISTEN_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$API_LISTEN_IP" = "" ] ; then
API_LISTEN_IP="127.0.0.1"
fi
API_HTTP_PORT="$(grep "^API_HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$API_HTTP_PORT" = "" ] ; then
API_HTTP_PORT="5000"
fi
API_SERVER_NAME="$(grep "^API_SERVER_NAME=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$API_SERVER_NAME" = "" ] ; then
API_SERVER_NAME="bwapi"
fi
API_WHITELIST_IP="$(grep "^API_WHITELIST_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$API_WHITELIST_IP" = "" ] ; then
API_WHITELIST_IP="127.0.0.0/8"
fi
USE_REAL_IP="$(grep "^USE_REAL_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$USE_REAL_IP" = "" ] ; then
USE_REAL_IP="no"
fi
USE_PROXY_PROTOCOL="$(grep "^USE_PROXY_PROTOCOL=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$USE_PROXY_PROTOCOL" = "" ] ; then
USE_PROXY_PROTOCOL="no"
fi
REAL_IP_FROM="$(grep "^REAL_IP_FROM=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$REAL_IP_FROM" = "" ] ; then
REAL_IP_FROM="192.168.0.0/16 172.16.0.0/12 10.0.0.0/8"
fi
REAL_IP_HEADER="$(grep "^REAL_IP_HEADER=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$REAL_IP_HEADER" = "" ] ; then
REAL_IP_HEADER="X-Forwarded-For"
fi
HTTP_PORT="$(grep "^HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$HTTP_PORT" = "" ] ; then
HTTP_PORT="80"
fi
HTTPS_PORT="$(grep "^HTTPS_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$HTTPS_PORT" = "" ] ; then
HTTPS_PORT="443"
fi
MODSECURITY_CRS_VERSION="$(grep "^MODSECURITY_CRS_VERSION=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$MODSECURITY_CRS_VERSION" = "" ] ; then
MODSECURITY_CRS_VERSION="4"
fi
sudo -E -u nginx -g nginx /bin/bash -c "echo -ne 'IS_LOADING=yes\nUSE_BUNKERNET=no\nSEND_ANONYMOUS_REPORT=no\nSERVER_NAME=\nMODSECURITY_CRS_VERSION=${MODSECURITY_CRS_VERSION}\nDNS_RESOLVERS=${DNS_RESOLVERS}\nAPI_HTTP_PORT=${API_HTTP_PORT}\nAPI_LISTEN_IP=${API_LISTEN_IP}\nAPI_SERVER_NAME=${API_SERVER_NAME}\nAPI_WHITELIST_IP=${API_WHITELIST_IP}\nUSE_REAL_IP=${USE_REAL_IP}\nUSE_PROXY_PROTOCOL=${USE_PROXY_PROTOCOL}\nREAL_IP_FROM=${REAL_IP_FROM}\nREAL_IP_HEADER=${REAL_IP_HEADER}\nHTTP_PORT=${HTTP_PORT}\nHTTPS_PORT=${HTTPS_PORT}\n' > /var/tmp/bunkerweb/tmp.env"
sudo -E -u nginx -g nginx /bin/bash -c "PYTHONPATH=/usr/share/bunkerweb/deps/python/ /usr/share/bunkerweb/gen/main.py --variables /var/tmp/bunkerweb/tmp.env --no-linux-reload"
# shellcheck disable=SC2181
if [ $? -ne 0 ] ; then
log "SYSTEMCTL" "❌" "Error while generating config from /var/tmp/bunkerweb/tmp.env"
exit 1
fi
# Check if we are in slave/master mode
export MASTER_MODE="$(grep "^MASTER_MODE=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
export SLAVE_MODE="$(grep "^SLAVE_MODE=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ ! -f /var/run/bunkerweb ] ; then
mkdir -p /var/run/bunkerweb
chown nginx:nginx /var/run/bunkerweb
fi
# Start nginx
log "SYSTEMCTL" "" "Starting nginx ..."
sudo -E -u nginx -g nginx /usr/sbin/nginx -e /var/log/bunkerweb/error.log
# shellcheck disable=SC2181
if [ $? -ne 0 ] ; then
log "SYSTEMCTL" "❌" "Error while executing temp nginx"
exit 1
fi
count=0
while [ $count -lt 10 ] ; do
check="$(curl -s -H "Host: healthcheck.bunkerweb.io" http://127.0.0.1:6000/healthz 2>&1)"
# shellcheck disable=SC2181
if [ $? -eq 0 ] && [ "$check" = "ok" ] ; then
break
if [ "$MASTER_MODE" != "yes" ] ; then
# Generate temp conf for jobs and start nginx
DNS_RESOLVERS="$(grep "^DNS_RESOLVERS=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$DNS_RESOLVERS" = "" ] ; then
DNS_RESOLVERS="8.8.8.8 8.8.4.4"
fi
count=$((count + 1))
sleep 1
log "SYSTEMCTL" "" "Waiting for nginx to start ..."
done
if [ $count -ge 10 ] ; then
log "SYSTEMCTL" "❌" "nginx is not started"
exit 1
API_LISTEN_IP="$(grep "^API_LISTEN_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$API_LISTEN_IP" = "" ] ; then
API_LISTEN_IP="127.0.0.1"
fi
API_HTTP_PORT="$(grep "^API_HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$API_HTTP_PORT" = "" ] ; then
API_HTTP_PORT="5000"
fi
API_SERVER_NAME="$(grep "^API_SERVER_NAME=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$API_SERVER_NAME" = "" ] ; then
API_SERVER_NAME="bwapi"
fi
API_WHITELIST_IP="$(grep "^API_WHITELIST_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$API_WHITELIST_IP" = "" ] ; then
API_WHITELIST_IP="127.0.0.0/8"
fi
USE_REAL_IP="$(grep "^USE_REAL_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$USE_REAL_IP" = "" ] ; then
USE_REAL_IP="no"
fi
USE_PROXY_PROTOCOL="$(grep "^USE_PROXY_PROTOCOL=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$USE_PROXY_PROTOCOL" = "" ] ; then
USE_PROXY_PROTOCOL="no"
fi
REAL_IP_FROM="$(grep "^REAL_IP_FROM=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$REAL_IP_FROM" = "" ] ; then
REAL_IP_FROM="192.168.0.0/16 172.16.0.0/12 10.0.0.0/8"
fi
REAL_IP_HEADER="$(grep "^REAL_IP_HEADER=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$REAL_IP_HEADER" = "" ] ; then
REAL_IP_HEADER="X-Forwarded-For"
fi
HTTP_PORT="$(grep "^HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$HTTP_PORT" = "" ] ; then
HTTP_PORT="80"
fi
HTTPS_PORT="$(grep "^HTTPS_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$HTTPS_PORT" = "" ] ; then
HTTPS_PORT="443"
fi
MODSECURITY_CRS_VERSION="$(grep "^MODSECURITY_CRS_VERSION=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)"
if [ "$MODSECURITY_CRS_VERSION" = "" ] ; then
MODSECURITY_CRS_VERSION="3"
fi
sudo -E -u nginx -g nginx /bin/bash -c "echo -ne 'IS_LOADING=yes\nUSE_BUNKERNET=no\nSEND_ANONYMOUS_REPORT=no\nSERVER_NAME=\nMODSECURITY_CRS_VERSION=${MODSECURITY_CRS_VERSION}\nDNS_RESOLVERS=${DNS_RESOLVERS}\nAPI_HTTP_PORT=${API_HTTP_PORT}\nAPI_LISTEN_IP=${API_LISTEN_IP}\nAPI_SERVER_NAME=${API_SERVER_NAME}\nAPI_WHITELIST_IP=${API_WHITELIST_IP}\nUSE_REAL_IP=${USE_REAL_IP}\nUSE_PROXY_PROTOCOL=${USE_PROXY_PROTOCOL}\nREAL_IP_FROM=${REAL_IP_FROM}\nREAL_IP_HEADER=${REAL_IP_HEADER}\nHTTP_PORT=${HTTP_PORT}\nHTTPS_PORT=${HTTPS_PORT}\n' > /var/tmp/bunkerweb/tmp.env"
sudo -E -u nginx -g nginx /bin/bash -c "PYTHONPATH=/usr/share/bunkerweb/deps/python/ /usr/share/bunkerweb/gen/main.py --variables /var/tmp/bunkerweb/tmp.env --no-linux-reload"
# shellcheck disable=SC2181
if [ $? -ne 0 ] ; then
log "SYSTEMCTL" "❌" "Error while generating config from /var/tmp/bunkerweb/tmp.env"
exit 1
fi
# Start nginx
log "SYSTEMCTL" "" "Starting nginx ..."
sudo -E -u nginx -g nginx /usr/sbin/nginx -e /var/log/bunkerweb/error.log
# shellcheck disable=SC2181
if [ $? -ne 0 ] ; then
log "SYSTEMCTL" "❌" "Error while executing temp nginx"
exit 1
fi
count=0
while [ $count -lt 10 ] ; do
check="$(curl -s -H "Host: healthcheck.bunkerweb.io" http://127.0.0.1:6000/healthz 2>&1)"
# shellcheck disable=SC2181
if [ $? -eq 0 ] && [ "$check" = "ok" ] ; then
break
fi
count=$((count + 1))
sleep 1
log "SYSTEMCTL" "" "Waiting for nginx to start ..."
done
if [ $count -ge 10 ] ; then
log "SYSTEMCTL" "❌" "nginx is not started"
exit 1
fi
log "SYSTEMCTL" "" "nginx started ..."
fi
log "SYSTEMCTL" "" "nginx started ..."
# Execute scheduler
log "SYSTEMCTL" " " "Executing scheduler ..."

View file

@ -30,6 +30,7 @@ from common_utils import bytes_hash, dict_to_frozenset, get_integration # type:
from logger import setup_logger # type: ignore
from Database import Database # type: ignore
from JobScheduler import JobScheduler
from API import API
RUN = True
SCHEDULER: Optional[JobScheduler] = None
@ -70,6 +71,9 @@ SCHEDULER_TMP_ENV_PATH.touch()
DB_LOCK_FILE = Path(sep, "var", "lib", "bunkerweb", "db.lock")
logger = setup_logger("Scheduler", getenv("LOG_LEVEL", "INFO"))
SLAVE_MODE = environ.get("SLAVE_MODE", "no") == "yes"
MASTER_MODE = environ.get("MASTER_MODE", "no") == "yes"
def handle_stop(signum, frame):
if SCHEDULER is not None:
@ -198,6 +202,53 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U
if not ret:
logger.error(f"Sending {'pro ' if pro else ''}external plugins failed, configuration will not work as expected...")
def generate_caches(plugins: List[Any], db: Database):
for plugin in plugins:
job_cache_files = db.get_jobs_cache_files(plugin_id=plugin["id"])
plugin_cache_files = set()
ignored_dirs = set()
job_path = Path(sep, "var", "cache", "bunkerweb", plugin["id"])
for job_cache_file in job_cache_files:
cache_path = job_path.joinpath(job_cache_file["service_id"] or "", job_cache_file["file_name"])
plugin_cache_files.add(cache_path)
try:
if job_cache_file["file_name"].endswith(".tgz"):
extract_path = cache_path.parent
if job_cache_file["file_name"].startswith("folder:"):
extract_path = Path(job_cache_file["file_name"].split("folder:", 1)[1].rsplit(".tgz", 1)[0])
ignored_dirs.add(extract_path.as_posix())
rmtree(extract_path, ignore_errors=True)
extract_path.mkdir(parents=True, exist_ok=True)
with tar_open(fileobj=BytesIO(job_cache_file["data"]), mode="r:gz") as tar:
try:
tar.extractall(extract_path, filter="fully_trusted")
except TypeError:
tar.extractall(extract_path)
else:
cache_path.parent.mkdir(parents=True, exist_ok=True)
cache_path.write_bytes(job_cache_file["data"])
except BaseException as e:
logger.error(f"Exception while restoring cache file {job_cache_file['file_name']} :\n{e}")
if job_path.is_dir():
for file in job_path.rglob("*"):
skipped = False
if file.as_posix().startswith(tuple(ignored_dirs)):
skipped = True
if skipped:
continue
logger.debug(f"Checking if {file} should be removed")
if file not in plugin_cache_files and file.is_file():
logger.debug(f"Removing non-cached file {file}")
file.unlink(missing_ok=True)
if file.parent.is_dir() and not list(file.parent.iterdir()):
logger.debug(f"Removing empty directory {file.parent}")
rmtree(file.parent, ignore_errors=True)
if file.parent == job_path:
break
elif file.is_dir() and not list(file.iterdir()):
logger.debug(f"Removing empty directory {file}")
rmtree(file, ignore_errors=True)
def api_to_instance(api):
hostname_port = api.endpoint.replace("http://", "").replace("https://", "").replace("/", "").split(":")
@ -206,6 +257,62 @@ def api_to_instance(api):
"env": {"API_HTTP_PORT": int(hostname_port[1]), "API_SERVER_NAME": api.host},
}
def run_in_slave_mode(db: Database, dotenv_env: Dict[str, Any]):
# Instantiate db
db = Database(logger, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None)))
# Wait for init
while not db.is_initialized():
logger.warning("Database is not initialized, retrying in 5s ...")
sleep(5)
# Wait for first config
env = db.get_config()
while not db.is_first_config_saved() or not env:
logger.warning("Database doesn't have any config saved yet, retrying in 5s ...")
sleep(5)
env = db.get_config()
# Download plugins
pro_plugins = db.get_plugins(_type="pro", with_data=True)
generate_external_plugins(pro_plugins, original_path=PRO_PLUGINS_PATH)
external_plugins = db.get_plugins(_type="external", with_data=True)
generate_external_plugins(external_plugins)
# Download custom configs
generate_custom_configs(db.get_custom_configs())
# Download caches
generate_caches(pro_plugins + external_plugins, db)
# Gen config
content = ""
for k, v in env.items():
content += f"{k}={v}\n"
SCHEDULER_TMP_ENV_PATH.write_text(content)
proc = subprocess_run(
[
"python3",
join(sep, "usr", "share", "bunkerweb", "gen", "main.py"),
"--settings",
join(sep, "usr", "share", "bunkerweb", "settings.json"),
"--templates",
join(sep, "usr", "share", "bunkerweb", "confs"),
"--output",
join(sep, "etc", "nginx"),
"--variables",
str(SCHEDULER_TMP_ENV_PATH),
],
stdin=DEVNULL,
stderr=STDOUT,
check=False,
)
if proc.returncode != 0:
logger.error("Config generator failed, configuration will not work as expected...")
# TODO : check nginx status + check DB status
while True:
sleep(5)
if __name__ == "__main__":
try:
@ -232,6 +339,10 @@ if __name__ == "__main__":
db = Database(logger, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None)))
if SLAVE_MODE:
run_in_slave_mode(db, dotenv_env)
stop(1)
if INTEGRATION in ("Swarm", "Kubernetes", "Autoconf"):
while not db.is_initialized():
logger.warning("Database is not initialized, retrying in 5s ...")
@ -280,8 +391,15 @@ if __name__ == "__main__":
env["DATABASE_URI"] = db.database_uri
# Override instances if needed
override_instances = env.get("OVERRIDE_INSTANCES", "")
apis=[]
if override_instances:
for instance in override_instances.split(" "):
apis.append(API(instance))
# Instantiate scheduler
SCHEDULER = JobScheduler(env | environ, logger, INTEGRATION, db=db)
SCHEDULER = JobScheduler(env | environ, logger, INTEGRATION, db=db, apis=apis)
if INTEGRATION in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
# Automatically setup the scheduler apis
@ -462,7 +580,7 @@ if __name__ == "__main__":
if event["Action"] == "start":
db.checked_changes(value=True)
if INTEGRATION == "Docker":
if INTEGRATION == "Docker" and not override_instances:
Thread(target=listen_for_instances_reload, args=(db,), name="listen_for_instances_reload").start()
while True:
@ -490,19 +608,22 @@ if __name__ == "__main__":
content += f"{k}={v}\n"
SCHEDULER_TMP_ENV_PATH.write_text(content)
# run the generator
args = [
"python3",
join(sep, "usr", "share", "bunkerweb", "gen", "main.py"),
"--settings",
join(sep, "usr", "share", "bunkerweb", "settings.json"),
"--templates",
join(sep, "usr", "share", "bunkerweb", "confs"),
"--output",
join(sep, "etc", "nginx"),
"--variables",
str(SCHEDULER_TMP_ENV_PATH),
]
if MASTER_MODE:
args.append("--no-linux-reload")
proc = subprocess_run(
[
"python3",
join(sep, "usr", "share", "bunkerweb", "gen", "main.py"),
"--settings",
join(sep, "usr", "share", "bunkerweb", "settings.json"),
"--templates",
join(sep, "usr", "share", "bunkerweb", "confs"),
"--output",
join(sep, "etc", "nginx"),
"--variables",
str(SCHEDULER_TMP_ENV_PATH),
],
args,
stdin=DEVNULL,
stderr=STDOUT,
check=False,

View file

@ -194,7 +194,7 @@ bw_version = Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text(encodin
try:
app.config.update(
DEBUG=True,
INSTANCES=Instances(docker_client, kubernetes_client, INTEGRATION),
INSTANCES=Instances(docker_client, kubernetes_client, INTEGRATION, db),
CONFIG=Config(db),
CONFIGFILES=ConfigFiles(),
WTF_CSRF_SSL_STRICT=False,
@ -634,8 +634,10 @@ def home():
if r and r.status_code == 200:
remote_version = basename(r.url).strip().replace("v", "")
instances = app.config["INSTANCES"].get_instances()
config = app.config["CONFIG"].get_config(with_drafts=True)
override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
instance_health_count = 0
for instance in instances:
@ -848,7 +850,9 @@ def instances():
)
# Display instances
instances = app.config["INSTANCES"].get_instances()
config = app.config["CONFIG"].get_config()
override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
return render_template("instances.html", title="Instances", instances=instances, username=current_user.get_id())
@ -1664,7 +1668,10 @@ def cache():
@app.route("/logs", methods=["GET"])
@login_required
def logs():
return render_template("logs.html", instances=app.config["INSTANCES"].get_instances(), username=current_user.get_id())
config = app.config["CONFIG"].get_config(with_drafts=True)
override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
return render_template("logs.html", instances=instances, username=current_user.get_id())
@app.route("/logs/local", methods=["GET"])

View file

@ -8,7 +8,8 @@ from typing import Any, List, Optional, Tuple, Union
from API import API # type: ignore
from ApiCaller import ApiCaller # type: ignore
from dotenv import dotenv_values
from dotenv import dotenv_values # type: ignore
from Database import Database # type: ignore
class Instance:
@ -134,10 +135,11 @@ class Instance:
class Instances:
def __init__(self, docker_client, kubernetes_client, integration: str):
def __init__(self, docker_client, kubernetes_client, integration: str, db):
self.__docker_client = docker_client
self.__kubernetes_client = kubernetes_client
self.__integration = integration
self.__db = db
def __instance_from_id(self, _id) -> Instance:
instances: list[Instance] = self.get_instances()
@ -147,8 +149,34 @@ class Instances:
raise ValueError(f"Can't find instance with _id {_id}")
def get_instances(self) -> list[Instance]:
def get_instances(self, override_instances=None) -> list[Instance]:
instances = []
# Override case : only return instances from DB
if override_instances is None:
config = self.__db.get_config()
override_instances = config["OVERRIDE_INSTANCES"] != ""
if override_instances:
for instance in self.__db.get_instances():
instances.append(
Instance(
instance["hostname"],
instance["hostname"],
instance["hostname"],
"override",
"up",
None,
ApiCaller(
[
API(
f"http://{instance['hostname']}:{str(instance['port'])}",
instance["server_name"],
)
]
)
)
)
return instances
# Docker instances (containers or services)
if self.__docker_client is not None:
for instance in self.__docker_client.containers.list(all=True, filters={"label": "bunkerweb.INSTANCE"}):

View file

@ -2,7 +2,7 @@
{% block content %}
{% set cards = [
{'name' : 'Version', 'title' : 'PRO' if is_pro_version else 'PRO LOCKED' if pro_status == 'active' and pro_overlapped else 'EXPIRED' if pro_status == 'expired' else 'SUSPENDED' if pro_status == 'suspended' else 'FREE', 'link' : 'https://panel.bunkerweb.io/?utm_campaign=self&utm_source=ui#pro', 'subtitle' : 'all features available' if is_pro_version else 'awaiting compliance' if pro_status == 'active' and pro_overlapped else 'renew license' if pro_status == 'expired' else 'talk to team' if pro_status == 'suspended' else 'upgrade to pro', 'subtitle_color' : 'success' if is_pro_version else 'warning' },
{'name' : 'Version number', 'title' : version, 'link' : 'https://github.com/bunkerity/bunkerweb', 'subtitle' : "couldn'd fint remote" if not remote_version else "latest version" if remote_version and check_version else 'Update to ' + remote_version , 'subtitle_color' : "error" if not remote_version else "success" if remote_version and check_version else 'warning'},
{'name' : 'Version number', 'title' : version, 'link' : 'https://github.com/bunkerity/bunkerweb', 'subtitle' : "couldn't find remote" if not remote_version else "latest version" if remote_version and check_version else 'Update to ' + remote_version , 'subtitle_color' : "error" if not remote_version else "success" if remote_version and check_version else 'warning'},
{'name' : 'Instances', 'title' : instances_number, 'link' : 'loading?next=' + url_for('instances') , 'subtitle' : instance_health_count|string + ' / ' + instances_number|string + ' is working' , 'subtitle_color' : "info"},
{'name' : 'Services', 'title' : services_number, 'link' : 'loading?next=' + url_for('services') , 'subtitle' : services_ui_count|string + ' ui, ' + services_scheduler_count|string + ' scheduler, ' + services_autoconf_count|string + ' autoconf ' , 'subtitle_color' : "info"},
{'name' : 'Plugins', 'title' : config["CONFIG"].get_plugins()|length, 'link' : 'loading?next=' + url_for('plugins') , 'subtitle' : plugins_errors|string + ' errors' if plugins_errors > 0 else 'no error' , 'subtitle_color' : "error" if plugins_errors > 0 else 'success'}