diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c699e79c..5f8bae2d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,17 +2,16 @@ ## v1.5.7 - ????/??/?? -- [LINUX] Fix potential issues when removing the bunkerweb package - [BUGFIX] Fix rare error when the cache is not properly initialized and jobs are executed -- [FEATURE] Add an automatic renaming of old database tables when upgrading to a new version in order to avoid data loss -- [FEATURE] Add the possibility to add custom bwcli commands in plugins +- [BUGFIX] Fix bug when downloading new mmdb files +- [BUGFIX] Remove potential false positives with ModSecurity on the jobs page of the web UI +- [BUGFIX] Fix bwcli not working with Redis sentinel +- [BUGFIX] Fix potential issues when removing the bunkerweb Linux package - [FEATURE] Add backup plugin to backup and restore easily the database - [FEATURE] Add LETS_ENCRYPT_CLEAR_OLD_CERTS setting to control if old certificates should be removed when generating Let's Encrypt certificates (default is no) - [FEATURE] Add DISABLE_DEFAULT_SERVER_STRICT_SNI setting to allow/block requests when SNI is unknown or unset (default is no) -- [MISC] Remove potential false positives with ModSecurity on the jobs page of the web UI -- [MISC] Fix rare bug when downloading new mmdb files -- [DOCUMENTATION] Add procedure to follow when upgrading from 1.5.7+ -- [DOCUMENTATION] Add documentation about the procedure to follow when upgrading from a version prior to 1.5.0 +- [DOCUMENTATION] Add upgrade procedure for 1.5.7+ +- [MISC] Support custom bwcli commands using plugins - [DEPS] Updated LuaJIT version to v2.1-20240314 ## v1.5.6 - 2024/03/25 diff --git a/misc/requirements-ansible.txt b/misc/requirements-ansible.txt index 61dace273..38f316302 100644 --- a/misc/requirements-ansible.txt +++ b/misc/requirements-ansible.txt @@ -8,9 +8,9 @@ ansible==9.2.0 \ --hash=sha256:39b19c252800aeed531413a626ccd07473b79615a3cea77568a1624c1aefaf7c \ --hash=sha256:a207a4a00a45e5cd178a7f94ca42afe26f23c9d27be49901ea8c45d18a07b7c6 # via -r requirements-ansible.in -ansible-core==2.16.3 \ - --hash=sha256:50c9f33a5b2ee645470a77f4bf99cf35d1ffdefef60388910020b0c58534bec1 \ - --hash=sha256:76a8765a8586064ef073a299562e308fa2c180a75b5f7569bbd0f61d4171cdb3 +ansible-core==2.16.5 \ + --hash=sha256:371b0bb11d109a58982684307c18cc44ff8d408b1b3350c0c5c78d9f096ee1f1 \ + --hash=sha256:cdd29b0ec3f20c35657355a2f6a9c1d0cf1131da99cc9a4a3401801b0ab36d6d # via ansible cffi==1.16.0 \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ @@ -66,39 +66,39 @@ cffi==1.16.0 \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via cryptography -cryptography==42.0.2 \ - --hash=sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380 \ - --hash=sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589 \ - --hash=sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea \ - --hash=sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65 \ - --hash=sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a \ - --hash=sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3 \ - --hash=sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008 \ - --hash=sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1 \ - --hash=sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2 \ - --hash=sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635 \ - --hash=sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2 \ - --hash=sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90 \ - --hash=sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee \ - --hash=sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a \ - --hash=sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242 \ - --hash=sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12 \ - --hash=sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2 \ - --hash=sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d \ - --hash=sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be \ - --hash=sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee \ - --hash=sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6 \ - --hash=sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529 \ - --hash=sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929 \ - --hash=sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1 \ - --hash=sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6 \ - --hash=sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a \ - --hash=sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446 \ - --hash=sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9 \ - --hash=sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888 \ - --hash=sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4 \ - --hash=sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33 \ - --hash=sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f +cryptography==42.0.5 \ + --hash=sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee \ + --hash=sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576 \ + --hash=sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d \ + --hash=sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30 \ + --hash=sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413 \ + --hash=sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb \ + --hash=sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da \ + --hash=sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4 \ + --hash=sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd \ + --hash=sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc \ + --hash=sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8 \ + --hash=sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1 \ + --hash=sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc \ + --hash=sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e \ + --hash=sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8 \ + --hash=sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940 \ + --hash=sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400 \ + --hash=sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7 \ + --hash=sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16 \ + --hash=sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278 \ + --hash=sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74 \ + --hash=sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec \ + --hash=sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1 \ + --hash=sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2 \ + --hash=sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c \ + --hash=sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922 \ + --hash=sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a \ + --hash=sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6 \ + --hash=sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1 \ + --hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \ + --hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \ + --hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7 # via ansible-core jinja2==3.1.3 \ --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \ @@ -166,13 +166,13 @@ markupsafe==2.1.5 \ --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \ --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68 # via jinja2 -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 +packaging==24.0 \ + --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ + --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 # via ansible-core -pycparser==2.21 \ - --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ - --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 +pycparser==2.22 \ + --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ + --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc # via cffi pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ diff --git a/src/common/cli/CLI.py b/src/common/cli/CLI.py index d9f804c8e..7517f6b0d 100644 --- a/src/common/cli/CLI.py +++ b/src/common/cli/CLI.py @@ -77,7 +77,8 @@ class CLI(ApiCaller): if self.__use_redis: self.__logger.info("Fetching redis configuration") redis_host = self.__get_variable("REDIS_HOST") - if redis_host: + sentinel_hosts = self.__get_variable("REDIS_SENTINEL_HOSTS") + if redis_host or sentinel_hosts: redis_port = self.__get_variable("REDIS_PORT", "6379") assert isinstance(redis_port, str), "REDIS_PORT is not a string" if not redis_port.isdigit(): @@ -107,8 +108,6 @@ class CLI(ApiCaller): redis_keepalive_pool = "10" redis_keepalive_pool = int(redis_keepalive_pool) - self.__logger.info("Redis configuration is valid") - redis_ssl = self.__get_variable("REDIS_SSL", "no") == "yes" username = self.__get_variable("REDIS_USERNAME", None) or None password = self.__get_variable("REDIS_PASSWORD", None) or None @@ -177,7 +176,7 @@ class CLI(ApiCaller): self.__use_redis = False self.__logger.info("Connected to redis") else: - self.__logger.error("USE_REDIS is set to yes but REDIS_HOST is not set, disabling redis") + self.__logger.error("USE_REDIS is set to yes but REDIS_HOST or REDIS_SENTINEL_HOSTS is not set, disabling redis") self.__use_redis = False if self.__integration == "linux": diff --git a/src/common/gen/Configurator.py b/src/common/gen/Configurator.py index d2b8fe4e9..bba78d72c 100644 --- a/src/common/gen/Configurator.py +++ b/src/common/gen/Configurator.py @@ -197,6 +197,8 @@ class Configurator: "NJS_VERSION", "PKG_RELEASE", "DOCKER_HOST", + "SLAVE_MODE", + "MASTER_MODE", ) ): self.__logger.warning(f"Ignoring variable {variable} : {err}") diff --git a/src/common/settings.json b/src/common/settings.json index 801835686..014b63240 100644 --- a/src/common/settings.json +++ b/src/common/settings.json @@ -316,5 +316,14 @@ "alert", "emerg" ] + }, + "OVERRIDE_INSTANCES": { + "context": "global", + "default": "", + "help": "List of BunkerWeb instances separated with spaces (format : fqdn-or-ip:5000 fqdn-or-ip:5000)", + "id": "override-instances", + "label": "Override instances", + "regex": "^.*$", + "type": "text" } } diff --git a/src/common/utils/ApiCaller.py b/src/common/utils/ApiCaller.py index 1523d0f72..436f8f3f3 100644 --- a/src/common/utils/ApiCaller.py +++ b/src/common/utils/ApiCaller.py @@ -138,7 +138,7 @@ class ApiCaller: else: responses[instance] = resp.json() - if response and responses: + if response: return ret, responses return ret diff --git a/src/linux/scripts/start.sh b/src/linux/scripts/start.sh index e84680e20..206bc8cb1 100644 --- a/src/linux/scripts/start.sh +++ b/src/linux/scripts/start.sh @@ -100,98 +100,104 @@ function start() { log "SYSTEMCTL" "ℹ️" "Created dummy variables.env file" fi + # Create PID folder + if [ ! -f /var/run/bunkerweb ] ; then + mkdir -p /var/run/bunkerweb + chown nginx:nginx /var/run/bunkerweb + fi + # Stop scheduler if it's running stop_scheduler # Stop nginx if it's running stop_nginx - # Generate temp conf for jobs and start nginx - DNS_RESOLVERS="$(grep "^DNS_RESOLVERS=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$DNS_RESOLVERS" = "" ] ; then - DNS_RESOLVERS="8.8.8.8 8.8.4.4" - fi - API_LISTEN_IP="$(grep "^API_LISTEN_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$API_LISTEN_IP" = "" ] ; then - API_LISTEN_IP="127.0.0.1" - fi - API_HTTP_PORT="$(grep "^API_HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$API_HTTP_PORT" = "" ] ; then - API_HTTP_PORT="5000" - fi - API_SERVER_NAME="$(grep "^API_SERVER_NAME=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$API_SERVER_NAME" = "" ] ; then - API_SERVER_NAME="bwapi" - fi - API_WHITELIST_IP="$(grep "^API_WHITELIST_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$API_WHITELIST_IP" = "" ] ; then - API_WHITELIST_IP="127.0.0.0/8" - fi - USE_REAL_IP="$(grep "^USE_REAL_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$USE_REAL_IP" = "" ] ; then - USE_REAL_IP="no" - fi - USE_PROXY_PROTOCOL="$(grep "^USE_PROXY_PROTOCOL=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$USE_PROXY_PROTOCOL" = "" ] ; then - USE_PROXY_PROTOCOL="no" - fi - REAL_IP_FROM="$(grep "^REAL_IP_FROM=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$REAL_IP_FROM" = "" ] ; then - REAL_IP_FROM="192.168.0.0/16 172.16.0.0/12 10.0.0.0/8" - fi - REAL_IP_HEADER="$(grep "^REAL_IP_HEADER=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$REAL_IP_HEADER" = "" ] ; then - REAL_IP_HEADER="X-Forwarded-For" - fi - HTTP_PORT="$(grep "^HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$HTTP_PORT" = "" ] ; then - HTTP_PORT="80" - fi - HTTPS_PORT="$(grep "^HTTPS_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$HTTPS_PORT" = "" ] ; then - HTTPS_PORT="443" - fi - MODSECURITY_CRS_VERSION="$(grep "^MODSECURITY_CRS_VERSION=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ "$MODSECURITY_CRS_VERSION" = "" ] ; then - MODSECURITY_CRS_VERSION="4" - fi - sudo -E -u nginx -g nginx /bin/bash -c "echo -ne 'IS_LOADING=yes\nUSE_BUNKERNET=no\nSEND_ANONYMOUS_REPORT=no\nSERVER_NAME=\nMODSECURITY_CRS_VERSION=${MODSECURITY_CRS_VERSION}\nDNS_RESOLVERS=${DNS_RESOLVERS}\nAPI_HTTP_PORT=${API_HTTP_PORT}\nAPI_LISTEN_IP=${API_LISTEN_IP}\nAPI_SERVER_NAME=${API_SERVER_NAME}\nAPI_WHITELIST_IP=${API_WHITELIST_IP}\nUSE_REAL_IP=${USE_REAL_IP}\nUSE_PROXY_PROTOCOL=${USE_PROXY_PROTOCOL}\nREAL_IP_FROM=${REAL_IP_FROM}\nREAL_IP_HEADER=${REAL_IP_HEADER}\nHTTP_PORT=${HTTP_PORT}\nHTTPS_PORT=${HTTPS_PORT}\n' > /var/tmp/bunkerweb/tmp.env" - sudo -E -u nginx -g nginx /bin/bash -c "PYTHONPATH=/usr/share/bunkerweb/deps/python/ /usr/share/bunkerweb/gen/main.py --variables /var/tmp/bunkerweb/tmp.env --no-linux-reload" - # shellcheck disable=SC2181 - if [ $? -ne 0 ] ; then - log "SYSTEMCTL" "❌" "Error while generating config from /var/tmp/bunkerweb/tmp.env" - exit 1 - fi + # Check if we are in slave/master mode + export MASTER_MODE="$(grep "^MASTER_MODE=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + export SLAVE_MODE="$(grep "^SLAVE_MODE=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" - if [ ! -f /var/run/bunkerweb ] ; then - mkdir -p /var/run/bunkerweb - chown nginx:nginx /var/run/bunkerweb - fi - - # Start nginx - log "SYSTEMCTL" "ℹ️" "Starting nginx ..." - sudo -E -u nginx -g nginx /usr/sbin/nginx -e /var/log/bunkerweb/error.log - # shellcheck disable=SC2181 - if [ $? -ne 0 ] ; then - log "SYSTEMCTL" "❌" "Error while executing temp nginx" - exit 1 - fi - count=0 - while [ $count -lt 10 ] ; do - check="$(curl -s -H "Host: healthcheck.bunkerweb.io" http://127.0.0.1:6000/healthz 2>&1)" - # shellcheck disable=SC2181 - if [ $? -eq 0 ] && [ "$check" = "ok" ] ; then - break + if [ "$MASTER_MODE" != "yes" ] ; then + # Generate temp conf for jobs and start nginx + DNS_RESOLVERS="$(grep "^DNS_RESOLVERS=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$DNS_RESOLVERS" = "" ] ; then + DNS_RESOLVERS="8.8.8.8 8.8.4.4" fi - count=$((count + 1)) - sleep 1 - log "SYSTEMCTL" "ℹ️" "Waiting for nginx to start ..." - done - if [ $count -ge 10 ] ; then - log "SYSTEMCTL" "❌" "nginx is not started" - exit 1 + API_LISTEN_IP="$(grep "^API_LISTEN_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$API_LISTEN_IP" = "" ] ; then + API_LISTEN_IP="127.0.0.1" + fi + API_HTTP_PORT="$(grep "^API_HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$API_HTTP_PORT" = "" ] ; then + API_HTTP_PORT="5000" + fi + API_SERVER_NAME="$(grep "^API_SERVER_NAME=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$API_SERVER_NAME" = "" ] ; then + API_SERVER_NAME="bwapi" + fi + API_WHITELIST_IP="$(grep "^API_WHITELIST_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$API_WHITELIST_IP" = "" ] ; then + API_WHITELIST_IP="127.0.0.0/8" + fi + USE_REAL_IP="$(grep "^USE_REAL_IP=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$USE_REAL_IP" = "" ] ; then + USE_REAL_IP="no" + fi + USE_PROXY_PROTOCOL="$(grep "^USE_PROXY_PROTOCOL=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$USE_PROXY_PROTOCOL" = "" ] ; then + USE_PROXY_PROTOCOL="no" + fi + REAL_IP_FROM="$(grep "^REAL_IP_FROM=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$REAL_IP_FROM" = "" ] ; then + REAL_IP_FROM="192.168.0.0/16 172.16.0.0/12 10.0.0.0/8" + fi + REAL_IP_HEADER="$(grep "^REAL_IP_HEADER=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$REAL_IP_HEADER" = "" ] ; then + REAL_IP_HEADER="X-Forwarded-For" + fi + HTTP_PORT="$(grep "^HTTP_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$HTTP_PORT" = "" ] ; then + HTTP_PORT="80" + fi + HTTPS_PORT="$(grep "^HTTPS_PORT=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$HTTPS_PORT" = "" ] ; then + HTTPS_PORT="443" + fi + MODSECURITY_CRS_VERSION="$(grep "^MODSECURITY_CRS_VERSION=" /etc/bunkerweb/variables.env | cut -d '=' -f 2)" + if [ "$MODSECURITY_CRS_VERSION" = "" ] ; then + MODSECURITY_CRS_VERSION="3" + fi + sudo -E -u nginx -g nginx /bin/bash -c "echo -ne 'IS_LOADING=yes\nUSE_BUNKERNET=no\nSEND_ANONYMOUS_REPORT=no\nSERVER_NAME=\nMODSECURITY_CRS_VERSION=${MODSECURITY_CRS_VERSION}\nDNS_RESOLVERS=${DNS_RESOLVERS}\nAPI_HTTP_PORT=${API_HTTP_PORT}\nAPI_LISTEN_IP=${API_LISTEN_IP}\nAPI_SERVER_NAME=${API_SERVER_NAME}\nAPI_WHITELIST_IP=${API_WHITELIST_IP}\nUSE_REAL_IP=${USE_REAL_IP}\nUSE_PROXY_PROTOCOL=${USE_PROXY_PROTOCOL}\nREAL_IP_FROM=${REAL_IP_FROM}\nREAL_IP_HEADER=${REAL_IP_HEADER}\nHTTP_PORT=${HTTP_PORT}\nHTTPS_PORT=${HTTPS_PORT}\n' > /var/tmp/bunkerweb/tmp.env" + sudo -E -u nginx -g nginx /bin/bash -c "PYTHONPATH=/usr/share/bunkerweb/deps/python/ /usr/share/bunkerweb/gen/main.py --variables /var/tmp/bunkerweb/tmp.env --no-linux-reload" + # shellcheck disable=SC2181 + if [ $? -ne 0 ] ; then + log "SYSTEMCTL" "❌" "Error while generating config from /var/tmp/bunkerweb/tmp.env" + exit 1 + fi + # Start nginx + log "SYSTEMCTL" "ℹ️" "Starting nginx ..." + sudo -E -u nginx -g nginx /usr/sbin/nginx -e /var/log/bunkerweb/error.log + # shellcheck disable=SC2181 + if [ $? -ne 0 ] ; then + log "SYSTEMCTL" "❌" "Error while executing temp nginx" + exit 1 + fi + count=0 + while [ $count -lt 10 ] ; do + check="$(curl -s -H "Host: healthcheck.bunkerweb.io" http://127.0.0.1:6000/healthz 2>&1)" + # shellcheck disable=SC2181 + if [ $? -eq 0 ] && [ "$check" = "ok" ] ; then + break + fi + count=$((count + 1)) + sleep 1 + log "SYSTEMCTL" "ℹ️" "Waiting for nginx to start ..." + done + if [ $count -ge 10 ] ; then + log "SYSTEMCTL" "❌" "nginx is not started" + exit 1 + fi + log "SYSTEMCTL" "ℹ️" "nginx started ..." fi - log "SYSTEMCTL" "ℹ️" "nginx started ..." # Execute scheduler log "SYSTEMCTL" "ℹ️ " "Executing scheduler ..." diff --git a/src/scheduler/main.py b/src/scheduler/main.py index e9ba7d623..03dfa1118 100644 --- a/src/scheduler/main.py +++ b/src/scheduler/main.py @@ -30,6 +30,7 @@ from common_utils import bytes_hash, dict_to_frozenset, get_integration # type: from logger import setup_logger # type: ignore from Database import Database # type: ignore from JobScheduler import JobScheduler +from API import API RUN = True SCHEDULER: Optional[JobScheduler] = None @@ -70,6 +71,9 @@ SCHEDULER_TMP_ENV_PATH.touch() DB_LOCK_FILE = Path(sep, "var", "lib", "bunkerweb", "db.lock") logger = setup_logger("Scheduler", getenv("LOG_LEVEL", "INFO")) +SLAVE_MODE = environ.get("SLAVE_MODE", "no") == "yes" +MASTER_MODE = environ.get("MASTER_MODE", "no") == "yes" + def handle_stop(signum, frame): if SCHEDULER is not None: @@ -198,6 +202,53 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U if not ret: logger.error(f"Sending {'pro ' if pro else ''}external plugins failed, configuration will not work as expected...") +def generate_caches(plugins: List[Any], db: Database): + for plugin in plugins: + job_cache_files = db.get_jobs_cache_files(plugin_id=plugin["id"]) + plugin_cache_files = set() + ignored_dirs = set() + job_path = Path(sep, "var", "cache", "bunkerweb", plugin["id"]) + for job_cache_file in job_cache_files: + cache_path = job_path.joinpath(job_cache_file["service_id"] or "", job_cache_file["file_name"]) + plugin_cache_files.add(cache_path) + + try: + if job_cache_file["file_name"].endswith(".tgz"): + extract_path = cache_path.parent + if job_cache_file["file_name"].startswith("folder:"): + extract_path = Path(job_cache_file["file_name"].split("folder:", 1)[1].rsplit(".tgz", 1)[0]) + ignored_dirs.add(extract_path.as_posix()) + rmtree(extract_path, ignore_errors=True) + extract_path.mkdir(parents=True, exist_ok=True) + with tar_open(fileobj=BytesIO(job_cache_file["data"]), mode="r:gz") as tar: + try: + tar.extractall(extract_path, filter="fully_trusted") + except TypeError: + tar.extractall(extract_path) + else: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_bytes(job_cache_file["data"]) + except BaseException as e: + logger.error(f"Exception while restoring cache file {job_cache_file['file_name']} :\n{e}") + if job_path.is_dir(): + for file in job_path.rglob("*"): + skipped = False + if file.as_posix().startswith(tuple(ignored_dirs)): + skipped = True + if skipped: + continue + logger.debug(f"Checking if {file} should be removed") + if file not in plugin_cache_files and file.is_file(): + logger.debug(f"Removing non-cached file {file}") + file.unlink(missing_ok=True) + if file.parent.is_dir() and not list(file.parent.iterdir()): + logger.debug(f"Removing empty directory {file.parent}") + rmtree(file.parent, ignore_errors=True) + if file.parent == job_path: + break + elif file.is_dir() and not list(file.iterdir()): + logger.debug(f"Removing empty directory {file}") + rmtree(file, ignore_errors=True) def api_to_instance(api): hostname_port = api.endpoint.replace("http://", "").replace("https://", "").replace("/", "").split(":") @@ -206,6 +257,62 @@ def api_to_instance(api): "env": {"API_HTTP_PORT": int(hostname_port[1]), "API_SERVER_NAME": api.host}, } +def run_in_slave_mode(db: Database, dotenv_env: Dict[str, Any]): + # Instantiate db + db = Database(logger, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None))) + + # Wait for init + while not db.is_initialized(): + logger.warning("Database is not initialized, retrying in 5s ...") + sleep(5) + + # Wait for first config + env = db.get_config() + while not db.is_first_config_saved() or not env: + logger.warning("Database doesn't have any config saved yet, retrying in 5s ...") + sleep(5) + env = db.get_config() + + # Download plugins + pro_plugins = db.get_plugins(_type="pro", with_data=True) + generate_external_plugins(pro_plugins, original_path=PRO_PLUGINS_PATH) + external_plugins = db.get_plugins(_type="external", with_data=True) + generate_external_plugins(external_plugins) + + # Download custom configs + generate_custom_configs(db.get_custom_configs()) + + # Download caches + generate_caches(pro_plugins + external_plugins, db) + + # Gen config + content = "" + for k, v in env.items(): + content += f"{k}={v}\n" + SCHEDULER_TMP_ENV_PATH.write_text(content) + proc = subprocess_run( + [ + "python3", + join(sep, "usr", "share", "bunkerweb", "gen", "main.py"), + "--settings", + join(sep, "usr", "share", "bunkerweb", "settings.json"), + "--templates", + join(sep, "usr", "share", "bunkerweb", "confs"), + "--output", + join(sep, "etc", "nginx"), + "--variables", + str(SCHEDULER_TMP_ENV_PATH), + ], + stdin=DEVNULL, + stderr=STDOUT, + check=False, + ) + if proc.returncode != 0: + logger.error("Config generator failed, configuration will not work as expected...") + + # TODO : check nginx status + check DB status + while True: + sleep(5) if __name__ == "__main__": try: @@ -232,6 +339,10 @@ if __name__ == "__main__": db = Database(logger, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None))) + if SLAVE_MODE: + run_in_slave_mode(db, dotenv_env) + stop(1) + if INTEGRATION in ("Swarm", "Kubernetes", "Autoconf"): while not db.is_initialized(): logger.warning("Database is not initialized, retrying in 5s ...") @@ -280,8 +391,15 @@ if __name__ == "__main__": env["DATABASE_URI"] = db.database_uri + # Override instances if needed + override_instances = env.get("OVERRIDE_INSTANCES", "") + apis=[] + if override_instances: + for instance in override_instances.split(" "): + apis.append(API(instance)) + # Instantiate scheduler - SCHEDULER = JobScheduler(env | environ, logger, INTEGRATION, db=db) + SCHEDULER = JobScheduler(env | environ, logger, INTEGRATION, db=db, apis=apis) if INTEGRATION in ("Docker", "Swarm", "Kubernetes", "Autoconf"): # Automatically setup the scheduler apis @@ -462,7 +580,7 @@ if __name__ == "__main__": if event["Action"] == "start": db.checked_changes(value=True) - if INTEGRATION == "Docker": + if INTEGRATION == "Docker" and not override_instances: Thread(target=listen_for_instances_reload, args=(db,), name="listen_for_instances_reload").start() while True: @@ -490,19 +608,22 @@ if __name__ == "__main__": content += f"{k}={v}\n" SCHEDULER_TMP_ENV_PATH.write_text(content) # run the generator + args = [ + "python3", + join(sep, "usr", "share", "bunkerweb", "gen", "main.py"), + "--settings", + join(sep, "usr", "share", "bunkerweb", "settings.json"), + "--templates", + join(sep, "usr", "share", "bunkerweb", "confs"), + "--output", + join(sep, "etc", "nginx"), + "--variables", + str(SCHEDULER_TMP_ENV_PATH), + ] + if MASTER_MODE: + args.append("--no-linux-reload") proc = subprocess_run( - [ - "python3", - join(sep, "usr", "share", "bunkerweb", "gen", "main.py"), - "--settings", - join(sep, "usr", "share", "bunkerweb", "settings.json"), - "--templates", - join(sep, "usr", "share", "bunkerweb", "confs"), - "--output", - join(sep, "etc", "nginx"), - "--variables", - str(SCHEDULER_TMP_ENV_PATH), - ], + args, stdin=DEVNULL, stderr=STDOUT, check=False, diff --git a/src/ui/main.py b/src/ui/main.py index 4142f3f06..24dc0759c 100755 --- a/src/ui/main.py +++ b/src/ui/main.py @@ -194,7 +194,7 @@ bw_version = Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text(encodin try: app.config.update( DEBUG=True, - INSTANCES=Instances(docker_client, kubernetes_client, INTEGRATION), + INSTANCES=Instances(docker_client, kubernetes_client, INTEGRATION, db), CONFIG=Config(db), CONFIGFILES=ConfigFiles(), WTF_CSRF_SSL_STRICT=False, @@ -634,8 +634,10 @@ def home(): if r and r.status_code == 200: remote_version = basename(r.url).strip().replace("v", "") - instances = app.config["INSTANCES"].get_instances() config = app.config["CONFIG"].get_config(with_drafts=True) + override_instances = config["OVERRIDE_INSTANCES"]["value"] != "" + instances = app.config["INSTANCES"].get_instances(override_instances=override_instances) + instance_health_count = 0 for instance in instances: @@ -848,7 +850,9 @@ def instances(): ) # Display instances - instances = app.config["INSTANCES"].get_instances() + config = app.config["CONFIG"].get_config() + override_instances = config["OVERRIDE_INSTANCES"]["value"] != "" + instances = app.config["INSTANCES"].get_instances(override_instances=override_instances) return render_template("instances.html", title="Instances", instances=instances, username=current_user.get_id()) @@ -1664,7 +1668,10 @@ def cache(): @app.route("/logs", methods=["GET"]) @login_required def logs(): - return render_template("logs.html", instances=app.config["INSTANCES"].get_instances(), username=current_user.get_id()) + config = app.config["CONFIG"].get_config(with_drafts=True) + override_instances = config["OVERRIDE_INSTANCES"]["value"] != "" + instances = app.config["INSTANCES"].get_instances(override_instances=override_instances) + return render_template("logs.html", instances=instances, username=current_user.get_id()) @app.route("/logs/local", methods=["GET"]) diff --git a/src/ui/src/Instances.py b/src/ui/src/Instances.py index 585f1f3f8..a99dead2d 100644 --- a/src/ui/src/Instances.py +++ b/src/ui/src/Instances.py @@ -8,7 +8,8 @@ from typing import Any, List, Optional, Tuple, Union from API import API # type: ignore from ApiCaller import ApiCaller # type: ignore -from dotenv import dotenv_values +from dotenv import dotenv_values # type: ignore +from Database import Database # type: ignore class Instance: @@ -134,10 +135,11 @@ class Instance: class Instances: - def __init__(self, docker_client, kubernetes_client, integration: str): + def __init__(self, docker_client, kubernetes_client, integration: str, db): self.__docker_client = docker_client self.__kubernetes_client = kubernetes_client self.__integration = integration + self.__db = db def __instance_from_id(self, _id) -> Instance: instances: list[Instance] = self.get_instances() @@ -147,8 +149,34 @@ class Instances: raise ValueError(f"Can't find instance with _id {_id}") - def get_instances(self) -> list[Instance]: + def get_instances(self, override_instances=None) -> list[Instance]: instances = [] + # Override case : only return instances from DB + if override_instances is None: + config = self.__db.get_config() + override_instances = config["OVERRIDE_INSTANCES"] != "" + if override_instances: + for instance in self.__db.get_instances(): + instances.append( + Instance( + instance["hostname"], + instance["hostname"], + instance["hostname"], + "override", + "up", + None, + ApiCaller( + [ + API( + f"http://{instance['hostname']}:{str(instance['port'])}", + instance["server_name"], + ) + ] + ) + ) + + ) + return instances # Docker instances (containers or services) if self.__docker_client is not None: for instance in self.__docker_client.containers.list(all=True, filters={"label": "bunkerweb.INSTANCE"}): diff --git a/src/ui/templates/home.html b/src/ui/templates/home.html index a22a4fea6..81259d053 100644 --- a/src/ui/templates/home.html +++ b/src/ui/templates/home.html @@ -2,7 +2,7 @@ {% block content %} {% set cards = [ {'name' : 'Version', 'title' : 'PRO' if is_pro_version else 'PRO LOCKED' if pro_status == 'active' and pro_overlapped else 'EXPIRED' if pro_status == 'expired' else 'SUSPENDED' if pro_status == 'suspended' else 'FREE', 'link' : 'https://panel.bunkerweb.io/?utm_campaign=self&utm_source=ui#pro', 'subtitle' : 'all features available' if is_pro_version else 'awaiting compliance' if pro_status == 'active' and pro_overlapped else 'renew license' if pro_status == 'expired' else 'talk to team' if pro_status == 'suspended' else 'upgrade to pro', 'subtitle_color' : 'success' if is_pro_version else 'warning' }, - {'name' : 'Version number', 'title' : version, 'link' : 'https://github.com/bunkerity/bunkerweb', 'subtitle' : "couldn'd fint remote" if not remote_version else "latest version" if remote_version and check_version else 'Update to ' + remote_version , 'subtitle_color' : "error" if not remote_version else "success" if remote_version and check_version else 'warning'}, + {'name' : 'Version number', 'title' : version, 'link' : 'https://github.com/bunkerity/bunkerweb', 'subtitle' : "couldn't find remote" if not remote_version else "latest version" if remote_version and check_version else 'Update to ' + remote_version , 'subtitle_color' : "error" if not remote_version else "success" if remote_version and check_version else 'warning'}, {'name' : 'Instances', 'title' : instances_number, 'link' : 'loading?next=' + url_for('instances') , 'subtitle' : instance_health_count|string + ' / ' + instances_number|string + ' is working' , 'subtitle_color' : "info"}, {'name' : 'Services', 'title' : services_number, 'link' : 'loading?next=' + url_for('services') , 'subtitle' : services_ui_count|string + ' ui, ' + services_scheduler_count|string + ' scheduler, ' + services_autoconf_count|string + ' autoconf ' , 'subtitle_color' : "info"}, {'name' : 'Plugins', 'title' : config["CONFIG"].get_plugins()|length, 'link' : 'loading?next=' + url_for('plugins') , 'subtitle' : plugins_errors|string + ' errors' if plugins_errors > 0 else 'no error' , 'subtitle_color' : "error" if plugins_errors > 0 else 'success'}