From 987af951d8a5c309ba54e8fc6aef6584b655d6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Thu, 4 May 2023 17:25:56 -0400 Subject: [PATCH 01/36] Fix often occurring bug when testing UI --- tests/ui/main.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/ui/main.py b/tests/ui/main.py index 8d0a919a1..eda52f4da 100644 --- a/tests/ui/main.py +++ b/tests/ui/main.py @@ -901,16 +901,6 @@ try: driver, By.XPATH, "//input[@type='file' and @name='file']" ).send_keys(join(Path.cwd(), "test.zip")) - access_page( - driver, - driver_wait, - "//div[@data-plugins-upload='']//button[@type='submit']", - "plugins", - False, - ) - - assert_alert_message(driver, "is not a valid plugin") - print( "The bad plugin has been rejected, trying to add a good plugin ...", flush=True, @@ -928,8 +918,6 @@ try: False, ) - assert_alert_message(driver, "Successfully created plugin") - external_plugins = safe_get_element( driver, By.XPATH, From 58db1352fa21e0b76d4dca84870b974aed329225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Thu, 4 May 2023 17:26:35 -0400 Subject: [PATCH 02/36] Revert "Fix often occurring bug when testing UI" This reverts commit 987af951d8a5c309ba54e8fc6aef6584b655d6c5. --- tests/ui/main.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/ui/main.py b/tests/ui/main.py index eda52f4da..8d0a919a1 100644 --- a/tests/ui/main.py +++ b/tests/ui/main.py @@ -901,6 +901,16 @@ try: driver, By.XPATH, "//input[@type='file' and @name='file']" ).send_keys(join(Path.cwd(), "test.zip")) + access_page( + driver, + driver_wait, + "//div[@data-plugins-upload='']//button[@type='submit']", + "plugins", + False, + ) + + assert_alert_message(driver, "is not a valid plugin") + print( "The bad plugin has been rejected, trying to add a good plugin ...", flush=True, @@ -918,6 +928,8 @@ try: False, ) + assert_alert_message(driver, "Successfully created plugin") + external_plugins = safe_get_element( driver, By.XPATH, From 39ace81755a1b7f9e325932ab16d866f428dc9c0 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 11:10:17 +0200 Subject: [PATCH 03/36] fix load-balancer example and add server_name to cache keys when required --- examples/load-balancer/docker-compose.yml | 2 +- src/common/core/blacklist/blacklist.lua | 4 ++-- src/common/core/country/country.lua | 4 ++-- src/common/core/greylist/greylist.lua | 4 ++-- src/common/core/whitelist/whitelist.lua | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/load-balancer/docker-compose.yml b/examples/load-balancer/docker-compose.yml index f848d7ff7..1de8dcb4a 100644 --- a/examples/load-balancer/docker-compose.yml +++ b/examples/load-balancer/docker-compose.yml @@ -18,7 +18,7 @@ services: - REVERSE_PROXY_URL=/ - REVERSE_PROXY_HOST=http://app - | - CUSTOM_CONF_HTTP_upstream.conf= + CUSTOM_CONF_HTTP_upstream= upstream app { server app1:80; server app2:80; diff --git a/src/common/core/blacklist/blacklist.lua b/src/common/core/blacklist/blacklist.lua index ee8aae24d..f23ffd5cb 100644 --- a/src/common/core/blacklist/blacklist.lua +++ b/src/common/core/blacklist/blacklist.lua @@ -162,7 +162,7 @@ function blacklist:kind_to_ele(kind) end function blacklist:is_in_cache(ele) - local ok, data = self.cachestore:get("plugin_blacklist_" .. ele) + local ok, data = self.cachestore:get("plugin_blacklist_" .. ngx.ctx.bw.server_name .. ele) if not ok then return false, data end @@ -170,7 +170,7 @@ function blacklist:is_in_cache(ele) end function blacklist:add_to_cache(ele, value) - local ok, err = self.cachestore:set("plugin_blacklist_" .. ele, value, 86400) + local ok, err = self.cachestore:set("plugin_blacklist_" .. ngx.ctx.bw.server_name .. ele, value, 86400) if not ok then return false, err end diff --git a/src/common/core/country/country.lua b/src/common/core/country/country.lua index 1d9d328fe..a1aef5b28 100644 --- a/src/common/core/country/country.lua +++ b/src/common/core/country/country.lua @@ -92,7 +92,7 @@ function country:preread() end function country:is_in_cache(ip) - local ok, data = self.cachestore:get("plugin_country_cache_" .. ip) + local ok, data = self.cachestore:get("plugin_country_cache_" .. ngx.ctx.bw.server_name .. ip) if not ok then return false, data end @@ -100,7 +100,7 @@ function country:is_in_cache(ip) end function country:add_to_cache(ip, country, result) - local ok, err = self.cachestore:set("plugin_country_cache_" .. ip, cjson.encode({country = country, result = result}), 86400) + local ok, err = self.cachestore:set("plugin_country_cache_" .. ngx.ctx.bw.server_name .. ip, cjson.encode({country = country, result = result}), 86400) if not ok then return false, err end diff --git a/src/common/core/greylist/greylist.lua b/src/common/core/greylist/greylist.lua index cea946b33..d24cd00c6 100644 --- a/src/common/core/greylist/greylist.lua +++ b/src/common/core/greylist/greylist.lua @@ -231,7 +231,7 @@ function greylist:is_greylisted_ua() end function greylist:is_in_cache(ele) - local ok, data = self.cachestore:get("plugin_greylist_" .. ele) + local ok, data = self.cachestore:get("plugin_greylist_" .. ngx.ctx.bw.server_name .. ele) if not ok then return false, data end @@ -239,7 +239,7 @@ function greylist:is_in_cache(ele) end function greylist:add_to_cache(ele, value) - local ok, err = self.cachestore:set("plugin_greylist_" .. ele, value, 86400) + local ok, err = self.cachestore:set("plugin_greylist_" .. ngx.ctx.bw.server_name .. ele, value, 86400) if not ok then return false, err end diff --git a/src/common/core/whitelist/whitelist.lua b/src/common/core/whitelist/whitelist.lua index 52fdb6f5d..e9e1e8a4b 100644 --- a/src/common/core/whitelist/whitelist.lua +++ b/src/common/core/whitelist/whitelist.lua @@ -189,7 +189,7 @@ function whitelist:check_cache() end function whitelist:is_in_cache(ele) - local ok, data = self.cachestore:get("plugin_whitelist_" .. ele) + local ok, data = self.cachestore:get("plugin_whitelist_" .. ngx.ctx.bw.server_name .. ele) if not ok then return false, data end @@ -197,7 +197,7 @@ function whitelist:is_in_cache(ele) end function whitelist:add_to_cache(ele, value) - local ok, err = self.cachestore:set("plugin_whitelist_" .. ele, value, 86400) + local ok, err = self.cachestore:set("plugin_whitelist_" .. ngx.ctx.bw.server_name .. ele, value, 86400) if not ok then return false, err end From 7592e5a84fbc97734030e7febba4bb5edf602206 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 11:11:44 +0200 Subject: [PATCH 04/36] ci/cd - fix typo in staging.yml --- .github/workflows/staging.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index a3c667d19..9a2f7ac9a 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -105,17 +105,17 @@ jobs: strategy: matrix: type: [docker, autoconf, swarm, k8s, linux] - include: - - type: docker - runs_on: "['self-hosted', 'bw-docker']" - - type: autoconf - runs_on: "['self-hosted', 'bw-autoconf']" - - type: swarm - runs_on: "['self-hosted', 'bw-swarm']" - - type: k8s - runs_on: "['ubuntu-latest']" - - type: linux - runs_on: "['self-hosted', 'bw-linux']" + include: + - type: docker + runs_on: "['self-hosted', 'bw-docker']" + - type: autoconf + runs_on: "['self-hosted', 'bw-autoconf']" + - type: swarm + runs_on: "['self-hosted', 'bw-swarm']" + - type: k8s + runs_on: "['ubuntu-latest']" + - type: linux + runs_on: "['self-hosted', 'bw-linux']" uses: ./.github/workflows/staging-tests.yml with: TYPE: ${{ matrix.type }} From 43cbc79c756c28420174d50aa6b8956ca254f709 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 11:15:17 +0200 Subject: [PATCH 05/36] ci/cd - move ARM_* to secrets in linux build wf --- .github/workflows/linux-build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index 5315c45a9..d243a54dc 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -19,13 +19,6 @@ on: required: false type: boolean default: false - ARM_SSH_KEY: - required: false - ARM_SSH_IP: - required: false - ARM_SSH_CONFIG: - required: false - secrets: DOCKER_USERNAME: required: true @@ -35,6 +28,13 @@ on: required: true PRIVATE_REGISTRY_TOKEN: required: true + ARM_SSH_KEY: + required: false + ARM_SSH_IP: + required: false + ARM_SSH_CONFIG: + required: false + jobs: build: From fb037334339dfe5e8d89e95b67da69419cfbda14 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 11:17:29 +0200 Subject: [PATCH 06/36] ci/cd - use single quote in linux build wf --- .github/workflows/linux-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index d243a54dc..806cfd238 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -49,7 +49,7 @@ jobs: env: PLATFORMS: ${{ inputs.PLATFORMS }} - name: Setup SSH for ARM node - if: startsWith(env.ARCH, "arm") == true + if: startsWith(env.ARCH, 'arm') == true run: | mkdir -p ~/.ssh echo "$SSH_KEY" > ~/.ssh/id_rsa_arm @@ -61,10 +61,10 @@ jobs: SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }} - name: Setup Buildx uses: docker/setup-buildx-action@v2 - if: startsWith(env.ARCH, "arm") == false + if: startsWith(env.ARCH, 'arm') == false - name: Setup Buildx (ARM) uses: docker/setup-buildx-action@v2 - if: startsWith(env.ARCH, "arm") == true + if: startsWith(env.ARCH, 'arm') == true with: endpoint: ssh://root@arm platforms: linux/arm64,linux/arm/v7,linux/arm/v6 From fd056102d4ef519695402a3bfcc294aa5116102c Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 11:21:07 +0200 Subject: [PATCH 07/36] fix centos repo command in rhel dockerfiles and fix delete infras order for staging wf --- .github/workflows/staging.yml | 2 +- src/linux/Dockerfile-rhel | 2 +- tests/linux/Dockerfile-rhel | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 9a2f7ac9a..657fc3109 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -133,7 +133,7 @@ jobs: # Delete infrastructures delete-infras: if: ${{ always() }} - needs: [create-infras] + needs: [staging-tests] strategy: matrix: type: [docker, autoconf, swarm, k8s, linux] diff --git a/src/linux/Dockerfile-rhel b/src/linux/Dockerfile-rhel index 67012f0ef..88a221689 100644 --- a/src/linux/Dockerfile-rhel +++ b/src/linux/Dockerfile-rhel @@ -5,7 +5,7 @@ ENV NGINX_VERSION 1.24.0 # Copy centos repo COPY src/linux/centos.repo /etc/yum.repos.d/centos.repo -RUN sed -i "s/%ARCH%/$(uname -m)/g" src/linux/centos.repo /etc/yum.repos.d/centos.repo +RUN sed -i "s/%ARCH%/$(uname -m)/g" /etc/yum.repos.d/centos.repo # Copy RPM-GPG-KEY-CentOS-Official COPY src/linux/RPM-GPG-KEY-centosofficial /etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial diff --git a/tests/linux/Dockerfile-rhel b/tests/linux/Dockerfile-rhel index aa41ef56a..dd481602e 100644 --- a/tests/linux/Dockerfile-rhel +++ b/tests/linux/Dockerfile-rhel @@ -4,7 +4,7 @@ ENV NGINX_VERSION 1.24.0 # Copy centos repo COPY src/linux/centos.repo /etc/yum.repos.d/centos.repo -RUN sed -i "s/%ARCH%/$(uname -m)/g" src/linux/centos.repo /etc/yum.repos.d/centos.repo +RUN sed -i "s/%ARCH%/$(uname -m)/g" /etc/yum.repos.d/centos.repo # Copy RPM-GPG-KEY-CentOS-Official COPY src/linux/RPM-GPG-KEY-centosofficial /etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial From 27b1dddb0dd1b635132679c452d4f956be30703f Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 11:37:45 +0200 Subject: [PATCH 08/36] linux - pin pip version --- src/linux/Dockerfile-centos | 2 +- src/linux/Dockerfile-debian | 2 +- src/linux/Dockerfile-fedora | 2 +- src/linux/Dockerfile-rhel | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/linux/Dockerfile-centos b/src/linux/Dockerfile-centos index 10d499101..f1a67dcda 100644 --- a/src/linux/Dockerfile-centos +++ b/src/linux/Dockerfile-centos @@ -27,7 +27,7 @@ RUN dnf install yum-utils redhat-lsb-core -y && \ # Compile and install dependencies RUN dnf install -y python39-pip brotli brotli-devel wget gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ kernel-devel curl znc-modtcl libmpc-devel gmp-devel gawk mpfr-devel libtool pcre-devel automake autoconf readline-devel gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel lmdb-libs && \ - pip3.9 install --no-cache-dir --upgrade pip && \ + pip3.9 install --no-cache-dir --upgrade pip=pip==20.2.4 && \ pip3.9 install wheel && \ #mkdir -p /usr/share/bunkerweb/deps && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ diff --git a/src/linux/Dockerfile-debian b/src/linux/Dockerfile-debian index 577f03c02..acd235667 100644 --- a/src/linux/Dockerfile-debian +++ b/src/linux/Dockerfile-debian @@ -31,7 +31,7 @@ RUN apt update && \ # Compile and install dependencies RUN apt update && \ apt install --no-install-recommends bash python3-pip libssl-dev git libpcre++-dev zlib1g-dev libxml2-dev libyajl-dev pkgconf libcurl4-openssl-dev libgeoip-dev liblmdb-dev apt-utils bash build-essential autoconf libtool automake g++ gcc libxml2-dev make musl-dev gnupg patch libreadline-dev libpcre3-dev libgd-dev -y && \ - pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir --upgrade pip==20.2.4 && \ pip install wheel && \ #mkdir -p /usr/share/bunkerweb/deps && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ diff --git a/src/linux/Dockerfile-fedora b/src/linux/Dockerfile-fedora index ecdef85a3..60fea2464 100644 --- a/src/linux/Dockerfile-fedora +++ b/src/linux/Dockerfile-fedora @@ -25,7 +25,7 @@ RUN mkdir -p /usr/share/bunkerweb/deps && \ # Compile and install dependencies RUN dnf install -y python3-pip brotli brotli-devel gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ kernel-devel curl znc-modtcl libmpc-devel gmp-devel gawk mpfr-devel libtool pcre-devel automake autoconf readline-devel gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel lmdb-devel && \ - pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir --upgrade pip==20.2.4 && \ pip install wheel && \ #mkdir -p /usr/share/bunkerweb/deps && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ diff --git a/src/linux/Dockerfile-rhel b/src/linux/Dockerfile-rhel index 88a221689..145b2ae83 100644 --- a/src/linux/Dockerfile-rhel +++ b/src/linux/Dockerfile-rhel @@ -40,7 +40,7 @@ RUN mkdir -p /usr/share/bunkerweb/deps && \ # Compile and install dependencies RUN dnf install -y readline-devel python39-pip brotli brotli-devel gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ curl znc-modtcl gawk libtool pcre-devel automake autoconf gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel --skip-broken && \ - pip3.9 install --no-cache-dir --upgrade pip && \ + pip3.9 install --no-cache-dir --upgrade pip==20.2.4 && \ pip3.9 install wheel && \ #mkdir -p /usr/share/bunkerweb/deps && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ From a58e5c60c051e453b4666efdece0d40f5e3f0079 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 12:12:44 +0200 Subject: [PATCH 09/36] deps - upgrade python dependencies --- docs/requirements.txt | 4 ++-- src/common/gen/requirements.in | 4 ++-- src/common/gen/requirements.txt | 24 ++++++++++++------------ src/linux/Dockerfile-centos | 2 +- src/linux/Dockerfile-debian | 2 +- src/linux/Dockerfile-fedora | 2 +- src/linux/Dockerfile-rhel | 2 +- src/scheduler/requirements.txt | 18 +++++++++--------- src/ui/requirements.in | 2 +- src/ui/requirements.txt | 16 ++++++++++++---- tests/ui/requirements.txt | 2 +- 11 files changed, 43 insertions(+), 35 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 93dcf966f..c8ec3f448 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ -mkdocs==1.4.2 -mkdocs-material==9.1.8 +mkdocs==1.4.3 +mkdocs-material==9.1.9 pytablewriter==0.64.2 mike==1.1.2 jinja2<3.1.0 diff --git a/src/common/gen/requirements.in b/src/common/gen/requirements.in index 20857ff44..d18a3aba8 100644 --- a/src/common/gen/requirements.in +++ b/src/common/gen/requirements.in @@ -1,6 +1,6 @@ -docker==6.0.1 +docker==6.1.0 kubernetes==26.1.0 jinja2==3.1.2 python-dotenv==1.0.0 -requests==2.29.0 +requests==2.30.0 redis==4.5.4 diff --git a/src/common/gen/requirements.txt b/src/common/gen/requirements.txt index bfdd725fb..45eece0d3 100644 --- a/src/common/gen/requirements.txt +++ b/src/common/gen/requirements.txt @@ -12,9 +12,9 @@ cachetools==5.3.0 \ --hash=sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14 \ --hash=sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4 # via google-auth -certifi==2022.12.7 \ - --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ - --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 +certifi==2023.5.7 \ + --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \ + --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716 # via # kubernetes # requests @@ -95,9 +95,9 @@ charset-normalizer==3.1.0 \ --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab # via requests -docker==6.0.1 \ - --hash=sha256:896c4282e5c7af5c45e8b683b0b0c33932974fe6e50fc6906a0a83616ab3da97 \ - --hash=sha256:dbcb3bd2fa80dca0788ed908218bf43972772009b881ed1e20dfc29a65e49782 +docker==6.1.0 \ + --hash=sha256:b65c999f87cb5c31700b6944dc17a631071170d1aab3ad6e23506068579f885d \ + --hash=sha256:cb697eccfeff55d232f7a7f4f88cd3770d27327c38d6c266b8f55c9f14a8491e # via -r requirements.in google-auth==2.17.3 \ --hash=sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc \ @@ -239,9 +239,9 @@ redis==4.5.4 \ --hash=sha256:2c19e6767c474f2e85167909061d525ed65bea9301c0770bb151e041b7ac89a2 \ --hash=sha256:73ec35da4da267d6847e47f68730fdd5f62e2ca69e3ef5885c6a78a9374c3893 # via -r requirements.in -requests==2.29.0 \ - --hash=sha256:e8f3c9be120d3333921d213eef078af392fba3933ab7ed2d1cba3b56f2568c3b \ - --hash=sha256:f2e34a75f4749019bb0e3effb66683630e4ffeaf75819fb51bebef1bf5aef059 +requests==2.30.0 \ + --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ + --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 # via # -r requirements.in # docker @@ -262,9 +262,9 @@ six==1.16.0 \ # google-auth # kubernetes # python-dateutil -urllib3==1.26.15 \ - --hash=sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305 \ - --hash=sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42 +urllib3==2.0.2 \ + --hash=sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc \ + --hash=sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e # via # docker # kubernetes diff --git a/src/linux/Dockerfile-centos b/src/linux/Dockerfile-centos index f1a67dcda..10d499101 100644 --- a/src/linux/Dockerfile-centos +++ b/src/linux/Dockerfile-centos @@ -27,7 +27,7 @@ RUN dnf install yum-utils redhat-lsb-core -y && \ # Compile and install dependencies RUN dnf install -y python39-pip brotli brotli-devel wget gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ kernel-devel curl znc-modtcl libmpc-devel gmp-devel gawk mpfr-devel libtool pcre-devel automake autoconf readline-devel gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel lmdb-libs && \ - pip3.9 install --no-cache-dir --upgrade pip=pip==20.2.4 && \ + pip3.9 install --no-cache-dir --upgrade pip && \ pip3.9 install wheel && \ #mkdir -p /usr/share/bunkerweb/deps && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ diff --git a/src/linux/Dockerfile-debian b/src/linux/Dockerfile-debian index acd235667..577f03c02 100644 --- a/src/linux/Dockerfile-debian +++ b/src/linux/Dockerfile-debian @@ -31,7 +31,7 @@ RUN apt update && \ # Compile and install dependencies RUN apt update && \ apt install --no-install-recommends bash python3-pip libssl-dev git libpcre++-dev zlib1g-dev libxml2-dev libyajl-dev pkgconf libcurl4-openssl-dev libgeoip-dev liblmdb-dev apt-utils bash build-essential autoconf libtool automake g++ gcc libxml2-dev make musl-dev gnupg patch libreadline-dev libpcre3-dev libgd-dev -y && \ - pip install --no-cache-dir --upgrade pip==20.2.4 && \ + pip install --no-cache-dir --upgrade pip && \ pip install wheel && \ #mkdir -p /usr/share/bunkerweb/deps && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ diff --git a/src/linux/Dockerfile-fedora b/src/linux/Dockerfile-fedora index 60fea2464..ecdef85a3 100644 --- a/src/linux/Dockerfile-fedora +++ b/src/linux/Dockerfile-fedora @@ -25,7 +25,7 @@ RUN mkdir -p /usr/share/bunkerweb/deps && \ # Compile and install dependencies RUN dnf install -y python3-pip brotli brotli-devel gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ kernel-devel curl znc-modtcl libmpc-devel gmp-devel gawk mpfr-devel libtool pcre-devel automake autoconf readline-devel gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel lmdb-devel && \ - pip install --no-cache-dir --upgrade pip==20.2.4 && \ + pip install --no-cache-dir --upgrade pip && \ pip install wheel && \ #mkdir -p /usr/share/bunkerweb/deps && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ diff --git a/src/linux/Dockerfile-rhel b/src/linux/Dockerfile-rhel index 145b2ae83..88a221689 100644 --- a/src/linux/Dockerfile-rhel +++ b/src/linux/Dockerfile-rhel @@ -40,7 +40,7 @@ RUN mkdir -p /usr/share/bunkerweb/deps && \ # Compile and install dependencies RUN dnf install -y readline-devel python39-pip brotli brotli-devel gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ curl znc-modtcl gawk libtool pcre-devel automake autoconf gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel --skip-broken && \ - pip3.9 install --no-cache-dir --upgrade pip==20.2.4 && \ + pip3.9 install --no-cache-dir --upgrade pip && \ pip3.9 install wheel && \ #mkdir -p /usr/share/bunkerweb/deps && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ diff --git a/src/scheduler/requirements.txt b/src/scheduler/requirements.txt index 52e879a7d..e49cbf0ed 100644 --- a/src/scheduler/requirements.txt +++ b/src/scheduler/requirements.txt @@ -12,9 +12,9 @@ certbot==2.5.0 \ --hash=sha256:76e6e5305021d3ee54c42fc471f8f0ed5dba790e6fd7fef6713060b0e42b97d7 \ --hash=sha256:a2d730753124508effe79f648264f5cab4d1e9120acfd695a4a0c2b7bab4a966 # via -r requirements.in -certifi==2022.12.7 \ - --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ - --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 +certifi==2023.5.7 \ + --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \ + --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716 # via requests cffi==1.15.1 \ --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ @@ -236,9 +236,9 @@ pytz==2023.3 \ # acme # certbot # pyrfc3339 -requests==2.29.0 \ - --hash=sha256:e8f3c9be120d3333921d213eef078af392fba3933ab7ed2d1cba3b56f2568c3b \ - --hash=sha256:f2e34a75f4749019bb0e3effb66683630e4ffeaf75819fb51bebef1bf5aef059 +requests==2.30.0 \ + --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ + --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 # via acme schedule==1.2.0 \ --hash=sha256:415908febaba0bc9a7c727a32efb407d646fe994367ef9157d123aabbe539ea8 \ @@ -248,9 +248,9 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via configobj -urllib3==1.26.15 \ - --hash=sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305 \ - --hash=sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42 +urllib3==2.0.2 \ + --hash=sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc \ + --hash=sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e # via requests # The following packages are considered to be unsafe in a requirements file: diff --git a/src/ui/requirements.in b/src/ui/requirements.in index 681b32a2d..9cf23173c 100644 --- a/src/ui/requirements.in +++ b/src/ui/requirements.in @@ -1,4 +1,4 @@ -Flask==2.3.1 +Flask==2.3.2 Flask_Login==0.6.2 Flask_WTF==1.1.1 beautifulsoup4==4.12.2 diff --git a/src/ui/requirements.txt b/src/ui/requirements.txt index cf3c3b5ed..334bf24ad 100644 --- a/src/ui/requirements.txt +++ b/src/ui/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --allow-unsafe --generate-hashes --resolver=backtracking @@ -39,9 +39,9 @@ click==8.1.3 \ --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 # via flask -flask==2.3.1 \ - --hash=sha256:8ba2a854608fdd603b67dccd4514a46450132227fb9df40127a8d0c1de8769ec \ - --hash=sha256:a6059db4297106e5a64b3215fa16ae641822c1cb97ecb498573549b2478602cb +flask==2.3.2 \ + --hash=sha256:77fd4e1249d8c9923de34907236b747ced06e5467ecac1a7bb7115ae0e9670b0 \ + --hash=sha256:8c2f9abd47a9e8df7f0c3f091ce9497d011dc3b31effcf4c85a6e2b50f4114ef # via # -r requirements.in # flask-login @@ -174,6 +174,10 @@ gunicorn==20.1.0 \ --hash=sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e \ --hash=sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8 # via -r requirements.in +importlib-metadata==6.6.0 \ + --hash=sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed \ + --hash=sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705 + # via flask itsdangerous==2.1.2 \ --hash=sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44 \ --hash=sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a @@ -351,6 +355,10 @@ wtforms==3.0.1 \ --hash=sha256:6b351bbb12dd58af57ffef05bc78425d08d1914e0fd68ee14143b7ade023c5bc \ --hash=sha256:837f2f0e0ca79481b92884962b914eba4e72b7a2daaf1f939c890ed0124b834b # via flask-wtf +zipp==3.15.0 \ + --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \ + --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556 + # via importlib-metadata zope-event==4.6 \ --hash=sha256:73d9e3ef750cca14816a9c322c7250b0d7c9dbc337df5d1b807ff8d3d0b9e97c \ --hash=sha256:81d98813046fc86cc4136e3698fee628a3282f9c320db18658c21749235fce80 diff --git a/tests/ui/requirements.txt b/tests/ui/requirements.txt index 0650e6655..9dd3989e0 100644 --- a/tests/ui/requirements.txt +++ b/tests/ui/requirements.txt @@ -1,2 +1,2 @@ selenium==4.9.0 -requests==2.29.0 +requests==2.30.0 From 59324526cfcb8d0d4dc81c5d753f601a39dc0d97 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 12:53:49 +0200 Subject: [PATCH 10/36] speedup build process for python deps and fix default env value for autoconf/k8s --- src/autoconf/Dockerfile | 1 + src/autoconf/IngressController.py | 2 +- src/linux/Dockerfile-centos | 1 + src/linux/Dockerfile-debian | 1 + src/linux/Dockerfile-fedora | 1 + src/linux/Dockerfile-rhel | 1 + src/linux/Dockerfile-ubuntu | 1 + src/scheduler/Dockerfile | 1 + src/ui/Dockerfile | 131 +++++++++++++++--------------- 9 files changed, 74 insertions(+), 66 deletions(-) diff --git a/src/autoconf/Dockerfile b/src/autoconf/Dockerfile index 87f8251b9..22401e2b5 100644 --- a/src/autoconf/Dockerfile +++ b/src/autoconf/Dockerfile @@ -13,6 +13,7 @@ RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev pip install --no-cache-dir --upgrade pip && \ pip install wheel && \ mkdir -p /usr/share/bunkerweb/deps/python && \ + export MAKEFLAGS="-j$(nproc)" && \ pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt && \ apk del .build-deps diff --git a/src/autoconf/IngressController.py b/src/autoconf/IngressController.py index 60dc99f48..4856b8fd2 100644 --- a/src/autoconf/IngressController.py +++ b/src/autoconf/IngressController.py @@ -51,7 +51,7 @@ class IngressController(Controller, ConfigCaller): self.__logger.warning(f"Missing container bunkerweb in pod {controller_instance.metadata.name}") else: for env in pod.env: - instance["env"][env.name] = env.value + instance["env"][env.name] = env.value or "" for controller_service in self._get_controller_services(): if controller_service.metadata.annotations: for ( diff --git a/src/linux/Dockerfile-centos b/src/linux/Dockerfile-centos index 10d499101..4ddc651b7 100644 --- a/src/linux/Dockerfile-centos +++ b/src/linux/Dockerfile-centos @@ -33,6 +33,7 @@ RUN dnf install -y python39-pip brotli brotli-devel wget gperftools-devel perl l chmod +x /tmp/bunkerweb/deps/install.sh && \ bash /tmp/bunkerweb/deps/install.sh && \ mkdir /usr/share/bunkerweb/deps/python && \ + export MAKEFLAGS="-j$(nproc)" && \ pip3.9 install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt #RUN pip3.9 install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /tmp/bunkerweb/deps/requirements.txt diff --git a/src/linux/Dockerfile-debian b/src/linux/Dockerfile-debian index 577f03c02..434b4b8b4 100644 --- a/src/linux/Dockerfile-debian +++ b/src/linux/Dockerfile-debian @@ -37,6 +37,7 @@ RUN apt update && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ bash /tmp/bunkerweb/deps/install.sh && \ mkdir /usr/share/bunkerweb/deps/python && \ + export MAKEFLAGS="-j$(nproc)" && \ pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt # Dirty fix to avoid errors with --target and packages same namespace diff --git a/src/linux/Dockerfile-fedora b/src/linux/Dockerfile-fedora index ecdef85a3..f05e39b24 100644 --- a/src/linux/Dockerfile-fedora +++ b/src/linux/Dockerfile-fedora @@ -31,6 +31,7 @@ RUN dnf install -y python3-pip brotli brotli-devel gperftools-devel perl libxslt chmod +x /tmp/bunkerweb/deps/install.sh && \ bash /tmp/bunkerweb/deps/install.sh && \ mkdir /usr/share/bunkerweb/deps/python && \ + export MAKEFLAGS="-j$(nproc)" && \ pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt # Dirty fix to avoid errors with --target and packages same namespace #cp -r /usr/lib64/python3.10/* /usr/lib/python3.10/ && \ diff --git a/src/linux/Dockerfile-rhel b/src/linux/Dockerfile-rhel index 88a221689..8bee3cd9c 100644 --- a/src/linux/Dockerfile-rhel +++ b/src/linux/Dockerfile-rhel @@ -46,6 +46,7 @@ RUN dnf install -y readline-devel python39-pip brotli brotli-devel gperftools-de chmod +x /tmp/bunkerweb/deps/install.sh && \ bash /tmp/bunkerweb/deps/install.sh && \ mkdir /usr/share/bunkerweb/deps/python && \ + export MAKEFLAGS="-j$(nproc)" && \ pip3.9 install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt #RUN pip3.10 install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /tmp/bunkerweb/deps/requirements.txt diff --git a/src/linux/Dockerfile-ubuntu b/src/linux/Dockerfile-ubuntu index da931b9e4..155228bf0 100644 --- a/src/linux/Dockerfile-ubuntu +++ b/src/linux/Dockerfile-ubuntu @@ -37,6 +37,7 @@ RUN apt update && \ chmod +x /tmp/bunkerweb/deps/install.sh && \ bash /tmp/bunkerweb/deps/install.sh && \ mkdir /usr/share/bunkerweb/deps/python && \ + export MAKEFLAGS="-j$(nproc)" && \ pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt # Copy files diff --git a/src/scheduler/Dockerfile b/src/scheduler/Dockerfile index 77e54c48c..95a461947 100644 --- a/src/scheduler/Dockerfile +++ b/src/scheduler/Dockerfile @@ -14,6 +14,7 @@ RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev pip install --no-cache-dir --upgrade pip && \ pip install wheel && \ mkdir -p /usr/share/bunkerweb/deps/python && \ + export MAKEFLAGS="-j$(nproc)" && \ pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt && \ pip install --no-cache-dir gunicorn && \ apk del .build-deps diff --git a/src/ui/Dockerfile b/src/ui/Dockerfile index 621335ed6..da9a592d8 100755 --- a/src/ui/Dockerfile +++ b/src/ui/Dockerfile @@ -1,66 +1,67 @@ -FROM python:3.11.3-alpine - -# Copy python requirements -COPY src/ui/requirements.txt /tmp/req/requirements.txt -COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.1 -COPY src/common/db/requirements.txt /tmp/req/requirements.txt.2 - -RUN mkdir -p /usr/share/bunkerweb/deps && \ - cat /tmp/req/requirements.txt /tmp/req/requirements.txt.1 /tmp/req/requirements.txt.2 > /usr/share/bunkerweb/deps/requirements.txt && \ - rm -rf /tmp/req - -# Install python requirements -RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev libffi-dev cairo-dev pango-dev gdk-pixbuf-dev openssl-dev cargo file make postgresql-dev && \ - pip install --no-cache-dir --upgrade pip && \ - pip install wheel && \ - mkdir -p /usr/share/bunkerweb/deps/python && \ - pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt && \ - apk del .build-deps - -# Copy files -# can't exclude specific files/dir from . so we are copying everything by hand -COPY src/common/api /usr/share/bunkerweb/api -COPY src/common/db /usr/share/bunkerweb/db -COPY src/common/core /usr/share/bunkerweb/core -COPY src/common/gen /usr/share/bunkerweb/gen -COPY src/common/settings.json /usr/share/bunkerweb/settings.json -COPY src/common/utils /usr/share/bunkerweb/utils -COPY src/ui /usr/share/bunkerweb/ui -COPY src/VERSION /usr/share/bunkerweb/VERSION - -# Add ui user -RUN apk add --no-cache bash && \ - addgroup -g 101 ui && \ - adduser -h /var/cache/nginx -g ui -s /bin/sh -G ui -D -H -u 101 ui && \ - echo "Docker" > /usr/share/bunkerweb/INTEGRATION && \ - mkdir -p /var/tmp/bunkerweb && \ - mkdir -p /etc/bunkerweb && \ - mkdir -p /data/cache && ln -s /data/cache /var/cache/bunkerweb && \ - mkdir -p /data/lib && ln -s /data/lib /var/lib/bunkerweb && \ - mkdir -p /var/log/nginx/ && touch /var/log/nginx/ui.log && \ - for dir in $(echo "configs plugins") ; do mkdir -p "/data/${dir}" && ln -s "/data/${dir}" "/etc/bunkerweb/${dir}" ; done && \ - for dir in $(echo "configs/http configs/stream configs/server-http configs/server-stream configs/default-server-http configs/default-server-stream configs/modsec configs/modsec-crs") ; do mkdir "/data/${dir}" ; done && \ - chown -R root:ui /data && \ - chmod -R 770 /data && \ - chown -R root:ui /usr/share/bunkerweb /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /var/log/nginx && \ - for dir in $(echo "/usr/share/bunkerweb /etc/bunkerweb") ; do find ${dir} -type f -exec chmod 0740 {} \; ; done && \ - for dir in $(echo "/usr/share/bunkerweb /etc/bunkerweb") ; do find ${dir} -type d -exec chmod 0750 {} \; ; done && \ - chmod 770 /var/cache/bunkerweb /var/lib/bunkerweb /var/tmp/bunkerweb /var/log/nginx/ui.log && \ - chmod 750 /usr/share/bunkerweb/gen/*.py /usr/share/bunkerweb/ui/*.py /usr/share/bunkerweb/ui/src/*.py /usr/share/bunkerweb/deps/python/bin/* && \ - chmod 660 /usr/share/bunkerweb/INTEGRATION - -# Fix CVEs -RUN apk add "libcrypto3>=3.0.8-r4" "libssl3>=3.0.8-r4" - -VOLUME /data /etc/nginx - -EXPOSE 7000 - -WORKDIR /usr/share/bunkerweb/ui - -USER ui:ui - -HEALTHCHECK --interval=10s --timeout=10s --start-period=30s --retries=6 CMD /usr/share/bunkerweb/helpers/healthcheck-ui.sh - -ENV PYTHONPATH /usr/share/bunkerweb/deps/python +FROM python:3.11.3-alpine + +# Copy python requirements +COPY src/ui/requirements.txt /tmp/req/requirements.txt +COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.1 +COPY src/common/db/requirements.txt /tmp/req/requirements.txt.2 + +RUN mkdir -p /usr/share/bunkerweb/deps && \ + cat /tmp/req/requirements.txt /tmp/req/requirements.txt.1 /tmp/req/requirements.txt.2 > /usr/share/bunkerweb/deps/requirements.txt && \ + rm -rf /tmp/req + +# Install python requirements +RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev libffi-dev cairo-dev pango-dev gdk-pixbuf-dev openssl-dev cargo file make postgresql-dev && \ + pip install --no-cache-dir --upgrade pip && \ + pip install wheel && \ + mkdir -p /usr/share/bunkerweb/deps/python && \ + export MAKEFLAGS="-j$(nproc)" && \ + pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt && \ + apk del .build-deps + +# Copy files +# can't exclude specific files/dir from . so we are copying everything by hand +COPY src/common/api /usr/share/bunkerweb/api +COPY src/common/db /usr/share/bunkerweb/db +COPY src/common/core /usr/share/bunkerweb/core +COPY src/common/gen /usr/share/bunkerweb/gen +COPY src/common/settings.json /usr/share/bunkerweb/settings.json +COPY src/common/utils /usr/share/bunkerweb/utils +COPY src/ui /usr/share/bunkerweb/ui +COPY src/VERSION /usr/share/bunkerweb/VERSION + +# Add ui user +RUN apk add --no-cache bash && \ + addgroup -g 101 ui && \ + adduser -h /var/cache/nginx -g ui -s /bin/sh -G ui -D -H -u 101 ui && \ + echo "Docker" > /usr/share/bunkerweb/INTEGRATION && \ + mkdir -p /var/tmp/bunkerweb && \ + mkdir -p /etc/bunkerweb && \ + mkdir -p /data/cache && ln -s /data/cache /var/cache/bunkerweb && \ + mkdir -p /data/lib && ln -s /data/lib /var/lib/bunkerweb && \ + mkdir -p /var/log/nginx/ && touch /var/log/nginx/ui.log && \ + for dir in $(echo "configs plugins") ; do mkdir -p "/data/${dir}" && ln -s "/data/${dir}" "/etc/bunkerweb/${dir}" ; done && \ + for dir in $(echo "configs/http configs/stream configs/server-http configs/server-stream configs/default-server-http configs/default-server-stream configs/modsec configs/modsec-crs") ; do mkdir "/data/${dir}" ; done && \ + chown -R root:ui /data && \ + chmod -R 770 /data && \ + chown -R root:ui /usr/share/bunkerweb /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /var/log/nginx && \ + for dir in $(echo "/usr/share/bunkerweb /etc/bunkerweb") ; do find ${dir} -type f -exec chmod 0740 {} \; ; done && \ + for dir in $(echo "/usr/share/bunkerweb /etc/bunkerweb") ; do find ${dir} -type d -exec chmod 0750 {} \; ; done && \ + chmod 770 /var/cache/bunkerweb /var/lib/bunkerweb /var/tmp/bunkerweb /var/log/nginx/ui.log && \ + chmod 750 /usr/share/bunkerweb/gen/*.py /usr/share/bunkerweb/ui/*.py /usr/share/bunkerweb/ui/src/*.py /usr/share/bunkerweb/deps/python/bin/* && \ + chmod 660 /usr/share/bunkerweb/INTEGRATION + +# Fix CVEs +RUN apk add "libcrypto3>=3.0.8-r4" "libssl3>=3.0.8-r4" + +VOLUME /data /etc/nginx + +EXPOSE 7000 + +WORKDIR /usr/share/bunkerweb/ui + +USER ui:ui + +HEALTHCHECK --interval=10s --timeout=10s --start-period=30s --retries=6 CMD /usr/share/bunkerweb/helpers/healthcheck-ui.sh + +ENV PYTHONPATH /usr/share/bunkerweb/deps/python CMD ["python3", "-m", "gunicorn", "--user", "ui", "--group", "ui", "main:app", "--worker-class", "gevent", "--bind", "0.0.0.0:7000", "--graceful-timeout", "0", "--access-logfile", "-", "--error-logfile", "-"] \ No newline at end of file From 541b646980df5cf32b09cb42e93304888d88967f Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 14:08:31 +0200 Subject: [PATCH 11/36] increase drupal delay time for tests, fix tmp dir not created for realip-download job and fix has_*_variable check when multisite is yes --- examples/drupal/tests.json | 2 +- src/bw/lua/bunkerweb/utils.lua | 8 ++++++-- src/common/core/realip/jobs/realip-download.py | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/drupal/tests.json b/examples/drupal/tests.json index 6158582cc..cc1edd375 100644 --- a/examples/drupal/tests.json +++ b/examples/drupal/tests.json @@ -3,7 +3,7 @@ "kinds": ["docker", "autoconf", "swarm", "kubernetes", "linux"], "no_copy_container": true, "timeout": 60, - "delay": 60, + "delay": 120, "tests": [ { "type": "string", diff --git a/src/bw/lua/bunkerweb/utils.lua b/src/bw/lua/bunkerweb/utils.lua index 4671c2e05..049bb386b 100644 --- a/src/bw/lua/bunkerweb/utils.lua +++ b/src/bw/lua/bunkerweb/utils.lua @@ -64,7 +64,9 @@ utils.has_variable = function(var, value) return true, "success" end end - return false, "success" + if servers ~= "" then + return false, "success" + end end return check_value == value, "success" end @@ -93,7 +95,9 @@ utils.has_not_variable = function(var, value) return true, "success" end end - return false, "success" + if servers ~= "" then + return false, "success" + end end return check_value ~= value, "success" end diff --git a/src/common/core/realip/jobs/realip-download.py b/src/common/core/realip/jobs/realip-download.py index 4923fb7a3..7843d7fb0 100755 --- a/src/common/core/realip/jobs/realip-download.py +++ b/src/common/core/realip/jobs/realip-download.py @@ -64,8 +64,9 @@ try: logger.info("RealIP is not activated, skipping download...") _exit(0) - # Create directory if it doesn't exist + # Create directories if they don't exist Path("/var/cache/bunkerweb/realip").mkdir(parents=True, exist_ok=True) + Path("/var/tmp/bunkerweb/realip").mkdir(parents=True, exist_ok=True) db = Database( logger, From 8b6d788c2e61527de3543ee3328f679cb097492d Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 14:49:37 +0200 Subject: [PATCH 12/36] ci/cd - fix bitnami chart values --- examples/drupal/drupal-chart-values.yml | 3 +++ examples/ghost/ghost-chart-values.yml | 3 +++ examples/joomla/joomla-chart-values.yml | 3 +++ examples/magento/magento-chart-values.yml | 3 +++ examples/moodle/moodle-chart-values.yml | 3 +++ examples/prestashop/prestashop-chart-values.yml | 3 +++ examples/redmine/redmine-chart-values.yml | 3 +++ examples/wordpress/wordpress-chart-values.yml | 3 +++ 8 files changed, 24 insertions(+) diff --git a/examples/drupal/drupal-chart-values.yml b/examples/drupal/drupal-chart-values.yml index 13bab1258..7f105fe7f 100644 --- a/examples/drupal/drupal-chart-values.yml +++ b/examples/drupal/drupal-chart-values.yml @@ -1,3 +1,6 @@ drupalUsername: "user" drupalPassword: "changeme42" drupalEmail: "contact@example.com" +mariadb: + auth: + password: "changeme1337" \ No newline at end of file diff --git a/examples/ghost/ghost-chart-values.yml b/examples/ghost/ghost-chart-values.yml index 3cb49478d..f9b7b6f5c 100644 --- a/examples/ghost/ghost-chart-values.yml +++ b/examples/ghost/ghost-chart-values.yml @@ -1,3 +1,6 @@ ghostUsername: "user" ghostPassword: "changeme42" ghostHost: "www.example.com" +mysql: + auth: + password: "changeme1337" \ No newline at end of file diff --git a/examples/joomla/joomla-chart-values.yml b/examples/joomla/joomla-chart-values.yml index 943dc8bdc..eb1d42965 100644 --- a/examples/joomla/joomla-chart-values.yml +++ b/examples/joomla/joomla-chart-values.yml @@ -1,3 +1,6 @@ joomlaUsername: "user" joomlaPassword: "changeme42" joomlaEmail: "contact@example.com" +mariadb: + auth: + password: "changeme1337" \ No newline at end of file diff --git a/examples/magento/magento-chart-values.yml b/examples/magento/magento-chart-values.yml index 411ca30c7..d65ff2d17 100644 --- a/examples/magento/magento-chart-values.yml +++ b/examples/magento/magento-chart-values.yml @@ -5,3 +5,6 @@ magentoEmail: "contact@example.com" magentoFirstName: "John" magentoLastName: "Doe" magentoAdminUri: "admin" +mariadb: + auth: + password: "changeme1337" \ No newline at end of file diff --git a/examples/moodle/moodle-chart-values.yml b/examples/moodle/moodle-chart-values.yml index 5f0c86f03..b4597275d 100644 --- a/examples/moodle/moodle-chart-values.yml +++ b/examples/moodle/moodle-chart-values.yml @@ -2,3 +2,6 @@ moodleSiteName: "My Moodle" moodleUsername: "admin" moodlePassword: "changeme42" moodleEmail: "admin@example.com" +mariadb: + auth: + password: "changeme1337" \ No newline at end of file diff --git a/examples/prestashop/prestashop-chart-values.yml b/examples/prestashop/prestashop-chart-values.yml index 12806fd76..d476d39ed 100644 --- a/examples/prestashop/prestashop-chart-values.yml +++ b/examples/prestashop/prestashop-chart-values.yml @@ -8,3 +8,6 @@ prestashopCountry: "us" prestashopLanguage: "en" service: type: ClusterIP +mariadb: + auth: + password: "changeme1337" \ No newline at end of file diff --git a/examples/redmine/redmine-chart-values.yml b/examples/redmine/redmine-chart-values.yml index ce82ab3ed..b5311f6a5 100644 --- a/examples/redmine/redmine-chart-values.yml +++ b/examples/redmine/redmine-chart-values.yml @@ -4,3 +4,6 @@ redmineEmail: "user@example.com" redmineLanguage: "en" service: type: ClusterIP +mariadb: + auth: + password: "changeme1337" \ No newline at end of file diff --git a/examples/wordpress/wordpress-chart-values.yml b/examples/wordpress/wordpress-chart-values.yml index 5a3d468a7..259236d8c 100644 --- a/examples/wordpress/wordpress-chart-values.yml +++ b/examples/wordpress/wordpress-chart-values.yml @@ -6,3 +6,6 @@ wordpressLastName: "LastName" wordpressBlogName: "User's Blog!" wordpressTablePrefix: "changeme_" wordpressScheme: "https" +mariadb: + auth: + password: "changeme1337" \ No newline at end of file From 93e567bb65b8284bf1ac9f2d10cf1c87fc61b36f Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 15:39:24 +0200 Subject: [PATCH 13/36] linux - fix fedora deps name and add architecture to fpm config --- src/linux/fpm-debian | 2 +- src/linux/fpm-fedora | 4 ++-- src/linux/fpm-rhel | 4 ++-- src/linux/fpm-ubuntu | 1 + src/linux/fpm.sh | 2 ++ 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/linux/fpm-debian b/src/linux/fpm-debian index b1a991fe1..011b9a493 100644 --- a/src/linux/fpm-debian +++ b/src/linux/fpm-debian @@ -2,7 +2,7 @@ --name bunkerweb --license agpl3 --version %VERSION% ---architecture amd64 +--architecture %ARCH% --depends bash --depends python3 --depends procps --depends python3-pip --depends 'nginx = 1.24.0-1~bullseye' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends lsof --description "BunkerWeb %VERSION% for Debian 11" --url "https://www.bunkerweb.io" diff --git a/src/linux/fpm-fedora b/src/linux/fpm-fedora index f68a9b9aa..de4eb14d1 100644 --- a/src/linux/fpm-fedora +++ b/src/linux/fpm-fedora @@ -2,8 +2,8 @@ --name bunkerweb --license agpl3 --version %VERSION% ---architecture x86_64 ---depends bash --depends python3 --depends 'nginx = 1:1.24.0-1.fc37' --depends libcurl-devel --depends libxml2 --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream +--architecture %ARCH% +--depends bash --depends python3 --depends 'nginx = 1:1.24.0-1.fc38' --depends libcurl-devel --depends libxml2 --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream --description "BunkerWeb %VERSION% for Fedora 38" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " diff --git a/src/linux/fpm-rhel b/src/linux/fpm-rhel index b84e22821..893dc3a6b 100644 --- a/src/linux/fpm-rhel +++ b/src/linux/fpm-rhel @@ -2,9 +2,9 @@ --name bunkerweb --license agpl3 --version %VERSION% ---architecture x86_64 +--architecture %ARCH% --depends bash --depends python39 --depends 'nginx = 1:1.24.0-1.el8.ngx' --depends libcurl-devel --depends libxml2 --depends file-libs --depends net-tools --depends gd --depends sudo --depends procps --depends lsof --depends geoip ---description "BunkerWeb %VERSION% for Rhel 8" +--description "BunkerWeb %VERSION% for RHEL 8" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " --before-install /usr/share/bunkerweb/scripts/beforeInstall.sh diff --git a/src/linux/fpm-ubuntu b/src/linux/fpm-ubuntu index b01d35ed0..429c30a5d 100644 --- a/src/linux/fpm-ubuntu +++ b/src/linux/fpm-ubuntu @@ -2,6 +2,7 @@ --name bunkerweb --license agpl3 --version %VERSION% +--architecture %ARCH% --depends bash --depends python3 --depends python3-pip --depends 'nginx = 1.24.0-1~jammy' --depends libcurl4 --depends libgeoip-dev --depends libxml2 --depends libyajl2 --depends libmagic1 --depends net-tools --depends sudo --depends procps --depends lsof --description "BunkerWeb %VERSION% for Ubuntu 22.04" --url "https://www.bunkerweb.io" diff --git a/src/linux/fpm.sh b/src/linux/fpm.sh index bc75586dd..e1b3f0c38 100644 --- a/src/linux/fpm.sh +++ b/src/linux/fpm.sh @@ -1,6 +1,8 @@ #!/bin/bash VERSION="$(cat /usr/share/bunkerweb/VERSION | tr -d '\n')" +ARCH="$(uname -m)" sed -i "s/%VERSION%/${VERSION}/g" .fpm +sed -i "s/%ARCH%/${ARCH}/g" .fpm fpm -t "$1" -p "/data/bunkerweb.$1" \ No newline at end of file From df787c75dc87a44f757d06609c1c5d5263ee41bf Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 16:24:27 +0200 Subject: [PATCH 14/36] linux - add pcre dep to fedora package --- src/linux/fpm-fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/linux/fpm-fedora b/src/linux/fpm-fedora index de4eb14d1..07b389fe9 100644 --- a/src/linux/fpm-fedora +++ b/src/linux/fpm-fedora @@ -3,7 +3,7 @@ --license agpl3 --version %VERSION% --architecture %ARCH% ---depends bash --depends python3 --depends 'nginx = 1:1.24.0-1.fc38' --depends libcurl-devel --depends libxml2 --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream +--depends bash --depends python3 --depends 'nginx = 1:1.24.0-1.fc38' --depends libcurl-devel --depends libxml2 --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream --depends pcre --description "BunkerWeb %VERSION% for Fedora 38" --url "https://www.bunkerweb.io" --maintainer "Bunkerity " From 00ba46ebf04288050e5361f0a83ee39d0e07bba8 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 17:38:20 +0200 Subject: [PATCH 15/36] prepare for 1.5.0-beta update --- .github/workflows/beta.yml | 22 +++------------------- docs/integrations.md | 4 ++-- docs/web-ui.md | 2 +- 3 files changed, 6 insertions(+), 22 deletions(-) diff --git a/.github/workflows/beta.yml b/.github/workflows/beta.yml index 1b45de982..7c52a1809 100644 --- a/.github/workflows/beta.yml +++ b/.github/workflows/beta.yml @@ -96,7 +96,7 @@ jobs: strategy: matrix: linux: [ubuntu, debian, fedora, rhel] - platforms: [linux/amd64, linux/arm64, linux/arm/v7] + platforms: [linux/amd64, linux/arm64] include: - release: beta - linux: ubuntu @@ -107,11 +107,6 @@ jobs: package: rpm - linux: rhel package: rpm - exclude: - - linux: fedora - platforms: linux/arm/v7 - - linux: rhel - platforms: linux/arm/v7 uses: ./.github/workflows/linux-build.yml with: RELEASE: ${{ matrix.release }} @@ -183,7 +178,7 @@ jobs: strategy: matrix: linux: [ubuntu, debian, fedora, el] - arch: [amd64, arm64, armv7] + arch: [amd64, arm64] include: - release: beta repo: bunkerweb @@ -229,19 +224,8 @@ jobs: arch: arm64 package_arch: aarch64 - linux: el - arch: amd64 + arch: arm64 package_arch: aarch64 - - linux: ubuntu - arch: armv7 - package_arch: armhf - - linux: debian - arch: armv7 - package_arch: armhf - exclude: - - linux: fedora - arch: armv7 - - linux: el - arch: armv7 uses: ./.github/workflows/push-packagecloud.yml with: SEPARATOR: ${{ matrix.separator }} diff --git a/docs/integrations.md b/docs/integrations.md index 1b66e40a1..f7f4ca7c4 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -813,7 +813,7 @@ spec:
Linux integration
-List of supported Linux distros : +List of supported Linux distros (amd64/x86_64 and arm64/aarch64 architectures) : - Debian 11 "Bullseye" - Ubuntu 22.04 "Jammy" @@ -1079,7 +1079,7 @@ BunkerWeb is managed using systemctl :
Ansible integration
-List of supported Linux distros : +List of supported Linux distros (amd64/x86_64 and arm64/aarch64 architectures) : - Debian 11 "Bullseye" - Ubuntu 22.04 "Jammy" diff --git a/docs/web-ui.md b/docs/web-ui.md index 8865fa69e..e0679eacb 100644 --- a/docs/web-ui.md +++ b/docs/web-ui.md @@ -26,7 +26,7 @@ Because the web UI is a web application, the recommended installation procedure The security of the web UI is really important. If someone manages to gain access to the application, not only he will be able to edit your configurations but he could execute some code in the context of BunkerWeb (with a custom configuration containing LUA code for example). We highly recommend you to follow minimal security best practices like : - * Choose a strong password for the login + * Choose a strong password for the login (**at least 8 chars with 1 lower case letter, 1 upper case letter, 1 digit and 1 special char is required**) * Put the web UI under a "hard to guess" URI * Do not open the web UI on the Internet without any further restrictions * Apply settings listed in the [security tuning section](security-tuning.md) of the documentation From 6e82fde8a1bf1be18efc788f53e380ebf0dab173 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 17:39:08 +0200 Subject: [PATCH 16/36] ci/cd - fix typo in beta wf --- .github/workflows/beta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beta.yml b/.github/workflows/beta.yml index 7c52a1809..37a7cd8a9 100644 --- a/.github/workflows/beta.yml +++ b/.github/workflows/beta.yml @@ -234,7 +234,7 @@ jobs: LINUX: ${{ matrix.linux }} VERSION: ${{ matrix.separator }} PACKAGE: ${{ matrix.package }} - BW_VERSION: ${{ $matrix.package == 'rpm' && needs.wait-builds.outputs.versionrpm || needs.wait-builds.outputs.version }} + BW_VERSION: ${{ matrix.package == 'rpm' && needs.wait-builds.outputs.versionrpm || needs.wait-builds.outputs.version }} PACKAGE_ARCH: ${{ matrix.package_arch }} secrets: PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} From 141f5a1d5be97fae7ed5876d2fb498f4b4e0553a Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 17:39:49 +0200 Subject: [PATCH 17/36] ci/cd - fix typo in beta wf (again) --- .github/workflows/beta.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/beta.yml b/.github/workflows/beta.yml index 37a7cd8a9..bec098e98 100644 --- a/.github/workflows/beta.yml +++ b/.github/workflows/beta.yml @@ -236,6 +236,7 @@ jobs: PACKAGE: ${{ matrix.package }} BW_VERSION: ${{ matrix.package == 'rpm' && needs.wait-builds.outputs.versionrpm || needs.wait-builds.outputs.version }} PACKAGE_ARCH: ${{ matrix.package_arch }} + ARCH: ${{ matrix.arch }} secrets: PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} From f4590749d7037a771acd1cf90f9c02e6719b4a81 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 17:50:44 +0200 Subject: [PATCH 18/36] linux - fix arch in rhel package image --- src/linux/Dockerfile-rhel | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/linux/Dockerfile-rhel b/src/linux/Dockerfile-rhel index 8bee3cd9c..197e21d65 100644 --- a/src/linux/Dockerfile-rhel +++ b/src/linux/Dockerfile-rhel @@ -23,9 +23,9 @@ RUN dnf install -y ruby ruby-devel make gcc redhat-rpm-config rpm-build wget && # Nginx COPY src/linux/nginx.repo /etc/yum.repos.d/nginx.repo RUN dnf install yum-utils -y && \ - wget https://nginx.org/packages/rhel/8/x86_64/RPMS/nginx-${NGINX_VERSION}-1.el8.ngx.x86_64.rpm && \ - dnf install nginx-${NGINX_VERSION}-1.el8.ngx.x86_64.rpm -y && \ - rm -rf nginx-${NGINX_VERSION}-1.el8.ngx.x86_64.rpm + wget https://nginx.org/packages/rhel/8/$(uname -m)/RPMS/nginx-${NGINX_VERSION}-1.el8.ngx.$(uname -m).rpm && \ + dnf install nginx-${NGINX_VERSION}-1.el8.ngx.$(uname -m).rpm -y && \ + rm -rf nginx-${NGINX_VERSION}-1.el8.ngx.$(uname -m).rpm # Copy dependencies sources folder COPY src/deps /tmp/bunkerweb/deps From 45c90527c4d0fd2d5b519644d82c5de20b944a27 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 18:27:45 +0200 Subject: [PATCH 19/36] ci/cd - fix linux package generation when arch is ARM --- .github/workflows/linux-build.yml | 24 ++++++++++++++++++++++-- src/linux/package.sh | 11 ++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index 806cfd238..e7f6b590e 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -35,7 +35,6 @@ on: ARM_SSH_CONFIG: required: false - jobs: build: runs-on: ubuntu-latest @@ -48,6 +47,18 @@ jobs: echo "ARCH=${{ env.PLATFORMS }}" | sed 's/linux//g' | sed 's@/@@g' >> "$GITHUB_ENV" env: PLATFORMS: ${{ inputs.PLATFORMS }} + - name: Extract linux arch + if: inputs.PACKAGE == 'rpm' + run : | + echo "LARCH=${{ env.ARCH }}" | sed 's/amd64/x86_64/g' | sed 's/arm64/aarch64/g' >> "$GITHUB_ENV" + env: + ARCH: ${{ env.ARCH }} + - name: Extract linux arch + if: inputs.PACKAGE == 'deb' + run : | + echo "LARCH=${{ env.ARCH }}" >> "$GITHUB_ENV" + env: + ARCH: ${{ env.ARCH }} - name: Setup SSH for ARM node if: startsWith(env.ARCH, 'arm') == true run: | @@ -103,7 +114,16 @@ jobs: tags: local/bunkerweb-${{ inputs.LINUX }}:latest # Generate package - name: Generate package - run: ./src/linux/package.sh ${{ inputs.LINUX }} + if: startsWith(env.ARCH, 'arm') == false + run: ./src/linux/package.sh ${{ inputs.LINUX }} ${{ inputs.LARCH }} + - name: Generate package (ARM) + if: startsWith(env.ARCH, 'arm') == true + run: | + docker save local/bunkerweb-${{ inputs.LINUX }}:latest | ssh -C root@arm docker load + scp ./src/linux/package.sh root@arm:/opt/package.sh + ssh root@arm chmod +x /opt/package.sh + ssh root@arm /opt/packages.sh ${{ inputs.LINUX }} ${{ inputs.LARCH }} + scp -r root@arm:/root/package-${{ inputs.LINUX }} ./package-${{ inputs.LINUX }} - uses: actions/upload-artifact@v3 with: name: package-${{ inputs.LINUX }}-${{ env.ARCH }} diff --git a/src/linux/package.sh b/src/linux/package.sh index 12ca31e7c..7c5328a60 100755 --- a/src/linux/package.sh +++ b/src/linux/package.sh @@ -15,12 +15,17 @@ function do_and_check_cmd() { return 0 } -# Check arg +# Check args if [ "$1" = "" ] ; then echo "āŒ Missing distro arg" exit 1 fi linux="$1" +if [ "$2" = "" ] ; then + echo "āŒ Missing arch arg" + exit 1 +fi +arch="$2" # Create empty directory package_dir="${PWD}/package-$linux" @@ -36,9 +41,9 @@ if [ "$linux" = "fedora" ] || [ "$linux" = "centos" ] || [ "$linux" = "rhel" ] ; type="rpm" fi do_and_check_cmd docker run --rm -v "${package_dir}:/data" "local/bunkerweb-${linux}:latest" "$type" -name="bunkerweb_${version}-1_amd64" +name="bunkerweb_${version}-1_${arch}" if [ "$type" = "rpm" ] ; then - name="bunkerweb-${version}-1.x86_64" + name="bunkerweb-${version}-1.${arch}" fi do_and_check_cmd mv "${package_dir}/bunkerweb.$type" "${package_dir}/${name}.${type}" From 73acbe0852b740dfb6c7b95a31ec291256795e84 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 18:41:54 +0200 Subject: [PATCH 20/36] ci/cd - fix typo in linux build wf --- .github/workflows/linux-build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index e7f6b590e..e4656be8c 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -115,15 +115,19 @@ jobs: # Generate package - name: Generate package if: startsWith(env.ARCH, 'arm') == false - run: ./src/linux/package.sh ${{ inputs.LINUX }} ${{ inputs.LARCH }} + run: ./src/linux/package.sh ${{ inputs.LINUX }} ${{ env.LARCH }} + env: + LARCH: ${{ env.LARCH }} - name: Generate package (ARM) if: startsWith(env.ARCH, 'arm') == true run: | docker save local/bunkerweb-${{ inputs.LINUX }}:latest | ssh -C root@arm docker load scp ./src/linux/package.sh root@arm:/opt/package.sh ssh root@arm chmod +x /opt/package.sh - ssh root@arm /opt/packages.sh ${{ inputs.LINUX }} ${{ inputs.LARCH }} + ssh root@arm /opt/packages.sh ${{ inputs.LINUX }} ${{ env.LARCH }} scp -r root@arm:/root/package-${{ inputs.LINUX }} ./package-${{ inputs.LINUX }} + env: + LARCH: ${{ env.LARCH }} - uses: actions/upload-artifact@v3 with: name: package-${{ inputs.LINUX }}-${{ env.ARCH }} From dc1cb6a6fa0c7121e01c811e45cbd9c3ae7cf1ae Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 19:32:38 +0200 Subject: [PATCH 21/36] ci/cd - fix scp command in linux build wf --- .github/workflows/linux-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index e4656be8c..a20330121 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -122,7 +122,7 @@ jobs: if: startsWith(env.ARCH, 'arm') == true run: | docker save local/bunkerweb-${{ inputs.LINUX }}:latest | ssh -C root@arm docker load - scp ./src/linux/package.sh root@arm:/opt/package.sh + scp ./src/linux/package.sh root@arm:/opt ssh root@arm chmod +x /opt/package.sh ssh root@arm /opt/packages.sh ${{ inputs.LINUX }} ${{ env.LARCH }} scp -r root@arm:/root/package-${{ inputs.LINUX }} ./package-${{ inputs.LINUX }} From 14ca85cdb6ab9e239b5944f1ff763624a6eeb391 Mon Sep 17 00:00:00 2001 From: florian Date: Sun, 7 May 2023 19:51:03 +0200 Subject: [PATCH 22/36] ci/cd - fix package.sh name in linux build wf --- .github/workflows/linux-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index a20330121..4e6b1be45 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -124,7 +124,7 @@ jobs: docker save local/bunkerweb-${{ inputs.LINUX }}:latest | ssh -C root@arm docker load scp ./src/linux/package.sh root@arm:/opt ssh root@arm chmod +x /opt/package.sh - ssh root@arm /opt/packages.sh ${{ inputs.LINUX }} ${{ env.LARCH }} + ssh root@arm /opt/package.sh ${{ inputs.LINUX }} ${{ env.LARCH }} scp -r root@arm:/root/package-${{ inputs.LINUX }} ./package-${{ inputs.LINUX }} env: LARCH: ${{ env.LARCH }} From c6f304b3718e2fed5484264d1ff88cb1504c1c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Sun, 7 May 2023 16:59:44 -0400 Subject: [PATCH 23/36] Update RDNS regex --- src/common/core/blacklist/plugin.json | 4 ++-- src/common/core/greylist/plugin.json | 2 +- src/common/core/whitelist/plugin.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/core/blacklist/plugin.json b/src/common/core/blacklist/plugin.json index 77003559b..73a29b5eb 100644 --- a/src/common/core/blacklist/plugin.json +++ b/src/common/core/blacklist/plugin.json @@ -48,7 +48,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to block.", "id": "blacklist-rdns", "label": "Blacklist reverse DNS", - "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", + "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", "type": "text" }, "BLACKLIST_RDNS_URLS": { @@ -138,7 +138,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to ignore in the blacklist.", "id": "blacklist-ignore-rdns", "label": "Blacklist ignore reverse DNS", - "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", + "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", "type": "text" }, "BLACKLIST_IGNORE_RDNS_URLS": { diff --git a/src/common/core/greylist/plugin.json b/src/common/core/greylist/plugin.json index 4e08ff9f8..a5f28d885 100644 --- a/src/common/core/greylist/plugin.json +++ b/src/common/core/greylist/plugin.json @@ -48,7 +48,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to put into the greylist.", "id": "greylist-rdns", "label": "Greylist reverse DNS", - "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", + "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", "type": "text" }, "GREYLIST_RDNS_URLS": { diff --git a/src/common/core/whitelist/plugin.json b/src/common/core/whitelist/plugin.json index 54fad023c..7bd2f18e8 100644 --- a/src/common/core/whitelist/plugin.json +++ b/src/common/core/whitelist/plugin.json @@ -48,7 +48,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to whitelist.", "id": "whitelist-rdns", "label": "Whitelist reverse DNS", - "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", + "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", "type": "text" }, "WHITELIST_RDNS_URLS": { From a544f18e2627f58642150901d68b2d4f33b2d213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Sun, 7 May 2023 17:00:07 -0400 Subject: [PATCH 24/36] Update update-check job to add stars so that the end of line shows --- src/common/core/misc/jobs/update-check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/core/misc/jobs/update-check.py b/src/common/core/misc/jobs/update-check.py index 21fff2f30..194c5797a 100644 --- a/src/common/core/misc/jobs/update-check.py +++ b/src/common/core/misc/jobs/update-check.py @@ -27,7 +27,7 @@ try: if current_version != latest_version: logger.warning( - f"\n\nšŸšØ A new version of BunkerWeb is available: {latest_version} (current: {current_version}) šŸšØ\n\n", + f"* \n* \n* šŸšØ A new version of BunkerWeb is available: {latest_version} (current: {current_version}) šŸšØ\n* \n* ", ) except: status = 2 From 48354fb26970c795326173d9d936e82256d58619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Sun, 7 May 2023 17:31:18 -0400 Subject: [PATCH 25/36] Revert "Update RDNS regex" This reverts commit c6f304b3718e2fed5484264d1ff88cb1504c1c7c. --- src/common/core/blacklist/plugin.json | 4 ++-- src/common/core/greylist/plugin.json | 2 +- src/common/core/whitelist/plugin.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/core/blacklist/plugin.json b/src/common/core/blacklist/plugin.json index 73a29b5eb..77003559b 100644 --- a/src/common/core/blacklist/plugin.json +++ b/src/common/core/blacklist/plugin.json @@ -48,7 +48,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to block.", "id": "blacklist-rdns", "label": "Blacklist reverse DNS", - "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", + "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", "type": "text" }, "BLACKLIST_RDNS_URLS": { @@ -138,7 +138,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to ignore in the blacklist.", "id": "blacklist-ignore-rdns", "label": "Blacklist ignore reverse DNS", - "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", + "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", "type": "text" }, "BLACKLIST_IGNORE_RDNS_URLS": { diff --git a/src/common/core/greylist/plugin.json b/src/common/core/greylist/plugin.json index a5f28d885..4e08ff9f8 100644 --- a/src/common/core/greylist/plugin.json +++ b/src/common/core/greylist/plugin.json @@ -48,7 +48,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to put into the greylist.", "id": "greylist-rdns", "label": "Greylist reverse DNS", - "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", + "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", "type": "text" }, "GREYLIST_RDNS_URLS": { diff --git a/src/common/core/whitelist/plugin.json b/src/common/core/whitelist/plugin.json index 7bd2f18e8..54fad023c 100644 --- a/src/common/core/whitelist/plugin.json +++ b/src/common/core/whitelist/plugin.json @@ -48,7 +48,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to whitelist.", "id": "whitelist-rdns", "label": "Whitelist reverse DNS", - "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", + "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", "type": "text" }, "WHITELIST_RDNS_URLS": { From b2e26fc8fcfefa1a538be715cc382055a6531701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Sun, 7 May 2023 17:41:08 -0400 Subject: [PATCH 26/36] Revert "Revert "Update RDNS regex"" This reverts commit 48354fb26970c795326173d9d936e82256d58619. --- src/common/core/blacklist/plugin.json | 4 ++-- src/common/core/greylist/plugin.json | 2 +- src/common/core/whitelist/plugin.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/core/blacklist/plugin.json b/src/common/core/blacklist/plugin.json index 77003559b..73a29b5eb 100644 --- a/src/common/core/blacklist/plugin.json +++ b/src/common/core/blacklist/plugin.json @@ -48,7 +48,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to block.", "id": "blacklist-rdns", "label": "Blacklist reverse DNS", - "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", + "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", "type": "text" }, "BLACKLIST_RDNS_URLS": { @@ -138,7 +138,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to ignore in the blacklist.", "id": "blacklist-ignore-rdns", "label": "Blacklist ignore reverse DNS", - "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", + "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", "type": "text" }, "BLACKLIST_IGNORE_RDNS_URLS": { diff --git a/src/common/core/greylist/plugin.json b/src/common/core/greylist/plugin.json index 4e08ff9f8..a5f28d885 100644 --- a/src/common/core/greylist/plugin.json +++ b/src/common/core/greylist/plugin.json @@ -48,7 +48,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to put into the greylist.", "id": "greylist-rdns", "label": "Greylist reverse DNS", - "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", + "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", "type": "text" }, "GREYLIST_RDNS_URLS": { diff --git a/src/common/core/whitelist/plugin.json b/src/common/core/whitelist/plugin.json index 54fad023c..7bd2f18e8 100644 --- a/src/common/core/whitelist/plugin.json +++ b/src/common/core/whitelist/plugin.json @@ -48,7 +48,7 @@ "help": "List of reverse DNS suffixes, separated with spaces, to whitelist.", "id": "whitelist-rdns", "label": "Whitelist reverse DNS", - "regex": "^( *((\\.([a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,})(?!.* \\3( |$))) *)*$", + "regex": "^( *(([^ ]+)(?!.*\\3( |$))) *)*$", "type": "text" }, "WHITELIST_RDNS_URLS": { From 86053d3dc5b6632b566bcb0adf87f8e4bb8ca3b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Sun, 7 May 2023 18:05:53 -0400 Subject: [PATCH 27/36] Update RDNS regex in jobs files --- src/common/core/blacklist/jobs/blacklist-download.py | 2 +- src/common/core/greylist/jobs/greylist-download.py | 2 +- src/common/core/whitelist/jobs/whitelist-download.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/core/blacklist/jobs/blacklist-download.py b/src/common/core/blacklist/jobs/blacklist-download.py index 734534069..6c966b871 100755 --- a/src/common/core/blacklist/jobs/blacklist-download.py +++ b/src/common/core/blacklist/jobs/blacklist-download.py @@ -23,7 +23,7 @@ from Database import Database from logger import setup_logger from jobs import cache_file, cache_hash, is_cached_file, file_hash -rdns_rx = re_compile(rb"^(\.?[a-z\d\-]+)*\.[a-z]{2,}$", IGNORECASE) +rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE) asn_rx = re_compile(rb"^\d+$") uri_rx = re_compile(rb"^/") diff --git a/src/common/core/greylist/jobs/greylist-download.py b/src/common/core/greylist/jobs/greylist-download.py index c5ce106b9..4dcdab421 100755 --- a/src/common/core/greylist/jobs/greylist-download.py +++ b/src/common/core/greylist/jobs/greylist-download.py @@ -23,7 +23,7 @@ from Database import Database from logger import setup_logger from jobs import cache_file, cache_hash, is_cached_file, file_hash -rdns_rx = re_compile(rb"^(\.?[a-z\d\-]+)*\.[a-z]{2,}$", IGNORECASE) +rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE) asn_rx = re_compile(rb"^\d+$") uri_rx = re_compile(rb"^/") diff --git a/src/common/core/whitelist/jobs/whitelist-download.py b/src/common/core/whitelist/jobs/whitelist-download.py index 9621a54eb..8fce9bff0 100755 --- a/src/common/core/whitelist/jobs/whitelist-download.py +++ b/src/common/core/whitelist/jobs/whitelist-download.py @@ -23,7 +23,7 @@ from Database import Database from logger import setup_logger from jobs import cache_file, cache_hash, is_cached_file, file_hash -rdns_rx = re_compile(rb"^(\.?[a-z\d\-]+)*\.[a-z]{2,}$", IGNORECASE) +rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE) asn_rx = re_compile(rb"^\d+$") uri_rx = re_compile(rb"^/") From ae042854f0c21b2cd19b512bcb99c644b434d9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Sun, 7 May 2023 19:06:22 -0400 Subject: [PATCH 28/36] Fix blacklist download jobs where ignore urls were not being downloaded --- src/common/core/blacklist/jobs/blacklist-download.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common/core/blacklist/jobs/blacklist-download.py b/src/common/core/blacklist/jobs/blacklist-download.py index 6c966b871..327930aee 100755 --- a/src/common/core/blacklist/jobs/blacklist-download.py +++ b/src/common/core/blacklist/jobs/blacklist-download.py @@ -29,7 +29,7 @@ uri_rx = re_compile(rb"^/") def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]: - if kind == "IP": + if kind in ("IP", "IGNORE_IP"): if b"/" in line: with suppress(ValueError): ip_network(line.decode("utf-8")) @@ -38,18 +38,18 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]: with suppress(ValueError): ip_address(line.decode("utf-8")) return True, line - elif kind == "RDNS": + elif kind in ("RDNS", "IGNORE_RDNS"): if rdns_rx.match(line): return True, line.lower() - elif kind == "ASN": + elif kind in ("ASN", "IGNORE_ASN"): real_line = line.replace(b"AS", b"").replace(b"as", b"") if asn_rx.match(real_line): return True, real_line - elif kind == "USER_AGENT": + elif kind in ("USER_AGENT", "IGNORE_USER_AGENT"): return True, line.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace( b"\\\\", b"\\" ).replace(b"-", b"%-") - elif kind == "URI": + elif kind in ("URI", "IGNORE_URI"): if uri_rx.match(line): return True, line From d83730cf7514effe556831f9777cb06dbddd5bd8 Mon Sep 17 00:00:00 2001 From: florian Date: Mon, 8 May 2023 15:36:57 +0200 Subject: [PATCH 29/36] ci/cd - fix linux package name in upload/download steps --- .github/workflows/linux-build.yml | 2 +- .github/workflows/push-packagecloud.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index 4e6b1be45..8c585267a 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -130,7 +130,7 @@ jobs: LARCH: ${{ env.LARCH }} - uses: actions/upload-artifact@v3 with: - name: package-${{ inputs.LINUX }}-${{ env.ARCH }} + name: package-${{ inputs.LINUX }}-${{ env.LARCH }} path: package-${{ inputs.LINUX }}/*.${{ inputs.PACKAGE }} # Build test image - name: Build test image diff --git a/.github/workflows/push-packagecloud.yml b/.github/workflows/push-packagecloud.yml index e4f72da15..135203a59 100644 --- a/.github/workflows/push-packagecloud.yml +++ b/.github/workflows/push-packagecloud.yml @@ -51,13 +51,13 @@ jobs: - uses: actions/download-artifact@v3 if: inputs.LINUX != 'el' with: - name: package-${{ inputs.LINUX }}-${{ inputs.ARCH }} + name: package-${{ inputs.LINUX }}-${{ inputs.PACKAGE_ARCH }} path: /tmp/${{ inputs.LINUX }} - uses: actions/download-artifact@v3 if: inputs.LINUX == 'el' with: - name: package-rhel - path: /tmp/${{ inputs.LINUX }}-${{ inputs.ARCH }} + name: package-rhel-${{ inputs.PACKAGE_ARCH }} + path: /tmp/${{ inputs.LINUX }} # Remove existing packages - name: Remove existing package run: package_cloud yank bunkerity/${{ inputs.REPO }}/${{ inputs.LINUX }}/${{ inputs.VERSION }} bunkerweb${{ inputs.SEPARATOR }}${{ inputs.BW_VERSION }}${{ inputs.SEPARATOR }}${{ inputs.SUFFIX }}${{ inputs.PACKAGE_ARCH }}.${{ inputs.PACKAGE }} From 136f68cd3b4f13d675ef96441b76c0ccbc7ed3a8 Mon Sep 17 00:00:00 2001 From: florian Date: Mon, 8 May 2023 16:06:20 +0200 Subject: [PATCH 30/36] ci/cd - fix typo in beta wf --- .github/workflows/beta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beta.yml b/.github/workflows/beta.yml index bec098e98..e1839b017 100644 --- a/.github/workflows/beta.yml +++ b/.github/workflows/beta.yml @@ -232,7 +232,7 @@ jobs: SUFFIX: ${{ matrix.suffix }} REPO: ${{ matrix.repo }} LINUX: ${{ matrix.linux }} - VERSION: ${{ matrix.separator }} + VERSION: ${{ matrix.version }} PACKAGE: ${{ matrix.package }} BW_VERSION: ${{ matrix.package == 'rpm' && needs.wait-builds.outputs.versionrpm || needs.wait-builds.outputs.version }} PACKAGE_ARCH: ${{ matrix.package_arch }} From e14475de4adc47b4708652cb65e472195c3997df Mon Sep 17 00:00:00 2001 From: florian Date: Mon, 8 May 2023 16:37:43 +0200 Subject: [PATCH 31/36] ci/cd - fix missing version in linux package name --- .github/workflows/linux-build.yml | 2 +- src/linux/package.sh | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index 8c585267a..5eaea0c4a 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -124,7 +124,7 @@ jobs: docker save local/bunkerweb-${{ inputs.LINUX }}:latest | ssh -C root@arm docker load scp ./src/linux/package.sh root@arm:/opt ssh root@arm chmod +x /opt/package.sh - ssh root@arm /opt/package.sh ${{ inputs.LINUX }} ${{ env.LARCH }} + ssh root@arm /opt/package.sh ${{ inputs.LINUX }} ${{ env.LARCH }} "$(cat src/VERSION | tr -d '\n')" scp -r root@arm:/root/package-${{ inputs.LINUX }} ./package-${{ inputs.LINUX }} env: LARCH: ${{ env.LARCH }} diff --git a/src/linux/package.sh b/src/linux/package.sh index 7c5328a60..6f8572986 100755 --- a/src/linux/package.sh +++ b/src/linux/package.sh @@ -35,7 +35,10 @@ fi do_and_check_cmd mkdir "$package_dir" # Generate package -version="$(cat src/VERSION | tr -d '\n')" +version="$3" +if [ -f "src/VERSION" ] ; then + version="$(cat src/VERSION | tr -d '\n')" +fi type="deb" if [ "$linux" = "fedora" ] || [ "$linux" = "centos" ] || [ "$linux" = "rhel" ] ; then type="rpm" From 63f4e44c6177c3ccad3dcb1818c5c41192c38afa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Mon, 8 May 2023 12:39:09 -0400 Subject: [PATCH 32/36] Fix CORS when sending an OPTIONS request --- src/common/core/cors/cors.lua | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/common/core/cors/cors.lua b/src/common/core/cors/cors.lua index f8ee707e6..c939ab995 100644 --- a/src/common/core/cors/cors.lua +++ b/src/common/core/cors/cors.lua @@ -1,8 +1,8 @@ -local class = require "middleclass" -local plugin = require "bunkerweb.plugin" -local utils = require "bunkerweb.utils" +local class = require "middleclass" +local plugin = require "bunkerweb.plugin" +local utils = require "bunkerweb.utils" -local cors = class("cors", plugin) +local cors = class("cors", plugin) function cors:initialize() -- Call parent initialize @@ -31,9 +31,21 @@ function cors:header() end ngx.header["Content-Type"] = "text/html" ngx.header["Content-Length"] = "0" - - -- Send CORS policy with a 204 (no content) status + return self:ret(true, "sent CORS policy") end -return cors \ No newline at end of file +function cors:access() + -- Check if access is needed + if self.variables["USE_CORS"] ~= "yes" then + return self:ret(true, "service doesn't use CORS") + end + if ngx.ctx.bw.request_method ~= "OPTIONS" then + return self:ret(true, "method is not OPTIONS") + end + + -- Send CORS policy with a 204 (no content) status + return self:ret(true, "sent CORS policy", ngx.HTTP_NO_CONTENT) +end + +return cors From b8d89fe79a2150eba96539ecd073dad701023d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Mon, 8 May 2023 22:03:38 -0400 Subject: [PATCH 33/36] Fix customcert plugin --- .../core/customcert/confs/server-http/custom-cert.conf | 9 ++++++--- .../core/customcert/confs/server-stream/custom-cert.conf | 9 ++++++--- src/common/core/customcert/jobs/custom-cert.py | 5 +++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/common/core/customcert/confs/server-http/custom-cert.conf b/src/common/core/customcert/confs/server-http/custom-cert.conf index 4b3e5a580..467ad931c 100644 --- a/src/common/core/customcert/confs/server-http/custom-cert.conf +++ b/src/common/core/customcert/confs/server-http/custom-cert.conf @@ -1,12 +1,15 @@ {% set os_path = import("os.path") %} -{% if USE_CUSTOM_SSL == "yes" and os_path.isfile("/data/cache/customcert/{}".format(CUSTOM_SSL_CERT.replace("/", "_"))) and os_path.isfile("/data/cache/customcert/{}".format(CUSTOM_SSL_KEY.replace("/", "_"))) +%} +{% set cert_file_path = "/data/cache/customcert/{}".format(CUSTOM_SSL_CERT.replace("/", "_")) %} +{% set key_file_path = "/data/cache/customcert/{}".format(CUSTOM_SSL_KEY.replace("/", "_")) %} + +{% if USE_CUSTOM_SSL == "yes" and os_path.isfile(cert_file_path) and os_path.isfile(key_file_path) +%} # listen on HTTPS PORT listen 0.0.0.0:{{ HTTPS_PORT }} ssl {% if HTTP2 == "yes" %}http2{% endif %} {% if USE_PROXY_PROTOCOL == "yes" %}proxy_protocol{% endif %}; # TLS config -ssl_certificate {{ CUSTOM_SSL_CERT }}; -ssl_certificate_key {{ CUSTOM_SSL_KEY }}; +ssl_certificate {{ cert_file_path }}; +ssl_certificate_key {{ key_file_path }}; ssl_protocols {{ SSL_PROTOCOLS }}; ssl_prefer_server_ciphers on; ssl_session_tickets off; diff --git a/src/common/core/customcert/confs/server-stream/custom-cert.conf b/src/common/core/customcert/confs/server-stream/custom-cert.conf index 9635d94b6..d24fa5bae 100644 --- a/src/common/core/customcert/confs/server-stream/custom-cert.conf +++ b/src/common/core/customcert/confs/server-stream/custom-cert.conf @@ -1,12 +1,15 @@ {% set os_path = import("os.path") %} -{% if USE_CUSTOM_SSL == "yes" and os_path.isfile(CUSTOM_SSL_CERT) and os_path.isfile(CUSTOM_SSL_KEY) +%} +{% set cert_file_path = "/data/cache/customcert/{}".format(CUSTOM_SSL_CERT.replace("/", "_")) %} +{% set key_file_path = "/data/cache/customcert/{}".format(CUSTOM_SSL_KEY.replace("/", "_")) %} + +{% if USE_CUSTOM_SSL == "yes" and os_path.isfile(cert_file_path) and os_path.isfile(key_file_path) +%} # listen listen 0.0.0.0:{{ LISTEN_STREAM_PORT_SSL }} ssl {% if USE_UDP == "yes" %} udp {% endif %}{% if USE_PROXY_PROTOCOL == "yes" %} proxy_protocol {% endif %}; # TLS config -ssl_certificate {{ CUSTOM_SSL_CERT }}; -ssl_certificate_key {{ CUSTOM_SSL_KEY }}; +ssl_certificate {{ cert_file_path }}; +ssl_certificate_key {{ key_file_path }}; ssl_protocols {{ SSL_PROTOCOLS }}; ssl_prefer_server_ciphers on; ssl_session_tickets off; diff --git a/src/common/core/customcert/jobs/custom-cert.py b/src/common/core/customcert/jobs/custom-cert.py index 5b8ae1571..c435a5084 100644 --- a/src/common/core/customcert/jobs/custom-cert.py +++ b/src/common/core/customcert/jobs/custom-cert.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 from os import getenv +from os.path import basename from pathlib import Path from shutil import copy from sys import exit as sys_exit, path as sys_path @@ -85,7 +86,7 @@ def check_cert(cert_path, key_path, first_server: Optional[str] = None) -> bool: err = db.update_job_cache( "custom-cert", first_server, - key_cache_path.replace(".hash", "").split("/")[-1], + basename(key_cache_path.replace(".hash", "")), Path(key_path).read_bytes(), checksum=key_hash, ) @@ -99,7 +100,7 @@ def check_cert(cert_path, key_path, first_server: Optional[str] = None) -> bool: err = db.update_job_cache( "custom-cert", first_server, - cert_cache_path.replace(".hash", "").split("/")[-1], + basename(cert_cache_path.replace(".hash", "")), Path(cert_path).read_bytes(), checksum=cert_hash, ) From 50ee37db0ae815799b5e76a655727a8d66e7df62 Mon Sep 17 00:00:00 2001 From: florian Date: Tue, 9 May 2023 11:42:03 +0200 Subject: [PATCH 34/36] cors - refactoring --- .../core/cors/confs/server-http/cors.conf | 5 -- src/common/core/cors/cors.lua | 54 +++++++++++-------- 2 files changed, 32 insertions(+), 27 deletions(-) delete mode 100644 src/common/core/cors/confs/server-http/cors.conf diff --git a/src/common/core/cors/confs/server-http/cors.conf b/src/common/core/cors/confs/server-http/cors.conf deleted file mode 100644 index 30cc0d46a..000000000 --- a/src/common/core/cors/confs/server-http/cors.conf +++ /dev/null @@ -1,5 +0,0 @@ -{% if USE_CORS == "yes" +%} -{% if CORS_ALLOW_ORIGIN != "" %}add_header Access-Control-Allow-Origin '{{ CORS_ALLOW_ORIGIN }}' always;{% endif %} -{% if CORS_EXPOSE_HEADERS != "" %}add_header Access-Control-Expose-Headers '{{ CORS_EXPOSE_HEADERS }}' always;{% endif %} -{% if CORS_ALLOW_CREDENTIALS != "no" %}add_header Access-Control-Allow-Credentials true always;{% endif %} -{% endif %} \ No newline at end of file diff --git a/src/common/core/cors/cors.lua b/src/common/core/cors/cors.lua index c939ab995..787c7f641 100644 --- a/src/common/core/cors/cors.lua +++ b/src/common/core/cors/cors.lua @@ -7,6 +7,16 @@ local cors = class("cors", plugin) function cors:initialize() -- Call parent initialize plugin.initialize(self, "cors") + self.all_headers = { + ["CORS_ALLOW_ORIGIN"] = "Access-Control-Allow-Origin", + ["CORS_EXPOSE_HEADERS"] = "Access-Control-Expose-Headers" + } + self.preflight_headers = { + ["CORS_MAX_AGE"] = "Access-Control-Max-Age", + ["CORS_ALLOW_CREDENTIALS"] = "Access-Control-Allow-Credentials", + ["CORS_ALLOW_METHODS"] = "Access-Control-Allow-Methods", + ["CORS_ALLOW_HEADERS"] = "Access-Control-Allow-Headers" + } end function cors:header() @@ -14,25 +24,26 @@ function cors:header() if self.variables["USE_CORS"] ~= "yes" then return self:ret(true, "service doesn't use CORS") end - if ngx.ctx.bw.request_method ~= "OPTIONS" then - return self:ret(true, "method is not OPTIONS") - end - -- Add headers - local cors_headers = { - ["CORS_MAX_AGE"] = "Access-Control-Max-Age", - ["CORS_ALLOW_METHODS"] = "Access-Control-Allow-Methods", - ["CORS_ALLOW_HEADERS"] = "Access-Control-Allow-Headers" - } - for variable, header in pairs(cors_headers) do - local value = self.variables[variable] - if value ~= "" then - ngx.header[header] = value + -- Standard headers + for variable, header in pairs(self.all_headers) do + if self.variables[variable] ~= "" then + ngx.header[header] = self.variables[variable] end end - ngx.header["Content-Type"] = "text/html" - ngx.header["Content-Length"] = "0" - - return self:ret(true, "sent CORS policy") + -- Preflight request + if ngx.ctx.bw.request_method == "OPTIONS" then + for variable, header in pairs(self.preflight_headers) do + if variable == "CORS_ALLOW_CREDENTIALS" and self.variables["CORS_ALLOW_CREDENTIALS"] == "yes" then + ngx.header[header] = "true" + elseif self.variables[variable] ~= "" then + ngx.header[header] = self.variables[variable] + end + end + ngx.header["Content-Type"] = "text/html" + ngx.header["Content-Length"] = "0" + return self:ret(true, "edited headers for preflight request") + end + return self:ret(true, "edited headers for standard request") end function cors:access() @@ -40,12 +51,11 @@ function cors:access() if self.variables["USE_CORS"] ~= "yes" then return self:ret(true, "service doesn't use CORS") end - if ngx.ctx.bw.request_method ~= "OPTIONS" then - return self:ret(true, "method is not OPTIONS") - end - -- Send CORS policy with a 204 (no content) status - return self:ret(true, "sent CORS policy", ngx.HTTP_NO_CONTENT) + if ngx.ctx.bw.request_method == "OPTIONS" then + return self:ret(true, "preflight request", ngx.HTTP_NO_CONTENT) + end + return self:ret(true, "standard request") end return cors From 30194f959996ac6f790313eead99d94f177de2d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 9 May 2023 10:05:48 -0400 Subject: [PATCH 35/36] Fix Access-Control-Allow-Credentials not being set to the right value when deactivated --- src/common/core/cors/cors.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common/core/cors/cors.lua b/src/common/core/cors/cors.lua index 787c7f641..21833e041 100644 --- a/src/common/core/cors/cors.lua +++ b/src/common/core/cors/cors.lua @@ -33,8 +33,10 @@ function cors:header() -- Preflight request if ngx.ctx.bw.request_method == "OPTIONS" then for variable, header in pairs(self.preflight_headers) do - if variable == "CORS_ALLOW_CREDENTIALS" and self.variables["CORS_ALLOW_CREDENTIALS"] == "yes" then - ngx.header[header] = "true" + if variable == "CORS_ALLOW_CREDENTIALS" then + if self.variables["CORS_ALLOW_CREDENTIALS"] == "yes" then + ngx.header[header] = "true" + end elseif self.variables[variable] ~= "" then ngx.header[header] = self.variables[variable] end From 64789276ac40410aabeacc0ec350a2072e121dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 9 May 2023 11:12:24 -0400 Subject: [PATCH 36/36] Update python deps --- docs/requirements.txt | 2 +- src/common/gen/requirements.in | 4 ++-- src/common/gen/requirements.txt | 12 ++++++------ src/ui/requirements.txt | 6 +++--- tests/ui/requirements.txt | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index c8ec3f448..a7f6c145b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ mkdocs==1.4.3 -mkdocs-material==9.1.9 +mkdocs-material==9.1.11 pytablewriter==0.64.2 mike==1.1.2 jinja2<3.1.0 diff --git a/src/common/gen/requirements.in b/src/common/gen/requirements.in index d18a3aba8..f8aaa5ff2 100644 --- a/src/common/gen/requirements.in +++ b/src/common/gen/requirements.in @@ -1,6 +1,6 @@ -docker==6.1.0 +docker==6.1.1 kubernetes==26.1.0 jinja2==3.1.2 python-dotenv==1.0.0 requests==2.30.0 -redis==4.5.4 +redis==4.5.5 diff --git a/src/common/gen/requirements.txt b/src/common/gen/requirements.txt index 45eece0d3..53ed92e22 100644 --- a/src/common/gen/requirements.txt +++ b/src/common/gen/requirements.txt @@ -95,9 +95,9 @@ charset-normalizer==3.1.0 \ --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab # via requests -docker==6.1.0 \ - --hash=sha256:b65c999f87cb5c31700b6944dc17a631071170d1aab3ad6e23506068579f885d \ - --hash=sha256:cb697eccfeff55d232f7a7f4f88cd3770d27327c38d6c266b8f55c9f14a8491e +docker==6.1.1 \ + --hash=sha256:5ec18b9c49d48ee145a5b5824bb126dc32fc77931e18444783fc07a7724badc0 \ + --hash=sha256:8308b23d3d0982c74f7aa0a3abd774898c0c4fba006e9c3bde4f68354e470fe2 # via -r requirements.in google-auth==2.17.3 \ --hash=sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc \ @@ -235,9 +235,9 @@ pyyaml==6.0 \ --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 # via kubernetes -redis==4.5.4 \ - --hash=sha256:2c19e6767c474f2e85167909061d525ed65bea9301c0770bb151e041b7ac89a2 \ - --hash=sha256:73ec35da4da267d6847e47f68730fdd5f62e2ca69e3ef5885c6a78a9374c3893 +redis==4.5.5 \ + --hash=sha256:77929bc7f5dab9adf3acba2d3bb7d7658f1e0c2f1cafe7eb36434e751c471119 \ + --hash=sha256:dc87a0bdef6c8bfe1ef1e1c40be7034390c2ae02d92dcd0c7ca1729443899880 # via -r requirements.in requests==2.30.0 \ --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ diff --git a/src/ui/requirements.txt b/src/ui/requirements.txt index 334bf24ad..f0df09995 100644 --- a/src/ui/requirements.txt +++ b/src/ui/requirements.txt @@ -345,9 +345,9 @@ soupsieve==2.4.1 \ --hash=sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8 \ --hash=sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea # via beautifulsoup4 -werkzeug==2.3.3 \ - --hash=sha256:4866679a0722de00796a74086238bb3b98d90f423f05de039abb09315487254a \ - --hash=sha256:a987caf1092edc7523edb139edb20c70571c4a8d5eed02e0b547b4739174d091 +werkzeug==2.3.4 \ + --hash=sha256:1d5a58e0377d1fe39d061a5de4469e414e78ccb1e1e59c0f5ad6fa1c36c52b76 \ + --hash=sha256:48e5e61472fee0ddee27ebad085614ebedb7af41e88f687aaf881afb723a162f # via # flask # flask-login diff --git a/tests/ui/requirements.txt b/tests/ui/requirements.txt index 9dd3989e0..f7f06f366 100644 --- a/tests/ui/requirements.txt +++ b/tests/ui/requirements.txt @@ -1,2 +1,2 @@ -selenium==4.9.0 +selenium==4.9.1 requests==2.30.0