Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #988 from bunkerity/dev

Browse source 
Merge branch "dev" into branch "staging"
This commit is contained in:
Théophile Diot 2024-03-16 13:12:40 +00:00 committed by GitHub
commit 1b60aa83b8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
216 changed files with 6029 additions and 4048 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/beta.yml vendored
View file

@ -133,7 +133,7 @@ jobs:
versionrpm: ${{ steps.getversionrpm.outputs.versionrpm }}
steps:
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Get VERSION
id: getversion
run: echo "version=$(cat src/VERSION | tr -d '\n')" >> "$GITHUB_OUTPUT"

6
.github/workflows/codeql.yml vendored
View file

@ -19,7 +19,7 @@ jobs:
language: ["python", "javascript"]
steps:
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Set up Python 3.9
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
if: matrix.language == 'python'
@ -35,12 +35,12 @@ jobs:
python -m pip install --no-cache-dir --require-hashes -r src/common/db/requirements.txt
echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
- name: Initialize CodeQL
uses: github/codeql-action/init@8a470fddafa5cbb6266ee11b37ef4d8aae19c571 # v3.24.6
uses: github/codeql-action/init@3ab4101902695724f9365a384f86c1074d94e18c # v3.24.7
with:
languages: ${{ matrix.language }}
config-file: ./.github/codeql.yml
setup-python-dependencies: false
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@8a470fddafa5cbb6266ee11b37ef4d8aae19c571 # v3.24.6
uses: github/codeql-action/analyze@3ab4101902695724f9365a384f86c1074d94e18c # v3.24.7
with:
category: "/language:${{matrix.language}}"

14
.github/workflows/container-build.yml vendored
View file

@ -45,7 +45,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Replace VERSION
if: inputs.RELEASE == 'testing'
run: ./misc/update-version.sh testing
@ -63,22 +63,22 @@ jobs:
SSH_IP: ${{ secrets.ARM_SSH_IP }}
SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }}
- name: Setup Buildx
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
if: inputs.CACHE_SUFFIX != 'arm'
- name: Setup Buildx (ARM)
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
if: inputs.CACHE_SUFFIX == 'arm'
with:
endpoint: ssh://root@arm
platforms: linux/arm64,linux/arm/v7,linux/arm/v6
- name: Login to Docker Hub
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
if: inputs.PUSH == true
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -92,7 +92,7 @@ jobs:
# Build cached image
- name: Build image
if: inputs.CACHE == true
uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
context: .
file: ${{ inputs.DOCKERFILE }}
@ -105,7 +105,7 @@ jobs:
# Build non-cached image
- name: Build image
if: inputs.CACHE != true
uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
context: .
file: ${{ inputs.DOCKERFILE }}

2
.github/workflows/create-arm.yml vendored
View file

@ -33,7 +33,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Get ARM availabilities
id: availabilities
uses: scaleway/action-scw@c718eca1fcb9fec1fb1433752d61599c6a0ad2e9

2
.github/workflows/dev-update-mmdb.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
fetch-depth: 0
token: ${{ secrets.BUNKERBOT_TOKEN }}

8
.github/workflows/dev.yml vendored
View file

@ -78,7 +78,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- id: set-matrix
run: |
tests=$(find ./tests/ui/ -name "*_page.py" -type f -printf "%f\n" | jq -c --raw-input --slurp 'split("\n")| .[0:-1]')
@ -111,7 +111,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- id: set-matrix
run: |
tests=$(find ./tests/core/ -maxdepth 1 -mindepth 1 -type d -printf "%f\n" | jq -c --raw-input --slurp 'split("\n")| .[0:-1]')
@ -149,12 +149,12 @@ jobs:
packages: write
steps:
- name: Login to Docker Hub
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}

2
.github/workflows/doc-to-pdf.yml vendored
View file

@ -13,7 +13,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Install Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:

16
.github/workflows/linux-build.yml vendored
View file

@ -37,7 +37,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Replace VERSION
if: inputs.RELEASE == 'testing' || inputs.RELEASE == 'dev' || inputs.RELEASE == 'ui'
run: ./misc/update-version.sh ${{ inputs.RELEASE }}
@ -72,21 +72,21 @@ jobs:
SSH_IP: ${{ secrets.ARM_SSH_IP }}
SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }}
- name: Setup Buildx
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
if: startsWith(env.ARCH, 'arm') == false
- name: Setup Buildx (ARM)
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
if: startsWith(env.ARCH, 'arm') == true
with:
endpoint: ssh://root@arm
platforms: linux/arm64,linux/arm/v7,linux/arm/v6
- name: Login to Docker Hub
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -94,7 +94,7 @@ jobs:
# Build testing package image
- name: Build package image
if: inputs.RELEASE == 'testing' || inputs.RELEASE == 'dev' || inputs.RELEASE == 'ui'
uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
context: .
load: true
@ -106,7 +106,7 @@ jobs:
# Build non-testing package image
- name: Build package image
if: inputs.RELEASE != 'testing' && inputs.RELEASE != 'dev'
uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
context: .
load: true
@ -142,7 +142,7 @@ jobs:
images: ghcr.io/bunkerity/${{ inputs.LINUX }}-tests:${{ inputs.RELEASE }}
- name: Build test image
if: inputs.TEST == true
uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
context: .
file: tests/linux/Dockerfile-${{ inputs.LINUX }}

2
.github/workflows/push-doc.yml vendored
View file

@ -18,7 +18,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
fetch-depth: 0
token: ${{ secrets.BUNKERBOT_TOKEN }}

10
.github/workflows/push-docker.yml vendored
View file

@ -33,14 +33,14 @@ jobs:
steps:
# Prepare
- name: Check out repository code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Login to Docker Hub
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -58,7 +58,7 @@ jobs:
SSH_IP: ${{ secrets.ARM_SSH_IP }}
SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }}
- name: Setup Buildx (ARM)
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
with:
endpoint: ssh://root@arm
platforms: linux/arm64,linux/arm/v7,linux/arm/v6
@ -70,7 +70,7 @@ jobs:
images: bunkerity/${{ inputs.IMAGE }}
# Build and push
- name: Build and push
uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
context: .
file: ${{ inputs.DOCKERFILE }}

6
.github/workflows/push-github.yml vendored
View file

@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
steps:
# Checkout
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
# Get PDF doc
- name: Get documentation
if: inputs.VERSION != 'testing'
@ -51,7 +51,7 @@ jobs:
# Create release
- name: Create release
if: inputs.VERSION != 'testing'
uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v1
uses: softprops/action-gh-release@9d7c94cfd0a1f3ed45544c887983e9fa900f0564 # v2.0.4
with:
body: |
Documentation : https://docs.bunkerweb.io/${{ inputs.VERSION }}/
@ -75,7 +75,7 @@ jobs:
# Create release
- name: Create release
if: inputs.VERSION == 'testing'
uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v1
uses: softprops/action-gh-release@9d7c94cfd0a1f3ed45544c887983e9fa900f0564 # v2.0.4
with:
body: |
**The testing version of BunkerWeb should not be used in production, please use the latest stable version instead.**

2
.github/workflows/push-packagecloud.yml vendored
View file

@ -40,7 +40,7 @@ jobs:
steps:
# Prepare
- name: Check out repository code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Install ruby
uses: ruby/setup-ruby@d4526a55538b775af234ba4af27118ed6f8f6677 # v1.172.0
with:

2
.github/workflows/release.yml vendored
View file

@ -141,7 +141,7 @@ jobs:
versionrpm: ${{ steps.getversionrpm.outputs.versionrpm }}
steps:
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Get VERSION
id: getversion
run: echo "version=$(cat src/VERSION | tr -d '\n')" >> "$GITHUB_OUTPUT"

2
.github/workflows/rm-arm.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Delete ARM VM
uses: scaleway/action-scw@c718eca1fcb9fec1fb1433752d61599c6a0ad2e9
with:

4
.github/workflows/scorecards-analysis.yml vendored
View file

@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: "Checkout code"
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
persist-credentials: false
- name: "Run analysis"
@ -25,6 +25,6 @@ jobs:
results_format: sarif
publish_results: true
- name: "Upload SARIF results to code scanning"
uses: github/codeql-action/upload-sarif@8a470fddafa5cbb6266ee11b37ef4d8aae19c571 # v3.24.6
uses: github/codeql-action/upload-sarif@3ab4101902695724f9365a384f86c1074d94e18c # v3.24.7
with:
sarif_file: results.sarif

2
.github/workflows/staging-create-infra.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
run: ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N "" && ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub && echo -e "Host *\n StrictHostKeyChecking no" > ~/.ssh/ssh_config
if: inputs.TYPE != 'k8s'
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Install terraform
uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0
- name: Install kubectl

2
.github/workflows/staging-delete-infra.yml vendored
View file

@ -20,7 +20,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Install terraform
uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0
- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4

4
.github/workflows/staging-tests.yml vendored
View file

@ -25,9 +25,9 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Login to ghcr
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}

8
.github/workflows/staging.yml vendored
View file

@ -89,7 +89,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- id: set-matrix
run: |
tests=$(find ./tests/core/ -maxdepth 1 -mindepth 1 -type d -printf "%f\n" | jq -c --raw-input --slurp 'split("\n")| .[0:-1]')
@ -100,7 +100,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- id: set-matrix
run: |
tests=$(find ./tests/ui/ -name "*_page.py" -type f -printf "%f\n" | jq -c --raw-input --slurp 'split("\n")| .[0:-1]')
@ -197,12 +197,12 @@ jobs:
packages: write
steps:
- name: Login to Docker Hub
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}

8
.github/workflows/test-core-linux.yml vendored
View file

@ -16,11 +16,11 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Python 3.12
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Set up Python 3.9
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: "3.12"
python-version: "3.9"
- name: Install Firefox manually and dependencies
run: |
sudo add-apt-repository ppa:mozillateam/ppa -y
@ -47,7 +47,7 @@ jobs:
sudo chmod +x /usr/local/bin/geckodriver
rm -f geckodriver.tar.gz
- name: Login to ghcr
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}

4
.github/workflows/test-core.yml vendored
View file

@ -16,9 +16,9 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Login to ghcr
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}

8
.github/workflows/tests-ui-linux.yml vendored
View file

@ -16,11 +16,11 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Python 3.12
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Set up Python 3.9
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: "3.12"
python-version: "3.9"
- name: Install Firefox manually and dependencies
run: |
sudo add-apt-repository ppa:mozillateam/ppa -y
@ -47,7 +47,7 @@ jobs:
sudo chmod +x /usr/local/bin/geckodriver
rm -f geckodriver.tar.gz
- name: Login to ghcr
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}

4
.github/workflows/tests-ui.yml vendored
View file

@ -15,9 +15,9 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- name: Login to ghcr
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}

1
.gitleaksignore
View file

@ -7,3 +7,4 @@ src/ui/templates/account.html:hashicorp-tf-password:417
src/ui/templates/account.html:hashicorp-tf-password:470
src/ui/templates/settings_plugins.html:hashicorp-tf-password:87
src/ui/templates/settings_plugins.html:hashicorp-tf-password:297
src/ui/templates/settings_plugins.html:hashicorp-tf-password:106

14
CHANGELOG.md
View file

@ -11,17 +11,20 @@
- [BUGFIX] Database update with external plugins reupload
- [LINUX] Add logrotate support for the logs
- [UI] New : add bans management page in the web UI
- [UI] New : add blocked requests page in the web UI
- [UI] New : some core plugins pages in the web UI
- [UI] General : enhance the Content-Security-Policy header in the web UI
- [UI] General : dark mode enhancement
- [UI] General : add visual feedback when filtering is matching nothing
- [UI] Add blocked requests page in the web UI
- [UI] Global config / service page : remove tabs for select and enhance filtering (plugin name includes)
- [UI] General : blog news working and add dynamic banner news
- [UI] Global config page : Add multisite edit, add context filter
- [UI] Global config / Service page : remove tabs for select and enhance filtering (plugin name, multiple settings and context now includes)
- [UI] Service page : add the possibility to clone a service in the web UI
- [UI] Service page : add the possibility to set a service as draft in the web UI
- [UI] Service page : add services filter when at least 4 services
- [UI] Configs page : add path filtering related to config presence
- [UI] Pro license : add home card, show pro plugis on menu and plugins page, resume in account page, alert in case issue with license usage
- [UI] Pro license : add home card, show pro plugins on menu and plugins page, resume in account page, alert in case issue with license usage
- [UI] Log page : enhance UX
- [FEATURE] Add setting REDIS_SSL_VERIFY to activate/disable the SSL certificate verification when using Redis
- [FEATURE] Add Redis Sentinel fallback to master automatically if no slaves are available
- [FEATURE] Add Redis Sentinel support for bwcli
@ -39,8 +42,11 @@
- [MISC] BunkerWeb will now load the default loading page even on 404 errors when generating the configuration
- [MISC] Update database schema to support the new pro version and optimize it
- [MISC] Refactor SSL/TLS logics to make it more consistent
- [MISC] Use ed5519 key instead of RSA for default/fallback certificates
- [MISC] Use ECDSA key instead of RSA for selfsigned/default/fallback certificates
- [MISC] Refactor certbot-new job to optimize the certbot requests
- [MISC] Refactor jobs utils to make it more consistent
- [MISC] Review jobs and utils to make it more consistent and better in general
- [MISC] Change BunkerWeb base Docker image to nginx:1.24.0-alpine-slim
- [DOCUMENTATION] Update web UI's setup wizard instructions in the documentation
- [DOCUMENTATION] Update plugins documentation to reflect the new plugin system
- [DOCUMENTATION] Update ModSecurity documentation to reflect the new changes in the Security Tuning section

BIN
docs/assets/img/demo.gif
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 48 MiB

7
docs/assets/img/pro-icon.svg Normal file
View file

@ -0,0 +1,7 @@
<svg style="height:1.5rem; width:1.5rem;"
viewBox="0 0 48 46"
fill="none"
xmlns="http://www.w3.org/2000/svg">
<path style="fill:#eab308" d="M43.218 28.2327L43.6765 23.971C43.921 21.6973 44.0825 20.1957 43.9557 19.2497L44 19.25C46.071 19.25 47.75 17.5711 47.75 15.5C47.75 13.4289 46.071 11.75 44 11.75C41.929 11.75 40.25 13.4289 40.25 15.5C40.25 16.4366 40.5935 17.2931 41.1613 17.9503C40.346 18.4535 39.2805 19.515 37.6763 21.1128C36.4405 22.3438 35.8225 22.9593 35.1333 23.0548C34.7513 23.1075 34.3622 23.0532 34.0095 22.898C33.373 22.6175 32.9485 21.8567 32.0997 20.335L27.6262 12.3135C27.1025 11.3747 26.6642 10.5889 26.2692 9.95662C27.89 9.12967 29 7.44445 29 5.5C29 2.73857 26.7615 0.5 24 0.5C21.2385 0.5 19 2.73857 19 5.5C19 7.44445 20.11 9.12967 21.7308 9.95662C21.3358 10.589 20.8975 11.3746 20.3738 12.3135L15.9002 20.335C15.0514 21.8567 14.627 22.6175 13.9905 22.898C13.6379 23.0532 13.2487 23.1075 12.8668 23.0548C12.1774 22.9593 11.5595 22.3438 10.3238 21.1128C8.71968 19.515 7.6539 18.4535 6.83882 17.9503C7.4066 17.2931 7.75 16.4366 7.75 15.5C7.75 13.4289 6.07107 11.75 4 11.75C1.92893 11.75 0.25 13.4289 0.25 15.5C0.25 17.5711 1.92893 19.25 4 19.25L4.04428 19.2497C3.91755 20.1957 4.07905 21.6973 4.32362 23.971L4.782 28.2327C5.03645 30.5982 5.24802 32.849 5.50717 34.875H42.4928C42.752 32.849 42.9635 30.5982 43.218 28.2327Z" fill="#1C274C" />
<path style="fill:#eab308" d="M21.2803 45.5H26.7198C33.8098 45.5 37.3545 45.5 39.7198 43.383C40.7523 42.4588 41.4057 40.793 41.8775 38.625H6.1224C6.59413 40.793 7.24783 42.4588 8.2802 43.383C10.6454 45.5 14.1903 45.5 21.2803 45.5Z" fill="#1C274C" />
</svg>
Side by side

After

Width:  |  Height:  |  Size: 1.6 KiB

68
docs/json2md.py
View file

@ -5,6 +5,10 @@ from json import loads
from glob import glob
from pathlib import Path
from pytablewriter import MarkdownTableWriter
import requests
import zipfile
import shutil
from contextlib import suppress
def print_md_table(settings) -> MarkdownTableWriter:
@ -71,9 +75,10 @@ print("## Core settings\n", file=doc)
core_settings = {}
for core in glob("src/common/core/*/plugin.json"):
with open(core, "r") as f:
core_plugin = loads(f.read())
if len(core_plugin["settings"]) > 0:
core_settings[core_plugin["name"]] = core_plugin
with suppress(Exception):
core_plugin = loads(f.read())
if len(core_plugin["settings"]) > 0:
core_settings[core_plugin["name"]] = core_plugin
for name, data in dict(sorted(core_settings.items())).items():
print(f"### {data['name']}\n", file=doc)
@ -81,6 +86,63 @@ for name, data in dict(sorted(core_settings.items())).items():
print(f"{data['description']}\n", file=doc)
print(print_md_table(data["settings"]), file=doc)
def pro_title(head_num: str, title: str) -> str:
markdown_header = "##" if head_num == "2" else "###"
return f"""
{markdown_header} {title}
<div style="display:flex; align-items:center">
<h{head_num} data-custom-header id="{title.lower().replace(" ", "-")}">{title}</h{head_num}>
<svg style="height:1.25rem; width:1.25rem; margin-top: 0.70rem; margin-left: 0.5rem"
viewBox="0 0 48 46"
fill="none"
xmlns="http://www.w3.org/2000/svg">
<path style="fill:#eab308" d="M43.218 28.2327L43.6765 23.971C43.921 21.6973 44.0825 20.1957 43.9557 19.2497L44 19.25C46.071 19.25 47.75 17.5711 47.75 15.5C47.75 13.4289 46.071 11.75 44 11.75C41.929 11.75 40.25 13.4289 40.25 15.5C40.25 16.4366 40.5935 17.2931 41.1613 17.9503C40.346 18.4535 39.2805 19.515 37.6763 21.1128C36.4405 22.3438 35.8225 22.9593 35.1333 23.0548C34.7513 23.1075 34.3622 23.0532 34.0095 22.898C33.373 22.6175 32.9485 21.8567 32.0997 20.335L27.6262 12.3135C27.1025 11.3747 26.6642 10.5889 26.2692 9.95662C27.89 9.12967 29 7.44445 29 5.5C29 2.73857 26.7615 0.5 24 0.5C21.2385 0.5 19 2.73857 19 5.5C19 7.44445 20.11 9.12967 21.7308 9.95662C21.3358 10.589 20.8975 11.3746 20.3738 12.3135L15.9002 20.335C15.0514 21.8567 14.627 22.6175 13.9905 22.898C13.6379 23.0532 13.2487 23.1075 12.8668 23.0548C12.1774 22.9593 11.5595 22.3438 10.3238 21.1128C8.71968 19.515 7.6539 18.4535 6.83882 17.9503C7.4066 17.2931 7.75 16.4366 7.75 15.5C7.75 13.4289 6.07107 11.75 4 11.75C1.92893 11.75 0.25 13.4289 0.25 15.5C0.25 17.5711 1.92893 19.25 4 19.25L4.04428 19.2497C3.91755 20.1957 4.07905 21.6973 4.32362 23.971L4.782 28.2327C5.03645 30.5982 5.24802 32.849 5.50717 34.875H42.4928C42.752 32.849 42.9635 30.5982 43.218 28.2327Z" fill="#1C274C" />
<path style="fill:#eab308" d="M21.2803 45.5H26.7198C33.8098 45.5 37.3545 45.5 39.7198 43.383C40.7523 42.4588 41.4057 40.793 41.8775 38.625H6.1224C6.59413 40.793 7.24783 42.4588 8.2802 43.383C10.6454 45.5 14.1903 45.5 21.2803 45.5Z" fill="#1C274C" />
</svg>
</div>
"""
# Read VERSION as file with permissions to read from src/
with open("src/VERSION", "r") as f:
version = f.read().strip()
# Get zip file from https://assets.bunkerity.com/bw-pro/preview/v{version}
url = f"https://assets.bunkerity.com/bw-pro/preview/v{version}.zip"
# Download zip
response = requests.get(url)
response.raise_for_status()
Path(f"v{version}.zip").write_bytes(response.content)
# Unzip file
with zipfile.ZipFile(f"v{version}.zip", "r") as zip_ref:
zip_ref.extractall(f"v{version}")
# Print pro settings
print("## Pro plugins", file=doc)
pro_settings = {}
for pro in glob(f"v{version}/*/plugin.json"):
with open(pro, "r") as f:
with suppress(Exception):
pro_plugin = loads(f.read())
if len(pro_plugin["settings"]) > 0:
pro_settings[pro_plugin["name"]] = pro_plugin
for name, data in dict(sorted(pro_settings.items())).items():
print(pro_title("3", data["name"]), file=doc)
print(f"{stream_support(data['stream'])}\n", file=doc)
print(f"{data['description']}\n", file=doc)
print(print_md_table(data["settings"]), file=doc)
# Remove zip file
Path(f"v{version}.zip").unlink()
# Remove folder using shutil
shutil.rmtree(f"v{version}")
doc.seek(0)
content = doc.read()
doc = StringIO(content.replace("\\|", "|"))

30
docs/overrides/main.html
View file

@ -7,8 +7,7 @@
</a>
{% endblock %}
{% block announce %}
📢 Looking for technical support, tailored
consulting or custom development for BunkerWeb ? Visit the
📢 Looking for BunkerWeb PRO version, technical support or tailored services ? Visit the
<a href="https://panel.bunkerweb.io/?utm_campaign=self&utm_source=doc"
style="color: #3f6ec6;
text-decoration: underline">BunkerWeb Panel</a>
@ -19,6 +18,21 @@
defer
data-domain="docs.bunkerweb.io"
src="https://data.bunkerity.com/js/script.js"></script>
<script defer>
window.addEventListener('DOMContentLoaded', (e) => {
const customHeaders = document.querySelectorAll('[data-custom-header]')
customHeaders.forEach(header => {
const getHeaders = document.querySelectorAll(`#${header.getAttribute('id')}`);
getHeaders.forEach(el => {
if(!el.hasAttribute('data-custom-header')) el.remove()
})})
})
</script>
<script defer>
// Lazy load images and embed youtube videos
window.addEventListener("load", () => {
@ -37,14 +51,14 @@
<script defer>
window.addEventListener('DOMContentLoaded', () => {
const bannerEl = document.querySelector('aside.md-banner');
let defaultContent = [{ "content" : 'Need premium support ? <a style="text-decoration:underline; color : white;" href="https://panel.bunkerweb.io/?utm_campaign=self&utm_source=doc">Check BunkerWeb Panel</a>'},
{ "content" : 'Try BunkerWeb on our <a style="text-decoration:underline; color : white;" href="https://demo.bunkerweb.io/link/?utm_campaign=self&utm_source=doc">demo web app !</a>'},
{ "content" : 'All information about BunkerWeb on our <a style="text-decoration:underline; color : white;" href="https://www.bunkerweb.io/?utm_campaign=self&utm_source=doc">website !</a>'}
let defaultContent = [{ "content" : '<p>Get the most of BunkerWeb by upgrading to the PRO version. More info and free trial <a style="text-decoration:underline; color : white;" href="https://panel.bunkerweb.io/?utm_campaign=self&utm_source=doc#pro">here</a>.</p>'},
{ "content" : '<p>Need premium support or tailored consulting around BunkerWeb ? Check out our <a style="text-decoration:underline; color : white;" href="https://panel.bunkerweb.io/?utm_campaign=self&utm_source=doc#services">professional services</a>.</p>'},
{ "content" : '<p>Be part of the Bunker community by joining the <a style="text-decoration:underline; color: white;" href="https://discord.bunkerweb.io">Discord chat</a> and following us on <a style="text-decoration:underline; color: white;" href="https://www.linkedin.com/company/bunkerity/">LinkedIn</a>.</p>'}
]
function setBannerStyle() {
const bannerItem = bannerEl.querySelector('.md-banner__inner');
bannerEl.style.backgroundColor = "#36ce7a";
bannerEl.style.backgroundColor = "#2eac68";
bannerItem.style.textAlign = "center";
bannerItem.style.transition = "all 0.5s ease-out";
}
@ -52,7 +66,7 @@
function setDefault() {
const bannerItem = bannerEl.querySelector('.md-banner__inner');
const clone = bannerItem.cloneNode(true);
clone.innerHTML = defaultContent[defaultContent.length - 1]["content"];
clone.innerHTML = defaultContent[0]["content"];
bannerEl.replaceChild(clone, bannerItem);
}
@ -62,7 +76,7 @@
setInterval(() => {
// Update or reset index
if(i + 1 === defaultContent.length - 1) {
if(i + 1 === defaultContent.length) {
i = 0;
} else {
i++;

116
docs/plugins.md
View file

@ -602,10 +602,6 @@ For example, you can get the request arguments in your template like this :
#### Actions.py
!!! info "Python libraries"
You can use Python libraries that are already available like :
`Flask`, `Flask-Login`, `Flask-WTF`, `beautifulsoup4`, `docker`, `Jinja2`, `python-magic` and `requests`. To see the full list, you can have a look at the Web UI [requirements.txt](https://github.com/bunkerity/bunkerweb/blobsrc/ui/requirements.txt). If you need external libraries, you can install them inside the **ui** folder of your plugin and then use the classical **import** directive.
!!! warning "CSRF Token"
Please note that every form submission is protected via a CSRF token, you will need to include the following snippet into your forms :
@ -613,16 +609,55 @@ For example, you can get the request arguments in your template like this :
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
```
You can power-up your plugin page with additional scripting with the **actions.py** file when sending a **POST /plugins/<*plugin_id*>**.
Here is what is send to the function :
You have two functions by default in **actions.py** :
**pre_render function**
This allows you to retrieve data when you **GET** the template, and to use the data with the pre_render variable available in Jinja to display content more dynamically.
```python
def pre_render(**kwargs)
return <pre_render_data>
```
BunkerWeb will send you this type of response :
```python
return jsonify({"status": "ok|ko", "code" : XXX, "data": <pre_render_data>}), 200
```
**<*plugin_id*> function**
This allows you to retrieve data when you make a **POST** from the template endpoint, which must be used in AJAX.
```python
def myplugin(**kwargs)
return <plugin_id_data>
```
BunkerWeb will send you this type of response :
```python
return jsonify({"message": "ok", "data": <plugin_id_data>}), 200
```
**What you can access from action.py**
Here are the arguments that are passed and access on action.py functions:
```python
function(app=app, args=request.args.to_dict() or request.json or None)
```
Some examples of what you can do :
!!! info "Python libraries"
In addition, you can use Python libraries that are already available like :
`Flask`, `Flask-Login`, `Flask-WTF`, `beautifulsoup4`, `docker`, `Jinja2`, `python-magic` and `requests`. To see the full list, you can have a look at the Web UI [requirements.txt](https://github.com/bunkerity/bunkerweb/blobsrc/ui/requirements.txt). If you need external libraries, you can install them inside the **ui** folder of your plugin and then use the classical **import** directive.
**Some examples**
- Retrieve form submitted data
@ -631,73 +666,22 @@ from flask import request
def myplugin(**kwargs) :
my_form_value = request.form["my_form_input"]
return my_form_value
```
- Access app methods
- Access app config
**action.py**
```python
from flask import request
def myplugin(**kwargs) :
def pre_render(**kwargs) :
config = kwargs['app'].config["CONFIG"].get_config(methods=False)
return config
```
**You need to retrieve JSON compatible data from this function**, app will return this as response :
```python
return jsonify({"message": "ok", "data": <function_output>}), 200
```
#### Updating template
To easily update the content of a template inside the UI with JSON, a **SetupPlugin class** is available in `src/ui/static/js/plugins/setup.js`.
!!! info "Check core plugins"
Core plugins are using this class. Feel free to look at them in order to see in details how it works.
For example, in case **actions.py** return this :
```python
def myplugin(**kwargs):
return {"name": "My awesome plugin"}
```
I can add this on my **template.html** :
**template**
```html
<p data-name></p>
<script>
new SetupPlugin({
name: {
el: document.querySelector('[data-name]'),
value: '',
type: 'text',
},
});
</script>
<!-- metadata + config -->
<div>{{ pre_render }}</div>
```
**This class will send a POST request, and will try to match the dict key to a JSON key and update your template**.
Here it will look for a `name` key in the JSON response, and will set the `data` on the defined `el`.
In case there is no `data` matching, this will keep or set the `value` key data.
This class has two arguments `SetupPlugin(setup, url)` :
- `url`(optional) : current endpoint by default. You can define another url or add arguments.
- `setup` : a dict of dict with needed data to update properly the template with the incoming data.
**setup details**
| key | Type | Description |
| :--------: | :--------: | :------------------------------------------------------------------------------------------- |
| `dict name` | string | Replace `dict name` by the JSON key to extract the related value. |
| `el` | DOM element| Select element you want the value to be updated. |
| `value` | any | Default value on template load or in case retrieving JSON failed. |
| `type` | string | Define the script behavior with the incoming value. Available : `text`, `list` and `status`. |
| `textEl` | DOM element| Optional additional text content when type is `status`. |
| `listNames`| string | List of data keys when type is `list`. |

2
docs/requirements.in
View file

@ -1,5 +1,5 @@
mike==2.0.0
mkdocs==1.5.3
mkdocs-material[imaging]==9.5.12
mkdocs-material[imaging]==9.5.13
mkdocs-print-site-plugin==2.3.6
pytablewriter==1.2.0

48
docs/requirements.txt
View file

@ -200,16 +200,16 @@ idna==3.6 \
--hash=sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca \
--hash=sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f
# via requests
importlib-metadata==7.0.1 \
--hash=sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e \
--hash=sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc
importlib-metadata==7.0.2 \
--hash=sha256:198f568f3230878cb1b44fbd7975f87906c22336dba2e4a7f05278c281fbd792 \
--hash=sha256:f4bc4c0c070c490abf4ce96d715f68e95923320370efb66143df00199bb6c100
# via
# markdown
# mike
# mkdocs
importlib-resources==6.1.2 \
--hash=sha256:308abf8474e2dba5f867d279237cd4076482c3de7104a40b41426370e891549b \
--hash=sha256:9a0a862501dc38b68adebc82970140c9e4209fc99601782925178f8386339938
importlib-resources==6.3.0 \
--hash=sha256:166072a97e86917a9025876f34286f549b9caf1d10b35a1b372bffa1600c6569 \
--hash=sha256:783407aa1cd05550e3aa123e8f7cfaebee35ffa9cb0242919e2d1e4172222705
# via mike
jinja2==3.1.3 \
--hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \
@ -218,9 +218,9 @@ jinja2==3.1.3 \
# mike
# mkdocs
# mkdocs-material
markdown==3.5.2 \
--hash=sha256:d43323865d89fc0cb9b20c75fc8ad313af307cc087e84b657d9eec768eddeadd \
--hash=sha256:e1ac7b3dc550ee80e602e71c1d168002f062e49f1b11e26a36264dafd4df2ef8
markdown==3.6 \
--hash=sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f \
--hash=sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224
# via
# mkdocs
# mkdocs-material
@ -311,9 +311,9 @@ mkdocs==1.5.3 \
# -r requirements.in
# mike
# mkdocs-material
mkdocs-material==9.5.12 \
--hash=sha256:5f69cef6a8aaa4050b812f72b1094fda3d079b9a51cf27a247244c03ec455e97 \
--hash=sha256:d6f0c269f015e48c76291cdc79efb70f7b33bbbf42d649cfe475522ebee61b1f
mkdocs-material==9.5.13 \
--hash=sha256:5cbe17fee4e3b4980c8420a04cc762d8dc052ef1e10532abd4fce88e5ea9ce6a \
--hash=sha256:d8e4caae576312a88fd2609b81cf43d233cdbe36860d67a68702b018b425bd87
# via
# -r requirements.in
# mkdocs-print-site-plugin
@ -325,9 +325,9 @@ mkdocs-print-site-plugin==2.3.6 \
--hash=sha256:01ccb1ceccc87f29e1612bebb77c3bf9980809fbce750fc2113f9d6acea589d4 \
--hash=sha256:82e5cabcfb7fe3074daecea018f28ccb4bff086f965e3103fe91019a76752f22
# via -r requirements.in
packaging==23.2 \
--hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \
--hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7
packaging==24.0 \
--hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \
--hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9
# via
# mkdocs
# typepy
@ -430,9 +430,9 @@ pymdown-extensions==10.7.1 \
--hash=sha256:c70e146bdd83c744ffc766b4671999796aba18842b268510a329f7f64700d584 \
--hash=sha256:f5cc7000d7ff0d1ce9395d216017fa4df3dde800afb1fb72d1c7d3fd35e710f4
# via mkdocs-material
pyparsing==3.1.1 \
--hash=sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb \
--hash=sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db
pyparsing==3.1.2 \
--hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \
--hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742
# via mike
pytablewriter==1.2.0 \
--hash=sha256:0204a4bb684a22140d640f2599f09e137bcdc18b3dd49426f4a555016e246b46 \
@ -612,9 +612,9 @@ requests==2.31.0 \
# importlib-resources
# The following packages are considered to be unsafe in a requirements file:
setuptools==69.1.1 \
--hash=sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56 \
--hash=sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8
setuptools==69.2.0 \
--hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \
--hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c
# via mkdocs-material
six==1.16.0 \
--hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
@ -686,7 +686,7 @@ webencodings==0.5.1 \
# via
# cssselect2
# tinycss2
zipp==3.17.0 \
--hash=sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31 \
--hash=sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0
zipp==3.18.1 \
--hash=sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b \
--hash=sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715
# via pytablewriter

96
docs/security-tuning.md
View file

@ -502,3 +502,99 @@ You can deploy complex authentication (e.g. SSO), by using the auth request sett
| `REVERSE_PROXY_AUTH_REQUEST` | | multisite | yes | Enable authentication using an external provider (value of auth_request directive). |
| `REVERSE_PROXY_AUTH_REQUEST_SIGNIN_URL` | | multisite | yes | Redirect clients to sign-in URL when using REVERSE_PROXY_AUTH_REQUEST (used when auth_request call returned 401). |
| `REVERSE_PROXY_AUTH_REQUEST_SET` | | multisite | yes | List of variables to set from the authentication provider, separated with ; (values of auth_request_set directives). |
## Monitoring and reporting
Monitoring and reporting means that you are kept informed of the slightest problem and can react as quickly as possible.
### Reporting
<div style="display:flex; align-items:center">
<h3 data-custom-header id="reporting">Reporting</h3>
<svg style="height:1.25rem; width:1.25rem; margin-top: 0.70rem; margin-left: 0.5rem"
viewBox="0 0 48 46"
fill="none"
xmlns="http://www.w3.org/2000/svg">
<path style="fill:#eab308" d="M43.218 28.2327L43.6765 23.971C43.921 21.6973 44.0825 20.1957 43.9557 19.2497L44 19.25C46.071 19.25 47.75 17.5711 47.75 15.5C47.75 13.4289 46.071 11.75 44 11.75C41.929 11.75 40.25 13.4289 40.25 15.5C40.25 16.4366 40.5935 17.2931 41.1613 17.9503C40.346 18.4535 39.2805 19.515 37.6763 21.1128C36.4405 22.3438 35.8225 22.9593 35.1333 23.0548C34.7513 23.1075 34.3622 23.0532 34.0095 22.898C33.373 22.6175 32.9485 21.8567 32.0997 20.335L27.6262 12.3135C27.1025 11.3747 26.6642 10.5889 26.2692 9.95662C27.89 9.12967 29 7.44445 29 5.5C29 2.73857 26.7615 0.5 24 0.5C21.2385 0.5 19 2.73857 19 5.5C19 7.44445 20.11 9.12967 21.7308 9.95662C21.3358 10.589 20.8975 11.3746 20.3738 12.3135L15.9002 20.335C15.0514 21.8567 14.627 22.6175 13.9905 22.898C13.6379 23.0532 13.2487 23.1075 12.8668 23.0548C12.1774 22.9593 11.5595 22.3438 10.3238 21.1128C8.71968 19.515 7.6539 18.4535 6.83882 17.9503C7.4066 17.2931 7.75 16.4366 7.75 15.5C7.75 13.4289 6.07107 11.75 4 11.75C1.92893 11.75 0.25 13.4289 0.25 15.5C0.25 17.5711 1.92893 19.25 4 19.25L4.04428 19.2497C3.91755 20.1957 4.07905 21.6973 4.32362 23.971L4.782 28.2327C5.03645 30.5982 5.24802 32.849 5.50717 34.875H42.4928C42.752 32.849 42.9635 30.5982 43.218 28.2327Z" fill="#1C274C" />
<path style="fill:#eab308" d="M21.2803 45.5H26.7198C33.8098 45.5 37.3545 45.5 39.7198 43.383C40.7523 42.4588 41.4057 40.793 41.8775 38.625H6.1224C6.59413 40.793 7.24783 42.4588 8.2802 43.383C10.6454 45.5 14.1903 45.5 21.2803 45.5Z" fill="#1C274C" />
</svg>
</div>
!!! warning "Used of cache data"
A comparison is made every hour with the cached data. If BunkerWeb no longer has access to the cache, the data to be compared will be reset.
#### Types of reporting
Pro reporting plugin gives you two types of reports :
- **regular report**: you can define a period of time, and you'll get a regular report showing the percentage change in data between the previous report and this one, and also key points about your BunkerWeb state.
- **alerts**: every hour, an analysis of the metrics will be carried out, and you can set a threshold for the percentage change in the data. If this threshold is reached, you will receive an alert.
!!! info "Example"
After one hour, if I go from 300 requests blocked to more than 600 after one hour : in case I have set a threshold of +100%, I'll be alerted.
#### Get reporting
To receive alerts or regular reports, you can use :
**1) webhook**
We are supporting multiple webhooks :
- **API** : we will send a JSON of type `{"message" : markdownReport }`.
- **Discord**
- **Slack**
!!! info "Specific webhook"
We listen to our customers, so if you need to make the plugin compatible with a particular webhook, don't hesitate to contact us to discuss it together.
**2) SMTP**
You can also use the SMTP protocol. You will need to set the various parameters (user auth, password auth, host...).
You need to **pay attention** using SMTP:
- Make sure that the address used to send the **message does not end up in the spam folder**.
- The address used must **not have double authentication** to work.
### Prometheus exporter
<div style="display:flex; align-items:center">
<h3 data-custom-header id="prometheus-exporter">Prometheus exporter</h3>
<svg style="height:1.25rem; width:1.25rem; margin-top: 0.70rem; margin-left: 0.5rem"
viewBox="0 0 48 46"
fill="none"
xmlns="http://www.w3.org/2000/svg">
<path style="fill:#eab308" d="M43.218 28.2327L43.6765 23.971C43.921 21.6973 44.0825 20.1957 43.9557 19.2497L44 19.25C46.071 19.25 47.75 17.5711 47.75 15.5C47.75 13.4289 46.071 11.75 44 11.75C41.929 11.75 40.25 13.4289 40.25 15.5C40.25 16.4366 40.5935 17.2931 41.1613 17.9503C40.346 18.4535 39.2805 19.515 37.6763 21.1128C36.4405 22.3438 35.8225 22.9593 35.1333 23.0548C34.7513 23.1075 34.3622 23.0532 34.0095 22.898C33.373 22.6175 32.9485 21.8567 32.0997 20.335L27.6262 12.3135C27.1025 11.3747 26.6642 10.5889 26.2692 9.95662C27.89 9.12967 29 7.44445 29 5.5C29 2.73857 26.7615 0.5 24 0.5C21.2385 0.5 19 2.73857 19 5.5C19 7.44445 20.11 9.12967 21.7308 9.95662C21.3358 10.589 20.8975 11.3746 20.3738 12.3135L15.9002 20.335C15.0514 21.8567 14.627 22.6175 13.9905 22.898C13.6379 23.0532 13.2487 23.1075 12.8668 23.0548C12.1774 22.9593 11.5595 22.3438 10.3238 21.1128C8.71968 19.515 7.6539 18.4535 6.83882 17.9503C7.4066 17.2931 7.75 16.4366 7.75 15.5C7.75 13.4289 6.07107 11.75 4 11.75C1.92893 11.75 0.25 13.4289 0.25 15.5C0.25 17.5711 1.92893 19.25 4 19.25L4.04428 19.2497C3.91755 20.1957 4.07905 21.6973 4.32362 23.971L4.782 28.2327C5.03645 30.5982 5.24802 32.849 5.50717 34.875H42.4928C42.752 32.849 42.9635 30.5982 43.218 28.2327Z" fill="#1C274C" />
<path style="fill:#eab308" d="M21.2803 45.5H26.7198C33.8098 45.5 37.3545 45.5 39.7198 43.383C40.7523 42.4588 41.4057 40.793 41.8775 38.625H6.1224C6.59413 40.793 7.24783 42.4588 8.2802 43.383C10.6454 45.5 14.1903 45.5 21.2803 45.5Z" fill="#1C274C" />
</svg>
</div>
TO DO
### Pro metrics
<div style="display:flex; align-items:center">
<h3 data-custom-header id="pro-metrics">Pro metrics</h3>
<svg style="height:1.25rem; width:1.25rem; margin-top: 0.70rem; margin-left: 0.5rem"
viewBox="0 0 48 46"
fill="none"
xmlns="http://www.w3.org/2000/svg">
<path style="fill:#eab308" d="M43.218 28.2327L43.6765 23.971C43.921 21.6973 44.0825 20.1957 43.9557 19.2497L44 19.25C46.071 19.25 47.75 17.5711 47.75 15.5C47.75 13.4289 46.071 11.75 44 11.75C41.929 11.75 40.25 13.4289 40.25 15.5C40.25 16.4366 40.5935 17.2931 41.1613 17.9503C40.346 18.4535 39.2805 19.515 37.6763 21.1128C36.4405 22.3438 35.8225 22.9593 35.1333 23.0548C34.7513 23.1075 34.3622 23.0532 34.0095 22.898C33.373 22.6175 32.9485 21.8567 32.0997 20.335L27.6262 12.3135C27.1025 11.3747 26.6642 10.5889 26.2692 9.95662C27.89 9.12967 29 7.44445 29 5.5C29 2.73857 26.7615 0.5 24 0.5C21.2385 0.5 19 2.73857 19 5.5C19 7.44445 20.11 9.12967 21.7308 9.95662C21.3358 10.589 20.8975 11.3746 20.3738 12.3135L15.9002 20.335C15.0514 21.8567 14.627 22.6175 13.9905 22.898C13.6379 23.0532 13.2487 23.1075 12.8668 23.0548C12.1774 22.9593 11.5595 22.3438 10.3238 21.1128C8.71968 19.515 7.6539 18.4535 6.83882 17.9503C7.4066 17.2931 7.75 16.4366 7.75 15.5C7.75 13.4289 6.07107 11.75 4 11.75C1.92893 11.75 0.25 13.4289 0.25 15.5C0.25 17.5711 1.92893 19.25 4 19.25L4.04428 19.2497C3.91755 20.1957 4.07905 21.6973 4.32362 23.971L4.782 28.2327C5.03645 30.5982 5.24802 32.849 5.50717 34.875H42.4928C42.752 32.849 42.9635 30.5982 43.218 28.2327Z" fill="#1C274C" />
<path style="fill:#eab308" d="M21.2803 45.5H26.7198C33.8098 45.5 37.3545 45.5 39.7198 43.383C40.7523 42.4588 41.4057 40.793 41.8775 38.625H6.1224C6.59413 40.793 7.24783 42.4588 8.2802 43.383C10.6454 45.5 14.1903 45.5 21.2803 45.5Z" fill="#1C274C" />
</svg>
</div>
TO DO

50
docs/settings.md
View file

@ -315,27 +315,17 @@ Manage HTTP headers sent to clients.
|`X_CONTENT_TYPE_OPTIONS` |`nosniff` |multisite|no |Value for the X-Content-Type-Options header. |
|`X_XSS_PROTECTION` |`1; mode=block` |multisite|no |Value for the X-XSS-Protection header. |
### Jobs
STREAM support :white_check_mark:
Fake core plugin for internal jobs.
| Setting |Default|Context|Multiple| Description |
|-----------------------|-------|-------|--------|-----------------------------------------------|
|`SEND_ANONYMOUS_REPORT`|`yes` |global |no |Send anonymous report to BunkerWeb maintainers.|
### Let's Encrypt
STREAM support :white_check_mark:
Automatic creation, renewal and configuration of Let's Encrypt certificates.
| Setting |Default| Context |Multiple| Description |
|--------------------------|-------|---------|--------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|`AUTO_LETS_ENCRYPT` |`no` |multisite|no |Activate automatic Let's Encrypt mode. |
|`EMAIL_LETS_ENCRYPT` | |multisite|no |Email used for Let's Encrypt notification and in certificate. |
|`USE_LETS_ENCRYPT_STAGING`|`no` |multisite|no |Use the staging environment for Let’s Encrypt certificate generation. Useful when you are testing your deployments to avoid being rate limited in the production environment.|
| Setting |Default| Context |Multiple| Description |
|--------------------------|-------|---------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|`AUTO_LETS_ENCRYPT` |`no` |multisite|no |Activate automatic Let's Encrypt mode. |
|`EMAIL_LETS_ENCRYPT` | |multisite|no |Email used for Let's Encrypt notification and in certificate. |
|`USE_LETS_ENCRYPT_STAGING`|`no` |multisite|no |Use the staging environment for Let’s Encrypt certificate generation. Useful when you are testing your deployments to avoid being rate limited in the production environment.|
### Limit
@ -390,6 +380,7 @@ Miscellaneous settings.
|`OPEN_FILE_CACHE_VALID` |`30s` |multisite|no |Open file cache valid time |
|`EXTERNAL_PLUGIN_URLS` | |global |no |List of external plugins URLs (direct download to .zip or .tar file) to download and install (URLs are separated with space).|
|`DENY_HTTP_STATUS` |`403` |global |no |HTTP status code to send when the request is denied (403 or 444). When using 444, BunkerWeb will close the connection. |
|`SEND_ANONYMOUS_REPORT` |`yes` |global |no |Send anonymous report to BunkerWeb maintainers. |
### ModSecurity
@ -587,3 +578,32 @@ Allow access based on internal and external IP/network/rDNS/ASN whitelists.
|`WHITELIST_USER_AGENT_URLS`| |global |no |List of URLs, separated with spaces, containing good User-Agent to whitelist. |
|`WHITELIST_URI` | |multisite|no |List of URI (PCRE regex), separated with spaces, to whitelist. |
|`WHITELIST_URI_URLS` | |global |no |List of URLs, separated with spaces, containing bad URI to whitelist. |
## Pro plugins
### Prometheus exporter
<div style="display:flex; align-items:center">
<h3 data-custom-header id="prometheus-exporter">Prometheus exporter</h3>
<svg style="height:1.25rem; width:1.25rem; margin-top: 0.70rem; margin-left: 0.5rem"
viewBox="0 0 48 46"
fill="none"
xmlns="http://www.w3.org/2000/svg">
<path style="fill:#eab308" d="M43.218 28.2327L43.6765 23.971C43.921 21.6973 44.0825 20.1957 43.9557 19.2497L44 19.25C46.071 19.25 47.75 17.5711 47.75 15.5C47.75 13.4289 46.071 11.75 44 11.75C41.929 11.75 40.25 13.4289 40.25 15.5C40.25 16.4366 40.5935 17.2931 41.1613 17.9503C40.346 18.4535 39.2805 19.515 37.6763 21.1128C36.4405 22.3438 35.8225 22.9593 35.1333 23.0548C34.7513 23.1075 34.3622 23.0532 34.0095 22.898C33.373 22.6175 32.9485 21.8567 32.0997 20.335L27.6262 12.3135C27.1025 11.3747 26.6642 10.5889 26.2692 9.95662C27.89 9.12967 29 7.44445 29 5.5C29 2.73857 26.7615 0.5 24 0.5C21.2385 0.5 19 2.73857 19 5.5C19 7.44445 20.11 9.12967 21.7308 9.95662C21.3358 10.589 20.8975 11.3746 20.3738 12.3135L15.9002 20.335C15.0514 21.8567 14.627 22.6175 13.9905 22.898C13.6379 23.0532 13.2487 23.1075 12.8668 23.0548C12.1774 22.9593 11.5595 22.3438 10.3238 21.1128C8.71968 19.515 7.6539 18.4535 6.83882 17.9503C7.4066 17.2931 7.75 16.4366 7.75 15.5C7.75 13.4289 6.07107 11.75 4 11.75C1.92893 11.75 0.25 13.4289 0.25 15.5C0.25 17.5711 1.92893 19.25 4 19.25L4.04428 19.2497C3.91755 20.1957 4.07905 21.6973 4.32362 23.971L4.782 28.2327C5.03645 30.5982 5.24802 32.849 5.50717 34.875H42.4928C42.752 32.849 42.9635 30.5982 43.218 28.2327Z" fill="#1C274C" />
<path style="fill:#eab308" d="M21.2803 45.5H26.7198C33.8098 45.5 37.3545 45.5 39.7198 43.383C40.7523 42.4588 41.4057 40.793 41.8775 38.625H6.1224C6.59413 40.793 7.24783 42.4588 8.2802 43.383C10.6454 45.5 14.1903 45.5 21.2803 45.5Z" fill="#1C274C" />
</svg>
</div>
STREAM support :x:
Prometheus export for BunkerWeb
| Setting | Default |Context|Multiple| Description |
|-------------------------------|-----------------------------------------------------|-------|--------|------------------------------------------------------------------------|
|`USE_PROMETHEUS_EXPORTER` |`no` |global |no |Enable the Prometheus export. |
|`PROMETHEUS_EXPORTER_IP` |`0.0.0.0` |global |no |Listening IP of the Prometheus exporter. |
|`PROMETHEUS_EXPORTER_PORT` |`9113` |global |no |Listening port of the Prometheus exporter. |
|`PROMETHEUS_EXPORTER_DICT_SIZE`|`10M` |global |no |Size of the dict to store Prometheus metrics. |
|`PROMETHEUS_EXPORTER_ALLOW_IP` |`127.0.0.1/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16`|global |no |List of IP/networks allowed to contact the Prometheus exporter endpoint.|
|`PROMETHEUS_EXPORTER_URL` |`/metrics` |global |no |HTTP URL of the Prometheus exporter. |

6
docs/web-ui.md
View file

@ -1738,11 +1738,9 @@ In case you have buy a (pro version)[https://panel.bunkerweb.io/?utm_campaign=se
- fill the **setting Pro License Key**
- **save** your changes
!!! warning "Download"
<figure markdown>
![Overview](assets/img/pro-from-ui.webp){ align=center, width="1000" }
<figcaption>Upgrade to PRO from UI</figcaption>
</figure>
The pro version is downloaded in the background by the scheduler. It may take some time before you see the changes to the UI.
If your license key is valid, the upgrade to the pro version will take place automatically in the background.

34
src/autoconf/Dockerfile
View file

@ -1,27 +1,19 @@
FROM python:3.12.2-alpine3.19@sha256:1a0501213b470de000d8432b3caab9d8de5489e9443c2cc7ccaa6b0aa5c3148e as builder
# Install python dependencies
RUN apk add --no-cache build-base postgresql-dev
# Copy python requirements
COPY src/deps/requirements.txt /tmp/requirements-deps.txt
COPY src/common/gen/requirements.txt /tmp/req/requirements.txt
COPY src/common/db/requirements.txt /tmp/req/requirements.txt.1
COPY src/common/gen/requirements.txt /tmp/req/requirements-gen.txt
COPY src/common/db/requirements.txt /tmp/req/requirements-db.txt
WORKDIR /usr/share/bunkerweb
RUN mkdir -p deps/python && \
cat /tmp/req/requirements.txt* > deps/requirements.txt && \
rm -rf /tmp/req
# Install python dependencies
RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev libffi-dev cairo-dev pango-dev gdk-pixbuf-dev openssl-dev cargo postgresql-dev
# Install python requirements
RUN export MAKEFLAGS="-j$(nproc)" && \
pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
# Remove build dependencies
RUN apk del .build-deps && \
rm -rf /var/cache/apk/*
pip install --no-cache-dir --require-hashes --break-system-packages -r /tmp/requirements-deps.txt && \
pip install --no-cache-dir --require-hashes --target deps/python $(for file in $(ls /tmp/req/requirements*.txt) ; do echo "-r ${file}" ; done | xargs)
# Copy files
# can't exclude specific files/dir from . so we are copying everything by hand
@ -49,9 +41,8 @@ RUN apk add --no-cache bash && \
addgroup -g 101 autoconf && \
adduser -h /var/cache/autoconf -g autoconf -s /bin/sh -G autoconf -D -H -u 101 autoconf && \
cp helpers/bwcli /usr/bin/ && \
mkdir -p /var/tmp/bunkerweb && \
mkdir -p /var/www && \
mkdir -p /etc/bunkerweb && \
echo "Docker" > INTEGRATION && \
mkdir -p /etc/bunkerweb /var/tmp/bunkerweb /var/run/bunkerweb /var/log/bunkerweb /var/www && \
mkdir -p /data/cache && ln -s /data/cache /var/cache/bunkerweb && \
mkdir -p /data/lib && ln -s /data/lib /var/lib/bunkerweb && \
mkdir -p /data/www && ln -s /data/www /var/www/html && \
@ -59,14 +50,15 @@ RUN apk add --no-cache bash && \
for dir in $(echo "configs/http configs/stream configs/server-http configs/server-stream configs/default-server-http configs/default-server-stream configs/modsec configs/modsec-crs") ; do mkdir "/data/${dir}" ; done && \
chown -R root:autoconf /data && \
chmod -R 770 /data && \
chown -R root:autoconf /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /usr/bin/bwcli && \
chown -R root:autoconf INTEGRATION /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /usr/bin/bwcli && \
chmod -R 770 /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb && \
chmod 750 cli/main.py helpers/*.sh /usr/bin/bwcli autoconf/main.py deps/python/bin/*
chmod 750 cli/main.py helpers/*.sh /usr/bin/bwcli autoconf/main.py deps/python/bin/* && \
chmod 660 INTEGRATION
# Fix CVEs
RUN apk add --no-cache "libexpat>=2.6.0-r0"
VOLUME /data /etc/nginx
VOLUME /data
WORKDIR /usr/share/bunkerweb/autoconf

32
src/bw/Dockerfile
View file

@ -1,7 +1,7 @@
FROM nginx:1.24.0-alpine@sha256:6845649eadc1f0a5dacaf5bb3f01b480ce200ae1249114be11fef9d389196eaf AS builder
FROM nginx:1.24.0-alpine-slim@sha256:9cec4fd40a4e5156b4f4f555ee44a597491b6e8b91380c32b63ed45a4053a763 AS builder
# Install temporary requirements for the dependencies
RUN apk add --no-cache --virtual .build-deps bash autoconf libtool automake geoip-dev g++ gcc curl-dev libxml2-dev pcre-dev make linux-headers musl-dev gd-dev gnupg brotli-dev openssl-dev patch readline-dev yajl yajl-dev yajl-tools py3-pip
RUN apk add --no-cache bash autoconf libtool automake geoip-dev g++ gcc curl-dev libxml2-dev pcre-dev make linux-headers musl-dev gd-dev gnupg brotli-dev openssl-dev patch readline-dev yajl yajl-dev yajl-tools py3-pip
WORKDIR /tmp/bunkerweb/deps
@ -9,27 +9,21 @@ WORKDIR /tmp/bunkerweb/deps
COPY src/deps/misc misc
COPY src/deps/src src
COPY src/deps/deps.json deps.json
COPY src/deps/install.sh install.sh
COPY --chmod=644 src/deps/install.sh install.sh
# Compile and install dependencies
RUN mkdir -p /usr/share/bunkerweb/deps/python && \
chmod +x install.sh && \
bash install.sh
RUN bash install.sh
WORKDIR /usr/share/bunkerweb
# Copy python requirements
COPY src/deps/requirements.txt /tmp/requirements-deps.txt
COPY src/common/gen/requirements.txt deps/requirements.txt
COPY src/common/gen/requirements.txt deps/requirements-gen.txt
# Install python requirements
RUN export MAKEFLAGS="-j$(nproc)" && \
pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
# Clean up temporary dependencies
RUN apk del .build-deps && \
rm -rf /var/cache/apk/*
pip install --no-cache-dir --require-hashes --ignore-installed -r /tmp/requirements-deps.txt && \
pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements-gen.txt
# Copy files
# can't exclude deps from . so we are copying everything by hand
@ -48,7 +42,7 @@ COPY src/common/utils utils
COPY src/VERSION VERSION
COPY misc/*.ascii misc/
FROM nginx:1.24.0-alpine@sha256:76ca7f6bfe01c3e22e3af85fd37c30149ece3ac2a444973687cab1765abca115
FROM nginx:1.24.0-alpine-slim@sha256:9cec4fd40a4e5156b4f4f555ee44a597491b6e8b91380c32b63ed45a4053a763
# Set default umask to prevent huge recursive chmod increasing the final image size
RUN umask 027
@ -59,18 +53,14 @@ COPY --from=builder --chown=0:101 /usr/share/bunkerweb /usr/share/bunkerweb
WORKDIR /usr/share/bunkerweb
# Install runtime dependencies, pypi packages, move bwcli, create data folders and set permissions
RUN apk add --no-cache pcre bash python3 yajl && \
RUN apk add --no-cache openssl pcre bash python3 yajl geoip libxml2 libgd && \
cp helpers/bwcli /usr/bin/ && \
mkdir -p /var/tmp/bunkerweb && \
mkdir -p /var/run/bunkerweb && \
mkdir -p /var/log/bunkerweb && \
mkdir -p /var/www/html && \
mkdir -p /etc/bunkerweb && \
mkdir -p /etc/bunkerweb /var/tmp/bunkerweb /var/run/bunkerweb /var/log/bunkerweb /var/www/html && \
mkdir -p /data/cache && ln -s /data/cache /var/cache/bunkerweb && \
for dir in $(echo "pro configs plugins") ; do mkdir -p "/data/${dir}" && ln -s "/data/${dir}" "/etc/bunkerweb/${dir}" ; done && \
for dir in $(echo "pro/plugins configs/http configs/stream configs/server-http configs/server-stream configs/default-server-http configs/default-server-stream configs/modsec configs/modsec-crs") ; do mkdir "/data/${dir}" ; done && \
chown -R root:nginx /data /etc/nginx /var/cache/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /var/run/bunkerweb /var/log/bunkerweb /usr/bin/bwcli && \
chmod -R 770 /data /etc/nginx /var/cache/bunkerweb /var/tmp/bunkerweb /var/log/bunkerweb /var/run/bunkerweb && \
chmod -R 770 /data /etc/nginx /var/cache/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /var/log/bunkerweb /var/run/bunkerweb && \
chmod 750 cli/main.py gen/main.py helpers/*.sh entrypoint.sh /usr/bin/bwcli deps/python/bin/* && \
rm -f /var/log/bunkerweb/* && \
ln -s /proc/1/fd/2 /var/log/bunkerweb/error.log && \

2
src/bw/lua/bunkerweb/api.lua
View file

@ -94,10 +94,12 @@ end
api.global.POST["^/reload$"] = function(self)
-- Check config
logger:log(NOTICE, "Checking Nginx configuration")
local status = execute("nginx -t")
if status ~= 0 then
return self:response(HTTP_INTERNAL_SERVER_ERROR, "error", "config check failed")
end
logger:log(NOTICE, "Nginx configuration is valid, reloading Nginx")
-- Send HUP signal to master process
local ok, err = kill(get_master_pid(), "HUP")
if not ok then

4
src/bw/lua/bunkerweb/mmdb.lua
View file

@ -1,6 +1,6 @@
local geoip = require "geoip.mmdb"
return {
country_db = geoip.load_database "/var/cache/bunkerweb/country.mmdb",
asn_db = geoip.load_database "/var/cache/bunkerweb/asn.mmdb",
country_db = geoip.load_database "/var/cache/bunkerweb/jobs/country.mmdb",
asn_db = geoip.load_database "/var/cache/bunkerweb/jobs/asn.mmdb",
}

9
src/common/confs/default-server-http.conf
View file

@ -17,7 +17,7 @@ server {
# HTTPS listen
{% set os = import("os") %}
{% if os.path.isfile("/var/cache/bunkerweb/default-server-cert/cert.pem") +%}
{% if os.path.isfile("/var/cache/bunkerweb/misc/default-server-cert.pem") +%}
ssl_protocols {{ SSL_PROTOCOLS }};
ssl_prefer_server_ciphers on;
ssl_session_tickets off;
@ -27,8 +27,8 @@ server {
ssl_dhparam /etc/nginx/dhparam;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
{% endif %}
ssl_certificate /var/cache/bunkerweb/default-server-cert/cert.pem;
ssl_certificate_key /var/cache/bunkerweb/default-server-cert/cert.key;
ssl_certificate /var/cache/bunkerweb/misc/default-server-cert.pem;
ssl_certificate_key /var/cache/bunkerweb/misc/default-server-cert.key;
listen 0.0.0.0:{{ HTTPS_PORT }} ssl {% if HTTP2 == "yes" %}http2{% endif %} default_server {% if USE_PROXY_PROTOCOL == "yes" %}proxy_protocol{% endif %};
{% if USE_IPV6 == "yes" +%}
listen [::]:{{ HTTPS_PORT }} ssl {% if HTTP2 == "yes" %}http2{% endif %} default_server {% if USE_PROXY_PROTOCOL == "yes" %}proxy_protocol{% endif %};
@ -38,6 +38,9 @@ server {
{% if IS_LOADING == "yes" +%}
root /usr/share/bunkerweb/loading;
try_files /index.html =404;
etag off;
add_header Last-Modified "";
server_tokens off;
{% endif %}
# include core and plugins default-server configurations

4
src/common/confs/server-http/ssl-certificate-lua.conf
View file

@ -1,5 +1,5 @@
ssl_certificate /var/cache/bunkerweb/default-server-cert/cert.pem;
ssl_certificate_key /var/cache/bunkerweb/default-server-cert/cert.key;
ssl_certificate /var/cache/bunkerweb/misc/default-server-cert.pem;
ssl_certificate_key /var/cache/bunkerweb/misc/default-server-cert.key;
ssl_protocols {{ SSL_PROTOCOLS }};
ssl_prefer_server_ciphers on;
ssl_session_tickets off;

4
src/common/confs/server-stream/ssl-certificate-stream-lua.conf
View file

@ -1,5 +1,5 @@
ssl_certificate /var/cache/bunkerweb/default-server-cert/cert.pem;
ssl_certificate_key /var/cache/bunkerweb/default-server-cert/cert.key;
ssl_certificate /var/cache/bunkerweb/misc/default-server-cert.pem;
ssl_certificate_key /var/cache/bunkerweb/misc/default-server-cert.key;
ssl_protocols {{ SSL_PROTOCOLS }};
ssl_prefer_server_ciphers on;
ssl_session_tickets off;

23
src/common/core/antibot/ui/actions.py
View file

@ -1,11 +1,18 @@
def antibot(**kwargs):
def pre_render(**kwargs):
try:
data = kwargs["app"].config["INSTANCES"].get_metrics("antibot")
if data.get("counter_failed_challenges") is None:
data["counter_failed_challenges"] = 0
return data
return {
"counter_failed_challenges": {
"value": data.get("counter_failed_challenges", 0),
"title": "Challenge",
"subtitle": "Failed",
"subtitle_color": "info",
"svg_color": "blue",
}
}
except:
return {"counter_failed_challenges": 0}
return {"counter_failed_challenges": {"value": "unknown", "title": "Challenge", "subtitle": "Failed", "subtitle_color": "info", "svg_color": "blue"}}
def antibot(**kwargs):
pass

147
src/common/core/antibot/ui/template.html
View file

@ -7,52 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-info-title">INFO</h5>
<div class="core-card-info-list">
<p data-info class="core-card-info-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">Challenges</p>
<h5 data-count class="core-card-title">"unknown"</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">total failed</span>
</p>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-metrics-svg-container blue">
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-base core-card-metrics-svg">
<path d="M11.7 2.805a.75.75 0 0 1 .6 0A60.65 60.65 0 0 1 22.83 8.72a.75.75 0 0 1-.231 1.337 49.948 49.948 0 0 0-9.902 3.912l-.003.002c-.114.06-.227.119-.34.18a.75.75 0 0 1-.707 0A50.88 50.88 0 0 0 7.5 12.173v-.224c0-.131.067-.248.172-.311a54.615 54.615 0 0 1 4.653-2.52.75.75 0 0 0-.65-1.352 56.123 56.123 0 0 0-4.78 2.589 1.858 1.858 0 0 0-.859 1.228 49.803 49.803 0 0 0-4.634-1.527.75.75 0 0 1-.231-1.337A60.653 60.653 0 0 1 11.7 2.805Z" />
<path d="M13.06 15.473a48.45 48.45 0 0 1 7.666-3.282c.134 1.414.22 2.843.255 4.284a.75.75 0 0 1-.46.711 47.87 47.87 0 0 0-8.105 4.342.75.75 0 0 1-.832 0 47.87 47.87 0 0 0-8.104-4.342.75.75 0 0 1-.461-.71c.035-1.442.121-2.87.255-4.286.921.304 1.83.634 2.726.99v1.27a1.5 1.5 0 0 0-.14 2.508c-.09.38-.222.753-.397 1.11.452.213.901.434 1.346.66a6.727 6.727 0 0 0 .551-1.607 1.5 1.5 0 0 0 .14-2.67v-.645a48.549 48.549 0 0 1 3.44 1.667 2.25 2.25 0 0 0 2.12 0Z" />
<path d="M4.462 19.462c.42-.419.753-.89 1-1.395.453.214.902.435 1.347.662a6.742 6.742 0 0 1-1.286 1.794.75.75 0 0 1-1.06-1.06Z" />
</svg>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
<!-- end icon -->
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
counter_failed_challenges: {
el: document.querySelector("[data-count]"),
value: "unknown",
type: "text",
},
});
</script>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -71,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

10
src/common/core/badbehavior/ui/actions.py
View file

@ -1,13 +1,17 @@
from operator import itemgetter
def badbehavior(**kwargs):
def pre_render(**kwargs):
try:
# Here we will have a list { 'counter_403': X, 'counter_401': Y ... }
data = kwargs["app"].config["INSTANCES"].get_metrics("badbehavior")
# Format to fit [{code: 403, count: X}, {code: 401, count: Y} ...]
format_data = [{"code": int(key.split("_")[1]), "count": int(value)} for key, value in data.items()]
format_data.sort(key=itemgetter("count"), reverse=True)
return {"items": format_data}
return {"top_bad_behavior": format_data}
except:
return {"items": []}
return {"top_bad_behavior": "unknown"}
def badbehavior(**kwargs):
pass

148
src/common/core/badbehavior/ui/template.html
View file

@ -7,53 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div data-fetch-success-show class="hidden core-card-list w-small">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">BAD BEHAVIOR LIST</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="w-small core-card-list-wrap">
<!-- header-->
<p class="core-card-list-header col-span-6">Error code</p>
<p class="core-card-list-header col-span-6">Count</p>
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
<li data-item class="core-card-list-item col-span-6">
<p data-name="code" class="core-card-list-item-content col-span-6"></p>
<p data-name="count" class="core-card-list-item-content col-span-6"></p>
</li>
</ul>
<!-- end list-->
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<!-- end list container-->
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
items: {
el: document.querySelector("[data-item]"),
value: [],
type: "list",
listNames: ["code", "count"],
},
});
</script>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -72,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

4
src/common/core/blacklist/blacklist.lua
View file

@ -355,10 +355,10 @@ end
-- luacheck: ignore 212
function blacklist:get_data(blacklisted)
local data = {}
if blacklisted == "ip" then
if blacklisted:lower() == "ip" then
data["id"] = "ip"
else
local id, value = blacklisted:match("^(.+) (.+)$")
local id, value = blacklisted:match("^(%w+) (.+)$")
if id and value then
id = id:lower()
data["id"] = id

104
src/common/core/blacklist/jobs/blacklist-download.py
View file

@ -2,10 +2,9 @@
from contextlib import suppress
from ipaddress import ip_address, ip_network
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join, normpath
from pathlib import Path
from re import IGNORECASE, compile as re_compile
from re import compile as re_compile
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from typing import Tuple
@ -16,11 +15,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from requests import get
from Database import Database # type: ignore
from common_utils import bytes_hash # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, del_file_in_db, is_cached_file, file_hash
from jobs import Job # type: ignore
rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE)
rdns_rx = re_compile(rb"^[^ ]+$")
asn_rx = re_compile(rb"^\d+$")
uri_rx = re_compile(rb"^/")
@ -51,7 +50,7 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]:
return False, b""
logger = setup_logger("BLACKLIST", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("BLACKLIST", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
@ -68,16 +67,10 @@ try:
blacklist_activated = True
if not blacklist_activated:
logger.info("Blacklist is not activated, skipping downloads...")
_exit(0)
LOGGER.info("Blacklist is not activated, skipping downloads...")
sys_exit(0)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
# Create directories if they don't exist
blacklist_path = Path(sep, "var", "cache", "bunkerweb", "blacklist")
blacklist_path.mkdir(parents=True, exist_ok=True)
tmp_blacklist_path = Path(sep, "var", "tmp", "bunkerweb", "blacklist")
tmp_blacklist_path.mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Get URLs
urls = {
@ -110,37 +103,35 @@ try:
"IGNORE_USER_AGENT": True,
"IGNORE_URI": True,
}
all_fresh = True
for kind in kinds_fresh:
if not is_cached_file(blacklist_path.joinpath(f"{kind}.list"), "hour", db):
kinds_fresh[kind] = False
all_fresh = False
logger.info(
f"Blacklist for {kind} is not cached, processing downloads..",
)
else:
logger.info(
f"Blacklist for {kind} is already in cache, skipping downloads...",
)
if not urls[kind]:
logger.warning(
f"Blacklist for {kind} is cached but no URL is configured, removing from cache...",
)
blacklist_path.joinpath(f"{kind}.list").unlink(missing_ok=True)
deleted, err = del_file_in_db(f"{kind}.list", db)
if not deleted:
logger.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all_fresh:
_exit(0)
if not JOB.is_cached_file(f"{kind}.list", "hour"):
if urls[kind]:
kinds_fresh[kind] = False
LOGGER.info(f"Blacklist for {kind} is not cached, processing downloads..")
continue
LOGGER.info(f"Blacklist for {kind} is already in cache, skipping downloads...")
if not urls[kind]:
LOGGER.warning(f"Blacklist for {kind} is cached but no URL is configured, removing from cache...")
deleted, err = JOB.del_cache(f"{kind}.list")
if not deleted:
LOGGER.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all(kinds_fresh.values()):
if not any(urls.values()):
LOGGER.info("No blacklist URL is configured, nothing to do...")
sys_exit(0)
# Loop on kinds
for kind, urls_list in urls.items():
if kinds_fresh[kind]:
continue
# Write combined data of the kind to a single temp file
# Write combined data of the kind in memory and check if it has changed
for url in urls_list:
try:
logger.info(f"Downloading blacklist data from {url} ...")
LOGGER.info(f"Downloading blacklist data from {url} ...")
if url.startswith("file://"):
with open(normpath(url[7:]), "rb") as f:
iterable = f.readlines()
@ -148,7 +139,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -168,39 +159,28 @@ try:
content += data + b"\n"
i += 1
tmp_blacklist_path.joinpath(f"{kind}.list").write_bytes(content)
logger.info(f"Downloaded {i} bad {kind}")
LOGGER.info(f"Downloaded {i} bad {kind}")
# Check if file has changed
new_hash = file_hash(tmp_blacklist_path.joinpath(f"{kind}.list"))
old_hash = cache_hash(blacklist_path.joinpath(f"{kind}.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash(f"{kind}.list")
if new_hash == old_hash:
logger.info(
f"New file {kind}.list is identical to cache file, reload is not needed",
)
LOGGER.info(f"New file {kind}.list is identical to cache file, reload is not needed")
else:
logger.info(
f"New file {kind}.list is different than cache file, reload is needed",
)
LOGGER.info(f"New file {kind}.list is different than cache file, reload is needed")
# Put file in cache
cached, err = cache_file(
tmp_blacklist_path.joinpath(f"{kind}.list"),
blacklist_path.joinpath(f"{kind}.list"),
new_hash,
db,
)
cached, err = JOB.cache_file(f"{kind}.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching blacklist : {err}")
LOGGER.error(f"Error while caching blacklist : {err}")
status = 2
else:
status = 1
except:
status = 2
logger.error(f"Exception while getting blacklist from {url} :\n{format_exc()}")
LOGGER.error(f"Exception while getting blacklist from {url} :\n{format_exc()}")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running blacklist-download.py :\n{format_exc()}")
LOGGER.error(f"Exception while running blacklist-download.py :\n{format_exc()}")
sys_exit(status)

35
src/common/core/blacklist/ui/actions.py
View file

@ -1,23 +1,20 @@
def blacklist(**kwargs):
keys = [
"counter_blacklist_url",
"counter_blacklist_ip",
"counter_blacklist_rdns",
"counter_blacklist_asn",
"counter_blacklist_ua",
]
def pre_render(**kwargs):
metrics = {
"counter_blacklist_url": {"value": "unknown", "title": "URL", "subtitle": "denied", "subtitle_color": "error", "svg_color": "red"},
"counter_blacklist_ip": {"value": "unknown", "title": "IP", "subtitle": "denied", "subtitle_color": "error", "svg_color": "orange"},
"counter_blacklist_rdns": {"value": "unknown", "title": "RDNS", "subtitle": "denied", "subtitle_color": "error", "svg_color": "amber"},
"counter_blacklist_asn": {"value": "unknown", "title": "ASN", "subtitle": "denied", "subtitle_color": "error", "svg_color": "emerald"},
"counter_blacklist_ua": {"value": "unknown", "title": "UA", "subtitle": "denied", "subtitle_color": "error", "svg_color": "pink"},
}
try:
data = kwargs["app"].config["INSTANCES"].get_metrics("blacklist")
for key in keys:
if data.get(key) is None:
data[key] = 0
return data
for key in metrics:
metrics[key]["value"] = data.get(key, 0)
return metrics
except:
data = {}
for key in keys:
data[key] = 0
return data
return metrics
def blacklist(**kwargs):
pass

255
src/common/core/blacklist/ui/template.html
View file

@ -7,158 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<div class="core-layout">
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
<!-- end info -->
</div>
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">URL</p>
<h5 data-count-url class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">denied</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container red">
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-medium core-card-metrics-svg">
<path fill-rule="evenodd" d="M19.902 4.098a3.75 3.75 0 0 0-5.304 0l-4.5 4.5a3.75 3.75 0 0 0 1.035 6.037.75.75 0 0 1-.646 1.353 5.25 5.25 0 0 1-1.449-8.45l4.5-4.5a5.25 5.25 0 1 1 7.424 7.424l-1.757 1.757a.75.75 0 1 1-1.06-1.06l1.757-1.757a3.75 3.75 0 0 0 0-5.304Zm-7.389 4.267a.75.75 0 0 1 1-.353 5.25 5.25 0 0 1 1.449 8.45l-4.5 4.5a5.25 5.25 0 1 1-7.424-7.424l1.757-1.757a.75.75 0 1 1 1.06 1.06l-1.757 1.757a3.75 3.75 0 1 0 5.304 5.304l4.5-4.5a3.75 3.75 0 0 0-1.035-6.037.75.75 0 0 1-.354-1Z" clip-rule="evenodd" />
</svg>
</div>
<!-- end icon -->
</div>
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">IP</p>
<h5 data-count-ip class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">denied</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container lime">
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg">
<path d="M3.53 2.47a.75.75 0 0 0-1.06 1.06l18 18a.75.75 0 1 0 1.06-1.06l-18-18ZM20.25 5.507v11.561L5.853 2.671c.15-.043.306-.075.467-.094a49.255 49.255 0 0 1 11.36 0c1.497.174 2.57 1.46 2.57 2.93ZM3.75 21V6.932l14.063 14.063L12 18.088l-7.165 3.583A.75.75 0 0 1 3.75 21Z" />
</svg>
</div>
<!-- end icon -->
</div>
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">RDNS</p>
<h5 data-count-rdns class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">denied</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-metrics-svg-container indigo">
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-medium core-card-metrics-svg">
<path d="M11.625 16.5a1.875 1.875 0 1 0 0-3.75 1.875 1.875 0 0 0 0 3.75Z" />
<path fill-rule="evenodd" d="M5.625 1.5H9a3.75 3.75 0 0 1 3.75 3.75v1.875c0 1.036.84 1.875 1.875 1.875H16.5a3.75 3.75 0 0 1 3.75 3.75v7.875c0 1.035-.84 1.875-1.875 1.875H5.625a1.875 1.875 0 0 1-1.875-1.875V3.375c0-1.036.84-1.875 1.875-1.875Zm6 16.5c.66 0 1.277-.19 1.797-.518l1.048 1.048a.75.75 0 0 0 1.06-1.06l-1.047-1.048A3.375 3.375 0 1 0 11.625 18Z" clip-rule="evenodd" />
<path d="M14.25 5.25a5.23 5.23 0 0 0-1.279-3.434 9.768 9.768 0 0 1 6.963 6.963A5.23 5.23 0 0 0 16.5 7.5h-1.875a.375.375 0 0 1-.375-.375V5.25Z" />
</svg>
</div>
<!-- end icon -->
</div>
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">ASN</p>
<h5 data-count-asn class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">denied</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container blue">
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-medium core-card-metrics-svg">
<path d="M21.721 12.752a9.711 9.711 0 0 0-.945-5.003 12.754 12.754 0 0 1-4.339 2.708 18.991 18.991 0 0 1-.214 4.772 17.165 17.165 0 0 0 5.498-2.477ZM14.634 15.55a17.324 17.324 0 0 0 .332-4.647c-.952.227-1.945.347-2.966.347-1.021 0-2.014-.12-2.966-.347a17.515 17.515 0 0 0 .332 4.647 17.385 17.385 0 0 0 5.268 0ZM9.772 17.119a18.963 18.963 0 0 0 4.456 0A17.182 17.182 0 0 1 12 21.724a17.18 17.18 0 0 1-2.228-4.605ZM7.777 15.23a18.87 18.87 0 0 1-.214-4.774 12.753 12.753 0 0 1-4.34-2.708 9.711 9.711 0 0 0-.944 5.004 17.165 17.165 0 0 0 5.498 2.477ZM21.356 14.752a9.765 9.765 0 0 1-7.478 6.817 18.64 18.64 0 0 0 1.988-4.718 18.627 18.627 0 0 0 5.49-2.098ZM2.644 14.752c1.682.971 3.53 1.688 5.49 2.099a18.64 18.64 0 0 0 1.988 4.718 9.765 9.765 0 0 1-7.478-6.816ZM13.878 2.43a9.755 9.755 0 0 1 6.116 3.986 11.267 11.267 0 0 1-3.746 2.504 18.63 18.63 0 0 0-2.37-6.49ZM12 2.276a17.152 17.152 0 0 1 2.805 7.121c-.897.23-1.837.353-2.805.353-.968 0-1.908-.122-2.805-.353A17.151 17.151 0 0 1 12 2.276ZM10.122 2.43a18.629 18.629 0 0 0-2.37 6.49 11.266 11.266 0 0 1-3.746-2.504 9.754 9.754 0 0 1 6.116-3.985Z" />
</svg>
</div>
<!-- end icon -->
</div>
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">User Agent</p>
<h5 data-count-user-agent class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">denied</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container amber">
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg">
<path fill-rule="evenodd" d="M7.5 6a4.5 4.5 0 1 1 9 0 4.5 4.5 0 0 1-9 0ZM3.751 20.105a8.25 8.25 0 0 1 16.498 0 .75.75 0 0 1-.437.695A18.683 18.683 0 0 1 12 22.5c-2.786 0-5.433-.608-7.812-1.7a.75.75 0 0 1-.437-.695Z" clip-rule="evenodd" />
</svg>
</div>
<!-- end icon -->
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
counter_blacklist_url: {
el: document.querySelector("[data-count-url]"),
value: "unknown",
type: "text",
},
counter_blacklist_ip: {
el: document.querySelector("[data-count-ip]"),
value: "unknown",
type: "text",
},
counter_blacklist_rdns: {
el: document.querySelector("[data-count-rdns]"),
value: "unknown",
type: "text",
},
counter_blacklist_asn: {
el: document.querySelector("[data-count-asn]"),
value: "unknown",
type: "text",
},
counter_blacklist_ua: {
el: document.querySelector("[data-count-user-agent]"),
value: "unknown",
type: "text",
},
});
</script>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -177,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

115
src/common/core/bunkernet/jobs/bunkernet-data.py
View file

@ -1,29 +1,21 @@
#!/usr/bin/env python3
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
("core", "bunkernet", "jobs"),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from bunkernet import data
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, file_hash, is_cached_file, get_file_in_db
from jobs import Job # type: ignore
from common_utils import bytes_hash # type: ignore
logger = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO"))
exit_status = 0
try:
@ -40,109 +32,90 @@ try:
bunkernet_activated = True
if not bunkernet_activated:
logger.info("BunkerNet is not activated, skipping download...")
_exit(0)
LOGGER.info("BunkerNet is not activated, skipping download...")
sys_exit(0)
# Create directory if it doesn't exist
bunkernet_path = Path(sep, "var", "cache", "bunkerweb", "bunkernet")
bunkernet_path.mkdir(parents=True, exist_ok=True)
bunkernet_tmp_path = Path(sep, "var", "tmp", "bunkerweb", "bunkernet")
bunkernet_tmp_path.mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Create empty file in case it doesn't exist
bunkernet_path.joinpath("ip.list").touch(exist_ok=True)
ip_list_path = bunkernet_path.joinpath("ip.list")
if not ip_list_path.is_file():
ip_list_path.touch(exist_ok=True)
# Get ID from cache
bunkernet_id = None
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
bunkernet_id = get_file_in_db("instance.id", db)
bunkernet_id = JOB.get_cache("instance.id")
if bunkernet_id:
bunkernet_path.joinpath("instance.id").write_bytes(bunkernet_id)
logger.info("Successfully retrieved BunkerNet ID from db cache")
LOGGER.info("Successfully retrieved BunkerNet ID from db cache")
else:
logger.info("No BunkerNet ID found in db cache")
LOGGER.info("No BunkerNet ID found in db cache")
# Check if ID is present
if not bunkernet_path.joinpath("instance.id").is_file():
logger.error(
"Not downloading BunkerNet data because instance is not registered",
)
_exit(2)
LOGGER.error("Not downloading BunkerNet data because instance is not registered")
sys_exit(2)
# Don't go further if the cache is fresh
if is_cached_file(bunkernet_path.joinpath("ip.list"), "day", db):
logger.info(
"BunkerNet list is already in cache, skipping download...",
)
_exit(0)
if JOB.is_cached_file("ip.list", "day"):
LOGGER.info("BunkerNet list is already in cache, skipping download...")
sys_exit(0)
exit_status = 1
# Download data
logger.info("Downloading BunkerNet data ...")
LOGGER.info("Downloading BunkerNet data ...")
ok, status, data = data()
if not ok:
logger.error(
f"Error while sending data request to BunkerNet API : {data}",
)
_exit(2)
LOGGER.error(f"Error while sending data request to BunkerNet API : {data}")
sys_exit(2)
elif status == 429:
logger.warning(
"BunkerNet API is rate limiting us, trying again later...",
)
_exit(0)
LOGGER.warning("BunkerNet API is rate limiting us, trying again later...")
sys_exit(0)
elif status == 403:
logger.warning(
"BunkerNet has banned this instance, retrying a register later...",
)
_exit(0)
LOGGER.warning("BunkerNet has banned this instance, retrying a register later...")
sys_exit(0)
try:
assert isinstance(data, dict)
except AssertionError:
logger.error(
f"Received invalid data from BunkerNet API while sending db request : {data}",
)
_exit(2)
LOGGER.error(f"Received invalid data from BunkerNet API while sending db request : {data}")
sys_exit(2)
if data["result"] != "ok":
logger.error(
f"Received error from BunkerNet API while sending db request : {data['data']}, removing instance ID",
)
_exit(2)
LOGGER.error(f"Received error from BunkerNet API while sending db request : {data['data']}, removing instance ID")
sys_exit(2)
logger.info("Successfully downloaded data from BunkerNet API")
LOGGER.info("Successfully downloaded data from BunkerNet API")
# Writing data to file
logger.info("Saving BunkerNet data ...")
LOGGER.info("Saving BunkerNet data ...")
content = "\n".join(data["data"]).encode("utf-8")
bunkernet_tmp_path.joinpath("ip.list").write_bytes(content)
# Check if file has changed
new_hash = file_hash(bunkernet_tmp_path.joinpath("ip.list"))
old_hash = cache_hash(bunkernet_path.joinpath("ip.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash("ip.list")
if new_hash == old_hash:
logger.info(
"New file is identical to cache file, reload is not needed",
)
_exit(0)
LOGGER.info("New file is identical to cache file, reload is not needed")
sys_exit(0)
# Put file in cache
cached, err = cache_file(
bunkernet_tmp_path.joinpath("ip.list"),
bunkernet_path.joinpath("ip.list"),
new_hash,
db,
)
cached, err = JOB.cache_file("ip.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching BunkerNet data : {err}")
_exit(2)
LOGGER.error(f"Error while caching BunkerNet data : {err}")
sys_exit(2)
logger.info("Successfully saved BunkerNet data")
LOGGER.info("Successfully saved BunkerNet data")
exit_status = 1
except SystemExit as e:
exit_status = e.code
except:
exit_status = 2
logger.error(f"Exception while running bunkernet-data.py :\n{format_exc()}")
LOGGER.error(f"Exception while running bunkernet-data.py :\n{format_exc()}")
sys_exit(exit_status)

151
src/common/core/bunkernet/jobs/bunkernet-register.py
View file

@ -1,30 +1,19 @@
#!/usr/bin/env python3
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from time import sleep
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
("core", "bunkernet", "jobs"),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from bunkernet import register, ping
from Database import Database # type: ignore
from bunkernet import register
from logger import setup_logger # type: ignore
from jobs import get_file_in_db, set_file_in_db, del_file_in_db # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO"))
exit_status = 0
try:
@ -41,132 +30,62 @@ try:
bunkernet_activated = True
if not bunkernet_activated:
logger.info("BunkerNet is not activated, skipping registration...")
_exit(0)
# Create directory if it doesn't exist
bunkernet_path = Path(sep, "var", "cache", "bunkerweb", "bunkernet")
bunkernet_path.mkdir(parents=True, exist_ok=True)
LOGGER.info("BunkerNet is not activated, skipping registration...")
sys_exit(0)
# Get ID from cache
bunkernet_id = None
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
bunkernet_id = get_file_in_db("instance.id", db)
if bunkernet_id:
bunkernet_path.joinpath("instance.id").write_bytes(bunkernet_id)
logger.info("Successfully retrieved BunkerNet ID from db cache")
else:
logger.info("No BunkerNet ID found in db cache")
JOB = Job(LOGGER)
bunkernet_id = JOB.get_cache("instance.id")
# Register instance
registered = False
instance_id_path = bunkernet_path.joinpath("instance.id")
if not instance_id_path.is_file():
logger.info("Registering instance on BunkerNet API ...")
if not bunkernet_id:
LOGGER.info("No BunkerNet ID found in db cache, Registering instance on BunkerNet API ...")
ok, status, data = register()
if not ok:
logger.error(f"Error while sending register request to BunkerNet API : {data}")
_exit(2)
LOGGER.error(f"Error while sending register request to BunkerNet API : {data}")
sys_exit(2)
elif status == 429:
logger.warning(
"BunkerNet API is rate limiting us, trying again later...",
)
_exit(0)
LOGGER.warning("BunkerNet API is rate limiting us, trying again later...")
sys_exit(0)
elif status == 403:
logger.warning(
"BunkerNet has banned this instance, retrying a register later...",
)
_exit(0)
LOGGER.warning("BunkerNet has banned this instance, retrying a register later...")
sys_exit(0)
try:
assert isinstance(data, dict)
except AssertionError:
logger.error(
f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...",
)
_exit(2)
LOGGER.error(f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...")
sys_exit(2)
bunkernet_id = data.get("data")
if status != 200:
logger.error(
f"Error {status} from BunkerNet API : {data['data']}",
)
_exit(2)
LOGGER.error(f"Error {status} from BunkerNet API : {bunkernet_id}")
sys_exit(2)
elif data.get("result", "ko") != "ok":
logger.error(f"Received error from BunkerNet API while sending register request : {data.get('data', {})}")
_exit(2)
bunkernet_id = data["data"]
instance_id_path.write_text(bunkernet_id, encoding="utf-8")
LOGGER.error(f"Received error from BunkerNet API while sending register request : {bunkernet_id}")
sys_exit(2)
assert isinstance(bunkernet_id, str), f"Received invalid bunkernet id : {bunkernet_id}"
registered = True
exit_status = 1
logger.info(f"Successfully registered on BunkerNet API with instance id {data['data']}")
LOGGER.info(f"Successfully registered on BunkerNet API with instance id {data['data']}")
else:
bunkernet_id = bunkernet_id or instance_id_path.read_bytes()
bunkernet_id = bunkernet_id.decode()
logger.info(f"Already registered on BunkerNet API with instance id {bunkernet_id}")
sleep(1)
LOGGER.info(f"Already registered on BunkerNet API with instance id {bunkernet_id}")
# Update cache with new bunkernet ID
if registered:
cached, err = set_file_in_db("instance.id", bunkernet_id.encode(), db)
cached, err = JOB.cache_file("instance.id", bunkernet_id.encode())
if not cached:
logger.error(f"Error while saving BunkerNet data to db cache : {err}")
LOGGER.error(f"Error while saving BunkerNet data to db cache : {err}")
else:
logger.info("Successfully saved BunkerNet data to db cache")
# Ping
logger.info("Checking connectivity with BunkerNet API ...")
bunkernet_ping = False
for i in range(0, 5):
ok, status, data = ping(bunkernet_id)
retry = False
if not ok:
logger.error(f"Error while sending ping request to BunkerNet API : {data}")
retry = True
elif status == 429:
logger.warning(
"BunkerNet API is rate limiting us, trying again later...",
)
retry = True
elif status == 403:
logger.warning(
"BunkerNet has banned this instance, retrying a register later...",
)
_exit(2)
elif status == 401:
logger.warning(
"Instance ID is not registered, removing it and retrying a register later...",
)
instance_id_path.unlink()
del_file_in_db("instance.id", db)
_exit(2)
try:
assert isinstance(data, dict)
except AssertionError:
logger.error(
f"Received invalid data from BunkerNet API while sending db request : {data}, retrying later...",
)
_exit(2)
if data.get("result", "ko") != "ok":
logger.error(
f"Received error from BunkerNet API while sending ping request : {data.get('data', {})}",
)
retry = True
if not retry:
bunkernet_ping = True
break
logger.warning("Waiting 1s and trying again ...")
sleep(1)
if bunkernet_ping:
logger.info("Connectivity with BunkerNet is successful !")
else:
logger.error("Connectivity with BunkerNet failed ...")
exit_status = 2
LOGGER.info("Successfully saved BunkerNet data to db cache")
except SystemExit as e:
exit_status = e.code
except:
exit_status = 2
logger.error(f"Exception while running bunkernet-register.py :\n{format_exc()}")
LOGGER.error(f"Exception while running bunkernet-register.py :\n{format_exc()}")
sys_exit(exit_status)

7
src/common/core/bunkernet/jobs/bunkernet.py
View file

@ -5,10 +5,10 @@ from pathlib import Path
from requests import request as requests_request, ReadTimeout
from typing import Literal, Optional, Tuple, Union
from jobs import get_os_info, get_integration, get_version # type: ignore
from common_utils import get_os_info, get_integration, get_version # type: ignore
def request(method: Union[Literal["POST"], Literal["GET"]], url: str, _id: Optional[str] = None) -> Tuple[bool, Optional[int], Union[str, dict]]:
def request(method: Literal["POST", "GET"], url: str, _id: Optional[str] = None) -> Tuple[bool, Optional[int], Union[str, dict]]:
data = {
"integration": get_integration(),
"version": get_version(),
@ -17,13 +17,12 @@ def request(method: Union[Literal["POST"], Literal["GET"]], url: str, _id: Optio
if _id:
data["id"] = _id
headers = {"User-Agent": f"BunkerWeb/{data['version']}"}
try:
resp = requests_request(
method,
f"{getenv('BUNKERNET_SERVER', 'https://api.bunkerweb.io')}{url}",
json=data,
headers=headers,
headers={"User-Agent": f"BunkerWeb/{data['version']}"},
timeout=5,
)
status = resp.status_code

10
src/common/core/bunkernet/ui/actions.py
View file

@ -1,6 +1,10 @@
def bunkernet(**kwargs):
def pre_render(**kwargs):
try:
ping_data = kwargs["app"].config["INSTANCES"].get_ping("bunkernet")
return {"ping_status": ping_data["status"]}
return {"ping_status": {"title": "BUNKERNET STATUS", "value": ping_data["status"]}}
except:
return {"ping_status": "error"}
return {"ping_status": {"title": "BUNKERNET STATUS", "value": "error"}}
def bunkernet(**kwargs):
pass

141
src/common/core/bunkernet/ui/template.html
View file

@ -7,44 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">STATUS</h5>
<svg data-status-svg
class="core-card-status-svg info"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
<p data-status-text class="core-card-text"></p>
</div>
<!-- end status -->
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
// value : active / inactive / unknown
ping_status: {
el: document.querySelector("[data-status-svg]"),
value: "unknown",
type: "status",
textEl: document.querySelector("[data-status-text]"),
},
});
</script>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -63,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

22
src/common/core/cors/ui/actions.py
View file

@ -1,11 +1,19 @@
def cors(**kwargs):
def pre_render(**kwargs):
try:
data = kwargs["app"].config["INSTANCES"].get_metrics("cors")
if data.get("counter_failed_cors") is None:
data["counter_failed_cors"] = 0
return data
return {
"counter_failed_cors": {
"value": data.get("counter_failed_cors", 0),
"title": "CORS",
"subtitle": "request blocked",
"subtitle_color": "error",
"svg_color": "red",
}
}
except:
return {"counter_failed_cors": 0}
return {"counter_failed_cors": {"value": "unknown", "title": "CORS", "subtitle": "request blocked", "subtitle_color": "error", "svg_color": "red"}}
def cors(**kwargs):
pass

147
src/common/core/cors/ui/template.html
View file

@ -7,52 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">CORS</p>
<h5 data-count class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">request blocked</span>
</p>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container red">
<svg xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="scale-75 leading-none text-lg relative fill-red-700 stroke-white">
<path stroke-linecap="round" stroke-linejoin="round" d="M18.364 18.364A9 9 0 0 0 5.636 5.636m12.728 12.728A9 9 0 0 1 5.636 5.636m12.728 12.728L5.636 5.636" />
</svg>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
<!-- end icon -->
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
counter_failed_cors: {
el: document.querySelector("[data-count]"),
value: "unknown",
type: "text",
},
});
</script>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -71,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

25
src/common/core/country/ui/actions.py
View file

@ -1,11 +1,20 @@
def country(**kwargs):
def pre_render(**kwargs):
try:
data = kwargs["app"].config["INSTANCES"].get_metrics("country")
if data.get("counter_failed_country") is None:
data["counter_failed_country"] = 0
return data
return {
"counter_failed_country": {
"value": data.get("counter_failed_country", 0),
"title": "Country",
"subtitle": "request blocked",
"subtitle_color": "error",
"svg_color": "red",
}
}
except:
return {"counter_failed_country": 0}
return {
"counter_failed_country": {"value": "unknown", "title": "Country", "subtitle": "request blocked", "subtitle_color": "error", "svg_color": "red"}
}
def country(**kwargs):
pass

147
src/common/core/country/ui/template.html
View file

@ -7,52 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">Country</p>
<h5 data-count class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">request blocked</span>
</p>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container red">
<svg xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="scale-75 leading-none text-lg relative fill-red-700 stroke-white">
<path stroke-linecap="round" stroke-linejoin="round" d="M18.364 18.364A9 9 0 0 0 5.636 5.636m12.728 12.728A9 9 0 0 1 5.636 5.636m12.728 12.728L5.636 5.636" />
</svg>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
<!-- end icon -->
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
counter_failed_country: {
el: document.querySelector("[data-count]"),
value: "",
type: "text",
},
});
</script>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -71,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

8
src/common/core/customcert/customcert.lua
View file

@ -44,8 +44,8 @@ function customcert:init()
for server_name, multisite_vars in pairs(vars) do
if multisite_vars["USE_CUSTOM_SSL"] == "yes" and server_name ~= "global" then
local check, data = read_files({
"/var/cache/bunkerweb/customcert/" .. server_name .. ".cert.pem",
"/var/cache/bunkerweb/customcert/" .. server_name .. ".key.pem",
"/var/cache/bunkerweb/customcert/" .. server_name .. "/cert.pem",
"/var/cache/bunkerweb/customcert/" .. server_name .. "/key.pem",
})
if not check then
self.logger:log(ERR, "error while reading files : " .. data)
@ -68,8 +68,8 @@ function customcert:init()
return self:ret(false, "can't get SERVER_NAME variable : " .. err)
end
local check, data = read_files({
"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. ".cert.pem",
"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. ".key.pem",
"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/cert.pem",
"/var/cache/bunkerweb/customcert/" .. server_name:match("%S+") .. "/key.pem",
})
if not check then
self.logger:log(ERR, "error while reading files : " .. data)

273
src/common/core/customcert/jobs/custom-cert.py
View file

@ -1,220 +1,143 @@
#!/usr/bin/env python3
from contextlib import suppress
from os import getenv, sep
from os.path import join, normpath
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from base64 import b64decode
from typing import Tuple, Union
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from jobs import del_file_in_db, cache_file, cache_hash, file_hash
from Database import Database # type: ignore
from common_utils import bytes_hash # type: ignore
from jobs import Job # type: ignore
from logger import setup_logger # type: ignore
logger = setup_logger("CUSTOM-CERT", getenv("LOG_LEVEL", "INFO"))
db = None
LOGGER = setup_logger("CUSTOM-CERT", getenv("LOG_LEVEL", "INFO"))
JOB = Job(LOGGER)
def check_cert(cert_path: str, key_path: str, first_server: str) -> bool:
try:
def check_cert(cert_file: Union[Path, bytes], key_file: Union[Path, bytes], first_server: str) -> Tuple[bool, str]:
with suppress(BaseException):
ret = False
if not cert_path or not key_path:
logger.warning("Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates")
return False
if not cert_file or not key_file:
return False, "Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates"
cert_path: Path = Path(normpath(cert_path))
key_path: Path = Path(normpath(key_path))
if isinstance(cert_file, Path):
if not cert_file.is_file():
return False, f"Certificate file {cert_file} is not a valid file, ignoring the custom certificate"
cert_file = cert_file.read_bytes()
if not cert_path.is_file():
logger.warning(f"Certificate file {cert_path} is not a valid file, ignoring the custom certificate")
return False
elif not key_path.is_file():
logger.warning(f"Key file {key_path} is not a valid file, ignoring the custom certificate")
return False
if isinstance(key_file, Path):
if not key_file.is_file():
return False, f"Key file {key_file} is not a valid file, ignoring the custom certificate"
key_file = key_file.read_bytes()
cert_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.cert.pem",
)
cert_cache_path.parent.mkdir(parents=True, exist_ok=True)
cert_hash = file_hash(cert_path)
old_hash = cache_hash(cert_cache_path, db)
cert_hash = bytes_hash(cert_file)
old_hash = JOB.cache_hash("cert.pem", service_id=first_server)
if old_hash != cert_hash:
ret = True
cached, err = cache_file(cert_path, cert_cache_path, cert_hash, db, delete_file=False)
cached, err = JOB.cache_file("cert.pem", cert_file, service_id=first_server, checksum=cert_hash, delete_file=False)
if not cached:
logger.error(f"Error while caching custom-cert cert.pem file : {err}")
elif not cert_cache_path.is_file():
cert_cache_path.write_bytes(cert_path.read_bytes())
ret = True
key_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.key.pem",
)
key_cache_path.parent.mkdir(parents=True, exist_ok=True)
LOGGER.error(f"Error while caching custom-cert cert.pem file : {err}")
key_hash = file_hash(key_path)
old_hash = cache_hash(key_cache_path, db)
key_hash = bytes_hash(key_file)
old_hash = JOB.cache_hash("key.pem", service_id=first_server)
if old_hash != key_hash:
ret = True
cached, err = cache_file(key_path, key_cache_path, key_hash, db, delete_file=False)
cached, err = JOB.cache_file("key.pem", key_file, service_id=first_server, checksum=key_hash, delete_file=False)
if not cached:
logger.error(f"Error while caching custom-cert key.pem file : {err}")
elif not key_cache_path.is_file():
key_cache_path.write_bytes(key_path.read_bytes())
ret = True
LOGGER.error(f"Error while caching custom-key key.pem file : {err}")
return ret
except:
logger.error(
f"Exception while running custom-cert.py (check_cert) :\n{format_exc()}",
)
return False
return ret, ""
return False, "exception"
status = 0
try:
Path(sep, "var", "cache", "bunkerweb", "customcert").mkdir(parents=True, exist_ok=True)
all_domains = getenv("SERVER_NAME") or []
if getenv("MULTISITE", "no") == "no" and getenv("USE_CUSTOM_SSL", "no") == "yes" and getenv("SERVER_NAME", "") != "":
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
if isinstance(all_domains, str):
all_domains = all_domains.split(" ")
cert_path = getenv("CUSTOM_SSL_CERT", "")
key_path = getenv("CUSTOM_SSL_KEY", "")
first_server = getenv("SERVER_NAME").split(" ")[0]
if not all_domains:
LOGGER.info("No services found, exiting ...")
sys_exit(0)
cert_data = b64decode(getenv("CUSTOM_SSL_CERT_DATA", ""))
key_data = b64decode(getenv("CUSTOM_SSL_KEY_DATA", ""))
for file, data in (("cert.pem", cert_data), ("key.pem", key_data)):
if data:
file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", f"{first_server}.{file}")
file_path.write_bytes(data)
if file == "cert.pem":
cert_path = str(file_path)
else:
key_path = str(file_path)
skipped_servers = []
if not getenv("MULTISITE", "no") == "yes":
all_domains = [all_domains[0]]
if getenv("USE_CUSTOM_SSL", "no") == "no":
LOGGER.info("Custom SSL is not enabled, skipping ...")
skipped_servers = all_domains
if cert_path and key_path:
logger.info(f"Checking certificate {cert_path} ...")
need_reload = check_cert(cert_path, key_path, first_server)
if need_reload:
logger.info(f"Detected change for certificate {cert_path}")
status = 1
else:
logger.info(f"No change for certificate {cert_path}")
elif not cert_path or not key_path:
logger.warning(
"Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY (or CUSTOM_SSL_CERT_DATA and CUSTOM_SSL_KEY_DATA) have to be set to use custom certificates, clearing cache ..."
)
cert_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.cert.pem",
)
cert_cache_path.unlink(missing_ok=True)
del_file_in_db(f"{first_server}.cert.pem", db, service_id=first_server)
key_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.key.pem",
)
key_cache_path.unlink(missing_ok=True)
del_file_in_db(f"{first_server}.key.pem", db, service_id=first_server)
elif getenv("MULTISITE", "no") == "yes":
servers = getenv("SERVER_NAME") or []
if isinstance(servers, str):
servers = servers.split(" ")
for first_server in servers:
if not first_server or (getenv(f"{first_server}_USE_CUSTOM_SSL", getenv("USE_CUSTOM_SSL", "no")) != "yes"):
if not skipped_servers:
for first_server in all_domains:
if getenv(f"{first_server}_USE_CUSTOM_SSL", getenv("USE_CUSTOM_SSL", "no")) == "no":
LOGGER.info(f"Custom SSL is not enabled for {first_server}, skipping ...")
skipped_servers.append(first_server)
continue
if not db:
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
cert_file = getenv(f"{first_server}_CUSTOM_SSL_CERT", getenv("CUSTOM_SSL_CERT", ""))
key_file = getenv(f"{first_server}_CUSTOM_SSL_KEY", getenv("CUSTOM_SSL_KEY", ""))
cert_data = getenv(f"{first_server}_CUSTOM_SSL_CERT_DATA", getenv("CUSTOM_SSL_CERT_DATA", ""))
key_data = getenv(f"{first_server}_CUSTOM_SSL_KEY_DATA", getenv("CUSTOM_SSL_KEY_DATA", ""))
cert_path = getenv(f"{first_server}_CUSTOM_SSL_CERT", "")
key_path = getenv(f"{first_server}_CUSTOM_SSL_KEY", "")
cert_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_CERT_DATA", ""))
key_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_KEY_DATA", ""))
for file, data in (("cert.pem", cert_data), ("key.pem", key_data)):
if data != b"":
file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", f"{first_server}.{file}")
file_path.parent.mkdir(parents=True, exist_ok=True)
file_path.write_bytes(data)
if file == "cert.pem":
cert_path = str(file_path)
else:
key_path = str(file_path)
if cert_path and key_path:
logger.info(
f"Checking certificate {cert_path} ...",
)
need_reload = check_cert(cert_path, key_path, first_server)
if need_reload:
logger.info(
f"Detected change for certificate {cert_path}",
)
status = 1
if cert_file or cert_data and key_file or key_data:
if isinstance(cert_file, str):
cert_file = Path(cert_file)
else:
logger.info(
f"No change for certificate {cert_path}",
)
elif not cert_path or not key_path:
logger.warning(
"Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY (or CUSTOM_SSL_CERT_DATA and CUSTOM_SSL_KEY_DATA) have to be set to use custom certificates, clearing cache ..."
try:
cert_file = b64decode(cert_data)
except BaseException:
LOGGER.exception(f"Error while decoding cert data, skipping server {first_server}...")
skipped_servers.append(first_server)
continue
if isinstance(key_file, str):
key_file = Path(key_file)
else:
try:
key_file = b64decode(key_data)
except BaseException:
LOGGER.exception(f"Error while decoding key data, skipping server {first_server}...")
skipped_servers.append(first_server)
continue
LOGGER.info(f"Checking certificate for {first_server} ...")
need_reload, err = check_cert(cert_file, key_file, first_server)
if err == "exception":
LOGGER.exception(f"Exception while checking {first_server}'s certificate, skipping ...")
skipped_servers.append(first_server)
continue
elif err:
LOGGER.warning(f"Error while checking {first_server}'s certificate : {err}")
skipped_servers.append(first_server)
continue
elif need_reload:
LOGGER.info(f"Detected change in {first_server}'s certificate")
status = 1
continue
LOGGER.info(f"No change in {first_server}'s certificate")
elif not cert_file or not key_file:
LOGGER.warning(
"Variables (CUSTOM_SSL_CERT or CUSTOM_SSL_CERT_DATA) and (CUSTOM_SSL_KEY or CUSTOM_SSL_KEY_DATA) have to be set to use custom certificates"
)
cert_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.cert.pem",
)
cert_cache_path.unlink(missing_ok=True)
del_file_in_db(f"{first_server}.cert.pem", db)
key_cache_path = Path(
sep,
"var",
"cache",
"bunkerweb",
"customcert",
f"{first_server}.key.pem",
)
key_cache_path.unlink(missing_ok=True)
del_file_in_db(f"{first_server}.key.pem", db)
skipped_servers.append(first_server)
for first_server in skipped_servers:
JOB.del_cache("cert.pem", service_id=first_server)
JOB.del_cache("key.pem", service_id=first_server)
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running custom-cert.py :\n{format_exc()}")
LOGGER.error(f"Exception while running custom-cert.py :\n{format_exc()}")
sys_exit(status)

23
src/common/core/dnsbl/ui/actions.py
View file

@ -1,11 +1,18 @@
def dnsbl(**kwargs):
def pre_render(**kwargs):
try:
data = kwargs["app"].config["INSTANCES"].get_metrics("dnsbl")
if data.get("counter_failed_dnsbl") is None:
data["counter_failed_dnsbl"] = 0
return data
return {
"counter_failed_dnsbl": {
"value": data.get("counter_failed_dnsbl", 0),
"title": "DNSBL",
"subtitle": "request blocked",
"subtitle_color": "error",
"svg_color": "red",
}
}
except:
return {"counter_failed_dnsbl": 0}
return {"counter_failed_dnsbl": {"value": "unknown", "title": "DNSBL", "subtitle": "request blocked", "subtitle_color": "error", "svg_color": "red"}}
def dnsbl(**kwargs):
pass

147
src/common/core/dnsbl/ui/template.html
View file

@ -7,52 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">DNSBL</p>
<h5 data-count class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">request blocked</span>
</p>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container red">
<svg xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="scale-75 leading-none text-lg relative fill-red-700 stroke-white">
<path stroke-linecap="round" stroke-linejoin="round" d="M18.364 18.364A9 9 0 0 0 5.636 5.636m12.728 12.728A9 9 0 0 1 5.636 5.636m12.728 12.728L5.636 5.636" />
</svg>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
<!-- end icon -->
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
counter_failed_dnsbl: {
el: document.querySelector("[data-count]"),
value: "unknown",
type: "text",
},
});
</script>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -71,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

10
src/common/core/errors/ui/actions.py
View file

@ -1,13 +1,17 @@
from operator import itemgetter
def errors(**kwargs):
def pre_render(**kwargs):
try:
# Here we will have a list { 'counter_403': X, 'counter_401': Y ... }
data = kwargs["app"].config["INSTANCES"].get_metrics("errors")
# Format to fit [{code: 403, count: X}, {code: 401, count: Y} ...]
format_data = [{"code": int(key.split("_")[1]), "count": int(value)} for key, value in data.items()]
format_data.sort(key=itemgetter("count"), reverse=True)
return {"items": format_data}
return {"top_errors": format_data}
except:
return {"items": []}
return {"top_errors": []}
def errors(**kwargs):
pass

140
src/common/core/errors/ui/template.html
View file

@ -6,52 +6,110 @@
class="hidden"
hidden />
<div class="core-layout">
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
</div>
<!-- end info -->
<div data-fetch-success-show class="hidden core-card-list w-medium">
<div class="core-card-list-container">
<h5 class="core-card-list-title">ERRORS LIST</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap w-medium">
<!-- header-->
<p class="core-card-list-header col-span-8">Code error</p>
<p class="core-card-list-header col-span-4">Count</p>
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
<li data-item class="core-card-list-item">
<p data-name="code" class="core-card-list-item-content col-span-8"></p>
<p data-name="count" class="core-card-list-item-content col-span-4"></p>
</li>
</ul>
<!-- end list-->
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
<!-- end list container-->
</div>
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
items: {
el: document.querySelector("[data-item]"),
value: [],
type: "list",
listNames: ["code", "count"],
},
});
</script>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
</div>
{% endblock %}

111
src/common/core/greylist/jobs/greylist-download.py
View file

@ -2,10 +2,9 @@
from contextlib import suppress
from ipaddress import ip_address, ip_network
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join, normpath
from pathlib import Path
from re import IGNORECASE, compile as re_compile
from re import compile as re_compile
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from typing import Tuple
@ -16,11 +15,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from requests import get
from Database import Database # type: ignore
from common_utils import bytes_hash # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, del_file_in_db, is_cached_file, file_hash
from jobs import Job # type: ignore
rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE)
rdns_rx = re_compile(rb"^[^ ]+$")
asn_rx = re_compile(rb"^\d+$")
uri_rx = re_compile(rb"^/")
@ -51,7 +50,7 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]:
return False, b""
logger = setup_logger("GREYLIST", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("GREYLIST", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
@ -68,16 +67,10 @@ try:
greylist_activated = True
if not greylist_activated:
logger.info("Greylist is not activated, skipping downloads...")
_exit(0)
LOGGER.info("Greylist is not activated, skipping downloads...")
sys_exit(0)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
# Create directories if they don't exist
greylist_path = Path(sep, "var", "cache", "bunkerweb", "greylist")
greylist_path.mkdir(parents=True, exist_ok=True)
tmp_greylist_path = Path(sep, "var", "tmp", "bunkerweb", "greylist")
tmp_greylist_path.mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Get URLs
urls = {"IP": [], "RDNS": [], "ASN": [], "USER_AGENT": [], "URI": []}
@ -87,45 +80,36 @@ try:
urls[kind].append(url)
# Don't go further if the cache is fresh
kinds_fresh = {
"IP": True,
"RDNS": True,
"ASN": True,
"USER_AGENT": True,
"URI": True,
}
all_fresh = True
kinds_fresh = {"IP": True, "RDNS": True, "ASN": True, "USER_AGENT": True, "URI": True}
for kind in kinds_fresh:
if not is_cached_file(greylist_path.joinpath(f"{kind}.list"), "hour", db):
kinds_fresh[kind] = False
all_fresh = False
logger.info(
f"Greylist for {kind} is not cached, processing downloads..",
)
else:
logger.info(
f"Greylist for {kind} is already in cache, skipping downloads...",
)
if not urls[kind]:
logger.warning(
f"Greylist for {kind} is cached but no URL is configured, removing from cache...",
)
greylist_path.joinpath(f"{kind}.list").unlink(missing_ok=True)
deleted, err = del_file_in_db(f"{kind}.list", db)
if not deleted:
logger.warning(f"Couldn't delete {kind}.list from cache : {err}")
if not JOB.is_cached_file(f"{kind}.list", "hour"):
if urls[kind]:
kinds_fresh[kind] = False
LOGGER.info(f"Greylist for {kind} is not cached, processing downloads..")
continue
if all_fresh:
_exit(0)
LOGGER.info(f"Greylist for {kind} is already in cache, skipping downloads...")
if not urls[kind]:
LOGGER.warning(f"Greylist for {kind} is cached but no URL is configured, removing from cache...")
deleted, err = JOB.del_cache(f"{kind}.list")
if not deleted:
LOGGER.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all(kinds_fresh.values()):
if not any(urls.values()):
LOGGER.info("No greylist URL is configured, nothing to do...")
sys_exit(0)
# Loop on kinds
for kind, urls_list in urls.items():
if kinds_fresh[kind]:
continue
# Write combined data of the kind to a single temp file
# Write combined data of the kind in memory and check if it has changed
for url in urls_list:
try:
logger.info(f"Downloading greylist data from {url} ...")
LOGGER.info(f"Downloading greylist data from {url} ...")
if url.startswith("file://"):
with open(normpath(url[7:]), "rb") as f:
iterable = f.readlines()
@ -133,7 +117,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -153,39 +137,28 @@ try:
content += data + b"\n"
i += 1
tmp_greylist_path.joinpath(f"{kind}.list").write_bytes(content)
logger.info(f"Downloaded {i} bad {kind}")
LOGGER.info(f"Downloaded {i} bad {kind}")
# Check if file has changed
new_hash = file_hash(tmp_greylist_path.joinpath(f"{kind}.list"))
old_hash = cache_hash(greylist_path.joinpath(f"{kind}.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash(f"{kind}.list")
if new_hash == old_hash:
logger.info(
f"New file {kind}.list is identical to cache file, reload is not needed",
)
LOGGER.info(f"New file {kind}.list is identical to cache file, reload is not needed")
else:
logger.info(
f"New file {kind}.list is different than cache file, reload is needed",
)
LOGGER.info(f"New file {kind}.list is different than cache file, reload is needed")
# Put file in cache
cached, err = cache_file(
tmp_greylist_path.joinpath(f"{kind}.list"),
greylist_path.joinpath(f"{kind}.list"),
new_hash,
db,
)
cached, err = JOB.cache_file(f"{kind}.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching greylist : {err}")
LOGGER.error(f"Error while caching greylist : {err}")
status = 2
else:
status = 1
except:
status = 2
logger.error(f"Exception while getting greylist from {url} :\n{format_exc()}")
LOGGER.error(f"Exception while getting greylist from {url} :\n{format_exc()}")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running greylist-download.py :\n{format_exc()}")
LOGGER.error(f"Exception while running greylist-download.py :\n{format_exc()}")
sys_exit(status)

25
src/common/core/greylist/ui/actions.py
View file

@ -1,11 +1,20 @@
def greylist(**kwargs):
def pre_render(**kwargs):
try:
data = kwargs["app"].config["INSTANCES"].get_metrics("greylist")
if data.get("counter_failed_greylist") is None:
data["counter_failed_greylist"] = 0
return data
return {
"counter_failed_greylist": {
"value": data.get("counter_failed_greylist", 0),
"title": "GREYLIST",
"subtitle": "request blocked",
"subtitle_color": "error",
"svg_color": "red",
}
}
except:
return {"counter_failed_greylist": 0}
return {
"counter_failed_greylist": {"value": "unknown", "title": "GREYLIST", "subtitle": "request blocked", "subtitle_color": "error", "svg_color": "red"}
}
def greylist(**kwargs):
pass

147
src/common/core/greylist/ui/template.html
View file

@ -7,52 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">GREYLIST</p>
<h5 data-count class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content error">request blocked</span>
</p>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container red">
<svg xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="scale-75 leading-none text-lg relative fill-red-700 stroke-white">
<path stroke-linecap="round" stroke-linejoin="round" d="M18.364 18.364A9 9 0 0 0 5.636 5.636m12.728 12.728A9 9 0 0 1 5.636 5.636m12.728 12.728L5.636 5.636" />
</svg>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
<!-- end icon -->
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
counter_failed_greylist: {
el: document.querySelector("[data-count]"),
value: "unknown",
type: "text",
},
});
</script>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -71,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

50
src/common/core/jobs/jobs/download-plugins.py
View file

@ -36,7 +36,7 @@ from logger import setup_logger # type: ignore
EXTERNAL_PLUGINS_DIR = Path(sep, "etc", "bunkerweb", "plugins")
logger = setup_logger("Jobs.download-plugins", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Jobs.download-plugins", getenv("LOG_LEVEL", "INFO"))
status = 0
@ -45,14 +45,14 @@ def install_plugin(plugin_dir: str, db) -> bool:
plugin_file = plugin_path.joinpath("plugin.json")
if not plugin_file.is_file():
logger.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json not found)")
LOGGER.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json not found)")
return False
# Load plugin.json
try:
metadata = loads(plugin_file.read_text(encoding="utf-8"))
except JSONDecodeError:
logger.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json is not valid)")
LOGGER.error(f"Skipping installation of plugin {plugin_path.name} (plugin.json is not valid)")
return False
# Don't go further if plugin is already installed
@ -65,12 +65,12 @@ def install_plugin(plugin_dir: str, db) -> bool:
break
if old_version == metadata["version"]:
logger.warning(
LOGGER.warning(
f"Skipping installation of plugin {metadata['id']} (version {metadata['version']} already installed)",
)
return False
logger.warning(
LOGGER.warning(
f"Plugin {metadata['id']} is already installed but version {metadata['version']} is different from database ({old_version}), updating it...",
)
rmtree(EXTERNAL_PLUGINS_DIR.joinpath(metadata["id"]), ignore_errors=True)
@ -81,7 +81,7 @@ def install_plugin(plugin_dir: str, db) -> bool:
for job_file in glob(join(sep, "etc", "bunkerweb", "plugins", "jobs", "*")):
st = Path(job_file).stat()
chmod(job_file, st.st_mode | S_IEXEC)
logger.info(f"Plugin {metadata['id']} installed")
LOGGER.info(f"Plugin {metadata['id']} installed")
return True
@ -89,14 +89,14 @@ try:
# Check if we have plugins to download
plugin_urls = getenv("EXTERNAL_PLUGIN_URLS")
if not plugin_urls:
logger.info("No external plugins to download")
LOGGER.info("No external plugins to download")
sys_exit(0)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"))
plugin_nbr = 0
# Loop on URLs
logger.info(f"Downloading external plugins from {plugin_urls}...")
LOGGER.info(f"Downloading external plugins from {plugin_urls}...")
for plugin_url in plugin_urls.split(" "):
# Download Plugin file
try:
@ -112,7 +112,7 @@ try:
)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
# Iterate over the response content in chunks
@ -120,7 +120,7 @@ try:
if chunk:
content += chunk
except:
logger.error(
LOGGER.error(
f"Exception while downloading plugin(s) from {plugin_url} :\n{format_exc()}",
)
status = 2
@ -137,15 +137,21 @@ try:
zf.extractall(path=temp_dir)
elif file_type == "application/gzip":
with tar_open(fileobj=BytesIO(content), mode="r:gz") as tar:
tar.extractall(path=temp_dir)
try:
tar.extractall(path=temp_dir, filter="data")
except TypeError:
tar.extractall(path=temp_dir)
elif file_type == "application/x-tar":
with tar_open(fileobj=BytesIO(content), mode="r") as tar:
tar.extractall(path=temp_dir)
try:
tar.extractall(path=temp_dir, filter="data")
except TypeError:
tar.extractall(path=temp_dir)
else:
logger.error(f"Unknown file type for {plugin_url}, either zip or tar are supported, skipping...")
LOGGER.error(f"Unknown file type for {plugin_url}, either zip or tar are supported, skipping...")
continue
except:
logger.error(f"Exception while decompressing plugin(s) from {plugin_url} :\n{format_exc()}")
LOGGER.error(f"Exception while decompressing plugin(s) from {plugin_url} :\n{format_exc()}")
status = 2
continue
@ -156,13 +162,13 @@ try:
if install_plugin(plugin_dir, db):
plugin_nbr += 1
except FileExistsError:
logger.warning(f"Skipping installation of plugin {basename(plugin_dir)} (already installed)")
LOGGER.warning(f"Skipping installation of plugin {basename(plugin_dir)} (already installed)")
except:
logger.error(f"Exception while installing plugin(s) from {plugin_url} :\n{format_exc()}")
LOGGER.error(f"Exception while installing plugin(s) from {plugin_url} :\n{format_exc()}")
status = 2
if not plugin_nbr:
logger.info("No external plugins to update to database")
LOGGER.info("No external plugins to update to database")
sys_exit(0)
external_plugins = []
@ -170,7 +176,7 @@ try:
for plugin in listdir(EXTERNAL_PLUGINS_DIR):
path = EXTERNAL_PLUGINS_DIR.joinpath(plugin)
if not path.joinpath("plugin.json").is_file():
logger.warning(f"Plugin {plugin} is not valid, deleting it...")
LOGGER.warning(f"Plugin {plugin} is not valid, deleting it...")
rmtree(path, ignore_errors=True)
continue
@ -208,18 +214,18 @@ try:
err = db.update_external_plugins(external_plugins)
if err:
logger.error(
LOGGER.error(
f"Couldn't update external plugins to database: {err}",
)
status = 1
logger.info("External plugins downloaded and installed")
LOGGER.info("External plugins downloaded and installed")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running download-plugins.py :\n{format_exc()}")
LOGGER.error(f"Exception while running download-plugins.py :\n{format_exc()}")
for plugin_tmp in glob(join(sep, "var", "tmp", "bunkerweb", "plugins", "*")):
rmtree(plugin_tmp, ignore_errors=True)

140
src/common/core/jobs/jobs/mmdb-asn.py
View file

@ -1,127 +1,137 @@
#!/usr/bin/env python3
from datetime import date
from datetime import date, datetime, timedelta
from gzip import decompress
from hashlib import sha1
from os import _exit, getenv, sep
from io import BytesIO
from os import getenv, sep
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
from typing import Optional
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from maxminddb import open_database
from requests import RequestException, get
from maxminddb import MODE_FD, open_database
from requests import RequestException, Response, get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, file_hash, is_cached_file
from common_utils import bytes_hash, file_hash # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("JOBS.mmdb-asn", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("JOBS.mmdb-asn", getenv("LOG_LEVEL", "INFO"))
status = 0
lock = Lock()
LOCK = Lock()
def request_mmdb() -> Optional[Response]:
try:
response = get("https://db-ip.com/db/download/ip-to-asn-lite", timeout=5)
response.raise_for_status()
return response
except RequestException:
return None
try:
dl_mmdb = True
tmp_path = Path(sep, "var", "tmp", "bunkerweb", "asn.mmdb")
cache_path = Path(sep, "var", "cache", "bunkerweb", "asn.mmdb")
new_hash = None
# Don't go further if the cache match the latest version
if tmp_path.exists():
with lock:
response = None
try:
response = get("https://db-ip.com/db/download/ip-to-asn-lite", timeout=5)
except RequestException:
logger.warning("Unable to check if asn.mmdb is the latest version")
response = None
if tmp_path.is_file():
response = request_mmdb()
if response and response.status_code == 200:
_sha1 = sha1()
with tmp_path.open("rb") as f:
while True:
data = f.read(1024)
if not data:
break
_sha1.update(data)
if response.content.decode().find(_sha1.hexdigest()) != -1:
logger.info("asn.mmdb is already the latest version, skipping download...")
if response.content.find(file_hash(tmp_path, algorithm="sha1").encode()) != -1:
LOGGER.info("asn.mmdb is already the latest version, skipping download...")
dl_mmdb = False
else:
logger.warning("Unable to check if asn.mmdb is the latest version, downloading it anyway...")
LOGGER.warning("Unable to check if the temporary mmdb file is the latest version, downloading it anyway...")
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
JOB = Job(LOGGER)
if dl_mmdb:
# Don't go further if the cache is fresh
if is_cached_file(cache_path, "month", db):
logger.info("asn.mmdb is already in cache, skipping download...")
_exit(0)
job_cache = JOB.get_cache("asn.mmdb", with_info=True, with_data=True)
if isinstance(job_cache, dict):
skip_dl = True
if response is None:
response = request_mmdb()
if response and response.status_code == 200:
skip_dl = response.content.find(bytes_hash(job_cache["data"], algorithm="sha1").encode()) != -1
elif job_cache["last_update"] < (datetime.now() - timedelta(weeks=1)).timestamp():
LOGGER.warning("Unable to check if the cache file is the latest version from db-ip.com and file is older than 1 week, checking anyway...")
skip_dl = False
if skip_dl:
LOGGER.info("asn.mmdb is already the latest version and is cached, skipping...")
sys_exit(0)
# Compute the mmdb URL
mmdb_url = f"https://download.db-ip.com/free/dbip-asn-lite-{date.today().strftime('%Y-%m')}.mmdb.gz"
# Download the mmdb file and save it to tmp
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = b""
LOGGER.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = BytesIO()
try:
with get(mmdb_url, stream=True, timeout=5) as resp:
resp.raise_for_status()
for chunk in resp.iter_content(chunk_size=4 * 1024):
if chunk:
file_content += chunk
file_content.write(chunk)
except RequestException:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}")
sys_exit(2)
try:
assert file_content
except AssertionError:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}")
sys_exit(2)
# Decompress it
logger.info("Decompressing mmdb file ...")
tmp_path.write_bytes(decompress(file_content))
LOGGER.info("Decompressing mmdb file ...")
file_content.seek(0)
content = BytesIO(decompress(file_content.getvalue()))
# Check if file has changed
new_hash = file_hash(tmp_path)
old_hash = cache_hash(cache_path, db)
if new_hash == old_hash:
logger.info("New file is identical to cache file, reload is not needed")
_exit(0)
if job_cache:
# Check if file has changed
new_hash = bytes_hash(content)
if new_hash == job_cache["checksum"]:
LOGGER.info("New file is identical to cache file, reload is not needed")
sys_exit(0)
# Try to load it
logger.info("Checking if mmdb file is valid ...")
with open_database(str(tmp_path)) as reader:
pass
LOGGER.info("Checking if mmdb file is valid ...")
if tmp_path.is_file():
with open_database(tmp_path.as_posix()) as reader:
pass
else:
with open_database(content, mode=MODE_FD) as reader:
pass
tmp_path = None
# Move it to cache folder
logger.info("Moving mmdb file to cache ...")
cached, err = cache_file(tmp_path, cache_path, new_hash, db)
LOGGER.info("Moving mmdb file to cache ...")
cached, err = JOB.cache_file("asn.mmdb", tmp_path or content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching mmdb file : {err}")
_exit(2)
LOGGER.error(f"Error while caching mmdb file : {err}")
sys_exit(2)
# Success
if dl_mmdb:
logger.info(f"Downloaded new mmdb from {mmdb_url}")
LOGGER.info(f"Downloaded new mmdb from {mmdb_url}")
status = 1
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running mmdb-asn.py :\n{format_exc()}")
LOGGER.error(f"Exception while running mmdb-asn.py :\n{format_exc()}")
sys_exit(status)

140
src/common/core/jobs/jobs/mmdb-country.py
View file

@ -1,127 +1,137 @@
#!/usr/bin/env python3
from datetime import date
from datetime import date, datetime, timedelta
from gzip import decompress
from hashlib import sha1
from os import _exit, getenv, sep
from io import BytesIO
from os import getenv, sep
from os.path import join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
from typing import Optional
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from maxminddb import open_database
from requests import RequestException, get
from maxminddb import MODE_FD, open_database
from requests import RequestException, Response, get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, file_hash, is_cached_file
from common_utils import bytes_hash, file_hash # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("JOBS.mmdb-country", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("JOBS.mmdb-country", getenv("LOG_LEVEL", "INFO"))
status = 0
lock = Lock()
LOCK = Lock()
def request_mmdb() -> Optional[Response]:
try:
response = get("https://db-ip.com/db/download/ip-to-country-lite", timeout=5)
response.raise_for_status()
return response
except RequestException:
return None
try:
dl_mmdb = True
tmp_path = Path(sep, "var", "tmp", "bunkerweb", "country.mmdb")
cache_path = Path(sep, "var", "cache", "bunkerweb", "country.mmdb")
new_hash = None
# Don't go further if the cache match the latest version
if tmp_path.exists():
with lock:
response = None
try:
response = get("https://db-ip.com/db/download/ip-to-country-lite", timeout=5)
except RequestException:
logger.warning("Unable to check if country.mmdb is the latest version")
response = None
if tmp_path.is_file():
response = request_mmdb()
if response and response.status_code == 200:
_sha1 = sha1()
with tmp_path.open("rb") as f:
while True:
data = f.read(1024)
if not data:
break
_sha1.update(data)
if response.content.decode().find(_sha1.hexdigest()) != -1:
logger.info("country.mmdb is already the latest version, skipping download...")
if response.content.find(file_hash(tmp_path, algorithm="sha1").encode()) != -1:
LOGGER.info("country.mmdb is already the latest version, skipping download...")
dl_mmdb = False
else:
logger.warning("Unable to check if country.mmdb is the latest version, downloading it anyway...")
LOGGER.warning("Unable to check if the temporary mmdb file is the latest version, downloading it anyway...")
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
JOB = Job(LOGGER)
if dl_mmdb:
# Don't go further if the cache is fresh
if is_cached_file(cache_path, "month", db):
logger.info("country.mmdb is already in cache, skipping download...")
_exit(0)
job_cache = JOB.get_cache("country.mmdb", with_info=True, with_data=True)
if isinstance(job_cache, dict):
skip_dl = True
if response is None:
response = request_mmdb()
if response and response.status_code == 200:
skip_dl = response.content.find(bytes_hash(job_cache["data"], algorithm="sha1").encode()) != -1
elif job_cache["last_update"] < (datetime.now() - timedelta(weeks=1)).timestamp():
LOGGER.warning("Unable to check if the cache file is the latest version from db-ip.com and file is older than 1 week, checking anyway...")
skip_dl = False
if skip_dl:
LOGGER.info("country.mmdb is already the latest version and is cached, skipping...")
sys_exit(0)
# Compute the mmdb URL
mmdb_url = f"https://download.db-ip.com/free/dbip-country-lite-{date.today().strftime('%Y-%m')}.mmdb.gz"
# Download the mmdb file and save it to tmp
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = b""
LOGGER.info(f"Downloading mmdb file from url {mmdb_url} ...")
file_content = BytesIO()
try:
with get(mmdb_url, stream=True, timeout=5) as resp:
resp.raise_for_status()
for chunk in resp.iter_content(chunk_size=4 * 1024):
if chunk:
file_content += chunk
file_content.write(chunk)
except RequestException:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}")
sys_exit(2)
try:
assert file_content
except AssertionError:
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
_exit(2)
LOGGER.error(f"Error while downloading mmdb file from {mmdb_url}")
sys_exit(2)
# Decompress it
logger.info("Decompressing mmdb file ...")
tmp_path.write_bytes(decompress(file_content))
LOGGER.info("Decompressing mmdb file ...")
file_content.seek(0)
content = BytesIO(decompress(file_content.getvalue()))
# Check if file has changed
new_hash = file_hash(tmp_path)
old_hash = cache_hash(cache_path, db)
if new_hash == old_hash:
logger.info("New file is identical to cache file, reload is not needed")
_exit(0)
if job_cache:
# Check if file has changed
new_hash = bytes_hash(content)
if new_hash == job_cache["checksum"]:
LOGGER.info("New file is identical to cache file, reload is not needed")
sys_exit(0)
# Try to load it
logger.info("Checking if mmdb file is valid ...")
with open_database(str(tmp_path)) as reader:
pass
LOGGER.info("Checking if mmdb file is valid ...")
if tmp_path.is_file():
with open_database(tmp_path.as_posix()) as reader:
pass
else:
with open_database(content, mode=MODE_FD) as reader:
pass
tmp_path = None
# Move it to cache folder
logger.info("Moving mmdb file to cache ...")
cached, err = cache_file(tmp_path, cache_path, new_hash, db)
LOGGER.info("Moving mmdb file to cache ...")
cached, err = JOB.cache_file("country.mmdb", tmp_path or content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching mmdb file : {err}")
_exit(2)
LOGGER.error(f"Error while caching mmdb file : {err}")
sys_exit(2)
# Success
if dl_mmdb:
logger.info(f"Downloaded new mmdb from {mmdb_url}")
LOGGER.info(f"Downloaded new mmdb from {mmdb_url}")
status = 1
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running mmdb-country.py :\n{format_exc()}")
LOGGER.error(f"Exception while running mmdb-country.py :\n{format_exc()}")
sys_exit(status)

22
src/common/core/jobs/plugin.json
View file

@ -4,28 +4,18 @@
"description": "Fake core plugin for internal jobs.",
"version": "1.0",
"stream": "yes",
"settings": {
"SEND_ANONYMOUS_REPORT": {
"context": "global",
"default": "yes",
"help": "Send anonymous report to BunkerWeb maintainers.",
"id": "send-anonymous-report",
"label": "Send anonymous report",
"regex": "^(yes|no)$",
"type": "check"
}
},
"settings": {},
"jobs": [
{
"name": "mmdb-country",
"file": "mmdb-country.py",
"every": "week",
"every": "day",
"reload": true
},
{
"name": "mmdb-asn",
"file": "mmdb-asn.py",
"every": "week",
"every": "day",
"reload": true
},
{
@ -33,12 +23,6 @@
"file": "download-plugins.py",
"every": "once",
"reload": false
},
{
"name": "anonymous-report",
"file": "anonymous-report.py",
"every": "day",
"reload": false
}
]
}

30
src/common/core/letsencrypt/jobs/certbot-auth.py
View file

@ -7,50 +7,46 @@ from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("api",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from jobs import get_integration # type: ignore
from common_utils import get_integration # type: ignore
from logger import setup_logger # type: ignore
from API import API # type: ignore
logger = setup_logger("Lets-encrypt.auth", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Lets-encrypt.auth", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
# Get env vars
token = getenv("CERTBOT_TOKEN", "")
validation = getenv("CERTBOT_VALIDATION", "")
integration = get_integration()
LOGGER.info(f"Detected {integration} integration")
# Cluster case
if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
if integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None))
lock = Lock()
with lock:
instances = db.get_instances()
LOGGER.info(f"Sending challenge to {len(instances)} instances")
for instance in instances:
api = API(f"http://{instance['hostname']}:{instance['port']}", host=instance["server_name"])
sent, err, status, resp = api.request("POST", "/lets-encrypt/challenge", data={"token": token, "validation": validation})
if not sent:
status = 1
logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
elif status != 200:
status = 1
logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}")
LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge")
LOGGER.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge")
# Linux case
else:
@ -59,6 +55,6 @@ try:
root_dir.joinpath(token).write_text(validation, encoding="utf-8")
except:
status = 1
logger.error(f"Exception while running certbot-auth.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-auth.py :\n{format_exc()}")
sys_exit(status)

30
src/common/core/letsencrypt/jobs/certbot-cleanup.py
View file

@ -7,53 +7,49 @@ from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("api",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from jobs import get_integration # type: ignore
from common_utils import get_integration # type: ignore
from logger import setup_logger # type: ignore
from API import API # type: ignore
logger = setup_logger("Lets-encrypt.cleanup", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Lets-encrypt.cleanup", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
# Get env vars
token = getenv("CERTBOT_TOKEN", "")
integration = get_integration()
LOGGER.info(f"Detected {integration} integration")
# Cluster case
if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
if integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None))
lock = Lock()
with lock:
instances = db.get_instances()
LOGGER.info(f"Cleaning challenge from {len(instances)} instances")
for instance in instances:
api = API(f"http://{instance['hostname']}:{instance['port']}", host=instance["server_name"])
sent, err, status, resp = api.request("DELETE", "/lets-encrypt/challenge", data={"token": token})
if not sent:
status = 1
logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
elif status != 200:
status = 1
logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}")
LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge")
LOGGER.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge")
# Linux case
else:
Path(sep, "var", "tmp", "bunkerweb", "lets-encrypt", ".well-known", "acme-challenge", token).unlink(missing_ok=True)
except:
status = 1
logger.error(f"Exception while running certbot-cleanup.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-cleanup.py :\n{format_exc()}")
sys_exit(status)

36
src/common/core/letsencrypt/jobs/certbot-deploy.py
View file

@ -9,31 +9,23 @@ from tarfile import open as tar_open
from threading import Lock
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("api",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from jobs import get_integration # type: ignore
from common_utils import get_integration # type: ignore
from logger import setup_logger # type: ignore
from API import API # type: ignore
logger = setup_logger("Lets-encrypt.deploy", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Lets-encrypt.deploy", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
# Get env vars
token = getenv("CERTBOT_TOKEN", "")
logger.info(f"Certificates renewal for {getenv('RENEWED_DOMAINS')} successful")
LOGGER.info(f"Certificates renewal for {getenv('RENEWED_DOMAINS')} successful")
# Cluster case
if get_integration() in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
@ -45,7 +37,7 @@ try:
tgz.seek(0, 0)
files = {"archive.tar.gz": tgz}
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI", None))
lock = Lock()
with lock:
@ -59,32 +51,32 @@ try:
sent, err, status, resp = api.request("POST", "/lets-encrypt/certificates", files=files)
if not sent:
status = 1
logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/certificates : {err}")
LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/certificates : {err}")
elif status != 200:
status = 1
logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/certificates : status = {resp['status']}, msg = {resp['msg']}")
LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/certificates : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(
LOGGER.info(
f"Successfully sent API request to {api.endpoint}/lets-encrypt/certificates",
)
sent, err, status, resp = api.request("POST", "/reload")
if not sent:
status = 1
logger.error(f"Can't send API request to {api.endpoint}/reload : {err}")
LOGGER.error(f"Can't send API request to {api.endpoint}/reload : {err}")
elif status != 200:
status = 1
logger.error(f"Error while sending API request to {api.endpoint}/reload : status = {resp['status']}, msg = {resp['msg']}")
LOGGER.error(f"Error while sending API request to {api.endpoint}/reload : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(f"Successfully sent API request to {api.endpoint}/reload")
LOGGER.info(f"Successfully sent API request to {api.endpoint}/reload")
# Linux case
else:
if run([join(sep, "usr", "sbin", "nginx"), "-s", "reload"], stdin=DEVNULL, stderr=STDOUT, check=False).returncode != 0:
status = 1
logger.error("Error while reloading nginx")
LOGGER.error("Error while reloading nginx")
else:
logger.info("Successfully reloaded nginx")
LOGGER.info("Successfully reloaded nginx")
except:
status = 1
logger.error(f"Exception while running certbot-deploy.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-deploy.py :\n{format_exc()}")
sys_exit(status)

102
src/common/core/letsencrypt/jobs/certbot-new.py
View file

@ -1,32 +1,22 @@
#!/usr/bin/env python3
from os import _exit, environ, getenv, sep
from os import environ, getenv, sep
from os.path import join
from pathlib import Path
from subprocess import DEVNULL, STDOUT, run, PIPE
from subprocess import DEVNULL, STDOUT, Popen, run, PIPE
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from tarfile import open as tar_open
from io import BytesIO
from shutil import rmtree
from re import MULTILINE, search
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import get_file_in_db, set_file_in_db # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("LETS-ENCRYPT.new", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("LETS-ENCRYPT.new", getenv("LOG_LEVEL", "INFO"))
LOGGER_CERTBOT = setup_logger("LETS-ENCRYPT.new.certbot", getenv("LOG_LEVEL", "INFO"))
status = 0
CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot")
@ -38,7 +28,7 @@ LETS_ENCRYPT_LOGS_DIR = join(sep, "var", "log", "bunkerweb")
def certbot_new(domains: str, email: str, use_letsencrypt_staging: bool = False) -> int:
return run(
process = Popen(
[
CERTBOT_BIN,
"certonly",
@ -64,9 +54,15 @@ def certbot_new(domains: str, email: str, use_letsencrypt_staging: bool = False)
]
+ (["--staging"] if use_letsencrypt_staging else []),
stdin=DEVNULL,
stderr=STDOUT,
stderr=PIPE,
universal_newlines=True,
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
).returncode
)
while process.poll() is None:
if process.stderr:
for line in process.stderr:
LOGGER_CERTBOT.info(line.strip())
return process.returncode
status = 0
@ -82,36 +78,21 @@ try:
use_letsencrypt = True
elif is_multisite:
for first_server in server_names:
if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes":
if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", getenv("AUTO_LETS_ENCRYPT", "no")) == "yes":
use_letsencrypt = True
break
if not use_letsencrypt:
logger.info("Let's Encrypt is not activated, skipping generation...")
_exit(0)
LOGGER.info("Let's Encrypt is not activated, skipping generation...")
sys_exit(0)
elif not getenv("SERVER_NAME"):
logger.warning("There are no server names, skipping generation...")
_exit(0)
LOGGER.warning("There are no server names, skipping generation...")
sys_exit(0)
# Create directories if they doesn't exist
LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True)
Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Extract letsencrypt folder if it exists in db
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
tgz = get_file_in_db("folder.tgz", db, job_name="certbot-renew")
if tgz:
# Delete folder if needed
if LETS_ENCRYPT_PATH.exists():
rmtree(LETS_ENCRYPT_PATH, ignore_errors=True)
LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True)
# Extract it
with tar_open(name="folder.tgz", mode="r:gz", fileobj=BytesIO(tgz)) as tf:
tf.extractall(LETS_ENCRYPT_PATH)
logger.info("Successfully retrieved Let's Encrypt data from db cache")
else:
logger.info("No Let's Encrypt data found in db cache")
# Restore Let's Encrypt data from db cache
JOB.restore_cache(job_name="certbot-renew")
domains_to_ask = []
# Multisite case
@ -142,11 +123,12 @@ try:
stderr=STDOUT,
text=True,
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
check=False,
)
stdout = proc.stdout
if proc.returncode != 0:
logger.error(f"Error while checking certificates :\n{proc.stdout}")
LOGGER.error(f"Error while checking certificates :\n{proc.stdout}")
domains_to_ask = server_names
else:
for first_server, domains in domains_sever_names.items():
@ -155,10 +137,10 @@ try:
domains_to_ask.append(first_server)
continue
elif set(f"{first_server}{current_domains.groupdict()['domains']}".strip().split(" ")) != set(domains.split(" ")):
logger.warning(f"Domains for {first_server} are not the same as in the certificate, asking new certificate...")
LOGGER.warning(f"Domains for {first_server} are not the same as in the certificate, asking new certificate...")
domains_to_ask.append(first_server)
continue
logger.info(f"Certificates already exists for domain(s) {domains}")
LOGGER.info(f"Certificates already exists for domain(s) {domains}")
for first_server, domains in domains_sever_names.items():
if first_server not in domains_to_ask:
@ -170,30 +152,26 @@ try:
use_letsencrypt_staging = getenv(f"{first_server}_USE_LETS_ENCRYPT_STAGING", getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes"
logger.info(f"Asking certificates for domain(s) : {domains} (email = {real_email}) to Let's Encrypt {'staging ' if use_letsencrypt_staging else ''}...")
LOGGER.info(f"Asking certificates for domain(s) : {domains} (email = {real_email}) to Let's Encrypt {'staging ' if use_letsencrypt_staging else ''}...")
if certbot_new(domains.replace(" ", ","), real_email, use_letsencrypt_staging) != 0:
status = 2
logger.error(f"Certificate generation failed for domain(s) {domains} ...")
LOGGER.error(f"Certificate generation failed for domain(s) {domains} ...")
continue
else:
status = 1 if status == 0 else status
logger.info(f"Certificate generation succeeded for domain(s) : {domains}")
LOGGER.info(f"Certificate generation succeeded for domain(s) : {domains}")
# Put new folder in cache
bio = BytesIO()
with tar_open("folder.tgz", mode="w:gz", fileobj=bio, compresslevel=9) as tgz:
tgz.add(LETS_ENCRYPT_PATH, arcname=".")
bio.seek(0, 0)
# Put tgz in cache
cached, err = set_file_in_db("folder.tgz", bio.read(), db, job_name="certbot-renew")
if not cached:
logger.error(f"Error while saving Let's Encrypt data to db cache : {err}")
else:
logger.info("Successfully saved Let's Encrypt data to db cache")
# Save Let's Encrypt data to db cache
if LETS_ENCRYPT_PATH.is_dir() and list(LETS_ENCRYPT_PATH.iterdir()):
cached, err = JOB.cache_dir(LETS_ENCRYPT_PATH, job_name="certbot-renew")
if not cached:
LOGGER.error(f"Error while saving Let's Encrypt data to db cache : {err}")
else:
LOGGER.info("Successfully saved Let's Encrypt data to db cache")
except SystemExit as e:
status = e.code
except:
status = 3
logger.error(f"Exception while running certbot-new.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-new.py :\n{format_exc()}")
sys_exit(status)

107
src/common/core/letsencrypt/jobs/certbot-renew.py
View file

@ -1,14 +1,11 @@
#!/usr/bin/env python3
from os import _exit, environ, getenv, sep
from os import environ, getenv, sep
from os.path import join
from pathlib import Path
from subprocess import DEVNULL, STDOUT, run
from subprocess import DEVNULL, PIPE, Popen
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from tarfile import open as tar_open
from io import BytesIO
from shutil import rmtree
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
@ -21,14 +18,18 @@ for deps_path in [
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import get_file_in_db, set_file_in_db # type: ignore
from jobs import Job # type: ignore
logger = setup_logger("LETS-ENCRYPT.renew", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("LETS-ENCRYPT.renew", getenv("LOG_LEVEL", "INFO"))
LOGGER_CERTBOT = setup_logger("LETS-ENCRYPT.renew.certbot", getenv("LOG_LEVEL", "INFO"))
status = 0
CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot")
LETS_ENCRYPT_PATH = Path(sep, "var", "cache", "bunkerweb", "letsencrypt")
LETS_ENCRYPT_WORK_DIR = join(sep, "var", "lib", "bunkerweb", "letsencrypt")
LETS_ENCRYPT_LOGS_DIR = join(sep, "var", "log", "bunkerweb")
try:
# Check if we're using let's encrypt
@ -42,66 +43,48 @@ try:
break
if not use_letsencrypt:
logger.info("Let's Encrypt is not activated, skipping renew...")
_exit(0)
LOGGER.info("Let's Encrypt is not activated, skipping renew...")
sys_exit(0)
# Create directory if it doesn't exist
LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True)
Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Extract letsencrypt folder if it exists in db
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
process = Popen(
[
CERTBOT_BIN,
"renew",
"--no-random-sleep-on-renew",
"--config-dir",
LETS_ENCRYPT_PATH.joinpath("etc").as_posix(),
"--work-dir",
LETS_ENCRYPT_WORK_DIR,
"--logs-dir",
LETS_ENCRYPT_LOGS_DIR,
],
stdin=DEVNULL,
stderr=PIPE,
universal_newlines=True,
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
)
while process.poll() is None:
if process.stderr:
for line in process.stderr:
LOGGER_CERTBOT.info(line.strip())
tgz = get_file_in_db("folder.tgz", db)
if tgz:
# Delete folder if needed
if LETS_ENCRYPT_PATH.exists():
rmtree(LETS_ENCRYPT_PATH, ignore_errors=True)
LETS_ENCRYPT_PATH.mkdir(parents=True, exist_ok=True)
# Extract it
with tar_open(name="folder.tgz", mode="r:gz", fileobj=BytesIO(tgz)) as tf:
tf.extractall(LETS_ENCRYPT_PATH)
logger.info("Successfully retrieved Let's Encrypt data from db cache")
else:
logger.info("No Let's Encrypt data found in db cache")
if (
run(
[
join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot"),
"renew",
"--no-random-sleep-on-renew",
"--config-dir",
LETS_ENCRYPT_PATH.joinpath("etc").as_posix(),
"--work-dir",
join(sep, "var", "lib", "bunkerweb", "letsencrypt"),
"--logs-dir",
join(sep, "var", "log", "bunkerweb"),
],
stdin=DEVNULL,
stderr=STDOUT,
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
check=False,
).returncode
!= 0
):
if process.returncode != 0:
status = 2
logger.error("Certificates renewal failed")
LOGGER.error("Certificates renewal failed")
# Put new folder in cache
bio = BytesIO()
with tar_open("folder.tgz", mode="w:gz", fileobj=bio, compresslevel=9) as tgz:
tgz.add(LETS_ENCRYPT_PATH, arcname=".")
bio.seek(0, 0)
# Put tgz in cache
cached, err = set_file_in_db("folder.tgz", bio.read(), db)
if not cached:
logger.error(f"Error while saving Let's Encrypt data to db cache : {err}")
else:
logger.info("Successfully saved Let's Encrypt data to db cache")
# Save Let's Encrypt data to db cache
if LETS_ENCRYPT_PATH.is_dir() and list(LETS_ENCRYPT_PATH.iterdir()):
cached, err = JOB.cache_dir(LETS_ENCRYPT_PATH)
if not cached:
LOGGER.error(f"Error while saving Let's Encrypt data to db cache : {err}")
else:
LOGGER.info("Successfully saved Let's Encrypt data to db cache")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running certbot-renew.py :\n{format_exc()}")
LOGGER.error(f"Exception while running certbot-renew.py :\n{format_exc()}")
sys_exit(status)

10
src/common/core/limit/ui/actions.py
View file

@ -1,7 +1,7 @@
from operator import itemgetter
def limit(**kwargs):
def pre_render(**kwargs):
try:
# Here we will have a list { 'limit_uri_url1': X, 'limit_uri_url2': Y ... }
data = kwargs["app"].config["INSTANCES"].get_metrics("limit")
@ -15,6 +15,10 @@ def limit(**kwargs):
key = ""
format_data.append({"url": f"/{key}", "count": int(value)})
format_data.sort(key=itemgetter("count"), reverse=True)
return {"items": format_data}
return {"top_limit": format_data}
except:
return {"items": []}
return {"top_limit": []}
def limit(**kwargs):
pass

148
src/common/core/limit/ui/template.html
View file

@ -7,53 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div data-fetch-success-show class="hidden core-card-list w-large">
<div class="core-card-list-container">
<h5 class="core-card-list-title">LIMIT AND REQUEST LIST</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap w-large">
<!-- header-->
<p class="core-card-list-header col-span-8">URL</p>
<p class="core-card-list-header col-span-4">Count</p>
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
<li data-item class="core-card-list-item">
<p data-name="url" class="core-card-list-item-content col-span-8"></p>
<p data-name="count" class="core-card-list-item-content col-span-4"></p>
</li>
</ul>
<!-- end list-->
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<!-- end list container-->
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
items: {
el: document.querySelector("[data-item]"),
value: [],
type: "list",
listNames: ["url", "count"],
},
});
</script>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -72,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

3
src/common/core/misc/confs/default-server-http/page.conf
View file

@ -2,5 +2,8 @@
root /usr/share/bunkerweb/core/misc/files;
location / {
try_files /default.html =404;
etag off;
add_header Last-Modified "";
server_tokens off;
}
{% endif %}

61
src/common/core/jobs/jobs/anonymous-report.py → src/common/core/misc/jobs/anonymous-report.py
View file

@ -3,8 +3,6 @@
from json import dumps
from os import getenv, sep
from os.path import join
from pathlib import Path
from platform import machine
from re import compile as re_compile
from sys import exit as sys_exit, path as sys_path, version
from traceback import format_exc
@ -14,33 +12,28 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from common_utils import get_os_info # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, is_cached_file # type: ignore
from jobs import Job # type: ignore
from requests import post
logger = setup_logger("ANONYMOUS-REPORT", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("ANONYMOUS-REPORT", getenv("LOG_LEVEL", "INFO"))
status = 0
if getenv("SEND_ANONYMOUS_REPORT", "yes") != "yes":
logger.info("Skipping the sending of anonymous report (disabled)")
sys_exit(status)
anonymous_report_path = Path(sep, "var", "cache", "bunkerweb", "anonymous_report")
anonymous_report_path.mkdir(parents=True, exist_ok=True)
tmp_anonymous_report_path = Path(sep, "var", "tmp", "bunkerweb", "anonymous_report")
tmp_anonymous_report_path.mkdir(parents=True, exist_ok=True)
try:
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
if is_cached_file(anonymous_report_path.joinpath("last_report.json"), "day", db):
logger.info("Skipping the sending of anonymous report (already sent today)")
if getenv("SEND_ANONYMOUS_REPORT", "yes") != "yes":
LOGGER.info("Skipping the sending of anonymous report (disabled)")
sys_exit(status)
JOB = Job(LOGGER)
if JOB.is_cached_file("last_report.json", "day"):
LOGGER.info("Skipping the sending of anonymous report (already sent today)")
sys_exit(0)
# ? Get version and integration of BunkerWeb
data: Dict[str, Any] = db.get_metadata()
data: Dict[str, Any] = JOB.db.get_metadata()
data["is_pro"] = "yes" if data["is_pro"] else "no"
@ -48,7 +41,7 @@ try:
if key not in ("version", "integration", "database_version", "is_pro"):
data.pop(key, None)
db_config = db.get_config(methods=True, with_drafts=True)
db_config = JOB.db.get_config(methods=True, with_drafts=True)
services = db_config.get("SERVER_NAME", {"value": ""})["value"].split(" ")
multisite = db_config.get("MULTISITE", {"value": "no"})["value"] == "yes"
@ -58,7 +51,7 @@ try:
database_version = database_version.group(1)
data["integration"] = data["integration"].lower()
data["database"] = f"{db.database_uri.split(':')[0].split('+')[0]}/{database_version}"
data["database"] = f"{JOB.db.database_uri.split(':')[0].split('+')[0]}/{database_version}"
data["service_number"] = str(len(services))
data["draft_service_number"] = 0
data["python_version"] = version.split(" ")[0]
@ -82,26 +75,13 @@ try:
data["external_plugins"] = []
data["pro_plugins"] = []
for plugin in db.get_plugins():
for plugin in JOB.db.get_plugins():
if plugin["type"] == "external":
data["external_plugins"].append(f"{plugin['id']}/{plugin['version']}")
elif plugin["type"] == "pro":
data["pro_plugins"].append(f"{plugin['id']}/{plugin['version']}")
data["os"] = {
"name": "Linux",
"version": "Unknown",
"version_id": "Unknown",
"version_codename": "Unknown",
"id": "Unknown",
"arch": machine(),
}
os_release = Path("/etc/os-release")
if os_release.exists():
for line in os_release.read_text().splitlines():
if "=" not in line or line.split("=")[0].strip().lower() not in data["os"]:
continue
data["os"][line.split("=")[0].lower()] = line.split("=")[1].strip('"')
data["os"] = get_os_info()
data["non_default_settings"] = {}
for setting, setting_data in db_config.items():
@ -121,18 +101,19 @@ try:
for key in data["non_default_settings"].copy():
data["non_default_settings"][key] = str(data["non_default_settings"][key])
data["bw_instances_number"] = str(len(db.get_instances()))
tmp_anonymous_report_path.joinpath("last_report.json").write_text(dumps(data, indent=4), encoding="utf-8")
data["bw_instances_number"] = str(len(JOB.db.get_instances()))
response = post("https://api.bunkerweb.io/data", json=data, headers={"User-Agent": f"BunkerWeb/{data['version']}"}, allow_redirects=True, timeout=10)
response.raise_for_status()
cached, err = cache_file(tmp_anonymous_report_path.joinpath("last_report.json"), anonymous_report_path.joinpath("last_report.json"), None, db)
cached, err = JOB.cache_file("last_report.json", dumps(data, indent=4).encode())
if not cached:
LOGGER.error(f"Failed to cache last_report.json :\n{err}")
status = 2
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running anonymous-report.py :\n{format_exc()}")
LOGGER.error(f"Exception while running anonymous-report.py :\n{format_exc()}")
sys_exit(status)

69
src/common/core/misc/jobs/default-server-cert.py
View file

@ -7,29 +7,24 @@ from subprocess import DEVNULL, run
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import set_file_in_db
from jobs import Job # type: ignore
logger = setup_logger("DEFAULT-SERVER-CERT", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("DEFAULT-SERVER-CERT", getenv("LOG_LEVEL", "INFO"))
LOGGER_OPENSSL = setup_logger("DEFAULT-SERVER-CERT.openssl", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
cert_path = Path(sep, "var", "cache", "bunkerweb", "default-server-cert")
cert_path.mkdir(parents=True, exist_ok=True)
if not cert_path.joinpath("cert.pem").is_file():
logger.info("Generating self-signed certificate for default server")
JOB = Job(LOGGER)
cert_path = Path(sep, "var", "cache", "bunkerweb", "misc")
if not JOB.is_cached_file("default-server-cert.pem", "month") or not JOB.is_cached_file("default-server-cert.key", "month"):
LOGGER.info("Generating self-signed certificate for default server")
cert_path.mkdir(parents=True, exist_ok=True)
if (
run(
@ -39,11 +34,13 @@ try:
"-nodes",
"-x509",
"-newkey",
"ed25519",
"ec",
"-pkeyopt",
"ec_paramgen_curve:prime256v1",
"-keyout",
str(cert_path.joinpath("cert.key")),
str(cert_path.joinpath("default-server-cert.key")),
"-out",
str(cert_path.joinpath("cert.pem")),
str(cert_path.joinpath("default-server-cert.pem")),
"-days",
"3650",
"-subj",
@ -55,43 +52,27 @@ try:
).returncode
!= 0
):
logger.error(
"Self-signed certificate generation failed for default server",
)
LOGGER.error("Self-signed certificate generation failed for default server")
status = 2
else:
LOGGER.info("Successfully generated self-signed certificate for default server")
status = 1
logger.info(
"Successfully generated self-signed certificate for default server",
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
cached, err = set_file_in_db(
"cert.pem",
cert_path.joinpath("cert.pem").read_bytes(),
db,
)
cached, err = JOB.cache_file("default-server-cert.pem", cert_path.joinpath("default-server-cert.pem"), overwrite_file=False)
if not cached:
logger.error(f"Error while saving default-server-cert cert.pem file to db cache : {err}")
LOGGER.error(f"Error while saving default-server-cert default-server-cert.pem file to db cache : {err}")
else:
logger.info("Successfully saved default-server-cert cert.pem file to db cache")
LOGGER.info("Successfully saved default-server-cert default-server-cert.pem file to db cache")
cached, err = set_file_in_db(
"cert.key",
cert_path.joinpath("cert.key").read_bytes(),
db,
)
cached, err = JOB.cache_file("default-server-cert.key", cert_path.joinpath("default-server-cert.key"), overwrite_file=False)
if not cached:
logger.error(f"Error while saving default-server-cert cert.key file to db cache : {err}")
LOGGER.error(f"Error while saving default-server-cert default-server-cert.key file to db cache : {err}")
else:
logger.info("Successfully saved default-server-cert cert.key file to db cache")
LOGGER.info("Successfully saved default-server-cert default-server-cert.key file to db cache")
else:
logger.info(
"Skipping generation of self-signed certificate for default server (already present)",
)
LOGGER.info("Skipping generation of self-signed certificate for default server (already present)")
except:
status = 2
logger.error(f"Exception while running default-server-cert.py :\n{format_exc()}")
LOGGER.error(f"Exception while running default-server-cert.py :\n{format_exc()}")
sys_exit(status)

30
src/common/core/misc/jobs/update-check.py
View file

@ -2,46 +2,34 @@
from os import getenv, sep
from os.path import basename, join
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from requests import get
from common_utils import get_version # type: ignore
from logger import setup_logger # type: ignore
logger = setup_logger("UPDATE-CHECK", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("UPDATE-CHECK", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
current_version = f"v{Path('/usr/share/bunkerweb/VERSION').read_text(encoding='utf-8').strip()}"
current_version = f"v{get_version().strip()}"
response = get(
"https://github.com/bunkerity/bunkerweb/releases/latest",
headers={"User-Agent": "BunkerWeb"},
allow_redirects=True,
timeout=10,
)
response = get("https://github.com/bunkerity/bunkerweb/releases/latest", headers={"User-Agent": "BunkerWeb"}, allow_redirects=True, timeout=10)
response.raise_for_status()
latest_version = basename(response.url)
if current_version != latest_version:
logger.warning(
f"* \n* \n* 🚨 A new version of BunkerWeb is available: {latest_version} (current: {current_version}) 🚨\n* \n* ",
)
LOGGER.warning(f"* \n* \n* 🚨 A new version of BunkerWeb is available: {latest_version} (current: {current_version}) 🚨\n* \n* ")
else:
logger.info(f"Latest version is already installed: {current_version}")
LOGGER.info(f"Latest version is already installed: {current_version}")
except:
status = 2
logger.error(f"Exception while running update-check.py :\n{format_exc()}")
LOGGER.error(f"Exception while running update-check.py :\n{format_exc()}")
sys_exit(status)

15
src/common/core/misc/plugin.json
View file

@ -158,6 +158,15 @@
"regex": "^(403|444)$",
"type": "select",
"select": ["403", "444"]
},
"SEND_ANONYMOUS_REPORT": {
"context": "global",
"default": "yes",
"help": "Send anonymous report to BunkerWeb maintainers.",
"id": "send-anonymous-report",
"label": "Send anonymous report",
"regex": "^(yes|no)$",
"type": "check"
}
},
"jobs": [
@ -172,6 +181,12 @@
"file": "update-check.py",
"every": "day",
"reload": false
},
{
"name": "anonymous-report",
"file": "anonymous-report.py",
"every": "day",
"reload": false
}
]
}

40
src/common/core/misc/ui/actions.py
View file

@ -1,14 +1,36 @@
def misc(**kwargs):
def pre_render(**kwargs):
try:
data = kwargs["app"].config["INSTANCES"].get_metrics("misc")
if "counter_failed_default" not in data:
data["counter_failed_default"] = 0
if "counter_failed_method" not in data:
data["counter_failed_method"] = 0
return data
return {
"counter_failed_default": {
"value": data.get("counter_failed_default", 0),
"title": "DEFAULT SERVER DISABLED",
"subtitle": "total",
"subtitle_color": "info",
"svg_color": "sky",
},
"counter_failed_method": {
"value": data.get("counter_failed_method", 0),
"title": "DISALLOWED METHODS",
"subtitle": "count",
"subtitle_color": "info",
"svg_color": "lime",
},
}
except:
return {"counter_failed_default": 0, "counter_failed_method": 0}
return {
"counter_failed_default": {
"value": "unknown",
"title": "DEFAULT SERVER DISABLED",
"subtitle": "total",
"subtitle_color": "info",
"svg_color": "sky",
},
"counter_failed_method": {"value": "unknown", "title": "DISALLOWED METHODS", "subtitle": "count", "subtitle_color": "info", "svg_color": "lime"},
}
def misc(**kwargs):
pass

176
src/common/core/misc/ui/template.html
View file

@ -7,79 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<div class="core-layout">
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
<!-- end info -->
</div>
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">DEFAULT SERVER DISABLED</p>
<h5 data-count-server-disabled class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content info">total</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container orange">
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="scale-[0.55] core-card-metrics-svg">
<path d="M4.08 5.227A3 3 0 0 1 6.979 3H17.02a3 3 0 0 1 2.9 2.227l2.113 7.926A5.228 5.228 0 0 0 18.75 12H5.25a5.228 5.228 0 0 0-3.284 1.153L4.08 5.227Z" />
<path fill-rule="evenodd" d="M5.25 13.5a3.75 3.75 0 1 0 0 7.5h13.5a3.75 3.75 0 1 0 0-7.5H5.25Zm10.5 4.5a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm3.75-.75a.75.75 0 1 1-1.5 0 .75.75 0 0 1 1.5 0Z" clip-rule="evenodd" />
</svg>
</div>
<!-- end icon -->
</div>
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">DISALLOWED METHODS</p>
<h5 data-count-disallowed-methods class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content info">count</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container lime">
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg">
<path d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z" />
</svg>
</div>
<!-- end icon -->
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
counter_failed_default: {
el: document.querySelector("[data-count-server-disabled]"),
value: "unknown",
type: "text",
},
counter_failed_method: {
el: document.querySelector("[data-count-disallowed-methods]"),
value: "unknown",
type: "text",
},
});
</script>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -98,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

128
src/common/core/pro/jobs/download-pro-plugins.py
View file

@ -4,7 +4,7 @@ from datetime import datetime
from hashlib import sha256
from io import BytesIO
from os import getenv, listdir, chmod, sep
from os.path import basename, join
from os.path import join
from pathlib import Path
from stat import S_IEXEC
from sys import exit as sys_exit, path as sys_path
@ -17,15 +17,7 @@ from tarfile import open as tar_open
from traceback import format_exc
from zipfile import ZipFile
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("api",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -33,7 +25,7 @@ from requests import get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import get_os_info, get_integration, get_version # type: ignore
from common_utils import get_os_info, get_integration, get_version # type: ignore
API_ENDPOINT = "https://api.bunkerweb.io"
PREVIEW_ENDPOINT = "https://assets.bunkerity.com/bw-pro/preview"
@ -44,31 +36,30 @@ STATUS_MESSAGES = {
"expired": "has expired",
"suspended": "has been suspended",
}
logger = setup_logger("Jobs.download-pro-plugins", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Jobs.download-pro-plugins", getenv("LOG_LEVEL", "INFO"))
status = 0
def clean_pro_plugins(db) -> None:
logger.debug("Cleaning up Pro plugins...")
LOGGER.debug("Cleaning up Pro plugins...")
# Clean Pro plugins
rmtree(PRO_PLUGINS_DIR.joinpath("*"), ignore_errors=True)
# Update database
db.update_external_plugins([], _type="pro")
def install_plugin(plugin_dir: str, db, preview: bool = True) -> bool:
plugin_path = Path(plugin_dir)
def install_plugin(plugin_path: Path, db, preview: bool = True) -> bool:
plugin_file = plugin_path.joinpath("plugin.json")
if not plugin_file.is_file():
logger.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json not found)")
LOGGER.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json not found)")
return False
# Load plugin.json
try:
metadata = loads(plugin_file.read_text(encoding="utf-8"))
except JSONDecodeError:
logger.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json is not valid)")
LOGGER.error(f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {plugin_path.name} (plugin.json is not valid)")
return False
# Don't go further if plugin is already installed
@ -81,37 +72,38 @@ def install_plugin(plugin_dir: str, db, preview: bool = True) -> bool:
break
if old_version == metadata["version"]:
logger.warning(
LOGGER.warning(
f"Skipping installation of {'preview version of ' if preview else ''}Pro plugin {metadata['id']} (version {metadata['version']} already installed)"
)
return False
logger.warning(
LOGGER.warning(
f"{'Preview version of ' if preview else ''}Pro plugin {metadata['id']} is already installed but version {metadata['version']} is different from database ({old_version}), updating it..."
)
rmtree(PRO_PLUGINS_DIR.joinpath(metadata["id"]), ignore_errors=True)
# Copy the plugin
copytree(plugin_dir, PRO_PLUGINS_DIR.joinpath(metadata["id"]))
copytree(plugin_path, PRO_PLUGINS_DIR.joinpath(metadata["id"]))
# Add u+x permissions to jobs files
for job_file in glob(PRO_PLUGINS_DIR.joinpath(metadata["id"], "jobs", "*").as_posix()):
st = Path(job_file).stat()
chmod(job_file, st.st_mode | S_IEXEC)
logger.info(f"✅ {'Preview version of ' if preview else ''}Pro plugin {metadata['id']} (version {metadata['version']}) installed successfully!")
LOGGER.info(f"✅ {'Preview version of ' if preview else ''}Pro plugin {metadata['id']} (version {metadata['version']}) installed successfully!")
return True
try:
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI"), pool=False)
db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI"))
db_metadata = db.get_metadata()
current_date = datetime.now()
pro_license_key = getenv("PRO_LICENSE_KEY")
# If we already checked in the last 10 minutes, skip the check
if db_metadata["last_pro_check"] and (current_date - db_metadata["last_pro_check"]).seconds < 600:
logger.info("Skipping the check for BunkerWeb Pro license (already checked in the last 10 minutes)")
# If we already checked today, skip the check
if pro_license_key == db_metadata["pro_license_key"] and db_metadata["last_pro_check"] and current_date.day == db_metadata["last_pro_check"].day:
LOGGER.info("Skipping the check for BunkerWeb Pro license (already checked today)")
sys_exit(0)
logger.info("Checking BunkerWeb Pro license key...")
LOGGER.info("Checking BunkerWeb Pro license key...")
data = {
"integration": get_integration(),
@ -122,63 +114,70 @@ try:
headers = {"User-Agent": f"BunkerWeb/{data['version']}"}
default_metadata = {
"is_pro": False,
"pro_license_key": None,
"pro_expire": None,
"pro_status": "invalid",
"pro_overlapped": False,
"pro_services": 0,
}
metadata = {}
pro_license_key = getenv("PRO_LICENSE_KEY")
error = False
temp_dir = TMP_DIR.joinpath(str(uuid4()))
temp_dir.mkdir(parents=True, exist_ok=True)
if pro_license_key:
logger.info("BunkerWeb Pro license provided, checking if it's valid...")
headers["Authorization"] = f"Bearer {pro_license_key.strip()}"
resp = get(f"{API_ENDPOINT}/pro-status", headers=headers, json=data, timeout=5, allow_redirects=True)
default_metadata["pro_license_key"] = (pro_license_key := pro_license_key.strip())
LOGGER.info("BunkerWeb Pro license provided, checking if it's valid...")
headers["Authorization"] = f"Bearer {pro_license_key}"
resp = get(f"{API_ENDPOINT}/pro/status", headers=headers, json=data, timeout=5, allow_redirects=True)
if resp.status_code == 403:
logger.error(f"Access denied to {API_ENDPOINT}/pro-status - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/")
LOGGER.error(f"Access denied to {API_ENDPOINT}/pro-status - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/")
error = True
if db_metadata["is_pro"]:
clean_pro_plugins(db)
if resp.headers.get("Content-Type", "") == "application/json":
resp_data = resp.json()
if db_metadata["is_pro"] and resp_data.get("action") == "clean":
clean_pro_plugins(db)
elif resp.status_code == 500:
logger.error("An error occurred with the remote server while checking BunkerWeb Pro license, please try again later")
LOGGER.error("An error occurred with the remote server while checking BunkerWeb Pro license, please try again later")
status = 2
sys_exit(status)
else:
resp.raise_for_status()
metadata = resp.json()["data"]
logger.debug(f"Got BunkerWeb Pro license metadata: {metadata}")
LOGGER.debug(f"Got BunkerWeb Pro license metadata: {metadata}")
metadata["pro_expire"] = datetime.strptime(metadata["pro_expire"], "%Y-%m-%d") if metadata["pro_expire"] else None
if metadata["pro_expire"] and metadata["pro_expire"] < datetime.now():
metadata["pro_status"] = "expired"
if metadata["pro_services"] < int(data["service_number"]):
metadata["pro_overlapped"] = True
metadata["is_pro"] = metadata["pro_status"] == "active" and not metadata["pro_overlapped"]
metadata["is_pro"] = metadata["pro_status"] == "active"
metadata = metadata or default_metadata
db.set_pro_metadata(metadata | {"last_pro_check": current_date})
metadata = default_metadata | metadata
db.set_pro_metadata(metadata)
if metadata["is_pro"] != db_metadata["is_pro"]:
clean_pro_plugins(db)
if metadata["is_pro"]:
logger.info("🚀 Your BunkerWeb Pro license is valid, checking if there are new or updated Pro plugins...")
LOGGER.info("🚀 Your BunkerWeb Pro license is valid, checking if there are new or updated Pro plugins...")
if not db_metadata["is_pro"]:
clean_pro_plugins(db)
resp = get(f"{API_ENDPOINT}/pro", headers=headers, json=data, timeout=5, allow_redirects=True)
resp = get(f"{API_ENDPOINT}/pro/download", headers=headers, json=data, timeout=5, allow_redirects=True)
if resp.status_code == 403:
logger.error(f"Access denied to {API_ENDPOINT}/pro - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/")
LOGGER.error(f"Access denied to {API_ENDPOINT}/pro - please check your BunkerWeb Pro access at https://panel.bunkerweb.io/")
error = True
metadata = default_metadata
db.set_pro_metadata(metadata | {"last_pro_check": current_date})
clean_pro_plugins(db)
if resp.headers.get("Content-Type", "") == "application/json":
resp_data = resp.json()
if resp_data.get("action") == "clean":
metadata = default_metadata.copy()
db.set_pro_metadata(metadata)
clean_pro_plugins(db)
elif resp.headers.get("Content-Type", "") != "application/octet-stream":
logger.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {API_ENDPOINT}/pro")
LOGGER.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {API_ENDPOINT}/pro")
status = 2
sys_exit(status)
@ -192,26 +191,23 @@ try:
STATUS_MESSAGES.get(metadata["pro_status"], "is not valid or has expired") if not error else "is not valid or has expired"
)
else:
logger.info("If you wish to purchase a BunkerWeb Pro license, please visit https://panel.bunkerweb.io/")
LOGGER.info("If you wish to purchase a BunkerWeb Pro license, please visit https://panel.bunkerweb.io/")
message = "No BunkerWeb Pro license key provided"
logger.warning(f"{message}, only checking if there are new or updated preview versions of Pro plugins...")
if metadata["is_pro"]:
clean_pro_plugins(db)
LOGGER.warning(f"{message}, only checking if there are new or updated preview versions of Pro plugins...")
resp = get(f"{PREVIEW_ENDPOINT}/v{data['version']}.zip", timeout=5, allow_redirects=True)
if resp.status_code == 404:
logger.error(f"Couldn't find Pro plugins for BunkerWeb version {data['version']} at {PREVIEW_ENDPOINT}/v{data['version']}.zip")
LOGGER.error(f"Couldn't find Pro plugins for BunkerWeb version {data['version']} at {PREVIEW_ENDPOINT}/v{data['version']}.zip")
status = 2
sys_exit(status)
elif resp.headers.get("Content-Type", "") != "application/zip":
logger.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {PREVIEW_ENDPOINT}/v{data['version']}.zip")
LOGGER.error(f"Got unexpected content type: {resp.headers.get('Content-Type', 'missing')} from {PREVIEW_ENDPOINT}/v{data['version']}.zip")
status = 2
sys_exit(status)
if resp.status_code == 500:
logger.error("An error occurred with the remote server, please try again later")
LOGGER.error("An error occurred with the remote server, please try again later")
status = 2
sys_exit(status)
resp.raise_for_status()
@ -224,19 +220,19 @@ try:
# Install plugins
try:
for plugin_dir in glob(temp_dir.joinpath(data["version"] if metadata["is_pro"] else "", "*").as_posix()):
for plugin_path in temp_dir.glob("*"):
try:
if install_plugin(plugin_dir, db, not metadata["is_pro"]):
if install_plugin(plugin_path, db, not metadata["is_pro"]):
plugin_nbr += 1
except FileExistsError:
logger.warning(f"Skipping installation of pro plugin {basename(plugin_dir)} (already installed)")
LOGGER.warning(f"Skipping installation of pro plugin {plugin_path.name} (already installed)")
except:
logger.exception("Exception while installing pro plugin(s)")
LOGGER.exception("Exception while installing pro plugin(s)")
status = 2
sys_exit(status)
if not plugin_nbr:
logger.info("All Pro plugins are up to date")
LOGGER.info("All Pro plugins are up to date")
sys_exit(0)
pro_plugins = []
@ -244,7 +240,7 @@ try:
for plugin in listdir(PRO_PLUGINS_DIR):
path = PRO_PLUGINS_DIR.joinpath(plugin)
if not path.joinpath("plugin.json").is_file():
logger.warning(f"Plugin {plugin} is not valid, deleting it...")
LOGGER.warning(f"Plugin {plugin} is not valid, deleting it...")
rmtree(path, ignore_errors=True)
continue
@ -282,15 +278,17 @@ try:
err = db.update_external_plugins(pro_plugins, _type="pro")
if err:
logger.error(f"Couldn't update Pro plugins to database: {err}")
LOGGER.error(f"Couldn't update Pro plugins to database: {err}")
sys_exit(2)
db.set_pro_metadata(metadata | {"last_pro_check": current_date})
status = 1
logger.info("🚀 Pro plugins downloaded and installed successfully!")
LOGGER.info("🚀 Pro plugins downloaded and installed successfully!")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running download-pro-plugins.py :\n{format_exc()}")
LOGGER.error(f"Exception while running download-pro-plugins.py :\n{format_exc()}")
for plugin_tmp in glob(TMP_DIR.joinpath("*").as_posix()):
rmtree(plugin_tmp, ignore_errors=True)

2
src/common/core/pro/plugin.json
View file

@ -19,7 +19,7 @@
{
"name": "download-pro-plugins",
"file": "download-pro-plugins.py",
"every": "hour",
"every": "day",
"reload": true
}
]

89
src/common/core/realip/jobs/realip-download.py
View file

@ -2,43 +2,35 @@
from contextlib import suppress
from ipaddress import ip_address, ip_network
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join, normpath
from pathlib import Path
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from requests import get
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, del_file_in_db, file_hash, is_cached_file
from common_utils import bytes_hash # type: ignore
from jobs import Job # type: ignore
def check_line(line):
if "/" in line:
with suppress(ValueError):
with suppress(ValueError):
if "/" in line:
ip_network(line)
return True, line
else:
with suppress(ValueError):
else:
ip_address(line)
return True, line
return False, b""
logger = setup_logger("REALIP", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("REALIP", getenv("LOG_LEVEL", "INFO"))
REALIP_CACHE_PATH = join(sep, "var", "cache", "bunkerweb", "realip")
status = 0
try:
@ -61,37 +53,34 @@ try:
realip_activated = True
if not realip_activated:
logger.info("RealIP is not activated, skipping download...")
_exit(0)
LOGGER.info("RealIP is not activated, skipping download...")
sys_exit(0)
# Create directories if they don't exist
realip_path = Path(sep, "var", "cache", "bunkerweb", "realip")
realip_path.mkdir(parents=True, exist_ok=True)
tmp_realip_path = Path(sep, "var", "tmp", "bunkerweb", "realip")
tmp_realip_path.mkdir(parents=True, exist_ok=True)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
JOB = Job(LOGGER)
# Get URLs
urls = [url for url in getenv("REAL_IP_FROM_URLS", "").split(" ") if url]
# Don't go further if the cache is fresh
if is_cached_file(realip_path.joinpath("combined.list"), "hour", db):
logger.info("RealIP list is already in cache, skipping download...")
if JOB.is_cached_file("combined.list", "hour"):
LOGGER.info("RealIP list is already in cache, skipping download...")
if not urls:
logger.warning("No URL found, deleting combined.list from cache...")
tmp_realip_path.joinpath("combined.list").unlink(missing_ok=True)
deleted, err = del_file_in_db("combined.list", db)
LOGGER.warning("No URL found, deleting combined.list from cache...")
deleted, err = JOB.del_cache("combined.list")
if not deleted:
logger.warning(f"Couldn't delete combined.list from cache : {err}")
_exit(0)
LOGGER.warning(f"Couldn't delete combined.list from cache : {err}")
sys_exit(0)
if not urls:
LOGGER.info("No URL found, skipping download...")
sys_exit(0)
# Download and write data to temp file
i = 0
content = b""
for url in urls:
try:
logger.info(f"Downloading RealIP list from {url} ...")
LOGGER.info(f"Downloading RealIP list from {url} ...")
if url.startswith("file://"):
with open(normpath(url[7:]), "rb") as f:
iterable = f.readlines()
@ -99,7 +88,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -116,34 +105,28 @@ try:
i += 1
except:
status = 2
logger.error(f"Exception while getting RealIP list from {url} :\n{format_exc()}")
tmp_realip_path.joinpath("combined.list").write_bytes(content)
LOGGER.error(f"Exception while getting RealIP list from {url} :\n{format_exc()}")
# Check if file has changed
new_hash = file_hash(tmp_realip_path.joinpath("combined.list"))
old_hash = cache_hash(realip_path.joinpath("combined.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash("combined.list")
if new_hash == old_hash:
logger.info("New file is identical to cache file, reload is not needed")
_exit(0)
LOGGER.info("New file is identical to cache file, reload is not needed")
sys_exit(0)
# Put file in cache
cached, err = cache_file(
tmp_realip_path.joinpath("combined.list"),
realip_path.joinpath("combined.list"),
new_hash,
db,
)
cached, err = JOB.cache_file("combined.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching list : {err}")
_exit(2)
LOGGER.error(f"Error while caching list : {err}")
sys_exit(2)
logger.info(f"Downloaded {i} trusted IP/net")
LOGGER.info(f"Downloaded {i} trusted IP/net")
status = 1
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running realip-download.py :\n{format_exc()}")
LOGGER.error(f"Exception while running realip-download.py :\n{format_exc()}")
sys_exit(status)

25
src/common/core/redis/ui/actions.py
View file

@ -1,20 +1,29 @@
def redis(**kwargs):
def pre_render(**kwargs):
ping = {}
data = {}
try:
ping_data = kwargs["app"].config["INSTANCES"].get_ping("redis")
ping = {"ping_status": ping_data["status"]}
ping = {"ping_status": {"title": "REDIS STATUS", "value": ping_data["status"]}}
except:
ping = {"ping_status": "error"}
ping = {"ping_status": {"title": "REDIS STATUS", "value": "error"}}
try:
metrics = kwargs["app"].config["INSTANCES"].get_metrics("redis")
data = {
"counter_redis_nb_keys": {
"value": metrics.get("redis_nb_keys", 0),
"title": "REDIS KEYS",
"subtitle": "total number",
"subtitle_color": "info",
"svg_color": "sky",
}
}
if metrics.get("redis_nb_keys") is None:
metrics["redis_nb_keys"] = 0
data = metrics
except:
data = {"redis_nb_keys": 0}
data = {"counter_redis_nb_keys": {"value": "unknown", "title": "REDIS KEYS", "subtitle": "total number", "subtitle_color": "info", "svg_color": "sky"}}
return ping | data
def redis(**kwargs):
pass

165
src/common/core/redis/ui/template.html
View file

@ -7,72 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">Keys</p>
<h5 data-count class="core-card-title">"unknown"</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content info">total number</span>
</p>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container sky">
<svg xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
class="scale-[0.6] leading-none text-lg relative-sky-700 stroke-white">
<path stroke-linecap="round" stroke-linejoin="round" d="M3.75 12h16.5m-16.5 3.75h16.5M3.75 19.5h16.5M5.625 4.5h12.75a1.875 1.875 0 0 1 0 3.75H5.625a1.875 1.875 0 0 1 0-3.75Z" />
</svg>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
<!-- end icon -->
</div>
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">STATUS</h5>
<svg data-status-svg
class="core-card-status-svg info"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
<p data-status-text class="core-card-text"></p>
</div>
<!-- end status -->
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
redis_nb_keys: {
el: document.querySelector("[data-count]"),
value: "unknown",
type: "text",
},
// value : active / inactive / unknown
ping_status: {
el: document.querySelector("[data-status-svg]"),
value: "unknown",
type: "status",
textEl: document.querySelector("[data-status-text]"),
},
});
</script>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -91,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

10
src/common/core/reversescan/ui/actions.py
View file

@ -1,13 +1,17 @@
from operator import itemgetter
def reversescan(**kwargs):
def pre_render(**kwargs):
try:
# Here we will have a list { 'counter_403': X, 'counter_401': Y ... }
data = kwargs["app"].config["INSTANCES"].get_metrics("reversescan")
# Format to fit [{code: 403, count: X}, {code: 401, count: Y} ...]
format_data = [{"port": int(key.split("_")[1]), "count": int(value)} for key, value in data.items()]
format_data.sort(key=itemgetter("count"), reverse=True)
return {"items": format_data}
return {"top_reverse_scan": format_data}
except:
return {"items": []}
return {"top_reverse_scan": []}
def reversescan(**kwargs):
pass

148
src/common/core/reversescan/ui/template.html
View file

@ -7,53 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div data-fetch-success-show class="hidden core-card-list w-large">
<div class="core-card-list-container">
<h5 class="core-card-list-title">REVERSE SCAN LIST</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap w-large">
<!-- header-->
<p class="core-card-list-header col-span-5">Port</p>
<p class="core-card-list-header col-span-7">Block count</p>
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
<li data-item class="core-card-list-item">
<p data-name="port" class="core-card-list-item-content col-span-5"></p>
<p data-name="count" class="core-card-list-item-content col-span-7"></p>
</li>
</ul>
<!-- end list-->
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<!-- end list container-->
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
items: {
el: document.querySelector("[data-item]"),
value: [],
type: "list",
listNames: ["port", "count"],
},
});
</script>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -72,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

155
src/common/core/selfsigned/jobs/self-signed.py
View file

@ -1,74 +1,61 @@
#!/usr/bin/env python3
from datetime import timedelta
from datetime import datetime, timedelta
from os import getenv, sep
from os.path import join
from pathlib import Path
from subprocess import DEVNULL, STDOUT, run
from subprocess import DEVNULL, run
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from traceback import format_exc
from typing import Tuple
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (
("deps", "python"),
("utils",),
("db",),
)
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import set_file_in_db
from jobs import Job # type: ignore
logger = setup_logger("self-signed", getenv("LOG_LEVEL", "INFO"))
db = None
lock = Lock()
LOGGER = setup_logger("self-signed", getenv("LOG_LEVEL", "INFO"))
JOB = Job(LOGGER)
status = 0
def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Path) -> Tuple[bool, int]:
if self_signed_path.joinpath(f"{first_server}.pem").is_file():
server_path = self_signed_path.joinpath(first_server)
cert_path = server_path.joinpath("cert.pem")
key_path = server_path.joinpath("key.pem")
if cert_path.is_file() and key_path.is_file():
if (
run(
[
"openssl",
"x509",
"-checkend",
"86400",
"-noout",
"-in",
str(self_signed_path.joinpath(f"{first_server}.pem")),
],
["openssl", "x509", "-checkend", "86400", "-noout", "-in", cert_path.as_posix()],
stdin=DEVNULL,
stderr=STDOUT,
stderr=DEVNULL,
check=False,
).returncode
== 0
):
logger.info(f"Self-signed certificate already present for {first_server}")
LOGGER.info(f"Self-signed certificate already present for {first_server}")
certificate = x509.load_pem_x509_certificate(
self_signed_path.joinpath(f"{first_server}.pem").read_bytes(),
default_backend(),
)
certificate = x509.load_pem_x509_certificate(JOB.get_cache("cert.pem", service_id=first_server), default_backend())
if sorted(attribute.rfc4514_string() for attribute in certificate.subject) != sorted(v for v in subj.split("/") if v):
logger.warning(f"Subject of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ...")
elif certificate.not_valid_after - certificate.not_valid_before != timedelta(days=int(days)):
logger.warning(
LOGGER.warning(f"Subject of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ...")
elif certificate.not_valid_after_utc - certificate.not_valid_before_utc != timedelta(days=int(days)):
LOGGER.warning(
f"Expiration date of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ..."
)
elif certificate.not_valid_after_utc < datetime.now(tz=certificate.not_valid_after_utc.timetz().tzinfo):
LOGGER.warning(f"Self-signed certificate for {first_server} has expired, regenerating ...")
else:
LOGGER.info(f"Self-signed certificate for {first_server} is valid")
return True, 0
logger.info(f"Generating self-signed certificate for {first_server}")
LOGGER.info(f"Generating self-signed certificate for {first_server}")
server_path.mkdir(parents=True, exist_ok=True)
if (
run(
[
@ -77,11 +64,13 @@ def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Pat
"-nodes",
"-x509",
"-newkey",
"ed25519",
"ec",
"-pkeyopt",
"ec_paramgen_curve:prime256v1",
"-keyout",
str(self_signed_path.joinpath(f"{first_server}.key")),
key_path.as_posix(),
"-out",
str(self_signed_path.joinpath(f"{first_server}.pem")),
cert_path.as_posix(),
"-days",
days,
"-subj",
@ -93,29 +82,19 @@ def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Pat
).returncode
!= 0
):
logger.error(f"Self-signed certificate generation failed for {first_server}")
LOGGER.error(f"Self-signed certificate generation failed for {first_server}")
return False, 2
# Update db
cached, err = set_file_in_db(
f"{first_server}.pem",
self_signed_path.joinpath(f"{first_server}.pem").read_bytes(),
db,
service_id=first_server,
)
cached, err = JOB.cache_file("cert.pem", self_signed_path.joinpath(first_server, "cert.pem"), service_id=first_server, overwrite_file=False)
if not cached:
logger.error(f"Error while caching self-signed {first_server}.pem file : {err}")
LOGGER.error(f"Error while caching self-signed cert.pem file for {first_server} : {err}")
cached, err = set_file_in_db(
f"{first_server}.key",
self_signed_path.joinpath(f"{first_server}.key").read_bytes(),
db,
service_id=first_server,
)
cached, err = JOB.cache_file("key.pem", self_signed_path.joinpath(first_server, "key.pem"), service_id=first_server, overwrite_file=False)
if not cached:
logger.error(f"Error while caching self-signed {first_server}.key file : {err}")
LOGGER.error(f"Error while caching self-signed {first_server}.key file : {err}")
logger.info(f"Successfully generated self-signed certificate for {first_server}")
LOGGER.info(f"Successfully generated self-signed certificate for {first_server}")
return True, 1
@ -123,58 +102,46 @@ status = 0
try:
self_signed_path = Path(sep, "var", "cache", "bunkerweb", "selfsigned")
self_signed_path.mkdir(parents=True, exist_ok=True)
servers = getenv("SERVER_NAME") or []
# Multisite case
if getenv("MULTISITE") == "yes":
servers = getenv("SERVER_NAME") or []
if isinstance(servers, str):
servers = servers.split(" ")
if isinstance(servers, str):
servers = servers.split(" ")
if not servers:
LOGGER.info("No server found, skipping self-signed certificate generation ...")
sys_exit(0)
skipped_servers = []
if getenv("MULTISITE", "no") == "no":
servers = [servers[0]]
if getenv("GENERATE_SELF_SIGNED_SSL", "no") == "no":
LOGGER.info("Generate self-signed SSL is not enabled, skipping certificate generation ...")
skipped_servers = servers
if not skipped_servers:
for first_server in servers:
if (
not first_server
or getenv(
f"{first_server}_GENERATE_SELF_SIGNED_SSL",
getenv("GENERATE_SELF_SIGNED_SSL", "no"),
)
!= "yes"
or self_signed_path.joinpath(f"{first_server}.pem").is_file()
):
if getenv(f"{first_server}_GENERATE_SELF_SIGNED_SSL", getenv("GENERATE_SELF_SIGNED_SSL", "no")) != "yes":
LOGGER.info(f"Self-signed SSL is not enabled for {first_server}, skipping certificate generation ...")
skipped_servers.append(first_server)
continue
if not db:
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
ret, ret_status = generate_cert(
first_server,
getenv(
f"{first_server}_SELF_SIGNED_SSL_EXPIRY",
getenv("SELF_SIGNED_SSL_EXPIRY", "365"),
),
getenv(
f"{first_server}_SELF_SIGNED_SSL_SUBJ",
getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/"),
),
getenv(f"{first_server}_SELF_SIGNED_SSL_EXPIRY", getenv("SELF_SIGNED_SSL_EXPIRY", "365")),
getenv(f"{first_server}_SELF_SIGNED_SSL_SUBJ", getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/")),
self_signed_path,
)
if not ret:
skipped_servers.append(first_server)
status = ret_status
# Singlesite case
elif getenv("GENERATE_SELF_SIGNED_SSL", "no") == "yes" and getenv("SERVER_NAME"):
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
first_server = getenv("SERVER_NAME", "").split(" ")[0]
ret, ret_status = generate_cert(
first_server,
getenv("SELF_SIGNED_SSL_EXPIRY", "365"),
getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/"),
self_signed_path,
)
status = ret_status
for first_server in skipped_servers:
JOB.del_cache("cert.pem", service_id=first_server)
JOB.del_cache("key.pem", service_id=first_server)
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running self-signed.py :\n{format_exc()}")
LOGGER.error(f"Exception while running self-signed.py :\n{format_exc()}")
sys_exit(status)

8
src/common/core/selfsigned/selfsigned.lua
View file

@ -44,8 +44,8 @@ function selfsigned:init()
for server_name, multisite_vars in pairs(vars) do
if multisite_vars["GENERATE_SELF_SIGNED_SSL"] == "yes" and server_name ~= "global" then
local check, data = read_files({
"/var/cache/bunkerweb/selfsigned/" .. server_name .. ".pem",
"/var/cache/bunkerweb/selfsigned/" .. server_name .. ".key",
"/var/cache/bunkerweb/selfsigned/" .. server_name .. "/cert.pem",
"/var/cache/bunkerweb/selfsigned/" .. server_name .. "/key.pem",
})
if not check then
self.logger:log(ERR, "error while reading files : " .. data)
@ -68,8 +68,8 @@ function selfsigned:init()
return self:ret(false, "can't get SERVER_NAME variable : " .. err)
end
local check, data = read_files({
"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. ".pem",
"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. ".key",
"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. "/cert.pem",
"/var/cache/bunkerweb/selfsigned/" .. server_name:match("%S+") .. "/key.pem",
})
if not check then
self.logger:log(ERR, "error while reading files : " .. data)

5
src/common/core/ui/confs/modsec/ui.conf
View file

@ -1,5 +1,4 @@
{% if USE_UI == "yes" +%}
SecRule REQUEST_FILENAME "@rx /services$" "id:7771,ctl:ruleRemoveByTag=attack-rce,ctl:ruleRemoveByTag=attack-xss,ctl:ruleRemoveByTag=attack-generic,ctl:ruleRemoveByTag=attack-lfi,ctl:ruleRemoveByTag=attack-rfi,ctl:ruleRemoveByTag=attack-ssrf,nolog"
SecRule REQUEST_FILENAME "@rx /global_config$" "id:7772,ctl:ruleRemoveByTag=platform-pgsql,ctl:ruleRemoveByTag=attack-lfi,ctl:ruleRemoveByTag=attack-rfi,ctl:ruleRemoveByTag=attack-ssrf,nolog"
SecRule REQUEST_FILENAME "@rx /configs$" "id:7773,ctl:ruleRemoveByTag=language-shell,ctl:ruleRemoveByTag=attack-lfi,ctl:ruleRemoveByTag=attack-rfi,ctl:ruleRemoveByTag=attack-ssrf,nolog"
SecRule REQUEST_FILENAME "@rx /(global_config|services)$" "id:7771,ctl:ruleRemoveByTag=language-shell,ctl:ruleRemoveByTag=platform-pgsql,ctl:ruleRemoveByTag=attack-xss,ctl:ruleRemoveByTag=attack-lfi,ctl:ruleRemoveByTag=attack-rfi,ctl:ruleRemoveByTag=attack-ssrf,nolog"
SecRule REQUEST_FILENAME "@rx /configs$" "id:7772,ctl:ruleRemoveByTag=language-shell,ctl:ruleRemoveByTag=attack-lfi,ctl:ruleRemoveByTag=attack-rfi,ctl:ruleRemoveByTag=attack-ssrf,nolog"
{% endif +%}

112
src/common/core/whitelist/jobs/whitelist-download.py
View file

@ -2,10 +2,9 @@
from contextlib import suppress
from ipaddress import ip_address, ip_network
from os import _exit, getenv, sep
from os import getenv, sep
from os.path import join, normpath
from pathlib import Path
from re import IGNORECASE, compile as re_compile
from re import compile as re_compile
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from typing import Tuple
@ -16,11 +15,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from requests import get
from Database import Database # type: ignore
from common_utils import bytes_hash # type: ignore
from logger import setup_logger # type: ignore
from jobs import cache_file, cache_hash, del_file_in_db, is_cached_file, file_hash
from jobs import Job # type: ignore
rdns_rx = re_compile(rb"^[^ ]+$", IGNORECASE)
rdns_rx = re_compile(rb"^[^ ]+$")
asn_rx = re_compile(rb"^\d+$")
uri_rx = re_compile(rb"^/")
@ -51,7 +50,7 @@ def check_line(kind: str, line: bytes) -> Tuple[bool, bytes]:
return False, b""
logger = setup_logger("WHITELIST", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("WHITELIST", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
@ -68,16 +67,10 @@ try:
whitelist_activated = True
if not whitelist_activated:
logger.info("Whitelist is not activated, skipping downloads...")
_exit(0)
LOGGER.info("Whitelist is not activated, skipping downloads...")
sys_exit(0)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
# Create directories if they don't exist
whitelist_path = Path(sep, "var", "cache", "bunkerweb", "whitelist")
whitelist_path.mkdir(parents=True, exist_ok=True)
tmp_whitelist_path = Path(sep, "var", "tmp", "bunkerweb", "whitelist")
tmp_whitelist_path.mkdir(parents=True, exist_ok=True)
JOB = Job(LOGGER)
# Get URLs
urls = {"IP": [], "RDNS": [], "ASN": [], "USER_AGENT": [], "URI": []}
@ -87,44 +80,36 @@ try:
urls[kind].append(url)
# Don't go further if the cache is fresh
kinds_fresh = {
"IP": True,
"RDNS": True,
"ASN": True,
"USER_AGENT": True,
"URI": True,
}
all_fresh = True
kinds_fresh = {"IP": True, "RDNS": True, "ASN": True, "USER_AGENT": True, "URI": True}
for kind in kinds_fresh:
if not is_cached_file(whitelist_path.joinpath(f"{kind}.list"), "hour", db):
kinds_fresh[kind] = False
all_fresh = False
logger.info(
f"Whitelist for {kind} is not cached, processing downloads..",
)
else:
logger.info(
f"Whitelist for {kind} is already in cache, skipping downloads...",
)
if not urls[kind]:
logger.info(
f"Whitelist for {kind} is already in cache, skipping downloads...",
)
whitelist_path.joinpath(f"{kind}.list").unlink(missing_ok=True)
deleted, err = del_file_in_db(f"{kind}.list", db)
if not deleted:
logger.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all_fresh:
_exit(0)
if not JOB.is_cached_file(f"{kind}.list", "hour"):
if urls[kind]:
kinds_fresh[kind] = False
LOGGER.info(f"Whitelist for {kind} is not cached, processing downloads..")
continue
LOGGER.info(f"Whitelist for {kind} is already in cache, skipping downloads...")
if not urls[kind]:
LOGGER.warning(f"Whitelist for {kind} is cached but no URL is configured, removing from cache...")
deleted, err = JOB.del_cache(f"{kind}.list")
if not deleted:
LOGGER.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all(kinds_fresh.values()):
if not any(urls.values()):
LOGGER.info("No whitelist URL is configured, nothing to do...")
sys_exit(0)
# Loop on kinds
for kind, urls_list in urls.items():
if kinds_fresh[kind]:
continue
# Write combined data of the kind to a single temp file
# Write combined data of the kind in memory and check if it has changed
for url in urls_list:
try:
logger.info(f"Downloading whitelist data from {url} ...")
LOGGER.info(f"Downloading whitelist data from {url} ...")
if url.startswith("file://"):
with open(normpath(url[7:]), "rb") as f:
iterable = f.readlines()
@ -132,7 +117,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(f"Got status code {resp.status_code}, skipping...")
LOGGER.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -152,39 +137,28 @@ try:
content += data + b"\n"
i += 1
tmp_whitelist_path.joinpath(f"{kind}.list").write_bytes(content)
logger.info(f"Downloaded {i} bad {kind}")
LOGGER.info(f"Downloaded {i} bad {kind}")
# Check if file has changed
new_hash = file_hash(tmp_whitelist_path.joinpath(f"{kind}.list"))
old_hash = cache_hash(whitelist_path.joinpath(f"{kind}.list"), db)
new_hash = bytes_hash(content)
old_hash = JOB.cache_hash(f"{kind}.list")
if new_hash == old_hash:
logger.info(
f"New file {kind}.list is identical to cache file, reload is not needed",
)
LOGGER.info(f"New file {kind}.list is identical to cache file, reload is not needed")
else:
logger.info(
f"New file {kind}.list is different than cache file, reload is needed",
)
LOGGER.info(f"New file {kind}.list is different than cache file, reload is needed")
# Put file in cache
cached, err = cache_file(
tmp_whitelist_path.joinpath(f"{kind}.list"),
whitelist_path.joinpath(f"{kind}.list"),
new_hash,
db,
)
cached, err = JOB.cache_file(f"{kind}.list", content, checksum=new_hash)
if not cached:
logger.error(f"Error while caching whitelist : {err}")
LOGGER.error(f"Error while caching whitelist : {err}")
status = 2
else:
status = 1
except:
status = 2
logger.error(f"Exception while getting whitelist from {url} :\n{format_exc()}")
LOGGER.error(f"Exception while getting whitelist from {url} :\n{format_exc()}")
except SystemExit as e:
status = e.code
except:
status = 2
logger.error(f"Exception while running whitelist-download.py :\n{format_exc()}")
LOGGER.error(f"Exception while running whitelist-download.py :\n{format_exc()}")
sys_exit(status)

30
src/common/core/whitelist/ui/actions.py
View file

@ -1,11 +1,27 @@
def whitelist(**kwargs):
def pre_render(**kwargs):
try:
data = kwargs["app"].config["INSTANCES"].get_metrics("whitelist")
if "counter_passed_whitelist" not in data:
data["counter_passed_whitelist"] = 0
return data
return {
"counter_passed_whitelist": {
"value": data.get("counter_passed_whitelist", 0),
"title": "WHITELIST",
"subtitle": "request passed",
"subtitle_color": "success",
"svg_color": "green",
}
}
except:
return {"counter_passed_whitelist": 0}
return {
"counter_passed_whitelist": {
"value": "unknown",
"title": "WHITELIST",
"subtitle": "request passed",
"subtitle_color": "success",
"svg_color": "green",
}
}
def whitelist(**kwargs):
pass

145
src/common/core/whitelist/ui/template.html
View file

@ -7,50 +7,111 @@
hidden />
<div class="core-layout">
{% if is_used and is_metrics %}
<!-- info-->
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text"></p>
</div>
<div class="core-card">
<h5 class="core-card-title">INFO</h5>
<div class="core-card-text-container">
<p data-info class="core-card-text">{{plugin.get('description')}}</p>
</div>
<!-- end info -->
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">WHITELIST</p>
<h5 data-count class="core-card-title"></h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content text-green-500 mx-0.5">request passed</span>
</p>
</div>
<!-- end info --> <div class="core-layout-separator"></div>
{% if pre_render["status"] and pre_render["status"] == "ko" or "error" in pre_render["data"] %}
<div class="flex justify-center col-span-12">
<p class="text-white">Error during pre rendering</p>
<div class="ml-2">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 stroke-red-500 fill-white">
<path stroke-linecap="round" stroke-linejoin="round" d="m9.75 9.75 4.5 4.5m0-4.5-4.5 4.5M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
</svg>
</div>
</div>
{% endif %}
{% if pre_render["status"] and pre_render["status"] == "ok" and "error" not in pre_render["data"] %}
{% for key, value in pre_render["data"].items() %}
{% if key.startswith("ping_") %}
<div class="core-card-status">
<div class="core-card-status-container">
<h5 class="core-card-status-title">{{ pre_render['data'][key].get('title', 'STATUS')}}</h5>
<svg data-status-svg
class="core-card-status-svg {{ 'fill-green-500' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'fill-red-500' }}"
viewBox="0 0 100 100"
xmlns="http://www.w3.org/2000/svg">
<circle cx="50" cy="50" r="50" />
</svg>
</div>
<p data-status-text class="core-card-text">{{ 'Active' if pre_render['data'][key].get('value') in ('up', 'yes', 'success', 'true') else 'Inactive' }}</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container green">
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-base core-card-metrics-svg">
<path fill-rule="evenodd" d="M2.25 12c0-5.385 4.365-9.75 9.75-9.75s9.75 4.365 9.75 9.75-4.365 9.75-9.75 9.75S2.25 17.385 2.25 12Zm13.36-1.814a.75.75 0 1 0-1.22-.872l-3.236 4.53L9.53 12.22a.75.75 0 0 0-1.06 1.06l2.25 2.25a.75.75 0 0 0 1.14-.094l3.75-5.25Z" clip-rule="evenodd" />
</svg>
{% endif %}
{% if key.startswith("count_") or key.startswith("counter_") %}
<div class="core-card-metrics">
<!-- text -->
<div>
<p class="core-card-metrics-name">{{pre_render['data'][key].get("title")}}</p>
<h5 data-count class="core-card-title">{{pre_render['data'][key].get("value")}}</h5>
<p class="core-card-metrics-subtitle">
<span class="core-card-metrics-subtitle-content {{pre_render['data'][key].get("subtitle_color", "info")}}">{{pre_render['data'][key].get("subtitle")}}</span>
</p>
</div>
<!-- end text -->
<!-- icon -->
<div role="img" class="core-card-svg-container {{pre_render['data'][key].get("svg_color")}}">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="size-small core-card-metrics-svg"
>
<path
d="M18.75 12.75h1.5a.75.75 0 0 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM12 6a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 6ZM12 18a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 12 18ZM3.75 6.75h1.5a.75.75 0 1 0 0-1.5h-1.5a.75.75 0 0 0 0 1.5ZM5.25 18.75h-1.5a.75.75 0 0 1 0-1.5h1.5a.75.75 0 0 1 0 1.5ZM3 12a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 3 12ZM9 3.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5ZM12.75 12a2.25 2.25 0 1 1 4.5 0 2.25 2.25 0 0 1-4.5 0ZM9 15.75a2.25 2.25 0 1 0 0 4.5 2.25 2.25 0 0 0 0-4.5Z"
/>
</svg>
</div>
<!-- end icon -->
</div>
<!-- end icon -->
</div>
<script nonce="{{script_nonce}}">
// Use SetupPlugin class that is on static/js/plugins/setup.js
const setPlugin = new SetupPlugin({
info: {
el: document.querySelector("[data-info]"),
value: "{{ plugin['description'] or ''}}",
type: "text",
},
counter_passed_whitelist: {
el: document.querySelector("[data-count]"),
value: "unknown",
type: "text",
},
});
</script>
{% endif %}
{% if (key.startswith("top_") and pre_render['data'][key]|length > 0) or (key.startswith("list_") and pre_render['data'][key]|length > 0) %}
<div class="core-card-list">
<div class="core-card-list-title-container">
<h5 class="core-card-list-title">{{ key.replace('_', ' ').upper()}}</h5>
</div>
<div class="core-card-list-container">
<!-- list container-->
<div class="core-card-list-wrap">
<!-- header-->
{% for val_key, val_value in pre_render['data'][key][0].items() %}
<p class="core-card-list-header {{'col-span-6' if pre_render['data'][key][0].keys()|length == 2 else "col-span-4" if pre_render['data'][key][0].keys()|length == 3 else "col-span-3" if pre_render['data'][key][0].keys()|length == 4}}">{{ val_key }}</p>
{% endfor%}
<!-- end header-->
<!-- list -->
<ul class="col-span-12 w-full">
{% for item in pre_render['data'][key] %}
<li class="core-card-list-item">
{% for top_key, top_value in item.items() %}
<p class="core-card-list-item-content {{'col-span-6' if item.keys()|length == 2 else "col-span-4" if item.keys()|length == 3 else "col-span-3" if item.keys()|length == 4}}">{{ top_value }}</p>
{% endfor %}
</li>
{% endfor %}
</ul>
<!-- end list-->
</div>
<!-- end list container-->
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% else %}
<div class="core-card">
<div class="core-card-wrap">
@ -69,7 +130,7 @@
<!-- end icon -->
</div>
<div class="core-card-text-container">
<p data-info class="core-card-text">This plugin need to be activated to get metrics.</p>
<p data-info class="core-card-text">This plugin need to be activated to access page.</p>
</div>
</div>
<!-- end info -->

265
src/common/db/Database.py
View file

@ -36,10 +36,11 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
if deps_path not in sys_path:
sys_path.append(deps_path)
from jobs import file_hash # type: ignore
from common_utils import file_hash # type: ignore
from pymysql import install_as_MySQLdb
from sqlalchemy import create_engine, MetaData as sql_metadata, text, inspect
from sqlalchemy import create_engine, event, MetaData as sql_metadata, text, inspect
from sqlalchemy.engine import Engine
from sqlalchemy.exc import (
ArgumentError,
DatabaseError,
@ -48,24 +49,31 @@ from sqlalchemy.exc import (
SQLAlchemyError,
)
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.pool import SingletonThreadPool
from sqlalchemy.pool import QueuePool
from sqlite3 import Connection as SQLiteConnection
install_as_MySQLdb()
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, _):
if isinstance(dbapi_connection, SQLiteConnection):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.execute("PRAGMA journal_mode=WAL")
cursor.close()
class Database:
DB_STRING_RX = re_compile(r"^(?P<database>(mariadb|mysql)(\+pymysql)?|sqlite(\+pysqlite)?|postgresql(\+psycopg)?):/+(?P<path>/[^\s]+)")
def __init__(
self,
logger: Logger,
sqlalchemy_string: Optional[str] = None,
*,
ui: bool = False,
pool: bool = True,
) -> None:
def __init__(self, logger: Logger, sqlalchemy_string: Optional[str] = None, *, ui: bool = False, pool: Optional[bool] = None) -> None:
"""Initialize the database"""
self.__logger = logger
self.logger = logger
if pool:
self.logger.warning("The pool parameter is deprecated, it will be removed in the next version")
self.__session_factory = None
self.__sql_engine = None
@ -74,21 +82,21 @@ class Database:
match = self.DB_STRING_RX.search(sqlalchemy_string)
if not match:
self.__logger.error(f"Invalid database string provided: {sqlalchemy_string}, exiting...")
self.logger.error(f"Invalid database string provided: {sqlalchemy_string}, exiting...")
_exit(1)
if match.group("database").startswith("sqlite"):
db_path = Path(normpath(match.group("path")))
if ui:
while not db_path.is_file():
self.__logger.warning(f"Waiting for the database file to be created: {db_path}")
self.logger.warning(f"Waiting for the database file to be created: {db_path}")
sleep(1)
else:
db_path.parent.mkdir(parents=True, exist_ok=True)
elif match.group("database").startswith("m") and not match.group("database").endswith("+pymysql"):
sqlalchemy_string = sqlalchemy_string.replace(
match.group("database"), f"{match.group('database')}+pymysql"
) # ? This is mandatory for alemic to work with mariadb and mysql
) # ? This is strongly recommended as pymysql is the new way to connect to mariadb and mysql
elif match.group("database").startswith("postgresql") and not match.group("database").endswith("+psycopg"):
sqlalchemy_string = sqlalchemy_string.replace(
match.group("database"), f"{match.group('database')}+psycopg"
@ -97,15 +105,22 @@ class Database:
self.database_uri = sqlalchemy_string
error = False
engine_kwargs = {"future": True, "poolclass": None if pool else SingletonThreadPool, "pool_pre_ping": True, "pool_recycle": 1800}
engine_kwargs = {
"future": True,
"poolclass": QueuePool,
"pool_pre_ping": True,
"pool_recycle": 1800,
"pool_size": 20,
"max_overflow": 10,
}
try:
self.__sql_engine = create_engine(sqlalchemy_string, **engine_kwargs)
except ArgumentError:
self.__logger.error(f"Invalid database URI: {sqlalchemy_string}")
self.logger.error(f"Invalid database URI: {sqlalchemy_string}")
error = True
except SQLAlchemyError:
self.__logger.error(f"Error when trying to create the engine: {format_exc()}")
self.logger.error(f"Error when trying to create the engine: {format_exc()}")
error = True
finally:
if error:
@ -114,7 +129,7 @@ class Database:
try:
assert self.__sql_engine is not None
except AssertionError:
self.__logger.error("The database engine is not initialized")
self.logger.error("The database engine is not initialized")
_exit(1)
not_connected = True
@ -128,37 +143,30 @@ class Database:
not_connected = False
except (OperationalError, DatabaseError) as e:
if retries <= 0:
self.__logger.error(
self.logger.error(
f"Can't connect to database : {format_exc()}",
)
_exit(1)
if "attempt to write a readonly database" in str(e):
self.__logger.warning("The database is read-only, waiting for it to become writable. Retrying in 5 seconds ...")
self.logger.warning("The database is read-only, waiting for it to become writable. Retrying in 5 seconds ...")
self.__sql_engine.dispose(close=True)
self.__sql_engine = create_engine(sqlalchemy_string, **engine_kwargs)
if "Unknown table" in str(e):
not_connected = False
continue
else:
self.__logger.warning(
self.logger.warning(
"Can't connect to database, retrying in 5 seconds ...",
)
retries -= 1
sleep(5)
except BaseException:
self.__logger.error(f"Error when trying to connect to the database: {format_exc()}")
self.logger.error(f"Error when trying to connect to the database: {format_exc()}")
exit(1)
self.__logger.info("✅ Database connection established")
self.__session_factory = sessionmaker(bind=self.__sql_engine, autoflush=True, expire_on_commit=False)
self.suffix_rx = re_compile(r"_\d+$")
if sqlalchemy_string.startswith("sqlite"):
with self.__db_session() as session:
session.execute(text("PRAGMA journal_mode=WAL"))
session.commit()
self.logger.info("✅ Database connection established")
def __del__(self) -> None:
"""Close the database"""
@ -171,20 +179,21 @@ class Database:
@contextmanager
def __db_session(self):
try:
assert self.__session_factory is not None
assert self.__sql_engine is not None
except AssertionError:
self.__logger.error("The database session is not initialized")
self.logger.error("The database engine is not initialized")
_exit(1)
session = scoped_session(self.__session_factory)
try:
yield session
except BaseException:
session.rollback()
raise
finally:
session.remove()
with self.__sql_engine.connect() as conn:
session_factory = sessionmaker(bind=conn, autoflush=True, expire_on_commit=False)
session = scoped_session(session_factory)
try:
yield session
except BaseException:
session.rollback()
raise
finally:
session.remove()
def set_autoconf_load(self, value: bool = True) -> str:
"""Set the autoconf_loaded value"""
@ -227,7 +236,7 @@ class Database:
return ""
def set_pro_metadata(self, data: Dict[Literal["is_pro", "pro_expire", "pro_status", "pro_overlapped", "pro_services"], Any] = {}) -> str:
def set_pro_metadata(self, data: Dict[Literal["is_pro", "pro_license_key", "pro_expire", "pro_status", "pro_overlapped", "pro_services"], Any] = {}) -> str:
"""Set the pro metadata values"""
with self.__db_session() as session:
try:
@ -303,15 +312,17 @@ class Database:
"integration": "unknown",
"database_version": "Unknown",
"is_pro": "no",
"pro_license_key": "",
"pro_expire": None,
"pro_services": 0,
"pro_overlapped": False,
"pro_status": "invalid",
"last_pro_check": None,
"default": True,
}
database = self.database_uri.split(":")[0].split("+")[0]
with self.__db_session() as session:
with suppress(ProgrammingError, OperationalError):
try:
database = self.database_uri.split(":")[0].split("+")[0]
data["database_version"] = (
session.execute(text("SELECT sqlite_version()" if database == "sqlite" else "SELECT VERSION()")).first() or ["unknown"]
)[0]
@ -321,6 +332,7 @@ class Database:
Metadata.version,
Metadata.integration,
Metadata.is_pro,
Metadata.pro_license_key,
Metadata.pro_expire,
Metadata.pro_services,
Metadata.pro_overlapped,
@ -336,13 +348,17 @@ class Database:
"version": metadata.version,
"integration": metadata.integration,
"is_pro": metadata.is_pro,
"pro_license_key": metadata.pro_license_key or "",
"pro_expire": metadata.pro_expire,
"pro_services": metadata.pro_services,
"pro_overlapped": metadata.pro_overlapped,
"pro_status": metadata.pro_status,
"last_pro_check": metadata.last_pro_check,
"default": False,
}
)
except BaseException:
self.logger.debug(f"Can't get the metadata: {format_exc()}")
return data
@ -417,16 +433,19 @@ class Database:
old_data = {}
if inspector and len(inspector.get_table_names()):
db_version = self.get_metadata()["version"]
metadata = self.get_metadata()
db_version = metadata["version"]
if metadata["default"]:
db_version = "error"
if db_version != bunkerweb_version:
self.__logger.warning(f"Database version ({db_version}) is different from Bunkerweb version ({bunkerweb_version}), migrating ...")
self.logger.warning(f"Database version ({db_version}) is different from Bunkerweb version ({bunkerweb_version}), migrating ...")
metadata = sql_metadata()
metadata.reflect(self.__sql_engine)
for table_name in Base.metadata.tables.keys():
if not inspector.has_table(table_name):
self.__logger.warning(f'Table "{table_name}" is missing')
self.logger.warning(f'Table "{table_name}" is missing')
has_all_tables = False
continue
@ -519,7 +538,7 @@ class Database:
updates[Plugins.checksum] = plugin.get("checksum")
if updates:
self.__logger.warning(f'Plugin "{plugin["id"]}" already exists, updating it with the new values')
self.logger.warning(f'Plugin "{plugin["id"]}" already exists, updating it with the new values')
session.query(Plugins).filter(Plugins.id == plugin["id"]).update(updates)
else:
to_put.append(
@ -578,11 +597,11 @@ class Database:
updates[Settings.multiple] = value.get("multiple")
if updates:
self.__logger.warning(f'Setting "{setting}" already exists, updating it with the new values')
self.logger.warning(f'Setting "{setting}" already exists, updating it with the new values')
session.query(Settings).filter(Settings.id == setting).update(updates)
else:
if db_plugin:
self.__logger.warning(f'Setting "{setting}" does not exist, creating it')
self.logger.warning(f'Setting "{setting}" does not exist, creating it')
to_put.append(Settings(**value))
db_values = [select.value for select in session.query(Selects).with_entities(Selects.value).filter_by(setting_id=value["id"])]
@ -591,7 +610,7 @@ class Database:
if select_values:
if missing_values:
# Remove selects that are no longer in the list
self.__logger.warning(f'Removing {len(missing_values)} selects from setting "{setting}" as they are no longer in the list')
self.logger.warning(f'Removing {len(missing_values)} selects from setting "{setting}" as they are no longer in the list')
session.query(Selects).filter(Selects.value.in_(missing_values)).delete()
for select in select_values:
@ -599,7 +618,7 @@ class Database:
to_put.append(Selects(setting_id=value["id"], value=select))
else:
if missing_values:
self.__logger.warning(f'Removing all selects from setting "{setting}" as there are no longer any in the list')
self.logger.warning(f'Removing all selects from setting "{setting}" as there are no longer any in the list')
session.query(Selects).filter_by(setting_id=value["id"]).delete()
db_names = [job.name for job in session.query(Jobs).with_entities(Jobs.name).filter_by(plugin_id=plugin["id"])]
@ -608,8 +627,9 @@ class Database:
if missing_names:
# Remove jobs that are no longer in the list
self.__logger.warning(f'Removing {len(missing_names)} jobs from plugin "{plugin["id"]}" as they are no longer in the list')
session.query(Jobs).filter(Jobs.name.in_(missing_names)).delete()
self.logger.warning(f'Removing {len(missing_names)} jobs from plugin "{plugin["id"]}" as they are no longer in the list')
session.query(Jobs).filter(Jobs.name.in_(missing_names), Jobs.plugin_id == plugin["id"]).delete()
session.query(Jobs_cache).filter(Jobs_cache.job_name.in_(missing_names)).delete()
for job in jobs:
db_job = (
@ -623,7 +643,7 @@ class Database:
job["file_name"] = job.pop("file")
job["reload"] = job.get("reload", False)
if db_plugin:
self.__logger.warning(f'Job "{job["name"]}" does not exist, creating it')
self.logger.warning(f'Job "{job["name"]}" does not exist, creating it')
to_put.append(Jobs(plugin_id=plugin["id"], **job))
else:
updates = {}
@ -638,13 +658,14 @@ class Database:
updates[Jobs.reload] = job.get("reload", False)
if updates:
self.__logger.warning(f'Job "{job["name"]}" already exists, updating it with the new values')
self.logger.warning(f'Job "{job["name"]}" already exists, updating it with the new values')
updates[Jobs.last_run] = None
session.query(Jobs_cache).filter(Jobs_cache.job_name == job["name"]).delete()
session.query(Jobs).filter(Jobs.name == job["name"]).update(updates)
core_ui_path = Path(sep, "usr", "share", "bunkerweb", "core", plugin["id"], "ui")
path_ui = core_ui_path if core_ui_path.exists() else Path(sep, "etc", "bunkerweb", "plugins", plugin["id"], "ui")
path_ui = path_ui if path_ui.exists() else Path(sep, "etc", "bunkerweb", "pro", "plugins", plugin["id"], "ui")
if path_ui.exists():
if {"template.html", "actions.py"}.issubset(listdir(str(path_ui))):
@ -681,12 +702,12 @@ class Database:
)
if updates:
self.__logger.warning(f'Page for plugin "{plugin["id"]}" already exists, updating it with the new values')
self.logger.warning(f'Page for plugin "{plugin["id"]}" already exists, updating it with the new values')
session.query(Plugin_pages).filter(Plugin_pages.plugin_id == plugin["id"]).update(updates)
continue
if db_plugin:
self.__logger.warning(f'Page for plugin "{plugin["id"]}" does not exist, creating it')
self.logger.warning(f'Page for plugin "{plugin["id"]}" does not exist, creating it')
to_put.append(
Plugin_pages(
@ -731,7 +752,8 @@ class Database:
# Remove services that are no longer in the list
session.query(Services).filter(Services.id.in_(missing_ids)).delete()
session.query(Services_settings).filter(Services_settings.service_id.in_(missing_ids)).delete()
session.query(Global_values).filter(Global_values.setting_id.in_(missing_ids)).delete()
session.query(Custom_configs).filter(Custom_configs.service_id.in_(missing_ids)).delete()
session.query(Jobs_cache).filter(Jobs_cache.service_id.in_(missing_ids)).delete()
drafts = {service for service in services if config.pop(f"{service}_IS_DRAFT", "no") == "yes"}
db_drafts = {service.id for service in db_services if service.is_draft}
@ -1082,11 +1104,15 @@ class Database:
)
.filter_by(service_id=service.id, setting_id=key)
):
value = service_setting.value
if key == "SERVER_NAME" and service.id not in value.split(" "):
value = f"{service.id} {value}".strip()
config[f"{service.id}_{key}" + (f"_{service_setting.suffix}" if service_setting.suffix > 0 else "")] = (
service_setting.value
value
if not methods
else {
"value": service_setting.value,
"value": value,
"global": False,
"method": service_setting.method,
}
@ -1162,10 +1188,16 @@ class Database:
def delete_job_cache(self, file_name: str, *, job_name: Optional[str] = None, service_id: Optional[str] = None):
job_name = job_name or basename(getsourcefile(_getframe(1))).replace(".py", "")
with self.__db_session() as session:
session.query(Jobs_cache).filter_by(job_name=job_name, file_name=file_name, service_id=service_id).delete()
filters = {"file_name": file_name}
if job_name:
filters["job_name"] = job_name
if service_id:
filters["service_id"] = service_id
def update_job_cache(
with self.__db_session() as session:
session.query(Jobs_cache).filter_by(**filters).delete()
def upsert_job_cache(
self,
service_id: Optional[str],
file_name: str,
@ -1176,6 +1208,7 @@ class Database:
) -> str:
"""Update the plugin cache in the database"""
job_name = job_name or basename(getsourcefile(_getframe(1))).replace(".py", "")
service_id = service_id or None
with self.__db_session() as session:
cache = session.query(Jobs_cache).filter_by(job_name=job_name, service_id=service_id, file_name=file_name).first()
@ -1254,7 +1287,7 @@ class Database:
if db_plugin:
if db_plugin.type not in ("external", "pro"):
self.__logger.warning(
self.logger.warning(
f"Plugin \"{plugin['id']}\" is not {_type}, skipping update (updating a non-external or non-pro plugin is forbidden for security reasons)", # noqa: E501
)
continue
@ -1297,6 +1330,9 @@ class Database:
changes = True
# Remove settings that are no longer in the list
session.query(Settings).filter(Settings.id.in_(missing_ids)).delete()
session.query(Selects).filter(Selects.setting_id.in_(missing_ids)).delete()
session.query(Services_settings).filter(Services_settings.setting_id.in_(missing_ids)).delete()
session.query(Global_values).filter(Global_values.setting_id.in_(missing_ids)).delete()
for setting, value in settings.items():
value.update({"plugin_id": plugin["id"], "name": value["id"], "id": setting})
@ -1375,6 +1411,7 @@ class Database:
changes = True
# Remove jobs that are no longer in the list
session.query(Jobs).filter(Jobs.name.in_(missing_names)).delete()
session.query(Jobs_cache).filter(Jobs_cache.job_name.in_(missing_names)).delete()
for job in jobs:
db_job = (
@ -1409,6 +1446,7 @@ class Database:
tmp_ui_path = Path(sep, "var", "tmp", "bunkerweb", "ui", plugin["id"], "ui")
path_ui = tmp_ui_path if tmp_ui_path.exists() else Path(sep, "etc", "bunkerweb", "plugins", plugin["id"], "ui")
path_ui = path_ui if path_ui.exists() else Path(sep, "etc", "bunkerweb", "pro", "plugins", plugin["id"], "ui")
if path_ui.exists():
if {"template.html", "actions.py"}.issubset(listdir(str(path_ui))):
@ -1484,7 +1522,7 @@ class Database:
db_setting = session.query(Settings).filter_by(id=setting).first()
if db_setting is not None:
self.__logger.warning(f"A setting with id {setting} already exists, therefore it will not be added.")
self.logger.warning(f"A setting with id {setting} already exists, therefore it will not be added.")
continue
value.update({"plugin_id": plugin["id"], "name": value["id"], "id": setting})
@ -1500,7 +1538,7 @@ class Database:
)
if db_job is not None:
self.__logger.warning(f"A job with the name {job['name']} already exists in the database, therefore it will not be added.")
self.logger.warning(f"A job with the name {job['name']} already exists in the database, therefore it will not be added.")
continue
job["file_name"] = job.pop("file")
@ -1510,6 +1548,7 @@ class Database:
if page:
tmp_ui_path = Path(sep, "var", "tmp", "bunkerweb", "ui", plugin["id"], "ui")
path_ui = tmp_ui_path if tmp_ui_path.exists() else Path(sep, "etc", "bunkerweb", "plugins", plugin["id"], "ui")
path_ui = path_ui if path_ui.exists() else Path(sep, "etc", "bunkerweb", "pro", "plugins", plugin["id"], "ui")
if path_ui.exists():
if {"template.html", "actions.py"}.issubset(listdir(str(path_ui))):
@ -1681,13 +1720,8 @@ class Database:
}
def get_job_cache_file(
self,
job_name: str,
file_name: str,
*,
with_info: bool = False,
with_data: bool = True,
) -> Optional[Any]:
self, job_name: str, file_name: str, *, service_id: str = "", plugin_id: str = "", with_info: bool = False, with_data: bool = True
) -> Optional[Union[Dict[str, Any], bytes]]:
"""Get job cache file."""
entities = []
if with_info:
@ -1695,27 +1729,74 @@ class Database:
if with_data:
entities.append(Jobs_cache.data)
with self.__db_session() as session:
return session.query(Jobs_cache).with_entities(*entities).filter_by(job_name=job_name, file_name=file_name).first()
filters = {"job_name": job_name, "file_name": file_name}
if service_id:
filters["service_id"] = service_id
def get_jobs_cache_files(self) -> List[Dict[str, Any]]:
with self.__db_session() as session:
if plugin_id:
job = session.query(Jobs).filter_by(name=job_name, plugin_id=plugin_id).first()
if not job:
return None
data = session.query(Jobs_cache).with_entities(*entities).filter_by(**filters).first()
if not data:
return None
elif with_data and not with_info:
return data.data
ret_data = {}
if with_info:
ret_data["last_update"] = data.last_update.timestamp() if data.last_update is not None else "Never"
ret_data["checksum"] = data.checksum
if with_data:
ret_data["data"] = data.data
return ret_data
def get_jobs_cache_files(self, *, job_name: str = "", plugin_id: str = "") -> List[Dict[str, Any]]:
"""Get jobs cache files."""
with self.__db_session() as session:
return [
{
"job_name": cache.job_name,
"service_id": cache.service_id,
"file_name": cache.file_name,
"data": "Download file to view content",
}
for cache in (
session.query(Jobs_cache).with_entities(
Jobs_cache.job_name,
Jobs_cache.service_id,
Jobs_cache.file_name,
)
filters = {}
query = session.query(Jobs_cache).with_entities(Jobs_cache.job_name, Jobs_cache.service_id, Jobs_cache.file_name, Jobs_cache.data)
if job_name:
query = query.filter_by(job_name=job_name)
filters["name"] = job_name
db_cache = query.all()
if not db_cache:
return []
if plugin_id:
filters["plugin_id"] = plugin_id
query = session.query(Jobs).with_entities(Jobs.name, Jobs.plugin_id)
if filters:
query = query.filter_by(**filters)
jobs = {}
for job in query:
jobs[job.name] = job.plugin_id
if not jobs:
return []
cache_files = []
for cache in db_cache:
if cache.job_name not in jobs:
continue
cache_files.append(
{
"plugin_id": jobs[cache.job_name],
"job_name": cache.job_name,
"service_id": cache.service_id,
"file_name": cache.file_name,
"data": cache.data,
}
)
]
return cache_files
def add_instance(self, hostname: str, port: int, server_name: str, changed: Optional[bool] = True) -> str:
"""Add instance."""

Some files were not shown because too many files have changed in this diff Show more