Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #673 from bunkerity/dev

Browse source 
Merge branch "dev" into branch "ui"
This commit is contained in:
Théophile Diot 2023-10-03 13:10:39 +01:00 committed by GitHub
commit fe87486f97
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
236 changed files with 1490 additions and 3136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/beta.yml vendored
View file

@ -131,7 +131,7 @@ jobs:
versionrpm: ${{ steps.getversionrpm.outputs.versionrpm }}
steps:
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Get VERSION
id: getversion
run: echo "version=$(cat src/VERSION | tr -d '\n')" >> "$GITHUB_OUTPUT"

6
.github/workflows/codeql.yml vendored
View file

@ -19,13 +19,13 @@ jobs:
language: ["python", "javascript"]
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
uses: github/codeql-action/init@ddccb873888234080b77e9bc2d4764d5ccaaccf9 # v2.21.9
with:
languages: ${{ matrix.language }}
config-file: ./.github/codeql.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
uses: github/codeql-action/analyze@ddccb873888234080b77e9bc2d4764d5ccaaccf9 # v2.21.9
with:
category: "/language:${{matrix.language}}"

18
.github/workflows/container-build.yml vendored
View file

@ -45,7 +45,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Replace VERSION
if: inputs.RELEASE == 'testing'
run: ./misc/update-version.sh testing
@ -61,22 +61,22 @@ jobs:
SSH_IP: ${{ secrets.ARM_SSH_IP }}
SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }}
- name: Setup Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
if: inputs.CACHE_SUFFIX != 'arm'
- name: Setup Buildx (ARM)
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
if: inputs.CACHE_SUFFIX == 'arm'
with:
endpoint: ssh://root@arm
platforms: linux/arm64,linux/arm/v7,linux/arm/v6
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
if: inputs.PUSH == true
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -84,13 +84,13 @@ jobs:
# Compute metadata
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
with:
images: bunkerity/${{ inputs.IMAGE }}
# Build cached image
- name: Build image
if: inputs.CACHE == true
uses: docker/build-push-action@v5
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: .
file: ${{ inputs.DOCKERFILE }}
@ -103,7 +103,7 @@ jobs:
# Build non-cached image
- name: Build image
if: inputs.CACHE != true
uses: docker/build-push-action@v5
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: .
file: ${{ inputs.DOCKERFILE }}
@ -115,7 +115,7 @@ jobs:
# Check OS vulnerabilities
- name: Check OS vulnerabilities
if: ${{ inputs.CACHE_SUFFIX != 'arm' }}
uses: aquasecurity/trivy-action@master
uses: aquasecurity/trivy-action@69cbbc0cbbf6a2b0bab8dcf0e9f2d7ead08e87e4 # master
with:
vuln-type: os
skip-dirs: /root/.cargo

2
.github/workflows/create-arm.yml vendored
View file

@ -33,7 +33,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Get ARM availabilities
id: availabilities
uses: scaleway/action-scw@c718eca1fcb9fec1fb1433752d61599c6a0ad2e9

6
.github/workflows/dev-update-mmdb.yml vendored
View file

@ -5,14 +5,14 @@ permissions:
on:
schedule:
- cron: "0 1 5 * *"
- cron: "0 12 1 * *"
jobs:
mmdb-update:
runs-on: ubuntu-latest
steps:
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
fetch-depth: 0
token: ${{ secrets.BUNKERBOT_TOKEN }}
@ -52,7 +52,7 @@ jobs:
rm -f asn.mmdb country.mmdb
gunzip asn.mmdb.gz country.mmdb.gz
- name: Commit and push changes
uses: stefanzweifel/git-auto-commit-action@v4
uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4.16.0
with:
branch: dev
commit_message: "Monthly mmdb update"

6
.github/workflows/dev.yml vendored
View file

@ -88,7 +88,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- id: set-matrix
run: |
tests=$(find ./tests/core/ -maxdepth 1 -mindepth 1 -type d -printf "%f\n" | jq -c --raw-input --slurp 'split("\n")| .[0:-1]')
@ -126,12 +126,12 @@ jobs:
packages: write
steps:
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}

8
.github/workflows/doc-to-pdf.yml vendored
View file

@ -13,9 +13,9 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Install Python
uses: actions/setup-python@v4
uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
with:
python-version: "3.10"
- name: Install doc requirements
@ -23,7 +23,7 @@ jobs:
- name: Install chromium
run: sudo apt install chromium-browser
- name: Install node
uses: actions/setup-node@v3
uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1
with:
node-version: 18
- name: Install puppeteer
@ -32,7 +32,7 @@ jobs:
run: mkdocs serve & sleep 10
- name: Run pdf script
run: node docs/misc/pdf.js http://localhost:8000/print_page/ BunkerWeb_documentation_v${{ inputs.VERSION }}.pdf 'BunkerWeb documentation v${{ inputs.VERSION }}'
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: BunkerWeb_documentation_v${{ inputs.VERSION }}.pdf
path: BunkerWeb_documentation_v${{ inputs.VERSION }}.pdf

20
.github/workflows/linux-build.yml vendored
View file

@ -37,7 +37,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Replace VERSION
if: inputs.RELEASE == 'testing' || inputs.RELEASE == 'dev' || inputs.RELEASE == 'ui'
run: ./misc/update-version.sh ${{ inputs.RELEASE }}
@ -70,21 +70,21 @@ jobs:
SSH_IP: ${{ secrets.ARM_SSH_IP }}
SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }}
- name: Setup Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
if: startsWith(env.ARCH, 'arm') == false
- name: Setup Buildx (ARM)
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
if: startsWith(env.ARCH, 'arm') == true
with:
endpoint: ssh://root@arm
platforms: linux/arm64,linux/arm/v7,linux/arm/v6
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -92,7 +92,7 @@ jobs:
# Build testing package image
- name: Build package image
if: inputs.RELEASE == 'testing' || inputs.RELEASE == 'dev' || inputs.RELEASE == 'ui'
uses: docker/build-push-action@v5
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: .
load: true
@ -104,7 +104,7 @@ jobs:
# Build non-testing package image
- name: Build package image
if: inputs.RELEASE != 'testing' && inputs.RELEASE != 'dev'
uses: docker/build-push-action@v5
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: .
load: true
@ -127,7 +127,7 @@ jobs:
scp -r root@arm:/root/package-${{ inputs.LINUX }} ./package-${{ inputs.LINUX }}
env:
LARCH: ${{ env.LARCH }}
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: package-${{ inputs.LINUX }}-${{ env.LARCH }}
path: package-${{ inputs.LINUX }}/*.${{ inputs.PACKAGE }}
@ -135,12 +135,12 @@ jobs:
- name: Extract metadata
if: inputs.TEST == true
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
with:
images: ghcr.io/bunkerity/${{ inputs.LINUX }}-tests:${{ inputs.RELEASE }}
- name: Build test image
if: inputs.TEST == true
uses: docker/build-push-action@v5
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: .
file: tests/linux/Dockerfile-${{ inputs.LINUX }}

4
.github/workflows/push-doc.yml vendored
View file

@ -18,7 +18,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
fetch-depth: 0
token: ${{ secrets.BUNKERBOT_TOKEN }}
@ -29,7 +29,7 @@ jobs:
run: |
git config --global user.name "BunkerBot"
git config --global user.email "bunkerbot@bunkerity.com"
- uses: actions/setup-python@v4
- uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
with:
python-version: "3.10"
- name: Install doc requirements

12
.github/workflows/push-docker.yml vendored
View file

@ -33,14 +33,14 @@ jobs:
steps:
# Prepare
- name: Check out repository code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -56,19 +56,19 @@ jobs:
SSH_IP: ${{ secrets.ARM_SSH_IP }}
SSH_CONFIG: ${{ secrets.ARM_SSH_CONFIG }}
- name: Setup Buildx (ARM)
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
with:
endpoint: ssh://root@arm
platforms: linux/arm64,linux/arm/v7,linux/arm/v6
# Compute metadata
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
with:
images: bunkerity/${{ inputs.IMAGE }}
# Build and push
- name: Build and push
uses: docker/build-push-action@v5
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: .
file: ${{ inputs.DOCKERFILE }}

12
.github/workflows/push-github.yml vendored
View file

@ -15,15 +15,15 @@ jobs:
runs-on: ubuntu-latest
steps:
# Checkout
- uses: actions/checkout@v4
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
# Get PDF doc
- name: Get documentation
if: inputs.VERSION != 'testing'
uses: actions/download-artifact@v3
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: BunkerWeb_documentation_v${{ inputs.VERSION }}.pdf
# Create tag
- uses: rickstaa/action-create-tag@v1
- uses: rickstaa/action-create-tag@88dbf7ff6fe2405f8e8f6c6fdfd78829bc631f83 # v1.6.3
name: Create tag
if: inputs.VERSION != 'testing'
with:
@ -31,7 +31,7 @@ jobs:
message: "v${{ inputs.VERSION }}"
force_push_tag: true
# Create tag
- uses: rickstaa/action-create-tag@v1
- uses: rickstaa/action-create-tag@88dbf7ff6fe2405f8e8f6c6fdfd78829bc631f83 # v1.6.3
name: Create tag
if: inputs.VERSION == 'testing'
with:
@ -51,7 +51,7 @@ jobs:
# Create release
- name: Create release
if: inputs.VERSION != 'testing'
uses: softprops/action-gh-release@v1
uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v1
with:
body: |
Documentation : https://docs.bunkerweb.io/${{ inputs.VERSION }}/
@ -75,7 +75,7 @@ jobs:
# Create release
- name: Create release
if: inputs.VERSION == 'testing'
uses: softprops/action-gh-release@v1
uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v1
with:
body: |
**The testing version of BunkerWeb should not be used in production, please use the latest stable version instead.**

10
.github/workflows/push-packagecloud.yml vendored
View file

@ -40,20 +40,20 @@ jobs:
steps:
# Prepare
- name: Check out repository code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Install ruby
uses: ruby/setup-ruby@v1
uses: ruby/setup-ruby@52b8784594ec115fd17094752708121dc5dabb47 # v1.154.0
with:
ruby-version: "3.0"
- name: Install packagecloud
run: gem install package_cloud
# Download packages
- uses: actions/download-artifact@v3
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
if: inputs.LINUX != 'el'
with:
name: package-${{ inputs.LINUX }}-${{ inputs.PACKAGE_ARCH }}
path: /tmp/${{ inputs.LINUX }}
- uses: actions/download-artifact@v3
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
if: inputs.LINUX == 'el'
with:
name: package-rhel-${{ inputs.PACKAGE_ARCH }}
@ -70,7 +70,7 @@ jobs:
# run: sudo apt install -y rename && rename 's/[0-9]\.[0-9]\.[0-9]/testing/' /tmp/${{ inputs.LINUX }}/*.${{ inputs.PACKAGE }}
# Push package
- name: Push package to packagecloud
uses: danielmundi/upload-packagecloud@v1
uses: danielmundi/upload-packagecloud@46cd0e61152bf952dbc0d1759e609d3d22649030 # v1
with:
PACKAGE-NAME: /tmp/${{ inputs.LINUX }}/*.${{ inputs.PACKAGE }}
PACKAGECLOUD-USERNAME: bunkerity

2
.github/workflows/release.yml vendored
View file

@ -139,7 +139,7 @@ jobs:
versionrpm: ${{ steps.getversionrpm.outputs.versionrpm }}
steps:
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Get VERSION
id: getversion
run: echo "version=$(cat src/VERSION | tr -d '\n')" >> "$GITHUB_OUTPUT"

2
.github/workflows/rm-arm.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Delete ARM VM
uses: scaleway/action-scw@c718eca1fcb9fec1fb1433752d61599c6a0ad2e9
with:

6
.github/workflows/scorecards-analysis.yml vendored
View file

@ -15,16 +15,16 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: "Checkout code"
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
persist-credentials: false
- name: "Run analysis"
uses: ossf/scorecard-action@v2.2.0
uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0
with:
results_file: results.sarif
results_format: sarif
publish_results: true
- name: "Upload SARIF results to code scanning"
uses: github/codeql-action/upload-sarif@v2
uses: github/codeql-action/upload-sarif@ddccb873888234080b77e9bc2d4764d5ccaaccf9 # v2.21.9
with:
sarif_file: results.sarif

10
.github/workflows/staging-create-infra.yml vendored
View file

@ -23,14 +23,14 @@ jobs:
run: ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N "" && ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub && echo -e "Host *\n StrictHostKeyChecking no" > ~/.ssh/ssh_config
if: inputs.TYPE != 'k8s'
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Install terraform
uses: hashicorp/setup-terraform@v2
uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
- name: Install kubectl
uses: azure/setup-kubectl@v3
uses: azure/setup-kubectl@901a10e89ea615cf61f57ac05cecdf23e7de06d8 # v3.2
if: inputs.TYPE == 'k8s'
- name: Set up Python 3.11
uses: actions/setup-python@v4
uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
if: inputs.TYPE != 'k8s'
with:
python-version: "3.11"
@ -54,7 +54,7 @@ jobs:
if: always()
env:
SECRET_KEY: ${{ secrets.SECRET_KEY }}
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
if: always()
with:
name: tf-${{ inputs.TYPE }}

8
.github/workflows/staging-delete-infra.yml vendored
View file

@ -20,10 +20,10 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Install terraform
uses: hashicorp/setup-terraform@v2
- uses: actions/download-artifact@v3
uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: tf-${{ inputs.TYPE }}
path: /tmp
@ -34,7 +34,7 @@ jobs:
tar xf /tmp/terraform.tar -C / && mkdir ~/.ssh && touch ~/.ssh/id_rsa.pub
env:
SECRET_KEY: ${{ secrets.SECRET_KEY }}
- uses: azure/setup-kubectl@v3
- uses: azure/setup-kubectl@901a10e89ea615cf61f57ac05cecdf23e7de06d8 # v3.2
if: inputs.TYPE == 'k8s'
# Remove infra
- run: kubectl delete daemonsets,replicasets,services,deployments,pods,rc,ingress,statefulsets --all --all-namespaces --timeout=60s ; kubectl delete pvc --all --timeout=60s ; kubectl delete pv --all --timeout=60s

10
.github/workflows/staging-tests.yml vendored
View file

@ -25,9 +25,9 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Login to ghcr
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -43,7 +43,7 @@ jobs:
if: inputs.TYPE == 'swarm'
- name: Install test dependencies
run: pip3 install --no-cache-dir --require-hashes -r tests/requirements.txt
- uses: actions/download-artifact@v3
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: tf-k8s
path: /tmp
@ -66,9 +66,9 @@ jobs:
REG_USER: ${{ github.actor }}
REG_TOKEN: ${{ secrets.GITHUB_TOKEN }}
if: inputs.TYPE == 'k8s'
- uses: azure/setup-kubectl@v3
- uses: azure/setup-kubectl@901a10e89ea615cf61f57ac05cecdf23e7de06d8 # v3.2
if: inputs.TYPE == 'k8s'
- uses: azure/setup-helm@v3
- uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
if: inputs.TYPE == 'k8s'
- name: Pull BW linux ubuntu test image
if: inputs.TYPE == 'linux'

6
.github/workflows/staging.yml vendored
View file

@ -89,7 +89,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- id: set-matrix
run: |
tests=$(find ./tests/core/ -maxdepth 1 -mindepth 1 -type d -printf "%f\n" | jq -c --raw-input --slurp 'split("\n")| .[0:-1]')
@ -174,12 +174,12 @@ jobs:
packages: write
steps:
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to ghcr
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}

8
.github/workflows/test-core-linux.yml vendored
View file

@ -16,9 +16,9 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Set up Python 3.11
uses: actions/setup-python@v4
uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
with:
python-version: "3.11"
cache: "pip"
@ -34,7 +34,7 @@ jobs:
sudo chmod 755 /opt/firefox /opt/firefox/firefox
rm -f firefox-setup.tar.bz2
- name: Download geckodriver
uses: nick-fields/retry@v2
uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0
with:
max_attempts: 3
timeout_minutes: 20
@ -45,7 +45,7 @@ jobs:
sudo chmod +x /usr/local/bin/geckodriver
rm -f geckodriver.tar.gz
- name: Login to ghcr
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}

4
.github/workflows/test-core.yml vendored
View file

@ -16,9 +16,9 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Login to ghcr
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}

8
.github/workflows/tests-ui-linux.yml vendored
View file

@ -13,9 +13,9 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Set up Python 3.11
uses: actions/setup-python@v4
uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
with:
python-version: "3.11"
cache: "pip"
@ -31,7 +31,7 @@ jobs:
sudo chmod 755 /opt/firefox /opt/firefox/firefox
rm -f firefox-setup.tar.bz2
- name: Download geckodriver
uses: nick-fields/retry@v2
uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0
with:
max_attempts: 3
timeout_minutes: 20
@ -42,7 +42,7 @@ jobs:
sudo chmod +x /usr/local/bin/geckodriver
rm -f geckodriver.tar.gz
- name: Login to ghcr
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}

4
.github/workflows/tests-ui.yml vendored
View file

@ -12,9 +12,9 @@ jobs:
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
- name: Login to ghcr
uses: docker/login-action@v3
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}

40
.pre-commit-config.yaml
View file

@ -3,26 +3,52 @@
exclude: (^LICENSE.md$|^src/VERSION$|^src/(bw/misc/root-ca.pem$|deps/src/|common/core/modsecurity/files|ui/static/js/(editor/|utils/purify/|tsparticles\.bundle\.min\.js))|\.(svg|drawio|patch\d?|ascii|tf|tftpl)$)
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: f71fa2c1f9cf5cb705f73dffe4b21f7c61470ba9 # frozen: v4.4.0
hooks:
- id: check-case-conflict
- id: detect-private-key
- id: end-of-file-fixer
- id: requirements-txt-fixer
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
exclude: ^(mkdocs.yml|examples/bigbluebutton/docker-compose.yml)$
args: ["--allow-multiple-documents"]
- id: check-case-conflict
- repo: https://github.com/ambv/black
rev: 23.9.1
rev: e87737140f32d3cd7c44ede75f02dcd58e55820e # frozen: 23.9.1
hooks:
- id: black
name: Black Python Formatter
language_version: python3.9
- repo: https://github.com/pre-commit/mirrors-prettier
rev: fc260393cc4ec09f8fc0a5ba4437f481c8b55dc1 # frozen: v3.0.3
hooks:
- id: prettier
name: Prettier Code Formatter
- repo: https://github.com/pycqa/flake8
rev: 10f4af6dbcf93456ba7df762278ae61ba3120dc6 # frozen: 6.1.0
hooks:
- id: flake8
name: Flake8 Python Linter
args: ["--max-line-length=250", "--ignore=E266,E402,E722,W503"]
- repo: https://github.com/codespell-project/codespell
rev: 6e41aba91fb32e9feb741a6258eefeb9c6e4a482 # frozen: v2.2.6
hooks:
- id: codespell
name: Codespell Spell Checker
exclude: (^src/(common/core/.+/files|bw/loading)/.+.html|modsecurity-rules.conf.*)$
entry: codespell --ignore-regex="(tabEl|Widgits)" --skip src/ui/static/js/utils/flatpickr.js,CHANGELOG.md
language: python
types: [text]
- repo: https://github.com/gitleaks/gitleaks
rev: v8.18.0
rev: b813e6fe08b87541cb77296359ba1b7a50a00c98 # frozen: v8.18.0
hooks:
- id: gitleaks
- repo: https://github.com/koalaman/shellcheck-precommit
rev: v0.9.0
rev: 3f77b826548d8dc2d26675f077361c92773b50a7 # frozen: v0.9.0
hooks:
- id: shellcheck

1
.prettierignore
View file

@ -7,6 +7,7 @@ src/deps/src/
mkdocs.yml
CHANGELOG.md
CONTRIBUTING.md
CODE_OF_CONDUCT.md
LICENSE.md
README.md
SECURITY.md

4
CHANGELOG.md
View file

@ -7,9 +7,13 @@
- [BUGFIX] Fix UI clearing configs folder at startup
- [BUGFIX] Fix Database not clearing old services when not using multisite
- [BUGFIX] Fix UI using the wrong database when generating the new config when using an external database
- [BUGFIX] Small fixes on linux paths creating unnecessary folders
- [MISC] Updated core dependencies
- [MISC] Updated self-signed job to regenerate the cert if the subject or the expiration date has changed
- [MISC] Jobs that download files from urls will now remove old cached files if urls are empty
- [MISC] Replaced gevent with gthread in UI for security reasons
- [MISC] Add HTML sanitization when injecting code in pages in the UI
- [MISC] Optimize the way the UI handles services creation and edition
## v1.5.2 - 2023/10/10

4
README.md
View file

@ -335,10 +335,10 @@ Please contact us at [contact@bunkerity.com](mailto:contact@bunkerity.com) if yo
## Community
To get free community support you can use the following medias :
To get free community support you can use the following media :
* The #help channel of BunkerWeb in the [Discord server](https://discord.com/invite/fTf46FmtyD)
* The help category of [GitHub dicussions](https://github.com/bunkerity/bunkerweb/discussions)
* The help category of [GitHub discussions](https://github.com/bunkerity/bunkerweb/discussions)
* The [/r/BunkerWeb](https://www.reddit.com/r/BunkerWeb) subreddit
* The [Server Fault](https://serverfault.com/) and [Super User](https://superuser.com/) forums

4
docs/about.md
View file

@ -17,10 +17,10 @@ Please contact us at [contact@bunkerity.com](mailto:contact@bunkerity.com) if yo
## Where to get community support ?
To get free community support, you can use the following medias :
To get free community support, you can use the following media :
- The #help channel of BunkerWeb in the [Discord server](https://discord.com/invite/fTf46FmtyD)
- The help category of [GitHub dicussions](https://github.com/bunkerity/bunkerweb/discussions)
- The help category of [GitHub discussions](https://github.com/bunkerity/bunkerweb/discussions)
- The [/r/BunkerWeb](https://www.reddit.com/r/BunkerWeb) subreddit
- The [Server Fault](https://serverfault.com/) and [Super User](https://superuser.com/) forums

2
docs/concepts.md
View file

@ -1,7 +1,7 @@
# Concepts
<figure markdown>
![Overwiew](assets/img/concepts.svg){ align=center, width="600" }
![Overview](assets/img/concepts.svg){ align=center, width="600" }
</figure>
## Integrations

12
docs/json2md.py
View file

@ -14,7 +14,7 @@ def print_md_table(settings) -> MarkdownTableWriter:
f"`{setting}`",
"" if data["default"] == "" else f"`{data['default']}`",
data["context"],
"no" if not "multiple" in data else "yes",
"no" if "multiple" not in data else "yes",
data["help"],
]
for setting, data in settings.items()
@ -42,15 +42,19 @@ print(
file=doc,
)
print(
"This section contains the full list of settings supported by BunkerWeb. If you are not yet familiar with BunkerWeb, you should first read the [concepts](concepts.md) section of the documentation. Please follow the instructions for your own [integration](integrations.md) on how to apply the settings.\n",
"This section contains the full list of settings supported by BunkerWeb."
+ " If you are not yet familiar with BunkerWeb, you should first read the [concepts](concepts.md) section of the documentation."
+ " Please follow the instructions for your own [integration](integrations.md) on how to apply the settings.\n",
file=doc,
)
print(
"As a general rule when multisite mode is enabled, if you want to apply settings with multisite context to a specific server, you will need to add the primary (first) server name as a prefix like `www.example.com_USE_ANTIBOT=captcha` or `myapp.example.com_USE_GZIP=yes` for example.\n",
"As a general rule when multisite mode is enabled, if you want to apply settings with multisite context to a specific server, you will need to add the primary"
+ " (first) server name as a prefix like `www.example.com_USE_ANTIBOT=captcha` or `myapp.example.com_USE_GZIP=yes` for example.\n",
file=doc,
)
print(
'When settings are considered as "multiple", it means that you can have multiple groups of settings for the same feature by adding numbers as suffix like `REVERSE_PROXY_URL_1=/subdir`, `REVERSE_PROXY_HOST_1=http://myhost1`, `REVERSE_PROXY_URL_2=/anotherdir`, `REVERSE_PROXY_HOST_2=http://myhost2`, ... for example.\n',
'When settings are considered as "multiple", it means that you can have multiple groups of settings for the same feature by adding numbers as suffix like `REVERSE_PROXY_URL_1=/subdir`,'
+ " `REVERSE_PROXY_HOST_1=http://myhost1`, `REVERSE_PROXY_URL_2=/anotherdir`, `REVERSE_PROXY_HOST_2=http://myhost2`, ... for example.\n",
file=doc,
)

4
docs/plugins.md
View file

@ -106,7 +106,7 @@ The first step is to install the plugin by putting the plugin files inside the c
When using the [Swarm integration](integrations.md#swarm), plugins must be written to the volume mounted on `/data/plugins` into the scheduler container.
!!! info "Swarm volume"
Configuring a Swarm volume that will persist when the scheduler service is running on different nodes is not covered is in this documentation. We will assume that you have a shared folder mounted on `/shared` accross all nodes.
Configuring a Swarm volume that will persist when the scheduler service is running on different nodes is not covered is in this documentation. We will assume that you have a shared folder mounted on `/shared` across all nodes.
The first thing to do is to create the plugins folder :
@ -460,7 +460,7 @@ Some helpers modules provide common helpful helpers :
- `self.logger` : print logs
- `bunkerweb.utils` : various useful functions
- `bunkerweb.datastore` : access the global shared data on one instance (key/value store)
- `bunkerweb.clusterstore` : access a Redis data store shared beetween BunkerWeb instances (key/value store)
- `bunkerweb.clusterstore` : access a Redis data store shared between BunkerWeb instances (key/value store)
To access the functions, you first need to **require** the modules :

2
docs/quickstart-guide.md
View file

@ -2322,7 +2322,7 @@ BunkerWeb supports PHP using external or remote [PHP-FPM](https://www.php.net/ma
!!! warning "Feature is in beta"
This feature is not production-ready. Feel free to test it and report us any bug using [issues](https://github.com/bunkerity/bunkerweb/issues) in the GitHub repository.
By default, BunkerWeb will only listen on IPv4 adresses and won't use IPv6 for network communications. If you want to enable IPv6 support, you need to set `USE_IPV6=yes`. Please note that IPv6 configuration of your network and environment is out-of-the-scope of this documentation.
By default, BunkerWeb will only listen on IPv4 addresses and won't use IPv6 for network communications. If you want to enable IPv6 support, you need to set `USE_IPV6=yes`. Please note that IPv6 configuration of your network and environment is out-of-the-scope of this documentation.
=== "Docker"

18
docs/requirements.txt
View file

@ -4,9 +4,9 @@
#
# pip-compile --allow-unsafe --generate-hashes --strip-extras requirements.in
#
babel==2.12.1 \
--hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \
--hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455
babel==2.13.0 \
--hash=sha256:04c3e2d28d2b7681644508f836be388ae49e0cfe91465095340395b60d00f210 \
--hash=sha256:fbfcae1575ff78e26c7449136f1abbefc3c13ce542eeb13d43d50d8b047216ec
# via mkdocs-material
certifi==2023.7.22 \
--hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \
@ -267,9 +267,9 @@ pathvalidate==3.2.0 \
--hash=sha256:5e8378cf6712bff67fbe7a8307d99fa8c1a0cb28aa477056f8fc374f0dff24ad \
--hash=sha256:cc593caa6299b22b37f228148257997e2fa850eea2daf7e4cc9205cef6908dee
# via pytablewriter
platformdirs==3.10.0 \
--hash=sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d \
--hash=sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d
platformdirs==3.11.0 \
--hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \
--hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e
# via mkdocs
pygments==2.16.1 \
--hash=sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692 \
@ -472,9 +472,9 @@ typepy==1.3.1 \
# dataproperty
# pytablewriter
# tabledata
urllib3==2.0.5 \
--hash=sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594 \
--hash=sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e
urllib3==2.0.6 \
--hash=sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2 \
--hash=sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564
# via requests
verspec==0.1.0 \
--hash=sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31 \

2
docs/troubleshooting.md
View file

@ -284,4 +284,4 @@ If you have bots that need to access your website, the recommended way to avoid
## Timezone
When using container-based integrations, the timezone of the container may not match the one of the host machine. To resolve that, you can set the `TZ` environment variable to the timezone of your choice on your containers (e.g. `TZ=Europe/Paris`). You will find the list of timezone identifers [here](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List).
When using container-based integrations, the timezone of the container may not match the one of the host machine. To resolve that, you can set the `TZ` environment variable to the timezone of your choice on your containers (e.g. `TZ=Europe/Paris`). You will find the list of timezone identifiers [here](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List).

3
examples/drupal/docker-compose.yml
View file

@ -18,7 +18,7 @@ services:
REVERSE_PROXY_URL: "/"
REVERSE_PROXY_HOST: "http://mydrupal"
CUSTOM_CONF_MODSEC_CRS_drupal: 'SecAction "id:900130,phase:1,nolog,pass,t:none,setvar:tx.crs_exclusions_drupal=1"'
# Onces the installation is done, you can remove these lines
# Once the installation is done, you can remove these lines
LIMIT_REQ_URL_1: "/core/install.php"
LIMIT_REQ_RATE_1: "5r/s"
labels:
@ -75,6 +75,7 @@ volumes:
bw-data:
db-data:
networks:
bw-universe:
ipam:

3
examples/prestashop/docker-compose.yml
View file

@ -18,7 +18,7 @@ services:
USE_REVERSE_PROXY: "yes"
REVERSE_PROXY_URL: "/"
REVERSE_PROXY_HOST: "http://myps"
# Onces the installation is done, you can remove these lines
# Once the installation is done, you can remove these lines
LIMIT_REQ_URL_1: "/install/index.php"
LIMIT_REQ_RATE_1: "8r/s"
labels:
@ -85,6 +85,7 @@ volumes:
ps-data:
db-data:
networks:
bw-universe:
ipam:

2
misc/integrations/swarm.postgres.ui.yml
View file

@ -1,4 +1,4 @@
version: "3.5"
version: "3.5"
services:
bunkerweb:

3
mkdocs.yml
View file

@ -2,7 +2,7 @@ site_name: BunkerWeb
site_url: https://docs.bunkerweb.io/
repo_url: https://github.com/bunkerity/bunkerweb
site_description: Make your web services secure by default.
copyright: Bunkerity
copyright: Copyright &copy; <script>document.write(new Date().getFullYear())</script> Bunkerity
#remote_branch: ?
#remote_name: ?
@ -49,7 +49,6 @@ markdown_extensions:
emoji_index: !!python/name:materialx.emoji.twemoji
emoji_generator: !!python/name:materialx.emoji.to_svg
copyright: Copyright &copy; <script>document.write(new Date().getFullYear())</script> Bunkerity
extra:
version:
provider: mike

2
pyproject.toml
View file

@ -8,6 +8,8 @@ authors = [
[tool.black]
py39 = true
line-length = 250
include = '\.pyi?$'
exclude = '''
/(
| \.git

26
src/autoconf/Config.py
View file

@ -2,7 +2,6 @@
from os import getenv
from time import sleep
from typing import Optional
from copy import deepcopy
from ConfigCaller import ConfigCaller # type: ignore
@ -25,13 +24,20 @@ class Config(ConfigCaller):
"modsec",
"modsec-crs",
]
self.__configs = {
config_type: {} for config_type in self._supported_config_types
}
self.__configs = {config_type: {} for config_type in self._supported_config_types}
self.__config = {}
self._db = Database(self.__logger)
def _update_settings(self):
plugins = self._db.get_plugins()
if not plugins:
self.__logger.error("No plugins in database, can't update settings...")
return
self._settings = []
for plugin in plugins:
self._settings.update(plugin["settings"])
def __get_full_env(self) -> dict:
env_instances = {"SERVER_NAME": ""}
for instance in self.__instances:
@ -106,9 +112,7 @@ class Config(ConfigCaller):
while True:
curr_changes = self._db.check_changes()
if isinstance(curr_changes, str):
self.__logger.error(
f"An error occurred when checking for changes in the database : {curr_changes}"
)
self.__logger.error(f"An error occurred when checking for changes in the database : {curr_changes}")
elif not any(curr_changes.values()):
break
else:
@ -134,9 +138,7 @@ class Config(ConfigCaller):
# save custom configs to database
if "custom_configs" in changes:
err = self._db.save_custom_configs(
custom_configs, "autoconf", changed=False
)
err = self._db.save_custom_configs(custom_configs, "autoconf", changed=False)
if err:
success = False
self.__logger.error(
@ -146,8 +148,6 @@ class Config(ConfigCaller):
# update changes in db
ret = self._db.checked_changes(changes, value=True)
if ret:
self.__logger.error(
f"An error occurred when setting the changes to checked in the database : {ret}"
)
self.__logger.error(f"An error occurred when setting the changes to checked in the database : {ret}")
return success

14
src/autoconf/Controller.py
View file

@ -1,9 +1,9 @@
#!/usr/bin/python3
from abc import ABC, abstractmethod
from abc import abstractmethod
from os import getenv
from time import sleep
from typing import Literal, Optional, Union
from typing import Literal, Union
from Config import Config
@ -20,12 +20,8 @@ class Controller(Config):
self._type = ctrl_type
self._instances = []
self._services = []
self._configs = {
config_type: {} for config_type in self._supported_config_types
}
self._logger = setup_logger(
f"{self._type}-controller", getenv("LOG_LEVEL", "INFO")
)
self._configs = {config_type: {} for config_type in self._supported_config_types}
self._logger = setup_logger(f"{self._type}-controller", getenv("LOG_LEVEL", "INFO"))
def wait(self, wait_time: int) -> list:
all_ready = False
@ -105,7 +101,7 @@ class Controller(Config):
def _is_service_present(self, server_name):
for service in self._services:
if not "SERVER_NAME" in service or not service["SERVER_NAME"]:
if "SERVER_NAME" not in service or not service["SERVER_NAME"]:
continue
if server_name == service["SERVER_NAME"].strip().split(" ")[0]:
return True

38
src/autoconf/DockerController.py
View file

@ -13,9 +13,7 @@ class DockerController(Controller):
def __init__(self, docker_host):
super().__init__("docker")
self.__client = DockerClient(base_url=docker_host)
self.__custom_confs_rx = re_compile(
r"^bunkerweb.CUSTOM_CONF_(SERVER_HTTP|MODSEC_CRS|MODSEC)_(.+)$"
)
self.__custom_confs_rx = re_compile(r"^bunkerweb.CUSTOM_CONF_(SERVER_HTTP|MODSEC_CRS|MODSEC)_(.+)$")
def _get_controller_instances(self) -> List[Container]:
return self.__client.containers.list(filters={"label": "bunkerweb.INSTANCE"})
@ -27,10 +25,7 @@ class DockerController(Controller):
instance = {}
instance["name"] = controller_instance.name
instance["hostname"] = controller_instance.name
instance["health"] = (
controller_instance.status == "running"
and controller_instance.attrs["State"]["Health"]["Status"] == "healthy"
)
instance["health"] = controller_instance.status == "running" and controller_instance.attrs["State"]["Health"]["Status"] == "healthy"
instance["env"] = {}
for env in controller_instance.attrs["Config"]["Env"]:
variable = env.split("=")[0]
@ -53,9 +48,7 @@ class DockerController(Controller):
def _get_static_services(self) -> List[dict]:
services = []
variables = {}
for instance in self.__client.containers.list(
filters={"label": "bunkerweb.INSTANCE"}
):
for instance in self.__client.containers.list(filters={"label": "bunkerweb.INSTANCE"}):
if not instance.attrs or not instance.attrs.get("Config", {}).get("Env"):
continue
@ -70,9 +63,7 @@ class DockerController(Controller):
for variable, value in variables.items():
prefix = variable.split("_")[0]
real_variable = variable.replace(f"{prefix}_", "", 1)
if prefix == server_name and self._is_setting_context(
real_variable, "multisite"
):
if prefix == server_name and self._is_setting_context(real_variable, "multisite"):
service[real_variable] = value
services.append(service)
return services
@ -80,9 +71,7 @@ class DockerController(Controller):
def get_configs(self) -> Dict[str, Dict[str, Any]]:
configs = {config_type: {} for config_type in self._supported_config_types}
# get site configs from labels
for container in self.__client.containers.list(
filters={"label": "bunkerweb.SERVER_NAME"}
):
for container in self.__client.containers.list(filters={"label": "bunkerweb.SERVER_NAME"}):
labels = container.labels # type: ignore (labels is inside a container)
if isinstance(labels, list):
labels = {label: "" for label in labels}
@ -100,9 +89,7 @@ class DockerController(Controller):
result = self.__custom_confs_rx.search(variable)
if result is None:
continue
configs[result.group(1).lower().replace("_", "-")][
f"{server_name}/{result.group(2)}"
] = value
configs[result.group(1).lower().replace("_", "-")][f"{server_name}/{result.group(2)}"] = value
return configs
def apply_config(self) -> bool:
@ -117,16 +104,13 @@ class DockerController(Controller):
self._set_autoconf_load_db()
for _ in self.__client.events(decode=True, filters={"type": "container"}):
try:
self._update_settings()
self._instances = self.get_instances()
self._services = self.get_services()
self._configs = self.get_configs()
if not self.update_needed(
self._instances, self._services, configs=self._configs
):
if not self.update_needed(self._instances, self._services, configs=self._configs):
continue
self._logger.info(
"Caught Docker event, deploying new configuration ..."
)
self._logger.info("Caught Docker event, deploying new configuration ...")
if not self.apply_config():
self._logger.error("Error while deploying new configuration")
else:
@ -136,6 +120,4 @@ class DockerController(Controller):
self._set_autoconf_load_db()
except:
self._logger.error(
f"Exception while processing events :\n{format_exc()}"
)
self._logger.error(f"Exception while processing events :\n{format_exc()}")

4
src/autoconf/Dockerfile
View file

@ -1,4 +1,4 @@
FROM python:3.11.5-alpine@sha256:cd311c6a0164f34a7edbf364e05258b07d66d3f7bc155139dcb9bef88a186ded AS builder
FROM python:3.12.0-alpine@sha256:ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac AS builder
# Copy python requirements
COPY src/deps/requirements.txt /tmp/requirements-deps.txt
@ -34,7 +34,7 @@ COPY src/common/helpers helpers
COPY src/common/settings.json settings.json
COPY src/common/utils utils
FROM python:3.11.5-alpine@sha256:cd311c6a0164f34a7edbf364e05258b07d66d3f7bc155139dcb9bef88a186ded
FROM python:3.12.0-alpine@sha256:ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac
# Set default umask to prevent huge recursive chmod increasing the final image size
RUN umask 027

55
src/autoconf/IngressController.py
View file

@ -19,21 +19,12 @@ class IngressController(Controller):
self.__networkingv1 = client.NetworkingV1Api()
def _get_controller_instances(self) -> list:
return [
pod
for pod in self.__corev1.list_pod_for_all_namespaces(watch=False).items
if (
pod.metadata.annotations
and "bunkerweb.io/INSTANCE" in pod.metadata.annotations
)
]
return [pod for pod in self.__corev1.list_pod_for_all_namespaces(watch=False).items if (pod.metadata.annotations and "bunkerweb.io/INSTANCE" in pod.metadata.annotations)]
def _to_instances(self, controller_instance) -> List[dict]:
instance = {}
instance["name"] = controller_instance.metadata.name
instance["hostname"] = (
controller_instance.status.pod_ip or controller_instance.metadata.name
)
instance["hostname"] = controller_instance.status.pod_ip or controller_instance.metadata.name
health = False
if controller_instance.status.conditions:
for condition in controller_instance.status.conditions:
@ -48,9 +39,7 @@ class IngressController(Controller):
pod = container
break
if not pod:
self._logger.warning(
f"Missing container bunkerweb in pod {controller_instance.metadata.name}"
)
self._logger.warning(f"Missing container bunkerweb in pod {controller_instance.metadata.name}")
else:
for env in pod.env:
instance["env"][env.name] = env.value or ""
@ -159,10 +148,7 @@ class IngressController(Controller):
services = []
variables = {}
for instance in self.__corev1.list_pod_for_all_namespaces(watch=False).items:
if (
not instance.metadata.annotations
or not "bunkerweb.io/INSTANCE" in instance.metadata.annotations
):
if not instance.metadata.annotations or "bunkerweb.io/INSTANCE" not in instance.metadata.annotations:
continue
pod = None
@ -181,22 +167,15 @@ class IngressController(Controller):
for variable, value in variables.items():
prefix = variable.split("_")[0]
real_variable = variable.replace(f"{prefix}_", "", 1)
if prefix == server_name and self._is_setting_context(
real_variable, "multisite"
):
if prefix == server_name and self._is_setting_context(real_variable, "multisite"):
service[real_variable] = value
services.append(service)
return services
def get_configs(self) -> dict:
configs = {config_type: {} for config_type in self._supported_config_types}
for configmap in self.__corev1.list_config_map_for_all_namespaces(
watch=False
).items:
if (
not configmap.metadata.annotations
or "bunkerweb.io/CONFIG_TYPE" not in configmap.metadata.annotations
):
for configmap in self.__corev1.list_config_map_for_all_namespaces(watch=False).items:
if not configmap.metadata.annotations or "bunkerweb.io/CONFIG_TYPE" not in configmap.metadata.annotations:
continue
config_type = configmap.metadata.annotations["bunkerweb.io/CONFIG_TYPE"]
@ -212,16 +191,12 @@ class IngressController(Controller):
continue
config_site = ""
if "bunkerweb.io/CONFIG_SITE" in configmap.metadata.annotations:
if not self._is_service_present(
configmap.metadata.annotations["bunkerweb.io/CONFIG_SITE"]
):
if not self._is_service_present(configmap.metadata.annotations["bunkerweb.io/CONFIG_SITE"]):
self._logger.warning(
f"Ignoring config {configmap.metadata.name} because {configmap.metadata.annotations['bunkerweb.io/CONFIG_SITE']} doesn't exist",
)
continue
config_site = (
f"{configmap.metadata.annotations['bunkerweb.io/CONFIG_SITE']}/"
)
config_site = f"{configmap.metadata.annotations['bunkerweb.io/CONFIG_SITE']}/"
for config_name, config_data in configmap.data.items():
configs[config_type][f"{config_site}{config_name}"] = config_data
return configs
@ -247,17 +222,16 @@ class IngressController(Controller):
for _ in w.stream(what):
self.__internal_lock.acquire()
locked = True
self._update_settings()
self._instances = self.get_instances()
self._services = self.get_services()
self._configs = self.get_configs()
if not self.update_needed(
self._instances, self._services, configs=self._configs
):
if not self.update_needed(self._instances, self._services, configs=self._configs):
self.__internal_lock.release()
locked = False
continue
self._logger.info(
f"Catched kubernetes event ({watch_type}), deploying new configuration ...",
f"Caught kubernetes event ({watch_type}), deploying new configuration ...",
)
try:
ret = self.apply_config()
@ -308,10 +282,7 @@ class IngressController(Controller):
def process_events(self):
self._set_autoconf_load_db()
watch_types = ("pod", "ingress", "configmap", "service")
threads = [
Thread(target=self.__watch, args=(watch_type,))
for watch_type in watch_types
]
threads = [Thread(target=self.__watch, args=(watch_type,)) for watch_type in watch_types]
for thread in threads:
thread.start()
for thread in threads:

65
src/autoconf/SwarmController.py
View file

@ -26,9 +26,7 @@ class SwarmController(Controller):
def _to_instances(self, controller_instance) -> List[dict]:
instances = []
instance_env = {}
for env in controller_instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"][
"Env"
]:
for env in controller_instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
variable = env.split("=")[0]
value = env.replace(f"{variable}=", "", 1)
if self._is_setting(variable):
@ -61,12 +59,8 @@ class SwarmController(Controller):
def _get_static_services(self) -> List[dict]:
services = []
variables = {}
for instance in self.__client.services.list(
filters={"label": "bunkerweb.INSTANCE"}
):
if not instance.attrs or not instance.attrs.get("Spec", {}).get(
"TaskTemplate", {}
).get("ContainerSpec", {}).get("Env"):
for instance in self.__client.services.list(filters={"label": "bunkerweb.INSTANCE"}):
if not instance.attrs or not instance.attrs.get("Spec", {}).get("TaskTemplate", {}).get("ContainerSpec", {}).get("Env"):
continue
for env in instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
@ -80,9 +74,7 @@ class SwarmController(Controller):
for variable, value in variables.items():
prefix = variable.split("_")[0]
real_variable = variable.replace(f"{prefix}_", "", 1)
if prefix == server_name and self._is_setting_context(
real_variable, "multisite"
):
if prefix == server_name and self._is_setting_context(real_variable, "multisite"):
service[real_variable] = value
services.append(service)
return services
@ -91,15 +83,8 @@ class SwarmController(Controller):
configs = {}
for config_type in self._supported_config_types:
configs[config_type] = {}
for config in self.__client.configs.list(
filters={"label": "bunkerweb.CONFIG_TYPE"}
):
if (
not config.name
or not config.attrs
or not config.attrs.get("Spec", {}).get("Labels", {})
or not config.attrs.get("Spec", {}).get("Data", {})
):
for config in self.__client.configs.list(filters={"label": "bunkerweb.CONFIG_TYPE"}):
if not config.name or not config.attrs or not config.attrs.get("Spec", {}).get("Labels", {}) or not config.attrs.get("Spec", {}).get("Data", {}):
continue
config_type = config.attrs["Spec"]["Labels"]["bunkerweb.CONFIG_TYPE"]
@ -111,19 +96,13 @@ class SwarmController(Controller):
continue
config_site = ""
if "bunkerweb.CONFIG_SITE" in config.attrs["Spec"]["Labels"]:
if not self._is_service_present(
config.attrs["Spec"]["Labels"]["bunkerweb.CONFIG_SITE"]
):
if not self._is_service_present(config.attrs["Spec"]["Labels"]["bunkerweb.CONFIG_SITE"]):
self._logger.warning(
f"Ignoring config {config_name} because {config.attrs['Spec']['Labels']['bunkerweb.CONFIG_SITE']} doesn't exist",
)
continue
config_site = (
f"{config.attrs['Spec']['Labels']['bunkerweb.CONFIG_SITE']}/"
)
configs[config_type][f"{config_site}{config_name}"] = b64decode(
config.attrs["Spec"]["Data"]
)
config_site = f"{config.attrs['Spec']['Labels']['bunkerweb.CONFIG_SITE']}/"
configs[config_type][f"{config_site}{config_name}"] = b64decode(config.attrs["Spec"]["Data"])
return configs
def apply_config(self) -> bool:
@ -139,36 +118,27 @@ class SwarmController(Controller):
locked = False
error = False
try:
for _ in self.__client.events(
decode=True, filters={"type": event_type}
):
for _ in self.__client.events(decode=True, filters={"type": event_type}):
self.__internal_lock.acquire()
locked = True
try:
self._update_settings()
self._instances = self.get_instances()
self._services = self.get_services()
self._configs = self.get_configs()
if not self.update_needed(
self._instances, self._services, configs=self._configs
):
if not self.update_needed(self._instances, self._services, configs=self._configs):
self.__internal_lock.release()
locked = False
continue
self._logger.info(
f"Catched Swarm event ({event_type}), deploying new configuration ..."
)
self._logger.info(f"Caught Swarm event ({event_type}), deploying new configuration ...")
if not self.apply_config():
self._logger.error(
"Error while deploying new configuration"
)
self._logger.error("Error while deploying new configuration")
else:
self._logger.info(
"Successfully deployed new configuration 🚀",
)
except:
self._logger.error(
f"Exception while processing Swarm event ({event_type}) :\n{format_exc()}"
)
self._logger.error(f"Exception while processing Swarm event ({event_type}) :\n{format_exc()}")
self.__internal_lock.release()
locked = False
except:
@ -187,10 +157,7 @@ class SwarmController(Controller):
def process_events(self):
self._set_autoconf_load_db()
event_types = ("service", "config")
threads = [
Thread(target=self.__event, args=(event_type,))
for event_type in event_types
]
threads = [Thread(target=self.__event, args=(event_type,)) for event_type in event_types]
for thread in threads:
thread.start()
for thread in threads:

5
src/autoconf/main.py
View file

@ -7,10 +7,7 @@ from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from pathlib import Path
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("deps", "python"), ("utils",), ("api",), ("db",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)

4
src/bw/entrypoint.sh
View file

@ -16,7 +16,7 @@ log "ENTRYPOINT" "ℹ️" "Starting BunkerWeb v$(cat /usr/share/bunkerweb/VERSIO
# trap SIGTERM and SIGINT
function trap_exit() {
# shellcheck disable=SC2317
log "ENTRYPOINT" "ℹ️" "Catched stop operation, stopping nginx ..."
log "ENTRYPOINT" "ℹ️" "Caught stop operation, stopping nginx ..."
# shellcheck disable=SC2317
nginx -s stop
}
@ -25,7 +25,7 @@ trap "trap_exit" TERM INT QUIT
# trap SIGHUP
function trap_reload() {
# shellcheck disable=SC2317
log "ENTRYPOINT" "ℹ️" "Catched reload operation"
log "ENTRYPOINT" "ℹ️" "Caught reload operation"
# shellcheck disable=SC2317
if [ -f /var/run/bunkerweb/nginx.pid ] ; then
# shellcheck disable=SC2317

272
src/bw/loading/index.html
View file

File diff suppressed because one or more lines are too long

2
src/bw/lua/bunkerweb/plugin.lua
View file

@ -20,7 +20,7 @@ function plugin:initialize(id, ctx)
end
end
self.is_request = multisite
-- Store common objets
-- Store common objects
self.logger = logger:new(self.id)
local use_redis, err = utils.get_variable("USE_REDIS", false)
if not use_redis then

28
src/common/cli/CLI.py
View file

@ -64,17 +64,13 @@ class CLI(ApiCaller):
if redis_host:
redis_port = self.__variables.get("REDIS_PORT", "6379")
if not redis_port.isdigit():
self.__logger.error(
f"REDIS_PORT is not a valid port number: {redis_port}, defaulting to 6379"
)
self.__logger.error(f"REDIS_PORT is not a valid port number: {redis_port}, defaulting to 6379")
redis_port = "6379"
redis_port = int(redis_port)
redis_db = self.__variables.get("REDIS_DB", "0")
if not redis_db.isdigit():
self.__logger.error(
f"REDIS_DB is not a valid database number: {redis_db}, defaulting to 0"
)
self.__logger.error(f"REDIS_DB is not a valid database number: {redis_db}, defaulting to 0")
redis_db = "0"
redis_db = int(redis_db)
@ -83,18 +79,12 @@ class CLI(ApiCaller):
try:
redis_timeout = float(redis_timeout)
except ValueError:
self.__logger.error(
f"REDIS_TIMEOUT is not a valid timeout: {redis_timeout}, defaulting to 1000 ms"
)
self.__logger.error(f"REDIS_TIMEOUT is not a valid timeout: {redis_timeout}, defaulting to 1000 ms")
redis_timeout = 1000.0
redis_keepalive_pool = self.__variables.get(
"REDIS_KEEPALIVE_POOL", "10"
)
redis_keepalive_pool = self.__variables.get("REDIS_KEEPALIVE_POOL", "10")
if not redis_keepalive_pool.isdigit():
self.__logger.error(
f"REDIS_KEEPALIVE_POOL is not a valid number of connections: {redis_keepalive_pool}, defaulting to 10"
)
self.__logger.error(f"REDIS_KEEPALIVE_POOL is not a valid number of connections: {redis_keepalive_pool}, defaulting to 10")
redis_keepalive_pool = "10"
redis_keepalive_pool = int(redis_keepalive_pool)
@ -109,9 +99,7 @@ class CLI(ApiCaller):
ssl=self.__variables.get("REDIS_SSL", "no") == "yes",
)
else:
self.__logger.error(
"USE_REDIS is set to yes but REDIS_HOST is not set, disabling redis"
)
self.__logger.error("USE_REDIS is set to yes but REDIS_HOST is not set, disabling redis")
self.__use_redis = False
if not db_path.is_dir() or self.__integration not in (
@ -143,9 +131,7 @@ class CLI(ApiCaller):
return "autoconf"
elif integration_path.is_file():
return integration_path.read_text(encoding="utf-8").strip().lower()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(
encoding="utf-8"
):
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
return "docker"
return "linux"

9
src/common/cli/main.py
View file

@ -6,10 +6,7 @@ from os.path import join
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("deps", "python"), ("utils",), ("api",), ("db",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -25,9 +22,7 @@ if __name__ == "__main__":
subparsers = parser.add_subparsers(help="command", dest="command")
# Unban subparser
parser_unban = subparsers.add_parser(
"unban", help="remove a ban from the cache"
)
parser_unban = subparsers.add_parser("unban", help="remove a ban from the cache")
parser_unban.add_argument("ip", type=str, help="IP address to unban")
# Ban subparser

2
src/common/core/antibot/files/captcha.html
View file

@ -1,5 +1,5 @@
{-raw-}
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />

2
src/common/core/antibot/files/hcaptcha.html
View file

@ -1,5 +1,5 @@
{-raw-}
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />

2
src/common/core/antibot/files/javascript.html
View file

@ -1,5 +1,5 @@
{-raw-}
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />

2
src/common/core/antibot/files/recaptcha.html
View file

@ -1,5 +1,5 @@
{-raw-}
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />

2
src/common/core/antibot/files/turnstile.html
View file

@ -1,5 +1,5 @@
{-raw-}
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />

20
src/common/core/blacklist/jobs/blacklist-download.py
View file

@ -10,10 +10,7 @@ from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from typing import Tuple
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("deps", "python"), ("utils",), ("db",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -63,10 +60,7 @@ try:
# Multisite case
if getenv("MULTISITE", "no") == "yes":
for first_server in getenv("SERVER_NAME", "").split(" "):
if (
getenv(f"{first_server}_USE_BLACKLIST", getenv("USE_BLACKLIST", "yes"))
== "yes"
):
if getenv(f"{first_server}_USE_BLACKLIST", getenv("USE_BLACKLIST", "yes")) == "yes":
blacklist_activated = True
break
# Singlesite case
@ -133,7 +127,7 @@ try:
blacklist_path.joinpath(f"{kind}.list").unlink(missing_ok=True)
deleted, err = del_file_in_db(f"{kind}.list", db)
if not deleted:
logger.warning(f"Coudn't delete {kind}.list from cache : {err}")
logger.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all_fresh:
_exit(0)
@ -152,9 +146,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(
f"Got status code {resp.status_code}, skipping..."
)
logger.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -203,9 +195,7 @@ try:
status = 1
except:
status = 2
logger.error(
f"Exception while getting blacklist from {url} :\n{format_exc()}"
)
logger.error(f"Exception while getting blacklist from {url} :\n{format_exc()}")
except:
status = 2

5
src/common/core/bunkernet/jobs/bunkernet-data.py
View file

@ -32,10 +32,7 @@ try:
# Multisite case
if getenv("MULTISITE", "no") == "yes":
for first_server in getenv("SERVER_NAME", "").split(" "):
if (
getenv(f"{first_server}_USE_BUNKERNET", getenv("USE_BUNKERNET", "yes"))
== "yes"
):
if getenv(f"{first_server}_USE_BUNKERNET", getenv("USE_BUNKERNET", "yes")) == "yes":
bunkernet_activated = True
break
# Singlesite case

25
src/common/core/bunkernet/jobs/bunkernet-register.py
View file

@ -19,10 +19,10 @@ for deps_path in [
if deps_path not in sys_path:
sys_path.append(deps_path)
from bunkernet import register, ping, get_id
from bunkernet import register, ping
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import get_file_in_db, set_file_in_db, del_file_in_db
from jobs import get_file_in_db, set_file_in_db, del_file_in_db # type: ignore
logger = setup_logger("BUNKERNET", getenv("LOG_LEVEL", "INFO"))
exit_status = 0
@ -38,10 +38,7 @@ try:
servers = servers.split(" ")
for first_server in servers:
if (
getenv(f"{first_server}_USE_BUNKERNET", getenv("USE_BUNKERNET", "yes"))
== "yes"
):
if getenv(f"{first_server}_USE_BUNKERNET", getenv("USE_BUNKERNET", "yes")) == "yes":
bunkernet_activated = True
break
# Singlesite case
@ -73,9 +70,7 @@ try:
logger.info("Registering instance on BunkerNet API ...")
ok, status, data = register()
if not ok:
logger.error(
f"Error while sending register request to BunkerNet API : {data}"
)
logger.error(f"Error while sending register request to BunkerNet API : {data}")
_exit(2)
elif status == 429:
logger.warning(
@ -102,23 +97,17 @@ try:
)
_exit(2)
elif data.get("result", "ko") != "ok":
logger.error(
f"Received error from BunkerNet API while sending register request : {data.get('data', {})}"
)
logger.error(f"Received error from BunkerNet API while sending register request : {data.get('data', {})}")
_exit(2)
bunkernet_id = data["data"]
instance_id_path.write_text(bunkernet_id, encoding="utf-8")
registered = True
exit_status = 1
logger.info(
f"Successfully registered on BunkerNet API with instance id {data['data']}"
)
logger.info(f"Successfully registered on BunkerNet API with instance id {data['data']}")
else:
bunkernet_id = bunkernet_id or instance_id_path.read_bytes()
bunkernet_id = bunkernet_id.decode()
logger.info(
f"Already registered on BunkerNet API with instance id {bunkernet_id}"
)
logger.info(f"Already registered on BunkerNet API with instance id {bunkernet_id}")
sleep(1)

20
src/common/core/bunkernet/jobs/bunkernet.py
View file

@ -6,9 +6,7 @@ from requests import request as requests_request, ReadTimeout
from typing import Literal, Optional, Tuple, Union
def request(
method: Union[Literal["POST"], Literal["GET"]], url: str, _id: Optional[str] = None
) -> Tuple[bool, Optional[int], Union[str, dict]]:
def request(method: Union[Literal["POST"], Literal["GET"]], url: str, _id: Optional[str] = None) -> Tuple[bool, Optional[int], Union[str, dict]]:
data = {"integration": get_integration(), "version": get_version()}
headers = {"User-Agent": f"BunkerWeb/{get_version()}"}
if _id is not None:
@ -51,19 +49,11 @@ def data() -> Tuple[bool, Optional[int], Union[str, dict]]:
def get_id() -> str:
return (
Path(sep, "var", "cache", "bunkerweb", "bunkernet", "instance.id")
.read_text(encoding="utf-8")
.strip()
)
return Path(sep, "var", "cache", "bunkerweb", "bunkernet", "instance.id").read_text(encoding="utf-8").strip()
def get_version() -> str:
return (
Path(sep, "usr", "share", "bunkerweb", "VERSION")
.read_text(encoding="utf-8")
.strip()
)
return Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text(encoding="utf-8").strip()
def get_integration() -> str:
@ -78,9 +68,7 @@ def get_integration() -> str:
return "autoconf"
elif integration_path.is_file():
return integration_path.read_text(encoding="utf-8").strip().lower()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(
encoding="utf-8"
):
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
return "docker"
return "linux"

41
src/common/core/customcert/jobs/custom-cert.py
View file

@ -26,28 +26,20 @@ logger = setup_logger("CUSTOM-CERT", getenv("LOG_LEVEL", "INFO"))
db = None
def check_cert(
cert_path: str, key_path: str, first_server: Optional[str] = None
) -> bool:
def check_cert(cert_path: str, key_path: str, first_server: Optional[str] = None) -> bool:
try:
if not cert_path or not key_path:
logger.warning(
"Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates"
)
logger.warning("Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates")
return False
cert_path: Path = Path(normpath(cert_path))
key_path: Path = Path(normpath(key_path))
if not cert_path.is_file():
logger.warning(
f"Certificate file {cert_path} is not a valid file, ignoring the custom certificate"
)
logger.warning(f"Certificate file {cert_path} is not a valid file, ignoring the custom certificate")
return False
elif not key_path.is_file():
logger.warning(
f"Key file {key_path} is not a valid file, ignoring the custom certificate"
)
logger.warning(f"Key file {key_path} is not a valid file, ignoring the custom certificate")
return False
cert_cache_path = Path(
@ -66,9 +58,7 @@ def check_cert(
if old_hash == cert_hash:
return False
cached, err = cache_file(
cert_path, cert_cache_path, cert_hash, db, delete_file=False
)
cached, err = cache_file(cert_path, cert_cache_path, cert_hash, db, delete_file=False)
if not cached:
logger.error(f"Error while caching custom-cert cert.pem file : {err}")
@ -86,9 +76,7 @@ def check_cert(
key_hash = file_hash(key_path)
old_hash = cache_hash(key_cache_path, db)
if old_hash != key_hash:
cached, err = cache_file(
key_path, key_cache_path, key_hash, db, delete_file=False
)
cached, err = cache_file(key_path, key_cache_path, key_hash, db, delete_file=False)
if not cached:
logger.error(f"Error while caching custom-cert key.pem file : {err}")
@ -103,14 +91,10 @@ def check_cert(
status = 0
try:
Path(sep, "var", "cache", "bunkerweb", "customcert").mkdir(
parents=True, exist_ok=True
)
Path(sep, "var", "cache", "bunkerweb", "customcert").mkdir(parents=True, exist_ok=True)
if getenv("USE_CUSTOM_SSL", "no") == "yes" and getenv("SERVER_NAME", "") != "":
db = Database(
logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
cert_path = getenv("CUSTOM_SSL_CERT", "")
key_path = getenv("CUSTOM_SSL_KEY", "")
@ -131,16 +115,11 @@ try:
servers = servers.split(" ")
for first_server in servers:
if not first_server or (
getenv(f"{first_server}_USE_CUSTOM_SSL", getenv("USE_CUSTOM_SSL", "no"))
!= "yes"
):
if not first_server or (getenv(f"{first_server}_USE_CUSTOM_SSL", getenv("USE_CUSTOM_SSL", "no")) != "yes"):
continue
if not db:
db = Database(
logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
cert_path = getenv(f"{first_server}_CUSTOM_SSL_CERT", "")
key_path = getenv(f"{first_server}_CUSTOM_SSL_KEY", "")

2
src/common/core/errors/files/error.html
View file

@ -1,4 +1,4 @@
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />

20
src/common/core/greylist/jobs/greylist-download.py
View file

@ -10,10 +10,7 @@ from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from typing import Tuple
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("deps", "python"), ("utils",), ("db",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -63,10 +60,7 @@ try:
# Multisite case
if getenv("MULTISITE", "no") == "yes":
for first_server in getenv("SERVER_NAME", "").split(" "):
if (
getenv(f"{first_server}_USE_GREYLIST", getenv("USE_GREYLIST", "no"))
== "yes"
):
if getenv(f"{first_server}_USE_GREYLIST", getenv("USE_GREYLIST", "no")) == "yes":
greylist_activated = True
break
# Singlesite case
@ -117,7 +111,7 @@ try:
greylist_path.joinpath(f"{kind}.list").unlink(missing_ok=True)
deleted, err = del_file_in_db(f"{kind}.list", db)
if not deleted:
logger.warning(f"Coudn't delete {kind}.list from cache : {err}")
logger.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all_fresh:
_exit(0)
@ -136,9 +130,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(
f"Got status code {resp.status_code}, skipping..."
)
logger.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -187,9 +179,7 @@ try:
status = 1
except:
status = 2
logger.error(
f"Exception while getting greylist from {url} :\n{format_exc()}"
)
logger.error(f"Exception while getting greylist from {url} :\n{format_exc()}")
except:
status = 2

4
src/common/core/jobs/jobs/download-plugins.py
View file

@ -114,9 +114,7 @@ try:
with tar_open(fileobj=BytesIO(content), mode="r") as tar:
tar.extractall(path=temp_dir)
else:
logger.error(
f"Unknown file type for {plugin_url}, either zip or tar are supported, skipping..."
)
logger.error(f"Unknown file type for {plugin_url}, either zip or tar are supported, skipping...")
continue
except:
logger.error(

12
src/common/core/jobs/jobs/mmdb-asn.py
View file

@ -43,9 +43,7 @@ try:
with lock:
response = None
try:
response = get(
"https://db-ip.com/db/download/ip-to-asn-lite", timeout=5
)
response = get("https://db-ip.com/db/download/ip-to-asn-lite", timeout=5)
except RequestException:
logger.warning("Unable to check if asn.mmdb is the latest version")
@ -59,14 +57,10 @@ try:
_sha1.update(data)
if response.content.decode().find(_sha1.hexdigest()) != -1:
logger.info(
"asn.mmdb is already the latest version, skipping download..."
)
logger.info("asn.mmdb is already the latest version, skipping download...")
dl_mmdb = False
else:
logger.warning(
"Unable to check if asn.mmdb is the latest version, downloading it anyway..."
)
logger.warning("Unable to check if asn.mmdb is the latest version, downloading it anyway...")
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)

12
src/common/core/jobs/jobs/mmdb-country.py
View file

@ -43,9 +43,7 @@ try:
with lock:
response = None
try:
response = get(
"https://db-ip.com/db/download/ip-to-country-lite", timeout=5
)
response = get("https://db-ip.com/db/download/ip-to-country-lite", timeout=5)
except RequestException:
logger.warning("Unable to check if country.mmdb is the latest version")
@ -59,14 +57,10 @@ try:
_sha1.update(data)
if response.content.decode().find(_sha1.hexdigest()) != -1:
logger.info(
"country.mmdb is already the latest version, skipping download..."
)
logger.info("country.mmdb is already the latest version, skipping download...")
dl_mmdb = False
else:
logger.warning(
"Unable to check if country.mmdb is the latest version, downloading it anyway..."
)
logger.warning("Unable to check if country.mmdb is the latest version, downloading it anyway...")
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)

12
src/common/core/letsencrypt/jobs/certbot-auth.py
View file

@ -39,9 +39,7 @@ try:
bw_integration = "Autoconf"
elif integration_path.is_file():
bw_integration = integration_path.read_text(encoding="utf-8").strip()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(
encoding="utf-8"
):
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
bw_integration = "Docker"
token = getenv("CERTBOT_TOKEN", "")
@ -49,9 +47,7 @@ try:
# Cluster case
if bw_integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
db = Database(
logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
lock = Lock()
with lock:
@ -69,9 +65,7 @@ try:
)
if not sent:
status = 1
logger.error(
f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}"
)
logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
elif status != 200:
status = 1
logger.error(

16
src/common/core/letsencrypt/jobs/certbot-cleanup.py
View file

@ -39,18 +39,14 @@ try:
bw_integration = "Autoconf"
elif integration_path.is_file():
bw_integration = integration_path.read_text(encoding="utf-8").strip()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(
encoding="utf-8"
):
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
bw_integration = "Docker"
token = getenv("CERTBOT_TOKEN", "")
# Cluster case
if bw_integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
db = Database(
logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
lock = Lock()
with lock:
instances = db.get_instances()
@ -60,14 +56,10 @@ try:
f"http://{instance['hostname']}:{instance['port']}",
host=instance["server_name"],
)
sent, err, status, resp = api.request(
"DELETE", "/lets-encrypt/challenge", data={"token": token}
)
sent, err, status, resp = api.request("DELETE", "/lets-encrypt/challenge", data={"token": token})
if not sent:
status = 1
logger.error(
f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}"
)
logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}")
elif status != 200:
status = 1
logger.error(

32
src/common/core/letsencrypt/jobs/certbot-deploy.py
View file

@ -42,9 +42,7 @@ try:
bw_integration = "Autoconf"
elif integration_path.is_file():
bw_integration = integration_path.read_text(encoding="utf-8").strip()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(
encoding="utf-8"
):
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
bw_integration = "Docker"
token = getenv("CERTBOT_TOKEN", "")
@ -64,9 +62,7 @@ try:
tgz.seek(0, 0)
files = {"archive.tar.gz": tgz}
db = Database(
logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
lock = Lock()
with lock:
@ -77,19 +73,13 @@ try:
host = instance["server_name"]
api = API(endpoint, host=host)
sent, err, status, resp = api.request(
"POST", "/lets-encrypt/certificates", files=files
)
sent, err, status, resp = api.request("POST", "/lets-encrypt/certificates", files=files)
if not sent:
status = 1
logger.error(
f"Can't send API request to {api.endpoint}/lets-encrypt/certificates : {err}"
)
logger.error(f"Can't send API request to {api.endpoint}/lets-encrypt/certificates : {err}")
elif status != 200:
status = 1
logger.error(
f"Error while sending API request to {api.endpoint}/lets-encrypt/certificates : status = {resp['status']}, msg = {resp['msg']}"
)
logger.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/certificates : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(
f"Successfully sent API request to {api.endpoint}/lets-encrypt/certificates",
@ -97,18 +87,12 @@ try:
sent, err, status, resp = api.request("POST", "/reload")
if not sent:
status = 1
logger.error(
f"Can't send API request to {api.endpoint}/reload : {err}"
)
logger.error(f"Can't send API request to {api.endpoint}/reload : {err}")
elif status != 200:
status = 1
logger.error(
f"Error while sending API request to {api.endpoint}/reload : status = {resp['status']}, msg = {resp['msg']}"
)
logger.error(f"Error while sending API request to {api.endpoint}/reload : status = {resp['status']}, msg = {resp['msg']}")
else:
logger.info(
f"Successfully sent API request to {api.endpoint}/reload"
)
logger.info(f"Successfully sent API request to {api.endpoint}/reload")
# Linux case
else:
if (

42
src/common/core/letsencrypt/jobs/certbot-new.py
View file

@ -24,15 +24,13 @@ for deps_path in [
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import get_file_in_db, set_file_in_db
from jobs import get_file_in_db, set_file_in_db # type: ignore
logger = setup_logger("LETS-ENCRYPT.new", getenv("LOG_LEVEL", "INFO"))
status = 0
def certbot_new(
domains: str, email: str, letsencrypt_path: Path, letsencrypt_job_path: Path
) -> int:
def certbot_new(domains: str, email: str, letsencrypt_path: Path, letsencrypt_job_path: Path) -> int:
return run(
[
join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot"),
@ -60,8 +58,7 @@ def certbot_new(
+ (["--staging"] if getenv("USE_LETS_ENCRYPT_STAGING", "no") == "yes" else []),
stdin=DEVNULL,
stderr=STDOUT,
env=environ.copy()
| {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
).returncode
@ -81,8 +78,7 @@ def certbot_check_domains(domains: list[str], letsencrypt_path: Path) -> int:
stdout=PIPE,
stderr=STDOUT,
text=True,
env=environ.copy()
| {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
)
if proc.returncode != 0:
logger.error(f"Error while checking certificates :\n{proc.stdout}")
@ -91,10 +87,7 @@ def certbot_check_domains(domains: list[str], letsencrypt_path: Path) -> int:
needed_domains = set(domains)
for raw_domains in findall(r"^ Domains: (.*)$", proc.stdout, MULTILINE):
current_domains = raw_domains.split(" ")
if (
current_domains[0] == first_needed_domain
and set(current_domains) == needed_domains
):
if current_domains[0] == first_needed_domain and set(current_domains) == needed_domains:
return 1
return 0
@ -108,10 +101,7 @@ try:
use_letsencrypt = True
elif getenv("MULTISITE", "no") == "yes":
for first_server in getenv("SERVER_NAME", "").split(" "):
if (
first_server
and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes"
):
if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes":
use_letsencrypt = True
break
@ -123,12 +113,8 @@ try:
letsencrypt_path = Path(sep, "var", "cache", "bunkerweb", "letsencrypt")
letsencrypt_path.mkdir(parents=True, exist_ok=True)
letsencrypt_job_path = Path(
sep, "usr", "share", "bunkerweb", "core", "letsencrypt", "jobs"
)
Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(
parents=True, exist_ok=True
)
letsencrypt_job_path = Path(sep, "usr", "share", "bunkerweb", "core", "letsencrypt", "jobs")
Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(parents=True, exist_ok=True)
# Get env vars
bw_integration = "Linux"
@ -142,9 +128,7 @@ try:
bw_integration = "Autoconf"
elif integration_path.is_file():
bw_integration = integration_path.read_text(encoding="utf-8").strip()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(
encoding="utf-8"
):
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
bw_integration = "Docker"
# Extract letsencrypt folder if it exists in db
@ -209,9 +193,7 @@ try:
continue
else:
status = 1 if status == 0 else status
logger.info(
f"Certificate generation succeeded for domain(s) : {domains}"
)
logger.info(f"Certificate generation succeeded for domain(s) : {domains}")
# Singlesite case
elif getenv("AUTO_LETS_ENCRYPT", "no") == "yes" and getenv("SERVER_NAME"):
@ -243,9 +225,7 @@ try:
logger.error(f"Certificate generation failed for domain(s) : {domains}")
else:
status = 1
logger.info(
f"Certificate generation succeeded for domain(s) : {domains}"
)
logger.info(f"Certificate generation succeeded for domain(s) : {domains}")
# Put new folder in cache
bio = BytesIO()

24
src/common/core/letsencrypt/jobs/certbot-renew.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/python3
from os import _exit, environ, getenv, listdir, sep
from os import _exit, environ, getenv, sep
from os.path import join
from pathlib import Path
from subprocess import DEVNULL, STDOUT, run
@ -23,7 +23,7 @@ for deps_path in [
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from jobs import get_file_in_db, set_file_in_db
from jobs import get_file_in_db, set_file_in_db # type: ignore
def renew(domain: str, letsencrypt_path: Path) -> int:
@ -53,8 +53,7 @@ def renew(domain: str, letsencrypt_path: Path) -> int:
],
stdin=DEVNULL,
stderr=STDOUT,
env=environ.copy()
| {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
env=environ.copy() | {"PYTHONPATH": join(sep, "usr", "share", "bunkerweb", "deps", "python")},
check=False,
).returncode
@ -69,10 +68,7 @@ try:
use_letsencrypt = True
elif getenv("MULTISITE", "no") == "yes":
for first_server in getenv("SERVER_NAME", "").split(" "):
if (
first_server
and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes"
):
if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes":
use_letsencrypt = True
break
@ -83,9 +79,7 @@ try:
# Create directory if it doesn't exist
letsencrypt_path = Path(sep, "var", "cache", "bunkerweb", "letsencrypt")
letsencrypt_path.mkdir(parents=True, exist_ok=True)
Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(
parents=True, exist_ok=True
)
Path(sep, "var", "lib", "bunkerweb", "letsencrypt").mkdir(parents=True, exist_ok=True)
# Get env vars
bw_integration = "Linux"
@ -99,9 +93,7 @@ try:
bw_integration = "Autoconf"
elif integration_path.is_file():
bw_integration = integration_path.read_text(encoding="utf-8").strip()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(
encoding="utf-8"
):
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
bw_integration = "Docker"
# Extract letsencrypt folder if it exists in db
@ -134,9 +126,7 @@ try:
getenv("AUTO_LETS_ENCRYPT", "no"),
)
!= "yes"
or not letsencrypt_path.joinpath(
"etc", "live", first_server, "cert.pem"
).exists()
or not letsencrypt_path.joinpath("etc", "live", first_server, "cert.pem").exists()
):
continue

2
src/common/core/misc/files/default.html
View file

@ -1,4 +1,4 @@
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />

25
src/common/core/misc/jobs/default-server-cert.py
View file

@ -35,10 +35,7 @@ try:
"AUTO_LETS_ENCRYPT",
"GENERATE_SELF_SIGNED_SSL",
):
if (
getenv(f"{first_server}_{check_var}", getenv(check_var, "no"))
== "yes"
):
if getenv(f"{first_server}_{check_var}", getenv(check_var, "no")) == "yes":
need_default_cert = True
break
if need_default_cert:
@ -99,9 +96,7 @@ try:
"Successfully generated self-signed certificate for default server",
)
db = Database(
logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
cached, err = set_file_in_db(
"cert.pem",
@ -109,13 +104,9 @@ try:
db,
)
if not cached:
logger.error(
f"Error while saving default-server-cert cert.pem file to db cache : {err}"
)
logger.error(f"Error while saving default-server-cert cert.pem file to db cache : {err}")
else:
logger.info(
"Successfully saved default-server-cert cert.pem file to db cache"
)
logger.info("Successfully saved default-server-cert cert.pem file to db cache")
cached, err = set_file_in_db(
"cert.key",
@ -123,13 +114,9 @@ try:
db,
)
if not cached:
logger.error(
f"Error while saving default-server-cert cert.key file to db cache : {err}"
)
logger.error(f"Error while saving default-server-cert cert.key file to db cache : {err}")
else:
logger.info(
"Successfully saved default-server-cert cert.key file to db cache"
)
logger.info("Successfully saved default-server-cert cert.key file to db cache")
else:
logger.info(
"Skipping generation of self-signed certificate for default server (already present)",

4
src/common/core/misc/jobs/update-check.py
View file

@ -23,9 +23,7 @@ logger = setup_logger("UPDATE-CHECK", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
current_version = (
f"v{Path('/usr/share/bunkerweb/VERSION').read_text(encoding='utf-8').strip()}"
)
current_version = f"v{Path('/usr/share/bunkerweb/VERSION').read_text(encoding='utf-8').strip()}"
response = get(
"https://github.com/bunkerity/bunkerweb/releases/latest",

11
src/common/core/realip/jobs/realip-download.py
View file

@ -52,10 +52,7 @@ try:
servers = servers.split(" ")
for first_server in servers:
if (
getenv(f"{first_server}_USE_REAL_IP", getenv("USE_REAL_IP", "no"))
== "yes"
):
if getenv(f"{first_server}_USE_REAL_IP", getenv("USE_REAL_IP", "no")) == "yes":
realip_activated = True
break
@ -84,7 +81,7 @@ try:
tmp_realip_path.joinpath("combined.list").unlink(missing_ok=True)
deleted, err = del_file_in_db("combined.list", db)
if not deleted:
logger.warning(f"Coudn't delete combined.list from cache : {err}")
logger.warning(f"Couldn't delete combined.list from cache : {err}")
logger.info("RealIP list is already in cache, skipping download...")
_exit(0)
@ -118,9 +115,7 @@ try:
i += 1
except:
status = 2
logger.error(
f"Exception while getting RealIP list from {url} :\n{format_exc()}"
)
logger.error(f"Exception while getting RealIP list from {url} :\n{format_exc()}")
tmp_realip_path.joinpath("combined.list").write_bytes(content)

29
src/common/core/selfsigned/jobs/self-signed.py
View file

@ -34,9 +34,7 @@ lock = Lock()
status = 0
def generate_cert(
first_server: str, days: str, subj: str, self_signed_path: Path
) -> Tuple[bool, int]:
def generate_cert(first_server: str, days: str, subj: str, self_signed_path: Path) -> Tuple[bool, int]:
if self_signed_path.joinpath(f"{first_server}.pem").is_file():
if (
run(
@ -61,19 +59,10 @@ def generate_cert(
self_signed_path.joinpath(f"{first_server}.pem").read_bytes(),
default_backend(),
)
if sorted(
attribute.rfc4514_string() for attribute in certificate.subject
) != sorted(v for v in subj.split("/") if v):
logger.warning(
f"Subject of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ..."
)
elif (
certificate.not_valid_after - certificate.not_valid_before
!= timedelta(days=int(days))
):
logger.warning(
f"Expiration date of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ..."
)
if sorted(attribute.rfc4514_string() for attribute in certificate.subject) != sorted(v for v in subj.split("/") if v):
logger.warning(f"Subject of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ...")
elif certificate.not_valid_after - certificate.not_valid_before != timedelta(days=int(days)):
logger.warning(f"Expiration date of self-signed certificate for {first_server} is different from the one in the configuration, regenerating ...")
else:
return True, 0
@ -154,9 +143,7 @@ try:
continue
if not db:
db = Database(
logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
ret, ret_status = generate_cert(
first_server,
@ -174,9 +161,7 @@ try:
# Singlesite case
elif getenv("GENERATE_SELF_SIGNED_SSL", "no") == "yes" and getenv("SERVER_NAME"):
db = Database(
logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False
)
db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
first_server = getenv("SERVER_NAME", "").split(" ")[0]
ret, ret_status = generate_cert(

20
src/common/core/whitelist/jobs/whitelist-download.py
View file

@ -10,10 +10,7 @@ from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
from typing import Tuple
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("deps", "python"), ("utils",), ("db",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -63,10 +60,7 @@ try:
# Multisite case
if getenv("MULTISITE", "no") == "yes":
for first_server in getenv("SERVER_NAME", "").split(" "):
if (
getenv(f"{first_server}_USE_WHITELIST", getenv("USE_WHITELIST", "no"))
== "yes"
):
if getenv(f"{first_server}_USE_WHITELIST", getenv("USE_WHITELIST", "no")) == "yes":
whitelist_activated = True
break
# Singlesite case
@ -117,7 +111,7 @@ try:
whitelist_path.joinpath(f"{kind}.list").unlink(missing_ok=True)
deleted, err = del_file_in_db(f"{kind}.list", db)
if not deleted:
logger.warning(f"Coudn't delete {kind}.list from cache : {err}")
logger.warning(f"Couldn't delete {kind}.list from cache : {err}")
if all_fresh:
_exit(0)
@ -136,9 +130,7 @@ try:
resp = get(url, stream=True, timeout=10)
if resp.status_code != 200:
logger.warning(
f"Got status code {resp.status_code}, skipping..."
)
logger.warning(f"Got status code {resp.status_code}, skipping...")
continue
iterable = resp.iter_lines()
@ -187,9 +179,7 @@ try:
status = 1
except:
status = 2
logger.error(
f"Exception while getting whitelist from {url} :\n{format_exc()}"
)
logger.error(f"Exception while getting whitelist from {url} :\n{format_exc()}")
except:
status = 2

542
src/common/db/Database.py
View file

File diff suppressed because it is too large Load diff

24
src/common/db/model.py
View file

@ -16,9 +16,7 @@ from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy.schema import UniqueConstraint
CONTEXTS_ENUM = Enum("global", "multisite", name="contexts_enum")
SETTINGS_TYPES_ENUM = Enum(
"password", "text", "check", "select", name="settings_types_enum"
)
SETTINGS_TYPES_ENUM = Enum("password", "text", "check", "select", name="settings_types_enum")
METHODS_ENUM = Enum("ui", "scheduler", "autoconf", "manual", name="methods_enum")
SCHEDULES_ENUM = Enum("once", "minute", "hour", "day", "week", name="schedules_enum")
CUSTOM_CONFIGS_TYPES_ENUM = Enum(
@ -57,9 +55,7 @@ class Plugins(Base):
data = Column(LargeBinary(length=(2**32) - 1), nullable=True)
checksum = Column(String(128), nullable=True)
settings = relationship(
"Settings", back_populates="plugin", cascade="all, delete-orphan"
)
settings = relationship("Settings", back_populates="plugin", cascade="all, delete-orphan")
jobs = relationship("Jobs", back_populates="plugin", cascade="all, delete-orphan")
pages = relationship("Plugin_pages", back_populates="plugin", cascade="all")
@ -88,12 +84,8 @@ class Settings(Base):
multiple = Column(String(128), nullable=True)
selects = relationship("Selects", back_populates="setting", cascade="all")
services = relationship(
"Services_settings", back_populates="setting", cascade="all"
)
global_value = relationship(
"Global_values", back_populates="setting", cascade="all"
)
services = relationship("Services_settings", back_populates="setting", cascade="all")
global_value = relationship("Global_values", back_populates="setting", cascade="all")
plugin = relationship("Plugins", back_populates="settings")
@ -118,12 +110,8 @@ class Services(Base):
id = Column(String(64), primary_key=True)
method = Column(METHODS_ENUM, nullable=False)
settings = relationship(
"Services_settings", back_populates="service", cascade="all"
)
custom_configs = relationship(
"Custom_configs", back_populates="service", cascade="all"
)
settings = relationship("Services_settings", back_populates="service", cascade="all")
custom_configs = relationship("Custom_configs", back_populates="service", cascade="all")
jobs_cache = relationship("Jobs_cache", back_populates="service", cascade="all")

129
src/common/db/requirements.txt
View file

@ -83,71 +83,70 @@ cryptography==41.0.4 \
--hash=sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb \
--hash=sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f
# via -r requirements.in
greenlet==2.0.2 \
--hash=sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a \
--hash=sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a \
--hash=sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1 \
--hash=sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43 \
--hash=sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33 \
--hash=sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8 \
--hash=sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088 \
--hash=sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca \
--hash=sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343 \
--hash=sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645 \
--hash=sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db \
--hash=sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df \
--hash=sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3 \
--hash=sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86 \
--hash=sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2 \
--hash=sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a \
--hash=sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf \
--hash=sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7 \
--hash=sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394 \
--hash=sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40 \
--hash=sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3 \
--hash=sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6 \
--hash=sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74 \
--hash=sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0 \
--hash=sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3 \
--hash=sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91 \
--hash=sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5 \
--hash=sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9 \
--hash=sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417 \
--hash=sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8 \
--hash=sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b \
--hash=sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6 \
--hash=sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb \
--hash=sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73 \
--hash=sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b \
--hash=sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df \
--hash=sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9 \
--hash=sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f \
--hash=sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0 \
--hash=sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857 \
--hash=sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a \
--hash=sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249 \
--hash=sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30 \
--hash=sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292 \
--hash=sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b \
--hash=sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d \
--hash=sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b \
--hash=sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c \
--hash=sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca \
--hash=sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7 \
--hash=sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75 \
--hash=sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae \
--hash=sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47 \
--hash=sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b \
--hash=sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470 \
--hash=sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c \
--hash=sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564 \
--hash=sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9 \
--hash=sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099 \
--hash=sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0 \
--hash=sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5 \
--hash=sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19 \
--hash=sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1 \
--hash=sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526
greenlet==3.0.0 \
--hash=sha256:02a807b2a58d5cdebb07050efe3d7deaf915468d112dfcf5e426d0564aa3aa4a \
--hash=sha256:0b72b802496cccbd9b31acea72b6f87e7771ccfd7f7927437d592e5c92ed703c \
--hash=sha256:0d3f83ffb18dc57243e0151331e3c383b05e5b6c5029ac29f754745c800f8ed9 \
--hash=sha256:10b5582744abd9858947d163843d323d0b67be9432db50f8bf83031032bc218d \
--hash=sha256:123910c58234a8d40eaab595bc56a5ae49bdd90122dde5bdc012c20595a94c14 \
--hash=sha256:19834e3f91f485442adc1ee440171ec5d9a4840a1f7bd5ed97833544719ce10b \
--hash=sha256:1d363666acc21d2c204dd8705c0e0457d7b2ee7a76cb16ffc099d6799744ac99 \
--hash=sha256:211ef8d174601b80e01436f4e6905aca341b15a566f35a10dd8d1e93f5dbb3b7 \
--hash=sha256:269d06fa0f9624455ce08ae0179430eea61085e3cf6457f05982b37fd2cefe17 \
--hash=sha256:2e7dcdfad252f2ca83c685b0fa9fba00e4d8f243b73839229d56ee3d9d219314 \
--hash=sha256:334ef6ed8337bd0b58bb0ae4f7f2dcc84c9f116e474bb4ec250a8bb9bd797a66 \
--hash=sha256:343675e0da2f3c69d3fb1e894ba0a1acf58f481f3b9372ce1eb465ef93cf6fed \
--hash=sha256:37f60b3a42d8b5499be910d1267b24355c495064f271cfe74bf28b17b099133c \
--hash=sha256:38ad562a104cd41e9d4644f46ea37167b93190c6d5e4048fcc4b80d34ecb278f \
--hash=sha256:3c0d36f5adc6e6100aedbc976d7428a9f7194ea79911aa4bf471f44ee13a9464 \
--hash=sha256:3fd2b18432e7298fcbec3d39e1a0aa91ae9ea1c93356ec089421fabc3651572b \
--hash=sha256:4a1a6244ff96343e9994e37e5b4839f09a0207d35ef6134dce5c20d260d0302c \
--hash=sha256:4cd83fb8d8e17633ad534d9ac93719ef8937568d730ef07ac3a98cb520fd93e4 \
--hash=sha256:527cd90ba3d8d7ae7dceb06fda619895768a46a1b4e423bdb24c1969823b8362 \
--hash=sha256:553d6fb2324e7f4f0899e5ad2c427a4579ed4873f42124beba763f16032959af \
--hash=sha256:56867a3b3cf26dc8a0beecdb4459c59f4c47cdd5424618c08515f682e1d46692 \
--hash=sha256:621fcb346141ae08cb95424ebfc5b014361621b8132c48e538e34c3c93ac7365 \
--hash=sha256:63acdc34c9cde42a6534518e32ce55c30f932b473c62c235a466469a710bfbf9 \
--hash=sha256:6512592cc49b2c6d9b19fbaa0312124cd4c4c8a90d28473f86f92685cc5fef8e \
--hash=sha256:6672fdde0fd1a60b44fb1751a7779c6db487e42b0cc65e7caa6aa686874e79fb \
--hash=sha256:6a5b2d4cdaf1c71057ff823a19d850ed5c6c2d3686cb71f73ae4d6382aaa7a06 \
--hash=sha256:6a68d670c8f89ff65c82b936275369e532772eebc027c3be68c6b87ad05ca695 \
--hash=sha256:6bb36985f606a7c49916eff74ab99399cdfd09241c375d5a820bb855dfb4af9f \
--hash=sha256:73b2f1922a39d5d59cc0e597987300df3396b148a9bd10b76a058a2f2772fc04 \
--hash=sha256:7709fd7bb02b31908dc8fd35bfd0a29fc24681d5cc9ac1d64ad07f8d2b7db62f \
--hash=sha256:8060b32d8586e912a7b7dac2d15b28dbbd63a174ab32f5bc6d107a1c4143f40b \
--hash=sha256:80dcd3c938cbcac986c5c92779db8e8ce51a89a849c135172c88ecbdc8c056b7 \
--hash=sha256:813720bd57e193391dfe26f4871186cf460848b83df7e23e6bef698a7624b4c9 \
--hash=sha256:831d6f35037cf18ca5e80a737a27d822d87cd922521d18ed3dbc8a6967be50ce \
--hash=sha256:871b0a8835f9e9d461b7fdaa1b57e3492dd45398e87324c047469ce2fc9f516c \
--hash=sha256:952256c2bc5b4ee8df8dfc54fc4de330970bf5d79253c863fb5e6761f00dda35 \
--hash=sha256:96d9ea57292f636ec851a9bb961a5cc0f9976900e16e5d5647f19aa36ba6366b \
--hash=sha256:9a812224a5fb17a538207e8cf8e86f517df2080c8ee0f8c1ed2bdaccd18f38f4 \
--hash=sha256:9adbd8ecf097e34ada8efde9b6fec4dd2a903b1e98037adf72d12993a1c80b51 \
--hash=sha256:9de687479faec7db5b198cc365bc34addd256b0028956501f4d4d5e9ca2e240a \
--hash=sha256:a048293392d4e058298710a54dfaefcefdf49d287cd33fb1f7d63d55426e4355 \
--hash=sha256:aa15a2ec737cb609ed48902b45c5e4ff6044feb5dcdfcf6fa8482379190330d7 \
--hash=sha256:abe1ef3d780de56defd0c77c5ba95e152f4e4c4e12d7e11dd8447d338b85a625 \
--hash=sha256:ad6fb737e46b8bd63156b8f59ba6cdef46fe2b7db0c5804388a2d0519b8ddb99 \
--hash=sha256:b1660a15a446206c8545edc292ab5c48b91ff732f91b3d3b30d9a915d5ec4779 \
--hash=sha256:b505fcfc26f4148551826a96f7317e02c400665fa0883fe505d4fcaab1dabfdd \
--hash=sha256:b822fab253ac0f330ee807e7485769e3ac85d5eef827ca224feaaefa462dc0d0 \
--hash=sha256:bdd696947cd695924aecb3870660b7545a19851f93b9d327ef8236bfc49be705 \
--hash=sha256:bdfaeecf8cc705d35d8e6de324bf58427d7eafb55f67050d8f28053a3d57118c \
--hash=sha256:be557119bf467d37a8099d91fbf11b2de5eb1fd5fc5b91598407574848dc910f \
--hash=sha256:c3692ecf3fe754c8c0f2c95ff19626584459eab110eaab66413b1e7425cd84e9 \
--hash=sha256:c6b5ce7f40f0e2f8b88c28e6691ca6806814157ff05e794cdd161be928550f4c \
--hash=sha256:c94e4e924d09b5a3e37b853fe5924a95eac058cb6f6fb437ebb588b7eda79870 \
--hash=sha256:cc3e2679ea13b4de79bdc44b25a0c4fcd5e94e21b8f290791744ac42d34a0353 \
--hash=sha256:d1e22c22f7826096ad503e9bb681b05b8c1f5a8138469b255eb91f26a76634f2 \
--hash=sha256:d5539f6da3418c3dc002739cb2bb8d169056aa66e0c83f6bacae0cd3ac26b423 \
--hash=sha256:d55db1db455c59b46f794346efce896e754b8942817f46a1bada2d29446e305a \
--hash=sha256:e09dea87cc91aea5500262993cbd484b41edf8af74f976719dd83fe724644cd6 \
--hash=sha256:e52a712c38e5fb4fd68e00dc3caf00b60cb65634d50e32281a9d6431b33b4af1 \
--hash=sha256:e693e759e172fa1c2c90d35dea4acbdd1d609b6936115d3739148d5e4cd11947 \
--hash=sha256:ecf94aa539e97a8411b5ea52fc6ccd8371be9550c4041011a091eb8b3ca1d810 \
--hash=sha256:f351479a6914fd81a55c8e68963609f792d9b067fb8a60a042c585a621e0de4f \
--hash=sha256:f47932c434a3c8d3c86d865443fadc1fbf574e9b11d6650b656e602b1797908a
# via sqlalchemy
psycopg2-binary==2.9.8 \
--hash=sha256:01f9731761f711e42459f87bd2ad5d744b9773b5dd05446f3b579a0f077e78e3 \

30
src/common/gen/Configurator.py
View file

@ -57,14 +57,10 @@ class Configurator:
def get_settings(self) -> Dict[str, Any]:
return self.__settings
def get_plugins(
self, _type: Union[Literal["core"], Literal["external"]]
) -> List[Dict[str, Any]]:
def get_plugins(self, _type: Union[Literal["core"], Literal["external"]]) -> List[Dict[str, Any]]:
return self.__core_plugins if _type == "core" else self.__external_plugins
def get_plugins_settings(
self, _type: Union[Literal["core"], Literal["external"]]
) -> Dict[str, Any]:
def get_plugins_settings(self, _type: Union[Literal["core"], Literal["external"]]) -> Dict[str, Any]:
if _type == "core":
plugins = self.__core_plugins
else:
@ -77,7 +73,7 @@ class Configurator:
return plugins_settings
def __map_servers(self) -> Dict[str, List[str]]:
if not self.__multisite or not "SERVER_NAME" in self.__variables:
if not self.__multisite or "SERVER_NAME" not in self.__variables:
return {}
servers = {}
for server_name in self.__variables["SERVER_NAME"].strip().split(" "):
@ -96,11 +92,7 @@ class Configurator:
f"Ignoring {server_name}_SERVER_NAME because regex is not valid",
)
else:
names = (
self.__variables[f"{server_name}_SERVER_NAME"]
.strip()
.split(" ")
)
names = self.__variables[f"{server_name}_SERVER_NAME"].strip().split(" ")
servers[server_name] = names
return servers
@ -132,9 +124,7 @@ class Configurator:
if _type == "external":
plugin_content = BytesIO()
with tar_open(
fileobj=plugin_content, mode="w:gz", compresslevel=9
) as tar:
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(
dirname(file),
arcname=basename(dirname(file)),
@ -170,10 +160,10 @@ class Configurator:
lines = f.readlines()
for line in lines:
line = line.strip()
if not line or line.startswith("#") or not "=" in line:
if not line or line.startswith("#") or "=" not in line:
continue
splitted = line.split("=", 1)
variables[splitted[0]] = splitted[1]
split = line.split("=", 1)
variables[split[0]] = split[1]
return variables
def get_config(self) -> Dict[str, Any]:
@ -268,9 +258,7 @@ class Configurator:
if variable in target:
return target, variable
for real_var, settings in target.items():
if "multiple" in settings and re_search(
f"^{real_var}_[0-9]+$", variable
):
if "multiple" in settings and re_search(f"^{real_var}_[0-9]+$", variable):
return target, real_var
return None, variable

16
src/common/gen/Templator.py
View file

@ -63,27 +63,19 @@ class Templator:
templates.append(template)
return templates
def __write_config(
self, subpath: Optional[str] = None, config: Optional[Dict[str, Any]] = None
):
def __write_config(self, subpath: Optional[str] = None, config: Optional[Dict[str, Any]] = None):
real_path = Path(self.__output, subpath or "", "variables.env")
real_path.parent.mkdir(parents=True, exist_ok=True)
real_path.write_text(
"\n".join(f"{k}={v}" for k, v in (config or self.__config).items())
)
real_path.write_text("\n".join(f"{k}={v}" for k, v in (config or self.__config).items()))
def __render_global(self):
self.__write_config()
templates = self.__find_templates(
["global", "http", "stream", "default-server-http"]
)
templates = self.__find_templates(["global", "http", "stream", "default-server-http"])
for template in templates:
self.__render_template(template)
def __render_server(self, server: str):
templates = self.__find_templates(
["modsec", "modsec-crs", "server-http", "server-stream"]
)
templates = self.__find_templates(["modsec", "modsec-crs", "server-http", "server-stream"])
if self.__config.get("MULTISITE", "no") == "yes":
config = self.__config.copy()
for variable, value in self.__config.items():

14
src/common/gen/main.py
View file

@ -12,10 +12,7 @@ from time import sleep
from traceback import format_exc
from typing import Any, Dict
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("deps", "python"), ("utils",), ("api",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -72,9 +69,7 @@ if __name__ == "__main__":
type=str,
help="path to the file containing environment variables",
)
parser.add_argument(
"--no-linux-reload", action="store_true", help="disable linux reload"
)
parser.add_argument("--no-linux-reload", action="store_true", help="disable linux reload")
args = parser.parse_args()
settings_path = Path(normpath(args.settings))
@ -183,10 +178,7 @@ if __name__ == "__main__":
)
templator.render()
if (
integration not in ("Autoconf", "Swarm", "Kubernetes", "Docker")
and not args.no_linux_reload
):
if integration not in ("Autoconf", "Swarm", "Kubernetes", "Docker") and not args.no_linux_reload:
retries = 0
while not Path(sep, "var", "run", "bunkerweb", "nginx.pid").exists():
if retries == 5:

6
src/common/gen/requirements.txt
View file

@ -298,9 +298,9 @@ six==1.16.0 \
# via
# kubernetes
# python-dateutil
urllib3==1.26.16 \
--hash=sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f \
--hash=sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14
urllib3==1.26.17 \
--hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \
--hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b
# via
# docker
# kubernetes

109
src/common/gen/save_config.py
View file

@ -10,10 +10,7 @@ from time import sleep
from traceback import format_exc
from typing import Any
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("deps", "python"), ("utils",), ("api",), ("db",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -24,9 +21,7 @@ from Database import Database # type: ignore
from Configurator import Configurator
from API import API # type: ignore
custom_confs_rx = re_compile(
r"^([0-9a-z\.-]*)_?CUSTOM_CONF_(HTTP|SERVER_STREAM|STREAM|DEFAULT_SERVER_HTTP|SERVER_HTTP|MODSEC_CRS|MODSEC)_(.+)$"
)
custom_confs_rx = re_compile(r"^([0-9a-z\.-]*)_?CUSTOM_CONF_(HTTP|SERVER_STREAM|STREAM|DEFAULT_SERVER_HTTP|SERVER_HTTP|MODSEC_CRS|MODSEC)_(.+)$")
def get_instance_configs_and_apis(instance: Any, db, _type="Docker"):
@ -36,17 +31,13 @@ def get_instance_configs_and_apis(instance: Any, db, _type="Docker"):
custom_confs = []
apis = []
for var in (
instance.attrs["Config"]["Env"]
if _type == "Docker"
else instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]
):
splitted = var.split("=", 1)
if custom_confs_rx.match(splitted[0]):
custom_conf = custom_confs_rx.search(splitted[0]).groups()
for var in instance.attrs["Config"]["Env"] if _type == "Docker" else instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
split = var.split("=", 1)
if custom_confs_rx.match(split[0]):
custom_conf = custom_confs_rx.search(split[0]).groups()
custom_confs.append(
{
"value": f"# CREATED BY ENV\n{splitted[1]}",
"value": f"# CREATED BY ENV\n{split[1]}",
"exploded": (
custom_conf[0],
custom_conf[1],
@ -54,18 +45,16 @@ def get_instance_configs_and_apis(instance: Any, db, _type="Docker"):
),
}
)
logger.info(
f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}"
)
logger.info(f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}")
else:
tmp_config[splitted[0]] = splitted[1]
tmp_config[split[0]] = split[1]
if not db and splitted[0] == "DATABASE_URI":
db = Database(logger, sqlalchemy_string=splitted[1], pool=False)
elif splitted[0] == "API_HTTP_PORT":
api_http_port = splitted[1]
elif splitted[0] == "API_SERVER_NAME":
api_server_name = splitted[1]
if not db and split[0] == "DATABASE_URI":
db = Database(logger, sqlalchemy_string=split[1], pool=False)
elif split[0] == "API_HTTP_PORT":
api_http_port = split[1]
elif split[0] == "API_SERVER_NAME":
api_server_name = split[1]
apis.append(
API(
@ -168,9 +157,7 @@ if __name__ == "__main__":
# Check existences and permissions
logger.info("Checking arguments ...")
files = [settings_path] + (
[Path(normpath(args.variables))] if args.variables else []
)
files = [settings_path] + ([Path(normpath(args.variables))] if args.variables else [])
paths_rx = [core_path, plugins_path]
for file in files:
if not file.is_file():
@ -217,19 +204,13 @@ if __name__ == "__main__":
),
}
)
logger.info(
f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}"
)
logger.info(f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}")
db = Database(logger, config_files.get("DATABASE_URI", None), pool=False)
else:
docker_client = DockerClient(
base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
)
docker_client = DockerClient(base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock"))
while not docker_client.containers.list(
filters={"label": "bunkerweb.INSTANCE"}
):
while not docker_client.containers.list(filters={"label": "bunkerweb.INSTANCE"}):
logger.info("Waiting for BunkerWeb instance ...")
sleep(5)
@ -239,16 +220,14 @@ if __name__ == "__main__":
custom_confs = []
apis = []
for instance in docker_client.containers.list(
filters={"label": "bunkerweb.INSTANCE"}
):
for instance in docker_client.containers.list(filters={"label": "bunkerweb.INSTANCE"}):
for var in instance.attrs["Config"]["Env"]:
splitted = var.split("=", 1)
if custom_confs_rx.match(splitted[0]):
custom_conf = custom_confs_rx.search(splitted[0]).groups()
split = var.split("=", 1)
if custom_confs_rx.match(split[0]):
custom_conf = custom_confs_rx.search(split[0]).groups()
custom_confs.append(
{
"value": f"# CREATED BY ENV\n{splitted[1]}",
"value": f"# CREATED BY ENV\n{split[1]}",
"exploded": (
custom_conf[0],
custom_conf[1],
@ -256,20 +235,16 @@ if __name__ == "__main__":
),
}
)
logger.info(
f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}"
)
logger.info(f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}")
else:
tmp_config[splitted[0]] = splitted[1]
tmp_config[split[0]] = split[1]
if not db and splitted[0] == "DATABASE_URI":
db = Database(
logger, sqlalchemy_string=splitted[1], pool=False
)
elif splitted[0] == "API_HTTP_PORT":
api_http_port = splitted[1]
elif splitted[0] == "API_SERVER_NAME":
api_server_name = splitted[1]
if not db and split[0] == "DATABASE_URI":
db = Database(logger, sqlalchemy_string=split[1], pool=False)
elif split[0] == "API_HTTP_PORT":
api_http_port = split[1]
elif split[0] == "API_SERVER_NAME":
api_server_name = split[1]
apis.append(
API(
@ -319,9 +294,7 @@ if __name__ == "__main__":
logger.info("Database tables initialized")
err = db.initialize_db(
version=Path(sep, "usr", "share", "bunkerweb", "VERSION")
.read_text()
.strip(),
version=Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text().strip(),
integration=integration,
)
@ -344,9 +317,7 @@ if __name__ == "__main__":
err = db.save_config(config_files, args.method, changed=False)
if err:
logger.warning(
f"Couldn't save config to database : {err}, config may not work as expected"
)
logger.warning(f"Couldn't save config to database : {err}, config may not work as expected")
else:
changes.append("config")
logger.info("Config successfully saved to database")
@ -355,9 +326,7 @@ if __name__ == "__main__":
err1 = db.save_custom_configs(custom_confs, args.method, changed=False)
if err1:
logger.warning(
f"Couldn't save custom configs to database : {err1}, custom configs may not work as expected"
)
logger.warning(f"Couldn't save custom configs to database : {err1}, custom configs may not work as expected")
else:
changes.append("custom_configs")
logger.info("Custom configs successfully saved to database")
@ -377,9 +346,7 @@ if __name__ == "__main__":
else:
if "instances" not in changes:
changes.append("instances")
logger.info(
f"Instance {endpoint_data[0]} successfully saved to database"
)
logger.info(f"Instance {endpoint_data[0]} successfully saved to database")
else:
err = db.add_instance(
"127.0.0.1",
@ -397,9 +364,7 @@ if __name__ == "__main__":
# update changes in db
ret = db.checked_changes(changes, value=True)
if ret:
logger.error(
f"An error occurred when setting the changes to checked in the database : {ret}"
)
logger.error(f"An error occurred when setting the changes to checked in the database : {ret}")
except SystemExit as e:
sys_exit(e.code)
except:

33
src/common/utils/ApiCaller.py
View file

@ -7,10 +7,7 @@ from sys import path as sys_path
from tarfile import open as tar_open
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("deps", "python"), ("utils",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -46,10 +43,7 @@ class ApiCaller:
config.load_incluster_config()
corev1 = kube_client.CoreV1Api()
for pod in corev1.list_pod_for_all_namespaces(watch=False).items:
if (
pod.metadata.annotations != None
and "bunkerweb.io/INSTANCE" in pod.metadata.annotations
):
if pod.metadata.annotations is not None and "bunkerweb.io/INSTANCE" in pod.metadata.annotations:
api_http_port = None
api_server_name = None
@ -66,20 +60,14 @@ class ApiCaller:
)
)
else:
docker_client = DockerClient(
base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
)
docker_client = DockerClient(base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock"))
if bw_integration == "Swarm":
for instance in docker_client.services.list(
filters={"label": "bunkerweb.INSTANCE"}
):
for instance in docker_client.services.list(filters={"label": "bunkerweb.INSTANCE"}):
api_http_port = None
api_server_name = None
for var in instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"][
"Env"
]:
for var in instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
if var.startswith("API_HTTP_PORT="):
api_http_port = var.replace("API_HTTP_PORT=", "", 1)
elif var.startswith("API_SERVER_NAME="):
@ -89,15 +77,12 @@ class ApiCaller:
self.__apis.append(
API(
f"http://{instance.name}.{task['NodeID']}.{task['ID']}:{api_http_port or getenv('API_HTTP_PORT', '5000')}",
host=api_server_name
or getenv("API_SERVER_NAME", "bwapi"),
host=api_server_name or getenv("API_SERVER_NAME", "bwapi"),
)
)
return
for instance in docker_client.containers.list(
filters={"label": "bunkerweb.INSTANCE"}
):
for instance in docker_client.containers.list(filters={"label": "bunkerweb.INSTANCE"}):
api_http_port = None
api_server_name = None
@ -160,9 +145,7 @@ class ApiCaller:
def send_files(self, path: str, url: str) -> bool:
ret = True
with BytesIO() as tgz:
with tar_open(
mode="w:gz", fileobj=tgz, dereference=True, compresslevel=3
) as tf:
with tar_open(mode="w:gz", fileobj=tgz, dereference=True, compresslevel=3) as tf:
tf.add(path, arcname=".")
tgz.seek(0, 0)
files = {"archive.tar.gz": tgz}

34
src/common/utils/ConfigCaller.py
View file

@ -15,18 +15,10 @@ from logger import setup_logger
class ConfigCaller:
def __init__(self):
self.__logger = setup_logger("Config", "INFO")
self._settings = loads(
Path(sep, "usr", "share", "bunkerweb", "settings.json").read_text(
encoding="utf-8"
)
)
for plugin in glob(
join(sep, "usr", "share", "bunkerweb", "core", "*", "plugin.json")
) + glob(join(sep, "etc", "bunkerweb", "plugins", "*", "plugin.json")):
self._settings = loads(Path(sep, "usr", "share", "bunkerweb", "settings.json").read_text(encoding="utf-8"))
for plugin in glob(join(sep, "usr", "share", "bunkerweb", "core", "*", "plugin.json")) + glob(join(sep, "etc", "bunkerweb", "plugins", "*", "plugin.json")):
try:
self._settings.update(
loads(Path(plugin).read_text(encoding="utf-8"))["settings"]
)
self._settings.update(loads(Path(plugin).read_text(encoding="utf-8"))["settings"])
except KeyError:
self.__logger.error(
f'Error while loading plugin metadata file at {plugin} : missing "settings" key',
@ -39,23 +31,15 @@ class ConfigCaller:
def _is_setting(self, setting) -> bool:
return setting in self._settings
def _is_setting_context(
self, setting: str, context: Union[Literal["global"], Literal["multisite"]]
) -> bool:
def _is_setting_context(self, setting: str, context: Union[Literal["global"], Literal["multisite"]]) -> bool:
if self._is_setting(setting):
return self._settings[setting]["context"] == context
elif match(r"^.+_\d+$", setting):
multiple_setting = "_".join(setting.split("_")[:-1])
return (
self._is_setting(multiple_setting)
and self._settings[multiple_setting]["context"] == context
and "multiple" in self._settings[multiple_setting]
)
return self._is_setting(multiple_setting) and self._settings[multiple_setting]["context"] == context and "multiple" in self._settings[multiple_setting]
return False
def _full_env(
self, env_instances: Dict[str, Any], env_services: Dict[str, Any]
) -> Dict[str, Any]:
def _full_env(self, env_instances: Dict[str, Any], env_services: Dict[str, Any]) -> Dict[str, Any]:
full_env = {}
# Fill with default values
for k, v in self._settings.items():
@ -63,11 +47,7 @@ class ConfigCaller:
# Replace with instances values
for k, v in env_instances.items():
full_env[k] = v
if (
not self._is_setting_context(k, "global")
and env_instances.get("MULTISITE", "no") == "yes"
and env_instances.get("SERVER_NAME", "") != ""
):
if not self._is_setting_context(k, "global") and env_instances.get("MULTISITE", "no") == "yes" and env_instances.get("SERVER_NAME", "") != "":
for server_name in env_instances["SERVER_NAME"].split(" "):
full_env[f"{server_name}_{k}"] = v
# Replace with services values

15
src/common/utils/jobs.py
View file

@ -70,9 +70,7 @@ def is_cached_file(
return is_cached and cached_file
def get_file_in_db(
file: Union[str, Path], db, *, job_name: Optional[str] = None
) -> Optional[bytes]:
def get_file_in_db(file: Union[str, Path], db, *, job_name: Optional[str] = None) -> Optional[bytes]:
cached_file = db.get_job_cache_file(
job_name or basename(getsourcefile(_getframe(1))).replace(".py", ""),
normpath(file),
@ -98,8 +96,7 @@ def set_file_in_db(
service_id,
name,
content,
job_name=job_name
or basename(getsourcefile(_getframe(1))).replace(".py", ""),
job_name=job_name or basename(getsourcefile(_getframe(1))).replace(".py", ""),
checksum=checksum,
)
@ -113,9 +110,7 @@ def set_file_in_db(
def del_file_in_db(name: str, db) -> Tuple[bool, str]:
ret, err = True, "success"
try:
db.delete_job_cache(
name, job_name=basename(getsourcefile(_getframe(1))).replace(".py", "")
)
db.delete_job_cache(name, job_name=basename(getsourcefile(_getframe(1))).replace(".py", ""))
except:
return False, f"exception :\n{format_exc()}"
return ret, err
@ -145,9 +140,7 @@ def bytes_hash(bio: BufferedReader) -> str:
def cache_hash(cache: Union[str, Path], db=None) -> Optional[str]:
with suppress(BaseException):
return loads(Path(normpath(f"{cache}.md")).read_text(encoding="utf-8")).get(
"checksum", None
)
return loads(Path(normpath(f"{cache}.md")).read_text(encoding="utf-8")).get("checksum", None)
if db:
cached_file = db.get_job_cache_file(
basename(getsourcefile(_getframe(1))).replace(".py", ""),

24
src/common/utils/logger.py
View file

@ -30,24 +30,12 @@ basicConfig(
level=default_level,
)
getLogger("sqlalchemy.orm.mapper.Mapper").setLevel(
default_level if default_level != INFO else WARNING
)
getLogger("sqlalchemy.orm.relationships.RelationshipProperty").setLevel(
default_level if default_level != INFO else WARNING
)
getLogger("sqlalchemy.orm.strategies.LazyLoader").setLevel(
default_level if default_level != INFO else WARNING
)
getLogger("sqlalchemy.pool.impl.QueuePool").setLevel(
default_level if default_level != INFO else WARNING
)
getLogger("sqlalchemy.pool.impl.NullPool").setLevel(
default_level if default_level != INFO else WARNING
)
getLogger("sqlalchemy.engine.Engine").setLevel(
default_level if default_level != INFO else WARNING
)
getLogger("sqlalchemy.orm.mapper.Mapper").setLevel(default_level if default_level != INFO else WARNING)
getLogger("sqlalchemy.orm.relationships.RelationshipProperty").setLevel(default_level if default_level != INFO else WARNING)
getLogger("sqlalchemy.orm.strategies.LazyLoader").setLevel(default_level if default_level != INFO else WARNING)
getLogger("sqlalchemy.pool.impl.QueuePool").setLevel(default_level if default_level != INFO else WARNING)
getLogger("sqlalchemy.pool.impl.NullPool").setLevel(default_level if default_level != INFO else WARNING)
getLogger("sqlalchemy.engine.Engine").setLevel(default_level if default_level != INFO else WARNING)
# Edit the default levels of the logging module
addLevelName(CRITICAL, "🚨")

6
src/deps/requirements-deps.txt
View file

@ -181,9 +181,9 @@ toposort==1.10 \
--hash=sha256:bfbb479c53d0a696ea7402601f4e693c97b0367837c8898bc6471adfca37a6bd \
--hash=sha256:cbdbc0d0bee4d2695ab2ceec97fe0679e9c10eab4b2a87a9372b929e70563a87
# via pip-compile-multi
urllib3==2.0.5 \
--hash=sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594 \
--hash=sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e
urllib3==2.0.6 \
--hash=sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2 \
--hash=sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564
# via requests
wheel==0.41.2 \
--hash=sha256:0c5ac5ff2afb79ac23ab82bab027a0be7b5dbcf2e54dc50efe4bf507de1f7985 \

2
src/linux/scripts/start.sh
View file

@ -226,7 +226,7 @@ function reload()
log "SYSTEMCTL" "ℹ️" "BunkerWeb service reloaded ..."
}
# List of differents args
# List of different args
case $1 in
"start")
start

10
src/scheduler/Dockerfile
View file

@ -1,4 +1,7 @@
FROM python:3.11.5-alpine@sha256:cd311c6a0164f34a7edbf364e05258b07d66d3f7bc155139dcb9bef88a186ded AS builder
FROM python:3.12.0-alpine@sha256:ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac AS builder
# Install python dependencies
RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev libffi-dev cairo-dev pango-dev gdk-pixbuf-dev openssl-dev cargo postgresql-dev
# Copy python requirements
COPY src/deps/requirements.txt /tmp/requirements-deps.txt
@ -12,9 +15,6 @@ RUN mkdir -p deps/python && \
cat /tmp/req/requirements.txt* > deps/requirements.txt && \
rm -rf /tmp/req
# Install python dependencies
RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev libffi-dev cairo-dev pango-dev gdk-pixbuf-dev openssl-dev cargo postgresql-dev
# Install python requirements
RUN export MAKEFLAGS="-j$(nproc)" && \
pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
@ -38,7 +38,7 @@ COPY src/common/utils utils
COPY src/scheduler scheduler
COPY src/VERSION VERSION
FROM python:3.11.5-alpine@sha256:cd311c6a0164f34a7edbf364e05258b07d66d3f7bc155139dcb9bef88a186ded
FROM python:3.12.0-alpine@sha256:ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac
# Set default umask to prevent huge recursive chmod increasing the final image size
RUN umask 027

36
src/scheduler/JobScheduler.py
View file

@ -21,9 +21,7 @@ from sys import path as sys_path
from threading import Lock, Semaphore, Thread
from traceback import format_exc
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("utils",), ("db",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("utils",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -94,16 +92,12 @@ class JobScheduler(ApiCaller):
def __get_jobs(self):
jobs = {}
for plugin_file in glob(
join(sep, "usr", "share", "bunkerweb", "core", "*", "plugin.json")
) + glob( # core plugins
join(sep, "etc", "bunkerweb", "plugins", "*", "plugin.json")
): # external plugins
for plugin_file in glob(join(sep, "usr", "share", "bunkerweb", "core", "*", "plugin.json")) + glob(join(sep, "etc", "bunkerweb", "plugins", "*", "plugin.json")): # core plugins # external plugins
plugin_name = basename(dirname(plugin_file))
jobs[plugin_name] = []
try:
plugin_data = loads(Path(plugin_file).read_text(encoding="utf-8"))
if not "jobs" in plugin_data:
if "jobs" not in plugin_data:
continue
plugin_jobs = plugin_data["jobs"]
@ -118,34 +112,24 @@ class JobScheduler(ApiCaller):
"reload",
)
):
self.__logger.warning(
f"missing keys for job {job['name']} in plugin {plugin_name}, must have name, file, every and reload, ignoring job"
)
self.__logger.warning(f"missing keys for job {job['name']} in plugin {plugin_name}, must have name, file, every and reload, ignoring job")
plugin_jobs.pop(x)
continue
if not match(r"^[\w.-]{1,128}$", job["name"]):
self.__logger.warning(
f"Invalid name for job {job['name']} in plugin {plugin_name} (Can only contain numbers, letters, underscores and hyphens (min 1 characters and max 128)), ignoring job"
)
self.__logger.warning(f"Invalid name for job {job['name']} in plugin {plugin_name} (Can only contain numbers, letters, underscores and hyphens (min 1 characters and max 128)), ignoring job")
plugin_jobs.pop(x)
continue
elif not match(r"^[\w./-]{1,256}$", job["file"]):
self.__logger.warning(
f"Invalid file for job {job['name']} in plugin {plugin_name} (Can only contain numbers, letters, underscores, hyphens and slashes (min 1 characters and max 256)), ignoring job"
)
self.__logger.warning(f"Invalid file for job {job['name']} in plugin {plugin_name} (Can only contain numbers, letters, underscores, hyphens and slashes (min 1 characters and max 256)), ignoring job")
plugin_jobs.pop(x)
continue
elif job["every"] not in ("once", "minute", "hour", "day", "week"):
self.__logger.warning(
f"Invalid every for job {job['name']} in plugin {plugin_name} (Must be once, minute, hour, day or week), ignoring job"
)
self.__logger.warning(f"Invalid every for job {job['name']} in plugin {plugin_name} (Must be once, minute, hour, day or week), ignoring job")
plugin_jobs.pop(x)
continue
elif job["reload"] is not True and job["reload"] is not False:
self.__logger.warning(
f"Invalid reload for job {job['name']} in plugin {plugin_name} (Must be true or false), ignoring job"
)
self.__logger.warning(f"Invalid reload for job {job['name']} in plugin {plugin_name} (Must be true or false), ignoring job")
plugin_jobs.pop(x)
continue
@ -255,9 +239,7 @@ class JobScheduler(ApiCaller):
file = job["file"]
every = job["every"]
if every != "once":
self.__str_to_schedule(every).do(
self.__job_wrapper, path, plugin, name, file
)
self.__str_to_schedule(every).do(self.__job_wrapper, path, plugin, name, file)
except:
self.__logger.error(
f"Exception while scheduling jobs for plugin {plugin} : {format_exc()}",

2
src/scheduler/entrypoint.sh
View file

@ -6,7 +6,7 @@
# trap SIGTERM and SIGINT
function trap_exit() {
# shellcheck disable=SC2317
log "ENTRYPOINT" "ℹ️ " "Catched stop operation"
log "ENTRYPOINT" "ℹ️ " "Caught stop operation"
# shellcheck disable=SC2317
if [ -f "/var/run/bunkerweb/scheduler.pid" ] ; then
# shellcheck disable=SC2317

93
src/scheduler/main.py
View file

@ -5,7 +5,7 @@ from copy import deepcopy
from glob import glob
from hashlib import sha256
from io import BytesIO
from json import dumps, load as json_load
from json import load as json_load
from os import (
_exit,
chmod,
@ -29,10 +29,7 @@ from time import sleep
from traceback import format_exc
from typing import Any, Dict, List, Optional, Union
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("deps", "python"), ("utils",), ("api",), ("db",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -41,7 +38,6 @@ from dotenv import dotenv_values
from logger import setup_logger # type: ignore
from Database import Database # type: ignore
from JobScheduler import JobScheduler
from ApiCaller import ApiCaller # type: ignore
RUN = True
SCHEDULER: Optional[JobScheduler] = None
@ -182,12 +178,7 @@ def dict_to_frozenset(d):
def api_to_instance(api):
hostname_port = (
api.endpoint.replace("http://", "")
.replace("https://", "")
.replace("/", "")
.split(":")
)
hostname_port = api.endpoint.replace("http://", "").replace("https://", "").replace("/", "").split(":")
return {
"hostname": hostname_port[0],
"env": {"API_HTTP_PORT": int(hostname_port[1]), "API_SERVER_NAME": api.host},
@ -228,27 +219,19 @@ if __name__ == "__main__":
INTEGRATION = "Autoconf"
elif integration_path.is_file():
INTEGRATION = integration_path.read_text(encoding="utf-8").strip()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(
encoding="utf-8"
):
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
INTEGRATION = "Docker"
del integration_path, os_release_path
tmp_variables_path = (
normpath(args.variables)
if args.variables
else join(sep, "var", "tmp", "bunkerweb", "variables.env")
)
tmp_variables_path = normpath(args.variables) if args.variables else join(sep, "var", "tmp", "bunkerweb", "variables.env")
tmp_variables_path = Path(tmp_variables_path)
nginx_variables_path = Path(sep, "etc", "nginx", "variables.env")
dotenv_env = dotenv_values(str(tmp_variables_path))
db = Database(
logger,
sqlalchemy_string=dotenv_env.get(
"DATABASE_URI", getenv("DATABASE_URI", None)
),
sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None)),
)
env = {}
@ -270,16 +253,7 @@ if __name__ == "__main__":
sleep(5)
env = db.get_config()
elif (
not tmp_variables_path.exists()
or not nginx_variables_path.exists()
or (
tmp_variables_path.read_text(encoding="utf-8")
!= nginx_variables_path.read_text(encoding="utf-8")
)
or db.is_initialized()
and db.get_config() != dotenv_env
):
elif not tmp_variables_path.exists() or not nginx_variables_path.exists() or (tmp_variables_path.read_text(encoding="utf-8") != nginx_variables_path.read_text(encoding="utf-8")) or db.is_initialized() and db.get_config() != dotenv_env:
# run the config saver
proc = subprocess_run(
[
@ -349,9 +323,7 @@ if __name__ == "__main__":
custom_conf = {
"value": content,
"exploded": (
f"{path_exploded.pop()}"
if path_exploded[-1] not in root_dirs
else None,
f"{path_exploded.pop()}" if path_exploded[-1] not in root_dirs else None,
path_exploded[-1],
file.replace(".conf", ""),
),
@ -360,10 +332,7 @@ if __name__ == "__main__":
saving = True
in_db = False
for db_conf in db_configs:
if (
db_conf["service_id"] == custom_conf["exploded"][0]
and db_conf["name"] == custom_conf["exploded"][2]
):
if db_conf["service_id"] == custom_conf["exploded"][0] and db_conf["name"] == custom_conf["exploded"][2]:
in_db = True
if db_conf["method"] != "manual":
saving = False
@ -376,9 +345,7 @@ if __name__ == "__main__":
if saving:
custom_configs.append(custom_conf)
changes = changes or {hash(dict_to_frozenset(d)) for d in custom_configs} != {
hash(dict_to_frozenset(d)) for d in db_configs
}
changes = changes or {hash(dict_to_frozenset(d)) for d in custom_configs} != {hash(dict_to_frozenset(d)) for d in db_configs}
if changes:
err = db.save_custom_configs(custom_configs, "manual")
@ -404,9 +371,7 @@ if __name__ == "__main__":
with open(filename, "r", encoding="utf-8") as f:
_dir = dirname(filename)
plugin_content = BytesIO()
with tar_open(
fileobj=plugin_content, mode="w:gz", compresslevel=9
) as tar:
with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar:
tar.add(_dir, arcname=basename(_dir), recursive=True)
plugin_content.seek(0, 0)
value = plugin_content.getvalue()
@ -435,9 +400,7 @@ if __name__ == "__main__":
db_plugin.pop("method", None)
tmp_db_plugins.append(db_plugin)
changes = {hash(dict_to_frozenset(d)) for d in tmp_external_plugins} != {
hash(dict_to_frozenset(d)) for d in tmp_db_plugins
}
changes = {hash(dict_to_frozenset(d)) for d in tmp_external_plugins} != {hash(dict_to_frozenset(d)) for d in tmp_db_plugins}
if changes:
err = db.update_external_plugins(external_plugins, delete_missing=True)
@ -463,9 +426,7 @@ if __name__ == "__main__":
ret = db.set_scheduler_first_start()
if ret:
logger.error(
f"An error occurred when setting the scheduler first start : {ret}"
)
logger.error(f"An error occurred when setting the scheduler first start : {ret}")
stop(1)
FIRST_RUN = True
@ -494,9 +455,7 @@ if __name__ == "__main__":
ret = db.checked_changes(CHANGES)
if ret:
logger.error(
f"An error occurred when setting the changes to checked in the database : {ret}"
)
logger.error(f"An error occurred when setting the changes to checked in the database : {ret}")
stop(1)
if RUN_JOBS_ONCE:
@ -578,17 +537,13 @@ if __name__ == "__main__":
logger.info("Successfully sent stop signal to temp nginx")
i = 0
while i < 20:
if not Path(
sep, "var", "run", "bunkerweb", "nginx.pid"
).is_file():
if not Path(sep, "var", "run", "bunkerweb", "nginx.pid").is_file():
break
logger.warning("Waiting for temp nginx to stop ...")
sleep(1)
i += 1
if i >= 20:
logger.error(
"Timeout error while waiting for temp nginx to stop"
)
logger.error("Timeout error while waiting for temp nginx to stop")
else:
# Start nginx
logger.info("Starting nginx ...")
@ -623,9 +578,7 @@ if __name__ == "__main__":
# infinite schedule for the jobs
logger.info("Executing job scheduler ...")
Path(sep, "var", "tmp", "bunkerweb", "scheduler.healthy").write_text(
"ok", encoding="utf-8"
)
Path(sep, "var", "tmp", "bunkerweb", "scheduler.healthy").write_text("ok", encoding="utf-8")
while RUN and not NEED_RELOAD:
SCHEDULER.run_pending()
sleep(1)
@ -633,9 +586,7 @@ if __name__ == "__main__":
changes = db.check_changes()
if isinstance(changes, str):
logger.error(
f"An error occurred when checking for changes in the database : {changes}"
)
logger.error(f"An error occurred when checking for changes in the database : {changes}")
stop(1)
# check if the plugins have changed since last time
@ -644,9 +595,7 @@ if __name__ == "__main__":
if FIRST_RUN:
# run the config saver to save potential ignored external plugins settings
logger.info(
"Running config saver to save potential ignored external plugins settings ..."
)
logger.info("Running config saver to save potential ignored external plugins settings ...")
proc = subprocess_run(
[
"python",
@ -707,9 +656,7 @@ if __name__ == "__main__":
if CONFIGS_NEED_GENERATION:
CHANGES.append("custom_configs")
generate_custom_configs(
db.get_custom_configs(), original_path=configs_path
)
generate_custom_configs(db.get_custom_configs(), original_path=configs_path)
if PLUGINS_NEED_GENERATION:
CHANGES.append("external_plugins")

2
src/scheduler/requirements.in
View file

@ -3,4 +3,4 @@ cryptography==41.0.4
maxminddb==2.4.0
python-magic==0.4.27
schedule==1.2.1
urllib3==1.26.16
urllib3==1.26.17

6
src/scheduler/requirements.txt
View file

@ -268,9 +268,9 @@ six==1.16.0 \
--hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
# via configobj
urllib3==1.26.16 \
--hash=sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f \
--hash=sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14
urllib3==1.26.17 \
--hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \
--hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b
# via
# acme
# certbot

10
src/ui/Dockerfile
View file

@ -1,4 +1,7 @@
FROM python:3.11.5-alpine@sha256:cd311c6a0164f34a7edbf364e05258b07d66d3f7bc155139dcb9bef88a186ded AS builder
FROM python:3.12.0-alpine@sha256:ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac AS builder
# Install python dependencies
RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev libffi-dev cairo-dev pango-dev gdk-pixbuf-dev openssl-dev cargo postgresql-dev file make
# Copy python requirements
COPY src/deps/requirements.txt /tmp/requirements-deps.txt
@ -12,9 +15,6 @@ RUN mkdir -p deps/python && \
cat /tmp/req/requirements.txt* > deps/requirements.txt && \
rm -rf /tmp/req
# Install python dependencies
RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev libffi-dev cairo-dev pango-dev gdk-pixbuf-dev openssl-dev cargo postgresql-dev file make
# Install python requirements
RUN export MAKEFLAGS="-j$(nproc)" && \
pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
@ -36,7 +36,7 @@ COPY src/common/helpers helpers
COPY src/ui ui
COPY src/VERSION VERSION
FROM python:3.11.5-alpine@sha256:cd311c6a0164f34a7edbf364e05258b07d66d3f7bc155139dcb9bef88a186ded
FROM python:3.12.0-alpine@sha256:ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac
# Set default umask to prevent huge recursive chmod increasing the final image size
RUN umask 027

269
src/ui/main.py
View file

@ -6,17 +6,12 @@ from sys import path as sys_path, modules as sys_modules
from pathlib import Path
os_release_path = Path(sep, "etc", "os-release")
if os_release_path.is_file() and "Alpine" not in os_release_path.read_text(
encoding="utf-8"
):
if os_release_path.is_file() and "Alpine" not in os_release_path.read_text(encoding="utf-8"):
sys_path.append(join(sep, "usr", "share", "bunkerweb", "deps", "python"))
del os_release_path
for deps_path in [
join(sep, "usr", "share", "bunkerweb", *paths)
for paths in (("utils",), ("api",), ("db",))
]:
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
@ -98,7 +93,7 @@ def stop(status, _stop=True):
def handle_stop(signum, frame):
app.logger.info("Catched stop operation")
app.logger.info("Caught stop operation")
app.logger.info("Stopping web ui ...")
stop(0, False)
@ -138,9 +133,7 @@ if not getenv("FLASK_DEBUG", False) and not regex_match(
r"^(?=.*?\p{Lowercase_Letter})(?=.*?\p{Uppercase_Letter})(?=.*?\d)(?=.*?[ !\"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]).{8,}$",
getenv("ADMIN_PASSWORD", "changeme"),
):
app.logger.error(
"The admin password is not strong enough. It must contain at least 8 characters, including at least 1 uppercase letter, 1 lowercase letter, 1 number and 1 special character (#@?!$%^&*-)."
)
app.logger.error("The admin password is not strong enough. It must contain at least 8 characters, including at least 1 uppercase letter, 1 lowercase letter, 1 number and 1 special character (#@?!$%^&*-).")
stop(1)
login_manager = LoginManager()
@ -173,9 +166,7 @@ docker_client = None
kubernetes_client = None
if INTEGRATION in ("Docker", "Swarm", "Autoconf"):
try:
docker_client: DockerClient = DockerClient(
base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
)
docker_client: DockerClient = DockerClient(base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock"))
except (docker_APIError, DockerException):
app.logger.warning("No docker host found")
elif INTEGRATION == "Kubernetes":
@ -203,11 +194,7 @@ while not db.is_initialized():
app.logger.info("Database is ready")
Path(sep, "var", "tmp", "bunkerweb", "ui.healthy").write_text("ok", encoding="utf-8")
bw_version = (
Path(sep, "usr", "share", "bunkerweb", "VERSION")
.read_text(encoding="utf-8")
.strip()
)
bw_version = Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text(encoding="utf-8").strip()
try:
app.config.update(
@ -238,9 +225,7 @@ app.jinja_env.globals.update(check_settings=check_settings)
csrf = CSRFProtect()
csrf.init_app(app)
LOG_RX = re_compile(
r"^(?P<date>\d+/\d+/\d+\s\d+:\d+:\d+)\s\[(?P<level>[a-z]+)\]\s\d+#\d+:\s(?P<message>[^\n]+)$"
)
LOG_RX = re_compile(r"^(?P<date>\d+/\d+/\d+\s\d+:\d+:\d+)\s\[(?P<level>[a-z]+)\]\s\d+#\d+:\s(?P<message>[^\n]+)$")
def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
@ -248,9 +233,7 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
error = False
if method == "services":
editing = operation == "edit"
service_custom_confs = glob(
join(sep, "etc", "bunkerweb", "configs", "*", args[1])
)
service_custom_confs = glob(join(sep, "etc", "bunkerweb", "configs", "*", args[1]))
moved = False
if operation == "new":
@ -261,9 +244,7 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
if listdir(service_custom_conf):
move(
service_custom_conf,
service_custom_conf.replace(
f"{sep}{args[1]}", f"{sep}{args[2]}"
).replace(join(sep, "etc"), join(sep, "var", "tmp")),
service_custom_conf.replace(f"{sep}{args[1]}", f"{sep}{args[2]}").replace(join(sep, "etc"), join(sep, "var", "tmp")),
)
moved = True
operation, error = app.config["CONFIG"].edit_service(args[1], args[0])
@ -276,9 +257,7 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
app.config["TO_FLASH"].append({"content": operation, "type": "success"})
if editing and moved and args[1] != args[2] and service_custom_confs:
for tmp_service_custom_conf in glob(
join(sep, "var", "tmp", "bunkerweb", "configs", "*", args[2])
):
for tmp_service_custom_conf in glob(join(sep, "var", "tmp", "bunkerweb", "configs", "*", args[2])):
move(
tmp_service_custom_conf,
tmp_service_custom_conf.replace(
@ -289,9 +268,7 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
error = app.config["CONFIGFILES"].save_configs()
if error:
app.config["TO_FLASH"].append({"content": error, "type": "error"})
rmtree(
join(sep, "var", "tmp", "bunkerweb", "configs"), ignore_errors=True
)
rmtree(join(sep, "var", "tmp", "bunkerweb", "configs"), ignore_errors=True)
if method == "global_config":
operation = app.config["CONFIG"].edit_global_conf(args[0])
elif method == "plugins":
@ -313,9 +290,7 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
if operation:
if isinstance(operation, list):
for op in operation:
app.config["TO_FLASH"].append(
{"content": f"Reload failed for the instance {op}", "type": "error"}
)
app.config["TO_FLASH"].append({"content": f"Reload failed for the instance {op}", "type": "error"})
elif operation.startswith("Can't"):
app.config["TO_FLASH"].append({"content": operation, "type": "error"})
else:
@ -327,9 +302,7 @@ def manage_bunkerweb(method: str, *args, operation: str = "reloads"):
@app.after_request
def set_csp_header(response):
"""Set the Content-Security-Policy header to prevent XSS attacks."""
response.headers[
"Content-Security-Policy"
] = "object-src 'none'; frame-ancestors 'self';"
response.headers["Content-Security-Policy"] = "object-src 'none'; frame-ancestors 'self';"
return response
@ -438,7 +411,7 @@ def instances():
# Manage instances
if request.method == "POST":
# Check operation
if not "operation" in request.form or not request.form["operation"] in (
if "operation" not in request.form or request.form["operation"] not in (
"reload",
"start",
"stop",
@ -448,7 +421,7 @@ def instances():
return redirect(url_for("loading", next=url_for("instances")))
# Check that all fields are present
if not "INSTANCE_ID" in request.form:
if "INSTANCE_ID" not in request.form:
flash("Missing INSTANCE_ID parameter.", "error")
return redirect(url_for("loading", next=url_for("instances")))
@ -465,12 +438,7 @@ def instances():
url_for(
"loading",
next=url_for("instances"),
message=(
f"{request.form['operation'].title()}ing"
if request.form["operation"] != "stop"
else "Stopping"
)
+ " instance",
message=(f"{request.form['operation'].title()}ing" if request.form["operation"] != "stop" else "Stopping") + " instance",
)
)
@ -489,7 +457,7 @@ def instances():
def services():
if request.method == "POST":
# Check operation
if not "operation" in request.form or not request.form["operation"] in (
if "operation" not in request.form or request.form["operation"] not in (
"new",
"edit",
"delete",
@ -501,10 +469,7 @@ def services():
variables = deepcopy(request.form.to_dict())
del variables["csrf_token"]
if (
not "OLD_SERVER_NAME" in request.form
and request.form["operation"] == "edit"
):
if "OLD_SERVER_NAME" not in request.form and request.form["operation"] == "edit":
flash("Missing OLD_SERVER_NAME parameter.", "error")
return redirect(url_for("loading", next=url_for("services")))
@ -527,19 +492,10 @@ def services():
elif value == "off":
value = "no"
if variable in variables and (
variable != "SERVER_NAME"
and value == config.get(variable, None)
or not value.strip()
):
if variable in variables and (variable != "SERVER_NAME" and value == config.get(variable, None) or not value.strip()):
del variables[variable]
if (
request.form["operation"] == "edit"
and len(variables) == 1
and "SERVER_NAME" in variables
and variables["SERVER_NAME"] == request.form.get("OLD_SERVER_NAME", "")
):
if request.form["operation"] == "edit" and len(variables) == 1 and "SERVER_NAME" in variables and variables["SERVER_NAME"] == request.form.get("OLD_SERVER_NAME", ""):
flash(
"The service was not edited because no values were changed.",
"error",
@ -553,13 +509,11 @@ def services():
# Delete
elif request.form["operation"] == "delete":
if not "SERVER_NAME" in request.form:
if "SERVER_NAME" not in request.form:
flash("Missing SERVER_NAME parameter.", "error")
return redirect(url_for("loading", next=url_for("services")))
error = app.config["CONFIG"].check_variables(
{"SERVER_NAME": request.form["SERVER_NAME"]}
)
error = app.config["CONFIG"].check_variables({"SERVER_NAME": request.form["SERVER_NAME"]})
if error:
return redirect(url_for("loading", next=url_for("services")))
@ -584,15 +538,11 @@ def services():
message = ""
if request.form["operation"] == "new":
message = (
f"Creating service {variables.get('SERVER_NAME', '').split(' ')[0]}"
)
message = f"Creating service {variables.get('SERVER_NAME', '').split(' ')[0]}"
elif request.form["operation"] == "edit":
message = f"Saving configuration for service {request.form.get('OLD_SERVER_NAME', '').split(' ')[0]}"
elif request.form["operation"] == "delete":
message = (
f"Deleting service {request.form.get('SERVER_NAME', '').split(' ')[0]}"
)
message = f"Deleting service {request.form.get('SERVER_NAME', '').split(' ')[0]}"
return redirect(url_for("loading", next=url_for("services"), message=message))
@ -644,9 +594,7 @@ def global_config():
del variables[variable]
if not variables:
flash(
"The global configuration was not edited because no values were changed."
)
flash("The global configuration was not edited because no values were changed.")
return redirect(url_for("loading", next=url_for("global_config")))
error = app.config["CONFIG"].check_variables(variables, True)
@ -688,7 +636,7 @@ def configs():
operation = ""
# Check operation
if not "operation" in request.form or not request.form["operation"] in (
if "operation" not in request.form or request.form["operation"] not in (
"new",
"edit",
"delete",
@ -720,21 +668,15 @@ def configs():
if "old_name" in variables:
variables["old_name"] = f"{variables['old_name']}.conf"
variables["content"] = BeautifulSoup(
variables["content"], "html.parser"
).get_text()
variables["content"] = BeautifulSoup(variables["content"], "html.parser").get_text()
error = False
if request.form["operation"] == "new":
if variables["type"] == "folder":
operation, error = app.config["CONFIGFILES"].create_folder(
variables["path"], variables["name"]
)
operation, error = app.config["CONFIGFILES"].create_folder(variables["path"], variables["name"])
elif variables["type"] == "file":
operation, error = app.config["CONFIGFILES"].create_file(
variables["path"], variables["name"], variables["content"]
)
operation, error = app.config["CONFIGFILES"].create_file(variables["path"], variables["name"], variables["content"])
elif request.form["operation"] == "edit":
if variables["type"] == "folder":
operation, error = app.config["CONFIGFILES"].edit_folder(
@ -774,10 +716,7 @@ def configs():
path_to_dict(
join(sep, "etc", "bunkerweb", "configs"),
db_data=db.get_custom_configs(),
services=app.config["CONFIG"]
.get_config(methods=False)
.get("SERVER_NAME", "")
.split(" "),
services=app.config["CONFIG"].get_config(methods=False).get("SERVER_NAME", "").split(" "),
)
],
dark_mode=app.config["DARK_MODE"],
@ -851,9 +790,7 @@ def plugins():
)
else:
try:
with tar_open(
str(tmp_ui_path.joinpath(file)), errorlevel=2
) as tar_file:
with tar_open(str(tmp_ui_path.joinpath(file)), errorlevel=2) as tar_file:
try:
tar_file.getmember("plugin.json")
except KeyError:
@ -882,24 +819,12 @@ def plugins():
)
if is_dir:
dirs = [
d
for d in listdir(str(temp_folder_path))
if temp_folder_path.joinpath(d).is_dir()
]
dirs = [d for d in listdir(str(temp_folder_path)) if temp_folder_path.joinpath(d).is_dir()]
if (
not dirs
or len(dirs) > 1
or not temp_folder_path.joinpath(
dirs[0], "plugin.json"
).is_file()
):
if not dirs or len(dirs) > 1 or not temp_folder_path.joinpath(dirs[0], "plugin.json").is_file():
raise KeyError
for file_name in listdir(
str(temp_folder_path.joinpath(dirs[0]))
):
for file_name in listdir(str(temp_folder_path.joinpath(dirs[0]))):
move(
str(temp_folder_path.joinpath(dirs[0], file_name)),
str(temp_folder_path.joinpath(file_name)),
@ -909,11 +834,7 @@ def plugins():
ignore_errors=True,
)
plugin_file = json_loads(
temp_folder_path.joinpath("plugin.json").read_text(
encoding="utf-8"
)
)
plugin_file = json_loads(temp_folder_path.joinpath("plugin.json").read_text(encoding="utf-8"))
if not all(key in plugin_file.keys() for key in PLUGIN_KEYS):
raise ValueError
@ -992,9 +913,7 @@ def plugins():
flash(f"{e}", "error")
finally:
if error != 1:
flash(
f"Successfully created plugin: <b><i>{folder_name}</i></b>"
)
flash(f"Successfully created plugin: <b><i>{folder_name}</i></b>")
error = 0
@ -1030,9 +949,7 @@ def plugins():
if tmp_ui_path.exists():
rmtree(str(tmp_ui_path), ignore_errors=True)
return redirect(
url_for("loading", next=url_for("plugins"), message="Reloading plugins")
)
return redirect(url_for("loading", next=url_for("plugins"), message="Reloading plugins"))
plugin_args = app.config["PLUGIN_ARGS"]
app.config["PLUGIN_ARGS"] = {}
@ -1051,11 +968,7 @@ def plugins():
csrf_token=generate_csrf,
url_for=url_for,
dark_mode=app.config["DARK_MODE"],
**(
plugin_args["args"]
if plugin_args.get("plugin", None) == plugin_id
else {}
),
**(plugin_args["args"] if plugin_args.get("plugin", None) == plugin_id else {}),
)
plugins = app.config["CONFIG"].get_plugins()
@ -1109,9 +1022,7 @@ def upload_plugin():
plugins.append(basename(dirname(file)))
if len(plugins) > 1:
tar_file.extractall(str(tmp_ui_path) + "/")
folder_name = uploaded_file.filename.replace(".tar.gz", "").replace(
".tar.xz", ""
)
folder_name = uploaded_file.filename.replace(".tar.gz", "").replace(".tar.xz", "")
if len(plugins) <= 1:
io.seek(0, 0)
@ -1120,12 +1031,8 @@ def upload_plugin():
for plugin in plugins:
with BytesIO() as tgz:
with tar_open(
mode="w:gz", fileobj=tgz, dereference=True, compresslevel=3
) as tf:
tf.add(
str(tmp_ui_path.joinpath(folder_name, plugin)), arcname=plugin
)
with tar_open(mode="w:gz", fileobj=tgz, dereference=True, compresslevel=3) as tf:
tf.add(str(tmp_ui_path.joinpath(folder_name, plugin)), arcname=plugin)
tgz.seek(0, 0)
tmp_ui_path.joinpath(f"{plugin}.tar.gz").write_bytes(tgz.read())
@ -1195,15 +1102,8 @@ def custom_plugin(plugin):
sys_modules.pop("actions")
del actions
if (
request.method != "POST"
or error is True
or res is None
or isinstance(res, dict) is False
):
return redirect(
url_for("loading", next=url_for("plugins", plugin_id=plugin))
)
if request.method != "POST" or error is True or res is None or isinstance(res, dict) is False:
return redirect(url_for("loading", next=url_for("plugins", plugin_id=plugin)))
app.config["PLUGIN_ARGS"] = {"plugin": plugin, "args": res}
@ -1221,10 +1121,7 @@ def cache():
join(sep, "var", "cache", "bunkerweb"),
is_cache=True,
db_data=db.get_jobs_cache_files(),
services=app.config["CONFIG"]
.get_config(methods=False)
.get("SERVER_NAME", "")
.split(" "),
services=app.config["CONFIG"].get_config(methods=False).get("SERVER_NAME", "").split(" "),
)
],
dark_mode=app.config["DARK_MODE"],
@ -1274,9 +1171,7 @@ def logs_linux():
nginx_error_file = Path(sep, "var", "log", "bunkerweb", "error.log")
if nginx_error_file.is_file():
with open(nginx_error_file, encoding="utf-8") as f:
for line in f.readlines()[
int(last_update.split(".")[0]) if last_update else 0 :
]:
for line in f.readlines()[int(last_update.split(".")[0]) if last_update else 0 :]: # noqa: E203
match = LOG_RX.search(line)
if not match:
continue
@ -1288,15 +1183,10 @@ def logs_linux():
logs_error[-1] += f"\n{line}"
continue
logs_error.append(line)
elif (
all(f"[{log_level}]" != level for log_level in NGINX_LOG_LEVELS)
and temp_multiple_lines
):
elif all(f"[{log_level}]" != level for log_level in NGINX_LOG_LEVELS) and temp_multiple_lines:
temp_multiple_lines.append(line)
else:
logs_error.append(
f"{datetime.strptime(date, '%Y/%m/%d %H:%M:%S').replace(tzinfo=timezone.utc).timestamp()} {line}"
)
logs_error.append(f"{datetime.strptime(date, '%Y/%m/%d %H:%M:%S').replace(tzinfo=timezone.utc).timestamp()} {line}")
if temp_multiple_lines:
logs_error.append("\n".join(temp_multiple_lines))
@ -1305,12 +1195,8 @@ def logs_linux():
nginx_access_file = Path(sep, "var", "log", "bunkerweb", "access.log")
if nginx_access_file.is_file():
with open(nginx_access_file, encoding="utf-8") as f:
for line in f.readlines()[
int(last_update.split(".")[1]) if last_update else 0 :
]:
logs_access.append(
f"{datetime.strptime(line[line.find('[') + 1: line.find(']')], '%d/%b/%Y:%H:%M:%S %z').replace(tzinfo=timezone.utc).timestamp()} {line}"
)
for line in f.readlines()[int(last_update.split(".")[1]) if last_update else 0 :]: # noqa: E203
logs_access.append(f"{datetime.strptime(line[line.find('[') + 1: line.find(']')], '%d/%b/%Y:%H:%M:%S %z').replace(tzinfo=timezone.utc).timestamp()} {line}")
raw_logs = logs_error + logs_access
@ -1339,17 +1225,8 @@ def logs_linux():
log_lower = log.lower()
error_type = (
"error"
if "[error]" in log_lower
or "[crit]" in log_lower
or "[alert]" in log_lower
or "❌" in log_lower
else (
"warn"
if "[warn]" in log_lower or "⚠️" in log_lower
else (
"info" if "[info]" in log_lower or "ℹ️" in log_lower else "message"
)
)
if "[error]" in log_lower or "[crit]" in log_lower or "[alert]" in log_lower or "❌" in log_lower
else ("warn" if "[warn]" in log_lower or "⚠️" in log_lower else ("info" if "[info]" in log_lower or "ℹ️" in log_lower else "message"))
)
logs.append(
@ -1370,9 +1247,7 @@ def logs_linux():
return jsonify(
{
"logs": logs,
"last_update": f"{count_error_logs + int(last_update.split('.')[0])}.{len(logs_access) + int(last_update.split('.')[1])}"
if last_update
else f"{count_error_logs}.{len(logs_access)}",
"last_update": f"{count_error_logs + int(last_update.split('.')[0])}.{len(logs_access) + int(last_update.split('.')[1])}" if last_update else f"{count_error_logs}.{len(logs_access)}",
}
)
@ -1398,10 +1273,7 @@ def logs_container(container_id):
422,
)
elif not last_update:
last_update = int(
datetime.now().timestamp()
- timedelta(days=1).total_seconds() # 1 day before
)
last_update = int(datetime.now().timestamp() - timedelta(days=1).total_seconds()) # 1 day before
else:
last_update = int(last_update) // 1000
@ -1458,13 +1330,13 @@ def logs_container(container_id):
)
for log in tmp_logs:
splitted = log.split(" ")
timestamp = splitted[0]
split = log.split(" ")
timestamp = split[0]
if to_date is not None and dateutil_parse(timestamp).timestamp() > to_date:
break
log = " ".join(splitted[1:])
log = " ".join(split[1:])
log_lower = log.lower()
if "[48;2" in log or not log.strip():
@ -1474,19 +1346,8 @@ def logs_container(container_id):
{
"content": log,
"type": "error"
if "[error]" in log_lower
or "[crit]" in log_lower
or "[alert]" in log_lower
or "❌" in log_lower
else (
"warn"
if "[warn]" in log_lower or "⚠️" in log_lower
else (
"info"
if "[info]" in log_lower or "ℹ️" in log_lower
else "message"
)
),
if "[error]" in log_lower or "[crit]" in log_lower or "[alert]" in log_lower or "❌" in log_lower
else ("warn" if "[warn]" in log_lower or "⚠️" in log_lower else ("info" if "[info]" in log_lower or "ℹ️" in log_lower else "message")),
}
)
@ -1541,14 +1402,8 @@ def jobs_download():
@app.route("/login", methods=["GET", "POST"])
def login():
fail = False
if (
request.method == "POST"
and "username" in request.form
and "password" in request.form
):
if app.config["USER"].get_id() == request.form["username"] and app.config[
"USER"
].check_password(request.form["password"]):
if request.method == "POST" and "username" in request.form and "password" in request.form:
if app.config["USER"].get_id() == request.form["username"] and app.config["USER"].check_password(request.form["password"]):
# log the user in
next_url = request.form.get("next")
login_user(app.config["USER"])
@ -1588,9 +1443,7 @@ def darkmode():
def check_reloading():
if not app.config["RELOADING"] or app.config["LAST_RELOAD"] + 60 < time():
if app.config["RELOADING"]:
app.logger.warning(
"Reloading took too long, forcing the state to be reloaded"
)
app.logger.warning("Reloading took too long, forcing the state to be reloaded")
flash("Forced the status to be reloaded", "error")
app.config["RELOADING"] = False

54
src/ui/src/Config.py
View file

@ -14,11 +14,7 @@ from uuid import uuid4
class Config:
def __init__(self, db) -> None:
self.__settings = json_loads(
Path(sep, "usr", "share", "bunkerweb", "settings.json").read_text(
encoding="utf-8"
)
)
self.__settings = json_loads(Path(sep, "usr", "share", "bunkerweb", "settings.json").read_text(encoding="utf-8"))
self.__db = db
def __gen_conf(self, global_conf: dict, services_conf: list[dict]) -> None:
@ -42,11 +38,7 @@ class Config:
server_name = service["SERVER_NAME"].split(" ")[0]
for k in service:
key_without_server_name = k.replace(f"{server_name}_", "")
if (
plugins_settings[key_without_server_name]["context"] != "global"
if key_without_server_name in plugins_settings
else True
):
if plugins_settings[key_without_server_name]["context"] != "global" if key_without_server_name in plugins_settings else True:
if not k.startswith(server_name) or k in plugins_settings:
conf[f"{server_name}_{k}"] = service[k]
else:
@ -86,9 +78,7 @@ class Config:
**self.__settings,
}
def get_plugins(
self, *, external: bool = False, with_data: bool = False
) -> List[dict]:
def get_plugins(self, *, external: bool = False, with_data: bool = False) -> List[dict]:
plugins = self.__db.get_plugins(external=external, with_data=with_data)
plugins.sort(key=lambda x: x["name"])
@ -153,18 +143,13 @@ class Config:
setting = k
else:
setting = k[0 : k.rfind("_")]
if (
setting not in plugins_settings
or "multiple" not in plugins_settings[setting]
):
setting = k[0 : k.rfind("_")] # noqa: E203
if setting not in plugins_settings or "multiple" not in plugins_settings[setting]:
error = 1
flash(f"Variable {k} is not valid.", "error")
continue
if not (
_global ^ (plugins_settings[setting]["context"] == "global")
) and re_search(plugins_settings[setting]["regex"], v):
if not (_global ^ (plugins_settings[setting]["context"] == "global")) and re_search(plugins_settings[setting]["regex"], v):
check = True
if not check:
@ -175,9 +160,7 @@ class Config:
return error
def reload_config(self) -> None:
self.__gen_conf(
self.get_config(methods=False), self.get_services(methods=False)
)
self.__gen_conf(self.get_config(methods=False), self.get_services(methods=False))
def new_service(self, variables: dict) -> Tuple[str, int]:
"""Creates a new service from the given variables
@ -200,10 +183,7 @@ class Config:
services = self.get_services(methods=False)
server_name_splitted = variables["SERVER_NAME"].split(" ")
for service in services:
if (
service["SERVER_NAME"] == variables["SERVER_NAME"]
or service["SERVER_NAME"] in server_name_splitted
):
if service["SERVER_NAME"] == variables["SERVER_NAME"] or service["SERVER_NAME"] in server_name_splitted:
return (
f"Service {service['SERVER_NAME'].split(' ')[0]} already exists.",
1,
@ -236,20 +216,14 @@ class Config:
server_name_splitted = variables["SERVER_NAME"].split(" ")
old_server_name_splitted = old_server_name.split(" ")
for i, service in enumerate(deepcopy(services)):
if (
service["SERVER_NAME"] == variables["SERVER_NAME"]
or service["SERVER_NAME"] in server_name_splitted
):
if service["SERVER_NAME"] == variables["SERVER_NAME"] or service["SERVER_NAME"] in server_name_splitted:
if changed_server_name:
return (
f"Service {service['SERVER_NAME'].split(' ')[0]} already exists.",
1,
)
services.pop(i)
elif changed_server_name and (
service["SERVER_NAME"] == old_server_name
or service["SERVER_NAME"] in old_server_name_splitted
):
elif changed_server_name and (service["SERVER_NAME"] == old_server_name or service["SERVER_NAME"] in old_server_name_splitted):
services.pop(i)
services.append(variables)
@ -279,9 +253,7 @@ class Config:
str
the confirmation message
"""
self.__gen_conf(
self.get_config(methods=False) | variables, self.get_services(methods=False)
)
self.__gen_conf(self.get_config(methods=False) | variables, self.get_services(methods=False))
return "The global configuration has been edited."
def delete_service(self, service_name: str) -> Tuple[str, int]:
@ -317,9 +289,7 @@ class Config:
if not found:
return f"Can't delete missing {service_name} configuration.", 1
full_env["SERVER_NAME"] = " ".join(
[s for s in full_env["SERVER_NAME"].split(" ") if s != service_name]
)
full_env["SERVER_NAME"] = " ".join([s for s in full_env["SERVER_NAME"].split(" ") if s != service_name])
new_env = deepcopy(full_env)

Some files were not shown because too many files have changed in this diff Show more