Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Start making scheduler agnostic to integrations and instances

Browse source
This commit is contained in:
Théophile Diot 2024-05-23 18:27:26 +01:00
parent 2fdfa1a3aa
commit d84629b7cb
No known key found for this signature in database
GPG key ID: 248FEA4BAE400D06
39 changed files with 952 additions and 1283 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
misc/dev/docker-compose.autoconf.misc.yml
View file

@ -2,7 +2,7 @@ x-env: &env
DATABASE_URI: "mariadb+pymysql://bunkerweb:secret@bw-db:3306/db"
DOCKER_HOST: "tcp://bw-docker:2375"
AUTOCONF_MODE: "yes"
LOG_LEVEL: "debug"
CUSTOM_LOG_LEVEL: "debug"
services:
bunkerweb:
@ -15,21 +15,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=
- MULTISITE=yes
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- AUTOCONF_MODE=yes
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- EXTERNAL_PLUGIN_URLS=https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip
- CUSTOM_CONF_MODSEC_CRS_reqbody-rule=SecRuleRemoveById 200002
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -47,10 +34,14 @@ services:
- bw-docker
environment:
<<: *env
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-autoconf
bw-db:
aliases:
- bw-autoconf
bw-docker:
aliases:
- bw-autoconf
@ -61,32 +52,35 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
- ./configs/server-http/hello.conf:/data/configs/server-http/hello.conf:ro
environment:
<<: *env
BUNKERWEB_INSTANCES: ""
SERVER_NAME: ""
MULTISITE: "yes"
API_WHITELIST_IP: "127.0.0.0/24 10.20.30.0/24"
USE_BUNKERNET: "no"
USE_BLACKLIST: "no"
USE_WHITELIST: "no"
SEND_ANONYMOUS_REPORT: "no"
LOG_LEVEL: "info"
SERVE_FILES: "no"
DISABLE_DEFAULT_SERVER: "yes"
USE_CLIENT_CACHE: "yes"
USE_GZIP: "yes"
EXTERNAL_PLUGIN_URLS: "https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip"
CUSTOM_CONF_MODSEC_CRS_reqbody-suppress: "SecRuleRemoveById 200002"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
bw-db:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
bw-db:
image: mariadb:11
environment:
@ -96,13 +90,28 @@ services:
- MYSQL_PASSWORD=secret
volumes:
- bw-db:/var/lib/mysql
restart: "unless-stopped"
networks:
bw-docker:
bw-db:
aliases:
- bw-db
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
restart: "unless-stopped"
networks:
bw-docker:
aliases:
- bw-docker
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -112,7 +121,6 @@ services:
- "bunkerweb.USE_REVERSE_PROXY=yes"
- "bunkerweb.REVERSE_PROXY_URL=/"
- "bunkerweb.REVERSE_PROXY_HOST=http://app1:8080"
- bunkerweb.CUSTOM_CONF_MODSEC_CRS_ip-host=SecRuleRemoveById 920350
volumes:
bw-data:
@ -127,5 +135,7 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-db:
name: bw-db
bw-docker:
name: bw-docker

77
misc/dev/docker-compose.autoconf.ui.misc.yml
View file

@ -2,7 +2,7 @@ x-env: &env
DATABASE_URI: "mariadb+pymysql://bunkerweb:secret@bw-db:3306/db"
DOCKER_HOST: "tcp://bw-docker:2375"
AUTOCONF_MODE: "yes"
LOG_LEVEL: "debug"
CUSTOM_LOG_LEVEL: "debug"
services:
bunkerweb:
@ -15,21 +15,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=
- MULTISITE=yes
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- AUTOCONF_MODE=yes
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- EXTERNAL_PLUGIN_URLS=https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip
- CUSTOM_CONF_MODSEC_CRS_reqbody-rule=SecRuleRemoveById 200002
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -47,10 +34,14 @@ services:
- bw-docker
environment:
<<: *env
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-autoconf
bw-db:
aliases:
- bw-autoconf
bw-docker:
aliases:
- bw-autoconf
@ -61,38 +52,39 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
- ./configs/server-http/hello.conf:/data/configs/server-http/hello.conf:ro
environment:
<<: *env
BUNKERWEB_INSTANCES: ""
SERVER_NAME: ""
MULTISITE: "yes"
API_WHITELIST_IP: "127.0.0.0/24 10.20.30.0/24"
USE_BUNKERNET: "no"
USE_BLACKLIST: "no"
USE_WHITELIST: "no"
SEND_ANONYMOUS_REPORT: "no"
LOG_LEVEL: "info"
SERVE_FILES: "no"
DISABLE_DEFAULT_SERVER: "yes"
USE_CLIENT_CACHE: "yes"
USE_GZIP: "yes"
EXTERNAL_PLUGIN_URLS: "https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip"
CUSTOM_CONF_MODSEC_CRS_reqbody-suppress: "SecRuleRemoveById 200002"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
bw-db:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
bw-ui:
build:
context: ../..
dockerfile: ./src/ui/Dockerfile
depends_on:
- bw-docker
volumes:
- ../../src/ui/src:/usr/share/bunkerweb/ui/src:ro
- ../../src/ui/static:/usr/share/bunkerweb/ui/static:ro
@ -105,11 +97,12 @@ services:
ADMIN_USERNAME: "admin"
ADMIN_PASSWORD: "P@ssw0rd"
DEBUG: "1"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-ui
bw-docker:
bw-db:
aliases:
- bw-ui
labels:
@ -119,7 +112,6 @@ services:
- "bunkerweb.REVERSE_PROXY_URL=/admin"
- "bunkerweb.REVERSE_PROXY_HOST=http://bw-ui:7000"
- "bunkerweb.INTERCEPTED_ERROR_CODES=400 404 405 413 429 500 501 502 503 504"
- bunkerweb.CUSTOM_CONF_MODSEC_CRS_ip-host=SecRuleRemoveById 920350
bw-db:
image: mariadb:11
@ -130,13 +122,28 @@ services:
- MYSQL_PASSWORD=secret
volumes:
- bw-db:/var/lib/mysql
restart: "unless-stopped"
networks:
bw-docker:
bw-db:
aliases:
- bw-db
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
restart: "unless-stopped"
networks:
bw-docker:
aliases:
- bw-docker
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -160,5 +167,7 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-db:
name: bw-db
bw-docker:
name: bw-docker

72
misc/dev/docker-compose.autoconf.ui.yml
View file

@ -2,7 +2,7 @@ x-env: &env
DATABASE_URI: "mariadb+pymysql://bunkerweb:secret@bw-db:3306/db"
DOCKER_HOST: "tcp://bw-docker:2375"
AUTOCONF_MODE: "yes"
LOG_LEVEL: "debug"
CUSTOM_LOG_LEVEL: "debug"
services:
bunkerweb:
@ -15,19 +15,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=
- MULTISITE=yes
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- AUTOCONF_MODE=yes
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -45,10 +34,14 @@ services:
- bw-docker
environment:
<<: *env
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-autoconf
bw-db:
aliases:
- bw-autoconf
bw-docker:
aliases:
- bw-autoconf
@ -59,37 +52,36 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
environment:
<<: *env
BUNKERWEB_INSTANCES: ""
SERVER_NAME: ""
MULTISITE: "yes"
API_WHITELIST_IP: "127.0.0.0/24 10.20.30.0/24"
USE_BUNKERNET: "no"
USE_BLACKLIST: "no"
USE_WHITELIST: "no"
SEND_ANONYMOUS_REPORT: "no"
LOG_LEVEL: "info"
SERVE_FILES: "no"
DISABLE_DEFAULT_SERVER: "yes"
USE_CLIENT_CACHE: "yes"
USE_GZIP: "yes"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
bw-db:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
bw-ui:
build:
context: ../..
dockerfile: ./src/ui/Dockerfile
depends_on:
- bw-docker
volumes:
- ../../src/ui/src:/usr/share/bunkerweb/ui/src:ro
- ../../src/ui/static:/usr/share/bunkerweb/ui/static:ro
@ -102,11 +94,12 @@ services:
ADMIN_USERNAME: "admin"
ADMIN_PASSWORD: "P@ssw0rd"
DEBUG: "1"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-ui
bw-docker:
bw-db:
aliases:
- bw-ui
labels:
@ -126,13 +119,28 @@ services:
- MYSQL_PASSWORD=secret
volumes:
- bw-db:/var/lib/mysql
restart: "unless-stopped"
networks:
bw-docker:
bw-db:
aliases:
- bw-db
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
restart: "unless-stopped"
networks:
bw-docker:
aliases:
- bw-docker
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -156,5 +164,7 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-db:
name: bw-db
bw-docker:
name: bw-docker

72
misc/dev/docker-compose.autoconf.wizard.yml
View file

@ -2,7 +2,7 @@ x-env: &env
DATABASE_URI: "mariadb+pymysql://bunkerweb:secret@bw-db:3306/db"
DOCKER_HOST: "tcp://bw-docker:2375"
AUTOCONF_MODE: "yes"
LOG_LEVEL: "debug"
CUSTOM_LOG_LEVEL: "debug"
services:
bunkerweb:
@ -15,19 +15,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=
- MULTISITE=yes
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- AUTOCONF_MODE=yes
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- UI_HOST=http://bw-ui:7000
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -45,10 +34,14 @@ services:
- bw-docker
environment:
<<: *env
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-autoconf
bw-db:
aliases:
- bw-autoconf
bw-docker:
aliases:
- bw-autoconf
@ -59,37 +52,36 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
environment:
<<: *env
BUNKERWEB_INSTANCES: ""
SERVER_NAME: ""
MULTISITE: "yes"
API_WHITELIST_IP: "127.0.0.0/24 10.20.30.0/24"
USE_BUNKERNET: "no"
USE_BLACKLIST: "no"
USE_WHITELIST: "no"
SEND_ANONYMOUS_REPORT: "no"
LOG_LEVEL: "info"
SERVE_FILES: "no"
USE_CLIENT_CACHE: "yes"
USE_GZIP: "yes"
UI_HOST: "http://bw-ui:7000"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
bw-db:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
bw-ui:
build:
context: ../..
dockerfile: ./src/ui/Dockerfile
depends_on:
- bw-docker
volumes:
- ../../src/ui/src:/usr/share/bunkerweb/ui/src:ro
- ../../src/ui/static:/usr/share/bunkerweb/ui/static:ro
@ -100,11 +92,12 @@ services:
environment:
<<: *env
DEBUG: "1"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-ui
bw-docker:
bw-db:
aliases:
- bw-ui
@ -117,13 +110,28 @@ services:
- MYSQL_PASSWORD=secret
volumes:
- bw-db:/var/lib/mysql
restart: "unless-stopped"
networks:
bw-docker:
bw-db:
aliases:
- bw-db
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
restart: "unless-stopped"
networks:
bw-docker:
aliases:
- bw-docker
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -147,5 +155,7 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-db:
name: bw-db
bw-docker:
name: bw-docker

67
misc/dev/docker-compose.autoconf.yml
View file

@ -2,7 +2,7 @@ x-env: &env
DATABASE_URI: "mariadb+pymysql://bunkerweb:secret@bw-db:3306/db"
DOCKER_HOST: "tcp://bw-docker:2375"
AUTOCONF_MODE: "yes"
LOG_LEVEL: "debug"
CUSTOM_LOG_LEVEL: "debug"
services:
bunkerweb:
@ -15,19 +15,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=
- MULTISITE=yes
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- AUTOCONF_MODE=yes
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -45,10 +34,14 @@ services:
- bw-docker
environment:
<<: *env
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-autoconf
bw-db:
aliases:
- bw-autoconf
bw-docker:
aliases:
- bw-autoconf
@ -59,31 +52,32 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
environment:
<<: *env
BUNKERWEB_INSTANCES: ""
SERVER_NAME: ""
MULTISITE: "yes"
API_WHITELIST_IP: "127.0.0.0/24 10.20.30.0/24"
USE_BUNKERNET: "no"
USE_BLACKLIST: "no"
USE_WHITELIST: "no"
SEND_ANONYMOUS_REPORT: "no"
LOG_LEVEL: "info"
SERVE_FILES: "no"
DISABLE_DEFAULT_SERVER: "yes"
USE_CLIENT_CACHE: "yes"
USE_GZIP: "yes"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
bw-db:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
bw-db:
image: mariadb:11
environment:
@ -93,13 +87,28 @@ services:
- MYSQL_PASSWORD=secret
volumes:
- bw-db:/var/lib/mysql
restart: "unless-stopped"
networks:
bw-docker:
bw-db:
aliases:
- bw-db
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
restart: "unless-stopped"
networks:
bw-docker:
aliases:
- bw-docker
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -123,5 +132,7 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-db:
name: bw-db
bw-docker:
name: bw-docker

58
misc/dev/docker-compose.misc.yml
View file

@ -9,22 +9,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=app1.example.com
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- USE_REVERSE_PROXY=yes
- REVERSE_PROXY_URL=/
- REVERSE_PROXY_HOST=http://app1:8080
- EXTERNAL_PLUGIN_URLS=https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip
- CUSTOM_CONF_MODSEC_CRS_reqbody-suppress=SecRuleRemoveById 200002
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -39,35 +25,39 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
- ./configs/server-http/hello.conf:/data/configs/server-http/hello.conf:ro
environment:
- DOCKER_HOST=tcp://bw-docker:2375
- LOG_LEVEL=debug
- BUNKERWEB_INSTANCES=bunkerweb
- SERVER_NAME=app1.example.com
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- CUSTOM_LOG_LEVEL=debug
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- EXTERNAL_PLUGIN_URLS=https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip
- USE_REVERSE_PROXY=yes
- REVERSE_PROXY_URL=/
- REVERSE_PROXY_HOST=http://app1:8080
- |
CUSTOM_CONF_MODSEC_CRS_reqbody-suppress=
SecRuleRemoveById 200002
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -85,5 +75,3 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-docker:
name: bw-docker

77
misc/dev/docker-compose.ui.misc.yml
View file

@ -1,6 +1,5 @@
x-env: &env
DATABASE_URI: "mariadb+pymysql://bunkerweb:secret@bw-db:3306/db"
DOCKER_HOST: "tcp://bw-docker:2375"
LOG_LEVEL: "debug"
services:
@ -14,29 +13,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=www.example.com app1.example.com
- MULTISITE=yes
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- EXTERNAL_PLUGIN_URLS=https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip
- CUSTOM_CONF_MODSEC_CRS_reqbody-suppress=SecRuleRemoveById 200002
- www.example.com_USE_UI=yes
- www.example.com_USE_REVERSE_PROXY=yes
- www.example.com_REVERSE_PROXY_URL=/admin
- www.example.com_REVERSE_PROXY_HOST=http://bw-ui:7000
- www.example.com_INTERCEPTED_ERROR_CODES=400 404 405 413 429 500 501 502 503 504
- www.example.com_CUSTOM_CONF_MODSEC_CRS_ip-host=SecRuleRemoveById 920350
- app1.example.com_USE_REVERSE_PROXY=yes
- app1.example.com_REVERSE_PROXY_URL=/
- app1.example.com_REVERSE_PROXY_HOST=http://app1:8080
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -51,38 +29,48 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
- ./configs/server-http/hello.conf:/data/configs/server-http/hello.conf:ro
environment:
<<: *env
BUNKERWEB_INSTANCES: "bunkerweb"
SERVER_NAME: "www.example.com app1.example.com"
MULTISITE: "yes"
API_WHITELIST_IP: "127.0.0.0/24 10.20.30.0/24"
USE_BUNKERNET: "no"
USE_BLACKLIST: "no"
USE_WHITELIST: "no"
SEND_ANONYMOUS_REPORT: "no"
CUSTOM_LOG_LEVEL: "debug"
LOG_LEVEL: "info"
SERVE_FILES: "no"
DISABLE_DEFAULT_SERVER: "yes"
USE_CLIENT_CACHE: "yes"
USE_GZIP: "yes"
EXTERNAL_PLUGIN_URLS: "https://github.com/bunkerity/bunkerweb-plugins/archive/refs/heads/dev.zip"
CUSTOM_CONF_MODSEC_CRS_reqbody-suppress: "SecRuleRemoveById 200002"
www.example.com_USE_UI: "yes"
www.example.com_USE_REVERSE_PROXY: "yes"
www.example.com_REVERSE_PROXY_URL: "/admin"
www.example.com_REVERSE_PROXY_HOST: "http://bw-ui:7000"
www.example.com_INTERCEPTED_ERROR_CODES: "400 404 405 413 429 500 501 502 503 504"
app1.example.com_USE_REVERSE_PROXY: "yes"
app1.example.com_REVERSE_PROXY_URL: "/"
app1.example.com_REVERSE_PROXY_HOST: "http://app1:8080"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
bw-db:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
bw-ui:
build:
context: ../..
dockerfile: ./src/ui/Dockerfile
depends_on:
- bw-docker
volumes:
- ../../src/ui/src:/usr/share/bunkerweb/ui/src:ro
- ../../src/ui/static:/usr/share/bunkerweb/ui/static:ro
@ -95,11 +83,12 @@ services:
ADMIN_USERNAME: "admin"
ADMIN_PASSWORD: "P@ssw0rd"
DEBUG: "1"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-ui
bw-docker:
bw-db:
aliases:
- bw-ui
@ -112,13 +101,15 @@ services:
- MYSQL_PASSWORD=secret
volumes:
- bw-db:/var/lib/mysql
restart: "unless-stopped"
networks:
bw-docker:
bw-db:
aliases:
- bw-db
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -137,5 +128,5 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-docker:
name: bw-docker
bw-db:
name: bw-db

72
misc/dev/docker-compose.ui.yml
View file

@ -1,6 +1,5 @@
x-env: &env
DATABASE_URI: "mariadb+pymysql://bunkerweb:secret@bw-db:3306/db"
DOCKER_HOST: "tcp://bw-docker:2375"
LOG_LEVEL: "debug"
services:
@ -14,26 +13,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=www.example.com app1.example.com
- MULTISITE=yes
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- www.example.com_USE_UI=yes
- www.example.com_USE_REVERSE_PROXY=yes
- www.example.com_REVERSE_PROXY_URL=/admin
- www.example.com_REVERSE_PROXY_HOST=http://bw-ui:7000
- www.example.com_INTERCEPTED_ERROR_CODES=400 404 405 413 429 500 501 502 503 504
- app1.example.com_USE_REVERSE_PROXY=yes
- app1.example.com_REVERSE_PROXY_URL=/
- app1.example.com_REVERSE_PROXY_HOST=http://app1:8080
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -48,37 +29,45 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
environment:
<<: *env
BUNKERWEB_INSTANCES: "bunkerweb"
SERVER_NAME: "www.example.com app1.example.com"
MULTISITE: "yes"
API_WHITELIST_IP: "127.0.0.0/24 10.20.30.0/24"
USE_BUNKERNET: "no"
USE_BLACKLIST: "no"
USE_WHITELIST: "no"
SEND_ANONYMOUS_REPORT: "no"
CUSTOM_LOG_LEVEL: "debug"
LOG_LEVEL: "info"
SERVE_FILES: "no"
DISABLE_DEFAULT_SERVER: "yes"
USE_CLIENT_CACHE: "yes"
USE_GZIP: "yes"
www.example.com_USE_UI: "yes"
www.example.com_USE_REVERSE_PROXY: "yes"
www.example.com_REVERSE_PROXY_URL: "/admin"
www.example.com_REVERSE_PROXY_HOST: "http://bw-ui:7000"
www.example.com_INTERCEPTED_ERROR_CODES: "400 404 405 413 429 500 501 502 503 504"
app1.example.com_USE_REVERSE_PROXY: "yes"
app1.example.com_REVERSE_PROXY_URL: "/"
app1.example.com_REVERSE_PROXY_HOST: "http://app1:8080"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
bw-db:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
bw-ui:
build:
context: ../..
dockerfile: ./src/ui/Dockerfile
depends_on:
- bw-docker
volumes:
- ../../src/ui/src:/usr/share/bunkerweb/ui/src:ro
- ../../src/ui/static:/usr/share/bunkerweb/ui/static:ro
@ -91,11 +80,12 @@ services:
ADMIN_USERNAME: "admin"
ADMIN_PASSWORD: "P@ssw0rd"
DEBUG: "1"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-ui
bw-docker:
bw-db:
aliases:
- bw-ui
@ -108,13 +98,15 @@ services:
- MYSQL_PASSWORD=secret
volumes:
- bw-db:/var/lib/mysql
restart: "unless-stopped"
networks:
bw-docker:
bw-db:
aliases:
- bw-db
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -133,5 +125,5 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-docker:
name: bw-docker
bw-db:
name: bw-db

62
misc/dev/docker-compose.wizard.yml
View file

@ -1,6 +1,5 @@
x-env: &env
DATABASE_URI: "mariadb+pymysql://bunkerweb:secret@bw-db:3306/db"
DOCKER_HOST: "tcp://bw-docker:2375"
LOG_LEVEL: "debug"
services:
@ -14,21 +13,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=app1.example.com
- MULTISITE=yes
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- UI_HOST=http://bw-ui:7000
- app1.example.com_USE_REVERSE_PROXY=yes
- app1.example.com_REVERSE_PROXY_URL=/
- app1.example.com_REVERSE_PROXY_HOST=http://app1:8080
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -43,37 +29,40 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
environment:
<<: *env
BUNKERWEB_INSTANCES: "bunkerweb"
SERVER_NAME: "app1.example.com"
MULTISITE: "yes"
API_WHITELIST_IP: "127.0.0.0/24 10.20.30.0/24"
USE_BUNKERNET: "no"
USE_BLACKLIST: "no"
USE_WHITELIST: "no"
SEND_ANONYMOUS_REPORT: "no"
CUSTOM_LOG_LEVEL: "debug"
LOG_LEVEL: "info"
SERVE_FILES: "no"
USE_CLIENT_CACHE: "yes"
USE_GZIP: "yes"
UI_HOST: "http://bw-ui:7000"
app1.example.com_USE_REVERSE_PROXY: "yes"
app1.example.com_REVERSE_PROXY_URL: "/"
app1.example.com_REVERSE_PROXY_HOST: "http://app1:8080"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
bw-db:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
bw-ui:
build:
context: ../..
dockerfile: ./src/ui/Dockerfile
depends_on:
- bw-docker
volumes:
- ../../src/ui/src:/usr/share/bunkerweb/ui/src:ro
- ../../src/ui/static:/usr/share/bunkerweb/ui/static:ro
@ -84,11 +73,12 @@ services:
environment:
<<: *env
DEBUG: "1"
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-ui
bw-docker:
bw-db:
aliases:
- bw-ui
@ -101,13 +91,15 @@ services:
- MYSQL_PASSWORD=secret
volumes:
- bw-db:/var/lib/mysql
restart: "unless-stopped"
networks:
bw-docker:
bw-db:
aliases:
- bw-db
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -126,5 +118,5 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-docker:
name: bw-docker
bw-db:
name: bw-db

52
misc/dev/docker-compose.yml
View file

@ -9,20 +9,8 @@ services:
labels:
- "bunkerweb.INSTANCE=yes"
environment:
- SERVER_NAME=app1.example.com
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- USE_REVERSE_PROXY=yes
- REVERSE_PROXY_URL=/
- REVERSE_PROXY_HOST=http://app1:8080
restart: "unless-stopped"
networks:
bw-universe:
aliases:
@ -37,34 +25,34 @@ services:
dockerfile: ./src/scheduler/Dockerfile
depends_on:
- bunkerweb
- bw-docker
volumes:
- bw-data:/data
environment:
- DOCKER_HOST=tcp://bw-docker:2375
- LOG_LEVEL=debug
- BUNKERWEB_INSTANCES=bunkerweb
- SERVER_NAME=app1.example.com
- API_WHITELIST_IP=127.0.0.0/24 10.20.30.0/24
- USE_BUNKERNET=no
- USE_BLACKLIST=no
- USE_WHITELIST=no
- SEND_ANONYMOUS_REPORT=no
- CUSTOM_LOG_LEVEL=debug
- LOG_LEVEL=info
- SERVE_FILES=no
- DISABLE_DEFAULT_SERVER=yes
- USE_CLIENT_CACHE=yes
- USE_GZIP=yes
- USE_REVERSE_PROXY=yes
- REVERSE_PROXY_URL=/
- REVERSE_PROXY_HOST=http://app1:8080
restart: "unless-stopped"
networks:
bw-universe:
aliases:
- bw-scheduler
bw-docker:
aliases:
- bw-scheduler
bw-docker:
image: tecnativa/docker-socket-proxy:nightly
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
- CONTAINERS=1
- LOG_LEVEL=warning
networks:
bw-docker:
aliases:
- bw-docker
app1:
image: nginxdemos/nginx-hello
restart: "unless-stopped"
networks:
bw-services:
aliases:
@ -82,5 +70,3 @@ networks:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
bw-docker:
name: bw-docker

15
src/autoconf/Config.py
View file

@ -4,7 +4,6 @@ from contextlib import suppress
from datetime import datetime
from os import getenv
from time import sleep
from copy import deepcopy
from ConfigCaller import ConfigCaller # type: ignore
from Database import Database # type: ignore
@ -98,7 +97,7 @@ class Config(ConfigCaller):
self.__configs = configs
changes.append("custom_configs")
if "instances" in changes or "services" in changes:
old_env = deepcopy(self.__config)
old_env = self.__config.copy()
new_env = self.__get_full_env()
if old_env != new_env or first:
self.__config = new_env
@ -150,7 +149,7 @@ class Config(ConfigCaller):
# update instances in database
if "instances" in changes:
err = self._db.update_instances(self.__instances, changed=False)
err = self._db.update_instances(self.__instances, "autoconf", changed=False)
if err:
self.__logger.error(f"Failed to update instances: {err}")
@ -159,22 +158,20 @@ class Config(ConfigCaller):
err = self._db.save_config(self.__config, "autoconf", changed=False)
if err:
success = False
self.__logger.error(
f"Can't save config in database: {err}, config may not work as expected",
)
self.__logger.error(f"Can't save config in database: {err}, config may not work as expected")
# save custom configs to database
if "custom_configs" in changes:
err = self._db.save_custom_configs(custom_configs, "autoconf", changed=False)
if err:
success = False
self.__logger.error(
f"Can't save autoconf custom configs in database: {err}, custom configs may not work as expected",
)
self.__logger.error(f"Can't save autoconf custom configs in database: {err}, custom configs may not work as expected")
# update changes in db
ret = self._db.checked_changes(changes, value=True)
if ret:
self.__logger.error(f"An error occurred when setting the changes to checked in the database : {ret}")
self.__logger.info("Successfully saved new configuration 🚀")
return success

56
src/autoconf/DockerController.py
View file

@ -22,17 +22,14 @@ class DockerController(Controller):
return self.__client.containers.list(filters={"label": "bunkerweb.SERVER_NAME"})
def _to_instances(self, controller_instance) -> List[dict]:
instance = {}
instance["name"] = controller_instance.name
instance["hostname"] = controller_instance.name
instance["health"] = controller_instance.status == "running" and controller_instance.attrs["State"]["Health"]["Status"] == "healthy"
instance["env"] = {}
for env in controller_instance.attrs["Config"]["Env"]:
variable = env.split("=")[0]
value = env.replace(f"{variable}=", "", 1)
if self._is_setting(variable):
instance["env"][variable] = value
return [instance]
return [
{
"name": controller_instance.name,
"hostname": controller_instance.name,
"health": controller_instance.status == "running" and controller_instance.attrs["State"]["Health"]["Status"] == "healthy",
"env": self._get_scheduler_env(),
}
]
def _to_services(self, controller_service) -> List[dict]:
service = {}
@ -45,20 +42,25 @@ class DockerController(Controller):
service[real_variable] = value
return [service]
def _get_static_services(self) -> List[dict]:
services = []
variables = {}
for instance in self.__client.containers.list(filters={"label": "bunkerweb.INSTANCE"}):
def _get_scheduler_env(self) -> Dict[str, str]:
env = {}
for instance in self.__client.containers.list(filters={"label": "bunkerweb.type=scheduler"}):
if not instance.attrs or not instance.attrs.get("Config", {}).get("Env"):
continue
for env in instance.attrs["Config"]["Env"]:
variable = env.split("=")[0]
value = env.replace(f"{variable}=", "", 1)
variables[variable] = value
for env_var in instance.attrs["Config"]["Env"]:
variable = env_var.split("=")[0]
value = env_var.replace(f"{variable}=", "", 1)
env[variable] = value
return env
def _get_static_services(self) -> List[dict]:
services = []
variables = self._get_scheduler_env()
if "SERVER_NAME" in variables and variables["SERVER_NAME"].strip():
for server_name in variables["SERVER_NAME"].strip().split(" "):
if not server_name:
continue
service = {"SERVER_NAME": server_name}
for variable, value in variables.items():
prefix = variable.split("_")[0]
@ -85,9 +87,7 @@ class DockerController(Controller):
# check if server_name exists
if not self._is_service_present(server_name):
self._logger.warning(
f"Ignoring config because {server_name} doesn't exist",
)
self._logger.warning(f"Ignoring config because {server_name} doesn't exist")
continue
for variable, value in labels.items():
@ -100,12 +100,7 @@ class DockerController(Controller):
return configs
def apply_config(self) -> bool:
return self.apply(
self._instances,
self._services,
configs=self._configs,
first=not self._loaded,
)
return self.apply(self._instances, self._services, configs=self._configs, first=not self._loaded)
def __process_event(self, event):
return (
@ -131,10 +126,7 @@ class DockerController(Controller):
if not self.apply_config():
self._logger.error("Error while deploying new configuration")
else:
self._logger.info(
"Successfully deployed new configuration 🚀",
)
self._logger.info("Successfully deployed new configuration 🚀")
self._set_autoconf_load_db()
except:
self._logger.error(f"Exception while processing events :\n{format_exc()}")

1
src/autoconf/Dockerfile
View file

@ -8,6 +8,7 @@ RUN apk add --no-cache build-base libffi-dev postgresql-dev cargo
# Copy python requirements
COPY src/deps/requirements.txt /tmp/requirements-deps.txt
COPY src/autoconf/requirements.txt /tmp/req/requirements-autoconf.txt
COPY src/common/gen/requirements.txt /tmp/req/requirements-gen.txt
COPY src/common/db/requirements.txt /tmp/req/requirements-db.txt
COPY src/common/db/requirements.armv7.txt /tmp/req/requirements-db.armv7.txt

51
src/autoconf/IngressController.py
View file

@ -3,7 +3,7 @@
from contextlib import suppress
from time import sleep
from traceback import format_exc
from typing import List
from typing import Dict, List
from kubernetes import client, config, watch
from kubernetes.client.exceptions import ApiException
from threading import Thread, Lock
@ -27,9 +27,11 @@ class IngressController(Controller):
]
def _to_instances(self, controller_instance) -> List[dict]:
instance = {}
instance["name"] = controller_instance.metadata.name
instance["hostname"] = controller_instance.status.pod_ip or controller_instance.metadata.name
instance = {
"name": controller_instance.metadata.name,
"hostname": controller_instance.metadata.name,
"env": self._get_scheduler_env(),
}
health = False
if controller_instance.status.conditions:
for condition in controller_instance.status.conditions:
@ -37,28 +39,6 @@ class IngressController(Controller):
health = True
break
instance["health"] = health
instance["env"] = {}
pod = None
for container in controller_instance.spec.containers:
if container.name == "bunkerweb":
pod = container
break
if not pod:
self._logger.warning(f"Missing container bunkerweb in pod {controller_instance.metadata.name}")
else:
for env in pod.env:
instance["env"][env.name] = env.value or ""
for controller_service in self._get_controller_services():
if controller_service.metadata.annotations:
for (
annotation,
value,
) in controller_service.metadata.annotations.items():
if not annotation.startswith("bunkerweb.io/"):
continue
variable = annotation.replace("bunkerweb.io/", "", 1)
if self._is_setting(variable):
instance["env"][variable] = value
return [instance]
def _get_controller_services(self) -> list:
@ -176,23 +156,26 @@ class IngressController(Controller):
service["CUSTOM_SSL_KEY_DATA"] = secret_tls.data["tls.key"]
return services
def _get_static_services(self) -> List[dict]:
services = []
def _get_scheduler_env(self) -> Dict[str, str]:
variables = {}
for instance in self.__corev1.list_pod_for_all_namespaces(watch=False).items:
if not instance.metadata.annotations or "bunkerweb.io/INSTANCE" not in instance.metadata.annotations:
if not instance.metadata.annotations or "bunkerweb.io/SCHEDULER" not in instance.metadata.annotations:
continue
pod = None
for container in instance.spec.containers:
if container.name == "bunkerweb":
if container.name == "bunkerweb-scheduler":
pod = container
break
if not pod:
continue
variables = {env.name: env.value or "" for env in pod.env}
return variables
def _get_static_services(self) -> List[dict]:
services = []
variables = self._get_scheduler_env()
if "SERVER_NAME" in variables and variables["SERVER_NAME"].strip():
for server_name in variables["SERVER_NAME"].strip().split(" "):
service = {"SERVER_NAME": server_name}
@ -299,14 +282,10 @@ class IngressController(Controller):
locked = False
except ApiException as e:
if e.status != 410:
self._logger.error(
f"API exception while reading k8s event (type = {watch_type}) :\n{format_exc()}",
)
self._logger.error(f"API exception while reading k8s event (type = {watch_type}) :\n{format_exc()}")
error = True
except:
self._logger.error(
f"Unknown exception while reading k8s event (type = {watch_type}) :\n{format_exc()}",
)
self._logger.error(f"Unknown exception while reading k8s event (type = {watch_type}) :\n{format_exc()}")
error = True
finally:
if locked:

29
src/autoconf/SwarmController.py
View file

@ -32,13 +32,6 @@ class SwarmController(Controller):
def _to_instances(self, controller_instance) -> List[dict]:
self.__swarm_instances.append(controller_instance.id)
instances = []
instance_env = {}
for env in controller_instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
variable = env.split("=")[0]
value = env.replace(f"{variable}=", "", 1)
if self._is_setting(variable):
instance_env[variable] = value
for task in controller_instance.tasks():
if task["DesiredState"] != "running":
continue
@ -47,7 +40,7 @@ class SwarmController(Controller):
"name": task["ID"],
"hostname": f"{controller_instance.name}.{task['NodeID']}.{task['ID']}",
"health": task["Status"]["State"] == "running",
"env": instance_env,
"env": self._get_scheduler_env(),
}
)
return instances
@ -64,19 +57,25 @@ class SwarmController(Controller):
service[real_variable] = value
return [service]
def _get_static_services(self) -> List[dict]:
services = []
variables = {}
for instance in self.__client.services.list(filters={"label": "bunkerweb.INSTANCE"}):
def _get_scheduler_env(self) -> Dict[str, str]:
env = {}
for instance in self.__client.services.list(filters={"label": "bunkerweb.type=scheduler"}):
if not instance.attrs or not instance.attrs.get("Spec", {}).get("TaskTemplate", {}).get("ContainerSpec", {}).get("Env"):
continue
for env in instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
variable = env.split("=")[0]
value = env.replace(f"{variable}=", "", 1)
variables[variable] = value
env[variable] = value
return env
def _get_static_services(self) -> List[dict]:
services = []
variables = self._get_scheduler_env()
if "SERVER_NAME" in variables and variables["SERVER_NAME"].strip():
for server_name in variables["SERVER_NAME"].strip().split(" "):
if not server_name:
continue
service = {}
service["SERVER_NAME"] = server_name
for variable, value in variables.items():
@ -175,9 +174,7 @@ class SwarmController(Controller):
self._logger.error(f"Exception while processing Swarm event ({event_type}) :\n{format_exc()}")
locked = False
except:
self._logger.error(
f"Exception while reading Swarm event ({event_type}) :\n{format_exc()}",
)
self._logger.error(f"Exception while reading Swarm event ({event_type}) :\n{format_exc()}")
error = True
finally:
if locked:

24
src/autoconf/main.py
View file

@ -17,21 +17,21 @@ from IngressController import IngressController
from DockerController import DockerController
# Get variables
logger = setup_logger("Autoconf", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Autoconf", getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")))
swarm = getenv("SWARM_MODE", "no").lower() == "yes"
kubernetes = getenv("KUBERNETES_MODE", "no").lower() == "yes"
docker_host = getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
wait_retry_interval = getenv("WAIT_RETRY_INTERVAL", "5")
if not wait_retry_interval.isdigit():
logger.error("Invalid WAIT_RETRY_INTERVAL value, must be an integer")
LOGGER.error("Invalid WAIT_RETRY_INTERVAL value, must be an integer")
_exit(1)
wait_retry_interval = int(wait_retry_interval)
def exit_handler(signum, frame):
logger.info("Stop signal received, exiting...")
LOGGER.info("Stop signal received, exiting...")
_exit(0)
@ -41,36 +41,36 @@ signal(SIGTERM, exit_handler)
try:
# Instantiate the controller
if swarm:
logger.info("Swarm mode detected")
LOGGER.info("Swarm mode detected")
controller = SwarmController(docker_host)
elif kubernetes:
logger.info("Kubernetes mode detected")
LOGGER.info("Kubernetes mode detected")
controller = IngressController()
else:
logger.info("Docker mode detected")
LOGGER.info("Docker mode detected")
controller = DockerController(docker_host)
# Wait for instances
logger.info("Waiting for BunkerWeb instances ...")
LOGGER.info("Waiting for BunkerWeb instances ...")
instances = controller.wait(wait_retry_interval)
logger.info("BunkerWeb instances are ready 🚀")
LOGGER.info("BunkerWeb instances are ready 🚀")
i = 1
for instance in instances:
logger.info(f"Instance #{i} : {instance['name']}")
LOGGER.info(f"Instance #{i} : {instance['name']}")
i += 1
# Run first configuration
ret = controller.apply_config()
if not ret:
logger.error("Error while applying initial configuration")
LOGGER.error("Error while applying initial configuration")
_exit(1)
# Process events
Path(sep, "var", "tmp", "bunkerweb", "autoconf.healthy").write_text("ok")
logger.info("Processing events ...")
LOGGER.info("Processing events ...")
controller.process_events()
except:
logger.error(f"Exception while running autoconf :\n{format_exc()}")
LOGGER.error(f"Exception while running autoconf :\n{format_exc()}")
sys_exit(1)
finally:
Path(sep, "var", "tmp", "bunkerweb", "autoconf.healthy").unlink(missing_ok=True)

2
src/autoconf/requirements.in Normal file
View file

@ -0,0 +1,2 @@
docker==7.1.0
kubernetes==29.0.0

229
src/autoconf/requirements.txt Normal file
View file

@ -0,0 +1,229 @@
#
# This file is autogenerated by pip-compile with Python 3.9
# by the following command:
#
# pip-compile --allow-unsafe --generate-hashes --strip-extras requirements.in
#
cachetools==5.3.3 \
--hash=sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945 \
--hash=sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105
# via google-auth
certifi==2024.2.2 \
--hash=sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f \
--hash=sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1
# via
# kubernetes
# requests
charset-normalizer==3.3.2 \
--hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \
--hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \
--hash=sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786 \
--hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \
--hash=sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09 \
--hash=sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185 \
--hash=sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574 \
--hash=sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e \
--hash=sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519 \
--hash=sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898 \
--hash=sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269 \
--hash=sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3 \
--hash=sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f \
--hash=sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6 \
--hash=sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8 \
--hash=sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a \
--hash=sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73 \
--hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \
--hash=sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714 \
--hash=sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2 \
--hash=sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc \
--hash=sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce \
--hash=sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d \
--hash=sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e \
--hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \
--hash=sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269 \
--hash=sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96 \
--hash=sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d \
--hash=sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a \
--hash=sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4 \
--hash=sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77 \
--hash=sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d \
--hash=sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0 \
--hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \
--hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \
--hash=sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac \
--hash=sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25 \
--hash=sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8 \
--hash=sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab \
--hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \
--hash=sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2 \
--hash=sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db \
--hash=sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f \
--hash=sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5 \
--hash=sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99 \
--hash=sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c \
--hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \
--hash=sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811 \
--hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \
--hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \
--hash=sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03 \
--hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \
--hash=sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04 \
--hash=sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c \
--hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \
--hash=sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458 \
--hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \
--hash=sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99 \
--hash=sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985 \
--hash=sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537 \
--hash=sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238 \
--hash=sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f \
--hash=sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d \
--hash=sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796 \
--hash=sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a \
--hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \
--hash=sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8 \
--hash=sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c \
--hash=sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5 \
--hash=sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5 \
--hash=sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711 \
--hash=sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4 \
--hash=sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6 \
--hash=sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c \
--hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \
--hash=sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4 \
--hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \
--hash=sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae \
--hash=sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12 \
--hash=sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c \
--hash=sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae \
--hash=sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8 \
--hash=sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887 \
--hash=sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b \
--hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \
--hash=sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f \
--hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 \
--hash=sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33 \
--hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \
--hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561
# via requests
docker==7.1.0 \
--hash=sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c \
--hash=sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0
# via -r requirements.in
google-auth==2.29.0 \
--hash=sha256:672dff332d073227550ffc7457868ac4218d6c500b155fe6cc17d2b13602c360 \
--hash=sha256:d452ad095688cd52bae0ad6fafe027f6a6d6f560e810fec20914e17a09526415
# via kubernetes
idna==3.7 \
--hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \
--hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0
# via requests
kubernetes==29.0.0 \
--hash=sha256:ab8cb0e0576ccdfb71886366efb102c6a20f268d817be065ce7f9909c631e43e \
--hash=sha256:c4812e227ae74d07d53c88293e564e54b850452715a59a927e7e1bc6b9a60459
# via -r requirements.in
oauthlib==3.2.2 \
--hash=sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca \
--hash=sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918
# via
# kubernetes
# requests-oauthlib
pyasn1==0.6.0 \
--hash=sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c \
--hash=sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473
# via
# pyasn1-modules
# rsa
pyasn1-modules==0.4.0 \
--hash=sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6 \
--hash=sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b
# via google-auth
python-dateutil==2.9.0.post0 \
--hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
--hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
# via kubernetes
pyyaml==6.0.1 \
--hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \
--hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \
--hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \
--hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \
--hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \
--hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \
--hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \
--hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \
--hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \
--hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \
--hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \
--hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \
--hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \
--hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \
--hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \
--hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \
--hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \
--hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \
--hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \
--hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \
--hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \
--hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \
--hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \
--hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \
--hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \
--hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \
--hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \
--hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \
--hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \
--hash=sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef \
--hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \
--hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \
--hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \
--hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \
--hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \
--hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \
--hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \
--hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \
--hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \
--hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \
--hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \
--hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \
--hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \
--hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \
--hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \
--hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \
--hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \
--hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \
--hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \
--hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \
--hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f
# via kubernetes
requests==2.32.2 \
--hash=sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289 \
--hash=sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c
# via
# docker
# kubernetes
# requests-oauthlib
requests-oauthlib==2.0.0 \
--hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \
--hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9
# via kubernetes
rsa==4.9 \
--hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \
--hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21
# via google-auth
six==1.16.0 \
--hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
# via
# kubernetes
# python-dateutil
urllib3==2.2.1 \
--hash=sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d \
--hash=sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19
# via
# docker
# kubernetes
# requests
websocket-client==1.8.0 \
--hash=sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 \
--hash=sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da
# via kubernetes

34
src/common/cli/CLI.py
View file

@ -71,7 +71,6 @@ class CLI(ApiCaller):
assert isinstance(self.__variables, dict), "Failed to get variables from database"
self.__integration = self.__detect_integration()
self.__use_redis = self.__get_variable("USE_REDIS", "no") == "yes"
self.__redis = None
if self.__use_redis:
@ -179,41 +178,12 @@ class CLI(ApiCaller):
self.__logger.error("USE_REDIS is set to yes but REDIS_HOST or REDIS_SENTINEL_HOSTS is not set, disabling redis")
self.__use_redis = False
if self.__integration == "linux":
super().__init__(
[
API(
f"http://127.0.0.1:{self.__get_variable('API_HTTP_PORT', '5000')}",
host=self.__get_variable("API_SERVER_NAME", "bwapi"),
)
]
)
else:
super().__init__()
self.auto_setup(self.__integration)
for db_instance in self.__db.get_instances():
self.apis.append(API(db_instance["hostname"], db_instance["port"], db_instance["server_name"]))
def __get_variable(self, variable: str, default: Optional[Any] = None) -> Optional[str]:
return getenv(variable, self.__variables.get(variable, default))
def __detect_integration(self) -> str:
if Path(sep, "usr", "sbin", "nginx").exists():
return "linux"
integration_path = Path(sep, "usr", "share", "bunkerweb", "INTEGRATION")
os_release_path = Path(sep, "etc", "os-release")
if self.__get_variable("KUBERNETES_MODE", "no").lower() == "yes": # type: ignore
return "kubernetes"
elif self.__get_variable("SWARM_MODE", "no").lower() == "yes": # type: ignore
return "swarm"
elif self.__get_variable("AUTOCONF_MODE", "no").lower() == "yes": # type: ignore
return "autoconf"
elif integration_path.is_file():
return integration_path.read_text(encoding="utf-8").strip().lower()
elif os_release_path.is_file() and "Alpine" in os_release_path.read_text(encoding="utf-8"):
return "docker"
return "linux"
def unban(self, ip: str) -> Tuple[bool, str]:
if self.__redis:
try:

25
src/common/db/Database.py
View file

@ -351,8 +351,9 @@ class Database:
if hasattr(metadata, key) and key not in ("database_version", "default"):
data[key] = getattr(metadata, key)
data["default"] = False
except BaseException:
self.logger.debug(f"Can't get the metadata: {format_exc()}")
except BaseException as e:
if "doesn't exist" not in str(e):
self.logger.debug(f"Can't get the metadata: {format_exc()}")
return data
@ -2084,7 +2085,7 @@ class Database:
)
return cache_files
def add_instance(self, hostname: str, port: int, server_name: str, changed: Optional[bool] = True) -> str:
def add_instance(self, hostname: str, port: int, server_name: str, method: str, changed: Optional[bool] = True) -> str:
"""Add instance."""
with self.__db_session() as session:
if self.readonly:
@ -2095,7 +2096,7 @@ class Database:
if db_instance is not None:
return f"Instance {hostname} already exists, will not be added."
session.add(Instances(hostname=hostname, port=port, server_name=server_name))
session.add(Instances(hostname=hostname, port=port, server_name=server_name, method=method))
if changed:
with suppress(ProgrammingError, OperationalError):
@ -2106,18 +2107,18 @@ class Database:
try:
session.commit()
except BaseException:
return f"An error occurred while adding the instance {hostname} (port: {port}, server name: {server_name}).\n{format_exc()}"
return f"An error occurred while adding the instance {hostname} (port: {port}, server name: {server_name}, method: {method}).\n{format_exc()}"
return ""
def update_instances(self, instances: List[Dict[str, Any]], changed: Optional[bool] = True) -> str:
def update_instances(self, instances: List[Dict[str, Any]], method: str, changed: Optional[bool] = True) -> str:
"""Update instances."""
to_put = []
with self.__db_session() as session:
if self.readonly:
return "The database is read-only, the changes will not be saved"
session.query(Instances).delete()
session.query(Instances).filter(Instances.method == method).delete()
for instance in instances:
to_put.append(
@ -2125,6 +2126,7 @@ class Database:
hostname=instance["hostname"],
port=instance["env"].get("API_HTTP_PORT", 5000),
server_name=instance["env"].get("API_SERVER_NAME", "bwapi"),
method=method,
)
)
@ -2142,16 +2144,21 @@ class Database:
return ""
def get_instances(self) -> List[Dict[str, Any]]:
def get_instances(self, *, method: Optional[str] = None) -> List[Dict[str, Any]]:
"""Get instances."""
with self.__db_session() as session:
query = session.query(Instances)
if method:
query = query.filter_by(method=method)
return [
{
"hostname": instance.hostname,
"port": instance.port,
"server_name": instance.server_name,
"method": instance.method,
}
for instance in (session.query(Instances).with_entities(Instances.hostname, Instances.port, Instances.server_name))
for instance in query
]
def get_plugin_actions(self, plugin: str) -> Optional[Any]:

1
src/common/db/model.py
View file

@ -201,6 +201,7 @@ class Instances(Base):
hostname = Column(String(256), primary_key=True)
port = Column(Integer, nullable=False)
server_name = Column(String(256), nullable=False)
method = Column(METHODS_ENUM, nullable=False, default="manual")
class Users(Base):

15
src/common/gen/Configurator.py
View file

@ -187,18 +187,25 @@ class Configurator:
config[variable] = value
elif (
"CUSTOM_CONF" not in variable
and not variable.startswith(("PYTHON", "KUBERNETES_SERVICE_", "KUBERNETES_PORT_", "SVC_"))
and not variable.startswith(("_", "PYTHON", "KUBERNETES_SERVICE_", "KUBERNETES_PORT_", "SVC_"))
and variable
not in (
"DOCKER_HOST",
"SLAVE_MODE",
"MASTER_MODE",
"CUSTOM_LOG_LEVEL",
"GPG_KEY",
"HOME",
"HOSTNAME",
"LANG",
"PATH",
"NGINX_VERSION",
"NJS_VERSION",
"PATH",
"PKG_RELEASE",
"DOCKER_HOST",
"SLAVE_MODE",
"MASTER_MODE",
"PWD",
"SHLVL",
"SERVER_SOFTWARE",
)
):
self.__logger.warning(f"Ignoring variable {variable} : {err}")

3
src/common/gen/requirements.in
View file

@ -1,5 +1,4 @@
docker==7.0.0
jinja2==3.1.4
kubernetes==29.0.0
python-dotenv==1.0.1
redis==5.0.4
requests==2.32.2

125
src/common/gen/requirements.txt
View file

@ -8,16 +8,10 @@ async-timeout==4.0.3 \
--hash=sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f \
--hash=sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028
# via redis
cachetools==5.3.3 \
--hash=sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945 \
--hash=sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105
# via google-auth
certifi==2024.2.2 \
--hash=sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f \
--hash=sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1
# via
# kubernetes
# requests
# via requests
charset-normalizer==3.3.2 \
--hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \
--hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \
@ -110,14 +104,6 @@ charset-normalizer==3.3.2 \
--hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \
--hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561
# via requests
docker==7.0.0 \
--hash=sha256:12ba681f2777a0ad28ffbcc846a69c31b4dfd9752b47eb425a274ee269c5e14b \
--hash=sha256:323736fb92cd9418fc5e7133bc953e11a9da04f4483f828b527db553f1e7e5a3
# via -r requirements.in
google-auth==2.29.0 \
--hash=sha256:672dff332d073227550ffc7457868ac4218d6c500b155fe6cc17d2b13602c360 \
--hash=sha256:d452ad095688cd52bae0ad6fafe027f6a6d6f560e810fec20914e17a09526415
# via kubernetes
idna==3.7 \
--hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \
--hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0
@ -126,10 +112,6 @@ jinja2==3.1.4 \
--hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \
--hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d
# via -r requirements.in
kubernetes==29.0.0 \
--hash=sha256:ab8cb0e0576ccdfb71886366efb102c6a20f268d817be065ce7f9909c631e43e \
--hash=sha256:c4812e227ae74d07d53c88293e564e54b850452715a59a927e7e1bc6b9a60459
# via -r requirements.in
markupsafe==2.1.5 \
--hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \
--hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \
@ -192,87 +174,10 @@ markupsafe==2.1.5 \
--hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \
--hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68
# via jinja2
oauthlib==3.2.2 \
--hash=sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca \
--hash=sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918
# via
# kubernetes
# requests-oauthlib
packaging==24.0 \
--hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \
--hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9
# via docker
pyasn1==0.6.0 \
--hash=sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c \
--hash=sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473
# via
# pyasn1-modules
# rsa
pyasn1-modules==0.4.0 \
--hash=sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6 \
--hash=sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b
# via google-auth
python-dateutil==2.9.0.post0 \
--hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
--hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
# via kubernetes
python-dotenv==1.0.1 \
--hash=sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca \
--hash=sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a
# via -r requirements.in
pyyaml==6.0.1 \
--hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \
--hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \
--hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \
--hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \
--hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \
--hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \
--hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \
--hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \
--hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \
--hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \
--hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \
--hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \
--hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \
--hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \
--hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \
--hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \
--hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \
--hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \
--hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \
--hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \
--hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \
--hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \
--hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \
--hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \
--hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \
--hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \
--hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \
--hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \
--hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \
--hash=sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef \
--hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \
--hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \
--hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \
--hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \
--hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \
--hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \
--hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \
--hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \
--hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \
--hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \
--hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \
--hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \
--hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \
--hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \
--hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \
--hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \
--hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \
--hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \
--hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \
--hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \
--hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f
# via kubernetes
redis==5.0.4 \
--hash=sha256:7adc2835c7a9b5033b7ad8f8918d09b7344188228809c98df07af226d39dec91 \
--hash=sha256:ec31f2ed9675cc54c21ba854cfe0462e6faf1d83c8ce5944709db8a4700b9c61
@ -280,32 +185,8 @@ redis==5.0.4 \
requests==2.32.2 \
--hash=sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289 \
--hash=sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c
# via
# docker
# kubernetes
# requests-oauthlib
requests-oauthlib==2.0.0 \
--hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \
--hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9
# via kubernetes
rsa==4.9 \
--hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \
--hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21
# via google-auth
six==1.16.0 \
--hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
# via
# kubernetes
# python-dateutil
# via -r requirements.in
urllib3==2.2.1 \
--hash=sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d \
--hash=sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19
# via
# docker
# kubernetes
# requests
websocket-client==1.8.0 \
--hash=sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 \
--hash=sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da
# via kubernetes
# via requests

294
src/common/gen/save_config.py
View file

@ -6,75 +6,31 @@ from os.path import join
from pathlib import Path
from re import compile as re_compile
from sys import exit as sys_exit, path as sys_path
from time import sleep
from traceback import format_exc
from typing import Any
for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]:
if deps_path not in sys_path:
sys_path.append(deps_path)
from docker import DockerClient
from common_utils import get_integration # type: ignore
from common_utils import get_integration, get_version # type: ignore
from logger import setup_logger # type: ignore
from Database import Database # type: ignore
from Configurator import Configurator
from API import API # type: ignore
custom_confs_rx = re_compile(r"^([0-9a-z\.-]*)_?CUSTOM_CONF_(HTTP|SERVER_STREAM|STREAM|DEFAULT_SERVER_HTTP|SERVER_HTTP|MODSEC_CRS|MODSEC)_(.+)$")
CUSTOM_CONF_RX = re_compile(
r"^(?P<service>[0-9a-z\.-]*)_?CUSTOM_CONF_(?P<type>HTTP|SERVER_STREAM|STREAM|DEFAULT_SERVER_HTTP|SERVER_HTTP|MODSEC_CRS|MODSEC)_(?P<name>.+)$"
)
BUNKERWEB_STATIC_INSTANCES_RX = re_compile(r"(http://)?(?P<hostname>(?<![:@])\b[^:@\s]+\b)(:(?P<port>\d+))?")
def get_instance_configs_and_apis(instance: Any, db, _type="Docker"):
api_http_port = None
api_server_name = None
tmp_config = {}
custom_confs = []
apis = []
for var in instance.attrs["Config"]["Env"] if _type == "Docker" else instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
split = var.split("=", 1)
if custom_confs_rx.match(split[0]):
custom_conf = custom_confs_rx.search(split[0]).groups()
custom_confs.append(
{
"value": f"# CREATED BY ENV\n{split[1]}",
"exploded": (
custom_conf[0],
custom_conf[1],
custom_conf[2].replace(".conf", ""),
),
}
)
logger.info(
f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}"
)
else:
tmp_config[split[0]] = split[1]
if not db and split[0] == "DATABASE_URI":
db = Database(logger, sqlalchemy_string=split[1])
elif split[0] == "API_HTTP_PORT":
api_http_port = split[1]
elif split[0] == "API_SERVER_NAME":
api_server_name = split[1]
apis.append(
API(
f"http://{instance.name}:{api_http_port or getenv('API_HTTP_PORT', '5000')}",
host=api_server_name or getenv("API_SERVER_NAME", "bwapi"),
)
)
return tmp_config, custom_confs, apis, db
LOGGER = setup_logger("Generator", getenv("LOG_LEVEL", "INFO"))
if __name__ == "__main__":
logger = setup_logger("Generator", getenv("LOG_LEVEL", "INFO"))
wait_retry_interval = getenv("WAIT_RETRY_INTERVAL", "5")
if not wait_retry_interval.isdigit():
logger.error("Invalid WAIT_RETRY_INTERVAL value, must be an integer")
LOGGER.error("Invalid WAIT_RETRY_INTERVAL value, must be an integer")
sys_exit(1)
wait_retry_interval = int(wait_retry_interval)
@ -97,227 +53,181 @@ if __name__ == "__main__":
plugins_path = Path(args.plugins)
pro_plugins_path = Path(args.pro_plugins)
logger.info("Save config started ...")
logger.info(f"Settings : {settings_path}")
logger.info(f"Core : {core_path}")
logger.info(f"Plugins : {plugins_path}")
logger.info(f"Pro plugins : {pro_plugins_path}")
logger.info(f"Init : {args.init}")
LOGGER.info("Save config started ...")
LOGGER.info(f"Settings : {settings_path}")
LOGGER.info(f"Core : {core_path}")
LOGGER.info(f"Plugins : {plugins_path}")
LOGGER.info(f"Pro plugins : {pro_plugins_path}")
LOGGER.info(f"Init : {args.init}")
integration = get_integration()
if args.init:
logger.info(f"Detected {integration} integration")
LOGGER.info(f"Detected {integration} integration")
if integration == "Linux" and not args.variables:
args.variables = join(sep, "etc", "bunkerweb", "variables.env")
config_files = None
db = None
apis = []
external_plugins = args.plugins
pro_plugins = args.pro_plugins
if not Path(sep, "usr", "sbin", "nginx").exists() and args.method == "ui":
db = Database(logger)
external_plugins = db.get_plugins(_type="external")
pro_plugins = db.get_plugins(_type="pro")
# Check existences and permissions
logger.info("Checking arguments ...")
LOGGER.info("Checking arguments ...")
files = [settings_path] + ([Path(args.variables)] if args.variables else [])
paths_rx = [core_path, plugins_path, pro_plugins_path]
for file in files:
if not file.is_file():
logger.error(f"Missing file : {file}")
LOGGER.error(f"Missing file : {file}")
sys_exit(1)
if not access(file, R_OK):
logger.error(f"Can't read file : {file}")
LOGGER.error(f"Can't read file : {file}")
sys_exit(1)
for path in paths_rx:
if not path.is_dir():
logger.error(f"Missing directory : {path}")
LOGGER.error(f"Missing directory : {path}")
sys_exit(1)
if not access(path, R_OK | X_OK):
logger.error(f"Missing RX rights on directory : {path}")
LOGGER.error(f"Missing RX rights on directory : {path}")
sys_exit(1)
if args.variables:
variables_path = Path(args.variables)
logger.info(f"Variables : {variables_path}")
# Compute the config
logger.info("Computing config ...")
config = Configurator(str(settings_path), str(core_path), external_plugins, pro_plugins, str(variables_path), logger)
config_files = config.get_config()
custom_confs = []
for k, v in environ.items():
if custom_confs_rx.match(k):
custom_conf = custom_confs_rx.search(k).groups()
custom_confs.append(
{
"value": f"# CREATED BY ENV\n{v}",
"exploded": (
custom_conf[0],
custom_conf[1],
custom_conf[2].replace(".conf", ""),
),
}
)
logger.info(
f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}"
)
db = Database(logger, config_files.get("DATABASE_URI", None))
elif getenv("KUBERNETES_MODE", "no") != "yes":
docker_client = DockerClient(base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock"))
while not docker_client.containers.list(filters={"label": "bunkerweb.INSTANCE"}):
logger.info("Waiting for BunkerWeb instance ...")
sleep(5)
api_http_port = None
api_server_name = None
tmp_config = {}
custom_confs = []
apis = []
for instance in docker_client.containers.list(filters={"label": "bunkerweb.INSTANCE"}):
for var in instance.attrs["Config"]["Env"]:
split = var.split("=", 1)
if custom_confs_rx.match(split[0]):
custom_conf = custom_confs_rx.search(split[0]).groups()
custom_confs.append(
{
"value": f"# CREATED BY ENV\n{split[1]}",
"exploded": (
custom_conf[0],
custom_conf[1],
custom_conf[2].replace(".conf", ""),
),
}
)
logger.info(
f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}"
)
else:
tmp_config[split[0]] = split[1]
if not db and split[0] == "DATABASE_URI":
db = Database(logger, sqlalchemy_string=split[1])
elif split[0] == "API_HTTP_PORT":
api_http_port = split[1]
elif split[0] == "API_SERVER_NAME":
api_server_name = split[1]
apis.append(
API(
f"http://{instance.name}:{api_http_port or getenv('API_HTTP_PORT', '5000')}",
host=api_server_name or getenv("API_SERVER_NAME", "bwapi"),
)
)
if not db:
db = Database(logger)
LOGGER.info(f"Variables : {variables_path}")
# Compute the config
if not config_files:
logger.info("Computing config ...")
config = Configurator(args.settings, args.core, external_plugins, pro_plugins, tmp_config, logger)
config_files = config.get_config()
LOGGER.info("Computing config ...")
config = Configurator(
str(settings_path), str(core_path), external_plugins, pro_plugins, str(variables_path) if args.variables else environ.copy(), LOGGER
)
settings = config.get_config()
bunkerweb_version = Path(sep, "usr", "share", "bunkerweb", "VERSION").read_text().strip()
# Parse BunkerWeb instances from environment
apis = []
hostnames = set()
for bw_instance in settings.get("BUNKERWEB_INSTANCES", "").split(" "):
if not bw_instance:
continue
match = BUNKERWEB_STATIC_INSTANCES_RX.search(bw_instance)
if match:
if match.group("hostname") in hostnames:
LOGGER.warning(f"Duplicate BunkerWeb instance hostname {match.group('hostname')}, skipping it")
hostnames.add(match.group("hostname"))
apis.append(
API(
f"http://{match.group('hostname')}:{match.group('port') or settings.get('API_HTTP_PORT', '5000')}",
host=settings.get("API_SERVER_NAME", "bwapi"),
)
)
else:
LOGGER.warning(
f"Invalid BunkerWeb instance {bw_instance}, it should match the following regex: (http://)<hostname>(:<port>) ({BUNKERWEB_STATIC_INSTANCES_RX.pattern}), skipping it"
)
custom_confs = []
for k, v in settings.items():
if CUSTOM_CONF_RX.match(k):
custom_conf = CUSTOM_CONF_RX.search(k)
custom_confs.append(
{
"value": f"# CREATED BY ENV\n{v}",
"exploded": (
custom_conf.group("service"),
custom_conf.group("type"),
custom_conf.group("name").replace(".conf", ""),
),
}
)
LOGGER.info(
f"Found custom conf env var {'for service ' + custom_conf.group('service') if custom_conf.group('service') else 'without service'} with type {custom_conf.group('type')} and name {custom_conf.group('name')}"
)
continue
db = Database(LOGGER)
bunkerweb_version = get_version()
db_metadata = db.get_metadata()
db_initialized = isinstance(db_metadata, str) or not db_metadata["is_initialized"]
if not db_initialized:
logger.info("Database not initialized, initializing ...")
LOGGER.info("Database not initialized, initializing ...")
ret, err = db.init_tables(
[config.get_settings(), config.get_plugins("core"), config.get_plugins("external"), config.get_plugins("pro")], bunkerweb_version
)
# Initialize database tables
if err:
logger.error(f"Exception while initializing database : {err}")
LOGGER.error(f"Exception while initializing database : {err}")
sys_exit(1)
elif not ret:
logger.info("Database tables are already initialized, skipping creation ...")
LOGGER.info("Database tables are already initialized, skipping creation ...")
else:
logger.info("Database tables initialized")
LOGGER.info("Database tables initialized")
else:
logger.info("Database is already initialized, checking for changes ...")
LOGGER.info("Database is already initialized, checking for changes ...")
ret, err = db.init_tables(
[config.get_settings(), config.get_plugins("core"), config.get_plugins("external"), config.get_plugins("pro")], bunkerweb_version
)
if not ret and err:
logger.error(f"Exception while checking database tables : {err}")
LOGGER.error(f"Exception while checking database tables : {err}")
sys_exit(1)
elif not ret:
logger.info("Database tables didn't change, skipping update ...")
LOGGER.info("Database tables didn't change, skipping update ...")
else:
logger.info("Database tables successfully updated")
LOGGER.info("Database tables successfully updated")
err = db.initialize_db(version=bunkerweb_version, integration=integration)
if err:
logger.error(f"Can't {'initialize' if not db_initialized else 'update'} database metadata : {err}")
LOGGER.error(f"Can't {'initialize' if not db_initialized else 'update'} database metadata : {err}")
sys_exit(1)
else:
logger.info("Database metadata successfully " + ("initialized" if not db_initialized else "updated"))
LOGGER.info("Database metadata successfully " + ("initialized" if not db_initialized else "updated"))
if args.init:
sys_exit(0)
changes = []
err = db.save_config(config_files, args.method, changed=False)
err = db.save_config(settings, args.method, changed=False)
if err:
logger.warning(f"Couldn't save config to database : {err}, config may not work as expected")
LOGGER.warning(f"Couldn't save config to database : {err}, config may not work as expected")
else:
changes.append("config")
logger.info("Config successfully saved to database")
LOGGER.info("Config successfully saved to database")
if args.method != "ui":
err1 = db.save_custom_configs(custom_confs, args.method, changed=False)
err1 = db.save_custom_configs(custom_confs, args.method, changed=False)
if err1:
logger.warning(f"Couldn't save custom configs to database : {err1}, custom configs may not work as expected")
if err1:
LOGGER.warning(f"Couldn't save custom configs to database : {err1}, custom configs may not work as expected")
else:
changes.append("custom_configs")
LOGGER.info("Custom configs successfully saved to database")
for api in apis:
endpoint_data = api.endpoint.replace("http://", "").split(":")
err = db.add_instance(endpoint_data[0], endpoint_data[1].replace("/", ""), api.host, method="manual", changed=False)
if err:
LOGGER.warning(err)
else:
changes.append("custom_configs")
logger.info("Custom configs successfully saved to database")
if apis:
for api in apis:
endpoint_data = api.endpoint.replace("http://", "").split(":")
err = db.add_instance(endpoint_data[0], endpoint_data[1].replace("/", ""), api.host, changed=False)
if err:
logger.warning(err)
else:
if "instances" not in changes:
changes.append("instances")
logger.info(f"Instance {endpoint_data[0]} successfully saved to database")
else:
err = db.add_instance("127.0.0.1", config_files.get("API_HTTP_PORT", 5000), config_files.get("API_SERVER_NAME", "bwapi"), changed=False)
if err:
logger.warning(err)
else:
if "instances" not in changes:
changes.append("instances")
logger.info("Instance 127.0.0.1 successfully saved to database")
LOGGER.info(f"Instance {endpoint_data[0]} successfully saved to database")
if not args.no_check_changes:
# update changes in db
ret = db.checked_changes(changes, value=True)
if ret:
logger.error(f"An error occurred when setting the changes to checked in the database : {ret}")
LOGGER.error(f"An error occurred when setting the changes to checked in the database : {ret}")
except SystemExit as e:
sys_exit(e.code)
except:
logger.error(f"Exception while executing config saver : {format_exc()}")
LOGGER.error(f"Exception while executing config saver : {format_exc()}")
sys_exit(1)
# We're done
logger.info("Config saver successfully executed !")
LOGGER.info("Config saver successfully executed !")

10
src/common/settings.json
View file

@ -317,12 +317,12 @@
"emerg"
]
},
"OVERRIDE_INSTANCES": {
"BUNKERWEB_INSTANCES": {
"context": "global",
"default": "",
"help": "List of BunkerWeb instances separated with spaces (format : fqdn-or-ip:5000 fqdn-or-ip:5000)",
"id": "override-instances",
"label": "Override instances",
"default": "127.0.0.1",
"help": "List of BunkerWeb instances separated with spaces (format : fqdn-or-ip:5000 http://fqdn-or-ip:5000)",
"id": "bunkerweb-instances",
"label": "BunkerWeb instances",
"regex": "^.*$",
"type": "text"
}

83
src/common/utils/ApiCaller.py
View file

@ -14,91 +14,12 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from API import API # type: ignore
from logger import setup_logger
from docker import DockerClient
from kubernetes import client as kube_client, config
class ApiCaller:
def __init__(self, apis: Optional[List[API]] = None):
self.__apis = apis or []
self.apis = apis or []
self.__logger = setup_logger("Api", getenv("LOG_LEVEL", "INFO"))
@property
def apis(self) -> List[API]:
return self.__apis
@apis.setter
def apis(self, apis: List[API]):
self.__apis = apis
def auto_setup(self, bw_integration: Optional[str] = None):
self.__apis.clear()
if bw_integration is None:
if getenv("KUBERNETES_MODE", "no") == "yes":
bw_integration = "Kubernetes"
elif getenv("SWARM_MODE", "no") == "yes":
bw_integration = "Swarm"
if bw_integration == "Kubernetes":
config.load_incluster_config()
corev1 = kube_client.CoreV1Api()
for pod in corev1.list_pod_for_all_namespaces(watch=False).items:
if pod.metadata.annotations is not None and "bunkerweb.io/INSTANCE" in pod.metadata.annotations:
api_http_port = None
api_server_name = None
for pod_env in pod.spec.containers[0].env:
if pod_env.name == "API_HTTP_PORT":
api_http_port = pod_env.value or "5000"
elif pod_env.name == "API_SERVER_NAME":
api_server_name = pod_env.value or "bwapi"
self.__apis.append(
API(
f"http://{pod.status.pod_ip}:{api_http_port or getenv('API_HTTP_PORT', '5000')}",
host=api_server_name or getenv("API_SERVER_NAME", "bwapi"),
)
)
else:
docker_client = DockerClient(base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock"))
if bw_integration == "Swarm":
for instance in docker_client.services.list(filters={"label": "bunkerweb.INSTANCE"}):
api_http_port = None
api_server_name = None
for var in instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
if var.startswith("API_HTTP_PORT="):
api_http_port = var.replace("API_HTTP_PORT=", "", 1)
elif var.startswith("API_SERVER_NAME="):
api_server_name = var.replace("API_SERVER_NAME=", "", 1)
for task in instance.tasks():
self.__apis.append(
API(
f"http://{instance.name}.{task['NodeID']}.{task['ID']}:{api_http_port or getenv('API_HTTP_PORT', '5000')}",
host=api_server_name or getenv("API_SERVER_NAME", "bwapi"),
)
)
return
for instance in docker_client.containers.list(filters={"label": "bunkerweb.INSTANCE"}):
api_http_port = None
api_server_name = None
for var in instance.attrs["Config"]["Env"]:
if var.startswith("API_HTTP_PORT="):
api_http_port = var.replace("API_HTTP_PORT=", "", 1)
elif var.startswith("API_SERVER_NAME="):
api_server_name = var.replace("API_SERVER_NAME=", "", 1)
self.__apis.append(
API(
f"http://{instance.name}:{api_http_port or getenv('API_HTTP_PORT', '5000')}",
host=api_server_name or getenv("API_SERVER_NAME", "bwapi"),
)
)
def send_to_apis(
self,
method: Union[Literal["POST"], Literal["GET"]],
@ -110,7 +31,7 @@ class ApiCaller:
ret = True
url = url if not url.startswith("/") else url[1:]
responses = {}
for api in self.__apis:
for api in self.apis:
if files is not None:
for buffer in files.values():
buffer.seek(0, 0)

2
src/common/utils/logger.py
View file

@ -23,7 +23,7 @@ class BWLogger(Logger):
setLoggerClass(BWLogger)
default_level = _nameToLevel.get(getenv("LOG_LEVEL", "INFO").upper(), INFO)
default_level = _nameToLevel.get(getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper(), INFO)
basicConfig(
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="[%Y-%m-%d %H:%M:%S]",

2
src/deps/update_python_deps.sh
View file

@ -40,7 +40,7 @@ pip install --no-cache-dir --require-hashes -r requirements-deps.txt
echo "Updating python requirements files"
files=("requirements.in" "../scheduler/requirements.in" "../ui/requirements.in")
files=("requirements.in" "../autoconf/requirements.in" "../scheduler/requirements.in" "../ui/requirements.in")
shopt -s globstar
for file in ../{common,../{docs,misc,tests}}/**/requirements*.in

19
src/scheduler/JobScheduler.py
View file

@ -29,7 +29,6 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from ApiCaller import ApiCaller # type: ignore
from API import API # type: ignore
class JobScheduler(ApiCaller):
@ -67,24 +66,6 @@ class JobScheduler(ApiCaller):
def set_integration(self, integration: str):
self.__integration = integration
def auto_setup(self):
super().auto_setup(bw_integration=self.__integration)
def update_instances(self):
super(JobScheduler, type(self)).apis.fset(self, self.__get_apis())
def __get_apis(self):
apis = []
try:
with self.__thread_lock:
instances = self.db.get_instances()
for instance in instances:
api = API(f"http://{instance['hostname']}:{instance['port']}", host=instance["server_name"])
apis.append(api)
except:
self.__logger.warning(f"Exception while getting jobs instances : {format_exc()}")
return apis
def update_jobs(self):
self.__jobs = self.__get_jobs()

223
src/scheduler/main.py
View file

@ -30,7 +30,7 @@ from common_utils import bytes_hash, dict_to_frozenset, get_integration # type:
from logger import setup_logger # type: ignore
from Database import Database # type: ignore
from JobScheduler import JobScheduler
from API import API
from API import API # type: ignore
RUN = True
SCHEDULER: Optional[JobScheduler] = None
@ -69,7 +69,7 @@ SCHEDULER_TMP_ENV_PATH = TMP_PATH.joinpath("scheduler.env")
SCHEDULER_TMP_ENV_PATH.touch()
DB_LOCK_FILE = Path(sep, "var", "lib", "bunkerweb", "db.lock")
logger = setup_logger("Scheduler", getenv("LOG_LEVEL", "INFO"))
LOGGER = setup_logger("Scheduler", getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")))
SLAVE_MODE = environ.get("SLAVE_MODE", "no") == "yes"
MASTER_MODE = environ.get("MASTER_MODE", "no") == "yes"
@ -104,11 +104,11 @@ def handle_reload(signum, frame):
check=False,
)
if proc.returncode != 0:
logger.error("Config saver failed, configuration will not work as expected...")
LOGGER.error("Config saver failed, configuration will not work as expected...")
else:
logger.warning("Ignored reload operation because scheduler is not running ...")
LOGGER.warning("Ignored reload operation because scheduler is not running ...")
except:
logger.error(f"Exception while reloading scheduler : {format_exc()}")
LOGGER.error(f"Exception while reloading scheduler : {format_exc()}")
signal(SIGHUP, handle_reload)
@ -125,7 +125,7 @@ def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Uni
original_path = Path(original_path)
# Remove old custom configs files
logger.info("Removing old custom configs files ...")
LOGGER.info("Removing old custom configs files ...")
if original_path.is_dir():
for file in original_path.glob("*/*"):
if file.is_symlink() or file.is_file():
@ -135,7 +135,7 @@ def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Uni
rmtree(file, ignore_errors=True)
if configs:
logger.info("Generating new custom configs ...")
LOGGER.info("Generating new custom configs ...")
original_path.mkdir(parents=True, exist_ok=True)
for custom_config in configs:
try:
@ -148,23 +148,23 @@ def generate_custom_configs(configs: List[Dict[str, Any]], *, original_path: Uni
tmp_path.parent.mkdir(parents=True, exist_ok=True)
tmp_path.write_bytes(custom_config["data"])
except OSError as e:
logger.debug(format_exc())
LOGGER.debug(format_exc())
if custom_config["method"] != "manual":
logger.error(
LOGGER.error(
f"Error while generating custom configs \"{custom_config['name']}\"{' for service ' + custom_config['service_id'] if custom_config['service_id'] else ''}: {e}"
)
except BaseException as e:
logger.debug(format_exc())
logger.error(
LOGGER.debug(format_exc())
LOGGER.error(
f"Error while generating custom configs \"{custom_config['name']}\"{' for service ' + custom_config['service_id'] if custom_config['service_id'] else ''}: {e}"
)
if SCHEDULER and SCHEDULER.apis:
logger.info("Sending custom configs to BunkerWeb")
LOGGER.info("Sending custom configs to BunkerWeb")
ret = SCHEDULER.send_files(original_path, "/custom_configs")
if not ret:
logger.error("Sending custom configs failed, configuration will not work as expected...")
LOGGER.error("Sending custom configs failed, configuration will not work as expected...")
def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: Union[Path, str] = EXTERNAL_PLUGINS_PATH):
@ -173,7 +173,7 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U
pro = "pro" in original_path.parts
# Remove old external/pro plugins files
logger.info(f"Removing old/changed {'pro ' if pro else ''}external plugins files ...")
LOGGER.info(f"Removing old/changed {'pro ' if pro else ''}external plugins files ...")
ignored_plugins = set()
if original_path.is_dir():
for file in original_path.glob("*"):
@ -187,7 +187,7 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U
if bytes_hash(plugin_content, algorithm="sha256") == plugins[index]["checksum"]:
ignored_plugins.add(file.name)
continue
logger.debug(f"Checksum of {file} has changed, removing it ...")
LOGGER.debug(f"Checksum of {file} has changed, removing it ...")
if file.is_symlink() or file.is_file():
with suppress(OSError):
@ -196,7 +196,7 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U
rmtree(file, ignore_errors=True)
if plugins:
logger.info(f"Generating new {'pro ' if pro else ''}external plugins ...")
LOGGER.info(f"Generating new {'pro ' if pro else ''}external plugins ...")
original_path.mkdir(parents=True, exist_ok=True)
for plugin in plugins:
if plugin["id"] in ignored_plugins:
@ -216,19 +216,19 @@ def generate_external_plugins(plugins: List[Dict[str, Any]], *, original_path: U
for job_file in chain(original_path.joinpath(plugin["id"], "jobs").glob("*"), original_path.joinpath(plugin["id"], "bwcli").glob("*")):
job_file.chmod(job_file.stat().st_mode | S_IEXEC)
except OSError as e:
logger.debug(format_exc())
LOGGER.debug(format_exc())
if plugin["method"] != "manual":
logger.error(f"Error while generating {'pro ' if pro else ''}external plugins \"{plugin['name']}\": {e}")
LOGGER.error(f"Error while generating {'pro ' if pro else ''}external plugins \"{plugin['name']}\": {e}")
except BaseException as e:
logger.debug(format_exc())
logger.error(f"Error while generating {'pro ' if pro else ''}external plugins \"{plugin['name']}\": {e}")
LOGGER.debug(format_exc())
LOGGER.error(f"Error while generating {'pro ' if pro else ''}external plugins \"{plugin['name']}\": {e}")
if SCHEDULER and SCHEDULER.apis:
logger.info(f"Sending {'pro ' if pro else ''}external plugins to BunkerWeb")
LOGGER.info(f"Sending {'pro ' if pro else ''}external plugins to BunkerWeb")
ret = SCHEDULER.send_files(original_path, "/pro_plugins" if original_path.as_posix().endswith("/pro/plugins") else "/plugins")
if not ret:
logger.error(f"Sending {'pro ' if pro else ''}external plugins failed, configuration will not work as expected...")
LOGGER.error(f"Sending {'pro ' if pro else ''}external plugins failed, configuration will not work as expected...")
def generate_caches(plugins: List[Any], db: Database):
@ -258,7 +258,7 @@ def generate_caches(plugins: List[Any], db: Database):
cache_path.parent.mkdir(parents=True, exist_ok=True)
cache_path.write_bytes(job_cache_file["data"])
except BaseException as e:
logger.error(f"Exception while restoring cache file {job_cache_file['file_name']} :\n{e}")
LOGGER.error(f"Exception while restoring cache file {job_cache_file['file_name']} :\n{e}")
if job_path.is_dir():
for file in job_path.rglob("*"):
skipped = False
@ -266,29 +266,21 @@ def generate_caches(plugins: List[Any], db: Database):
skipped = True
if skipped:
continue
logger.debug(f"Checking if {file} should be removed")
LOGGER.debug(f"Checking if {file} should be removed")
if file not in plugin_cache_files and file.is_file():
logger.debug(f"Removing non-cached file {file}")
LOGGER.debug(f"Removing non-cached file {file}")
file.unlink(missing_ok=True)
if file.parent.is_dir() and not list(file.parent.iterdir()):
logger.debug(f"Removing empty directory {file.parent}")
LOGGER.debug(f"Removing empty directory {file.parent}")
rmtree(file.parent, ignore_errors=True)
if file.parent == job_path:
break
elif file.is_dir() and not list(file.iterdir()):
logger.debug(f"Removing empty directory {file}")
LOGGER.debug(f"Removing empty directory {file}")
rmtree(file, ignore_errors=True)
def api_to_instance(api):
hostname_port = api.endpoint.replace("http://", "").replace("https://", "").replace("/", "").split(":")
return {
"hostname": hostname_port[0],
"env": {"API_HTTP_PORT": int(hostname_port[1]), "API_SERVER_NAME": api.host},
}
def run_in_slave_mode():
def run_in_slave_mode(): # TODO: Refactor this feature
assert SCHEDULER is not None
ready = False
@ -296,16 +288,16 @@ def run_in_slave_mode():
db_metadata = SCHEDULER.db.get_metadata()
env = SCHEDULER.db.get_config()
if isinstance(db_metadata, str) or not db_metadata["is_initialized"]:
logger.warning("Database is not initialized, retrying in 5s ...")
LOGGER.warning("Database is not initialized, retrying in 5s ...")
elif not db_metadata["first_config_saved"] or not env:
logger.warning("Database doesn't have any config saved yet, retrying in 5s ...")
LOGGER.warning("Database doesn't have any config saved yet, retrying in 5s ...")
else:
ready = True
continue
sleep(5)
# Instantiate scheduler environment
SCHEDULER.env = env | environ
SCHEDULER.env = env
# Download plugins
pro_plugins = SCHEDULER.db.get_plugins(_type="pro", with_data=True)
@ -342,19 +334,21 @@ def run_in_slave_mode():
check=False,
)
if proc.returncode != 0:
logger.error("Config generator failed, configuration will not work as expected...")
LOGGER.error("Config generator failed, configuration will not work as expected...")
# TODO : check nginx status + check DB status
while True:
sleep(5)
# TODO: add a health check for the scheduler's BunkerWeb instances
if __name__ == "__main__":
try:
# Don't execute if pid file exists
pid_path = Path(sep, "var", "run", "bunkerweb", "scheduler.pid")
if pid_path.is_file():
logger.error("Scheduler is already running, skipping execution ...")
LOGGER.error("Scheduler is already running, skipping execution ...")
_exit(1)
# Write pid to file
@ -372,7 +366,7 @@ if __name__ == "__main__":
nginx_variables_path = Path(sep, "etc", "nginx", "variables.env")
dotenv_env = dotenv_values(str(tmp_variables_path))
SCHEDULER = JobScheduler(environ, logger, INTEGRATION, db=Database(logger, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None)))) # type: ignore
SCHEDULER = JobScheduler(environ, LOGGER, INTEGRATION, db=Database(LOGGER, sqlalchemy_string=dotenv_env.get("DATABASE_URI", getenv("DATABASE_URI", None)))) # type: ignore
if SLAVE_MODE:
run_in_slave_mode()
@ -401,15 +395,13 @@ if __name__ == "__main__":
check=False,
)
if proc.returncode != 0:
logger.error("Config saver failed, configuration will not work as expected...")
LOGGER.error("Config saver failed, configuration will not work as expected...")
ready = False
while not ready:
db_metadata = SCHEDULER.db.get_metadata()
if isinstance(db_metadata, str) or not db_metadata["is_initialized"]:
logger.warning("Database is not initialized, retrying in 5s ...")
elif INTEGRATION in ("Swarm", "Kubernetes", "Autoconf") and not db_metadata["autoconf_loaded"]:
logger.warning("Autoconf is not loaded yet in the database, retrying in 5s ...")
LOGGER.warning("Database is not initialized, retrying in 5s ...")
else:
ready = True
continue
@ -418,38 +410,25 @@ if __name__ == "__main__":
env = SCHEDULER.db.get_config()
env["DATABASE_URI"] = SCHEDULER.db.database_uri
# Override instances if needed
override_instances = env.get("OVERRIDE_INSTANCES", "")
apis = []
if override_instances:
for instance in override_instances.split(" "):
apis.append(API(instance))
# Instantiate scheduler environment
SCHEDULER.env = env | environ
SCHEDULER.env = env
if INTEGRATION in ("Docker", "Swarm", "Kubernetes", "Autoconf"):
# Automatically setup the scheduler apis
while not SCHEDULER.apis:
SCHEDULER.auto_setup()
if not SCHEDULER.apis:
logger.warning("No BunkerWeb API found, retrying in 5s ...")
sleep(5)
SCHEDULER.db.update_instances([api_to_instance(api) for api in SCHEDULER.apis])
SCHEDULER.apis = []
for db_instance in SCHEDULER.db.get_instances():
SCHEDULER.apis.append(API(f"http://{db_instance['hostname']}:{db_instance['port']}", db_instance["server_name"]))
scheduler_first_start = db_metadata["scheduler_first_start"]
logger.info("Scheduler started ...")
LOGGER.info("Scheduler started ...")
# Checking if any custom config has been created by the user
logger.info("Checking if there are any changes in custom configs ...")
LOGGER.info("Checking if there are any changes in custom configs ...")
custom_configs = []
db_configs = SCHEDULER.db.get_custom_configs()
changes = False
for file in CUSTOM_CONFIGS_PATH.rglob("*.conf"):
if len(file.parts) > len(CUSTOM_CONFIGS_PATH.parts) + 3:
logger.warning(f"Custom config file {file} is not in the correct path, skipping ...")
LOGGER.warning(f"Custom config file {file} is not in the correct path, skipping ...")
content = file.read_text(encoding="utf-8")
service_id = file.parent.name if file.parent.name not in CUSTOM_CONFIGS_DIRS else None
@ -473,7 +452,7 @@ if __name__ == "__main__":
if changes:
err = SCHEDULER.db.save_custom_configs(custom_configs, "manual")
if err:
logger.error(f"Couldn't save some manually created custom configs to database: {err}")
LOGGER.error(f"Couldn't save some manually created custom configs to database: {err}")
if (scheduler_first_start and db_configs) or changes:
generate_custom_configs(SCHEDULER.db.get_custom_configs())
@ -482,7 +461,7 @@ if __name__ == "__main__":
def check_plugin_changes(_type: Literal["external", "pro"] = "external"):
# Check if any external or pro plugin has been added by the user
logger.info(f"Checking if there are any changes in {_type} plugins ...")
LOGGER.info(f"Checking if there are any changes in {_type} plugins ...")
plugin_path = EXTERNAL_PLUGINS_PATH if _type == "external" else PRO_PLUGINS_PATH
db_plugins = SCHEDULER.db.get_plugins(_type=_type)
external_plugins = []
@ -527,7 +506,7 @@ if __name__ == "__main__":
if changes:
err = SCHEDULER.db.update_external_plugins(external_plugins, _type=_type, delete_missing=True)
if err:
logger.error(f"Couldn't save some manually added {_type} plugins to database: {err}")
LOGGER.error(f"Couldn't save some manually added {_type} plugins to database: {err}")
if (scheduler_first_start and db_plugins) or changes:
generate_external_plugins(SCHEDULER.db.get_plugins(_type=_type, with_data=True), original_path=plugin_path)
@ -535,24 +514,24 @@ if __name__ == "__main__":
check_plugin_changes("external")
check_plugin_changes("pro")
logger.info("Running plugins download jobs ...")
LOGGER.info("Running plugins download jobs ...")
# Update the environment variables of the scheduler
SCHEDULER.env = env | environ
SCHEDULER.env = env
if not SCHEDULER.run_single("download-plugins"):
logger.warning("download-plugins job failed at first start, plugins settings set by the user may not be up to date ...")
LOGGER.warning("download-plugins job failed at first start, plugins settings set by the user may not be up to date ...")
if not SCHEDULER.run_single("download-pro-plugins"):
logger.warning("download-pro-plugins job failed at first start, pro plugins settings set by the user may not be up to date ...")
LOGGER.warning("download-pro-plugins job failed at first start, pro plugins settings set by the user may not be up to date ...")
db_metadata = SCHEDULER.db.get_metadata()
if INTEGRATION not in ("Swarm", "Kubernetes", "Autoconf") and (db_metadata["pro_plugins_changed"] or db_metadata["external_plugins_changed"]):
if db_metadata["pro_plugins_changed"] or db_metadata["external_plugins_changed"]:
if db_metadata["pro_plugins_changed"]:
generate_external_plugins(SCHEDULER.db.get_plugins(_type="pro", with_data=True), original_path=PRO_PLUGINS_PATH)
if db_metadata["external_plugins_changed"]:
generate_external_plugins(SCHEDULER.db.get_plugins(_type="external", with_data=True))
# run the config saver to save potential ignored external plugins settings
logger.info("Running config saver to save potential ignored external plugins settings ...")
LOGGER.info("Running config saver to save potential ignored external plugins settings ...")
proc = subprocess_run(
[
"python3",
@ -565,15 +544,13 @@ if __name__ == "__main__":
check=False,
)
if proc.returncode != 0:
logger.error(
"Config saver failed, configuration will not work as expected...",
)
LOGGER.error("Config saver failed, configuration will not work as expected...")
SCHEDULER.update_jobs()
env = SCHEDULER.db.get_config()
env["DATABASE_URI"] = SCHEDULER.db.database_uri
logger.info("Executing scheduler ...")
LOGGER.info("Executing scheduler ...")
del dotenv_env
@ -583,44 +560,27 @@ if __name__ == "__main__":
threads = []
def send_nginx_configs():
logger.info(f"Sending {join(sep, 'etc', 'nginx')} folder ...")
LOGGER.info(f"Sending {join(sep, 'etc', 'nginx')} folder ...")
ret = SCHEDULER.send_files(join(sep, "etc", "nginx"), "/confs")
if not ret:
logger.error("Sending nginx configs failed, configuration will not work as expected...")
LOGGER.error("Sending nginx configs failed, configuration will not work as expected...")
def send_nginx_cache():
logger.info(f"Sending {CACHE_PATH} folder ...")
LOGGER.info(f"Sending {CACHE_PATH} folder ...")
if not SCHEDULER.send_files(CACHE_PATH, "/cache"):
logger.error(f"Error while sending {CACHE_PATH} folder")
LOGGER.error(f"Error while sending {CACHE_PATH} folder")
else:
logger.info(f"Successfully sent {CACHE_PATH} folder")
def listen_for_instances_reload():
from docker import DockerClient
global SCHEDULER
docker_client = DockerClient(base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock"))
for event in docker_client.events(decode=True, filters={"type": "container", "label": "bunkerweb.INSTANCE"}):
if event["Action"] in ("start", "die"):
logger.info(f"🐋 Detected {event['Action']} event on container {event['Actor']['Attributes']['name']}")
SCHEDULER.auto_setup()
SCHEDULER.db.update_instances([api_to_instance(api) for api in SCHEDULER.apis], changed=event["Action"] == "die")
if event["Action"] == "start":
SCHEDULER.db.checked_changes(value=True)
if INTEGRATION == "Docker" and not override_instances:
Thread(target=listen_for_instances_reload, name="listen_for_instances_reload").start()
LOGGER.info(f"Successfully sent {CACHE_PATH} folder")
while True:
threads.clear()
if RUN_JOBS_ONCE:
# Only run jobs once
if not SCHEDULER.reload(env | environ):
logger.error("At least one job in run_once() failed")
if not SCHEDULER.reload(env):
LOGGER.error("At least one job in run_once() failed")
else:
logger.info("All jobs in run_once() were successful")
LOGGER.info("All jobs in run_once() were successful")
if CONFIG_NEED_GENERATION:
content = ""
@ -642,15 +602,10 @@ if __name__ == "__main__":
]
if MASTER_MODE:
args.append("--no-linux-reload")
proc = subprocess_run(
args,
stdin=DEVNULL,
stderr=STDOUT,
check=False,
)
proc = subprocess_run(args, stdin=DEVNULL, stderr=STDOUT, check=False)
if proc.returncode != 0:
logger.error("Config generator failed, configuration will not work as expected...")
LOGGER.error("Config generator failed, configuration will not work as expected...")
else:
copy(str(nginx_variables_path), join(sep, "var", "tmp", "bunkerweb", "variables.env"))
@ -660,7 +615,7 @@ if __name__ == "__main__":
thread.start()
threads.append(thread)
elif INTEGRATION != "Linux":
logger.warning("No BunkerWeb instance found, skipping nginx configs sending ...")
LOGGER.warning("No BunkerWeb instance found, skipping nginx configs sending ...")
try:
if SCHEDULER.apis:
@ -673,12 +628,12 @@ if __name__ == "__main__":
thread.join()
if SCHEDULER.send_to_apis("POST", "/reload"):
logger.info("Successfully reloaded nginx")
LOGGER.info("Successfully reloaded nginx")
else:
logger.error("Error while reloading nginx")
LOGGER.error("Error while reloading nginx")
elif INTEGRATION == "Linux":
# Reload nginx
logger.info("Reloading nginx ...")
LOGGER.info("Reloading nginx ...")
proc = subprocess_run(
[join(sep, "usr", "sbin", "nginx"), "-s", "reload"],
stdin=DEVNULL,
@ -688,20 +643,20 @@ if __name__ == "__main__":
stdout=PIPE,
)
if proc.returncode == 0:
logger.info("Successfully sent reload signal to nginx")
LOGGER.info("Successfully sent reload signal to nginx")
else:
logger.error(
LOGGER.error(
f"Error while reloading nginx - returncode: {proc.returncode} - error: {proc.stdout.decode('utf-8') if proc.stdout else 'no output'}"
)
else:
logger.warning("No BunkerWeb instance found, skipping nginx reload ...")
LOGGER.warning("No BunkerWeb instance found, skipping nginx reload ...")
except:
logger.error(f"Exception while reloading after running jobs once scheduling : {format_exc()}")
LOGGER.error(f"Exception while reloading after running jobs once scheduling : {format_exc()}")
ret = SCHEDULER.db.checked_changes(CHANGES)
if ret:
logger.error(f"An error occurred when setting the changes to checked in the database : {ret}")
LOGGER.error(f"An error occurred when setting the changes to checked in the database : {ret}")
stop(1)
NEED_RELOAD = False
@ -716,9 +671,9 @@ if __name__ == "__main__":
ret = SCHEDULER.db.set_metadata({"scheduler_first_start": False})
if ret == "The database is read-only, the changes will not be saved":
logger.warning("The database is read-only, the scheduler first start will not be saved")
LOGGER.warning("The database is read-only, the scheduler first start will not be saved")
elif ret:
logger.error(f"An error occurred when setting the scheduler first start : {ret}")
LOGGER.error(f"An error occurred when setting the scheduler first start : {ret}")
stop(1)
scheduler_first_start = False
@ -726,7 +681,7 @@ if __name__ == "__main__":
HEALTHY_PATH.write_text(datetime.now().isoformat(), encoding="utf-8")
# infinite schedule for the jobs
logger.info("Executing job scheduler ...")
LOGGER.info("Executing job scheduler ...")
errors = 0
while RUN and not NEED_RELOAD:
try:
@ -735,7 +690,7 @@ if __name__ == "__main__":
current_time = datetime.now()
while DB_LOCK_FILE.is_file() and DB_LOCK_FILE.stat().st_ctime + 30 > current_time.timestamp():
logger.debug("Database is locked, waiting for it to be unlocked (timeout: 30s) ...")
LOGGER.debug("Database is locked, waiting for it to be unlocked (timeout: 30s) ...")
sleep(1)
DB_LOCK_FILE.unlink(missing_ok=True)
@ -747,14 +702,14 @@ if __name__ == "__main__":
# check if the plugins have changed since last time
if db_metadata["pro_plugins_changed"]:
logger.info("Pro plugins changed, generating ...")
LOGGER.info("Pro plugins changed, generating ...")
PRO_PLUGINS_NEED_GENERATION = True
CONFIG_NEED_GENERATION = True
RUN_JOBS_ONCE = True
NEED_RELOAD = True
if db_metadata["external_plugins_changed"]:
logger.info("External plugins changed, generating ...")
LOGGER.info("External plugins changed, generating ...")
PLUGINS_NEED_GENERATION = True
CONFIG_NEED_GENERATION = True
RUN_JOBS_ONCE = True
@ -762,29 +717,29 @@ if __name__ == "__main__":
# check if the custom configs have changed since last time
if db_metadata["custom_configs_changed"]:
logger.info("Custom configs changed, generating ...")
LOGGER.info("Custom configs changed, generating ...")
CONFIGS_NEED_GENERATION = True
CONFIG_NEED_GENERATION = True
NEED_RELOAD = True
# check if the config have changed since last time
if db_metadata["config_changed"]:
logger.info("Config changed, generating ...")
LOGGER.info("Config changed, generating ...")
CONFIG_NEED_GENERATION = True
RUN_JOBS_ONCE = True
NEED_RELOAD = True
# check if the instances have changed since last time
if db_metadata["instances_changed"]:
logger.info("Instances changed, generating ...")
LOGGER.info("Instances changed, generating ...")
INSTANCES_NEED_GENERATION = True
CONFIGS_NEED_GENERATION = True
CONFIG_NEED_GENERATION = True
NEED_RELOAD = True
except BaseException:
logger.debug(format_exc())
LOGGER.debug(format_exc())
if errors > 5:
logger.error(f"An error occurred when executing the scheduler : {format_exc()}")
LOGGER.error(f"An error occurred when executing the scheduler : {format_exc()}")
stop(1)
errors += 1
sleep(5)
@ -794,7 +749,9 @@ if __name__ == "__main__":
if INSTANCES_NEED_GENERATION:
CHANGES.append("instances")
SCHEDULER.update_instances()
SCHEDULER.apis = []
for db_instance in SCHEDULER.db.get_instances():
SCHEDULER.apis.append(API(f"http://{db_instance['hostname']}:{db_instance['port']}", db_instance["server_name"]))
if CONFIGS_NEED_GENERATION:
CHANGES.append("custom_configs")
@ -816,5 +773,5 @@ if __name__ == "__main__":
env["DATABASE_URI"] = SCHEDULER.db.database_uri
except:
logger.error(f"Exception while executing scheduler : {format_exc()}")
LOGGER.error(f"Exception while executing scheduler : {format_exc()}")
stop(1)

1
src/scheduler/requirements.in
View file

@ -1,5 +1,4 @@
certbot==2.10.0
configobj==5.0.8
cryptography==42.0.7
maxminddb==2.6.1
python-magic==0.4.27

4
src/scheduler/requirements.txt
View file

@ -169,9 +169,7 @@ configargparse==1.7 \
configobj==5.0.8 \
--hash=sha256:6f704434a07dc4f4dc7c9a745172c1cad449feb548febd9f7fe362629c627a97 \
--hash=sha256:a7a8c6ab7daade85c3f329931a807c8aee750a2494363934f8ea84d8a54c87ea
# via
# -r requirements.in
# certbot
# via certbot
cryptography==42.0.7 \
--hash=sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55 \
--hash=sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785 \

5
src/ui/gunicorn.conf.py
View file

@ -11,7 +11,6 @@ for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in ((
if deps_path not in sys_path:
sys_path.append(deps_path)
from common_utils import get_integration # type: ignore
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
@ -56,15 +55,11 @@ def on_starting(server):
db = Database(LOGGER, ui=True)
INTEGRATION = get_integration()
ready = False
while not ready:
db_metadata = db.get_metadata()
if isinstance(db_metadata, str) or not db_metadata["is_initialized"]:
LOGGER.warning("Database is not initialized, retrying in 5s ...")
elif INTEGRATION in ("Swarm", "Kubernetes", "Autoconf") and not db_metadata["autoconf_loaded"]:
LOGGER.warning("Autoconf is not loaded yet in the database, retrying in 5s ...")
else:
ready = True
continue

130
src/ui/main.py
View file

@ -19,8 +19,6 @@ from bs4 import BeautifulSoup
from copy import deepcopy
from datetime import datetime, timedelta, timezone
from dateutil.parser import parse as dateutil_parse
from docker import DockerClient
from docker.errors import NotFound as docker_NotFound, APIError as docker_APIError, DockerException
from flask import Flask, Response, flash, jsonify, redirect, render_template, request, send_file, session, url_for
from flask_login import current_user, LoginManager, login_required, login_user, logout_user
from flask_wtf.csrf import CSRFProtect, CSRFError
@ -29,9 +27,6 @@ from importlib.machinery import SourceFileLoader
from io import BytesIO
from json import JSONDecodeError, dumps, loads as json_loads
from jinja2 import Environment, FileSystemLoader, select_autoescape
from kubernetes import client as kube_client
from kubernetes import config as kube_config
from kubernetes.client.exceptions import ApiException as kube_ApiException
from redis import Redis, Sentinel
from regex import compile as re_compile, match as regex_match
from requests import get
@ -53,7 +48,7 @@ from src.User import AnonymousUser, User
from src.Templates import get_ui_templates
from utils import check_settings, get_b64encoded_qr_image, path_to_dict, get_remain
from common_utils import get_integration, get_version # type: ignore
from common_utils import get_version # type: ignore
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
@ -96,7 +91,7 @@ app = Flask(__name__, static_url_path="/", static_folder="static", template_fold
PROXY_NUMBERS = int(getenv("PROXY_NUMBERS", "1"))
app.wsgi_app = ReverseProxied(app.wsgi_app, x_for=PROXY_NUMBERS, x_proto=PROXY_NUMBERS, x_host=PROXY_NUMBERS, x_prefix=PROXY_NUMBERS)
app.logger = setup_logger("UI")
app.logger = setup_logger("UI", getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")))
FLASK_SECRET = getenv("FLASK_SECRET")
@ -114,19 +109,6 @@ login_manager.login_view = "login"
login_manager.anonymous_user = AnonymousUser
PLUGIN_KEYS = ["id", "name", "description", "version", "stream", "settings"]
INTEGRATION = get_integration()
docker_client = None
kubernetes_client = None
if INTEGRATION in ("Docker", "Swarm", "Autoconf"):
try:
docker_client: DockerClient = DockerClient(base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock"))
except (docker_APIError, DockerException):
app.logger.warning("No docker host found")
elif INTEGRATION == "Kubernetes":
kube_config.load_incluster_config()
kubernetes_client = kube_client.CoreV1Api()
db = Database(app.logger, ui=True, log=False)
USER = "Error"
@ -146,7 +128,7 @@ if not TMP_DIR.joinpath(".ui.json").is_file():
try:
app.config.update(
INSTANCES=Instances(docker_client, kubernetes_client, INTEGRATION, db),
INSTANCES=Instances(db),
CONFIG=Config(db),
CONFIGFILES=ConfigFiles(),
WTF_CSRF_SSL_STRICT=False,
@ -655,8 +637,7 @@ def home():
remote_version = basename(r.url).strip().replace("v", "")
config = app.config["CONFIG"].get_config(with_drafts=True)
override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
instances = app.config["INSTANCES"].get_instances()
instance_health_count = 0
@ -892,9 +873,7 @@ def instances():
)
# Display instances
config = app.config["CONFIG"].get_config()
override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
instances = app.config["INSTANCES"].get_instances()
return render_template("instances.html", title="Instances", instances=instances, username=current_user.get_id())
@ -1837,9 +1816,7 @@ def cache():
@app.route("/logs", methods=["GET"])
@login_required
def logs():
config = app.config["CONFIG"].get_config(with_drafts=True)
override_instances = config["OVERRIDE_INSTANCES"]["value"] != ""
instances = app.config["INSTANCES"].get_instances(override_instances=override_instances)
instances = app.config["INSTANCES"].get_instances()
return render_template("logs.html", instances=instances, username=current_user.get_id())
@ -1992,53 +1969,56 @@ def logs_container(container_id):
logs = []
tmp_logs = []
if docker_client:
try:
if INTEGRATION != "Swarm":
docker_logs = docker_client.containers.get(container_id).logs( # type: ignore
stdout=True,
stderr=True,
since=datetime.fromtimestamp(last_update),
timestamps=True,
)
else:
docker_logs = docker_client.services.get(container_id).logs( # type: ignore
stdout=True,
stderr=True,
since=datetime.fromtimestamp(last_update),
timestamps=True,
)
return jsonify({"logs": logs, "last_update": int(time() * 1000)})
tmp_logs = docker_logs.decode("utf-8", errors="replace").split("\n")[0:-1]
except docker_NotFound:
return (
jsonify(
{
"status": "ko",
"message": f"Container with ID {container_id} not found!",
}
),
404,
)
elif kubernetes_client:
try:
kubernetes_logs = kubernetes_client.read_namespaced_pod_log(
container_id,
getenv("KUBERNETES_NAMESPACE", "default"),
since_seconds=int(datetime.now().timestamp() - last_update),
timestamps=True,
)
tmp_logs = kubernetes_logs.split("\n")[0:-1]
except kube_ApiException:
return (
jsonify(
{
"status": "ko",
"message": f"Pod with ID {container_id} not found!",
}
),
404,
)
# TODO: find a solution for this
# if docker_client:
# try:
# if INTEGRATION != "Swarm":
# docker_logs = docker_client.containers.get(container_id).logs( # type: ignore
# stdout=True,
# stderr=True,
# since=datetime.fromtimestamp(last_update),
# timestamps=True,
# )
# else:
# docker_logs = docker_client.services.get(container_id).logs( # type: ignore
# stdout=True,
# stderr=True,
# since=datetime.fromtimestamp(last_update),
# timestamps=True,
# )
# tmp_logs = docker_logs.decode("utf-8", errors="replace").split("\n")[0:-1]
# except docker_NotFound:
# return (
# jsonify(
# {
# "status": "ko",
# "message": f"Container with ID {container_id} not found!",
# }
# ),
# 404,
# )
# elif kubernetes_client:
# try:
# kubernetes_logs = kubernetes_client.read_namespaced_pod_log(
# container_id,
# getenv("KUBERNETES_NAMESPACE", "default"),
# since_seconds=int(datetime.now().timestamp() - last_update),
# timestamps=True,
# )
# tmp_logs = kubernetes_logs.split("\n")[0:-1]
# except kube_ApiException:
# return (
# jsonify(
# {
# "status": "ko",
# "message": f"Pod with ID {container_id} not found!",
# }
# ),
# 404,
# )
for log in tmp_logs:
split = log.split(" ")

1
src/ui/requirements.in
View file

@ -4,7 +4,6 @@ Flask==3.0.3
Flask-Login==0.6.3
Flask_WTF==1.2.1
gunicorn[gthread]==22.0.0
importlib-metadata==7.1.0
pyotp==2.9.0
python-magic==0.4.27
python_dateutil==2.9.0.post0

4
src/ui/requirements.txt
View file

@ -67,9 +67,7 @@ gunicorn==22.0.0 \
importlib-metadata==7.1.0 \
--hash=sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570 \
--hash=sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2
# via
# -r requirements.in
# flask
# via flask
itsdangerous==2.2.0 \
--hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \
--hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173

164
src/ui/src/Instances.py
View file

@ -2,13 +2,11 @@
from operator import itemgetter
from os import sep
from os.path import join
from pathlib import Path
from subprocess import DEVNULL, STDOUT, run
from typing import Any, List, Optional, Tuple, Union
from API import API # type: ignore
from ApiCaller import ApiCaller # type: ignore
from dotenv import dotenv_values # type: ignore
class Instance:
@ -134,10 +132,7 @@ class Instance:
class Instances:
def __init__(self, docker_client, kubernetes_client, integration: str, db):
self.__docker_client = docker_client
self.__kubernetes_client = kubernetes_client
self.__integration = integration
def __init__(self, db):
self.__db = db
def __instance_from_id(self, _id) -> Instance:
@ -148,151 +143,26 @@ class Instances:
raise ValueError(f"Can't find instance with _id {_id}")
def get_instances(self, override_instances=None) -> list[Instance]:
instances = []
# Override case : only return instances from DB
if override_instances is None:
config = self.__db.get_config()
override_instances = config["OVERRIDE_INSTANCES"] != ""
if override_instances:
for instance in self.__db.get_instances():
instances.append(
Instance(
instance["hostname"],
instance["hostname"],
instance["hostname"],
"override",
"up",
None,
ApiCaller(
[
API(
f"http://{instance['hostname']}:{instance['port']}",
instance["server_name"],
)
]
),
)
)
return instances
# Docker instances (containers or services)
if self.__docker_client is not None:
for instance in self.__docker_client.containers.list(all=True, filters={"label": "bunkerweb.INSTANCE"}):
env_variables = {x[0]: (x[1] if len(x) > 1 else "") for x in [env.split("=") for env in instance.attrs["Config"]["Env"]]}
instances.append(
Instance(
instance.id,
instance.name,
instance.name,
"container",
"up" if instance.status == "running" else "down",
instance,
ApiCaller(
[
API(
f"http://{instance.name}:{env_variables.get('API_HTTP_PORT', '5000')}",
env_variables.get("API_SERVER_NAME", "bwapi"),
)
]
),
)
)
elif self.__integration == "Swarm":
for instance in self.__docker_client.services.list(filters={"label": "bunkerweb.INSTANCE"}):
status = "down"
desired_tasks = instance.attrs["ServiceStatus"]["DesiredTasks"]
running_tasks = instance.attrs["ServiceStatus"]["RunningTasks"]
if desired_tasks > 0 and (desired_tasks == running_tasks):
status = "up"
apiCaller = ApiCaller()
api_http_port = None
api_server_name = None
for var in instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
if var.startswith("API_HTTP_PORT="):
api_http_port = var.replace("API_HTTP_PORT=", "", 1)
elif var.startswith("API_SERVER_NAME="):
api_server_name = var.replace("API_SERVER_NAME=", "", 1)
for task in instance.tasks():
apiCaller.append(
def get_instances(self) -> list[Instance]:
return [
Instance(
instance["hostname"],
instance["hostname"],
instance["hostname"],
instance["method"],
"up",
None,
ApiCaller(
[
API(
f"http://{instance.name}.{task['NodeID']}.{task['ID']}:{api_http_port or '5000'}",
host=api_server_name or "bwapi",
f"http://{instance['hostname']}:{instance['port']}",
instance["server_name"],
)
)
instances.append(
Instance(
instance.id,
instance.name,
instance.name,
"service",
status,
instance,
apiCaller,
)
)
elif self.__integration == "Kubernetes":
for pod in self.__kubernetes_client.list_pod_for_all_namespaces(watch=False).items:
if pod.metadata.annotations is not None and "bunkerweb.io/INSTANCE" in pod.metadata.annotations:
env_variables = {env.name: env.value or "" for env in pod.spec.containers[0].env}
status = "up"
if pod.status.conditions is not None:
for condition in pod.status.conditions:
if condition.type == "Ready" and condition.status == "True":
status = "down"
break
instances.append(
Instance(
pod.metadata.uid,
pod.metadata.name,
pod.status.pod_ip,
"pod",
status,
pod,
ApiCaller(
[
API(
f"http://{pod.status.pod_ip}:{env_variables.get('API_HTTP_PORT', '5000')}",
host=env_variables.get("API_SERVER_NAME", "bwapi"),
)
]
),
)
)
instances.sort(key=lambda x: x.name)
# Local instance
if Path(sep, "usr", "sbin", "nginx").exists():
env_variables = dotenv_values(join(sep, "etc", "bunkerweb", "variables.env"))
instances.insert(
0,
Instance(
"local",
"local",
"127.0.0.1",
"local",
"up" if Path(sep, "var", "run", "bunkerweb", "nginx.pid").exists() else "down",
None,
ApiCaller(
[
API(
f"http://127.0.0.1:{env_variables.get('API_HTTP_PORT', '5000')}",
env_variables.get("API_SERVER_NAME", "bwapi"),
)
]
),
]
),
)
return instances
for instance in self.__db.get_instances()
]
def reload_instances(self) -> Union[list[str], str]:
not_reloaded: list[str] = []

2
src/ui/templates/settings_plugins.html vendored
View file

@ -54,7 +54,7 @@
{% for setting, value in plugin["settings"].items() %}
{% set setting_input = { "is_pro_plugin" : True if plugin["type"] == "pro" else False, "name" : setting, "context" : value.get("context"), "method" : value.get("method"), "help" : value.get("help"), "label" : value.get("label"), "id" : value.get("id"), "type" : value.get("type"), "default" : value.get("default"), "select" : value.get("select"), "regex" : value.get("regex"), "value" : value.get("value"), "is_multiple" : False, "levels" : value.get('levels', {})} %}
{% if setting != "IS_DRAFT" and (current_endpoint == "global-config" and setting not in ["SERVER_NAME", "IS_LOADING"] or current_endpoint == "services" and value['context'] == "multisite") %}
{% if setting != "IS_DRAFT" and (current_endpoint == "global-config" and setting not in ["SERVER_NAME", "IS_LOADING", "BUNKERWEB_INSTANCES"] or current_endpoint == "services" and value['context'] == "multisite") %}
{% if value['multiple'] and value['multiple'] not in multList %}
{% if multList.append(value['multiple']) %}{% endif %}
{% endif %}