From ad64ce22e92c22516c3e83afab8ba2a1b5a511af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 15:12:56 +0200 Subject: [PATCH 01/10] Remove no longer needed packages that were fixing old CVEs --- src/autoconf/Dockerfile | 3 --- src/bw/Dockerfile | 3 --- src/scheduler/Dockerfile | 3 --- src/ui/Dockerfile | 3 --- 4 files changed, 12 deletions(-) diff --git a/src/autoconf/Dockerfile b/src/autoconf/Dockerfile index 6156bed78..c7c57ab21 100644 --- a/src/autoconf/Dockerfile +++ b/src/autoconf/Dockerfile @@ -53,9 +53,6 @@ RUN apk add --no-cache bash && \ chown root:nginx /var/log/letsencrypt /var/lib/letsencrypt && \ chmod 770 /var/log/letsencrypt /var/lib/letsencrypt -# Fix CVEs -RUN apk add "libcrypto3>=3.0.8-r2" "libssl3>=3.0.8-r2" - VOLUME /data /etc/nginx WORKDIR /usr/share/bunkerweb/autoconf diff --git a/src/bw/Dockerfile b/src/bw/Dockerfile index 10d1f0e50..ee5cb15a8 100644 --- a/src/bw/Dockerfile +++ b/src/bw/Dockerfile @@ -74,9 +74,6 @@ RUN apk add --no-cache pcre bash python3 && \ ln -s /proc/1/fd/1 /var/log/nginx/jobs.log && \ chmod 660 /usr/share/bunkerweb/INTEGRATION -# Fix CVEs -RUN apk add "curl>=7.83.1-r6" "libcrypto1.1>=1.1.1t-r1" "libcurl>=7.83.1-r6" "libssl1.1>=1.1.1t-r1" "libcrypto3>=3.0.8-r2" "libssl3>=3.0.8-r2" - VOLUME /data /etc/nginx EXPOSE 8080/tcp 8443/tcp diff --git a/src/scheduler/Dockerfile b/src/scheduler/Dockerfile index 3ddb43866..60712dc6f 100644 --- a/src/scheduler/Dockerfile +++ b/src/scheduler/Dockerfile @@ -63,9 +63,6 @@ RUN apk add --no-cache bash libgcc libstdc++ openssl && \ ln -s /proc/1/fd/1 /var/log/letsencrypt/letsencrypt.log && \ chmod 660 /usr/share/bunkerweb/INTEGRATION -# Fix CVEs -RUN apk add "libcrypto3>=3.0.8-r2" "libssl3>=3.0.8-r2" - VOLUME /data /etc/nginx WORKDIR /usr/share/bunkerweb/scheduler diff --git a/src/ui/Dockerfile b/src/ui/Dockerfile index 4b9e84d36..ac1c9de21 100755 --- a/src/ui/Dockerfile +++ b/src/ui/Dockerfile @@ -49,9 +49,6 @@ RUN apk add --no-cache bash && \ chmod 750 /usr/share/bunkerweb/gen/*.py /usr/share/bunkerweb/ui/*.py /usr/share/bunkerweb/ui/src/*.py /usr/share/bunkerweb/deps/python/bin/* && \ chmod 660 /usr/share/bunkerweb/INTEGRATION -# Fix CVEs -RUN apk add "libcrypto3>=3.0.8-r2" "libssl3>=3.0.8-r2" - VOLUME /data /etc/nginx EXPOSE 7000 From 8c919c676843cffb6dd82982787716bb224c8ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 15:13:23 +0200 Subject: [PATCH 02/10] Update links in the home page of the web UI --- src/ui/templates/home.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ui/templates/home.html b/src/ui/templates/home.html index 88865efbe..33f84bef6 100644 --- a/src/ui/templates/home.html +++ b/src/ui/templates/home.html @@ -3,6 +3,7 @@
@@ -139,7 +140,7 @@ From 9578ace0268c73a085e8dd913988627415f273a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 15:29:34 +0200 Subject: [PATCH 03/10] Remove not used INTEGRATION file in BunkerWeb container --- src/bw/Dockerfile | 6 ++---- src/bw/entrypoint.sh | 10 +--------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/bw/Dockerfile b/src/bw/Dockerfile index ee5cb15a8..ae574fb97 100644 --- a/src/bw/Dockerfile +++ b/src/bw/Dockerfile @@ -42,12 +42,11 @@ COPY src/common/helpers /usr/share/bunkerweb/helpers COPY src/common/settings.json /usr/share/bunkerweb/settings.json COPY src/common/utils /usr/share/bunkerweb/utils COPY src/VERSION /usr/share/bunkerweb/VERSION -COPY misc/*.ascii /usr/share/bunkerweb +COPY misc/*.ascii /usr/share/bunkerweb/ # Install runtime dependencies, pypi packages, move bwcli, create data folders and set permissions RUN apk add --no-cache pcre bash python3 && \ cp /usr/share/bunkerweb/helpers/bwcli /usr/bin/ && \ - echo "Docker" > /usr/share/bunkerweb/INTEGRATION && \ mkdir -p /var/tmp/bunkerweb && \ mkdir -p /var/www/html && \ mkdir -p /etc/bunkerweb && \ @@ -71,8 +70,7 @@ RUN apk add --no-cache pcre bash python3 && \ ln -s /proc/1/fd/2 /var/log/nginx/error.log && \ ln -s /proc/1/fd/2 /var/log/nginx/modsec_audit.log && \ ln -s /proc/1/fd/1 /var/log/nginx/access.log && \ - ln -s /proc/1/fd/1 /var/log/nginx/jobs.log && \ - chmod 660 /usr/share/bunkerweb/INTEGRATION + ln -s /proc/1/fd/1 /var/log/nginx/jobs.log VOLUME /data /etc/nginx diff --git a/src/bw/entrypoint.sh b/src/bw/entrypoint.sh index cc71b5f54..7010c27f3 100644 --- a/src/bw/entrypoint.sh +++ b/src/bw/entrypoint.sh @@ -35,14 +35,6 @@ function trap_reload() { } trap "trap_reload" HUP -if [ "$SWARM_MODE" == "yes" ] ; then - echo "Swarm" > /usr/share/bunkerweb/INTEGRATION -elif [ "$KUBERNETES_MODE" == "yes" ] ; then - echo "Kubernetes" > /usr/share/bunkerweb/INTEGRATION -elif [ "$AUTOCONF_MODE" == "yes" ] ; then - echo "Autoconf" > /usr/share/bunkerweb/INTEGRATION -fi - if [ -f "/etc/nginx/variables.env" ] ; then log "ENTRYPOINT" "⚠️ " "Looks like BunkerWeb has already been loaded, will not generate temp config" else @@ -63,4 +55,4 @@ while [ -f "/var/tmp/bunkerweb/nginx.pid" ] ; do done log "ENTRYPOINT" "ℹ️" "BunkerWeb stopped" -exit 0 \ No newline at end of file +exit 0 From 707256076ae16d1e16568d628229e3daa6988ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 15:30:09 +0200 Subject: [PATCH 04/10] Add now the scheduler will pass his own env as well to jobs --- src/scheduler/main.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/scheduler/main.py b/src/scheduler/main.py index 3b05a61f6..ad783f311 100644 --- a/src/scheduler/main.py +++ b/src/scheduler/main.py @@ -6,6 +6,7 @@ from glob import glob from os import ( _exit, chmod, + environ, getenv, getpid, listdir, @@ -214,13 +215,6 @@ if __name__ == "__main__": "Kubernetes", "Autoconf", ): - # err = db.set_autoconf_load(False) - # if err: - # success = False - # logger.error( - # f"Can't set autoconf loaded metadata to false in database: {err}", - # ) - while not db.is_autoconf_loaded(): logger.warning( "Autoconf is not loaded yet in the database, retrying in 5s ...", @@ -327,7 +321,7 @@ if __name__ == "__main__": while True: # Instantiate scheduler scheduler = JobScheduler( - env=deepcopy(env), + env=deepcopy(env) | environ, apis=api_caller._get_apis(), logger=logger, integration=integration, @@ -484,10 +478,6 @@ if __name__ == "__main__": ) else: need_reload = True - # if api_caller._send_to_apis("POST", "/reload"): - # logger.info("Successfully reloaded nginx") - # else: - # logger.error("Error while reloading nginx") # check if the plugins have changed since last time tmp_external_plugins = db.get_plugins(external=True) From 9a325c7a9a994b2b6821e7593e097c3fe4959099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 15:44:35 +0200 Subject: [PATCH 05/10] Add new check for integrations in BunkerNet job --- src/common/core/bunkernet/jobs/bunkernet.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/common/core/bunkernet/jobs/bunkernet.py b/src/common/core/bunkernet/jobs/bunkernet.py index 598dc14ab..7f0814dfc 100644 --- a/src/common/core/bunkernet/jobs/bunkernet.py +++ b/src/common/core/bunkernet/jobs/bunkernet.py @@ -62,13 +62,17 @@ def get_integration(): try: if getenv("AUTOCONF_MODE") == "yes": return "autoconf" - if getenv("SWARM_MODE") == "yes": + elif getenv("SWARM_MODE") == "yes": return "swarm" elif getenv("KUBERNETES_MODE") == "yes": return "kubernetes" - elif Path("/usr/share/bunkerweb/INTEGRATION").exists(): - with open("/usr/share/bunkerweb/INTEGRATION", "r") as f: - return f.read().strip().lower() + elif Path("/usr/share/bunkerweb/INTEGRATION").is_file(): + return Path("/usr/share/bunkerweb/INTEGRATION").read_text().strip().lower() + elif ( + Path("/etc/os-release").is_file() + and "Alpine" in Path("/etc/os-release").read_text() + ): + return "docker" return "linux" except: From 79a46e2cf65e298169b33966f832e0f17d8e52a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 17:23:41 +0200 Subject: [PATCH 06/10] Update the logic behind the check for linux os --- src/common/core/bunkernet/jobs/bunkernet.py | 10 +++++----- src/common/gen/main.py | 12 ++++++++++-- src/common/gen/save_config.py | 5 +++++ src/scheduler/JobScheduler.py | 2 +- src/scheduler/main.py | 8 ++++---- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/common/core/bunkernet/jobs/bunkernet.py b/src/common/core/bunkernet/jobs/bunkernet.py index 7f0814dfc..4b0fdb89e 100644 --- a/src/common/core/bunkernet/jobs/bunkernet.py +++ b/src/common/core/bunkernet/jobs/bunkernet.py @@ -60,12 +60,12 @@ def get_version(): def get_integration(): try: - if getenv("AUTOCONF_MODE") == "yes": - return "autoconf" - elif getenv("SWARM_MODE") == "yes": - return "swarm" - elif getenv("KUBERNETES_MODE") == "yes": + if getenv("KUBERNETES_MODE", "no") == "yes": return "kubernetes" + elif getenv("SWARM_MODE", "no") == "yes": + return "swarm" + elif getenv("AUTOCONF_MODE", "no") == "yes": + return "autoconf" elif Path("/usr/share/bunkerweb/INTEGRATION").is_file(): return Path("/usr/share/bunkerweb/INTEGRATION").read_text().strip().lower() elif ( diff --git a/src/common/gen/main.py b/src/common/gen/main.py index f8afb03c9..10961dbf1 100644 --- a/src/common/gen/main.py +++ b/src/common/gen/main.py @@ -90,8 +90,13 @@ if __name__ == "__main__": integration = "Swarm" elif getenv("AUTOCONF_MODE", "no") == "yes": integration = "Autoconf" - elif Path("/usr/share/bunkerweb/INTEGRATION").exists(): + elif Path("/usr/share/bunkerweb/INTEGRATION").is_file(): integration = Path("/usr/share/bunkerweb/INTEGRATION").read_text().strip() + elif ( + Path("/etc/os-release").is_file() + and "Alpine" in Path("/etc/os-release").read_text() + ): + integration = "Docker" if args.variables: logger.info(f"Variables : {args.variables}") @@ -163,7 +168,10 @@ if __name__ == "__main__": ) templator.render() - if integration == "Linux" and not args.no_linux_reload: + if ( + integration not in ("Autoconf", "Swarm", "Kubernetes", "Docker") + and not args.no_linux_reload + ): retries = 0 while not Path("/var/tmp/bunkerweb/nginx.pid").exists(): if retries == 5: diff --git a/src/common/gen/save_config.py b/src/common/gen/save_config.py index 82a510c32..477378043 100644 --- a/src/common/gen/save_config.py +++ b/src/common/gen/save_config.py @@ -142,6 +142,11 @@ if __name__ == "__main__": integration = "Autoconf" elif Path("/usr/share/bunkerweb/INTEGRATION").is_file(): integration = Path("/usr/share/bunkerweb/INTEGRATION").read_text().strip() + elif ( + Path("/etc/os-release").is_file() + and "Alpine" in Path("/etc/os-release").read_text() + ): + integration = "Docker" if args.init: logger.info(f"Detected {integration} integration") diff --git a/src/scheduler/JobScheduler.py b/src/scheduler/JobScheduler.py index ee436d65e..0c7ae3181 100644 --- a/src/scheduler/JobScheduler.py +++ b/src/scheduler/JobScheduler.py @@ -76,7 +76,7 @@ class JobScheduler(ApiCaller): def __reload(self): reload = True - if self.__integration == "Linux": + if self.__integration not in ("Autoconf", "Swarm", "Kubernetes", "Docker"): self.__logger.info("Reloading nginx ...") proc = run( ["nginx", "-s", "reload"], stdin=DEVNULL, stderr=PIPE, env=self.__env diff --git a/src/scheduler/main.py b/src/scheduler/main.py index ad783f311..f5ea30638 100644 --- a/src/scheduler/main.py +++ b/src/scheduler/main.py @@ -106,7 +106,7 @@ def generate_custom_configs( Path(dirname(tmp_path)).mkdir(parents=True, exist_ok=True) Path(tmp_path).write_bytes(custom_config["data"]) - if integration != "Linux": + if integration not in ("Autoconf", "Swarm", "Kubernetes", "Docker"): logger.info("Sending custom configs to BunkerWeb") ret = api_caller._send_files("/data/configs", "/custom_configs") @@ -137,7 +137,7 @@ def generate_external_plugins( st = stat(job_file) chmod(job_file, st.st_mode | S_IEXEC) - if integration != "Linux": + if integration not in ("Autoconf", "Swarm", "Kubernetes", "Docker"): logger.info("Sending plugins to BunkerWeb") ret = api_caller._send_files("/data/plugins", "/plugins") @@ -377,7 +377,7 @@ if __name__ == "__main__": logger.info("Successfully sent /data/cache folder") # restart nginx - if integration == "Linux": + if integration not in ("Autoconf", "Swarm", "Kubernetes", "Docker"): # Stop temp nginx logger.info("Stopping temp nginx ...") proc = subprocess_run( @@ -461,7 +461,7 @@ if __name__ == "__main__": # reload nginx logger.info("Reloading nginx ...") - if integration == "Linux": + if integration not in ("Autoconf", "Swarm", "Kubernetes", "Docker"): # Reloading the nginx server. proc = subprocess_run( # Reload nginx From cf2938bf2eda5d50394bd5671d9f05b9c35398c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 17:40:27 +0200 Subject: [PATCH 07/10] Update web-ui docs according to the next major version --- docs/web-ui.md | 230 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 153 insertions(+), 77 deletions(-) diff --git a/docs/web-ui.md b/docs/web-ui.md index 54ffe7d1c..9abf07d49 100644 --- a/docs/web-ui.md +++ b/docs/web-ui.md @@ -82,7 +82,7 @@ Because the web UI is a web application, the recommended installation procedure * replace the `/changeme` URL with a custom one of your choice * the `bunkerweb.INSTANCE` label is mandatory - The web UI will need to access the Docker API in order to get metadata about the running containers. It can be done easily by mounting the **docker.sock** file into the container. But there is a security risk : if the web UI is exploited, all your container(s) and the host will be impacted because, at the moment, Docker doesn't provide any restriction feature. We highly recommend using something like a [docker socket proxy](https://github.com/Tecnativa/docker-socket-proxy) to mitigate that risk (only a subset of read-only API endpoints will be available to the web UI container). + The web UI and the Scheduler will need to access the Docker API in order to get metadata about the running containers. It can be done easily by mounting the **docker.sock** file into the container. But there is a security risk : if the web UI is exploited, all your container(s) and the host will be impacted because, at the moment, Docker doesn't provide any restriction feature. We highly recommend using something like a [docker socket proxy](https://github.com/Tecnativa/docker-socket-proxy) to mitigate that risk (only a subset of read-only API endpoints will be available to the web UI container). To connect the docker socket proxy and the web UI, you will need another network : ```shell @@ -236,7 +236,7 @@ Because the web UI is a web application, the recommended installation procedure * replace `10.20.30.0/24` with the same network address used for the `bw-universe` network * the `bunkerweb.INSTANCE` label is mandatory - The Autoconf and web UI will need to access the Docker API in order to get metadata about the running containers. It can be done easily by mounting the **docker.sock** file into the container. But there is a security risk : if the web UI is exploited, all your container(s) and the host will be impacted because, at the moment, Docker doesn't provide any restriction feature. We highly recommend using something like a [docker socket proxy](https://github.com/Tecnativa/docker-socket-proxy) to mitigate that risk (only a subset of read-only API endpoints will be available to the web UI container). + The Autoconf, the Scheduler and web UI will need to access the Docker API in order to get metadata about the running containers. It can be done easily by mounting the **docker.sock** file into the container. But there is a security risk : if the web UI is exploited, all your container(s) and the host will be impacted because, at the moment, Docker doesn't provide any restriction feature. We highly recommend using something like a [docker socket proxy](https://github.com/Tecnativa/docker-socket-proxy) to mitigate that risk (only a subset of read-only API endpoints will be available to the web UI container). To connect the docker socket proxy and the web UI, you will need another network : ```shell @@ -271,6 +271,7 @@ Because the web UI is a web application, the recommended installation procedure --network bw-universe \ -v bw-data:/data \ -e DOCKER_HOST=tcp://bw-docker:2375 \ + -e AUTOCONF_MODE=yes \ bunkerity/bunkerweb-scheduler:1.5.0-beta && \ docker network connect bw-docker bw-scheduler ``` @@ -282,6 +283,7 @@ Because the web UI is a web application, the recommended installation procedure --network bw-universe \ -v bw-data:/data \ -e DOCKER_HOST=tcp://bw-docker:2375 \ + -e AUTOCONF_MODE=yes \ -e ADMIN_USERNAME=admin \ -e ADMIN_PASSWORD=changeme \ -e ABSOLUTE_URI=http(s)://bwadm.example.com/changeme/ \ @@ -320,7 +322,6 @@ Because the web UI is a web application, the recommended installation procedure - SERVER_NAME= - MULTISITE=yes - API_WHITELIST_IP=127.0.0.0/8 10.20.30.0/24 - - AUTOCONF_MODE=yes networks: - bw-universe - bw-services @@ -369,7 +370,7 @@ Because the web UI is a web application, the recommended installation procedure - AUTOCONF_MODE=yes - ADMIN_USERNAME=admin - ADMIN_PASSWORD=changeme - - ABSOLUTE_URI=http://bwadm.example.com/changeme/ + - ABSOLUTE_URI=http(s)://bwadm.example.com/changeme/ labels: - "bunkerweb.SERVER_NAME=bwadm.example.com" - "bunkerweb.USE_UI=yes" @@ -403,27 +404,36 @@ Because the web UI is a web application, the recommended installation procedure Let's start by creating the networks (replace 10.20.30.0/24 with an unused network of your choice) : ```shell - docker network create --subnet 10.20.30.0/24 bw-universe && \ - docker network create bw-services + docker network create -d overlay --attachable --subnet 10.20.30.0/24 bw-universe && \ + docker network create -d overlay --attachable bw-services ``` - You will also need the data volume, which where BunkerWeb's data will be stored : + Swarm will need a redis instance to work properly, you can use the [official redis image](https://hub.docker.com/_/redis) : ```shell - docker volume create bw-data + docker service create \ + --name bw-redis \ + --constraint 'node.role == worker' \ + --network bw-universe \ + redis:7-alpine ``` You can now create the BunkerWeb container, please note the special `bunkerweb.INSTANCE` label which is mandatory for the scheduler as well as the web UI to work properly : ```shell - docker run -d \ - --name bunkerweb \ - --network bw-universe \ - -p 80:8080 \ - -p 443:8443 \ - -e SERVER_NAME= \ - -e MULTISITE=yes \ - -e "API_WHITELIST_IP=127.0.0.0/8 10.20.30.0/24" \ - -l bunkerweb.INSTANCE \ - bunkerity/bunkerweb:1.5.0-beta + docker service create \ + --name bunkerweb \ + --mode global \ + --constraint 'node.role == worker' \ + --network bw-universe \ + --network bw-services \ + -p published=80,target=8080,mode=host \ + -p published=443,target=8443,mode=host \ + -e SERVER_NAME= \ + -e MULTISITE=yes \ + -e USE_REDIS=yes \ + -e REDIS_HOST=bw-redis \ + -e "API_WHITELIST_IP=127.0.0.0/8 10.20.30.0/24" \ + -l "bunkerweb.INSTANCE" \ + bunkerity/bunkerweb:1.5.0-beta ``` Important things to note : @@ -431,7 +441,7 @@ Because the web UI is a web application, the recommended installation procedure * replace `10.20.30.0/24` with the same network address used for the `bw-universe` network * the `bunkerweb.INSTANCE` label is mandatory - The Autoconf and web UI will need to access the Docker API in order to get metadata about the running containers. It can be done easily by mounting the **docker.sock** file into the container. But there is a security risk : if the web UI is exploited, all your container(s) and the host will be impacted because, at the moment, Docker doesn't provide any restriction feature. We highly recommend using something like a [docker socket proxy](https://github.com/Tecnativa/docker-socket-proxy) to mitigate that risk (only a subset of read-only API endpoints will be available to the web UI container). + Swarm, the Scheduler and the web UI will need to access the Docker API in order to get metadata about the running containers. It can be done easily by mounting the **docker.sock** file into the container. But there is a security risk : if the web UI is exploited, all your container(s) and the host will be impacted because, at the moment, Docker doesn't provide any restriction feature. We highly recommend using something like a [docker socket proxy](https://github.com/Tecnativa/docker-socket-proxy) to mitigate that risk (only a subset of read-only API endpoints will be available to the web UI container). To connect the docker socket proxy and the web UI, you will need another network : ```shell @@ -440,43 +450,72 @@ Because the web UI is a web application, the recommended installation procedure Once the network is created, you can now create the docker socket proxy container : ```shell - docker run -d \ + docker service create \ --name bw-docker \ + --constraint 'node.role == manager' \ --network bw-docker \ - -v /var/run/docker.sock:/var/run/docker.sock:ro \ + --mount type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock,ro \ + -e CONFIGS=1 \ -e CONTAINERS=1 \ + -e SERVICES=1 \ + -e SWARM=1 \ + -e TASKS=1 \ tecnativa/docker-socket-proxy ``` - You can then create the autoconf container connected to the docker socket proxy network : + Swarm, the Scheduler and the web UI will also need to share data among them. We recommend using a dedicated database container for that purpose. For this example, we will use a [MariaDB](https://mariadb.org/) container : ```shell - docker run -d \ - --name bw-autoconf \ - --network bw-universe \ - -v bw-data:/data \ - -e DOCKER_HOST=tcp://bw-docker:2375 \ - bunkerity/bunkerweb-autoconf:1.5.0-beta && \ - docker network connect bw-docker bw-autoconf + docker service create \ + --name bw-db \ + --network bw-docker \ + --mount type=volume,source=bw-data,destination=/var/lib/mysql \ + -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ + -e MYSQL_DATABASE=db \ + -e MYSQL_USER=bunkerweb \ + -e MYSQL_PASSWORD=changeme \ + mariadb:10.10 ``` - You can then create the scheduler container with the bw-data volume and the docker socket proxy network : + Important things to note : + + * For security purposes the database container should not be on the same network as the BunkerWeb container + * Replace the password `changeme` with strong ones, the same password will be used in the url provided to Swarm, the Scheduler and the web UI + * The URL provided to Swarm, the Scheduler and the web UI will be `mariadb+pymysql://bunkerweb:changeme@bw-db:3306/db` + + You can then create the autoconf container connected to the database and the docker socket proxy network : ```shell - docker run -d \ + docker service create \ + --name bw-autoconf \ + --network bw-universe \ + --network bw-docker \ + -e SWARM_MODE=yes \ + -e DOCKER_HOST=tcp://bw-docker:2375 + -e DATABASE_URI=mariadb+pymysql://bunkerweb:changeme@bw-db:3306/db \ + bunkerity/bunkerweb-autoconf:1.5.0-beta + ``` + + You can then create the scheduler container connected to the database and the docker socket proxy network : + ```shell + docker service create \ --name bw-scheduler \ --network bw-universe \ - -v bw-data:/data \ - -e DOCKER_HOST=tcp://bw-docker:2375 \ - bunkerity/bunkerweb-scheduler:1.5.0-beta && \ - docker network connect bw-docker bw-scheduler + --network bw-docker \ + -e SWARM_MODE=yes \ + -e DOCKER_HOST=tcp://bw-docker:2375 + -e DATABASE_URI=mariadb+pymysql://bunkerweb:changeme@bw-db:3306/db \ + bunkerity/bunkerweb-scheduler:1.5.0-beta ``` We can finally create the web UI container : ```shell - docker run -d \ + docker service create \ --name bw-ui \ --network bw-universe \ + --network bw-docker \ -v bw-data:/data \ + -e SWARM_MODE=yes \ -e DOCKER_HOST=tcp://bw-docker:2375 \ + -e DATABASE_URI=mariadb+pymysql://bunkerweb:changeme@bw-db:3306/db \ -e ADMIN_USERNAME=admin \ -e ADMIN_PASSWORD=changeme \ -e ABSOLUTE_URI=http(s)://bwadm.example.com/changeme/ \ @@ -488,8 +527,7 @@ Because the web UI is a web application, the recommended installation procedure -l "bunkerweb.REVERSE_PROXY_HOST=http://bw-ui:7000" \ -l "bunkerweb.REVERSE_PROXY_HEADERS=X-Script-Name /changeme" \ -l "bunkerweb.INTERCEPTED_ERROR_CODES=400 401.5.0-beta 405 413 429 500 501 502 503 504" \ - bunkerity/bunkerweb-ui:1.5.0-beta && \ - docker network connect bw-docker bw-ui + bunkerity/bunkerweb-ui:1.5.0-beta ``` Important things to note : @@ -507,36 +545,37 @@ Because the web UI is a web application, the recommended installation procedure bunkerweb: image: bunkerity/bunkerweb:1.5.0-beta ports: - - 80:8080 - - 443:8443 - labels: - - "bunkerweb.INSTANCE" + - published: 80 + target: 8080 + mode: host + protocol: tcp + - published: 443 + target: 8443 + mode: host + protocol: tcp environment: - SERVER_NAME= - MULTISITE=yes + - USE_REDIS=yes + - REDIS_HOST=bw-redis - API_WHITELIST_IP=127.0.0.0/8 10.20.30.0/24 - - AUTOCONF_MODE=yes networks: - bw-universe - bw-services + deploy: + mode: global + placement: + constraints: + - "node.role == worker" + labels: + - "bunkerweb.INSTANCE" bw-autoconf: image: bunkerity/bunkerweb-autoconf:1.5.0-beta - volumes: - - bw-data:/data environment: + - SWARM_MODE=yes - DOCKER_HOST=tcp://bw-docker:2375 - networks: - - bw-universe - - bw-docker - - bw-scheduler: - image: bunkerity/bunkerweb-scheduler:1.5.0-beta - volumes: - - bw-data:/data - environment: - - DOCKER_HOST=tcp://bw-docker:2375 - - AUTOCONF_MODE=yes + - DATABASE_URI=mariadb+pymysql://bunkerweb:changeme@bw-db:3306/db networks: - bw-universe - bw-docker @@ -546,34 +585,67 @@ Because the web UI is a web application, the recommended installation procedure volumes: - /var/run/docker.sock:/var/run/docker.sock:ro environment: + - CONFIGS=1 - CONTAINERS=1 + - SERVICES=1 + - SWARM=1 + - TASKS=1 networks: - bw-docker - + deploy: + placement: + constraints: + - "node.role == manager" + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + environment: + - SWARM_MODE=yes + - DOCKER_HOST=tcp://bw-docker:2375 + - DATABASE_URI=mariadb+pymysql://bunkerweb:changeme@bw-db:3306/db + networks: + - bw-universe + - bw-docker + + bw-db: + image: mariadb:10.10 + environment: + - MYSQL_RANDOM_ROOT_PASSWORD=yes + - MYSQL_DATABASE=db + - MYSQL_USER=bunkerweb + - MYSQL_PASSWORD=changeme + volumes: + - bw-data:/var/lib/mysql + networks: + - bw-docker + + bw-redis: + image: redis:7-alpine + networks: + - bw-universe + bw-ui: image: bunkerity/bunkerweb-ui:1.5.0-beta - networks: - bw-docker: - bw-universe: - aliases: - - bw-ui - volumes: - - bw-data:/data environment: + - SWARM_MODE=yes + - DATABASE_URI=mariadb+pymysql://bunkerweb:changeme@bw-db:3306/db - DOCKER_HOST=tcp://bw-docker:2375 - - AUTOCONF_MODE=yes - ADMIN_USERNAME=admin - ADMIN_PASSWORD=changeme - - ABSOLUTE_URI=http://bwadm.example.com/changeme/ - labels: - - "bunkerweb.SERVER_NAME=bwadm.example.com" - - "bunkerweb.USE_UI=yes" - - "bunkerweb.CONTENT_SECURITY_POLICY=object-src 'none'; frame-ancestors 'self';" - - "bunkerweb.USE_REVERSE_PROXY=yes" - - "bunkerweb.REVERSE_PROXY_URL=/changeme" - - "bunkerweb.REVERSE_PROXY_HOST=http://bw-ui:7000" - - "bunkerweb.REVERSE_PROXY_HEADERS=X-Script-Name /changeme" - - "bunkerweb.INTERCEPTED_ERROR_CODES=400 401.5.0-beta 405 413 429 500 501 502 503 504" + - ABSOLUTE_URI=http://bwadm.example.com/changeme + networks: + - bw-universe + - bw-docker + deploy: + labels: + - bunkerweb.SERVER_NAME=bwadm.example.com + - bunkerweb.USE_UI=yes + - "bunkerweb.CONTENT_SECURITY_POLICY=object-src 'none'; frame-ancestors 'self';" + - bunkerweb.USE_REVERSE_PROXY=yes + - bunkerweb.REVERSE_PROXY_URL=/changeme + - bunkerweb.REVERSE_PROXY_HOST=http://bw-ui:7000 + - bunkerweb.REVERSE_PROXY_HEADERS=X-Script-Name /changeme + - bunkerweb.INTERCEPTED_ERROR_CODES=400 401 405 413 429 500 501 502 503 504 volumes: bw-data: @@ -581,15 +653,19 @@ Because the web UI is a web application, the recommended installation procedure networks: bw-universe: name: bw-universe + driver: overlay + attachable: true ipam: - driver: default config: - subnet: 10.20.30.0/24 bw-services: name: bw-services + driver: overlay + attachable: true bw-docker: name: bw-docker - + driver: overlay + attachable: true ``` === "Kubernetes" From 348ab7a1e2886981c54fd0e6b1fb5e9009da9ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 17:40:59 +0200 Subject: [PATCH 08/10] Add feature that allow the copy of code blocks in markdown + Update copyright --- mkdocs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 9673b1437..68186bd22 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -30,6 +30,7 @@ theme: - navigation.tabs - navigation.tabs.sticky - toc.integrate + - content.code.copy custom_dir: docs/overrides extra_css: @@ -48,7 +49,7 @@ markdown_extensions: emoji_index: !!python/name:materialx.emoji.twemoji emoji_generator: !!python/name:materialx.emoji.to_svg -copyright: Copyright © 2022 Bunkerity +copyright: Copyright © Bunkerity extra: version: provider: mike From 0bd3e273b71f65d4fcec6933570f4a6a7ad9d160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 17:54:23 +0200 Subject: [PATCH 09/10] Update compression_level of sent tarfiles to 5 instead of 9 --- src/common/utils/ApiCaller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/utils/ApiCaller.py b/src/common/utils/ApiCaller.py index 45605c6b1..38be644b3 100644 --- a/src/common/utils/ApiCaller.py +++ b/src/common/utils/ApiCaller.py @@ -139,7 +139,7 @@ class ApiCaller: def _send_files(self, path, url): ret = True with BytesIO() as tgz: - with taropen(mode="w:gz", fileobj=tgz, dereference=True) as tf: + with taropen(mode="w:gz", fileobj=tgz, dereference=True, compresslevel=5) as tf: tf.add(path, arcname=".") tgz.seek(0, 0) files = {"archive.tar.gz": tgz} From ed495b99ff663cbd26e265553de1b34bbdf5bf6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 18 Apr 2023 18:00:45 +0200 Subject: [PATCH 10/10] Add CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 128 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..a7fc73ed9 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +contact@bunkerity.com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. \ No newline at end of file