diff --git a/src/deps/update_python_deps.sh b/src/deps/update_python_deps.sh index c8d82e5a4..c0e0ad4c3 100755 --- a/src/deps/update_python_deps.sh +++ b/src/deps/update_python_deps.sh @@ -10,7 +10,12 @@ pip install pip --upgrade > /dev/null && pip install pip-compile-multi pip-upgra echo "Updating requirements.in files" -files=("../../docs/requirements.txt" "../common/db/requirements.in" "../common/gen/requirements.in" "../scheduler/requirements.in" "../ui/requirements.in" "../../tests/requirements.txt" "../../tests/ui/requirements.txt") +files=("../../docs/requirements.txt" "../common/db/requirements.in" "../common/gen/requirements.in" "../scheduler/requirements.in" "../ui/requirements.in") + +for file in $(find ../../tests -iname "requirements.txt") +do + files+=("$file") +done for file in "${files[@]}" do @@ -31,6 +36,8 @@ do echo "No need to generate hashes for $file" fi + echo " " + cd - done diff --git a/tests/core/.dockerignore b/tests/core/.dockerignore new file mode 100644 index 000000000..8fa5b33d8 --- /dev/null +++ b/tests/core/.dockerignore @@ -0,0 +1 @@ +env \ No newline at end of file diff --git a/tests/core/Dockerfile.dev b/tests/core/Dockerfile.dev new file mode 100644 index 000000000..e876e1513 --- /dev/null +++ b/tests/core/Dockerfile.dev @@ -0,0 +1,5 @@ +FROM docker + +COPY . . + +ENTRYPOINT [ "./tests.sh" ] \ No newline at end of file diff --git a/tests/core/antibot/Dockerfile b/tests/core/antibot/Dockerfile new file mode 100644 index 000000000..c6b6dd4bc --- /dev/null +++ b/tests/core/antibot/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11.3-alpine + +# Install firefox and geckodriver +RUN apk add --no-cache --virtual .build-deps curl grep zip && \ + apk add --no-cache firefox + +# Installing geckodriver for firefox... +RUN GECKODRIVER_VERSION=`curl -i https://github.com/mozilla/geckodriver/releases/latest | grep -Po 'v[0-9]+.[0-9]+.[0-9]+'` && \ + wget -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \ + tar -C /usr/local/bin -xzvf geckodriver.tar.gz && \ + chmod +x /usr/local/bin/geckodriver && \ + rm geckodriver.tar.gz + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/antibot/docker-compose.test.yml b/tests/core/antibot/docker-compose.test.yml new file mode 100644 index 000000000..949b2c980 --- /dev/null +++ b/tests/core/antibot/docker-compose.test.yml @@ -0,0 +1,18 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_ANTIBOT: "no" + ANTIBOT_URI: "/challenge" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/antibot/docker-compose.yml b/tests/core/antibot/docker-compose.yml new file mode 100644 index 000000000..06b9b8b9e --- /dev/null +++ b/tests/core/antibot/docker-compose.yml @@ -0,0 +1,68 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + USE_REVERSE_PROXY: "yes" + REVERSE_PROXY_HOST: "http://app1:8080" + REVERSE_PROXY_URL: "/" + LOG_LEVEL: "info" + + # ? ANTIBOT settings + USE_ANTIBOT: "no" + ANTIBOT_URI: "/challenge" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + app1: + image: nginxdemos/nginx-hello + networks: + bw-services: + ipv4_address: 192.168.0.4 + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/antibot/main.py b/tests/core/antibot/main.py new file mode 100644 index 000000000..88800bc76 --- /dev/null +++ b/tests/core/antibot/main.py @@ -0,0 +1,95 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.common.exceptions import NoSuchElementException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + firefox_options = Options() + firefox_options.add_argument("--headless") + + test_type = getenv("USE_ANTIBOT", "no") + antibot_uri = getenv("ANTIBOT_URI", "/challenge") + + if test_type != "javascript": + print("ℹ️ Starting Firefox ...", flush=True) + with webdriver.Firefox(options=firefox_options) as driver: + driver.delete_all_cookies() + driver.maximize_window() + + print("ℹ️ Navigating to http://www.example.com ...", flush=True) + + driver.get("http://www.example.com") + + if driver.current_url.endswith(antibot_uri) and test_type == "no": + print("❌ Antibot is enabled, it shouldn't be ...", flush=True) + exit(1) + elif test_type == "captcha": + if not driver.current_url.endswith(antibot_uri): + print( + "❌ Antibot is disabled or the endpoint is wrong ...", flush=True + ) + exit(1) + try: + driver.find_element(By.XPATH, "//input[@name='captcha']") + except NoSuchElementException: + print("❌ The captcha input is missing ...", flush=True) + exit(1) + + print( + f"✅ The captcha input is present{' and the endpoint is correct' if antibot_uri != '/challenge' else ''} ...", + flush=True, + ) + else: + print("✅ Antibot is disabled, as expected ...", flush=True) + else: + status_code = get( + "http://www.example.com", + headers={"Host": "www.example.com"}, + allow_redirects=False, + ).status_code + if status_code >= 500: + print("ℹ️ An error occurred with the server, exiting ...", flush=True) + exit(1) + elif status_code != 302: + print( + "❌ The server should have redirected to the antibot page ...", + flush=True, + ) + exit(1) + + print("✅ Status code is 302, as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/antibot/requirements.txt b/tests/core/antibot/requirements.txt new file mode 100644 index 000000000..6f7b13f79 --- /dev/null +++ b/tests/core/antibot/requirements.txt @@ -0,0 +1,2 @@ +requests==2.30.0 +selenium==4.9.1 diff --git a/tests/core/antibot/test.sh b/tests/core/antibot/test.sh new file mode 100755 index 000000000..2df21aa1e --- /dev/null +++ b/tests/core/antibot/test.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +echo "🤖 Building antibot stack ..." + +# Starting stack +docker compose pull bw-docker app1 +if [ $? -ne 0 ] ; then + echo "🤖 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🤖 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@ANTIBOT_URI: "/custom"@ANTIBOT_URI: "/challenge"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_ANTIBOT: ".*"$@USE_ANTIBOT: "no"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🤖 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🤖 Down failed ❌" + exit 1 + fi + + echo "🤖 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "deactivated" "javascript" "captcha" "endpoint" +do + if [ "$test" = "deactivated" ] ; then + echo "🤖 Running tests without antibot ..." + elif [ "$test" = "endpoint" ] ; then + echo "🤖 Running tests where antibot is on a different endpoint ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@ANTIBOT_URI: "/challenge"@ANTIBOT_URI: "/custom"@' {} \; + elif [ "$test" != "deactivated" ] ; then + echo "🤖 Running tests with antibot \"$test\" ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_ANTIBOT: ".*"$@USE_ANTIBOT: "'"${test}"'"@' {} \; + fi + + echo "🤖 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🤖 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🤖 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("antibot-bw-1" "antibot-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🤖 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🤖 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🤖 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🤖 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🤖 Tests are done ! ✅" diff --git a/tests/core/authbasic/Dockerfile b/tests/core/authbasic/Dockerfile new file mode 100644 index 000000000..c6b6dd4bc --- /dev/null +++ b/tests/core/authbasic/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11.3-alpine + +# Install firefox and geckodriver +RUN apk add --no-cache --virtual .build-deps curl grep zip && \ + apk add --no-cache firefox + +# Installing geckodriver for firefox... +RUN GECKODRIVER_VERSION=`curl -i https://github.com/mozilla/geckodriver/releases/latest | grep -Po 'v[0-9]+.[0-9]+.[0-9]+'` && \ + wget -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \ + tar -C /usr/local/bin -xzvf geckodriver.tar.gz && \ + chmod +x /usr/local/bin/geckodriver && \ + rm geckodriver.tar.gz + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/authbasic/docker-compose.test.yml b/tests/core/authbasic/docker-compose.test.yml new file mode 100644 index 000000000..920cfaf9d --- /dev/null +++ b/tests/core/authbasic/docker-compose.test.yml @@ -0,0 +1,20 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_AUTH_BASIC: "no" + AUTH_BASIC_LOCATION: "sitewide" + AUTH_BASIC_USER: "bunkerity" + AUTH_BASIC_PASSWORD: "Secr3tP@ssw0rd" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/authbasic/docker-compose.yml b/tests/core/authbasic/docker-compose.yml new file mode 100644 index 000000000..3fdfa0ad8 --- /dev/null +++ b/tests/core/authbasic/docker-compose.yml @@ -0,0 +1,70 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + USE_REVERSE_PROXY: "yes" + REVERSE_PROXY_HOST: "http://app1:8080" + REVERSE_PROXY_URL: "/" + LOG_LEVEL: "info" + + # ? AUTH_BASIC settings + USE_AUTH_BASIC: "no" + AUTH_BASIC_LOCATION: "sitewide" + AUTH_BASIC_USER: "bunkerity" + AUTH_BASIC_PASSWORD: "Secr3tP@ssw0rd" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + app1: + image: nginxdemos/nginx-hello + networks: + bw-services: + ipv4_address: 192.168.0.4 + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/authbasic/main.py b/tests/core/authbasic/main.py new file mode 100644 index 000000000..2ffb1afe7 --- /dev/null +++ b/tests/core/authbasic/main.py @@ -0,0 +1,106 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.common.exceptions import NoSuchElementException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code <= 401 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + firefox_options = Options() + firefox_options.add_argument("--headless") + + use_auth_basic = getenv("USE_AUTH_BASIC", "no") + auth_basic_location = getenv("AUTH_BASIC_LOCATION", "sitewide") + auth_basic_username = getenv("AUTH_BASIC_USER", "bunkerity") + auth_basic_password = getenv("AUTH_BASIC_PASSWORD", "Secr3tP@ssw0rd") + + print("ℹ️ Starting Firefox ...", flush=True) + with webdriver.Firefox(options=firefox_options) as driver: + driver.delete_all_cookies() + driver.maximize_window() + + if use_auth_basic == "no" or auth_basic_location != "sitewide": + print("ℹ️ Navigating to http://www.example.com ...", flush=True) + driver.get("http://www.example.com") + + try: + driver.find_element(By.XPATH, "//img[@alt='NGINX Logo']") + except NoSuchElementException: + print("❌ The page is not accessible ...", flush=True) + exit(1) + + if use_auth_basic == "no": + print("✅ Auth-basic is disabled, as expected ...", flush=True) + else: + print( + f"ℹ️ Trying to access http://www.example.com{auth_basic_location} ...", + flush=True, + ) + status_code = get( + f"http://www.example.com{auth_basic_location}", + headers={"Host": "www.example.com"}, + ).status_code + + if status_code != 401: + print("❌ The page is accessible without auth-basic ...", flush=True) + exit(1) + print( + "✅ Auth-basic is enabled and working in the expected location ...", + ) + else: + print(f"ℹ️ Trying to access http://www.example.com ...", flush=True) + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code != 401: + print("❌ The page is accessible without auth-basic ...", flush=True) + exit(1) + + print( + f"ℹ️ Trying to access http://{auth_basic_username}:{auth_basic_password}@www.example.com ...", + flush=True, + ) + driver.get( + f"http://{auth_basic_username}:{auth_basic_password}@www.example.com" + ) + + try: + driver.find_element(By.XPATH, "//img[@alt='NGINX Logo']") + except NoSuchElementException: + print("❌ The page is not accessible ...", flush=True) + exit(1) + print("✅ Auth-basic is enabled and working, as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/authbasic/requirements.txt b/tests/core/authbasic/requirements.txt new file mode 100644 index 000000000..6f7b13f79 --- /dev/null +++ b/tests/core/authbasic/requirements.txt @@ -0,0 +1,2 @@ +requests==2.30.0 +selenium==4.9.1 diff --git a/tests/core/authbasic/test.sh b/tests/core/authbasic/test.sh new file mode 100755 index 000000000..cb3b8fcdb --- /dev/null +++ b/tests/core/authbasic/test.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +echo "🔐 Building authbasic stack ..." + +# Starting stack +docker compose pull bw-docker app1 +if [ $? -ne 0 ] ; then + echo "🔐 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🔐 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_AUTH_BASIC: "yes"@USE_AUTH_BASIC: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@AUTH_BASIC_LOCATION: "/auth"@AUTH_BASIC_LOCATION: "sitewide"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@AUTH_BASIC_USER: "admin"@AUTH_BASIC_USER: "bunkerity"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@AUTH_BASIC_PASSWORD: "password"@AUTH_BASIC_PASSWORD: "Secr3tP\@ssw0rd"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🔐 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🔐 Down failed ❌" + exit 1 + fi + + echo "🔐 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "deactivated" "sitewide" "location" "user" "password" +do + if [ "$test" = "deactivated" ] ; then + echo "🔐 Running tests without authbasic ..." + elif [ "$test" = "sitewide" ] ; then + echo "🔐 Running tests with sitewide authbasic ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_AUTH_BASIC: "no"@USE_AUTH_BASIC: "yes"@' {} \; + elif [ "$test" = "location" ] ; then + echo "🔐 Running tests with the location changed ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@AUTH_BASIC_LOCATION: "sitewide"@AUTH_BASIC_LOCATION: "/auth"@' {} \; + elif [ "$test" = "user" ] ; then + echo "🔐 Running tests with the user changed ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@AUTH_BASIC_LOCATION: "/auth"@AUTH_BASIC_LOCATION: "sitewide"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@AUTH_BASIC_USER: "bunkerity"@AUTH_BASIC_USER: "admin"@' {} \; + elif [ "$test" = "password" ] ; then + echo "🔐 Running tests with the password changed ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@AUTH_BASIC_PASSWORD: "Secr3tP\@ssw0rd"@AUTH_BASIC_PASSWORD: "password"@' {} \; + fi + + echo "🔐 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🔐 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🔐 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("authbasic-bw-1" "authbasic-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🔐 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🔐 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🔐 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🔐 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🔐 Tests are done ! ✅" diff --git a/tests/core/badbehavior/Dockerfile b/tests/core/badbehavior/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/badbehavior/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/badbehavior/docker-compose.test.yml b/tests/core/badbehavior/docker-compose.test.yml new file mode 100644 index 000000000..038cb7c20 --- /dev/null +++ b/tests/core/badbehavior/docker-compose.test.yml @@ -0,0 +1,25 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + DOCKER_HOST: "tcp://bw-docker:2375" + USE_BAD_BEHAVIOR: "yes" + BAD_BEHAVIOR_STATUS_CODES: "400 401 403 404 405 429 444" + BAD_BEHAVIOR_BAN_TIME: "86400" + BAD_BEHAVIOR_THRESHOLD: "10" + BAD_BEHAVIOR_COUNT_TIME: "60" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-docker: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true + bw-docker: + external: true diff --git a/tests/core/badbehavior/docker-compose.yml b/tests/core/badbehavior/docker-compose.yml new file mode 100644 index 000000000..ec3629b2e --- /dev/null +++ b/tests/core/badbehavior/docker-compose.yml @@ -0,0 +1,65 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? BAD_BEHAVIOR settings + USE_BAD_BEHAVIOR: "yes" + BAD_BEHAVIOR_STATUS_CODES: "400 401 403 404 405 429 444" + BAD_BEHAVIOR_BAN_TIME: "86400" + BAD_BEHAVIOR_THRESHOLD: "10" + BAD_BEHAVIOR_COUNT_TIME: "60" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: + name: bw-docker diff --git a/tests/core/badbehavior/index.html b/tests/core/badbehavior/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/badbehavior/main.py b/tests/core/badbehavior/main.py new file mode 100644 index 000000000..3e7cee3e3 --- /dev/null +++ b/tests/core/badbehavior/main.py @@ -0,0 +1,134 @@ +from contextlib import suppress +from datetime import datetime +from docker import DockerClient +from os import getenv +from requests import get +from requests.exceptions import RequestException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_bad_behavior = getenv("USE_BAD_BEHAVIOR", "yes") + bad_behavior_status_codes = getenv( + "BAD_BEHAVIOR_STATUS_CODES", "400 401 403 404 405 429 444" + ) + bad_behavior_ban_time = getenv("BAD_BEHAVIOR_BAN_TIME", "86400") + bad_behavior_threshold = getenv("BAD_BEHAVIOR_THRESHOLD", "10") + bad_behavior_count_time = getenv("BAD_BEHAVIOR_COUNT_TIME", "60") + + print( + "ℹ️ Sending 15 requests to http://www.example.com/?id=/etc/passwd ...", + flush=True, + ) + + for _ in range(15): + get( + "http://www.example.com/?id=/etc/passwd", + headers={"Host": "www.example.com"}, + ) + + sleep(1) + + status_code = get( + f"http://www.example.com", + headers={"Host": "www.example.com"}, + ).status_code + + if status_code == 403: + if use_bad_behavior == "no": + print("❌ Bad Behavior is enabled, it shouldn't be ...", flush=True) + exit(1) + elif bad_behavior_status_codes != "400 401 403 404 405 429 444": + print("❌ Bad Behavior's status codes didn't changed ...", flush=True) + exit(1) + elif bad_behavior_ban_time != "86400": + print( + "ℹ️ Sleeping for 7s to wait if Bad Behavior's ban time changed ...", + flush=True, + ) + sleep(7) + + status_code = get( + f"http://www.example.com", + headers={"Host": "www.example.com"}, + ).status_code + + if status_code == 403: + print("❌ Bad Behavior's ban time didn't changed ...", flush=True) + exit(1) + elif bad_behavior_threshold != "10": + print("❌ Bad Behavior's threshold didn't changed ...", flush=True) + exit(1) + elif bad_behavior_count_time != "60": + print( + "ℹ️ Sleeping for 7s to wait if Bad Behavior's count time changed ...", + flush=True, + ) + current_time = datetime.now().timestamp() + sleep(7) + + print( + "ℹ️ Checking BunkerWeb's logs to see if Bad Behavior's count time changed ...", + flush=True, + ) + + docker_host = getenv("DOCKER_HOST", "unix:///var/run/docker.sock") + docker_client = DockerClient(base_url=docker_host) + + bw_instances = docker_client.containers.list( + filters={"label": "bunkerweb.INSTANCE"} + ) + + if not bw_instances: + print("❌ BunkerWeb instance not found ...", flush=True) + exit(1) + + bw_instance = bw_instances[0] + + found = False + for log in bw_instance.logs(since=current_time).split(b"\n"): + if b"decreased counter for IP 192.168.0.3 (0/10)" in log: + found = True + break + + if not found: + print("❌ Bad Behavior's count time didn't changed ...", flush=True) + exit(1) + elif ( + use_bad_behavior == "yes" + and bad_behavior_status_codes == "400 401 403 404 405 429 444" + and bad_behavior_threshold == "10" + ): + print("❌ Bad Behavior is disabled, it shouldn't be ...", flush=True) + exit(1) + + print("✅ Bad Behavior is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/badbehavior/requirements.txt b/tests/core/badbehavior/requirements.txt new file mode 100644 index 000000000..a5d538d5f --- /dev/null +++ b/tests/core/badbehavior/requirements.txt @@ -0,0 +1,2 @@ +requests==2.30.0 +docker==6.1.2 diff --git a/tests/core/badbehavior/test.sh b/tests/core/badbehavior/test.sh new file mode 100755 index 000000000..1afda4660 --- /dev/null +++ b/tests/core/badbehavior/test.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +echo "📟 Building badbehavior stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "📟 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "📟 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BAD_BEHAVIOR: "no"@USE_BAD_BEHAVIOR: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_STATUS_CODES: "400 401 404 405 429 444"@BAD_BEHAVIOR_STATUS_CODES: "400 401 403 404 405 429 444"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_BAN_TIME: "5"@BAD_BEHAVIOR_BAN_TIME: "86400"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_THRESHOLD: "20"@BAD_BEHAVIOR_THRESHOLD: "10"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_COUNT_TIME: "5"@BAD_BEHAVIOR_COUNT_TIME: "60"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "📟 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "📟 Down failed ❌" + exit 1 + fi + + echo "📟 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "activated" "deactivated" "status_codes" "ban_time" "threshold" "count_time" +do + if [ "$test" = "activated" ] ; then + echo "📟 Running tests with badbehavior activated ..." + elif [ "$test" = "deactivated" ] ; then + echo "📟 Running tests without badbehavior ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BAD_BEHAVIOR: "yes"@USE_BAD_BEHAVIOR: "no"@' {} \; + elif [ "$test" = "status_codes" ] ; then + echo "📟 Running tests with badbehavior's 403 status code removed from the list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BAD_BEHAVIOR: "no"@USE_BAD_BEHAVIOR: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_STATUS_CODES: "400 401 403 404 405 429 444"@BAD_BEHAVIOR_STATUS_CODES: "400 401 404 405 429 444"@' {} \; + elif [ "$test" = "ban_time" ] ; then + echo "📟 Running tests with badbehavior's ban time changed to 5 seconds ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_STATUS_CODES: "400 401 404 405 429 444"@BAD_BEHAVIOR_STATUS_CODES: "400 401 403 404 405 429 444"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_BAN_TIME: "86400"@BAD_BEHAVIOR_BAN_TIME: "5"@' {} \; + elif [ "$test" = "threshold" ] ; then + echo "📟 Running tests with badbehavior's threshold set to 20 ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_BAN_TIME: "5"@BAD_BEHAVIOR_BAN_TIME: "86400"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_THRESHOLD: "10"@BAD_BEHAVIOR_THRESHOLD: "20"@' {} \; + elif [ "$test" = "count_time" ] ; then + echo "📟 Running tests with badbehavior's count time set to 5 seconds ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_THRESHOLD: "20"@BAD_BEHAVIOR_THRESHOLD: "10"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BAD_BEHAVIOR_COUNT_TIME: "60"@BAD_BEHAVIOR_COUNT_TIME: "5"@' {} \; + fi + + echo "📟 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "📟 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "📟 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("badbehavior-bw-1" "badbehavior-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "📟 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "📟 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "📟 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "📟 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "📟 Tests are done ! ✅" diff --git a/tests/core/blacklist/Dockerfile b/tests/core/blacklist/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/blacklist/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/blacklist/api/Dockerfile b/tests/core/blacklist/api/Dockerfile new file mode 100644 index 000000000..47b275094 --- /dev/null +++ b/tests/core/blacklist/api/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/blacklist_api + +COPY main.py . + +ENTRYPOINT [ "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers", "--forwarded-allow-ips", "\"*\"" ] \ No newline at end of file diff --git a/tests/core/blacklist/api/main.py b/tests/core/blacklist/api/main.py new file mode 100644 index 000000000..b7a8b92a1 --- /dev/null +++ b/tests/core/blacklist/api/main.py @@ -0,0 +1,30 @@ +from fastapi import FastAPI +from fastapi.responses import PlainTextResponse + + +app = FastAPI() + + +@app.get("/ip") +async def ip(): + return PlainTextResponse("192.168.0.3\n10.0.0.0/8\n127.0.0.1/32") + + +@app.get("/rdns") +async def rdns(): + return PlainTextResponse(".example.com\n.example.org\n.bw-services") + + +@app.get("/asn") +async def asn(): + return PlainTextResponse("1234\n13335\n5678") + + +@app.get("/user_agent") +async def user_agent(): + return PlainTextResponse("BunkerBot\nCensysInspect\nShodanInspect\nZmEu\nmasscan") + + +@app.get("/uri") +async def uri(): + return PlainTextResponse("/admin\n/login") diff --git a/tests/core/blacklist/api/requirements.txt b/tests/core/blacklist/api/requirements.txt new file mode 100644 index 000000000..c06221a5a --- /dev/null +++ b/tests/core/blacklist/api/requirements.txt @@ -0,0 +1,2 @@ +fastapi==0.95.1 +uvicorn[standard]==0.22.0 diff --git a/tests/core/blacklist/docker-compose.init.yml b/tests/core/blacklist/docker-compose.init.yml new file mode 100644 index 000000000..36d606b46 --- /dev/null +++ b/tests/core/blacklist/docker-compose.init.yml @@ -0,0 +1,9 @@ +version: "3.5" + +services: + init: + build: init + environment: + PYTHONUNBUFFERED: "1" + volumes: + - ./init/output:/output diff --git a/tests/core/blacklist/docker-compose.test.yml b/tests/core/blacklist/docker-compose.test.yml new file mode 100644 index 000000000..d7d8bb8d6 --- /dev/null +++ b/tests/core/blacklist/docker-compose.test.yml @@ -0,0 +1,72 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_BLACKLIST: "yes" + BLACKLIST_IP: "" + BLACKLIST_IP_URLS: "" + BLACKLIST_RDNS_GLOBAL: "yes" + BLACKLIST_RDNS: "" + BLACKLIST_RDNS_URLS: "" + BLACKLIST_ASN: "" + BLACKLIST_ASN_URLS: "" + BLACKLIST_USER_AGENT: "" + BLACKLIST_USER_AGENT_URLS: "" + BLACKLIST_URI: "" + BLACKLIST_URI_URLS: "" + BLACKLIST_IGNORE_IP: "" + BLACKLIST_IGNORE_IP_URLS: "" + BLACKLIST_IGNORE_RDNS: "" + BLACKLIST_IGNORE_RDNS_URLS: "" + BLACKLIST_IGNORE_ASN: "" + BLACKLIST_IGNORE_ASN_URLS: "" + BLACKLIST_IGNORE_USER_AGENT: "" + BLACKLIST_IGNORE_USER_AGENT_URLS: "" + BLACKLIST_IGNORE_URI: "" + BLACKLIST_IGNORE_URI_URLS: "" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + + global-tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_BLACKLIST: "yes" + BLACKLIST_IP: "" + BLACKLIST_IP_URLS: "" + BLACKLIST_RDNS_GLOBAL: "yes" + BLACKLIST_RDNS: "" + BLACKLIST_RDNS_URLS: "" + BLACKLIST_ASN: "" + BLACKLIST_ASN_URLS: "" + BLACKLIST_USER_AGENT: "" + BLACKLIST_USER_AGENT_URLS: "" + BLACKLIST_URI: "" + BLACKLIST_URI_URLS: "" + BLACKLIST_IGNORE_IP: "" + BLACKLIST_IGNORE_IP_URLS: "" + BLACKLIST_IGNORE_RDNS: "" + BLACKLIST_IGNORE_RDNS_URLS: "" + BLACKLIST_IGNORE_ASN: "" + BLACKLIST_IGNORE_ASN_URLS: "" + BLACKLIST_IGNORE_USER_AGENT: "" + BLACKLIST_IGNORE_USER_AGENT_URLS: "" + BLACKLIST_IGNORE_URI: "" + BLACKLIST_IGNORE_URI_URLS: "" + extra_hosts: + - "www.example.com:1.0.0.2" + networks: + bw-global-network: + ipv4_address: 1.0.0.3 + +networks: + bw-services: + external: true + bw-global-network: + external: true diff --git a/tests/core/blacklist/docker-compose.yml b/tests/core/blacklist/docker-compose.yml new file mode 100644 index 000000000..1d8a45dee --- /dev/null +++ b/tests/core/blacklist/docker-compose.yml @@ -0,0 +1,102 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + LOG_LEVEL: "info" + + # ? BLACKLIST settings + USE_BLACKLIST: "yes" + BLACKLIST_IP: "" + BLACKLIST_IP_URLS: "" + BLACKLIST_RDNS_GLOBAL: "yes" + BLACKLIST_RDNS: "" + BLACKLIST_RDNS_URLS: "" + BLACKLIST_ASN: "" + BLACKLIST_ASN_URLS: "" + BLACKLIST_USER_AGENT: "" + BLACKLIST_USER_AGENT_URLS: "" + BLACKLIST_URI: "" + BLACKLIST_URI_URLS: "" + BLACKLIST_IGNORE_IP: "" + BLACKLIST_IGNORE_IP_URLS: "" + BLACKLIST_IGNORE_RDNS: "" + BLACKLIST_IGNORE_RDNS_URLS: "" + BLACKLIST_IGNORE_ASN: "" + BLACKLIST_IGNORE_ASN_URLS: "" + BLACKLIST_IGNORE_USER_AGENT: "" + BLACKLIST_IGNORE_USER_AGENT_URLS: "" + BLACKLIST_IGNORE_URI: "" + BLACKLIST_IGNORE_URI_URLS: "" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + bw-global-network: + ipv4_address: 1.0.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + volumes: + - bw-data:/data + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + blacklist-api: + build: api + networks: + bw-docker: + bw-services: + ipv4_address: 192.168.0.4 + +volumes: + bw-data: + + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-global-network: + name: bw-global-network + ipam: + driver: default + config: + - subnet: 1.0.0.0/8 + bw-docker: + name: bw-docker diff --git a/tests/core/blacklist/index.html b/tests/core/blacklist/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/blacklist/init/Dockerfile b/tests/core/blacklist/init/Dockerfile new file mode 100644 index 000000000..2bb13a4f9 --- /dev/null +++ b/tests/core/blacklist/init/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/blacklist_init + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/blacklist/init/main.py b/tests/core/blacklist/init/main.py new file mode 100644 index 000000000..ef1409902 --- /dev/null +++ b/tests/core/blacklist/init/main.py @@ -0,0 +1,33 @@ +from datetime import date +from gzip import GzipFile +from io import BytesIO +from pathlib import Path +from maxminddb import MODE_FD, open_database +from requests import get + +# Compute the mmdb URL +mmdb_url = f"https://download.db-ip.com/free/dbip-asn-lite-{date.today().strftime('%Y-%m')}.mmdb.gz" + +# Download the mmdb file in memory +print(f"Downloading mmdb file from url {mmdb_url} ...", flush=True) +file_content = BytesIO() +with get(mmdb_url, stream=True) as resp: + resp.raise_for_status() + for chunk in resp.iter_content(chunk_size=4 * 1024): + if chunk: + file_content.write(chunk) +file_content.seek(0) + +with open_database(GzipFile(fileobj=file_content, mode="rb"), mode=MODE_FD) as reader: + dbip_asn = reader.get("1.0.0.3") + + if not dbip_asn: + print(f"❌ Error while reading mmdb file from {mmdb_url}", flush=True) + exit(1) + + print( + f"✅ ASN for IP 1.0.0.3 is {dbip_asn['autonomous_system_number']}, saving it to /output/ip_asn.txt", + flush=True, + ) + + Path("/output/ip_asn.txt").write_text(str(dbip_asn["autonomous_system_number"])) diff --git a/tests/core/blacklist/init/requirements.txt b/tests/core/blacklist/init/requirements.txt new file mode 100644 index 000000000..f91deab71 --- /dev/null +++ b/tests/core/blacklist/init/requirements.txt @@ -0,0 +1,2 @@ +maxminddb==2.3.0 +requests==2.30.0 diff --git a/tests/core/blacklist/main.py b/tests/core/blacklist/main.py new file mode 100644 index 000000000..d6b2350ec --- /dev/null +++ b/tests/core/blacklist/main.py @@ -0,0 +1,212 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 or status_code == 403 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_blacklist = getenv("USE_BLACKLIST", "yes") == "yes" + + blacklist_ip = getenv("BLACKLIST_IP", "") + blacklist_ip_urls = getenv("BLACKLIST_IP_URLS", "") + blacklist_rdns_global = getenv("BLACKLIST_RDNS_GLOBAL", "yes") == "yes" + blacklist_rdns = getenv("BLACKLIST_RDNS", "") + blacklist_rdns_urls = getenv("BLACKLIST_RDNS_URLS", "") + blacklist_asn = getenv("BLACKLIST_ASN", "") + blacklist_asn_urls = getenv("BLACKLIST_ASN_URLS", "") + blacklist_user_agent = getenv("BLACKLIST_USER_AGENT", "") + blacklist_user_agent_urls = getenv("BLACKLIST_USER_AGENT_URLS", "") + blacklist_uri = getenv("BLACKLIST_URI", "") + blacklist_uri_urls = getenv("BLACKLIST_URI_URLS", "") + + blacklist_ignore_ip = getenv("BLACKLIST_IGNORE_IP", "") + blacklist_ignore_ip_urls = getenv("BLACKLIST_IGNORE_IP_URLS", "") + blacklist_ignore_rdns = getenv("BLACKLIST_IGNORE_RDNS", "") + blacklist_ignore_rdns_urls = getenv("BLACKLIST_IGNORE_RDNS_URLS", "") + blacklist_ignore_asn = getenv("BLACKLIST_IGNORE_ASN", "") + blacklist_ignore_asn_urls = getenv("BLACKLIST_IGNORE_ASN_URLS", "") + blacklist_ignore_user_agent = getenv("BLACKLIST_IGNORE_USER_AGENT", "") + blacklist_ignore_user_agent_urls = getenv("BLACKLIST_IGNORE_USER_AGENT_URLS", "") + blacklist_ignore_uri = getenv("BLACKLIST_IGNORE_URI", "") + blacklist_ignore_uri_urls = getenv("BLACKLIST_IGNORE_URI_URLS", "") + + print( + "ℹ️ Sending a request to http://www.example.com/admin with User-Agent: BunkerBot ...", + flush=True, + ) + + status_code = get( + f"http://www.example.com/admin", + headers={"Host": "www.example.com", "User-Agent": "BunkerBot"}, + ).status_code + + if status_code == 403: + if not use_blacklist: + print( + "❌ The request was rejected, but the blacklist is disabled, exiting ..." + ) + exit(1) + elif blacklist_rdns_global and ( + blacklist_rdns != "" or blacklist_rdns_urls != "" + ): + print( + "❌ Blacklist's RDNS global didn't work as expected, exiting ...", + ) + exit(1) + elif blacklist_ignore_ip != "": + print("❌ Blacklist's ignore IP didn't work as expected, exiting ...") + exit(1) + elif blacklist_ignore_ip_urls != "": + print( + "❌ Blacklist's ignore IP urls didn't work as expected, exiting ...", + flush=True, + ) + exit(1) + elif blacklist_ignore_rdns != "": + print("❌ Blacklist's ignore RDNS didn't work as expected, exiting ...") + exit(1) + elif blacklist_ignore_rdns_urls != "": + print( + "❌ Blacklist's ignore RDNS urls didn't work as expected, exiting ...", + flush=True, + ) + exit(1) + elif blacklist_ignore_asn != "": + print("❌ Blacklist's ignore ASN didn't work as expected, exiting ...") + exit(1) + elif blacklist_ignore_asn_urls != "": + print( + "❌ Blacklist's ignore ASN urls didn't work as expected, exiting ...", + flush=True, + ) + exit(1) + elif blacklist_ignore_user_agent != "": + print( + "❌ Blacklist's ignore user agent didn't work as expected, exiting ...", + flush=True, + ) + exit(1) + elif blacklist_ignore_user_agent_urls != "": + print( + "❌ Blacklist's ignore user agent urls didn't work as expected, exiting ...", + flush=True, + ) + exit(1) + elif blacklist_ignore_uri != "": + print("❌ Blacklist's ignore URI didn't work as expected, exiting ...") + exit(1) + elif blacklist_ignore_uri_urls != "": + print( + "❌ Blacklist's ignore URI urls didn't work as expected, exiting ...", + flush=True, + ) + exit(1) + elif blacklist_ip != "" and not any( + [blacklist_ignore_ip, blacklist_ignore_ip_urls, not use_blacklist] + ): + print("❌ Blacklist's IP didn't work as expected, exiting ...", flush=True) + exit(1) + elif blacklist_ip_urls != "": + print("❌ Blacklist's IP urls didn't work as expected, exiting ...", flush=True) + exit(1) + elif blacklist_rdns != "" and not any( + [ + blacklist_ignore_rdns, + blacklist_ignore_rdns_urls, + blacklist_rdns_global, + ] + ): + print("❌ Blacklist's RDNS didn't work as expected, exiting ...", flush=True) + exit(1) + elif blacklist_rdns_urls != "" and blacklist_rdns_global: + print( + "❌ Blacklist's RDNS urls didn't work as expected, exiting ...", flush=True + ) + exit(1) + elif blacklist_asn != "" and not any( + [blacklist_ignore_asn, blacklist_ignore_asn_urls] + ): + print("❌ Blacklist's ASN didn't work as expected, exiting ...", flush=True) + exit(1) + elif blacklist_asn_urls != "": + print("❌ Blacklist's ASN urls didn't work as expected, exiting ...", flush=True) + exit(1) + elif blacklist_user_agent != "" and not any( + [blacklist_ignore_user_agent, blacklist_ignore_user_agent_urls] + ): + print( + "❌ Blacklist's User Agent didn't work as expected, exiting ...", flush=True + ) + exit(1) + elif blacklist_user_agent_urls != "": + print( + "❌ Blacklist's User Agent urls didn't work as expected, exiting ...", + flush=True, + ) + exit(1) + elif blacklist_uri != "" and not any( + [blacklist_ignore_uri, blacklist_ignore_uri_urls] + ): + print("❌ Blacklist's URI didn't work as expected, exiting ...", flush=True) + exit(1) + elif blacklist_uri_urls != "": + print("❌ Blacklist's URI urls didn't work as expected, exiting ...", flush=True) + exit(1) + elif use_blacklist and not any( + [ + blacklist_ip, + blacklist_ip_urls, + blacklist_rdns, + blacklist_rdns_urls, + blacklist_asn, + blacklist_asn_urls, + blacklist_user_agent, + blacklist_user_agent_urls, + blacklist_uri, + blacklist_uri_urls, + blacklist_ignore_ip, + blacklist_ignore_ip_urls, + blacklist_ignore_rdns, + blacklist_ignore_rdns_urls, + blacklist_ignore_asn, + blacklist_ignore_asn_urls, + blacklist_ignore_user_agent, + blacklist_ignore_user_agent_urls, + blacklist_ignore_uri, + blacklist_ignore_uri_urls, + ] + ): + print("❌ Blacklist is disabled, it shouldn't be ...", flush=True) + exit(1) + + print("✅ Blacklist is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/blacklist/requirements.txt b/tests/core/blacklist/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/blacklist/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/blacklist/test.sh b/tests/core/blacklist/test.sh new file mode 100755 index 000000000..745d11ddf --- /dev/null +++ b/tests/core/blacklist/test.sh @@ -0,0 +1,258 @@ +#!/bin/bash + +echo "🏴 Building blacklist stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🏴 Pull failed ❌" + exit 1 +fi + +echo "🏴 Building custom api image ..." +docker compose build blacklist-api +if [ $? -ne 0 ] ; then + echo "🏴 Build failed ❌" + exit 1 +fi + +echo "🏴 Building tests images ..." +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🏴 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +as_number=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + rm -rf init/output + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BLACKLIST: "no"@USE_BLACKLIST: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IP: "0.0.0.0/0"@BLACKLIST_IP: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_IP: "192.168.0.3"@BLACKLIST_IGNORE_IP: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IP_URLS: "http://blacklist-api:8080/ip"@BLACKLIST_IP_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_IP_URLS: "http://blacklist-api:8080/ip"@BLACKLIST_IGNORE_IP_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_RDNS_GLOBAL: "no"@BLACKLIST_RDNS_GLOBAL: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_RDNS: ".bw-services"@BLACKLIST_RDNS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_RDNS: ".bw-services"@BLACKLIST_IGNORE_RDNS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_RDNS_URLS: "http://blacklist-api:8080/rdns"@BLACKLIST_RDNS_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_RDNS_URLS: "http://blacklist-api:8080/rdns"@BLACKLIST_IGNORE_RDNS_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_ASN: "[0-9]*"@BLACKLIST_ASN: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_ASN: "[0-9]*"@BLACKLIST_IGNORE_ASN: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_ASN_URLS: "http://blacklist-api:8080/asn"@BLACKLIST_ASN_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_ASN_URLS: "http://blacklist-api:8080/asn"@BLACKLIST_IGNORE_ASN_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_USER_AGENT: "BunkerBot"@BLACKLIST_USER_AGENT: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_USER_AGENT: "BunkerBot"@BLACKLIST_IGNORE_USER_AGENT: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_USER_AGENT_URLS: "http://blacklist-api:8080/user_agent"@BLACKLIST_USER_AGENT_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_USER_AGENT_URLS: "http://blacklist-api:8080/user_agent"@BLACKLIST_IGNORE_USER_AGENT_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_URI: "/admin"@BLACKLIST_URI: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_URI: "/admin"@BLACKLIST_IGNORE_URI: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_URI_URLS: "http://blacklist-api:8080/uri"@BLACKLIST_URI_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_URI_URLS: "http://blacklist-api:8080/uri"@BLACKLIST_IGNORE_URI_URLS: ""@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🏴 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🏴 Down failed ❌" + exit 1 + fi + + echo "🏴 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +echo "🏴 Initializing workspace ..." +rm -rf init/output +mkdir -p init/output +docker compose -f docker-compose.init.yml up --build +if [ $? -ne 0 ] ; then + echo "🏴 Build failed ❌" + exit 1 +elif ! [[ -f "init/output/ip_asn.txt" ]]; then + echo "🏴 ip_asn.txt not found ❌" + exit 1 +fi + +as_number=$(cat init/output/ip_asn.txt) + +if [[ $as_number = "" ]]; then + echo "🏴 AS number not found ❌" + exit 1 +fi + +rm -rf init/output + +for test in "ip" "deactivated" "ignore_ip" "ignore_ip_urls" "ip_urls" "rdns" "rdns_global" "ignore_rdns" "ignore_rdns_urls" "rdns_urls" "asn" "ignore_asn" "ignore_asn_urls" "asn_urls" "user_agent" "ignore_user_agent" "ignore_user_agent_urls" "user_agent_urls" "uri" "ignore_uri" "ignore_uri_urls" "uri_urls" +do + if [ "$test" = "ip" ] ; then + echo "🏴 Running tests with the network 0.0.0.0/0 in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IP: ""@BLACKLIST_IP: "0.0.0.0/0"@' {} \; + elif [ "$test" = "deactivated" ] ; then + echo "🏴 Running tests when deactivating the blacklist ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BLACKLIST: "yes"@USE_BLACKLIST: "no"@' {} \; + elif [ "$test" = "ignore_ip" ] ; then + echo "🏴 Running tests with blacklist's ignore_ip set to 192.168.0.3 ..." + echo "ℹ️ Keeping the network 0.0.0.0/0 in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BLACKLIST: "no"@USE_BLACKLIST: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_IP: ""@BLACKLIST_IGNORE_IP: "192.168.0.3"@' {} \; + elif [ "$test" = "ignore_ip_urls" ] ; then + echo "🏴 Running tests with blacklist's ignore_ip_urls set to http://blacklist-api:8080/ip ..." + echo "ℹ️ Keeping the network 0.0.0.0/0 in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_IP: "192.168.0.3"@BLACKLIST_IGNORE_IP: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_IP_URLS: ""@BLACKLIST_IGNORE_IP_URLS: "http://blacklist-api:8080/ip"@' {} \; + elif [ "$test" = "ip_urls" ] ; then + echo "🏴 Running tests with blacklist's ip url set to http://blacklist-api:8080/ip ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_IP_URLS: "http://blacklist-api:8080/ip"@BLACKLIST_IGNORE_IP_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IP: "0.0.0.0/0"@BLACKLIST_IP: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IP_URLS: ""@BLACKLIST_IP_URLS: "http://blacklist-api:8080/ip"@' {} \; + elif [ "$test" = "rdns" ] ; then + echo "🏴 Running tests with blacklist's rdns set to .bw-services ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IP_URLS: "http://blacklist-api:8080/ip"@BLACKLIST_IP_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_RDNS: ""@BLACKLIST_RDNS: ".bw-services"@' {} \; + elif [ "$test" = "rdns_global" ] ; then + echo "🏴 Running tests when blacklist's rdns also scans local ip addresses ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_RDNS_GLOBAL: "yes"@BLACKLIST_RDNS_GLOBAL: "no"@' {} \; + elif [ "$test" = "ignore_rdns" ] ; then + echo "🏴 Running tests with blacklist's ignore_rdns set to .bw-services ..." + echo "ℹ️ Keeping the rdns also scanning local ip addresses ..." + echo "ℹ️ Keeping the rdns .bw-services in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_RDNS: ""@BLACKLIST_IGNORE_RDNS: ".bw-services"@' {} \; + elif [ "$test" = "ignore_rdns_urls" ] ; then + echo "🏴 Running tests with blacklist's ignore_rdns_urls set to http://blacklist-api:8080/rdns ..." + echo "ℹ️ Keeping the rdns also scanning local ip addresses ..." + echo "ℹ️ Keeping the rdns .bw-services in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_RDNS: ".bw-services"@BLACKLIST_IGNORE_RDNS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_RDNS_URLS: ""@BLACKLIST_IGNORE_RDNS_URLS: "http://blacklist-api:8080/rdns"@' {} \; + elif [ "$test" = "rdns_urls" ] ; then + echo "🏴 Running tests with blacklist's rdns url set to http://blacklist-api:8080/rdns ..." + echo "ℹ️ Keeping the rdns also scanning local ip addresses ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_RDNS_URLS: "http://blacklist-api:8080/rdns"@BLACKLIST_IGNORE_RDNS_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_RDNS: ".bw-services"@BLACKLIST_RDNS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_RDNS_URLS: ""@BLACKLIST_RDNS_URLS: "http://blacklist-api:8080/rdns"@' {} \; + elif [ "$test" = "asn" ] ; then + echo "🏴 Running tests with blacklist's asn set to $as_number ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_RDNS_GLOBAL: "no"@BLACKLIST_RDNS_GLOBAL: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_RDNS_URLS: "http://blacklist-api:8080/rdns"@BLACKLIST_RDNS_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_ASN: ""@BLACKLIST_ASN: "'"$as_number"'"@' {} \; + elif [ "$test" = "ignore_asn" ] ; then + echo "🏴 Running tests with blacklist's ignore_asn set to $as_number ..." + echo "ℹ️ Keeping the asn $as_number in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_ASN: ""@BLACKLIST_IGNORE_ASN: "'"$as_number"'"@' {} \; + elif [ "$test" = "ignore_asn_urls" ] ; then + echo "🏴 Running tests with blacklist's ignore_asn_urls set to http://blacklist-api:8080/asn ..." + echo "ℹ️ Keeping the asn $as_number in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_ASN: "'"$as_number"'"@BLACKLIST_IGNORE_ASN: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_ASN_URLS: ""@BLACKLIST_IGNORE_ASN_URLS: "http://blacklist-api:8080/asn"@' {} \; + elif [ "$test" = "asn_urls" ] ; then + echo "🏴 Running tests with blacklist's asn url set to http://blacklist-api:8080/asn ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_ASN_URLS: "http://blacklist-api:8080/asn"@BLACKLIST_IGNORE_ASN_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_ASN: "'"$as_number"'"@BLACKLIST_ASN: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_ASN_URLS: ""@BLACKLIST_ASN_URLS: "http://blacklist-api:8080/asn"@' {} \; + elif [ "$test" = "user_agent" ] ; then + echo "🏴 Running tests with blacklist's user_agent set to BunkerBot ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_ASN_URLS: "http://blacklist-api:8080/asn"@BLACKLIST_ASN_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_USER_AGENT: ""@BLACKLIST_USER_AGENT: "BunkerBot"@' {} \; + elif [ "$test" = "ignore_user_agent" ] ; then + echo "🏴 Running tests with blacklist's ignore_user_agent set to BunkerBot ..." + echo "ℹ️ Keeping the user_agent BunkerBot in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_USER_AGENT: ""@BLACKLIST_IGNORE_USER_AGENT: "BunkerBot"@' {} \; + elif [ "$test" = "ignore_user_agent_urls" ] ; then + echo "🏴 Running tests with blacklist's ignore_user_agent_urls set to http://blacklist-api:8080/user_agent ..." + echo "ℹ️ Keeping the user_agent BunkerBot in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_USER_AGENT: "BunkerBot"@BLACKLIST_IGNORE_USER_AGENT: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_USER_AGENT_URLS: ""@BLACKLIST_IGNORE_USER_AGENT_URLS: "http://blacklist-api:8080/user_agent"@' {} \; + elif [ "$test" = "user_agent_urls" ] ; then + echo "🏴 Running tests with blacklist's user_agent url set to http://blacklist-api:8080/user_agent ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_USER_AGENT_URLS: "http://blacklist-api:8080/user_agent"@BLACKLIST_IGNORE_USER_AGENT_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_USER_AGENT: "BunkerBot"@BLACKLIST_USER_AGENT: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_USER_AGENT_URLS: ""@BLACKLIST_USER_AGENT_URLS: "http://blacklist-api:8080/user_agent"@' {} \; + elif [ "$test" = "uri" ] ; then + echo "🏴 Running tests with blacklist's uri set to /admin ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_USER_AGENT_URLS: "http://blacklist-api:8080/user_agent"@BLACKLIST_USER_AGENT_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_URI: ""@BLACKLIST_URI: "/admin"@' {} \; + elif [ "$test" = "ignore_uri" ] ; then + echo "🏴 Running tests with blacklist's ignore_uri set to /admin ..." + echo "ℹ️ Keeping the uri /admin in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_URI: ""@BLACKLIST_IGNORE_URI: "/admin"@' {} \; + elif [ "$test" = "ignore_uri_urls" ] ; then + echo "🏴 Running tests with blacklist's ignore_ip_urls set to http://blacklist-api:8080/uri ..." + echo "ℹ️ Keeping the uri /admin in the ban list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_URI: "/admin"@BLACKLIST_IGNORE_URI: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_URI_URLS: ""@BLACKLIST_IGNORE_URI_URLS: "http://blacklist-api:8080/uri"@' {} \; + elif [ "$test" = "uri_urls" ] ; then + echo "🏴 Running tests with blacklist's uri url set to http://blacklist-api:8080/uri ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IGNORE_URI_URLS: "http://blacklist-api:8080/uri"@BLACKLIST_IGNORE_URI_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_URI: "/admin"@BLACKLIST_URI: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_URI_URLS: ""@BLACKLIST_URI_URLS: "http://blacklist-api:8080/uri"@' {} \; + fi + + echo "🏴 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🏴 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🏴 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("blacklist-bw-1" "blacklist-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🏴 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🏴 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + if [[ "$test" = "asn" || "$test" = "ignore_asn" || "$test" = "ignore_asn_urls" || "$test" = "asn_urls" ]] ; then + docker compose -f docker-compose.test.yml up global-tests --abort-on-container-exit --exit-code-from global-tests 2>/dev/null + else + docker compose -f docker-compose.test.yml up tests --abort-on-container-exit --exit-code-from tests 2>/dev/null + fi + + if [ $? -ne 0 ] ; then + echo "🏴 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb, BunkerWeb Scheduler and Custom API logs ..." + docker compose logs bw bw-scheduler blacklist-api + exit 1 + else + echo "🏴 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🏴 Tests are done ! ✅" diff --git a/tests/core/brotli/Dockerfile b/tests/core/brotli/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/brotli/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/brotli/docker-compose.test.yml b/tests/core/brotli/docker-compose.test.yml new file mode 100644 index 000000000..8fd33c7b0 --- /dev/null +++ b/tests/core/brotli/docker-compose.test.yml @@ -0,0 +1,17 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_BROTLI: "no" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/brotli/docker-compose.yml b/tests/core/brotli/docker-compose.yml new file mode 100644 index 000000000..f6e103c9a --- /dev/null +++ b/tests/core/brotli/docker-compose.yml @@ -0,0 +1,67 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + USE_REVERSE_PROXY: "yes" + REVERSE_PROXY_HOST: "http://app1:8080" + REVERSE_PROXY_URL: "/" + LOG_LEVEL: "info" + + # ? BROTLI settings + USE_BROTLI: "no" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + app1: + image: nginxdemos/nginx-hello + networks: + bw-services: + ipv4_address: 192.168.0.4 + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/brotli/main.py b/tests/core/brotli/main.py new file mode 100644 index 000000000..7f3a92ca7 --- /dev/null +++ b/tests/core/brotli/main.py @@ -0,0 +1,62 @@ +from contextlib import suppress +from os import getenv +from requests import RequestException, get, head +from traceback import format_exc +from time import sleep + + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_brotli = getenv("USE_BROTLI", "no") == "yes" + + print( + "ℹ️ Sending a HEAD request to http://www.example.com ...", + flush=True, + ) + + response = head( + "http://www.example.com", + headers={"Host": "www.example.com", "Accept-Encoding": "br"}, + ) + response.raise_for_status() + + if not use_brotli and response.headers.get("Content-Encoding", "").lower() == "br": + print( + f"❌ Content-Encoding header is present even if Brotli is deactivated, exiting ...\nheaders: {response.headers}" + ) + exit(1) + elif use_brotli and response.headers.get("Content-Encoding", "").lower() != "br": + print( + f"❌ Content-Encoding header is not present or with the wrong value even if Brotli is activated, exiting ...\nheaders: {response.headers}" + ) + exit(1) + + print("✅ Brotli is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/brotli/requirements.txt b/tests/core/brotli/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/brotli/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/brotli/test.sh b/tests/core/brotli/test.sh new file mode 100755 index 000000000..a70c41e11 --- /dev/null +++ b/tests/core/brotli/test.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +echo "📦 Building brotli stack ..." + +# Starting stack +docker compose pull bw-docker app1 +if [ $? -ne 0 ] ; then + echo "📦 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "📦 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BROTLI: "yes"@USE_BROTLI: "no"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "📦 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "📦 Down failed ❌" + exit 1 + fi + + echo "📦 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "deactivated" "activated" +do + if [ "$test" = "deactivated" ] ; then + echo "📦 Running tests without brotli ..." + elif [ "$test" = "activated" ] ; then + echo "📦 Running tests with brotli ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BROTLI: "no"@USE_BROTLI: "yes"@' {} \; + fi + + echo "📦 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "📦 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "📦 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("brotli-bw-1" "brotli-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "📦 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "📦 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "📦 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "📦 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "📦 Tests are done ! ✅" diff --git a/tests/core/bunkernet/Dockerfile b/tests/core/bunkernet/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/bunkernet/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/bunkernet/api/Dockerfile b/tests/core/bunkernet/api/Dockerfile new file mode 100644 index 000000000..47b275094 --- /dev/null +++ b/tests/core/bunkernet/api/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/blacklist_api + +COPY main.py . + +ENTRYPOINT [ "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers", "--forwarded-allow-ips", "\"*\"" ] \ No newline at end of file diff --git a/tests/core/bunkernet/api/main.py b/tests/core/bunkernet/api/main.py new file mode 100644 index 000000000..bdac9f9ab --- /dev/null +++ b/tests/core/bunkernet/api/main.py @@ -0,0 +1,46 @@ +from uuid import uuid4 +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + + +app = FastAPI() +instance_id = None +report_num = 0 + + +@app.get("/ping") +async def ping(_: Request): + return JSONResponse(status_code=200, content={"result": "ok", "data": "pong"}) + + +@app.post("/register") +async def register(_: Request): + global instance_id + instance_id = str(uuid4()) + return JSONResponse(status_code=200, content={"result": "ok", "data": instance_id}) + + +@app.post("/report") +async def report(_: Request): + global report_num + report_num += 1 + return JSONResponse( + status_code=200, content={"result": "ok", "data": "Report acknowledged."} + ) + + +@app.get("/db") +async def db(_: Request): + return JSONResponse(status_code=200, content={"result": "ok", "data": []}) + + +@app.get("/instance_id") +async def get_instance_id(_: Request): + global instance_id + return JSONResponse(status_code=200, content={"result": "ok", "data": instance_id}) + + +@app.get("/report_num") +async def get_report_num(_: Request): + global report_num + return JSONResponse(status_code=200, content={"result": "ok", "data": report_num}) diff --git a/tests/core/bunkernet/api/requirements.txt b/tests/core/bunkernet/api/requirements.txt new file mode 100644 index 000000000..c06221a5a --- /dev/null +++ b/tests/core/bunkernet/api/requirements.txt @@ -0,0 +1,2 @@ +fastapi==0.95.1 +uvicorn[standard]==0.22.0 diff --git a/tests/core/bunkernet/docker-compose.test.yml b/tests/core/bunkernet/docker-compose.test.yml new file mode 100644 index 000000000..fd362fe96 --- /dev/null +++ b/tests/core/bunkernet/docker-compose.test.yml @@ -0,0 +1,18 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_BUNKERNET: "yes" + BUNKERNET_SERVER: "http://bunkernet-api:8080" + extra_hosts: + - "www.example.com:1.0.0.2" + networks: + bw-services: + ipv4_address: 1.0.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/bunkernet/docker-compose.yml b/tests/core/bunkernet/docker-compose.yml new file mode 100644 index 000000000..89b5f6e60 --- /dev/null +++ b/tests/core/bunkernet/docker-compose.yml @@ -0,0 +1,68 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? BUNKERNET settings + USE_BUNKERNET: "yes" + BUNKERNET_SERVER: "http://bunkernet-api:8080" + networks: + bw-universe: + bw-services: + ipv4_address: 1.0.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + bunkernet-api: + build: api + networks: + bw-docker: + bw-services: + ipv4_address: 1.0.0.4 + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 1.0.0.0/24 + bw-docker: + name: bw-docker diff --git a/tests/core/bunkernet/index.html b/tests/core/bunkernet/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/bunkernet/main.py b/tests/core/bunkernet/main.py new file mode 100644 index 000000000..95ba055f1 --- /dev/null +++ b/tests/core/bunkernet/main.py @@ -0,0 +1,80 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_bunkernet = getenv("USE_BUNKERNET", "yes") == "yes" + bunkernet_server = getenv("BUNKERNET_SERVER") + + if not bunkernet_server: + print("❌ BunkerNet server not specified, exiting ...", flush=True) + exit(1) + + instance_id = get(f"{bunkernet_server}/instance_id").json()["data"] + + if use_bunkernet and not instance_id: + print("❌ BunkerNet plugin did not register, exiting ...", flush=True) + exit(1) + elif not use_bunkernet and instance_id: + print("❌ BunkerNet plugin registered but it shouldn't, exiting ...", flush=True) + exit(1) + elif not use_bunkernet and not instance_id: + print("✅ BunkerNet plugin is disabled and not registered ...", flush=True) + exit(0) + + print( + "ℹ️ Sending a request to http://www.example.com/?id=/etc/passwd ...", flush=True + ) + + status_code = get( + f"http://www.example.com/?id=/etc/passwd", + headers={"Host": "www.example.com"}, + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code != 403: + print("❌ The request was not blocked, exiting ...", flush=True) + exit(1) + + sleep(2) + + report_num = get(f"{bunkernet_server}/report_num").json()["data"] + + if report_num < 1: + print("❌ The report was not sent, exiting ...", flush=True) + exit(1) + + print("✅ BunkerNet is working as expected ...", flush=True) +except SystemExit as e: + exit(e.code) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/bunkernet/requirements.txt b/tests/core/bunkernet/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/bunkernet/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/bunkernet/test.sh b/tests/core/bunkernet/test.sh new file mode 100755 index 000000000..f5aa48cd8 --- /dev/null +++ b/tests/core/bunkernet/test.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +echo "🕸️ Building bunkernet stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🕸️ Pull failed ❌" + exit 1 +fi + +echo "🕸️ Building custom api image ..." +docker compose build bunkernet-api +if [ $? -ne 0 ] ; then + echo "🕸️ Build failed ❌" + exit 1 +fi + +echo "🕸️ Building tests images ..." +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🕸️ Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BUNKERNET: "no"@USE_BUNKERNET: "yes"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🕸️ Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🕸️ Down failed ❌" + exit 1 + fi + + echo "🕸️ Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "activated" "deactivated" +do + if [ "$test" = "activated" ] ; then + echo "🕸️ Running tests with bunkernet activated ..." + elif [ "$test" = "deactivated" ] ; then + echo "🕸️ Running tests without bunkernet ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_BUNKERNET: "yes"@USE_BUNKERNET: "no"@' {} \; + fi + + echo "🕸️ Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🕸️ Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🕸️ Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("bunkernet-bw-1" "bunkernet-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🕸️ Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🕸️ Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🏴 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb, BunkerWeb Scheduler and Custom API logs ..." + docker compose logs bw bw-scheduler bunkernet-api + exit 1 + else + echo "🏴 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🕸️ Tests are done ! ✅" diff --git a/tests/core/clientcache/Dockerfile b/tests/core/clientcache/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/clientcache/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/clientcache/docker-compose.test.yml b/tests/core/clientcache/docker-compose.test.yml new file mode 100644 index 000000000..4250ed80b --- /dev/null +++ b/tests/core/clientcache/docker-compose.test.yml @@ -0,0 +1,20 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_CLIENT_CACHE: "no" + CLIENT_CACHE_EXTENSIONS: "jpg|jpeg|png|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2" + CLIENT_CACHE_ETAG: "yes" + CLIENT_CACHE_CONTROL: "public, max-age=15552000" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/clientcache/docker-compose.yml b/tests/core/clientcache/docker-compose.yml new file mode 100644 index 000000000..e6714a462 --- /dev/null +++ b/tests/core/clientcache/docker-compose.yml @@ -0,0 +1,65 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + ports: + - 80:80 + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./image.png:/var/www/html/image.png + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? CLIENT_CACHE settings + USE_CLIENT_CACHE: "no" + CLIENT_CACHE_EXTENSIONS: "jpg|jpeg|png|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2" + CLIENT_CACHE_ETAG: "yes" + CLIENT_CACHE_CONTROL: "public, max-age=15552000" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/clientcache/image.png b/tests/core/clientcache/image.png new file mode 100644 index 000000000..44f08f1f8 Binary files /dev/null and b/tests/core/clientcache/image.png differ diff --git a/tests/core/clientcache/main.py b/tests/core/clientcache/main.py new file mode 100644 index 000000000..4e7c9ddda --- /dev/null +++ b/tests/core/clientcache/main.py @@ -0,0 +1,89 @@ +from contextlib import suppress +from os import getenv +from requests import RequestException, get +from traceback import format_exc +from time import sleep + + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com/image.png", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_client_cache = getenv("USE_CLIENT_CACHE", "no") == "yes" + default_cache_extensions = ( + getenv( + "CLIENT_CACHE_EXTENSIONS", + "jpg|jpeg|png|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2", + ) + == "jpg|jpeg|png|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2" + ) + client_cache_etag = getenv("CLIENT_CACHE_ETAG", "yes") == "yes" + client_cache_control = getenv("CLIENT_CACHE_CONTROL", "public, max-age=15552000") + + print( + "ℹ️ Sending a request to http://www.example.com/image.png ...", + flush=True, + ) + + response = get( + "http://www.example.com/image.png", headers={"Host": "www.example.com"} + ) + response.raise_for_status() + + if not use_client_cache: + if "Cache-Control" in response.headers: + print( + f"❌ Cache-Control header is present even if Client cache is deactivated, exiting ...\nheaders: {response.headers}" + ) + exit(1) + else: + if "Cache-Control" not in response.headers and default_cache_extensions: + print( + f"❌ Cache-Control header is not present even if Client cache is activated, exiting ...\nheaders: {response.headers}" + ) + exit(1) + elif not default_cache_extensions and "Cache-Control" in response.headers: + print( + f"❌ Cache-Control header is present even if the png extension is not in the list of extensions, exiting ...\nheaders: {response.headers}", + flush=True, + ) + elif not client_cache_etag and "ETag" in response.headers: + print( + f"❌ ETag header is present even if Client cache ETag is deactivated, exiting ...\nheaders: {response.headers}" + ) + exit(1) + elif default_cache_extensions and client_cache_control != response.headers.get( + "Cache-Control" + ): + print( + f"❌ Cache-Control header is not equal to the expected value, exiting ...\nheaders: {response.headers}" + ) + exit(1) + + print("✅ Client cache is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/clientcache/requirements.txt b/tests/core/clientcache/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/clientcache/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/clientcache/test.sh b/tests/core/clientcache/test.sh new file mode 100755 index 000000000..0b7ad9bb9 --- /dev/null +++ b/tests/core/clientcache/test.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +echo "📝 Building clientcache stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "📝 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "📝 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_CLIENT_CACHE: "yes"@USE_CLIENT_CACHE: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CLIENT_CACHE_EXTENSIONS: "jpg|jpeg|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2"@CLIENT_CACHE_EXTENSIONS: "jpg|jpeg|png|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CLIENT_CACHE_ETAG: "no"@CLIENT_CACHE_ETAG: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CLIENT_CACHE_CONTROL: "public, max-age=3600"@CLIENT_CACHE_CONTROL: "public, max-age=15552000"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "📝 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "📝 Down failed ❌" + exit 1 + fi + + echo "📝 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "deactivated" "activated" "cache_extensions" "cache_etag" "cache_control" +do + if [ "$test" = "deactivated" ] ; then + echo "📝 Running tests without clientcache ..." + elif [ "$test" = "activated" ] ; then + echo "📝 Running tests with clientcache ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_CLIENT_CACHE: "no"@USE_CLIENT_CACHE: "yes"@' {} \; + elif [ "$test" = "cache_extensions" ] ; then + echo "📝 Running tests when removing png from the cache extensions ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CLIENT_CACHE_EXTENSIONS: "jpg|jpeg|png|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2"@CLIENT_CACHE_EXTENSIONS: "jpg|jpeg|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2"@' {} \; + elif [ "$test" = "cache_etag" ] ; then + echo "📝 Running tests when deactivating the etag ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CLIENT_CACHE_EXTENSIONS: "jpg|jpeg|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2"@CLIENT_CACHE_EXTENSIONS: "jpg|jpeg|png|bmp|ico|svg|tif|css|js|otf|ttf|eot|woff|woff2"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CLIENT_CACHE_ETAG: "yes"@CLIENT_CACHE_ETAG: "no"@' {} \; + elif [ "$test" = "cache_control" ] ; then + echo "📝 Running tests whith clientcache control set to public, max-age=3600 ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CLIENT_CACHE_ETAG: "no"@CLIENT_CACHE_ETAG: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CLIENT_CACHE_CONTROL: "public, max-age=15552000"@CLIENT_CACHE_CONTROL: "public, max-age=3600"@' {} \; + fi + + echo "📝 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "📝 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "📝 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("clientcache-bw-1" "clientcache-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "📝 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "📝 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "📝 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "📝 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "📝 Tests are done ! ✅" diff --git a/tests/core/cors/Dockerfile b/tests/core/cors/Dockerfile new file mode 100644 index 000000000..c6b6dd4bc --- /dev/null +++ b/tests/core/cors/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11.3-alpine + +# Install firefox and geckodriver +RUN apk add --no-cache --virtual .build-deps curl grep zip && \ + apk add --no-cache firefox + +# Installing geckodriver for firefox... +RUN GECKODRIVER_VERSION=`curl -i https://github.com/mozilla/geckodriver/releases/latest | grep -Po 'v[0-9]+.[0-9]+.[0-9]+'` && \ + wget -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \ + tar -C /usr/local/bin -xzvf geckodriver.tar.gz && \ + chmod +x /usr/local/bin/geckodriver && \ + rm geckodriver.tar.gz + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/cors/docker-compose.test.yml b/tests/core/cors/docker-compose.test.yml new file mode 100644 index 000000000..5e024c352 --- /dev/null +++ b/tests/core/cors/docker-compose.test.yml @@ -0,0 +1,23 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_CORS: "no" + CORS_ALLOW_ORIGIN: "*" + CORS_EXPOSE_HEADERS: "Content-Length,Content-Range" + CORS_MAX_AGE: "86400" + CORS_ALLOW_CREDENTIALS: "no" + CORS_ALLOW_METHODS: "GET, POST, OPTIONS" + CORS_ALLOW_HEADERS: "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/cors/docker-compose.yml b/tests/core/cors/docker-compose.yml new file mode 100644 index 000000000..cd988787b --- /dev/null +++ b/tests/core/cors/docker-compose.yml @@ -0,0 +1,69 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + HTTPS_PORT: "443" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + GENERATE_SELF_SIGNED_SSL: "yes" + ALLOWED_METHODS: "GET|POST|HEAD|OPTIONS" + + # ? CORS settings + USE_CORS: "no" + CORS_ALLOW_ORIGIN: "*" + CORS_EXPOSE_HEADERS: "Content-Length,Content-Range" + CORS_MAX_AGE: "86400" + CORS_ALLOW_CREDENTIALS: "no" + CORS_ALLOW_METHODS: "GET, POST, OPTIONS" + CORS_ALLOW_HEADERS: "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/cors/index.html b/tests/core/cors/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/cors/main.py b/tests/core/cors/main.py new file mode 100644 index 000000000..c0ad5189b --- /dev/null +++ b/tests/core/cors/main.py @@ -0,0 +1,220 @@ +from contextlib import suppress +from os import getenv +from requests import RequestException, get, head, options +from selenium import webdriver +from selenium.webdriver.firefox.options import Options +from selenium.common.exceptions import JavascriptException +from traceback import format_exc +from time import sleep + + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "https://www.example.com", + headers={"Host": "www.example.com"}, + verify=False, + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + firefox_options = Options() + firefox_options.add_argument("--headless") + + use_cors = getenv("USE_CORS", "no") + cors_allow_origin = getenv("CORS_ALLOW_ORIGIN", "*") + cors_expose_headers = getenv("CORS_EXPOSE_HEADERS", "Content-Length,Content-Range") + cors_max_age = getenv("CORS_MAX_AGE", "86400") + cors_allow_credentials = getenv("CORS_ALLOW_CREDENTIALS", "no") == "yes" + cors_allow_methods = getenv("CORS_ALLOW_METHODS", "GET, POST, OPTIONS") + cors_allow_headers = getenv( + "CORS_ALLOW_HEADERS", + "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range", + ) + + if any( + [ + cors_allow_origin != "*", + cors_expose_headers != "Content-Length,Content-Range", + ] + ): + print( + "ℹ️ Sending a HEAD request to https://www.example.com ...", + flush=True, + ) + + response = head( + "https://www.example.com", headers={"Host": "www.example.com"}, verify=False + ) + response.raise_for_status() + + if any( + header in response.headers + for header in ( + "Access-Control-Max-Age", + "Access-Control-Allow-Credentials", + "Access-Control-Allow-Methods", + "Access-Control-Allow-Headers", + ) + ): + print( + f"❌ One of the preflight request headers is present in the response headers, it should not be ...\nheaders: {response.headers}", + ) + exit(1) + elif cors_allow_origin != response.headers.get("Access-Control-Allow-Origin"): + print( + f"❌ The Access-Control-Allow-Origin header is set to {response.headers.get('Access-Control-Allow-Origin', 'missing')}, it should be {cors_allow_origin} ...\nheaders: {response.headers}", + flush=True, + ) + exit(1) + elif cors_allow_origin != "*": + print( + f"✅ The Access-Control-Allow-Origin header is set to {cors_allow_origin} ...", + flush=True, + ) + elif cors_expose_headers != response.headers.get( + "Access-Control-Expose-Headers" + ): + print( + f"❌ The Access-Control-Expose-Headers header is set to {response.headers.get('Access-Control-Expose-Headers', 'missing')}, it should be {cors_expose_headers} ...\nheaders: {response.headers}", + flush=True, + ) + exit(1) + elif cors_expose_headers != "Content-Length,Content-Range": + print( + f"✅ The Access-Control-Expose-Headers header is set to {cors_expose_headers} ...", + flush=True, + ) + + exit(0) + elif any( + [ + cors_max_age != "86400", + cors_allow_credentials, + cors_allow_methods != "GET, POST, OPTIONS", + cors_allow_headers + != "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range", + ] + ): + print( + "ℹ️ Sending a preflight request to https://www.example.com ...", + flush=True, + ) + + response = options( + "https://www.example.com", headers={"Host": "www.example.com"}, verify=False + ) + response.raise_for_status() + + if ( + not cors_allow_credentials + and "Access-Control-Allow-Credentials" in response.headers + ): + print( + f"❌ The Access-Control-Allow-Credentials header is present in the response headers while the setting CORS_ALLOW_CREDENTIALS is set to {cors_allow_credentials}, it should not be ...\nheaders: {response.headers}", + ) + exit(1) + elif cors_max_age != response.headers.get("Access-Control-Max-Age"): + print( + f"❌ The Access-Control-Max-Age header is set to {response.headers.get('Access-Control-Max-Age', 'missing')}, it should be {cors_max_age} ...\nheaders: {response.headers}", + flush=True, + ) + exit(1) + elif cors_max_age != "86400": + print( + f"✅ The Access-Control-Max-Age header is set to {cors_max_age} ...", + flush=True, + ) + elif ( + cors_allow_credentials + and "Access-Control-Allow-Credentials" not in response.headers + ): + print( + f"❌ The Access-Control-Allow-Credentials header is not present in the response headers while the setting CORS_ALLOW_CREDENTIALS is set to {cors_allow_credentials}, it should be ...\nheaders: {response.headers}", + ) + exit(1) + elif cors_allow_methods != response.headers.get("Access-Control-Allow-Methods"): + print( + f"❌ The Access-Control-Allow-Methods header is set to {response.headers.get('Access-Control-Allow-Methods', 'missing')}, it should be {cors_allow_methods} ...\nheaders: {response.headers}", + ) + exit(1) + elif cors_allow_methods != "GET, POST, OPTIONS": + print( + f"✅ The Access-Control-Allow-Methods is set to {cors_allow_methods} ...", + flush=True, + ) + elif cors_allow_headers != response.headers.get("Access-Control-Allow-Headers"): + print( + f"❌ The Access-Control-Allow-Headers header is set to {response.headers.get('Access-Control-Allow-Headers', 'missing')}, it should be {cors_allow_headers} ...\nheaders: {response.headers}", + flush=True, + ) + exit(1) + elif ( + cors_allow_headers + != "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" + ): + print( + f"✅ The Access-Control-Allow-Headers header is set to {cors_allow_headers} ...", + flush=True, + ) + else: + print( + f"✅ The Access-Control-Allow-Credentials header is present and set to {cors_allow_credentials} ...", + flush=True, + ) + + exit(0) + + print("ℹ️ Starting Firefox ...", flush=True) + with webdriver.Firefox(options=firefox_options) as driver: + driver.delete_all_cookies() + driver.maximize_window() + + print( + "ℹ️ Sending a javascript request to https://www.example.com ...", + flush=True, + ) + error = False + + try: + driver.execute_script( + """var xhttp = new XMLHttpRequest(); +xhttp.open("GET", "https://www.example.com", false); +xhttp.setRequestHeader("Host", "www.example.com"); +xhttp.send();""" + ) + except JavascriptException as e: + if not f"{e}".startswith("Message: NetworkError"): + print(f"❌ {e}", flush=True) + error = True + + if use_cors == "no" and not error: + print("❌ CORS is enabled, it shouldn't be, exiting ...", flush=True) + exit(1) + elif use_cors == "yes" and error: + print("❌ CORS are not working as expected, exiting ...", flush=True) + exit(1) + + print("✅ CORS are working as expected ...", flush=True) +except SystemExit as e: + exit(e.code) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/cors/requirements.txt b/tests/core/cors/requirements.txt new file mode 100644 index 000000000..6f7b13f79 --- /dev/null +++ b/tests/core/cors/requirements.txt @@ -0,0 +1,2 @@ +requests==2.30.0 +selenium==4.9.1 diff --git a/tests/core/cors/test.sh b/tests/core/cors/test.sh new file mode 100755 index 000000000..0e9924e88 --- /dev/null +++ b/tests/core/cors/test.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +echo "🛰️ Building cors stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🛰️ Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🛰️ Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_CORS: "yes"@USE_CORS: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_ORIGIN: "http://www.example.com"@CORS_ALLOW_ORIGIN: "\*"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_EXPOSE_HEADERS: "X-Test"@CORS_EXPOSE_HEADERS: "Content-Length,Content-Range"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_MAX_AGE: "3600"@CORS_MAX_AGE: "86400"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_CREDENTIALS: "yes"@CORS_ALLOW_CREDENTIALS: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_METHODS: "GET, HEAD, POST, OPTIONS"@CORS_ALLOW_METHODS: "GET, POST, OPTIONS"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_HEADERS: "X-Test"@CORS_ALLOW_HEADERS: "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🛰️ Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🛰️ Down failed ❌" + exit 1 + fi + + echo "🛰️ Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "deactivated" "activated" "allow_origin" "expose_headers" "max_age" "allow_credentials" "allow_methods" "allow_headers" +do + if [ "$test" = "deactivated" ] ; then + echo "🛰️ Running tests without cors ..." + elif [ "$test" = "activated" ] ; then + echo "🛰️ Running tests with cors ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_CORS: "no"@USE_CORS: "yes"@' {} \; + elif [ "$test" = "allow_origin" ] ; then + echo "🛰️ Running tests with cors allow origin set to http://www.example.com ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_ORIGIN: "\*"@CORS_ALLOW_ORIGIN: "http://www.example.com"@' {} \; + elif [ "$test" = "expose_headers" ] ; then + echo "🛰️ Running tests with cors expose headers set to X-Test ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_ORIGIN: "http://www.example.com"@CORS_ALLOW_ORIGIN: "\*"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_EXPOSE_HEADERS: "Content-Length,Content-Range"@CORS_EXPOSE_HEADERS: "X-Test"@' {} \; + elif [ "$test" = "max_age" ] ; then + echo "🛰️ Running tests with cors max age set to 3600 ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_EXPOSE_HEADERS: "X-Test"@CORS_EXPOSE_HEADERS: "Content-Length,Content-Range"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_MAX_AGE: "86400"@CORS_MAX_AGE: "3600"@' {} \; + elif [ "$test" = "allow_credentials" ] ; then + echo "🛰️ Running tests with cors allow credentials is set to yes ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_MAX_AGE: "3600"@CORS_MAX_AGE: "86400"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_CREDENTIALS: "no"@CORS_ALLOW_CREDENTIALS: "yes"@' {} \; + elif [ "$test" = "allow_methods" ] ; then + echo "🛰️ Running tests with cors allow methods is set to GET, HEAD, POST, OPTIONS ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_CREDENTIALS: "yes"@CORS_ALLOW_CREDENTIALS: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_METHODS: "GET, POST, OPTIONS"@CORS_ALLOW_METHODS: "GET, HEAD, POST, OPTIONS"@' {} \; + elif [ "$test" = "allow_headers" ] ; then + echo "🛰️ Running tests with cors allow headers is set to X-Test ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_METHODS: "GET, HEAD, POST, OPTIONS"@CORS_ALLOW_METHODS: "GET, POST, OPTIONS"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CORS_ALLOW_HEADERS: "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range"@CORS_ALLOW_HEADERS: "X-Test"@' {} \; + fi + + echo "🛰️ Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🛰️ Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🛰️ Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("cors-bw-1" "cors-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🛰️ Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🛰️ Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🛰️ Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🛰️ Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🛰️ Tests are done ! ✅" diff --git a/tests/core/country/Dockerfile b/tests/core/country/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/country/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/country/docker-compose.test.yml b/tests/core/country/docker-compose.test.yml new file mode 100644 index 000000000..a748f4b77 --- /dev/null +++ b/tests/core/country/docker-compose.test.yml @@ -0,0 +1,34 @@ +version: "3.5" + +services: + tests-fr: + build: . + environment: + PYTHONUNBUFFERED: "1" + COUNTRY: "FR" + BLACKLIST_COUNTRY: "" + WHITELIST_COUNTRY: "" + extra_hosts: + - "www.example.com:2.0.0.2" + networks: + bw-fr-network: + ipv4_address: 2.0.0.3 + + tests-us: + build: . + environment: + PYTHONUNBUFFERED: "1" + COUNTRY: "US" + BLACKLIST_COUNTRY: "" + WHITELIST_COUNTRY: "" + extra_hosts: + - "www.example.com:8.0.0.2" + networks: + bw-us-network: + ipv4_address: 8.0.0.3 + +networks: + bw-fr-network: + external: true + bw-us-network: + external: true diff --git a/tests/core/country/docker-compose.yml b/tests/core/country/docker-compose.yml new file mode 100644 index 000000000..ef3e5be1f --- /dev/null +++ b/tests/core/country/docker-compose.yml @@ -0,0 +1,70 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? COUNTRY settings + BLACKLIST_COUNTRY: "" + WHITELIST_COUNTRY: "" + networks: + bw-universe: + bw-us-network: + ipv4_address: 8.0.0.2 + bw-fr-network: + ipv4_address: 2.0.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-us-network: + name: bw-us-network + ipam: + driver: default + config: + - subnet: 8.0.0.0/8 + bw-fr-network: + name: bw-fr-network + ipam: + driver: default + config: + - subnet: 2.0.0.0/8 + bw-docker: + name: bw-docker diff --git a/tests/core/country/index.html b/tests/core/country/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/country/main.py b/tests/core/country/main.py new file mode 100644 index 000000000..c51afecdf --- /dev/null +++ b/tests/core/country/main.py @@ -0,0 +1,75 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 or status_code == 403 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + country = getenv("COUNTRY") + blacklist_country = getenv("BLACKLIST_COUNTRY", "") + whitelist_country = getenv("WHITELIST_COUNTRY", "") + + print( + "ℹ️ Sending a request to http://www.example.com ...", + flush=True, + ) + + status_code = get( + f"http://www.example.com", + headers={"Host": "www.example.com"}, + ).status_code + + if status_code == 403: + if not blacklist_country and not whitelist_country: + print( + "❌ Got rejected even though there are no country blacklisted or whitelisted, exiting ...", + flush=True, + ) + exit(1) + elif country == whitelist_country: + print( + f"❌ Got rejected even if the current country ({country}) is whitelisted, exiting ...", + flush=True, + ) + exit(1) + + print("✅ Got rejected, as expected ...") + else: + if country == blacklist_country: + print( + f"❌ Didn't get rejected even if the current country ({country}) is blacklisted, exiting ...", + flush=True, + ) + exit(1) + + print("✅ Didn't get rejected, as expected ...") +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/country/requirements.txt b/tests/core/country/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/country/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/country/test.sh b/tests/core/country/test.sh new file mode 100755 index 000000000..2a5bf9d15 --- /dev/null +++ b/tests/core/country/test.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +echo "🌍 Building country stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🌍 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🌍 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_COUNTRY: "US"@BLACKLIST_COUNTRY: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_COUNTRY: "FR"@WHITELIST_COUNTRY: ""@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🌍 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🌍 Down failed ❌" + exit 1 + fi + + echo "🌍 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "deactivated" "blacklist" "whitelist" +do + if [ "$test" = "deactivated" ] ; then + echo "🌍 Running tests without the country plugin ..." + elif [ "$test" = "blacklist" ] ; then + echo "🌍 Running tests when blacklisting United States ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_COUNTRY: ""@BLACKLIST_COUNTRY: "US"@' {} \; + elif [ "$test" = "whitelist" ] ; then + echo "🌍 Running tests when whitelisting France ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_COUNTRY: "US"@BLACKLIST_COUNTRY: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_COUNTRY: ""@WHITELIST_COUNTRY: "FR"@' {} \; + fi + + echo "🌍 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🌍 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🌍 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("country-bw-1" "country-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🌍 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🌍 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + echo "🌍 Starting the FR container" + docker compose -f docker-compose.test.yml up tests-fr --abort-on-container-exit --exit-code-from tests-fr 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🌍 Test \"$test\" failed for the FR container ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🌍 Test \"$test\" succeeded for the FR container ✅" + fi + + echo "🌍 Starting the US container" + docker compose -f docker-compose.test.yml up tests-us --abort-on-container-exit --exit-code-from tests-us 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🌍 Test \"$test\" failed for the US container ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🌍 Test \"$test\" succeeded for the US container ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🌍 Tests are done ! ✅" diff --git a/tests/core/customcert/Dockerfile b/tests/core/customcert/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/customcert/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/customcert/docker-compose.init.yml b/tests/core/customcert/docker-compose.init.yml new file mode 100644 index 000000000..72838efad --- /dev/null +++ b/tests/core/customcert/docker-compose.init.yml @@ -0,0 +1,9 @@ +version: "3.5" + +services: + init: + build: init + environment: + PYTHONUNBUFFERED: "1" + volumes: + - ./init/certs:/certs diff --git a/tests/core/customcert/docker-compose.test.yml b/tests/core/customcert/docker-compose.test.yml new file mode 100644 index 000000000..b487d2288 --- /dev/null +++ b/tests/core/customcert/docker-compose.test.yml @@ -0,0 +1,17 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_CUSTOM_SSL: "no" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/customcert/docker-compose.yml b/tests/core/customcert/docker-compose.yml new file mode 100644 index 000000000..7697758ca --- /dev/null +++ b/tests/core/customcert/docker-compose.yml @@ -0,0 +1,69 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + ports: + - 80:80 + - 443:443 + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + HTTPS_PORT: "443" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? CUSTOM_CERT settings + USE_CUSTOM_SSL: "no" + CUSTOM_SSL_CERT: "/certs/certificate.pem" + CUSTOM_SSL_KEY: "/certs/privatekey.key" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + volumes: + - ./init/certs:/certs + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: + name: bw-docker diff --git a/tests/core/customcert/index.html b/tests/core/customcert/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/customcert/init/Dockerfile b/tests/core/customcert/init/Dockerfile new file mode 100644 index 000000000..da14e6e82 --- /dev/null +++ b/tests/core/customcert/init/Dockerfile @@ -0,0 +1,11 @@ +FROM alpine + +RUN apk add --no-cache bash openssl + +WORKDIR /opt/init + +COPY entrypoint.sh . + +RUN chmod +x entrypoint.sh + +ENTRYPOINT [ "./entrypoint.sh" ] \ No newline at end of file diff --git a/tests/core/customcert/init/entrypoint.sh b/tests/core/customcert/init/entrypoint.sh new file mode 100644 index 000000000..87a2aca12 --- /dev/null +++ b/tests/core/customcert/init/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "ℹ️ Generating certificate for www.example.com ..." +openssl req -nodes -x509 -newkey rsa:4096 -keyout /certs/privatekey.key -out /certs/certificate.pem -days 365 -subj /CN=www.example.com/ + +chown -R root:101 /certs +chmod -R 777 /certs diff --git a/tests/core/customcert/main.py b/tests/core/customcert/main.py new file mode 100644 index 000000000..4955a5eb8 --- /dev/null +++ b/tests/core/customcert/main.py @@ -0,0 +1,49 @@ +from os import getenv +from requests import get +from requests.exceptions import RequestException +from traceback import format_exc + +try: + use_custom_ssl = getenv("USE_CUSTOM_SSL", "no") == "yes" + + print( + "ℹ️ Sending a request to http://www.example.com ...", + flush=True, + ) + + try: + get("http://www.example.com", headers={"Host": "www.example.com"}) + except RequestException: + if not use_custom_ssl: + print( + "❌ The request failed even though the Custom Cert isn't activated, exiting ...", + flush=True, + ) + exit(1) + + if not use_custom_ssl: + print("✅ The Custom Cert isn't activated, as expected ...", flush=True) + exit(0) + + print( + "ℹ️ Sending a request to https://www.example.com ...", + flush=True, + ) + + try: + get( + "https://www.example.com", headers={"Host": "www.example.com"}, verify=False + ) + except RequestException: + print( + "❌ The request failed even though the Custom Cert is activated, exiting ...", + flush=True, + ) + exit(1) + + print("✅ The Custom Cert is activated, as expected ...", flush=True) +except SystemExit as e: + exit(e.code) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/customcert/requirements.txt b/tests/core/customcert/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/customcert/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/customcert/test.sh b/tests/core/customcert/test.sh new file mode 100755 index 000000000..5fdf6fc18 --- /dev/null +++ b/tests/core/customcert/test.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +echo "🔏 Building customcert stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🔏 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🔏 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + rm -rf init/certs + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_CUSTOM_SSL: "yes"@USE_CUSTOM_SSL: "no"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🔏 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🔏 Down failed ❌" + exit 1 + fi + + echo "🔏 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +echo "🔏 Initializing workspace ..." +rm -rf init/certs +mkdir -p init/certs +docker compose -f docker-compose.init.yml up --build +if [ $? -ne 0 ] ; then + echo "🔏 Build failed ❌" + exit 1 +elif ! [[ -f "init/certs/certificate.pem" ]]; then + echo "🔏 certificate.pem not found ❌" + exit 1 +elif ! [[ -f "init/certs/privatekey.key" ]]; then + echo "🔏 privatekey.key not found ❌" + exit 1 +fi + +for test in "deactivated" "activated" +do + if [ "$test" = "deactivated" ] ; then + echo "🔏 Running tests without the custom cert ..." + elif [ "$test" = "activated" ] ; then + echo "🔏 Running tests with the custom cert activated ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_CUSTOM_SSL: "no"@USE_CUSTOM_SSL: "yes"@' {} \; + fi + + echo "🔏 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🔏 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🔏 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("customcert-bw-1" "customcert-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🔏 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🔏 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🔏 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🔏 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🔏 Tests are done ! ✅" diff --git a/tests/core/db/Dockerfile b/tests/core/db/Dockerfile new file mode 100644 index 000000000..32316b8c0 --- /dev/null +++ b/tests/core/db/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +RUN addgroup -g 101 nginx && \ + adduser -h /opt/tests -g nginx -s /bin/sh -G nginx -D -H -u 101 nginx + +COPY --chown=nginx:nginx main.py . +ADD ./init/plugins external + +RUN chown -R nginx:nginx external && \ + chmod -R 777 external + +USER nginx:nginx + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/db/docker-compose.init.yml b/tests/core/db/docker-compose.init.yml new file mode 100644 index 000000000..b02a3af05 --- /dev/null +++ b/tests/core/db/docker-compose.init.yml @@ -0,0 +1,9 @@ +version: "3.5" + +services: + init: + build: init + environment: + PYTHONUNBUFFERED: "1" + volumes: + - ./init/plugins:/plugins diff --git a/tests/core/db/docker-compose.test.yml b/tests/core/db/docker-compose.test.yml new file mode 100644 index 000000000..db99851bd --- /dev/null +++ b/tests/core/db/docker-compose.test.yml @@ -0,0 +1,42 @@ +version: "3.5" + +services: + tests: + build: . + volumes: + - bw-data:/data/lib + - bw-db:/opt/tests/db + - bw-core-plugins:/opt/tests/core + environment: + PYTHONUNBUFFERED: "1" + DATABASE_URI: "sqlite:////var/lib/bunkerweb/db.sqlite3" + GLOBAL_API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + GLOBAL_MULTISITE: "no" + GLOBAL_HTTP_PORT: "80" + GLOBAL_USE_BUNKERNET: "no" + GLOBAL_USE_BLACKLIST: "no" + GLOBAL_USE_REVERSE_PROXY: "yes" + GLOBAL_REVERSE_PROXY_HOST: "http://app1:8080" + GLOBAL_REVERSE_PROXY_URL: "/" + GLOBAL_LOG_LEVEL: "info" + CUSTOM_CONF_MODSEC_test_custom_conf: 'SecRule REQUEST_FILENAME "@rx ^/db" "id:1,ctl:ruleRemoveByTag=attack-generic,ctl:ruleRemoveByTag=attack-protocol,nolog"' + extra_hosts: + - "bwadm.example.com:192.168.0.2" + networks: + bw-docker: + bw-services: + ipv4_address: 192.168.0.3 + +volumes: + bw-data: + external: true + bw-db: + external: true + bw-core-plugins: + external: true + +networks: + bw-services: + external: true + bw-docker: + external: true diff --git a/tests/core/db/docker-compose.yml b/tests/core/db/docker-compose.yml new file mode 100644 index 000000000..d57a693ca --- /dev/null +++ b/tests/core/db/docker-compose.yml @@ -0,0 +1,112 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + environment: + SERVER_NAME: "bwadm.example.com" + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + MULTISITE: "no" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + USE_REVERSE_PROXY: "yes" + REVERSE_PROXY_HOST: "http://app1:8080" + REVERSE_PROXY_URL: "/" + LOG_LEVEL: "info" + CUSTOM_CONF_MODSEC_test_custom_conf: 'SecRule REQUEST_FILENAME "@rx ^/db" "id:1,ctl:ruleRemoveByTag=attack-generic,ctl:ruleRemoveByTag=attack-protocol,nolog"' + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + volumes: + - bw-data:/data/lib + - bw-db:/usr/share/bunkerweb/db + - bw-core-plugins:/usr/share/bunkerweb/core + - ./init/plugins:/data/plugins + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + # ? DATABASE settings + DATABASE_URI: "sqlite:////var/lib/bunkerweb/db.sqlite3" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + app1: + image: nginxdemos/nginx-hello + networks: + bw-services: + ipv4_address: 192.168.0.4 + + bw-maria-db: + image: mariadb:10.10 + environment: + - MYSQL_RANDOM_ROOT_PASSWORD=yes + - MYSQL_DATABASE=db + - MYSQL_USER=bunkerweb + - MYSQL_PASSWORD=secret + networks: + - bw-docker + + bw-mysql-db: + image: mysql:8.0 + environment: + - MYSQL_RANDOM_ROOT_PASSWORD=yes + - MYSQL_DATABASE=db + - MYSQL_USER=bunkerweb + - MYSQL_PASSWORD=secret + networks: + - bw-docker + + bw-postgres-db: + image: postgres:15.1 + environment: + - POSTGRES_USER=bunkerweb + - POSTGRES_PASSWORD=secret + - POSTGRES_DB=db + networks: + - bw-docker + +volumes: + bw-data: + name: bw-data + bw-db: + name: bw-db + bw-core-plugins: + name: bw-core-plugins + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: + name: bw-docker diff --git a/tests/core/db/init/Dockerfile b/tests/core/db/init/Dockerfile new file mode 100644 index 000000000..d0d6b9276 --- /dev/null +++ b/tests/core/db/init/Dockerfile @@ -0,0 +1,11 @@ +FROM alpine + +RUN apk add --no-cache bash git + +WORKDIR /opt/init + +COPY entrypoint.sh . + +RUN chmod +x entrypoint.sh + +ENTRYPOINT [ "./entrypoint.sh" ] \ No newline at end of file diff --git a/tests/core/db/init/entrypoint.sh b/tests/core/db/init/entrypoint.sh new file mode 100644 index 000000000..d4b788896 --- /dev/null +++ b/tests/core/db/init/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo "ℹ️ Cloning BunkerWeb Plugins ..." + +git clone https://github.com/bunkerity/bunkerweb-plugins.git + +echo "ℹ️ Checking out to dev branch ..." + +cd bunkerweb-plugins +git checkout dev # TODO: remove this when the next release of bw-plugins is out + +echo "ℹ️ Extracting ClamAV plugin ..." + +cp -r clamav /plugins/ + +chown -R root:101 /plugins +chmod -R 777 /plugins \ No newline at end of file diff --git a/tests/core/db/main.py b/tests/core/db/main.py new file mode 100644 index 000000000..e9ae7bd5d --- /dev/null +++ b/tests/core/db/main.py @@ -0,0 +1,984 @@ +from contextlib import contextmanager +from glob import iglob +from hashlib import sha512 +from json import dumps, load +from os import environ, getenv +from os.path import dirname, join +from pathlib import Path +from re import compile as re_compile +from sqlalchemy import create_engine, text +from sqlalchemy.exc import ( + ArgumentError, + DatabaseError, + OperationalError, + SQLAlchemyError, +) +from sqlalchemy.orm import scoped_session, sessionmaker +from traceback import format_exc +from time import sleep + +from db.model import ( + Custom_configs, + Global_values, + Jobs, + Metadata, + Plugins, + Plugin_pages, + Services, + Services_settings, + Settings, +) + +try: + database_uri = getenv("DATABASE_URI", "sqlite:////var/lib/bunkerweb/db.sqlite3") + + if database_uri == "sqlite:////var/lib/bunkerweb/db.sqlite3": + database_uri = "sqlite:////data/lib/db.sqlite3" + + error = False + + print(f"ℹ️ Connecting to database: {database_uri}", flush=True) + + try: + sql_engine = create_engine( + database_uri, + future=True, + ) + except ArgumentError: + print(f"❌ Invalid database URI: {database_uri}", flush=True) + error = True + except SQLAlchemyError: + print(f"❌ Error when trying to create the engine: {format_exc()}", flush=True) + error = True + finally: + if error: + exit(1) + + try: + assert sql_engine is not None + except AssertionError: + print("❌ The database engine is not initialized", flush=True) + exit(1) + + not_connected = True + retries = 15 + + while not_connected: + try: + with sql_engine.connect() as conn: + conn.execute(text("CREATE TABLE IF NOT EXISTS test (id INT)")) + conn.execute(text("DROP TABLE test")) + not_connected = False + except (OperationalError, DatabaseError) as e: + if retries <= 0: + print(f"❌ Can't connect to database : {format_exc()}", flush=True) + exit(1) + + if "attempt to write a readonly database" in str(e): + print( + "⚠️ The database is read-only, waiting for it to become writable. Retrying in 5 seconds ...", + flush=True, + ) + sql_engine.dispose(close=True) + sql_engine = create_engine( + database_uri, + future=True, + ) + if "Unknown table" in str(e): + not_connected = False + continue + else: + print( + "⚠️ Can't connect to database, retrying in 5 seconds ...", + flush=True, + ) + retries -= 1 + sleep(5) + except BaseException: + print( + f"❌ Error when trying to connect to the database: {format_exc()}", + flush=True, + ) + exit(1) + + print("ℹ️ Database connection established, launching tests ...", flush=True) + + session = sessionmaker() + sql_session = scoped_session(session) + sql_session.remove() + sql_session.configure(bind=sql_engine, autoflush=False, expire_on_commit=False) + + @contextmanager + def db_session(): + try: + assert sql_session is not None + except AssertionError: + print("❌ The database session is not initialized", flush=True) + exit(1) + + session = sql_session() + session.expire_on_commit = False + + try: + yield session + except BaseException: + session.rollback() + raise + finally: + session.close() + + print("ℹ️ Checking if database is initialized ...", flush=True) + + with db_session() as session: + metadata = ( + session.query(Metadata) + .with_entities(Metadata.is_initialized) + .filter_by(id=1) + .first() + ) + + if metadata is None or not metadata.is_initialized: + print( + "❌ The database is not initialized, it should be, exiting ...", + flush=True, + ) + exit(1) + + print("✅ Database is initialized", flush=True) + print(" ", flush=True) + print("ℹ️ Checking if service bwadm.example.com is in the database ...", flush=True) + + with db_session() as session: + services = session.query(Services).all() + + if not services: + print( + "❌ The bw_services database table is empty, it shouldn't be, exiting ...", + flush=True, + ) + exit(1) + + if services[0].id != "bwadm.example.com": + print( + "❌ The service bwadm.example.com is not in the database, it should be, exiting ...", + flush=True, + ) + exit(1) + + print("✅ Service bwadm.example.com is in the database", flush=True) + print(" ", flush=True) + print( + "ℹ️ Checking if global values are in the database and are correct ...", + flush=True, + ) + + global_settings = {} + service_settings = {} + multisite = getenv("GLOBAL_MULTISITE", "no") == "yes" + for env in environ: + if env.startswith("GLOBAL_"): + if env == "GLOBAL_MULTISITE" and environ[env] == "no": + continue + global_settings[env[7:]] = {"value": environ[env], "checked": False} + elif env.startswith("SERVICE_"): + service_settings[env[8:]] = {"value": environ[env], "checked": False} + + with db_session() as session: + global_values = session.query(Global_values).all() + + for global_value in global_values: + if global_value.setting_id in global_settings: + if ( + global_value.value + != global_settings[global_value.setting_id]["value"] + ): + print( + f"❌ The global value {global_value.setting_id} is in the database but is not correct, exiting ...\n{global_value.value} (database) != {global_settings[global_value.setting_id]['value']} (env)", + flush=True, + ) + exit(1) + elif global_value.suffix != 0: + print( + f"❌ The global value {global_value.setting_id} is in the database but has the wrong suffix, exiting ...\n{global_value.suffix} (database) != 0 (env)", + flush=True, + ) + exit(1) + elif global_value.method != "scheduler": + print( + f"❌ The global value {global_value.setting_id} is in the database but has the wrong method, exiting ...\n{global_value.method} (database) != scheduler (env)", + flush=True, + ) + exit(1) + + global_settings[global_value.setting_id]["checked"] = True + else: + print( + f"❌ The global value {global_value.setting_id} is in the database but should not be, exiting ...", + flush=True, + ) + exit(1) + + if not all( + [global_settings[global_value]["checked"] for global_value in global_settings] + ): + print( + f"❌ Not all global values are in the database, exiting ...\nmissing values: {', '.join([global_value for global_value in global_settings if not global_settings[global_value]['checked']])}", + flush=True, + ) + exit(1) + + print("✅ Global values are in the database and are correct", flush=True) + print(" ", flush=True) + print( + "ℹ️ Checking if service values are in the database and are correct ...", + flush=True, + ) + + with db_session() as session: + services_settings = session.query(Services_settings).all() + + if not multisite and service_settings: + print( + '❌ The bw_services_settings database table is not empty, it should be when multisite is set to "no", exiting ...', + flush=True, + ) + exit(1) + else: + for service_setting in services_settings: + if service_setting.setting_id in service_settings: + if ( + service_setting.value + != service_settings[service_setting.setting_id]["value"] + ): + print( + f"❌ The service value {service_setting.setting_id} is in the database but is not correct, exiting ...\n{service_setting.value} (database) != {service_settings[service_setting.setting_id]['value']} (env)", + flush=True, + ) + exit(1) + elif service_setting.suffix != 0: + print( + f"❌ The service value {service_setting.setting_id} is in the database but has the wrong suffix, exiting ...\n{service_setting.suffix} (database) != 0 (env)", + flush=True, + ) + exit(1) + elif service_setting.method != "scheduler": + print( + f"❌ The service value {service_setting.setting_id} is in the database but has the wrong method, exiting ...\n{service_setting.method} (database) != scheduler (env)", + flush=True, + ) + exit(1) + + service_settings[service_setting.setting_id]["checked"] = True + else: + print( + f"❌ The service value {service_setting.setting_id} is in the database but should not be, exiting ...", + flush=True, + ) + exit(1) + + if not all( + [ + service_settings[service_setting]["checked"] + for service_setting in service_settings + ] + ): + print( + f"❌ Not all service values are in the database, exiting ...\nmissing values: {', '.join([service_setting for service_setting in service_settings if not service_settings[service_setting]['checked']])}", + flush=True, + ) + exit(1) + + print("✅ Service values are correct", flush=True) + print(" ", flush=True) + print("ℹ️ Checking if the plugins are correct ...", flush=True) + + core_plugins = { + "general": { + "order": 999, + "name": "General", + "description": "The general settings for the server", + "version": "0.1", + "stream": "partial", + "external": False, + "checked": False, + "page_checked": True, + "settings": { + "IS_LOADING": { + "context": "global", + "default": "no", + "help": "Internal use : set to yes when BW is loading.", + "id": "internal-use", + "label": "internal use", + "regex": "^(yes|no)$", + "type": "check", + }, + "NGINX_PREFIX": { + "context": "global", + "default": "/etc/nginx/", + "help": "Where nginx will search for configurations.", + "id": "nginx-prefix", + "label": "nginx prefix", + "regex": "^(/[\\w. -]+)*/$", + "type": "text", + }, + "HTTP_PORT": { + "context": "global", + "default": "8080", + "help": "HTTP port number which bunkerweb binds to.", + "id": "http-port", + "label": "HTTP port", + "regex": "^\\d+$", + "type": "text", + }, + "HTTPS_PORT": { + "context": "global", + "default": "8443", + "help": "HTTPS port number which bunkerweb binds to.", + "id": "https-port", + "label": "HTTPS port", + "regex": "^\\d+$", + "type": "text", + }, + "MULTISITE": { + "context": "global", + "default": "no", + "help": "Multi site activation.", + "id": "multisite", + "label": "Multisite", + "regex": "^(yes|no)$", + "type": "check", + }, + "SERVER_NAME": { + "context": "multisite", + "default": "www.example.com", + "help": "List of the virtual hosts served by bunkerweb.", + "id": "server-name", + "label": "Server name", + "regex": "^(?! )( ?((?=.{1,255}$)[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?(?:\\.[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?)*\\.?)(?!.* \\2))*$", + "type": "text", + }, + "WORKER_PROCESSES": { + "context": "global", + "default": "auto", + "help": "Number of worker processes.", + "id": "worker-processes", + "label": "Worker processes", + "regex": "^(auto|\\d+)$", + "type": "text", + }, + "WORKER_RLIMIT_NOFILE": { + "context": "global", + "default": "2048", + "help": "Maximum number of open files for worker processes.", + "id": "worker-rlimit-nofile", + "label": "Open files per worker", + "regex": "^\\d+$", + "type": "text", + }, + "WORKER_CONNECTIONS": { + "context": "global", + "default": "1024", + "help": "Maximum number of connections per worker.", + "id": "worker-connections", + "label": "Connections per worker", + "regex": "^\\d+$", + "type": "text", + }, + "LOG_FORMAT": { + "context": "global", + "default": '$host $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"', + "help": "The format to use for access logs.", + "id": "log-format", + "label": "Log format", + "regex": "^.*$", + "type": "text", + }, + "LOG_LEVEL": { + "context": "global", + "default": "notice", + "help": "The level to use for error logs.", + "id": "log-level", + "label": "Log level", + "regex": "^(debug|info|notice|warn|error|crit|alert|emerg)$", + "type": "select", + "select": [ + "debug", + "info", + "notice", + "warn", + "error", + "crit", + "alert", + "emerg", + ], + }, + "DNS_RESOLVERS": { + "context": "global", + "default": "127.0.0.11", + "help": "DNS addresses of resolvers to use.", + "id": "dns-resolvers", + "label": "DNS resolvers", + "regex": "^(?! )( *(((\\b25[0-5]|\\b2[0-4]\\d|\\b[01]?\\d\\d?)(\\.(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)){3})(\\/([1-2][0-9]?|3[0-2]?|[04-9]))?|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]Z{0,4}){0,4}%[0-9a-zA-Z]+|::(ffff(:0{1,4})?:)?((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d)|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d))(\\/(12[0-8]|1[01][0-9]|[0-9][0-9]?))?)(?!.*\\D\\2([^\\d\\/]|$)) *)*$", + "type": "text", + }, + "DATASTORE_MEMORY_SIZE": { + "context": "global", + "default": "64m", + "help": "Size of the internal datastore.", + "id": "datastore-memory-size", + "label": "Datastore memory size", + "regex": "^\\d+[kKmMgG]?$", + "type": "text", + }, + "CACHESTORE_MEMORY_SIZE": { + "context": "global", + "default": "64m", + "help": "Size of the internal cachestore.", + "id": "cachestore-memory-size", + "label": "Cachestore memory size", + "regex": "^\\d+[kKmMgG]?$", + "type": "text", + }, + "CACHESTORE_IPC_MEMORY_SIZE": { + "context": "global", + "default": "16m", + "help": "Size of the internal cachestore (ipc).", + "id": "cachestore-ipc-memory-size", + "label": "Cachestore ipc memory size", + "regex": "^\\d+[kKmMgG]?$", + "type": "text", + }, + "CACHESTORE_MISS_MEMORY_SIZE": { + "context": "global", + "default": "16m", + "help": "Size of the internal cachestore (miss).", + "id": "cachestore-miss-memory-size", + "label": "Cachestore miss memory size", + "regex": "^\\d+[kKmMgG]?$", + "type": "text", + }, + "CACHESTORE_LOCKS_MEMORY_SIZE": { + "context": "global", + "default": "16m", + "help": "Size of the internal cachestore (locks).", + "id": "cachestore-locks-memory-size", + "label": "Cachestore locks memory size", + "regex": "^\\d+[kKmMgG]?$", + "type": "text", + }, + "USE_API": { + "context": "global", + "default": "yes", + "help": "Activate the API to control BunkerWeb.", + "id": "use-api", + "label": "Activate API", + "regex": "^(yes|no)$", + "type": "check", + }, + "API_HTTP_PORT": { + "context": "global", + "default": "5000", + "help": "Listen port number for the API.", + "id": "api-http-listen", + "label": "API port number", + "regex": "^\\d+$", + "type": "text", + }, + "API_LISTEN_IP": { + "context": "global", + "default": "0.0.0.0", + "help": "Listen IP address for the API.", + "id": "api-ip-listen", + "label": "API listen IP", + "regex": "^.*$", + "type": "text", + }, + "API_SERVER_NAME": { + "context": "global", + "default": "bwapi", + "help": "Server name (virtual host) for the API.", + "id": "api-server-name", + "label": "API server name", + "regex": "^(?=.{1,255}$)[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?(?:\\.[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?)*\\.?$", + "type": "text", + }, + "API_WHITELIST_IP": { + "context": "global", + "default": "127.0.0.0/8", + "help": "List of IP/network allowed to contact the API.", + "id": "api-whitelist-ip", + "label": "API whitelist IP", + "regex": "^(?! )( *(((\\b25[0-5]|\\b2[0-4]\\d|\\b[01]?\\d\\d?)(\\.(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)){3})(\\/([1-2][0-9]?|3[0-2]?|[04-9]))?|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]Z{0,4}){0,4}%[0-9a-zA-Z]+|::(ffff(:0{1,4})?:)?((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d)|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d))(\\/(12[0-8]|1[01][0-9]|[0-9][0-9]?))?)(?!.*\\D\\2([^\\d\\/]|$)) *)*$", + "type": "text", + }, + "AUTOCONF_MODE": { + "context": "global", + "default": "no", + "help": "Enable Autoconf Docker integration.", + "id": "autoconf-mode", + "label": "Autoconf mode", + "regex": "^(yes|no)$", + "type": "check", + }, + "SWARM_MODE": { + "context": "global", + "default": "no", + "help": "Enable Docker Swarm integration.", + "id": "swarm-mode", + "label": "Swarm mode", + "regex": "^(yes|no)$", + "type": "check", + }, + "KUBERNETES_MODE": { + "context": "global", + "default": "no", + "help": "Enable Kubernetes integration.", + "id": "kubernetes-mode", + "label": "Kubernetes mode", + "regex": "^(yes|no)$", + "type": "check", + }, + "SERVER_TYPE": { + "context": "multisite", + "default": "http", + "help": "Server type : http or stream.", + "id": "server-type", + "label": "Server type", + "regex": "^(http|stream)$", + "type": "select", + "select": ["http", "stream"], + }, + "LISTEN_STREAM": { + "context": "multisite", + "default": "yes", + "help": "Enable listening for non-ssl (passthrough).", + "id": "listen-stream", + "label": "Listen stream", + "regex": "^(yes|no)$", + "type": "check", + }, + "LISTEN_STREAM_PORT": { + "context": "multisite", + "default": "1337", + "help": "Listening port for non-ssl (passthrough).", + "id": "listen-stream-port", + "label": "Listen stream port", + "regex": "^[0-9]+$", + "type": "text", + }, + "LISTEN_STREAM_PORT_SSL": { + "context": "multisite", + "default": "4242", + "help": "Listening port for ssl (passthrough).", + "id": "listen-stream-port-ssl", + "label": "Listen stream port ssl", + "regex": "^[0-9]+$", + "type": "text", + }, + "USE_UDP": { + "context": "multisite", + "default": "no", + "help": "UDP listen instead of TCP (stream).", + "id": "use-udp", + "label": "Listen UDP", + "regex": "^(yes|no)$", + "type": "check", + }, + }, + } + } + for filename in iglob(join("core", "*", "plugin.json")): + with open(filename, "r") as f: + data = load(f) + data["checked"] = False + for x, job in enumerate(data.get("jobs", [])): + data["jobs"][x]["checked"] = False + data["page_checked"] = not Path(f"{dirname(filename)}/ui").exists() or False + core_plugins[data.pop("id")] = data + + external_plugins = {} + for filename in iglob(join("external", "*", "plugin.json")): + with open(filename, "r") as f: + data = load(f) + data["checked"] = False + for x, job in enumerate(data.get("jobs", [])): + data["jobs"][x]["checked"] = False + data["page_checked"] = not Path(f"{dirname(filename)}/ui").exists() or False + external_plugins[data.pop("id")] = data + + with db_session() as session: + plugins = ( + session.query(Plugins) + .with_entities( + Plugins.id, + Plugins.order, + Plugins.name, + Plugins.description, + Plugins.version, + Plugins.stream, + Plugins.external, + Plugins.method, + ) + .all() + ) + + for plugin in plugins: + if not plugin.external and plugin.id in core_plugins: + current_plugin = core_plugins + elif plugin.external and plugin.id in external_plugins: + current_plugin = external_plugins + else: + print( + f"❌ The {'external' if plugin.external else 'core'} plugin {plugin.name} (id: {plugin.id}) is in the database but should not be, exiting ...", + flush=True, + ) + exit(1) + + if ( + plugin.order != current_plugin[plugin.id]["order"] + or plugin.name != current_plugin[plugin.id]["name"] + or plugin.description != current_plugin[plugin.id]["description"] + or plugin.version != current_plugin[plugin.id]["version"] + or plugin.stream != current_plugin[plugin.id]["stream"] + ): + print( + f"❌ The {'external' if plugin.external else 'core'} plugin {plugin.name} (id: {plugin.id}) is in the database but is not correct, exiting ...\n{dumps({'order': plugin.order, 'name': plugin.name, 'description': plugin.description, 'version': plugin.version, 'stream': plugin.stream})} (database) != {dumps({'order': current_plugin[plugin.id]['order'], 'name': current_plugin[plugin.id]['name'], 'description': current_plugin[plugin.id]['description'], 'version': current_plugin[plugin.id]['version'], 'stream': current_plugin[plugin.id]['stream']})} (file)", + flush=True, + ) + exit(1) + else: + settings = session.query(Settings).filter_by(plugin_id=plugin.id).all() + + for setting in settings: + if ( + setting.name + != current_plugin[plugin.id]["settings"][setting.id]["id"] + or setting.context + != current_plugin[plugin.id]["settings"][setting.id]["context"] + or setting.default + != current_plugin[plugin.id]["settings"][setting.id]["default"] + or setting.help + != current_plugin[plugin.id]["settings"][setting.id]["help"] + or setting.label + != current_plugin[plugin.id]["settings"][setting.id]["label"] + or setting.regex + != current_plugin[plugin.id]["settings"][setting.id]["regex"] + or setting.type + != current_plugin[plugin.id]["settings"][setting.id]["type"] + or setting.multiple + != current_plugin[plugin.id]["settings"][setting.id].get( + "multiple", None + ) + ): + print( + f"❌ The {'external' if plugin.external else 'core'} plugin {plugin.name} (id: {plugin.id}) is in the database but is not correct, exiting ...\n{dumps({'default': setting.default, 'help': setting.help, 'label': setting.label, 'regex': setting.regex, 'type': setting.type})} (database) != {dumps({'default': current_plugin[plugin.id]['settings'][setting.id]['default'], 'help': current_plugin[plugin.id]['settings'][setting.id]['help'], 'label': current_plugin[plugin.id]['settings'][setting.id]['label'], 'regex': current_plugin[plugin.id]['settings'][setting.id]['regex'], 'type': current_plugin[plugin.id]['settings'][setting.id]['type']})} (file)", + flush=True, + ) + exit(1) + + current_plugin[plugin.id]["checked"] = True + + if not all([core_plugins[plugin]["checked"] for plugin in core_plugins]): + print( + f"❌ Not all core plugins are in the database, exiting ...\nmissing plugins: {', '.join([plugin for plugin in core_plugins if not core_plugins[plugin]])}", + flush=True, + ) + exit(1) + elif not all([external_plugins[plugin]["checked"] for plugin in external_plugins]): + print( + f"❌ Not all external plugins are in the database, exiting ...\nmissing plugins: {', '.join([plugin for plugin in external_plugins if not external_plugins[plugin]])}", + flush=True, + ) + exit(1) + + print("✅ The ClamAV plugin and all core plugins are in the database", flush=True) + print(" ", flush=True) + print("ℹ️ Checking if the jobs are in the database ...", flush=True) + + with db_session() as session: + jobs = session.query(Jobs).all() + + for job in jobs: + if not job.success: + print( + f"❌ The job {job.name} (plugin_id: {job.plugin_id}) is in the database but failed, exiting ...", + flush=True, + ) + exit(1) + + if job.plugin_id in core_plugins: + current_plugin = core_plugins + elif job.plugin_id in external_plugins: + current_plugin = external_plugins + else: + print( + f"❌ The job {job.name} (plugin_id: {job.plugin_id}) is in the database but should not be, exiting ...", + flush=True, + ) + exit(1) + + index = next( + index + for (index, d) in enumerate( + current_plugin[job.plugin_id].get("jobs", []) + ) + if d["name"] == job.name + ) + core_job = current_plugin[job.plugin_id]["jobs"][index] + + if ( + job.name != core_job["name"] + or job.file_name != core_job["file"] + or job.every != core_job["every"] + or job.reload != core_job["reload"] + ): + print( + f"❌ The job {job.name} (plugin_id: {job.plugin_id}) is in the database but is not correct, exiting ...\n{dumps({'name': job.name, 'file': job.file_name, 'every': job.every, 'reload': job.reload})} (database) != {dumps({'name': core_job['name'], 'file': core_job['file'], 'every': core_job['every'], 'reload': core_job['reload']})} (file)", + flush=True, + ) + exit(1) + + current_plugin[job.plugin_id]["jobs"][index]["checked"] = True + + if not all( + [ + all([job["checked"] for job in core_plugins[plugin].get("jobs", [])]) + for plugin in core_plugins + ] + ): + print( + f"❌ Not all jobs from core plugins are in the database, exiting ...\nmissing jobs: {dumps({plugin: [job['name'] for job in core_plugins[plugin]['jobs'] if not job['checked']] for plugin in core_plugins})}", + flush=True, + ) + exit(1) + elif not all( + [ + all([job["checked"] for job in external_plugins[plugin].get("jobs", [])]) + for plugin in external_plugins + ] + ): + print( + f"❌ Not all jobs from external plugins are in the database, exiting ...\nmissing jobs: {dumps({plugin: [job['name'] for job in external_plugins[plugin]['jobs'] if not job['checked']] for plugin in external_plugins})}", + flush=True, + ) + exit(1) + + print("✅ All jobs are in the database and have successfully ran", flush=True) + print(" ", flush=True) + print("ℹ️ Checking if all plugin pages are in the database ...", flush=True) + + def file_hash(file: str) -> str: + _sha512 = sha512() + with open(file, "rb") as f: + while True: + data = f.read(1024) + if not data: + break + _sha512.update(data) + return _sha512.hexdigest() + + with db_session() as session: + plugin_pages = ( + session.query(Plugin_pages) + .with_entities( + Plugin_pages.id, + Plugin_pages.plugin_id, + Plugin_pages.template_checksum, + Plugin_pages.actions_checksum, + ) + .all() + ) + + for plugin_page in plugin_pages: + if plugin_page.plugin_id in core_plugins: + current_plugin = core_plugins + elif plugin_page.plugin_id in external_plugins: + current_plugin = external_plugins + else: + print( + f"❌ The plugin page from {plugin_page.plugin_id} is in the database but should not be, exiting ...", + flush=True, + ) + exit(1) + + path_ui = ( + Path(join("core", plugin_page.plugin_id, "ui")) + if Path(join("core", plugin_page.plugin_id, "ui")).exists() + else Path(join("external", plugin_page.plugin_id, "ui")) + ) + + if not path_ui.exists(): + print( + f'❌ The plugin page from {plugin_page.plugin_id} is in the database but should not be because the "ui" folder is missing from the plugin, exiting ...', + flush=True, + ) + exit(1) + + template_checksum = file_hash(f"{path_ui}/template.html") + actions_checksum = file_hash(f"{path_ui}/actions.py") + + if plugin_page.template_checksum != template_checksum: + print( + f"❌ The plugin page from {plugin_page.plugin_id} is in the database but the template file checksum differ, exiting ...\n{plugin_page.template_checksum} (database) != {template_checksum} (file)", + flush=True, + ) + exit(1) + elif plugin_page.actions_checksum != actions_checksum: + print( + f"❌ The plugin page from {plugin_page.plugin_id} is in the database but the actions file checksum differ, exiting ...\n{plugin_page.actions_checksum} (database) != {actions_checksum} (file)", + flush=True, + ) + exit(1) + + current_plugin[plugin_page.plugin_id]["page_checked"] = True + + if not all([core_plugins[plugin]["page_checked"] for plugin in core_plugins]): + print( + f"❌ Not all core plugins pages are in the database, exiting ...\nmissing plugins pages: {', '.join([plugin for plugin in core_plugins if not core_plugins[plugin]['page_checked']])}", + flush=True, + ) + exit(1) + elif not all( + [external_plugins[plugin]["page_checked"] for plugin in external_plugins] + ): + print( + f"❌ Not all external plugins pages are in the database, exiting ...\nmissing plugins pages: {', '.join([plugin for plugin in external_plugins if not external_plugins[plugin]['page_checked']])}", + flush=True, + ) + exit(1) + + print("✅ All plugin pages are in the database and have the right value", flush=True) + print(" ", flush=True) + print("ℹ️ Checking if all custom configs are in the database ...", flush=True) + + custom_confs_rx = re_compile( + r"^([0-9a-z\.-]*)_?CUSTOM_CONF_(SERVICE_)?(HTTP|SERVER_STREAM|STREAM|DEFAULT_SERVER_HTTP|SERVER_HTTP|MODSEC_CRS|MODSEC)_(.+)$" + ) + + global_custom_configs = {} + service_custom_configs = {} + for env in environ: + if not custom_confs_rx.match(env): + continue + + custom_conf = custom_confs_rx.search(env).groups() + if custom_conf[1]: + service_custom_configs[custom_conf[3]] = { + "value": environ[env].encode(), + "type": custom_conf[2].lower(), + "method": "scheduler", + "checked": False, + } + continue + + global_custom_configs[custom_conf[3]] = { + "value": environ[env].encode(), + "type": custom_conf[2].lower(), + "method": "scheduler", + "checked": False, + } + + with db_session() as session: + custom_configs = ( + session.query(Custom_configs) + .with_entities( + Custom_configs.service_id, + Custom_configs.type, + Custom_configs.name, + Custom_configs.data, + Custom_configs.method, + ) + .all() + ) + + for custom_config in custom_configs: + if ( + not multisite + and custom_config.name in global_custom_configs + and custom_config.service_id + ): + print( + f"❌ The custom config {custom_config.name} is in the database but should not be owned by the service {custom_config.service_id} because multisite is not enabled, exiting ...", + flush=True, + ) + exit(1) + elif ( + multisite + and custom_config.name in service_custom_configs + and not custom_config.service_id + ): + print( + f"❌ The custom config {custom_config.name} is in the database but should be owned by the service bwadm.example.com because it's a service config, exiting ...", + flush=True, + ) + exit(1) + + if custom_config.name in global_custom_configs: + current_custom_configs = global_custom_configs + elif custom_config.name in service_custom_configs: + current_custom_configs = service_custom_configs + else: + print( + f"❌ The custom config {custom_config.name} is in the database but should not be, exiting ...", + flush=True, + ) + exit(1) + + if custom_config.type != current_custom_configs[custom_config.name]["type"]: + print( + f"❌ The custom config {custom_config.name} is in the database but the type differ, exiting ...\n{custom_config.type} (database) != {current_custom_configs[custom_config.name]['type']} (env)", + flush=True, + ) + exit(1) + elif ( + custom_config.data + != current_custom_configs[custom_config.name]["value"] + ): + print( + f"❌ The custom config {custom_config.name} is in the database but the value differ, exiting ...\n{custom_config.data} (database) != {current_custom_configs[custom_config.name]['value']} (env)", + flush=True, + ) + exit(1) + elif ( + custom_config.method + != current_custom_configs[custom_config.name]["method"] + ): + print( + f"❌ The custom config {custom_config.name} is in the database but the method differ, exiting ...\n{custom_config.method} (database) != {current_custom_configs[custom_config.name]['method']} (env)", + flush=True, + ) + exit(1) + + current_custom_configs[custom_config.name]["checked"] = True + + if not all( + [ + global_custom_configs[custom_config]["checked"] + for custom_config in global_custom_configs + ] + ): + print( + f"❌ Not all global custom configs are in the database, exiting ...\nmissing custom configs: {', '.join([custom_config for custom_config in global_custom_configs if not global_custom_configs[custom_config]['checked']])}", + flush=True, + ) + exit(1) + elif not all( + [ + service_custom_configs[custom_config]["checked"] + for custom_config in service_custom_configs + ] + ): + print( + f"❌ Not all service custom configs are in the database, exiting ...\nmissing custom configs: {', '.join([custom_config for custom_config in service_custom_configs if not service_custom_configs[custom_config]['checked']])}", + flush=True, + ) + exit(1) + + print( + "✅ All custom configs are in the database and have the right value", flush=True + ) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/db/requirements.txt b/tests/core/db/requirements.txt new file mode 100644 index 000000000..e051fe3c0 --- /dev/null +++ b/tests/core/db/requirements.txt @@ -0,0 +1,4 @@ +sqlalchemy==2.0.13 +psycopg2-binary==2.9.6 +PyMySQL==1.0.3 +cryptography==40.0.2 diff --git a/tests/core/db/test.sh b/tests/core/db/test.sh new file mode 100755 index 000000000..8df748676 --- /dev/null +++ b/tests/core/db/test.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +echo "💾 Building db stack ..." + +# Starting stack +docker compose pull bw-docker app1 bw-maria-db bw-mysql-db bw-postgres-db +if [ $? -ne 0 ] ; then + echo "💾 Pull failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + rm -rf init/plugins + find . -type f -name 'docker-compose.*' -exec sed -i 's@DATABASE_URI: ".*"$@DATABASE_URI: "sqlite:////var/lib/bunkerweb/db.sqlite3"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@MULTISITE: "yes"$@MULTISITE: "no"@' {} \; + sed -i 's@bwadm.example.com_USE_REVERSE_PROXY@USE_REVERSE_PROXY@' docker-compose.yml + sed -i 's@bwadm.example.com_REVERSE_PROXY_HOST@REVERSE_PROXY_HOST@' docker-compose.yml + sed -i 's@bwadm.example.com_REVERSE_PROXY_URL@REVERSE_PROXY_URL@' docker-compose.yml + sed -i 's@SERVICE_USE_REVERSE_PROXY@GLOBAL_USE_REVERSE_PROXY@' docker-compose.test.yml + sed -i 's@SERVICE_REVERSE_PROXY_HOST@GLOBAL_REVERSE_PROXY_HOST@' docker-compose.test.yml + sed -i 's@SERVICE_REVERSE_PROXY_URL@GLOBAL_REVERSE_PROXY_URL@' docker-compose.test.yml + + if [[ $(sed '20!d' docker-compose.yml) = ' bwadm.example.com_SERVER_NAME: "bwadm.example.com"' ]] ; then + sed -i '20d' docker-compose.yml + fi + + if [[ $(sed '24!d' docker-compose.yml) = " bwadm.example.com_CUSTOM_CONF_MODSEC_CRS_test_service_conf: 'SecRule REQUEST_FILENAME \"@rx ^/test\" \"id:2,ctl:ruleRemoveByTag=attack-generic,ctl:ruleRemoveByTag=attack-protocol,nolog\"'" ]] ; then + sed -i '24d' docker-compose.yml + fi + + if [[ $(sed '18!d' docker-compose.test.yml) = ' SERVICE_SERVER_NAME: "bwadm.example.com"' ]] ; then + sed -i '18d' docker-compose.test.yml + fi + + if [[ $(sed '23!d' docker-compose.test.yml) = " CUSTOM_CONF_SERVICE_MODSEC_CRS_test_service_conf: 'SecRule REQUEST_FILENAME \"@rx ^/test\" \"id:2,ctl:ruleRemoveByTag=attack-generic,ctl:ruleRemoveByTag=attack-protocol,nolog\"'" ]] ; then + sed -i '23d' docker-compose.test.yml + fi + + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "💾 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "💾 Down failed ❌" + exit 1 + fi + + echo "💾 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +echo "💾 Initializing workspace ..." +rm -rf init/plugins +mkdir -p init/plugins +docker compose -f docker-compose.init.yml up --build +if [ $? -ne 0 ] ; then + echo "💾 Build failed ❌" + exit 1 +elif ! [[ -d "init/plugins/clamav" ]]; then + echo "💾 ClamAV plugin not found ❌" + exit 1 +fi + +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "💾 Build failed ❌" + exit 1 +fi + +for test in "local" "multisite" "mariadb" "mysql" "postgres" +do + if [ "$test" = "local" ] ; then + echo "💾 Running tests with a local database ..." + elif [ "$test" = "multisite" ] ; then + echo "💾 Running tests with MULTISITE set to yes and with multisite settings ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@MULTISITE: "no"$@MULTISITE: "yes"@' {} \; + sed -i '20i \ bwadm.example.com_SERVER_NAME: "bwadm.example.com"' docker-compose.yml + sed -i "25i \ bwadm.example.com_CUSTOM_CONF_MODSEC_CRS_test_service_conf: 'SecRule REQUEST_FILENAME \"@rx ^/test\" \"id:2,ctl:ruleRemoveByTag=attack-generic,ctl:ruleRemoveByTag=attack-protocol,nolog\"'" docker-compose.yml + sed -i 's@USE_REVERSE_PROXY@bwadm.example.com_USE_REVERSE_PROXY@' docker-compose.yml + sed -i 's@REVERSE_PROXY_HOST@bwadm.example.com_REVERSE_PROXY_HOST@' docker-compose.yml + sed -i 's@REVERSE_PROXY_URL@bwadm.example.com_REVERSE_PROXY_URL@' docker-compose.yml + sed -i '18i \ SERVICE_SERVER_NAME: "bwadm.example.com"' docker-compose.test.yml + sed -i "24i \ CUSTOM_CONF_SERVICE_MODSEC_CRS_test_service_conf: 'SecRule REQUEST_FILENAME \"@rx ^/test\" \"id:2,ctl:ruleRemoveByTag=attack-generic,ctl:ruleRemoveByTag=attack-protocol,nolog\"'" docker-compose.test.yml + sed -i 's@GLOBAL_USE_REVERSE_PROXY@SERVICE_USE_REVERSE_PROXY@' docker-compose.test.yml + sed -i 's@GLOBAL_REVERSE_PROXY_HOST@SERVICE_REVERSE_PROXY_HOST@' docker-compose.test.yml + sed -i 's@GLOBAL_REVERSE_PROXY_URL@SERVICE_REVERSE_PROXY_URL@' docker-compose.test.yml + elif [ "$test" = "mariadb" ] ; then + echo "💾 Running tests with MariaDB database ..." + echo "ℹ️ Keeping the MULTISITE variable to yes and multisite settings ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@DATABASE_URI: ".*"$@DATABASE_URI: "mariadb+pymysql://bunkerweb:secret\@bw-maria-db:3306/db"@' {} \; + elif [ "$test" = "mysql" ] ; then + echo "💾 Running tests with MySQL database ..." + echo "ℹ️ Keeping the MULTISITE variable to yes and multisite settings ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@DATABASE_URI: ".*"$@DATABASE_URI: "mysql+pymysql://bunkerweb:secret\@bw-mysql-db:3306/db"@' {} \; + elif [ "$test" = "postgres" ] ; then + echo "💾 Running tests with PostgreSQL database ..." + echo "ℹ️ Keeping the MULTISITE variable to yes and multisite settings ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@DATABASE_URI: ".*"$@DATABASE_URI: "postgresql://bunkerweb:secret\@bw-postgres-db:5432/db"@' {} \; + fi + + echo "💾 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "💾 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "💾 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("db-bw-1" "db-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "💾 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "💾 Docker stack is not healthy ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "💾 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "💾 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "💾 Tests are done ! ✅" diff --git a/tests/core/dnsbl/Dockerfile b/tests/core/dnsbl/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/dnsbl/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/dnsbl/docker-compose.init.yml b/tests/core/dnsbl/docker-compose.init.yml new file mode 100644 index 000000000..36d606b46 --- /dev/null +++ b/tests/core/dnsbl/docker-compose.init.yml @@ -0,0 +1,9 @@ +version: "3.5" + +services: + init: + build: init + environment: + PYTHONUNBUFFERED: "1" + volumes: + - ./init/output:/output diff --git a/tests/core/dnsbl/docker-compose.test.yml b/tests/core/dnsbl/docker-compose.test.yml new file mode 100644 index 000000000..9c192a915 --- /dev/null +++ b/tests/core/dnsbl/docker-compose.test.yml @@ -0,0 +1,18 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_DNSBL: "yes" + DNSBL_LIST: "bl.blocklist.de problems.dnsbl.sorbs.net" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/dnsbl/docker-compose.yml b/tests/core/dnsbl/docker-compose.yml new file mode 100644 index 000000000..768bee0ca --- /dev/null +++ b/tests/core/dnsbl/docker-compose.yml @@ -0,0 +1,65 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + ports: + - 80:80 + - 443:443 + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? DNSBL settings + USE_DNSBL: "yes" + DNSBL_LIST: "bl.blocklist.de problems.dnsbl.sorbs.net" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: + name: bw-docker diff --git a/tests/core/dnsbl/index.html b/tests/core/dnsbl/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/dnsbl/init/Dockerfile b/tests/core/dnsbl/init/Dockerfile new file mode 100644 index 000000000..c6b6dd4bc --- /dev/null +++ b/tests/core/dnsbl/init/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11.3-alpine + +# Install firefox and geckodriver +RUN apk add --no-cache --virtual .build-deps curl grep zip && \ + apk add --no-cache firefox + +# Installing geckodriver for firefox... +RUN GECKODRIVER_VERSION=`curl -i https://github.com/mozilla/geckodriver/releases/latest | grep -Po 'v[0-9]+.[0-9]+.[0-9]+'` && \ + wget -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \ + tar -C /usr/local/bin -xzvf geckodriver.tar.gz && \ + chmod +x /usr/local/bin/geckodriver && \ + rm geckodriver.tar.gz + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/dnsbl/init/main.py b/tests/core/dnsbl/init/main.py new file mode 100644 index 000000000..35763fa42 --- /dev/null +++ b/tests/core/dnsbl/init/main.py @@ -0,0 +1,59 @@ +from contextlib import suppress +from ipaddress import IPv4Address +from pathlib import Path +from traceback import format_exc +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.remote.webelement import WebElement +from selenium.webdriver.support import expected_conditions as EC +from socket import gaierror, gethostbyname +from typing import List + +try: + firefox_options = Options() + firefox_options.add_argument("--headless") + + dnsbl_servers = [] + + print("ℹ️ Starting Firefox ...", flush=True) + with webdriver.Firefox(options=firefox_options) as driver: + driver.delete_all_cookies() + driver.maximize_window() + driver_wait = WebDriverWait(driver, 10) + + print("ℹ️ Navigating to https://www.dnsbl.info/dnsbl-list.php ...", flush=True) + driver.get("https://www.dnsbl.info/dnsbl-list.php") + + print("ℹ️ Getting the DNSBL servers ...") + links: List[WebElement] = driver_wait.until( + EC.presence_of_all_elements_located( + (By.XPATH, "//table[@class='body_sub_body']//td") + ) + ) + + for link in links: + content = link.text + if content: + dnsbl_servers.append(content) + + print("ℹ️ Checking the DNSBL servers for a banned IP ...", flush=True) + + for ip_address in [IPv4Address(f"{x}.0.0.3") for x in range(1, 256)]: + for dnsbl_server in dnsbl_servers: + with suppress(gaierror): + gethostbyname( + f"{ip_address.reverse_pointer.replace('.in-addr.arpa', '')}.{dnsbl_server}" + ) + print( + f"✅ {ip_address} is banned on {dnsbl_server}, saving it to /output/dnsbl_ip.txt", + flush=True, + ) + Path("/output/dnsbl_ip.txt").write_text(f"{ip_address} {dnsbl_server}") + exit(0) +except SystemExit as e: + exit(e.code) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/dnsbl/init/requirements.txt b/tests/core/dnsbl/init/requirements.txt new file mode 100644 index 000000000..d2389eee9 --- /dev/null +++ b/tests/core/dnsbl/init/requirements.txt @@ -0,0 +1 @@ +selenium==4.9.1 diff --git a/tests/core/dnsbl/main.py b/tests/core/dnsbl/main.py new file mode 100644 index 000000000..82605274e --- /dev/null +++ b/tests/core/dnsbl/main.py @@ -0,0 +1,65 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"}, timeout=3 + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 or status_code == 403 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_dnsbl = getenv("USE_DNSBL", "yes") == "yes" + dnsbl_list = getenv("DNSBL_LIST", "bl.blocklist.de problems.dnsbl.sorbs.net") + + print( + "ℹ️ Sending a request to http://www.example.com ...", + flush=True, + ) + + status_code = get( + f"http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code == 403: + if not use_dnsbl: + print("❌ The request was rejected, but DNSBL is disabled, exiting ...") + exit(1) + elif dnsbl_list == "bl.blocklist.de problems.dnsbl.sorbs.net": + print( + '❌ The request was rejected, but DNSBL list is equal to "bl.blocklist.de problems.dnsbl.sorbs.net", exiting ...' + ) + exit(1) + elif use_dnsbl and dnsbl_list != "bl.blocklist.de problems.dnsbl.sorbs.net": + print( + f'❌ The request was not rejected, but DNSBL list is equal to "{dnsbl_list}", exiting ...' + ) + exit(1) + + print("✅ DNSBL is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/dnsbl/requirements.txt b/tests/core/dnsbl/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/dnsbl/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/dnsbl/test.sh b/tests/core/dnsbl/test.sh new file mode 100755 index 000000000..54b97685a --- /dev/null +++ b/tests/core/dnsbl/test.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +echo "🚫 Building dnsbl stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🚫 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🚫 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + rm -rf init/output + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_DNSBL: "no"@USE_DNSBL: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@DNSBL_LIST: ".*"@DNSBL_LIST: "bl.blocklist.de problems.dnsbl.sorbs.net"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@ipv4_address: [0-9][0-9]*\.0@ipv4_address: 192.168@' {} \; + sed -i 's@subnet: [0-9][0-9]*\.0@subnet: 192.168@' docker-compose.yml + sed -i 's@www.example.com:[0-9][0-9]*\.0@www.example.com:192.168@' docker-compose.test.yml + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🚫 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🚫 Down failed ❌" + exit 1 + fi + + echo "🚫 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +echo "🚫 Initializing workspace ..." +rm -rf init/output +mkdir -p init/output +docker compose -f docker-compose.init.yml up --build +if [ $? -ne 0 ] ; then + echo "🚫 Build failed ❌" + exit 1 +elif ! [[ -f "init/output/dnsbl_ip.txt" ]] ; then + echo "🚫 Initialization failed, dnsbl_ip.txt not found ❌" + exit 1 +fi + +content=($(cat init/output/dnsbl_ip.txt)) +ip=${content[0]} +server=${content[1]} + +echo "🚫 Will use IP: $ip" +echo "🚫 Will use DNSBL Server: $server" + +for test in "activated" "deactivated" "list" +do + if [ "$test" = "activated" ] ; then + echo "🚫 Running tests with DNSBL activated and the server $server added to the list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@DNSBL_LIST: ".*"@DNSBL_LIST: "bl.blocklist.de problems.dnsbl.sorbs.net '"$server"'"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@ipv4_address: 192.168@ipv4_address: '"${ip%%.*}"'.0@' {} \; + sed -i 's@subnet: 192.168@subnet: '"${ip%%.*}"'.0@' docker-compose.yml + sed -i 's@www.example.com:192.168@www.example.com:'"${ip%%.*}"'.0@' docker-compose.test.yml + elif [ "$test" = "deactivated" ] ; then + echo "🚫 Running tests without DNSBL ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_DNSBL: "yes"@USE_DNSBL: "no"@' {} \; + elif [ "$test" = "list" ] ; then + echo "🚫 Running tests with DNSBL activated and without the server $server added to the list ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_DNSBL: "no"@USE_DNSBL: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@DNSBL_LIST: ".*"@DNSBL_LIST: "bl.blocklist.de problems.dnsbl.sorbs.net"@' {} \; + fi + + echo "🚫 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🚫 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🚫 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("dnsbl-bw-1" "dnsbl-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🚫 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🚫 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🚫 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🚫 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🚫 Tests are done ! ✅" diff --git a/tests/core/docker-compose.test.yml b/tests/core/docker-compose.test.yml new file mode 100644 index 000000000..b9893e2a4 --- /dev/null +++ b/tests/core/docker-compose.test.yml @@ -0,0 +1,7 @@ +version: '3.5' + +services: + tests: + build: + context: . + dockerfile: Dockerfile.dev diff --git a/tests/core/errors/403.html b/tests/core/errors/403.html new file mode 100644 index 000000000..3926a0bc7 --- /dev/null +++ b/tests/core/errors/403.html @@ -0,0 +1,5 @@ + + +

It Works!

+ + diff --git a/tests/core/errors/Dockerfile b/tests/core/errors/Dockerfile new file mode 100644 index 000000000..c6b6dd4bc --- /dev/null +++ b/tests/core/errors/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11.3-alpine + +# Install firefox and geckodriver +RUN apk add --no-cache --virtual .build-deps curl grep zip && \ + apk add --no-cache firefox + +# Installing geckodriver for firefox... +RUN GECKODRIVER_VERSION=`curl -i https://github.com/mozilla/geckodriver/releases/latest | grep -Po 'v[0-9]+.[0-9]+.[0-9]+'` && \ + wget -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \ + tar -C /usr/local/bin -xzvf geckodriver.tar.gz && \ + chmod +x /usr/local/bin/geckodriver && \ + rm geckodriver.tar.gz + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/errors/docker-compose.test.yml b/tests/core/errors/docker-compose.test.yml new file mode 100644 index 000000000..d80ff82bb --- /dev/null +++ b/tests/core/errors/docker-compose.test.yml @@ -0,0 +1,18 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + ERRORS: "" + INTERCEPTED_ERROR_CODES: "400 401 403 404 405 413 429 500 501 502 503 504" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/errors/docker-compose.yml b/tests/core/errors/docker-compose.yml new file mode 100644 index 000000000..ccfdc413c --- /dev/null +++ b/tests/core/errors/docker-compose.yml @@ -0,0 +1,62 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./403.html:/var/www/html/errors/403.html + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? ERRORS settings + ERRORS: "" + INTERCEPTED_ERROR_CODES: "400 401 403 404 405 413 429 500 501 502 503 504" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/errors/index.html b/tests/core/errors/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/errors/main.py b/tests/core/errors/main.py new file mode 100644 index 000000000..ea038cc40 --- /dev/null +++ b/tests/core/errors/main.py @@ -0,0 +1,117 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.common.exceptions import NoSuchElementException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + firefox_options = Options() + firefox_options.add_argument("--headless") + + errors = getenv("ERRORS", "") + intercepted_error_codes = getenv( + "INTERCEPTED_ERROR_CODES", "400 401 403 404 405 413 429 500 501 502 503 504" + ) + + print("ℹ️ Starting Firefox ...", flush=True) + with webdriver.Firefox(options=firefox_options) as driver: + driver.delete_all_cookies() + driver.maximize_window() + + print( + "ℹ️ Navigating to http://www.example.com/?id=/etc/passwd ...", + flush=True, + ) + driver.get("http://www.example.com/?id=/etc/passwd") + + default_message = None + with suppress(NoSuchElementException): + default_message = driver.find_element( + By.XPATH, "//p[contains(text(), 'This website is protected with')]" + ) + + if default_message and ( + errors + or intercepted_error_codes + != "400 401 403 404 405 413 429 500 501 502 503 504" + ): + print( + "❌ The default error page is being displayed, exiting ...", + flush=True, + ) + exit(1) + elif ( + not default_message + and not errors + and intercepted_error_codes + == "400 401 403 404 405 413 429 500 501 502 503 504" + ): + print( + "❌ The default error page is not being displayed, exiting ...", + flush=True, + ) + exit(1) + + if errors: + custom_message = None + with suppress(NoSuchElementException): + custom_message = driver.find_element( + By.XPATH, "//h1[contains(text(), 'It Works!')]" + ) + + if not custom_message: + print( + "❌ The custom error page is not being displayed while a custom one has been provided, exiting ...", + flush=True, + ) + exit(1) + + if intercepted_error_codes != "400 401 403 404 405 413 429 500 501 502 503 504": + nginx_message = None + with suppress(NoSuchElementException): + nginx_message = driver.find_element( + By.XPATH, "//center[contains(text(), 'nginx')]" + ) + + if not nginx_message: + print( + "❌ The default nginx error page is not being displayed while a custom one has been provided, exiting ...", + flush=True, + ) + exit(1) + + print("✅ Errors are working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/errors/requirements.txt b/tests/core/errors/requirements.txt new file mode 100644 index 000000000..6f7b13f79 --- /dev/null +++ b/tests/core/errors/requirements.txt @@ -0,0 +1,2 @@ +requests==2.30.0 +selenium==4.9.1 diff --git a/tests/core/errors/test.sh b/tests/core/errors/test.sh new file mode 100755 index 000000000..068a6c929 --- /dev/null +++ b/tests/core/errors/test.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +echo "⭕ Building errors stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "⭕ Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "⭕ Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@ERRORS: "403=/errors/403.html"@ERRORS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@INTERCEPTED_ERROR_CODES: "400 401 404 405 413 429 500 501 502 503 504"@INTERCEPTED_ERROR_CODES: "400 401 403 404 405 413 429 500 501 502 503 504"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "⭕ Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "⭕ Down failed ❌" + exit 1 + fi + + echo "⭕ Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "default" "custom_403" "without_403" +do + if [ "$test" = "default" ] ; then + echo "⭕ Running tests with default configuration ..." + elif [ "$test" = "custom_403" ] ; then + echo "⭕ Running tests with a custom 403 page ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@ERRORS: ""@ERRORS: "403=/errors/403.html"@' {} \; + elif [ "$test" = "without_403" ] ; then + echo "⭕ Running tests without a 403 being intercepted ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@ERRORS: "403=/errors/403.html"@ERRORS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@INTERCEPTED_ERROR_CODES: "400 401 403 404 405 413 429 500 501 502 503 504"@INTERCEPTED_ERROR_CODES: "400 401 404 405 413 429 500 501 502 503 504"@' {} \; + fi + + echo "⭕ Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "⭕ Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "⭕ Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("errors-bw-1" "errors-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "⭕ Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "⭕ Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "⭕ Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "⭕ Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "⭕ Tests are done ! ✅" diff --git a/tests/core/greylist/Dockerfile b/tests/core/greylist/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/greylist/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/greylist/api/Dockerfile b/tests/core/greylist/api/Dockerfile new file mode 100644 index 000000000..714dc8a89 --- /dev/null +++ b/tests/core/greylist/api/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/greylist_api + +COPY main.py . + +ENTRYPOINT [ "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers", "--forwarded-allow-ips", "\"*\"" ] \ No newline at end of file diff --git a/tests/core/greylist/api/main.py b/tests/core/greylist/api/main.py new file mode 100644 index 000000000..b7a8b92a1 --- /dev/null +++ b/tests/core/greylist/api/main.py @@ -0,0 +1,30 @@ +from fastapi import FastAPI +from fastapi.responses import PlainTextResponse + + +app = FastAPI() + + +@app.get("/ip") +async def ip(): + return PlainTextResponse("192.168.0.3\n10.0.0.0/8\n127.0.0.1/32") + + +@app.get("/rdns") +async def rdns(): + return PlainTextResponse(".example.com\n.example.org\n.bw-services") + + +@app.get("/asn") +async def asn(): + return PlainTextResponse("1234\n13335\n5678") + + +@app.get("/user_agent") +async def user_agent(): + return PlainTextResponse("BunkerBot\nCensysInspect\nShodanInspect\nZmEu\nmasscan") + + +@app.get("/uri") +async def uri(): + return PlainTextResponse("/admin\n/login") diff --git a/tests/core/greylist/api/requirements.txt b/tests/core/greylist/api/requirements.txt new file mode 100644 index 000000000..c06221a5a --- /dev/null +++ b/tests/core/greylist/api/requirements.txt @@ -0,0 +1,2 @@ +fastapi==0.95.1 +uvicorn[standard]==0.22.0 diff --git a/tests/core/greylist/docker-compose.init.yml b/tests/core/greylist/docker-compose.init.yml new file mode 100644 index 000000000..36d606b46 --- /dev/null +++ b/tests/core/greylist/docker-compose.init.yml @@ -0,0 +1,9 @@ +version: "3.5" + +services: + init: + build: init + environment: + PYTHONUNBUFFERED: "1" + volumes: + - ./init/output:/output diff --git a/tests/core/greylist/docker-compose.test.yml b/tests/core/greylist/docker-compose.test.yml new file mode 100644 index 000000000..724694aae --- /dev/null +++ b/tests/core/greylist/docker-compose.test.yml @@ -0,0 +1,54 @@ +version: "3.5" + +services: + local-tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + GLOBAL: "0" + USE_GREYLIST: "no" + GREYLIST_IP: "" + GREYLIST_IP_URLS: "" + GREYLIST_RDNS_GLOBAL: "yes" + GREYLIST_RDNS: "" + GREYLIST_RDNS_URLS: "" + GREYLIST_ASN: "" + GREYLIST_ASN_URLS: "" + GREYLIST_USER_AGENT: "" + GREYLIST_USER_AGENT_URLS: "" + GREYLIST_URI: "" + GREYLIST_URI_URLS: "" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + + global-tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + GLOBAL: "1" + USE_GREYLIST: "no" + GREYLIST_IP: "" + GREYLIST_IP_URLS: "" + GREYLIST_RDNS_GLOBAL: "yes" + GREYLIST_RDNS: "" + GREYLIST_RDNS_URLS: "" + GREYLIST_ASN: "" + GREYLIST_ASN_URLS: "" + GREYLIST_USER_AGENT: "" + GREYLIST_USER_AGENT_URLS: "" + GREYLIST_URI: "" + GREYLIST_URI_URLS: "" + extra_hosts: + - "www.example.com:1.0.0.2" + networks: + bw-global-network: + ipv4_address: 1.0.0.3 + +networks: + bw-services: + external: true + bw-global-network: + external: true diff --git a/tests/core/greylist/docker-compose.yml b/tests/core/greylist/docker-compose.yml new file mode 100644 index 000000000..03d2e928d --- /dev/null +++ b/tests/core/greylist/docker-compose.yml @@ -0,0 +1,92 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + LOG_LEVEL: "info" + + # ? GREYLIST settings + USE_GREYLIST: "no" + GREYLIST_IP: "" + GREYLIST_IP_URLS: "" + GREYLIST_RDNS_GLOBAL: "yes" + GREYLIST_RDNS: "" + GREYLIST_RDNS_URLS: "" + GREYLIST_ASN: "" + GREYLIST_ASN_URLS: "" + GREYLIST_USER_AGENT: "" + GREYLIST_USER_AGENT_URLS: "" + GREYLIST_URI: "" + GREYLIST_URI_URLS: "" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + bw-global-network: + ipv4_address: 1.0.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + volumes: + - bw-data:/data + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + greylist-api: + build: api + networks: + bw-docker: + bw-services: + ipv4_address: 192.168.0.4 + +volumes: + bw-data: + + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-global-network: + name: bw-global-network + ipam: + driver: default + config: + - subnet: 1.0.0.0/8 + bw-docker: + name: bw-docker diff --git a/tests/core/greylist/index.html b/tests/core/greylist/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/greylist/init/Dockerfile b/tests/core/greylist/init/Dockerfile new file mode 100644 index 000000000..2bb13a4f9 --- /dev/null +++ b/tests/core/greylist/init/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/blacklist_init + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/greylist/init/main.py b/tests/core/greylist/init/main.py new file mode 100644 index 000000000..ef1409902 --- /dev/null +++ b/tests/core/greylist/init/main.py @@ -0,0 +1,33 @@ +from datetime import date +from gzip import GzipFile +from io import BytesIO +from pathlib import Path +from maxminddb import MODE_FD, open_database +from requests import get + +# Compute the mmdb URL +mmdb_url = f"https://download.db-ip.com/free/dbip-asn-lite-{date.today().strftime('%Y-%m')}.mmdb.gz" + +# Download the mmdb file in memory +print(f"Downloading mmdb file from url {mmdb_url} ...", flush=True) +file_content = BytesIO() +with get(mmdb_url, stream=True) as resp: + resp.raise_for_status() + for chunk in resp.iter_content(chunk_size=4 * 1024): + if chunk: + file_content.write(chunk) +file_content.seek(0) + +with open_database(GzipFile(fileobj=file_content, mode="rb"), mode=MODE_FD) as reader: + dbip_asn = reader.get("1.0.0.3") + + if not dbip_asn: + print(f"❌ Error while reading mmdb file from {mmdb_url}", flush=True) + exit(1) + + print( + f"✅ ASN for IP 1.0.0.3 is {dbip_asn['autonomous_system_number']}, saving it to /output/ip_asn.txt", + flush=True, + ) + + Path("/output/ip_asn.txt").write_text(str(dbip_asn["autonomous_system_number"])) diff --git a/tests/core/greylist/init/requirements.txt b/tests/core/greylist/init/requirements.txt new file mode 100644 index 000000000..f91deab71 --- /dev/null +++ b/tests/core/greylist/init/requirements.txt @@ -0,0 +1,2 @@ +maxminddb==2.3.0 +requests==2.30.0 diff --git a/tests/core/greylist/main.py b/tests/core/greylist/main.py new file mode 100644 index 000000000..c41afe570 --- /dev/null +++ b/tests/core/greylist/main.py @@ -0,0 +1,189 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 or status_code == 403 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_greylist = getenv("USE_GREYLIST", "yes") == "yes" + _global = getenv("GLOBAL", "0") == "1" + + greylist_ip = getenv("GREYLIST_IP", "") + greylist_ip_urls = getenv("GREYLIST_IP_URLS", "") + greylist_rdns_global = getenv("GREYLIST_RDNS_GLOBAL", "yes") == "yes" + greylist_rdns = getenv("GREYLIST_RDNS", "") + greylist_rdns_urls = getenv("GREYLIST_RDNS_URLS", "") + greylist_asn = getenv("GREYLIST_ASN", "") + greylist_asn_urls = getenv("GREYLIST_ASN_URLS", "") + greylist_user_agent = getenv("GREYLIST_USER_AGENT", "") + greylist_user_agent_urls = getenv("GREYLIST_USER_AGENT_URLS", "") + greylist_uri = getenv("GREYLIST_URI", "") + greylist_uri_urls = getenv("GREYLIST_URI_URLS", "") + + print("ℹ️ Sending a request to http://www.example.com ...", flush=True) + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code == 403: + if not use_greylist: + print( + "❌ Request was rejected, even though greylist is supposed to be disabled, exiting ..." + ) + exit(1) + elif (greylist_ip or greylist_ip_urls) and not _global: + print( + "❌ Request was rejected, even though IP is supposed to be in the greylist, exiting ..." + ) + exit(1) + elif ( + (greylist_rdns or greylist_rdns_urls) + and not greylist_rdns_global + and not _global + ): + print( + "❌ Request was rejected, even though RDNS is supposed to be in the greylist, exiting ..." + ) + exit(1) + elif (greylist_asn or greylist_asn_urls) and _global: + print( + "❌ Request was rejected, even though ASN is supposed to be in the greylist, exiting ..." + ) + exit(1) + elif greylist_user_agent or greylist_user_agent_urls: + print( + "ℹ️ Sending a request to http://www.example.com with User-Agent BunkerBot ...", + flush=True, + ) + status_code = get( + "http://www.example.com", + headers={"Host": "www.example.com", "User-Agent": "BunkerBot"}, + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code == 403: + print( + "❌ Request was rejected, even though User Agent is supposed to be in the greylist ..." + ) + exit(1) + + print("✅ Request was not rejected, User Agent is in the greylist ...") + print( + "ℹ️ Sending a request to http://www.example.com/?id=/etc/passwd with User-Agent BunkerBot ...", + flush=True, + ) + + status_code = get( + "http://www.example.com/?id=/etc/passwd", + headers={"Host": "www.example.com", "User-Agent": "BunkerBot"}, + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code != 403: + print("❌ Request was not rejected, exiting ...", flush=True) + exit(1) + elif greylist_uri or greylist_uri_urls: + print( + "ℹ️ Sending a request to http://www.example.com/admin ...", + flush=True, + ) + status_code = get( + "http://www.example.com/admin", headers={"Host": "www.example.com"} + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code == 403: + print( + "❌ Request was rejected, even though URI is supposed to be in the greylist ..." + ) + exit(1) + + print("✅ Request was not rejected, URI is in the greylist ...") + print( + "ℹ️ Sending a request to http://www.example.com/admin/?id=/etc/passwd ...", + flush=True, + ) + + status_code = get( + "http://www.example.com/admin/?id=/etc/passwd", + headers={"Host": "www.example.com"}, + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code != 403: + print("❌ Request was not rejected, exiting ...", flush=True) + exit(1) + else: + if (greylist_ip or greylist_ip_urls) and _global: + print( + "❌ Request was not rejected, but IP is not in the greylist, exiting ..." + ) + exit(1) + elif (greylist_rdns or greylist_rdns_urls) and _global: + print( + "❌ Request was not rejected, but RDNS is not in the greylist, exiting ..." + ) + exit(1) + elif (greylist_asn or greylist_asn_urls) and not _global: + print("❌ Request was rejected, but ASN is not in the greylist, exiting ...") + exit(1) + elif greylist_user_agent or greylist_user_agent_urls: + print("❌ Request was rejected, but User Agent is not in the greylist ...") + exit(1) + elif greylist_uri or greylist_uri_urls: + print("❌ Request was rejected, but URI is not in the greylist ...") + exit(1) + + print( + "ℹ️ Sending a request to http://www.example.com/?id=/etc/passwd ...", + flush=True, + ) + + status_code = get( + "http://www.example.com/?id=/etc/passwd", + headers={"Host": "www.example.com"}, + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code != 403: + print("❌ Request was not rejected, exiting ...", flush=True) + exit(1) + + print("✅ Greylist is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/greylist/requirements.txt b/tests/core/greylist/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/greylist/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/greylist/test.sh b/tests/core/greylist/test.sh new file mode 100755 index 000000000..405a04dd0 --- /dev/null +++ b/tests/core/greylist/test.sh @@ -0,0 +1,209 @@ +#!/bin/bash + +echo "🏁 Building greylist stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🏁 Pull failed ❌" + exit 1 +fi + +echo "🏁 Building custom api image ..." +docker compose build greylist-api +if [ $? -ne 0 ] ; then + echo "🏁 Build failed ❌" + exit 1 +fi + +echo "🏁 Building tests images ..." +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🏁 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +as_number=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + rm -rf init/output + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_GREYLIST: "yes"@USE_GREYLIST: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_IP: "192.168.0.0/24"@GREYLIST_IP: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_IP_URLS: "http://greylist-api:8080/ip"@GREYLIST_IP_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_RDNS_GLOBAL: "no"@GREYLIST_RDNS_GLOBAL: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_RDNS: ".bw-services"@GREYLIST_RDNS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_RDNS_URLS: "http://greylist-api:8080/rdns"@GREYLIST_RDNS_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_ASN: "[0-9]*"@GREYLIST_ASN: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_ASN_URLS: "http://greylist-api:8080/asn"@GREYLIST_ASN_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_USER_AGENT: "BunkerBot"@GREYLIST_USER_AGENT: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_USER_AGENT_URLS: "http://greylist-api:8080/user_agent"@GREYLIST_USER_AGENT_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_URI: "/admin"@GREYLIST_URI: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_URI_URLS: "http://greylist-api:8080/uri"@GREYLIST_URI_URLS: ""@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🏁 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🏁 Down failed ❌" + exit 1 + fi + + echo "🏁 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +echo "🏁 Initializing workspace ..." +rm -rf init/output +mkdir -p init/output +docker compose -f docker-compose.init.yml up --build +if [ $? -ne 0 ] ; then + echo "🏁 Build failed ❌" + exit 1 +elif ! [[ -f "init/output/ip_asn.txt" ]]; then + echo "🏁 ip_asn.txt not found ❌" + exit 1 +fi + +as_number=$(cat init/output/ip_asn.txt) + +if [[ $as_number = "" ]]; then + echo "🏁 AS number not found ❌" + exit 1 +fi + +rm -rf init/output + +for test in "deactivated" "ip" "ip_urls" "rdns" "rdns_global" "rdns_urls" "asn" "asn_urls" "user_agent" "user_agent_urls" "uri" "uri_urls" +do + if [ "$test" = "deactivated" ] ; then + echo "🏁 Running tests when the greylist is deactivated ..." + elif [ "$test" = "ip" ] ; then + echo "🏁 Running tests with the network 192.168.0.0/24 in the grey list ..." + echo "ℹ️ Activating the greylist for all the future tests ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_GREYLIST: "no"@USE_GREYLIST: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_IP: ""@GREYLIST_IP: "192.168.0.0/24"@' {} \; + elif [ "$test" = "ip_urls" ] ; then + echo "🏁 Running tests with greylist's ip url set to http://greylist-api:8080/ip ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_IP: "192.168.0.0/24"@GREYLIST_IP: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_IP_URLS: ""@GREYLIST_IP_URLS: "http://greylist-api:8080/ip"@' {} \; + elif [ "$test" = "rdns" ] ; then + echo "🏁 Running tests with greylist's rdns set to .bw-services ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_IP_URLS: "http://greylist-api:8080/ip"@GREYLIST_IP_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_RDNS: ""@GREYLIST_RDNS: ".bw-services"@' {} \; + elif [ "$test" = "rdns_global" ] ; then + echo "🏴 Running tests when greylist's rdns also scans local ip addresses ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_RDNS_GLOBAL: "yes"@GREYLIST_RDNS_GLOBAL: "no"@' {} \; + elif [ "$test" = "rdns_urls" ] ; then + echo "🏁 Running tests with greylist's rdns url set to http://greylist-api:8080/rdns ..." + echo "ℹ️ Keeping the rdns also scanning local ip addresses ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_RDNS: ".bw-services"@GREYLIST_RDNS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_RDNS_URLS: ""@GREYLIST_RDNS_URLS: "http://greylist-api:8080/rdns"@' {} \; + elif [ "$test" = "asn" ] ; then + echo "🏁 Running tests with greylist's asn set to $as_number ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_RDNS_GLOBAL: "no"@GREYLIST_RDNS_GLOBAL: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_RDNS_URLS: "http://greylist-api:8080/rdns"@GREYLIST_RDNS_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_ASN: ""@GREYLIST_ASN: "'"$as_number"'"@' {} \; + elif [ "$test" = "asn_urls" ] ; then + echo "🏁 Running tests with greylist's asn url set to http://greylist-api:8080/asn ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_ASN: "'"$as_number"'"@GREYLIST_ASN: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_ASN_URLS: ""@GREYLIST_ASN_URLS: "http://greylist-api:8080/asn"@' {} \; + elif [ "$test" = "user_agent" ] ; then + echo "🏁 Running tests with greylist's user_agent set to BunkerBot ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_ASN_URLS: "http://greylist-api:8080/asn"@GREYLIST_ASN_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_USER_AGENT: ""@GREYLIST_USER_AGENT: "BunkerBot"@' {} \; + elif [ "$test" = "user_agent_urls" ] ; then + echo "🏁 Running tests with greylist's user_agent url set to http://greylist-api:8080/user_agent ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_USER_AGENT: "BunkerBot"@GREYLIST_USER_AGENT: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_USER_AGENT_URLS: ""@GREYLIST_USER_AGENT_URLS: "http://greylist-api:8080/user_agent"@' {} \; + elif [ "$test" = "uri" ] ; then + echo "🏁 Running tests with greylist's uri set to /admin ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_USER_AGENT_URLS: "http://greylist-api:8080/user_agent"@GREYLIST_USER_AGENT_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_URI: ""@GREYLIST_URI: "/admin"@' {} \; + elif [ "$test" = "uri_urls" ] ; then + echo "🏁 Running tests with greylist's uri url set to http://greylist-api:8080/uri ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_URI: "/admin"@GREYLIST_URI: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GREYLIST_URI_URLS: ""@GREYLIST_URI_URLS: "http://greylist-api:8080/uri"@' {} \; + fi + + echo "🏁 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🏁 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🏁 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("greylist-bw-1" "greylist-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🏁 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🏁 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + if ! [[ "$test" = "user_agent" || "$test" = "user_agent_urls" || "$test" = "uri" || "$test" = "uri_urls" ]] ; then + echo "🏁 Running global container tests ..." + + docker compose -f docker-compose.test.yml up global-tests --abort-on-container-exit --exit-code-from global-tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🏁 Test \"$test\" failed for global tests ❌" + echo "🛡️ Showing BunkerWeb, BunkerWeb Scheduler and Custom API logs ..." + docker compose logs bw bw-scheduler greylist-api + exit 1 + else + echo "🏁 Test \"$test\" succeeded for global tests ✅" + fi + fi + + echo "🏁 Running local container tests ..." + + docker compose -f docker-compose.test.yml up local-tests --abort-on-container-exit --exit-code-from local-tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🏁 Test \"$test\" failed for local tests ❌" + echo "🛡️ Showing BunkerWeb, BunkerWeb Scheduler and Custom API logs ..." + docker compose logs bw bw-scheduler greylist-api + exit 1 + else + echo "🏁 Test \"$test\" succeeded for local tests ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🏁 Tests are done ! ✅" diff --git a/tests/core/gzip/Dockerfile b/tests/core/gzip/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/gzip/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/gzip/docker-compose.test.yml b/tests/core/gzip/docker-compose.test.yml new file mode 100644 index 000000000..b02abf5ee --- /dev/null +++ b/tests/core/gzip/docker-compose.test.yml @@ -0,0 +1,17 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_GZIP: "no" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/gzip/docker-compose.yml b/tests/core/gzip/docker-compose.yml new file mode 100644 index 000000000..6c2a94930 --- /dev/null +++ b/tests/core/gzip/docker-compose.yml @@ -0,0 +1,67 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + USE_REVERSE_PROXY: "yes" + REVERSE_PROXY_HOST: "http://app1:8080" + REVERSE_PROXY_URL: "/" + LOG_LEVEL: "info" + + # ? GZIP settings + USE_GZIP: "no" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + app1: + image: nginxdemos/nginx-hello + networks: + bw-services: + ipv4_address: 192.168.0.4 + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/gzip/main.py b/tests/core/gzip/main.py new file mode 100644 index 000000000..da12e4455 --- /dev/null +++ b/tests/core/gzip/main.py @@ -0,0 +1,62 @@ +from contextlib import suppress +from os import getenv +from requests import RequestException, get, head +from traceback import format_exc +from time import sleep + + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_gzip = getenv("USE_GZIP", "no") == "yes" + + print( + "ℹ️ Sending a HEAD request to http://www.example.com ...", + flush=True, + ) + + response = head( + "http://www.example.com", + headers={"Host": "www.example.com", "Accept-Encoding": "gzip"}, + ) + response.raise_for_status() + + if not use_gzip and response.headers.get("Content-Encoding", "").lower() == "gzip": + print( + f"❌ Content-Encoding header is present even if Gzip is deactivated, exiting ...\nheaders: {response.headers}" + ) + exit(1) + elif use_gzip and response.headers.get("Content-Encoding", "").lower() != "gzip": + print( + f"❌ Content-Encoding header is not present or with the wrong value even if Gzip is activated, exiting ...\nheaders: {response.headers}" + ) + exit(1) + + print("✅ Gzip is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/gzip/requirements.txt b/tests/core/gzip/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/gzip/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/gzip/test.sh b/tests/core/gzip/test.sh new file mode 100755 index 000000000..d7627a554 --- /dev/null +++ b/tests/core/gzip/test.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +echo "🗜️ Building gzip stack ..." + +# Starting stack +docker compose pull bw-docker app1 +if [ $? -ne 0 ] ; then + echo "🗜️ Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🗜️ Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_GZIP: "yes"@USE_GZIP: "no"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🗜️ Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🗜️ Down failed ❌" + exit 1 + fi + + echo "🗜️ Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "deactivated" "activated" +do + if [ "$test" = "deactivated" ] ; then + echo "🗜️ Running tests without gzip ..." + elif [ "$test" = "activated" ] ; then + echo "🗜️ Running tests with gzip ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_GZIP: "no"@USE_GZIP: "yes"@' {} \; + fi + + echo "🗜️ Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🗜️ Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🗜️ Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("gzip-bw-1" "gzip-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🗜️ Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🗜️ Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🗜️ Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🗜️ Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🗜️ Tests are done ! ✅" diff --git a/tests/core/headers/Dockerfile b/tests/core/headers/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/headers/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/headers/docker-compose.test.yml b/tests/core/headers/docker-compose.test.yml new file mode 100644 index 000000000..096e82fc6 --- /dev/null +++ b/tests/core/headers/docker-compose.test.yml @@ -0,0 +1,29 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + GENERATE_SELF_SIGNED_SSL: "no" + CUSTOM_HEADER: "" + REMOVE_HEADERS: "Server X-Powered-By X-AspNet-Version X-AspNetMvc-Version" + STRICT_TRANSPORT_SECURITY: "max-age=31536000" + COOKIE_FLAGS: "* HttpOnly SameSite=Lax" + COOKIE_AUTO_SECURE_FLAG: "yes" + CONTENT_SECURITY_POLICY: "object-src 'none'; form-action 'self'; frame-ancestors 'self';" + REFERRER_POLICY: "strict-origin-when-cross-origin" + PERMISSIONS_POLICY: "accelerometer=(), ambient-light-sensor=(), autoplay=(), battery=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), fullscreen=(), geolocation=(), gyroscope=(), hid=(), idle-detection=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), serial=(), usb=(), web-share=(), xr-spatial-tracking=()" + FEATURE_POLICY: "accelerometer 'none'; ambient-light-sensor 'none'; autoplay 'none'; battery 'none'; camera 'none'; display-capture 'none'; document-domain 'none'; encrypted-media 'none'; execution-while-not-rendered 'none'; execution-while-out-of-viewport 'none'; fullscreen 'none'; geolocation 'none'; gyroscope 'none'; layout-animation 'none'; legacy-image-formats 'none'; magnetometer 'none'; microphone 'none'; midi 'none'; navigation-override 'none'; payment 'none'; picture-in-picture 'none'; publickey-credentials-get 'none'; speaker-selection 'none'; sync-xhr 'none'; unoptimized-images 'none'; unsized-media 'none'; usb 'none'; screen-wake-lock 'none'; web-share 'none'; xr-spatial-tracking 'none';" + X_FRAME_OPTIONS: "SAMEORIGIN" + X_CONTENT_TYPE_OPTIONS: "nosniff" + X_XSS_PROTECTION: "1; mode=block" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/headers/docker-compose.yml b/tests/core/headers/docker-compose.yml new file mode 100644 index 000000000..606245451 --- /dev/null +++ b/tests/core/headers/docker-compose.yml @@ -0,0 +1,83 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./www:/var/www/html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + HTTPS_PORT: "443" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + GENERATE_SELF_SIGNED_SSL: "no" + REMOTE_PHP: "bw-php" + REMOTE_PHP_PATH: "/app" + + # ? HEADERS settings + CUSTOM_HEADER: "" + REMOVE_HEADERS: "Server X-Powered-By X-AspNet-Version X-AspNetMvc-Version" + STRICT_TRANSPORT_SECURITY: "max-age=31536000" + COOKIE_FLAGS: "* HttpOnly SameSite=Lax" + COOKIE_AUTO_SECURE_FLAG: "yes" + CONTENT_SECURITY_POLICY: "object-src 'none'; form-action 'self'; frame-ancestors 'self';" + REFERRER_POLICY: "strict-origin-when-cross-origin" + PERMISSIONS_POLICY: "accelerometer=(), ambient-light-sensor=(), autoplay=(), battery=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), fullscreen=(), geolocation=(), gyroscope=(), hid=(), idle-detection=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), serial=(), usb=(), web-share=(), xr-spatial-tracking=()" + FEATURE_POLICY: "accelerometer 'none'; ambient-light-sensor 'none'; autoplay 'none'; battery 'none'; camera 'none'; display-capture 'none'; document-domain 'none'; encrypted-media 'none'; execution-while-not-rendered 'none'; execution-while-out-of-viewport 'none'; fullscreen 'none'; geolocation 'none'; gyroscope 'none'; layout-animation 'none'; legacy-image-formats 'none'; magnetometer 'none'; microphone 'none'; midi 'none'; navigation-override 'none'; payment 'none'; picture-in-picture 'none'; publickey-credentials-get 'none'; speaker-selection 'none'; sync-xhr 'none'; unoptimized-images 'none'; unsized-media 'none'; usb 'none'; screen-wake-lock 'none'; web-share 'none'; xr-spatial-tracking 'none';" + X_FRAME_OPTIONS: "SAMEORIGIN" + X_CONTENT_TYPE_OPTIONS: "nosniff" + X_XSS_PROTECTION: "1; mode=block" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + bw-php: + image: php:fpm-alpine3.17 + volumes: + - ./www:/app + networks: + bw-services: + ipv4_address: 192.168.0.4 + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/headers/main.py b/tests/core/headers/main.py new file mode 100644 index 000000000..37eb37de5 --- /dev/null +++ b/tests/core/headers/main.py @@ -0,0 +1,185 @@ +from contextlib import suppress +from os import getenv +from requests import RequestException, get, head +from traceback import format_exc +from time import sleep + + +try: + ssl = getenv("GENERATE_SELF_SIGNED_SSL", "no") == "yes" + + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + f"http{'s' if ssl else ''}://www.example.com", + headers={"Host": "www.example.com"}, + verify=False, + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + custom_headers = getenv("CUSTOM_HEADER", "") + remove_headers = getenv( + "REMOVE_HEADERS", "Server X-Powered-By X-AspNet-Version X-AspNetMvc-Version" + ) + strict_transport_security = getenv("STRICT_TRANSPORT_SECURITY", "max-age=31536000") + cookie_flags = getenv("COOKIE_FLAGS", "* HttpOnly SameSite=Lax") + cookie_flags_1 = getenv("COOKIE_FLAGS_1") + cookie_auto_secure_flag = getenv("COOKIE_AUTO_SECURE_FLAG", "yes") == "yes" + content_security_policy = getenv( + "CONTENT_SECURITY_POLICY", + "object-src 'none'; form-action 'self'; frame-ancestors 'self';", + ) + referrer_policy = getenv("REFERRER_POLICY", "strict-origin-when-cross-origin") + permissions_policy = getenv( + "PERMISSIONS_POLICY", + "accelerometer=(), ambient-light-sensor=(), autoplay=(), battery=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), fullscreen=(), geolocation=(), gyroscope=(), hid=(), idle-detection=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), serial=(), usb=(), web-share=(), xr-spatial-tracking=()", + ) + feature_policy = getenv( + "FEATURE_POLICY", + "accelerometer 'none'; ambient-light-sensor 'none'; autoplay 'none'; battery 'none'; camera 'none'; display-capture 'none'; document-domain 'none'; encrypted-media 'none'; execution-while-not-rendered 'none'; execution-while-out-of-viewport 'none'; fullscreen 'none'; geolocation 'none'; gyroscope 'none'; layout-animation 'none'; legacy-image-formats 'none'; magnetometer 'none'; microphone 'none'; midi 'none'; navigation-override 'none'; payment 'none'; picture-in-picture 'none'; publickey-credentials-get 'none'; speaker-selection 'none'; sync-xhr 'none'; unoptimized-images 'none'; unsized-media 'none'; usb 'none'; screen-wake-lock 'none'; web-share 'none'; xr-spatial-tracking 'none';", + ) + x_frame_options = getenv("X_FRAME_OPTIONS", "SAMEORIGIN") + x_content_type_options = getenv("X_CONTENT_TYPE_OPTIONS", "nosniff") + x_xss_protection = getenv("X_XSS_PROTECTION", "1; mode=block") + + print( + f"ℹ️ Sending a HEAD request to http{'s' if ssl else ''}://www.example.com ...", + flush=True, + ) + + response = head( + f"http{'s' if ssl else ''}://www.example.com", + headers={"Host": "www.example.com"}, + verify=False, + ) + response.raise_for_status() + + if custom_headers: + splitted = custom_headers.split(":") + + if response.headers.get(splitted[0].strip()) != splitted[1].strip(): + print( + f"❌ Header {splitted[0].strip()} is not set to {splitted[1].strip()}, exiting ...\nheaders: {response.headers}", + flush=True, + ) + exit(1) + elif "Server" not in remove_headers and "Server" not in response.headers: + print( + f'❌ Header "Server" is not removed, exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + elif ( + ssl + and response.headers.get("Strict-Transport-Security") + != strict_transport_security + ): + print( + f'❌ Header "Strict-Transport-Security" doesn\'t have the right value. {response.headers.get("Strict-Transport-Security", "missing header")} (header) != {strict_transport_security} (env), exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + elif not ssl and "Strict-Transport-Security" in response.headers: + print( + f'❌ Header "Strict-Transport-Security" is present even though ssl is deactivated, exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + elif response.headers.get("Content-Security-Policy") != content_security_policy: + print( + f'❌ Header "Content-Security-Policy" doesn\'t have the right value. {response.headers.get("Content-Security-Policy", "missing header")} (header) != {content_security_policy} (env), exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + elif response.headers.get("Referrer-Policy") != referrer_policy: + print( + f'❌ Header "Referrer-Policy" doesn\'t have the right value. {response.headers.get("Referrer-Policy", "missing header")} (header) != {referrer_policy} (env), exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + elif response.headers.get("Permissions-Policy") != permissions_policy: + print( + f'❌ Header "Permissions-Policy" doesn\'t have the right value. {response.headers.get("Permissions-Policy", "missing header")} (header) != {permissions_policy} (env), exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + elif response.headers.get("Feature-Policy") != feature_policy: + print( + f'❌ Header "Feature-Policy" doesn\'t have the right value. {response.headers.get("Feature-Policy", "missing header")} (header) != {feature_policy} (env), exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + elif response.headers.get("X-Frame-Options") != x_frame_options: + print( + f'❌ Header "X-Frame-Options" doesn\'t have the right value. {response.headers.get("X-Frame-Options", "missing header")} (header) != {x_frame_options} (env), exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + elif response.headers.get("X-Content-Type-Options") != x_content_type_options: + print( + f'❌ Header "X-Content-Type-Options" doesn\'t have the right value. {response.headers.get("X-Content-Type-Options", "missing header")} (header) != {x_content_type_options} (env), exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + elif response.headers.get("X-XSS-Protection") != x_xss_protection: + print( + f'❌ Header "X-XSS-Protection" doesn\'t have the right value. {response.headers.get("X-XSS-Protection", "missing header")} (header) != {x_xss_protection} (env), exiting ...\nheaders: {response.headers}', + flush=True, + ) + exit(1) + + if not response.cookies: + print("❌ No cookies were set, exiting ...", flush=True) + exit(1) + + cookie = next(iter(response.cookies)) + + # Iterate over the cookies and print their values and flags + if ssl and cookie_auto_secure_flag and not cookie.secure: + print( + f"❌ Cookie {cookie.name} doesn't have the secure flag, exiting ...\ncookie: name = {cookie.name}, secure = {cookie.secure}, HttpOnly = {cookie.has_nonstandard_attr('HttpOnly')}", + ) + exit(1) + elif (not ssl or not cookie_auto_secure_flag) and cookie.secure: + print( + f"❌ Cookie {cookie.name} has the secure flag even though it's not supposed to, exiting ...\ncookie: name = {cookie.name}, secure = {cookie.secure}, HttpOnly = {cookie.has_nonstandard_attr('HttpOnly')}", + ) + exit(1) + elif "HttpOnly" not in cookie_flags and cookie.has_nonstandard_attr("HttpOnly"): + print( + f"❌ Cookie {cookie.name} has the HttpOnly flag even though it's not supposed to, exiting ...\ncookie: name = {cookie.name}, secure = {cookie.secure}, HttpOnly = {cookie.has_nonstandard_attr('HttpOnly')}", + ) + exit(1) + elif ( + not cookie_flags_1 + and "HttpOnly" in cookie_flags + and not cookie.has_nonstandard_attr("HttpOnly") + ): + print( + f"❌ Cookie {cookie.name} doesn't have the HttpOnly flag even though it's set in the env, exiting ...\ncookie: name = {cookie.name}, secure = {cookie.secure}, HttpOnly = {cookie.has_nonstandard_attr('HttpOnly')}", + ) + exit(1) + + print("✅ Headers are working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/headers/requirements.txt b/tests/core/headers/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/headers/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/headers/test.sh b/tests/core/headers/test.sh new file mode 100755 index 000000000..6df8f142e --- /dev/null +++ b/tests/core/headers/test.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +echo "🎛️ Building headers stack ..." + +# Starting stack +docker compose pull bw-docker bw-php +if [ $? -ne 0 ] ; then + echo "🎛️ Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🎛️ Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@CUSTOM_HEADER: "X-Test: test"@CUSTOM_HEADER: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@REMOVE_HEADERS: ".*"$@REMOVE_HEADERS: "Server X-Powered-By X-AspNet-Version X-AspNetMvc-Version"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@STRICT_TRANSPORT_SECURITY: "max-age=86400"@STRICT_TRANSPORT_SECURITY: "max-age=31536000"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@COOKIE_FLAGS: ".*"$@COOKIE_FLAGS: "* HttpOnly SameSite=Lax"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GENERATE_SELF_SIGNED_SSL: "yes"@GENERATE_SELF_SIGNED_SSL: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@COOKIE_AUTO_SECURE_FLAG: "no"@COOKIE_AUTO_SECURE_FLAG: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CONTENT_SECURITY_POLICY: ".*"$@CONTENT_SECURITY_POLICY: "object-src '"'"'none'"'"'; form-action '"'"'self'"'"'; frame-ancestors '"'"'self'"'"';"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@REFERRER_POLICY: "no-referrer"@REFERRER_POLICY: "strict-origin-when-cross-origin"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@PERMISSIONS_POLICY: ".*"$@PERMISSIONS_POLICY: "accelerometer=(), ambient-light-sensor=(), autoplay=(), battery=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), fullscreen=(), geolocation=(), gyroscope=(), hid=(), idle-detection=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), serial=(), usb=(), web-share=(), xr-spatial-tracking=()"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@FEATURE_POLICY: ".*"$@FEATURE_POLICY: "accelerometer '"'"'none'"'"'; ambient-light-sensor '"'"'none'"'"'; autoplay '"'"'none'"'"'; battery '"'"'none'"'"'; camera '"'"'none'"'"'; display-capture '"'"'none'"'"'; document-domain '"'"'none'"'"'; encrypted-media '"'"'none'"'"'; execution-while-not-rendered '"'"'none'"'"'; execution-while-out-of-viewport '"'"'none'"'"'; fullscreen '"'"'none'"'"'; geolocation '"'"'none'"'"'; gyroscope '"'"'none'"'"'; layout-animation '"'"'none'"'"'; legacy-image-formats '"'"'none'"'"'; magnetometer '"'"'none'"'"'; microphone '"'"'none'"'"'; midi '"'"'none'"'"'; navigation-override '"'"'none'"'"'; payment '"'"'none'"'"'; picture-in-picture '"'"'none'"'"'; publickey-credentials-get '"'"'none'"'"'; speaker-selection '"'"'none'"'"'; sync-xhr '"'"'none'"'"'; unoptimized-images '"'"'none'"'"'; unsized-media '"'"'none'"'"'; usb '"'"'none'"'"'; screen-wake-lock '"'"'none'"'"'; web-share '"'"'none'"'"'; xr-spatial-tracking '"'"'none'"'"';"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@X_FRAME_OPTIONS: "DENY"@X_FRAME_OPTIONS: "SAMEORIGIN"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@X_CONTENT_TYPE_OPTIONS: ""@X_CONTENT_TYPE_OPTIONS: "nosniff"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@X_XSS_PROTECTION: "0"@X_XSS_PROTECTION: "1; mode=block"@' {} \; + + if [[ $(sed '27!d' docker-compose.yml) = ' COOKIE_FLAGS_1: "bw_cookie SameSite=Lax"' ]] ; then + sed -i '27d' docker-compose.yml + fi + + if [[ $(sed '13!d' docker-compose.test.yml) = ' COOKIE_FLAGS_1: "bw_cookie SameSite=Lax"' ]] ; then + sed -i '13d' docker-compose.test.yml + fi + + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🎛️ Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🎛️ Down failed ❌" + exit 1 + fi + + echo "🎛️ Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "without_ssl" "no_httponly_flag" "multiple_no_httponly_flag" "with_ssl" "no_cookie_auto_secure_flag" +do + if [ "$test" = "without_ssl" ] ; then + echo "🎛️ Running tests without ssl and with tweaked settings ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@CUSTOM_HEADER: ""@CUSTOM_HEADER: "X-Test: test"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@REMOVE_HEADERS: ".*"$@REMOVE_HEADERS: "X-Powered-By X-AspNet-Version X-AspNetMvc-Version"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@STRICT_TRANSPORT_SECURITY: "max-age=31536000"@STRICT_TRANSPORT_SECURITY: "max-age=86400"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CONTENT_SECURITY_POLICY: ".*"$@CONTENT_SECURITY_POLICY: "object-src '"'"'none'"'"'; frame-ancestors '"'"'self'"'"';"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@REFERRER_POLICY: "strict-origin-when-cross-origin"@REFERRER_POLICY: "no-referrer"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@PERMISSIONS_POLICY: ".*"$@PERMISSIONS_POLICY: "geolocation=(self), microphone=()"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@FEATURE_POLICY: ".*"$@FEATURE_POLICY: "geolocation '"'"'self'"'"'; microphone '"'"'none'"'"';"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@X_FRAME_OPTIONS: "SAMEORIGIN"@X_FRAME_OPTIONS: "DENY"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@X_CONTENT_TYPE_OPTIONS: "nosniff"@X_CONTENT_TYPE_OPTIONS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@X_XSS_PROTECTION: "1; mode=block"@X_XSS_PROTECTION: "0"@' {} \; + elif [ "$test" = "no_httponly_flag" ] ; then + echo "🎛️ Running tests without HttpOnly flag for cookies and with default values ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@COOKIE_FLAGS: ".*"$@COOKIE_FLAGS: "* SameSite=Lax"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CUSTOM_HEADER: "X-Test: test"@CUSTOM_HEADER: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@REMOVE_HEADERS: ".*"$@REMOVE_HEADERS: "Server X-Powered-By X-AspNet-Version X-AspNetMvc-Version"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@STRICT_TRANSPORT_SECURITY: "max-age=86400"@STRICT_TRANSPORT_SECURITY: "max-age=31536000"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@GENERATE_SELF_SIGNED_SSL: "yes"@GENERATE_SELF_SIGNED_SSL: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@CONTENT_SECURITY_POLICY: ".*"$@CONTENT_SECURITY_POLICY: "object-src '"'"'none'"'"'; form-action '"'"'self'"'"'; frame-ancestors '"'"'self'"'"';"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@REFERRER_POLICY: "no-referrer"@REFERRER_POLICY: "strict-origin-when-cross-origin"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@PERMISSIONS_POLICY: ".*"$@PERMISSIONS_POLICY: "accelerometer=(), ambient-light-sensor=(), autoplay=(), battery=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), fullscreen=(), geolocation=(), gyroscope=(), hid=(), idle-detection=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), serial=(), usb=(), web-share=(), xr-spatial-tracking=()"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@FEATURE_POLICY: ".*"$@FEATURE_POLICY: "accelerometer '"'"'none'"'"'; ambient-light-sensor '"'"'none'"'"'; autoplay '"'"'none'"'"'; battery '"'"'none'"'"'; camera '"'"'none'"'"'; display-capture '"'"'none'"'"'; document-domain '"'"'none'"'"'; encrypted-media '"'"'none'"'"'; execution-while-not-rendered '"'"'none'"'"'; execution-while-out-of-viewport '"'"'none'"'"'; fullscreen '"'"'none'"'"'; geolocation '"'"'none'"'"'; gyroscope '"'"'none'"'"'; layout-animation '"'"'none'"'"'; legacy-image-formats '"'"'none'"'"'; magnetometer '"'"'none'"'"'; microphone '"'"'none'"'"'; midi '"'"'none'"'"'; navigation-override '"'"'none'"'"'; payment '"'"'none'"'"'; picture-in-picture '"'"'none'"'"'; publickey-credentials-get '"'"'none'"'"'; speaker-selection '"'"'none'"'"'; sync-xhr '"'"'none'"'"'; unoptimized-images '"'"'none'"'"'; unsized-media '"'"'none'"'"'; usb '"'"'none'"'"'; screen-wake-lock '"'"'none'"'"'; web-share '"'"'none'"'"'; xr-spatial-tracking '"'"'none'"'"';"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@X_FRAME_OPTIONS: "DENY"@X_FRAME_OPTIONS: "SAMEORIGIN"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@X_CONTENT_TYPE_OPTIONS: ""@X_CONTENT_TYPE_OPTIONS: "nosniff"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@X_XSS_PROTECTION: "0"@X_XSS_PROTECTION: "1; mode=block"@' {} \; + elif [ "$test" = "multiple_no_httponly_flag" ] ; then + echo "🎛️ Running tests with HttpOnly flag overriden for cookie \"bw_cookie\" and default cookies flags ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@COOKIE_FLAGS: ".*"$@COOKIE_FLAGS: "* HttpOnly SameSite=Lax"@' {} \; + sed -i '27i \ COOKIE_FLAGS_1: "bw_cookie SameSite=Lax"' docker-compose.yml + sed -i '13i \ COOKIE_FLAGS_1: "bw_cookie SameSite=Lax"' docker-compose.test.yml + elif [ "$test" = "with_ssl" ] ; then + echo "🎛️ Running tests with ssl ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GENERATE_SELF_SIGNED_SSL: "no"@GENERATE_SELF_SIGNED_SSL: "yes"@' {} \; + sed -i '27d' docker-compose.yml + sed -i '13d' docker-compose.test.yml + elif [ "$test" = "no_cookie_auto_secure_flag" ] ; then + echo "🎛️ Running tests without cookie_auto_secure_flag ..." + echo "ℹ️ Keeping the generated self-signed SSL certificate" + find . -type f -name 'docker-compose.*' -exec sed -i 's@COOKIE_AUTO_SECURE_FLAG: "yes"@COOKIE_AUTO_SECURE_FLAG: "no"@' {} \; + fi + + echo "🎛️ Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🎛️ Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🎛️ Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("headers-bw-1" "headers-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🎛️ Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🎛️ Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🎛️ Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🎛️ Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🎛️ Tests are done ! ✅" diff --git a/tests/core/headers/www/index.php b/tests/core/headers/www/index.php new file mode 100644 index 000000000..c66d0c75b --- /dev/null +++ b/tests/core/headers/www/index.php @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/tests/core/inject/Dockerfile b/tests/core/inject/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/inject/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/inject/docker-compose.test.yml b/tests/core/inject/docker-compose.test.yml new file mode 100644 index 000000000..1849f9f77 --- /dev/null +++ b/tests/core/inject/docker-compose.test.yml @@ -0,0 +1,17 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + INJECT_BODY: "TEST" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/inject/docker-compose.yml b/tests/core/inject/docker-compose.yml new file mode 100644 index 000000000..15686e92f --- /dev/null +++ b/tests/core/inject/docker-compose.yml @@ -0,0 +1,60 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? INJECT settings + INJECT_BODY: "TEST" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/inject/index.html b/tests/core/inject/index.html new file mode 100644 index 000000000..c7806182b --- /dev/null +++ b/tests/core/inject/index.html @@ -0,0 +1,5 @@ + + + EMPTY PAGE + + diff --git a/tests/core/inject/main.py b/tests/core/inject/main.py new file mode 100644 index 000000000..95e9ec9d2 --- /dev/null +++ b/tests/core/inject/main.py @@ -0,0 +1,52 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + inject_body = getenv("INJECT_BODY", "") + + page_text = get("http://www.example.com", headers={"Host": "www.example.com"}).text + + if inject_body not in page_text: + print( + f"❌ The service is ready but the injected body is not present, exiting ...", + flush=True, + ) + exit(1) + + print( + "✅ The service is ready and the injected body is present, exiting ...", + flush=True, + ) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/inject/requirements.txt b/tests/core/inject/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/inject/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/inject/test.sh b/tests/core/inject/test.sh new file mode 100755 index 000000000..190f1c55c --- /dev/null +++ b/tests/core/inject/test.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +echo "💉 Building inject stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "💉 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "💉 Build failed ❌" + exit 1 +fi + +cleanup_stack () { + echo "💉 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "💉 Down failed ❌" + exit 1 + fi + + echo "💉 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +echo "💉 Running tests while injecting TEST into the HTML page ..." + +echo "💉 Starting stack ..." +docker compose up -d 2>/dev/null +if [ $? -ne 0 ] ; then + echo "💉 Up failed ❌" + exit 1 +fi + +# Check if stack is healthy +echo "💉 Waiting for stack to be healthy ..." +i=0 +while [ $i -lt 120 ] ; do + containers=("inject-bw-1" "inject-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "💉 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) +done +if [ $i -ge 120 ] ; then + docker compose logs + echo "💉 Docker stack is not healthy ❌" + exit 1 +fi + +# Start tests + +docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + +if [ $? -ne 0 ] ; then + echo "💉 Test \"inject\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 +else + echo "💉 Test \"inject\" succeeded ✅" +fi + +echo "💉 Tests are done ! ✅" diff --git a/tests/core/limit/Dockerfile b/tests/core/limit/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/limit/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/limit/docker-compose.test.yml b/tests/core/limit/docker-compose.test.yml new file mode 100644 index 000000000..2b8f3946d --- /dev/null +++ b/tests/core/limit/docker-compose.test.yml @@ -0,0 +1,21 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_LIMIT_REQ: "no" + LIMIT_REQ_URL: "/" + LIMIT_REQ_RATE: "2r/s" + USE_LIMIT_CONN: "yes" + LIMIT_CONN_MAX_HTTP1: "1" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/limit/docker-compose.yml b/tests/core/limit/docker-compose.yml new file mode 100644 index 000000000..e50cba098 --- /dev/null +++ b/tests/core/limit/docker-compose.yml @@ -0,0 +1,64 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? LIMIT settings + USE_LIMIT_REQ: "no" + LIMIT_REQ_URL: "/" + LIMIT_REQ_RATE: "2r/s" + USE_LIMIT_CONN: "yes" + LIMIT_CONN_MAX_HTTP1: "1" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/limit/index.html b/tests/core/limit/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/limit/main.py b/tests/core/limit/main.py new file mode 100644 index 000000000..fbaf214d5 --- /dev/null +++ b/tests/core/limit/main.py @@ -0,0 +1,178 @@ +from asyncio import Semaphore, gather, run +from httpx import AsyncClient, Client +from os import getenv +from time import time +from traceback import format_exc + + +try: + use_limit_request = getenv("USE_LIMIT_REQ", "yes") == "yes" + limit_req_url = getenv("LIMIT_REQ_URL", "/") + limit_req_rate = getenv("LIMIT_REQ_RATE", "2r/s") + limit_req_url_1 = getenv("LIMIT_REQ_URL_1") + limit_req_rate_1 = getenv("LIMIT_REQ_RATE_1") + + use_limit_conn = getenv("USE_LIMIT_CONN", "yes") == "yes" + limit_conn_http1 = getenv("LIMIT_CONN_MAX_HTTP1", "1") + + if not limit_conn_http1.isdigit(): + print("❌ LIMIT_CONN_MAX_HTTP1 is not a number, exiting ...", flush=True) + exit(1) + + limit_conn_http1 = int(limit_conn_http1) + + async def fetch(domain, semaphore): + async with semaphore: + async with AsyncClient() as client: + return await client.get(f"http://{domain}", headers={"Host": domain}) + + async def main(limit) -> list: + domains = ["www.example.com"] * limit + semaphore = Semaphore(limit) + tasks = [fetch(domain, semaphore) for domain in domains] + return await gather(*tasks) + + if use_limit_conn: + print( + f"ℹ️ Making requests to the service with HTTP/1.1 with the limit_conn_http1 directive set to {limit_conn_http1} ...", + ) + + responses = run(main(5)) + + if any(response.status_code >= 500 for response in responses): + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + elif ( + not any(response.status_code == 429 for response in responses) + and limit_conn_http1 == 1 + ): + print( + f"❌ The limit_conn for HTTP1 directive is not working correctly, the limit was set to {limit_conn_http1} and the limit was not reached with 5 simultaneous connections, exiting ...", + flush=True, + ) + exit(1) + elif ( + any(response.status_code == 429 for response in responses) + and limit_conn_http1 > 1 + ): + print( + f"❌ The limit_conn for HTTP1 directive is not working correctly, the limit was set to {limit_conn_http1} and the limit was reached with 5 simultaneous connections, exiting ...", + flush=True, + ) + exit(1) + + print( + f"✅ The limit_conn for HTTP1 directive is working correctly, the limit was set to {limit_conn_http1} and the limit was reached with 5 simultaneous connections", + flush=True, + ) + exit(0) + + start = time() + status_code = 200 + request_number = 0 + stopped = False + + print( + "ℹ️ Sending requests to the service until it reaches the limit ...", flush=True + ) + + while status_code != 429: + with Client() as client: + status_code = client.get( + f"http://www.example.com{limit_req_url}", + headers={"Host": "www.example.com"}, + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + request_number += 1 + + if request_number >= 20: + stopped = True + break + total = time() - start + + if use_limit_request and stopped: + print( + f"❌ The limit_req directive is not working correctly, the limit was not reached in 20 requests in {total:.2f}s, exiting ...", + flush=True, + ) + exit(1) + elif not use_limit_request and not stopped: + print( + f"❌ The limit_req directive is not working correctly, {request_number} requests were made in {total:.2f}s to reach the limit, exiting ...", + flush=True, + ) + exit(1) + elif not use_limit_request and stopped: + print( + f"✅ The limit_req directive was successfully disabled, {request_number} requests were made in {total:.2f}s and the limit was not reached", + flush=True, + ) + exit(0) + elif request_number != int(limit_req_rate[:-3]) + 1: + print( + f"❌ The limit_req directive is not working correctly, {request_number} requests were made in {total:.2f}s while the limit was set to {limit_req_rate}, exiting ...", + flush=True, + ) + exit(1) + + print( + f"✅ The limit_req directive is working correctly, {request_number} requests were made in {total:.2f}s to reach the limit", + flush=True, + ) + + if limit_req_url_1: + start = time() + status_code = 200 + request_number = 0 + stopped = False + + print( + f"ℹ️ Sending requests to the url {limit_req_url_1} until it reaches the limit ...", + flush=True, + ) + + while status_code != 429: + with Client() as client: + status_code = client.get( + f"http://www.example.com{limit_req_url_1}", + headers={"Host": "www.example.com"}, + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + request_number += 1 + + if request_number >= 20: + stopped = True + break + total = time() - start + + if stopped: + print( + f"❌ The limit_req_1 directive is not working correctly, the limit was not reached in 20 requests in {total:.2f}s, exiting ...", + flush=True, + ) + exit(1) + elif request_number != int(limit_req_rate_1[:-3]) + 1: + print( + f"❌ The limit_req_1 directive is not working correctly, {request_number} requests were made in {total:.2f}s while the limit was set to {limit_req_rate}, exiting ...", + flush=True, + ) + exit(1) + + print( + f"✅ The limit_req_1 directive is working correctly, {request_number} requests were made in {total:.2f}s to reach the limit", + flush=True, + ) + exit(0) +except SystemExit as e: + exit(e.code) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/limit/requirements.txt b/tests/core/limit/requirements.txt new file mode 100644 index 000000000..024d27a80 --- /dev/null +++ b/tests/core/limit/requirements.txt @@ -0,0 +1 @@ +httpx==0.24.0 diff --git a/tests/core/limit/test.sh b/tests/core/limit/test.sh new file mode 100755 index 000000000..4856c5c4e --- /dev/null +++ b/tests/core/limit/test.sh @@ -0,0 +1,140 @@ +#!/bin/bash + +echo "🎚️ Building limit stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🎚️ Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🎚️ Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_LIMIT_REQ: "yes"@USE_LIMIT_REQ: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@LIMIT_REQ_URL: ".*"$@LIMIT_REQ_URL: "/"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@LIMIT_REQ_RATE: ".*"$@LIMIT_REQ_RATE: "2r/s"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_LIMIT_CONN: "no"@USE_LIMIT_CONN: "yes"@' {} \; + + if [[ $(sed '22!d' docker-compose.yml) = ' LIMIT_REQ_URL_1: "/custom"' ]] ; then + sed -i '22d' docker-compose.yml + fi + + if [[ $(sed '22!d' docker-compose.yml) = ' LIMIT_REQ_RATE_1: "4r/s"' ]] ; then + sed -i '22d' docker-compose.yml + fi + + if [[ $(sed '11!d' docker-compose.test.yml) = ' LIMIT_REQ_URL_1: "/custom"' ]] ; then + sed -i '11d' docker-compose.test.yml + fi + + if [[ $(sed '11!d' docker-compose.test.yml) = ' LIMIT_REQ_RATE_1: "4r/s"' ]] ; then + sed -i '11d' docker-compose.test.yml + fi + + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🎚️ Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🎚️ Down failed ❌" + exit 1 + fi + + echo "🎚️ Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "http1" "limit_req" "augmented" "custom_endpoint_rate" "deactivated_req" +do + if [ "$test" = "http1" ] ; then + echo "🎚️ Running tests with limit conn activated and the limit conn max http1 set to 1 ..." + elif [ "$test" = "limit_req" ] ; then + echo "🎚️ Running tests with limit req activated ..." + echo "ℹ️ Deactivating limit conn ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_LIMIT_CONN: "yes"@USE_LIMIT_CONN: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_LIMIT_REQ: "no"@USE_LIMIT_REQ: "yes"@' {} \; + elif [ "$test" = "augmented" ] ; then + echo "🎚️ Running tests with limit req rate set to 10r/s ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@LIMIT_REQ_RATE: ".*"$@LIMIT_REQ_RATE: "10r/s"@' {} \; + elif [ "$test" = "custom_endpoint_rate" ] ; then + echo "🎚️ Running tests with a custom endpoint rate ..." + sed -i '22i \ LIMIT_REQ_URL_1: "/custom"' docker-compose.yml + sed -i '23i \ LIMIT_REQ_RATE_1: "4r/s"' docker-compose.yml + sed -i '11i \ LIMIT_REQ_URL_1: "/custom"' docker-compose.test.yml + sed -i '12i \ LIMIT_REQ_RATE_1: "4r/s"' docker-compose.test.yml + elif [ "$test" = "deactivated_req" ] ; then + echo "🎚️ Running tests without limit req ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_LIMIT_REQ: "yes"@USE_LIMIT_REQ: "no"@' {} \; + fi + + echo "🎚️ Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🎚️ Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🎚️ Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("limit-bw-1" "limit-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🎚️ Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🎚️ Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🎚️ Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🎚️ Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🎚️ Tests are done ! ✅" diff --git a/tests/core/misc/test.sh b/tests/core/misc/test.sh new file mode 100644 index 000000000..f87f5c14c --- /dev/null +++ b/tests/core/misc/test.sh @@ -0,0 +1 @@ +# TODO \ No newline at end of file diff --git a/tests/core/modsecurity/Dockerfile b/tests/core/modsecurity/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/modsecurity/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/modsecurity/docker-compose.test.yml b/tests/core/modsecurity/docker-compose.test.yml new file mode 100644 index 000000000..0e178da09 --- /dev/null +++ b/tests/core/modsecurity/docker-compose.test.yml @@ -0,0 +1,21 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_MODSECURITY: "yes" + USE_MODSECURITY_CRS: "yes" + MODSECURITY_SEC_AUDIT_ENGINE: "RelevantOnly" + MODSECURITY_SEC_RULE_ENGINE: "On" + MODSECURITY_SEC_AUDIT_LOG_PARTS: "ABCFHZ" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/modsecurity/docker-compose.yml b/tests/core/modsecurity/docker-compose.yml new file mode 100644 index 000000000..32599202d --- /dev/null +++ b/tests/core/modsecurity/docker-compose.yml @@ -0,0 +1,64 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? MODECURITY settings + USE_MODSECURITY: "yes" + USE_MODSECURITY_CRS: "yes" + MODSECURITY_SEC_AUDIT_ENGINE: "RelevantOnly" + MODSECURITY_SEC_RULE_ENGINE: "On" + MODSECURITY_SEC_AUDIT_LOG_PARTS: "ABCFHZ" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/modsecurity/index.html b/tests/core/modsecurity/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/modsecurity/main.py b/tests/core/modsecurity/main.py new file mode 100644 index 000000000..2492c1db9 --- /dev/null +++ b/tests/core/modsecurity/main.py @@ -0,0 +1,70 @@ +from contextlib import suppress +from datetime import datetime +from os import getenv +from requests import get +from requests.exceptions import RequestException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_modsecurity = getenv("USE_MODSECURITY", "yes") == "yes" + use_modsecurity_crs = getenv("USE_MODSECURITY_CRS", "yes") == "yes" + + print( + "ℹ️ Sending a requests to http://www.example.com/?id=/etc/passwd ...", + flush=True, + ) + + status_code = get( + "http://www.example.com/?id=/etc/passwd", headers={"Host": "www.example.com"} + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code == 403: + if not use_modsecurity: + print( + "❌ ModSecurity should have been disabled, exiting ...", + flush=True, + ) + exit(1) + elif not use_modsecurity_crs: + print( + "❌ ModSecurity CRS should have been disabled, exiting ...", + flush=True, + ) + exit(1) + elif use_modsecurity and use_modsecurity_crs: + print("❌ ModSecurity is not working as expected, exiting ...", flush=True) + exit(1) + + print("✅ ModSecurity is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/modsecurity/requirements.txt b/tests/core/modsecurity/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/modsecurity/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/modsecurity/test.sh b/tests/core/modsecurity/test.sh new file mode 100755 index 000000000..bfced6eaf --- /dev/null +++ b/tests/core/modsecurity/test.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +echo "👮 Building modsecurity stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "👮 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "👮 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_MODSECURITY: "no"@USE_MODSECURITY: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_MODSECURITY_CRS: "no"@USE_MODSECURITY_CRS: "yes"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "👮 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "👮 Down failed ❌" + exit 1 + fi + + echo "👮 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "activated" "crs_deactivated" "deactivated" +do + if [ "$test" = "activated" ] ; then + echo "👮 Running tests with modsecurity activated ..." + elif [ "$test" = "crs_deactivated" ] ; then + echo "👮 Running tests without the CRS ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_MODSECURITY_CRS: "yes"@USE_MODSECURITY_CRS: "no"@' {} \; + elif [ "$test" = "deactivated" ] ; then + echo "👮 Running tests without modsecurity ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_MODSECURITY_CRS: "no"@USE_MODSECURITY_CRS: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_MODSECURITY: "yes"@USE_MODSECURITY: "no"@' {} \; + fi + + echo "👮 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "👮 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "👮 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("modsecurity-bw-1" "modsecurity-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "👮 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "👮 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "👮 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "👮 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "👮 Tests are done ! ✅" diff --git a/tests/core/redirect/Dockerfile b/tests/core/redirect/Dockerfile new file mode 100644 index 000000000..c6b6dd4bc --- /dev/null +++ b/tests/core/redirect/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11.3-alpine + +# Install firefox and geckodriver +RUN apk add --no-cache --virtual .build-deps curl grep zip && \ + apk add --no-cache firefox + +# Installing geckodriver for firefox... +RUN GECKODRIVER_VERSION=`curl -i https://github.com/mozilla/geckodriver/releases/latest | grep -Po 'v[0-9]+.[0-9]+.[0-9]+'` && \ + wget -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \ + tar -C /usr/local/bin -xzvf geckodriver.tar.gz && \ + chmod +x /usr/local/bin/geckodriver && \ + rm geckodriver.tar.gz + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/redirect/docker-compose.test.yml b/tests/core/redirect/docker-compose.test.yml new file mode 100644 index 000000000..36755d5cb --- /dev/null +++ b/tests/core/redirect/docker-compose.test.yml @@ -0,0 +1,18 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + REDIRECT_TO: "" + REDIRECT_TO_REQUEST_URI: "no" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/redirect/docker-compose.yml b/tests/core/redirect/docker-compose.yml new file mode 100644 index 000000000..adfc45e5a --- /dev/null +++ b/tests/core/redirect/docker-compose.yml @@ -0,0 +1,70 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + ports: + - 80:80 + labels: + - "bunkerweb.INSTANCE" + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + USE_REVERSE_PROXY: "yes" + REVERSE_PROXY_HOST: "http://app1:8080" + REVERSE_PROXY_URL: "/" + LOG_LEVEL: "info" + + # ? REDIRECT settings + REDIRECT_TO: "" + REDIRECT_TO_REQUEST_URI: "no" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + app1: + image: nginxdemos/nginx-hello + networks: + bw-services: + ipv4_address: 192.168.0.4 + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/redirect/main.py b/tests/core/redirect/main.py new file mode 100644 index 000000000..f0182cc5f --- /dev/null +++ b/tests/core/redirect/main.py @@ -0,0 +1,64 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.common.exceptions import NoSuchElementException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + firefox_options = Options() + firefox_options.add_argument("--headless") + + redirect_to = getenv("REDIRECT_TO", "") + redirect_to_request_uri = getenv("REDIRECT_TO_REQUEST_URI", "no") + + print("ℹ️ Starting Firefox ...", flush=True) + with webdriver.Firefox(options=firefox_options) as driver: + driver.delete_all_cookies() + driver.maximize_window() + + print("ℹ️ Navigating to http://www.example.com/test ...", flush=True) + driver.get("http://www.example.com/test") + + if redirect_to_request_uri == "yes": + redirect_to += "/test" + + if not driver.current_url == redirect_to: + print( + f"❌ Expected redirect to {redirect_to}, got {driver.current_url} instead, exiting ...", + flush=True, + ) + exit(1) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/redirect/requirements.txt b/tests/core/redirect/requirements.txt new file mode 100644 index 000000000..6f7b13f79 --- /dev/null +++ b/tests/core/redirect/requirements.txt @@ -0,0 +1,2 @@ +requests==2.30.0 +selenium==4.9.1 diff --git a/tests/core/redirect/test.sh b/tests/core/redirect/test.sh new file mode 100755 index 000000000..b8c3159de --- /dev/null +++ b/tests/core/redirect/test.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +echo "↩️ Building redirect stack ..." + +# Starting stack +docker compose pull bw-docker app1 +if [ $? -ne 0 ] ; then + echo "↩️ Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "↩️ Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIRECT_TO: "http://brightlushsilveryawn\.neverssl\.com/online/"@REDIRECT_TO: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIRECT_TO_REQUEST_URI: "yes"@REDIRECT_TO_REQUEST_URI: "no"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "↩️ Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "↩️ Down failed ❌" + exit 1 + fi + + echo "↩️ Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "redirect" "redirect_uri" +do + if [ "$test" = "redirect" ] ; then + echo "↩️ Running tests when redirecting to http://brightlushsilveryawn.neverssl.com/online/ ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIRECT_TO: ""@REDIRECT_TO: "http://brightlushsilveryawn.neverssl.com/online/"@' {} \; + elif [ "$test" = "redirect_uri" ] ; then + echo "↩️ Running tests when redirecting to uri test ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIRECT_TO_REQUEST_URI: "no"@REDIRECT_TO_REQUEST_URI: "yes"@' {} \; + fi + + echo "↩️ Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "↩️ Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "↩️ Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("redirect-bw-1" "redirect-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "↩️ Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "↩️ Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "↩️ Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "↩️ Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "↩️ Tests are done ! ✅" diff --git a/tests/core/redis/test.sh b/tests/core/redis/test.sh new file mode 100644 index 000000000..f87f5c14c --- /dev/null +++ b/tests/core/redis/test.sh @@ -0,0 +1 @@ +# TODO \ No newline at end of file diff --git a/tests/core/reversescan/Dockerfile b/tests/core/reversescan/Dockerfile new file mode 100644 index 000000000..47759b595 --- /dev/null +++ b/tests/core/reversescan/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11.3-alpine + +# Install firefox and geckodriver +RUN apk add --no-cache --virtual .build-deps curl grep zip && \ + apk add --no-cache firefox + +# Installing geckodriver for firefox... +RUN GECKODRIVER_VERSION=`curl -i https://github.com/mozilla/geckodriver/releases/latest | grep -Po 'v[0-9]+.[0-9]+.[0-9]+'` && \ + wget -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \ + tar -C /usr/local/bin -xzvf geckodriver.tar.gz && \ + chmod +x /usr/local/bin/geckodriver && \ + rm geckodriver.tar.gz + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +EXPOSE 80 + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/reversescan/docker-compose.test.yml b/tests/core/reversescan/docker-compose.test.yml new file mode 100644 index 000000000..cd6f71b8b --- /dev/null +++ b/tests/core/reversescan/docker-compose.test.yml @@ -0,0 +1,19 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + USE_REVERSE_SCAN: "yes" + REVERSE_SCAN_PORTS: "22 80 443 3128 8000 8080" + REVERSE_SCAN_TIMEOUT: "500" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/reversescan/docker-compose.yml b/tests/core/reversescan/docker-compose.yml new file mode 100644 index 000000000..5cf42bdd1 --- /dev/null +++ b/tests/core/reversescan/docker-compose.yml @@ -0,0 +1,65 @@ +version: "3.5" + +services: + bw: + # image: bunkerity/bunkerweb:1.5.0-beta + # pull_policy: never + build: + context: /home/theophile/dev/bunkerweb + dockerfile: src/bw/Dockerfile + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? REVERSE_SCAN settings + USE_REVERSE_SCAN: "yes" + REVERSE_SCAN_PORTS: "22 80 443 3128 8000 8080" + REVERSE_SCAN_TIMEOUT: "500" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/reversescan/index.html b/tests/core/reversescan/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/reversescan/main.py b/tests/core/reversescan/main.py new file mode 100644 index 000000000..5e7c00b4c --- /dev/null +++ b/tests/core/reversescan/main.py @@ -0,0 +1,43 @@ +from time import sleep +from fastapi import FastAPI +from os import getenv +from requests import get +from multiprocessing import Process +from traceback import format_exc +from uvicorn import run + + +app = FastAPI() +fastapi_proc = Process(target=run, args=(app,), kwargs=dict(host="0.0.0.0", port=80)) +fastapi_proc.start() + +sleep(1) + +try: + use_reverse_scan = getenv("USE_REVERSE_SCAN", "no") == "yes" + reverse_scan_ports = getenv("REVERSE_SCAN_PORTS", "22 80 443 3128 8000 8080") + + print(f"ℹ️ Trying to access http://www.example.com ...", flush=True) + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code == 403: + pass + elif use_reverse_scan and " 80 " in reverse_scan_ports: + print( + "❌ Request didn't return 403, but reverse scan is enabled and port 80 is in the reverse scan ports list, exiting ...", + flush=True, + ) + exit(1) + + print("✅ Reverse scan is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) +finally: + fastapi_proc.terminate() diff --git a/tests/core/reversescan/requirements.txt b/tests/core/reversescan/requirements.txt new file mode 100644 index 000000000..f8a512e89 --- /dev/null +++ b/tests/core/reversescan/requirements.txt @@ -0,0 +1,3 @@ +requests==2.30.0 +fastapi==0.95.1 +uvicorn[standard]==0.22.0 diff --git a/tests/core/reversescan/test.sh b/tests/core/reversescan/test.sh new file mode 100755 index 000000000..4e01a64e1 --- /dev/null +++ b/tests/core/reversescan/test.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +echo "🕵️ Building reversescan stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🕵️ Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🕵️ Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_REVERSE_SCAN: "no"@USE_REVERSE_SCAN: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@REVERSE_SCAN_PORTS: ".*"$@REVERSE_SCAN_PORTS: "22 80 443 3128 8000 8080"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🕵️ Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🕵️ Down failed ❌" + exit 1 + fi + + echo "🕵️ Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "reverse_scan" "tweaked_ports" "deactivated" +do + if [ "$test" = "reverse_scan" ] ; then + echo "🕵️ Running tests with default reverse scan ..." + elif [ "$test" = "tweaked_ports" ] ; then + echo "🕵️ Running tests while removing the 80 port being scanned ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@REVERSE_SCAN_PORTS: ".*"$@REVERSE_SCAN_PORTS: "22 443 3128 8000 8080"@' {} \; + elif [ "$test" = "deactivated" ] ; then + echo "🕵️ Running tests without the reverse scan ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_REVERSE_SCAN: "yes"@USE_REVERSE_SCAN: "no"@' {} \; + fi + + echo "🕵️ Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🕵️ Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🕵️ Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("reversescan-bw-1" "reversescan-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🕵️ Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🕵️ Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🕵️ Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🕵️ Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🕵️ Tests are done ! ✅" diff --git a/tests/core/selfsigned/Dockerfile b/tests/core/selfsigned/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/selfsigned/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/selfsigned/docker-compose.test.yml b/tests/core/selfsigned/docker-compose.test.yml new file mode 100644 index 000000000..f5ce8dc10 --- /dev/null +++ b/tests/core/selfsigned/docker-compose.test.yml @@ -0,0 +1,19 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + GENERATE_SELF_SIGNED_SSL: "no" + SELF_SIGNED_SSL_EXPIRY: "365" + SELF_SIGNED_SSL_SUBJ: "/CN=www.example.com/" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/selfsigned/docker-compose.yml b/tests/core/selfsigned/docker-compose.yml new file mode 100644 index 000000000..d822fc850 --- /dev/null +++ b/tests/core/selfsigned/docker-compose.yml @@ -0,0 +1,63 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + HTTPS_PORT: "443" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + + # ? SELF_SIGNED settings + GENERATE_SELF_SIGNED_SSL: "no" + SELF_SIGNED_SSL_EXPIRY: "365" + SELF_SIGNED_SSL_SUBJ: "/CN=www.example.com/" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/selfsigned/index.html b/tests/core/selfsigned/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/selfsigned/main.py b/tests/core/selfsigned/main.py new file mode 100644 index 000000000..ac3e4573e --- /dev/null +++ b/tests/core/selfsigned/main.py @@ -0,0 +1,83 @@ +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from datetime import datetime, timedelta +from os import getenv +from requests import get +from socket import create_connection +from ssl import CERT_NONE, DER_cert_to_PEM_cert, create_default_context +from time import sleep +from traceback import format_exc + +try: + ssl_generated = getenv("GENERATE_SELF_SIGNED_SSL", "no") == "yes" + self_signed_ssl_expiry = getenv("SELF_SIGNED_SSL_EXPIRY", "365") + + self_signed_ssl_expiry = ( + datetime.now() + + timedelta(days=int(self_signed_ssl_expiry)) + - timedelta(hours=1) + ) + + self_signed_ssl_subj = getenv("SELF_SIGNED_SSL_SUBJ", "/CN=www.example.com/") + + response = get( + f"http://www.example.com", headers={"Host": "www.example.com"}, verify=False + ) + + if not ssl_generated and response.status_code == 200: + print( + "✅ The SSL generation is disabled and the request returned 200, exiting ...", + flush=True, + ) + exit(0) + + sleep(1) + + response = get( + f"https://www.example.com", headers={"Host": "www.example.com"}, verify=False + ) + + if ssl_generated and response.status_code != 200: + print( + f"❌ The SSL generation is enabled and the request returned {response.status_code}, exiting ...", + flush=True, + ) + exit(1) + + sleep(1) + + context = create_default_context() + context.check_hostname = False + context.verify_mode = CERT_NONE + with create_connection(("www.example.com", 443)) as sock: + with context.wrap_socket(sock, server_hostname="www.example.com") as ssock: + # Retrieve the SSL certificate + pem_data = DER_cert_to_PEM_cert(ssock.getpeercert(True)) + + # Parse the PEM certificate + certificate = x509.load_pem_x509_certificate(pem_data.encode(), default_backend()) + + common_name = certificate.subject.get_attributes_for_oid( + x509.oid.NameOID.COMMON_NAME + )[0].value + if common_name != self_signed_ssl_subj.replace("/", "").replace("CN=", ""): + print( + f"❌ The SSL generation is enabled and the Common Name (CN) is not {self_signed_ssl_subj} but {common_name}, exiting ...", + flush=True, + ) + exit(1) + + expiration_date = certificate.not_valid_after + if expiration_date < self_signed_ssl_expiry: + print( + f"❌ The SSL generation is enabled and the expiration date is {expiration_date} but should be {self_signed_ssl_expiry}, exiting ...", + flush=True, + ) + exit(1) + + print("✅ The SSL generation is enabled and the certificate is valid", flush=True) +except SystemExit as e: + exit(e.code) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/selfsigned/requirements.txt b/tests/core/selfsigned/requirements.txt new file mode 100644 index 000000000..854cb0f93 --- /dev/null +++ b/tests/core/selfsigned/requirements.txt @@ -0,0 +1,2 @@ +requests==2.30.0 +cryptography==40.0.2 diff --git a/tests/core/selfsigned/test.sh b/tests/core/selfsigned/test.sh new file mode 100755 index 000000000..85bb6cee9 --- /dev/null +++ b/tests/core/selfsigned/test.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +echo "🔐 Building selfsigned stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🔐 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🔐 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@GENERATE_SELF_SIGNED_SSL: "yes"@GENERATE_SELF_SIGNED_SSL: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@SELF_SIGNED_SSL_EXPIRY: "30"@SELF_SIGNED_SSL_EXPIRY: "365"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@SELF_SIGNED_SSL_SUBJ: "/CN=example.com/"@SELF_SIGNED_SSL_SUBJ: "/CN=www.example.com/"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🔐 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🔐 Down failed ❌" + exit 1 + fi + + echo "🔐 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "deactivated" "activated" "tweaked_options" +do + if [ "$test" = "deactivated" ] ; then + echo "🔐 Running tests without selfsigned ..." + elif [ "$test" = "activated" ] ; then + echo "🔐 Running tests with selfsigned activated ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@GENERATE_SELF_SIGNED_SSL: "no"@GENERATE_SELF_SIGNED_SSL: "yes"@' {} \; + elif [ "$test" = "tweaked_options" ] ; then + echo "🔐 Running tests with selfsigned's options tweaked ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@SELF_SIGNED_SSL_EXPIRY: "365"@SELF_SIGNED_SSL_EXPIRY: "30"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@SELF_SIGNED_SSL_SUBJ: "/CN=www.example.com/"@SELF_SIGNED_SSL_SUBJ: "/CN=example.com/"@' {} \; + fi + + echo "🔐 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🔐 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🔐 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("selfsigned-bw-1" "selfsigned-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🔐 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🔐 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🔐 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🔐 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🔐 Tests are done ! ✅" diff --git a/tests/core/sessions/Dockerfile b/tests/core/sessions/Dockerfile new file mode 100644 index 000000000..c6b6dd4bc --- /dev/null +++ b/tests/core/sessions/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11.3-alpine + +# Install firefox and geckodriver +RUN apk add --no-cache --virtual .build-deps curl grep zip && \ + apk add --no-cache firefox + +# Installing geckodriver for firefox... +RUN GECKODRIVER_VERSION=`curl -i https://github.com/mozilla/geckodriver/releases/latest | grep -Po 'v[0-9]+.[0-9]+.[0-9]+'` && \ + wget -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \ + tar -C /usr/local/bin -xzvf geckodriver.tar.gz && \ + chmod +x /usr/local/bin/geckodriver && \ + rm geckodriver.tar.gz + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/sessions/docker-compose.test.yml b/tests/core/sessions/docker-compose.test.yml new file mode 100644 index 000000000..1b90422a7 --- /dev/null +++ b/tests/core/sessions/docker-compose.test.yml @@ -0,0 +1,18 @@ +version: "3.5" + +services: + tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + SESSIONS_SECRET: "random" + SESSIONS_NAME: "random" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + +networks: + bw-services: + external: true diff --git a/tests/core/sessions/docker-compose.yml b/tests/core/sessions/docker-compose.yml new file mode 100644 index 000000000..1a35a6646 --- /dev/null +++ b/tests/core/sessions/docker-compose.yml @@ -0,0 +1,62 @@ +version: "3.5" + +services: + bw: + image: bunkerity/bunkerweb:1.5.0-beta + pull_policy: never + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 192.168.0.3" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + USE_BLACKLIST: "no" + LOG_LEVEL: "info" + USE_ANTIBOT: "cookie" + + # ? SESSIONS settings + SESSIONS_SECRET: "random" + SESSIONS_NAME: "random" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + + bw-scheduler: + image: bunkerity/bunkerweb-scheduler:1.5.0-beta + pull_policy: never + depends_on: + - bw + - bw-docker + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-docker: diff --git a/tests/core/sessions/index.html b/tests/core/sessions/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/sessions/main.py b/tests/core/sessions/main.py new file mode 100644 index 000000000..d48f07e63 --- /dev/null +++ b/tests/core/sessions/main.py @@ -0,0 +1,95 @@ +from contextlib import suppress +from os import getenv +from requests import get, post +from requests.exceptions import RequestException +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options +from selenium.common.exceptions import NoSuchElementException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + firefox_options = Options() + firefox_options.add_argument("--headless") + + sessions_secret = getenv("SESSIONS_SECRET", "random") + sessions_name = getenv("SESSIONS_NAME", "random") + first_cookie = None + + print("ℹ️ Starting Firefox ...", flush=True) + with webdriver.Firefox(options=firefox_options) as driver: + driver.delete_all_cookies() + driver.maximize_window() + + print("ℹ️ Navigating to http://www.example.com ...", flush=True) + driver.get("http://www.example.com") + + if sessions_name != "random": + if not driver.get_cookie(sessions_name): + print(f"❌ Cookie {sessions_name} not found, exiting ...", flush=True) + exit(1) + print(f"✅ Cookie {sessions_name} found", flush=True) + exit(0) + + first_cookie = driver.get_cookies()[0] + print(first_cookie, flush=True) + + print("ℹ️ Reloading BunkerWeb ...", flush=True) + + response = post("http://192.168.0.2:5000/reload", headers={"Host": "bwapi"}) + + if response.status_code != 200: + print("❌ An error occurred when restarting BunkerWeb, exiting ...", flush=True) + exit(1) + + data = response.json() + + if data["status"] != "success": + print("❌ An error occurred when restarting BunkerWeb, exiting ...", flush=True) + exit(1) + + sleep(10) + + print("ℹ️ Starting Firefox again ...", flush=True) + with webdriver.Firefox(options=firefox_options) as driver: + driver.delete_all_cookies() + driver.maximize_window() + + print("ℹ️ Navigating to http://www.example.com ...", flush=True) + driver.get("http://www.example.com") + + cookie = driver.get_cookies()[0] + + if sessions_name == "random" and first_cookie["name"] == cookie["name"]: + print("❌ The cookie name has not been changed, exiting ...", flush=True) + exit(1) +except SystemExit as e: + exit(e.code) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/sessions/requirements.txt b/tests/core/sessions/requirements.txt new file mode 100644 index 000000000..6f7b13f79 --- /dev/null +++ b/tests/core/sessions/requirements.txt @@ -0,0 +1,2 @@ +requests==2.30.0 +selenium==4.9.1 diff --git a/tests/core/sessions/test.sh b/tests/core/sessions/test.sh new file mode 100755 index 000000000..5d440cb6b --- /dev/null +++ b/tests/core/sessions/test.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +echo "🧳 Building sessions stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🧳 Pull failed ❌" + exit 1 +fi +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🧳 Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + find . -type f -name 'docker-compose.*' -exec sed -i 's@SESSIONS_SECRET: ".*"$@SESSIONS_SECRET: "random"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@SESSIONS_NAME: ".*"$@SESSIONS_NAME: "random"@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🧳 Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🧳 Down failed ❌" + exit 1 + fi + + echo "🧳 Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +for test in "random" "manual_name" # TODO: "manual_secret" +do + if [ "$test" = "random" ] ; then + echo "🧳 Running tests with random secret and random name ..." + elif [ "$test" = "manual_name" ] ; then + echo "🧳 Running tests where session name is equal to \"test\" ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@SESSIONS_NAME: ".*"$@SESSIONS_NAME: "test"@' {} \; + elif [ "$test" = "manual_secret" ] ; then + echo "🧳 Running tests where session secret is equal to \"test\" ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@SESSIONS_NAME: ".*"$@SESSIONS_NAME: "random"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@SESSIONS_SECRET: ".*"$@SESSIONS_SECRET: "test"@' {} \; + fi + + echo "🧳 Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🧳 Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🧳 Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("sessions-bw-1" "sessions-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🧳 Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🧳 Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🧳 Test \"$test\" failed ❌" + echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..." + docker compose logs bw bw-scheduler + exit 1 + else + echo "🧳 Test \"$test\" succeeded ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🧳 Tests are done ! ✅" diff --git a/tests/core/tests.sh b/tests/core/tests.sh new file mode 100755 index 000000000..9b1ab23fe --- /dev/null +++ b/tests/core/tests.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +echo "🚀 Starting core tests ..." + +# Prepare environment +# TODO: uncomment this in production +# find . -type f -name 'docker-compose.*' -exec sed -i "s@bunkerity/bunkerweb:.*@bunkerweb-tests@" {} \; +# find . -type f -name 'docker-compose.*' -exec sed -i "s@bunkerity/bunkerweb-scheduler:.*@scheduler-tests@" {} \; + +for dir in $(ls -d */) +do + if [ -f "$dir/test.sh" ] ; then + echo "Testing ${dir%?} ..." + cd $dir + ./test.sh + + if [ $? -ne 0 ] ; then + echo "❌ Core test ${dir%?} failed" + exit 1 + fi + + cd .. + + echo " " + else + echo "⚠️ No tests in ${dir%?}, skipping." + fi +done + +echo "🚀 Core tests are done ! ✅" \ No newline at end of file diff --git a/tests/core/whitelist/Dockerfile b/tests/core/whitelist/Dockerfile new file mode 100644 index 000000000..9cdc4ff12 --- /dev/null +++ b/tests/core/whitelist/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/tests + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/whitelist/api/Dockerfile b/tests/core/whitelist/api/Dockerfile new file mode 100644 index 000000000..714dc8a89 --- /dev/null +++ b/tests/core/whitelist/api/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/greylist_api + +COPY main.py . + +ENTRYPOINT [ "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers", "--forwarded-allow-ips", "\"*\"" ] \ No newline at end of file diff --git a/tests/core/whitelist/api/main.py b/tests/core/whitelist/api/main.py new file mode 100644 index 000000000..b7a8b92a1 --- /dev/null +++ b/tests/core/whitelist/api/main.py @@ -0,0 +1,30 @@ +from fastapi import FastAPI +from fastapi.responses import PlainTextResponse + + +app = FastAPI() + + +@app.get("/ip") +async def ip(): + return PlainTextResponse("192.168.0.3\n10.0.0.0/8\n127.0.0.1/32") + + +@app.get("/rdns") +async def rdns(): + return PlainTextResponse(".example.com\n.example.org\n.bw-services") + + +@app.get("/asn") +async def asn(): + return PlainTextResponse("1234\n13335\n5678") + + +@app.get("/user_agent") +async def user_agent(): + return PlainTextResponse("BunkerBot\nCensysInspect\nShodanInspect\nZmEu\nmasscan") + + +@app.get("/uri") +async def uri(): + return PlainTextResponse("/admin\n/login") diff --git a/tests/core/whitelist/api/requirements.txt b/tests/core/whitelist/api/requirements.txt new file mode 100644 index 000000000..c06221a5a --- /dev/null +++ b/tests/core/whitelist/api/requirements.txt @@ -0,0 +1,2 @@ +fastapi==0.95.1 +uvicorn[standard]==0.22.0 diff --git a/tests/core/whitelist/docker-compose.init.yml b/tests/core/whitelist/docker-compose.init.yml new file mode 100644 index 000000000..36d606b46 --- /dev/null +++ b/tests/core/whitelist/docker-compose.init.yml @@ -0,0 +1,9 @@ +version: "3.5" + +services: + init: + build: init + environment: + PYTHONUNBUFFERED: "1" + volumes: + - ./init/output:/output diff --git a/tests/core/whitelist/docker-compose.test.yml b/tests/core/whitelist/docker-compose.test.yml new file mode 100644 index 000000000..aea42e7cf --- /dev/null +++ b/tests/core/whitelist/docker-compose.test.yml @@ -0,0 +1,54 @@ +version: "3.5" + +services: + local-tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + GLOBAL: "0" + USE_WHITELIST: "no" + WHITELIST_IP: "" + WHITELIST_IP_URLS: "" + WHITELIST_RDNS_GLOBAL: "yes" + WHITELIST_RDNS: "" + WHITELIST_RDNS_URLS: "" + WHITELIST_ASN: "" + WHITELIST_ASN_URLS: "" + WHITELIST_USER_AGENT: "" + WHITELIST_USER_AGENT_URLS: "" + WHITELIST_URI: "" + WHITELIST_URI_URLS: "" + extra_hosts: + - "www.example.com:192.168.0.2" + networks: + bw-services: + ipv4_address: 192.168.0.3 + + global-tests: + build: . + environment: + PYTHONUNBUFFERED: "1" + GLOBAL: "1" + USE_WHITELIST: "no" + WHITELIST_IP: "" + WHITELIST_IP_URLS: "" + WHITELIST_RDNS_GLOBAL: "yes" + WHITELIST_RDNS: "" + WHITELIST_RDNS_URLS: "" + WHITELIST_ASN: "" + WHITELIST_ASN_URLS: "" + WHITELIST_USER_AGENT: "" + WHITELIST_USER_AGENT_URLS: "" + WHITELIST_URI: "" + WHITELIST_URI_URLS: "" + extra_hosts: + - "www.example.com:1.0.0.2" + networks: + bw-global-network: + ipv4_address: 1.0.0.3 + +networks: + bw-services: + external: true + bw-global-network: + external: true diff --git a/tests/core/whitelist/docker-compose.yml b/tests/core/whitelist/docker-compose.yml new file mode 100644 index 000000000..fc9e30189 --- /dev/null +++ b/tests/core/whitelist/docker-compose.yml @@ -0,0 +1,103 @@ +version: "3.5" + +services: + bw: + # TODO: use image + # image: bunkerity/bunkerweb:1.5.0-beta + # pull_policy: never + build: + context: /home/theophile/dev/bunkerweb + dockerfile: src/bw/Dockerfile + labels: + - "bunkerweb.INSTANCE" + volumes: + - ./index.html:/var/www/html/index.html + environment: + API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24" + HTTP_PORT: "80" + USE_BUNKERNET: "no" + LOG_LEVEL: "info" + USE_BLACKLIST: "yes" + BLACKLIST_IP: "0.0.0.0/0" + BLACKLIST_IP_URLS: "" + + # ? WHITELIST settings + USE_WHITELIST: "no" + WHITELIST_IP: "" + WHITELIST_IP_URLS: "" + WHITELIST_RDNS_GLOBAL: "yes" + WHITELIST_RDNS: "" + WHITELIST_RDNS_URLS: "" + WHITELIST_ASN: "" + WHITELIST_ASN_URLS: "" + WHITELIST_USER_AGENT: "" + WHITELIST_USER_AGENT_URLS: "" + WHITELIST_URI: "" + WHITELIST_URI_URLS: "" + networks: + bw-universe: + bw-services: + ipv4_address: 192.168.0.2 + bw-global-network: + ipv4_address: 1.0.0.2 + + bw-scheduler: + # TODO: use image + # image: bunkerity/bunkerweb-scheduler:1.5.0-beta + # pull_policy: never + build: + context: /home/theophile/dev/bunkerweb + dockerfile: src/scheduler/Dockerfile + depends_on: + - bw + - bw-docker + volumes: + - bw-data:/data + environment: + DOCKER_HOST: "tcp://bw-docker:2375" + LOG_LEVEL: "info" + networks: + - bw-universe + - bw-docker + + bw-docker: + image: tecnativa/docker-socket-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: "1" + networks: + - bw-docker + + whitelist-api: + build: api + networks: + bw-docker: + bw-services: + ipv4_address: 192.168.0.4 + +volumes: + bw-data: + + +networks: + bw-universe: + name: bw-universe + ipam: + driver: default + config: + - subnet: 10.20.30.0/24 + bw-services: + name: bw-services + ipam: + driver: default + config: + - subnet: 192.168.0.0/24 + bw-global-network: + name: bw-global-network + ipam: + driver: default + config: + - subnet: 1.0.0.0/8 + bw-docker: + name: bw-docker diff --git a/tests/core/whitelist/index.html b/tests/core/whitelist/index.html new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/whitelist/init/Dockerfile b/tests/core/whitelist/init/Dockerfile new file mode 100644 index 000000000..2bb13a4f9 --- /dev/null +++ b/tests/core/whitelist/init/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11.3-alpine + +WORKDIR /tmp + +COPY requirements.txt . + +RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \ + rm -f requirements.txt + +WORKDIR /opt/blacklist_init + +COPY main.py . + +ENTRYPOINT [ "python3", "main.py" ] \ No newline at end of file diff --git a/tests/core/whitelist/init/main.py b/tests/core/whitelist/init/main.py new file mode 100644 index 000000000..ef1409902 --- /dev/null +++ b/tests/core/whitelist/init/main.py @@ -0,0 +1,33 @@ +from datetime import date +from gzip import GzipFile +from io import BytesIO +from pathlib import Path +from maxminddb import MODE_FD, open_database +from requests import get + +# Compute the mmdb URL +mmdb_url = f"https://download.db-ip.com/free/dbip-asn-lite-{date.today().strftime('%Y-%m')}.mmdb.gz" + +# Download the mmdb file in memory +print(f"Downloading mmdb file from url {mmdb_url} ...", flush=True) +file_content = BytesIO() +with get(mmdb_url, stream=True) as resp: + resp.raise_for_status() + for chunk in resp.iter_content(chunk_size=4 * 1024): + if chunk: + file_content.write(chunk) +file_content.seek(0) + +with open_database(GzipFile(fileobj=file_content, mode="rb"), mode=MODE_FD) as reader: + dbip_asn = reader.get("1.0.0.3") + + if not dbip_asn: + print(f"❌ Error while reading mmdb file from {mmdb_url}", flush=True) + exit(1) + + print( + f"✅ ASN for IP 1.0.0.3 is {dbip_asn['autonomous_system_number']}, saving it to /output/ip_asn.txt", + flush=True, + ) + + Path("/output/ip_asn.txt").write_text(str(dbip_asn["autonomous_system_number"])) diff --git a/tests/core/whitelist/init/requirements.txt b/tests/core/whitelist/init/requirements.txt new file mode 100644 index 000000000..f91deab71 --- /dev/null +++ b/tests/core/whitelist/init/requirements.txt @@ -0,0 +1,2 @@ +maxminddb==2.3.0 +requests==2.30.0 diff --git a/tests/core/whitelist/main.py b/tests/core/whitelist/main.py new file mode 100644 index 000000000..af3aad38c --- /dev/null +++ b/tests/core/whitelist/main.py @@ -0,0 +1,140 @@ +from contextlib import suppress +from os import getenv +from requests import get +from requests.exceptions import RequestException +from time import sleep +from traceback import format_exc + +try: + ready = False + retries = 0 + while not ready: + with suppress(RequestException): + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + if status_code >= 500: + print("❌ An error occurred with the server, exiting ...", flush=True) + exit(1) + + ready = status_code < 400 or status_code == 403 + + if retries > 10: + print("❌ The service took too long to be ready, exiting ...", flush=True) + exit(1) + elif not ready: + retries += 1 + print( + "⚠️ Waiting for the service to be ready, retrying in 5s ...", flush=True + ) + sleep(5) + + use_whitelist = getenv("USE_WHITELIST", "yes") == "yes" + _global = getenv("GLOBAL", "0") == "1" + + whitelist_ip = getenv("WHITELIST_IP", "") + whitelist_ip_urls = getenv("WHITELIST_IP_URLS", "") + whitelist_rdns_global = getenv("WHITELIST_RDNS_GLOBAL", "yes") == "yes" + whitelist_rdns = getenv("WHITELIST_RDNS", "") + whitelist_rdns_urls = getenv("WHITELIST_RDNS_URLS", "") + whitelist_asn = getenv("WHITELIST_ASN", "") + whitelist_asn_urls = getenv("WHITELIST_ASN_URLS", "") + whitelist_user_agent = getenv("WHITELIST_USER_AGENT", "") + whitelist_user_agent_urls = getenv("WHITELIST_USER_AGENT_URLS", "") + whitelist_uri = getenv("WHITELIST_URI", "") + whitelist_uri_urls = getenv("WHITELIST_URI_URLS", "") + + print("ℹ️ Sending a request to http://www.example.com ...", flush=True) + status_code = get( + "http://www.example.com", headers={"Host": "www.example.com"} + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code == 403: + if (whitelist_ip or whitelist_ip_urls) and not _global: + print( + "❌ Request was rejected, even though IP is supposed to be in the whitelist, exiting ..." + ) + exit(1) + elif ( + (whitelist_rdns or whitelist_rdns_urls) + and not whitelist_rdns_global + and not _global + ): + print( + "❌ Request was rejected, even though RDNS is supposed to be in the whitelist, exiting ..." + ) + exit(1) + elif (whitelist_asn or whitelist_asn_urls) and _global: + print( + "❌ Request was rejected, even though ASN is supposed to be in the whitelist, exiting ..." + ) + exit(1) + elif whitelist_user_agent or whitelist_user_agent_urls: + print( + "ℹ️ Sending a request to http://www.example.com with User-Agent BunkerBot ...", + flush=True, + ) + status_code = get( + "http://www.example.com", + headers={"Host": "www.example.com", "User-Agent": "BunkerBot"}, + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code == 403: + print( + "❌ Request was rejected, even though User Agent is supposed to be in the whitelist ..." + ) + exit(1) + + print("✅ Request was not rejected, User Agent is in the whitelist ...") + elif whitelist_uri or whitelist_uri_urls: + print( + "ℹ️ Sending a request to http://www.example.com/admin ...", + flush=True, + ) + status_code = get( + "http://www.example.com/admin", headers={"Host": "www.example.com"} + ).status_code + + print(f"ℹ️ Status code: {status_code}", flush=True) + + if status_code == 403: + print( + "❌ Request was rejected, even though URI is supposed to be in the whitelist ..." + ) + exit(1) + + print("✅ Request was not rejected, URI is in the whitelist ...") + else: + if (whitelist_ip or whitelist_ip_urls) and _global: + print( + "❌ Request was not rejected, but IP is not in the whitelist, exiting ..." + ) + exit(1) + elif (whitelist_rdns or whitelist_rdns_urls) and _global: + print( + "❌ Request was not rejected, but RDNS is not in the whitelist, exiting ..." + ) + exit(1) + elif (whitelist_asn or whitelist_asn_urls) and not _global: + print( + "❌ Request was rejected, but ASN is not in the whitelist, exiting ..." + ) + exit(1) + elif whitelist_user_agent or whitelist_user_agent_urls: + print("❌ Request was rejected, but User Agent is not in the whitelist ...") + exit(1) + elif whitelist_uri or whitelist_uri_urls: + print("❌ Request was rejected, but URI is not in the whitelist ...") + exit(1) + + print("✅ Whitelist is working as expected ...", flush=True) +except SystemExit: + exit(1) +except: + print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True) + exit(1) diff --git a/tests/core/whitelist/requirements.txt b/tests/core/whitelist/requirements.txt new file mode 100644 index 000000000..becc27ff2 --- /dev/null +++ b/tests/core/whitelist/requirements.txt @@ -0,0 +1 @@ +requests==2.30.0 diff --git a/tests/core/whitelist/test.sh b/tests/core/whitelist/test.sh new file mode 100755 index 000000000..1c4756784 --- /dev/null +++ b/tests/core/whitelist/test.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +echo "🏳️ Building whitelist stack ..." + +# Starting stack +docker compose pull bw-docker +if [ $? -ne 0 ] ; then + echo "🏳️ Pull failed ❌" + exit 1 +fi + +echo "🏳️ Building custom api image ..." +docker compose build whitelist-api +if [ $? -ne 0 ] ; then + echo "🏳️ Build failed ❌" + exit 1 +fi + +echo "🏳️ Building tests images ..." +docker compose -f docker-compose.test.yml build +if [ $? -ne 0 ] ; then + echo "🏳️ Build failed ❌" + exit 1 +fi + +manual=0 +end=0 +as_number=0 +cleanup_stack () { + exit_code=$? + if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then + rm -rf init/output + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_WHITELIST: "yes"@USE_WHITELIST: "no"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_IP: "192.168.0.0/24"@WHITELIST_IP: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_IP_URLS: "http://whitelist-api:8080/ip"@WHITELIST_IP_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_RDNS_GLOBAL: "no"@WHITELIST_RDNS_GLOBAL: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_RDNS: ".bw-services"@WHITELIST_RDNS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_RDNS_URLS: "http://whitelist-api:8080/rdns"@WHITELIST_RDNS_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_ASN: "[0-9]*"@WHITELIST_ASN: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_ASN_URLS: "http://whitelist-api:8080/asn"@WHITELIST_ASN_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_USER_AGENT: "BunkerBot"@WHITELIST_USER_AGENT: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_USER_AGENT_URLS: "http://whitelist-api:8080/user_agent"@WHITELIST_USER_AGENT_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_URI: "/admin"@WHITELIST_URI: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_URI_URLS: "http://whitelist-api:8080/uri"@WHITELIST_URI_URLS: ""@' {} \; + if [[ $end -eq 1 && $exit_code = 0 ]] ; then + return + fi + fi + + echo "🏳️ Cleaning up current stack ..." + + docker compose down -v --remove-orphans 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🏳️ Down failed ❌" + exit 1 + fi + + echo "🏳️ Cleaning up current stack done ✅" +} + +# Cleanup stack on exit +trap cleanup_stack EXIT + +echo "🏳️ Initializing workspace ..." +rm -rf init/output +mkdir -p init/output +docker compose -f docker-compose.init.yml up --build +if [ $? -ne 0 ] ; then + echo "🏳️ Build failed ❌" + exit 1 +elif ! [[ -f "init/output/ip_asn.txt" ]]; then + echo "🏳️ ip_asn.txt not found ❌" + exit 1 +fi + +as_number=$(cat init/output/ip_asn.txt) + +if [[ $as_number = "" ]]; then + echo "🏳️ AS number not found ❌" + exit 1 +fi + +rm -rf init/output + +for test in "deactivated" "ip" "ip_urls" "rdns" "rdns_global" "rdns_urls" "asn" "asn_urls" "user_agent" "user_agent_urls" "uri" "uri_urls" +do + if [ "$test" = "deactivated" ] ; then + echo "🏳️ Running tests when the whitelist is deactivated ..." + echo "ℹ️ Activating the blacklist and banning 0.0.0.0/0 network for all the future tests ..." + elif [ "$test" = "ip" ] ; then + echo "🏳️ Running tests with the network 192.168.0.0/24 in the white list ..." + echo "ℹ️ Activating the whitelist for all the future tests ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_WHITELIST: "no"@USE_WHITELIST: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_IP: ""@WHITELIST_IP: "192.168.0.0/24"@' {} \; + elif [ "$test" = "ip_urls" ] ; then + echo "🏳️ Running tests with whitelist's ip url set to http://whitelist-api:8080/ip ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_IP: "192.168.0.0/24"@WHITELIST_IP: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_IP_URLS: ""@WHITELIST_IP_URLS: "http://whitelist-api:8080/ip"@' {} \; + elif [ "$test" = "rdns" ] ; then + echo "🏳️ Running tests with whitelist's rdns set to .bw-services ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_IP_URLS: "http://whitelist-api:8080/ip"@WHITELIST_IP_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_RDNS: ""@WHITELIST_RDNS: ".bw-services"@' {} \; + elif [ "$test" = "rdns_global" ] ; then + echo "🏴 Running tests when whitelist's rdns also scans local ip addresses ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_RDNS_GLOBAL: "yes"@WHITELIST_RDNS_GLOBAL: "no"@' {} \; + elif [ "$test" = "rdns_urls" ] ; then + echo "🏳️ Running tests with whitelist's rdns url set to http://whitelist-api:8080/rdns ..." + echo "ℹ️ Keeping the rdns also scanning local ip addresses ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_RDNS: ".bw-services"@WHITELIST_RDNS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_RDNS_URLS: ""@WHITELIST_RDNS_URLS: "http://whitelist-api:8080/rdns"@' {} \; + elif [ "$test" = "asn" ] ; then + echo "🏳️ Running tests with whitelist's asn set to $as_number ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_RDNS_GLOBAL: "no"@WHITELIST_RDNS_GLOBAL: "yes"@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_RDNS_URLS: "http://whitelist-api:8080/rdns"@WHITELIST_RDNS_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_ASN: ""@WHITELIST_ASN: "'"$as_number"'"@' {} \; + elif [ "$test" = "asn_urls" ] ; then + echo "🏳️ Running tests with whitelist's asn url set to http://whitelist-api:8080/asn ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_ASN: "'"$as_number"'"@WHITELIST_ASN: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_ASN_URLS: ""@WHITELIST_ASN_URLS: "http://whitelist-api:8080/asn"@' {} \; + elif [ "$test" = "user_agent" ] ; then + echo "🏳️ Running tests with whitelist's user_agent set to BunkerBot ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_ASN_URLS: "http://whitelist-api:8080/asn"@WHITELIST_ASN_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_USER_AGENT: ""@WHITELIST_USER_AGENT: "BunkerBot"@' {} \; + elif [ "$test" = "user_agent_urls" ] ; then + echo "🏳️ Running tests with whitelist's user_agent url set to http://whitelist-api:8080/user_agent ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_USER_AGENT: "BunkerBot"@WHITELIST_USER_AGENT: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_USER_AGENT_URLS: ""@WHITELIST_USER_AGENT_URLS: "http://whitelist-api:8080/user_agent"@' {} \; + elif [ "$test" = "uri" ] ; then + echo "🏳️ Running tests with whitelist's uri set to /admin ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_USER_AGENT_URLS: "http://whitelist-api:8080/user_agent"@WHITELIST_USER_AGENT_URLS: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_URI: ""@WHITELIST_URI: "/admin"@' {} \; + elif [ "$test" = "uri_urls" ] ; then + echo "🏳️ Running tests with whitelist's uri url set to http://whitelist-api:8080/uri ..." + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_URI: "/admin"@WHITELIST_URI: ""@' {} \; + find . -type f -name 'docker-compose.*' -exec sed -i 's@WHITELIST_URI_URLS: ""@WHITELIST_URI_URLS: "http://whitelist-api:8080/uri"@' {} \; + fi + + echo "🏳️ Starting stack ..." + docker compose up -d 2>/dev/null + if [ $? -ne 0 ] ; then + echo "🏳️ Up failed ❌" + exit 1 + fi + + # Check if stack is healthy + echo "🏳️ Waiting for stack to be healthy ..." + i=0 + while [ $i -lt 120 ] ; do + containers=("whitelist-bw-1" "whitelist-bw-scheduler-1") + healthy="true" + for container in "${containers[@]}" ; do + check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")" + if [ "$check" = "" ] ; then + healthy="false" + break + fi + done + if [ "$healthy" = "true" ] ; then + echo "🏳️ Docker stack is healthy ✅" + break + fi + sleep 1 + i=$((i+1)) + done + if [ $i -ge 120 ] ; then + docker compose logs + echo "🏳️ Docker stack is not healthy ❌" + exit 1 + fi + + # Start tests + + if ! [[ "$test" = "user_agent" || "$test" = "user_agent_urls" || "$test" = "uri" || "$test" = "uri_urls" ]] ; then + echo "🏳️ Running global container tests ..." + + docker compose -f docker-compose.test.yml up global-tests --abort-on-container-exit --exit-code-from global-tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🏳️ Test \"$test\" failed for global tests ❌" + echo "🛡️ Showing BunkerWeb, BunkerWeb Scheduler and Custom API logs ..." + docker compose logs bw bw-scheduler whitelist-api + exit 1 + else + echo "🏳️ Test \"$test\" succeeded for global tests ✅" + fi + fi + + echo "🏳️ Running local container tests ..." + + docker compose -f docker-compose.test.yml up local-tests --abort-on-container-exit --exit-code-from local-tests 2>/dev/null + + if [ $? -ne 0 ] ; then + echo "🏳️ Test \"$test\" failed for local tests ❌" + echo "🛡️ Showing BunkerWeb, BunkerWeb Scheduler and Custom API logs ..." + docker compose logs bw bw-scheduler whitelist-api + exit 1 + else + echo "🏳️ Test \"$test\" succeeded for local tests ✅" + fi + + manual=1 + cleanup_stack + manual=0 + + echo " " +done + +end=1 +echo "🏳️ Tests are done ! ✅"