Fix db automatic tests

tests/core/db/docker-compose.old.yml

@@ -0,0 +1,90 @@
+version: "3.5"
+
+services:
+  bw:
+    image: bunkerity/bunkerweb:1.5.9
+    pull_policy: never
+    labels:
+      - "bunkerweb.INSTANCE=yes"
+    environment:
+      SERVER_NAME: "bwadm.example.com"
+      API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24"
+      MULTISITE: "no"
+      HTTP_PORT: "80"
+      USE_BUNKERNET: "no"
+      USE_BLACKLIST: "no"
+      SEND_ANONYMOUS_REPORT: "no"
+      USE_REVERSE_PROXY: "yes"
+      REVERSE_PROXY_HOST: "http://app1:8080"
+      REVERSE_PROXY_URL: "/"
+      LOG_LEVEL: "info"
+      CUSTOM_CONF_MODSEC_test_custom_conf: 'SecRule REQUEST_FILENAME "@rx ^/db" "id:10000,ctl:ruleRemoveByTag=attack-generic,ctl:ruleRemoveByTag=attack-protocol,nolog"'
+      CUSTOM_CONF_SERVER_HTTP_ready: |
+        location /ready {
+          default_type 'text/plain';
+          rewrite_by_lua_block {
+            ngx.print('ready')
+            ngx.flush(true)
+            ngx.exit(ngx.HTTP_OK)
+          }
+        }
+    networks:
+      bw-universe:
+      bw-services:
+        ipv4_address: 192.168.0.2
+
+  bw-scheduler:
+    image: bunkerity/bunkerweb-scheduler:1.5.9
+    pull_policy: never
+    depends_on:
+      - bw
+      - bw-docker
+    volumes:
+      - bw-data:/data/lib
+      - bw-volume:/usr/share/bunkerweb
+      - ./init/plugins:/data/plugins
+    environment:
+      DOCKER_HOST: "tcp://bw-docker:2375"
+      LOG_LEVEL: "debug"
+      # ? DATABASE settings
+      DATABASE_URI: "sqlite:////var/lib/bunkerweb/db.sqlite3"
+    networks:
+      - bw-universe
+      - bw-docker
+
+  bw-docker:
+    image: tecnativa/docker-socket-proxy:nightly
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+      CONTAINERS: "1"
+    networks:
+      - bw-docker
+
+  app1:
+    image: nginxdemos/nginx-hello
+    networks:
+      bw-services:
+        ipv4_address: 192.168.0.4
+
+volumes:
+  bw-data:
+    name: bw-data
+  bw-volume:
+    name: bw-volume
+
+networks:
+  bw-universe:
+    name: bw-universe
+    ipam:
+      driver: default
+      config:
+        - subnet: 10.20.30.0/24
+  bw-services:
+    name: bw-services
+    ipam:
+      driver: default
+      config:
+        - subnet: 192.168.0.0/24
+  bw-docker:
+    external: true

tests/core/db/main.py

@@ -21,6 +21,7 @@ from bunkerweb.db.model import (
     Custom_configs,
     Global_values,
     Jobs,
+    Jobs_runs,
     Metadata,
     Plugins,
     Plugin_pages,
@@ -399,7 +400,7 @@ try:
             if job.plugin_id in pro_plugin_ids:
                 continue
 
-            if job.name != "download-pro-plugins" and not job.success:
+            if job.name != "download-pro-plugins" and not all(job_run.success for job_run in session.query(Jobs_runs).filter_by(job_name=job.name)):
                 print(
                     f"❌ The job {job.name} (plugin_id: {job.plugin_id}) is in the database but failed, exiting ...",
                     flush=True,
@@ -463,8 +464,7 @@ try:
             .with_entities(
                 Plugin_pages.id,
                 Plugin_pages.plugin_id,
-                Plugin_pages.template_checksum,
-                Plugin_pages.actions_checksum,
+                Plugin_pages.checksum,
             )
             .all()
         )

tests/core/db/test.sh

@@ -132,10 +132,15 @@ cleanup_stack () {
 
     if [ "$integration" == "docker" ] ; then
         soft_cleanup=$1
+        compose_file="docker-compose.yml"
+        if [ "$2" == "old" ] ; then
+            compose_file="docker-compose.old.yml"
+        fi
+
         if [ "$soft_cleanup" = "1" ] ; then
-            docker compose down
+            docker compose -f "$compose_file" down
         else
-            docker compose down -v --remove-orphans
+            docker compose -f "$compose_file" down -v --remove-orphans
         fi
 
         if [[ $end -eq 0 && $exit_code = 1 ]] && [ $manual = 0 ] ; then
@@ -169,7 +174,12 @@ trap cleanup_stack EXIT
 starting_stack () {
     echo "💾 Starting stack ..."
     if [ "$integration" == "docker" ] ; then
-        docker compose up -d
+        compose_file="docker-compose.yml"
+        if [ "$1" == "old" ] ; then
+            compose_file="docker-compose.old.yml"
+        fi
+
+        docker compose -f "$compose_file" up -d
         # shellcheck disable=SC2181
         if [ $? -ne 0 ] ; then
             echo "💾 Up failed, retrying ... ⚠️"
@@ -195,7 +205,7 @@ starting_stack () {
                 fi
             fi
             manual=0
-            docker compose up -d
+            docker compose -f "$compose_file" up -d
             # shellcheck disable=SC2181
             if [ $? -ne 0 ] ; then
                 echo "💾 Up failed ❌"
@@ -217,6 +227,11 @@ waiting_stack () {
     echo "💾 Waiting for stack to be healthy ..."
     i=0
     if [ "$integration" == "docker" ] ; then
+        compose_file="docker-compose.yml"
+        if [ "$1" == "old" ] ; then
+            compose_file="docker-compose.old.yml"
+        fi
+
         while [ $i -lt 120 ] ; do
             containers=("db-bw-1" "db-bw-scheduler-1")
             healthy="true"
@@ -235,10 +250,9 @@ waiting_stack () {
             i=$((i+1))
         done
         if [ $i -ge 120 ] ; then
-            docker compose logs
             echo "💾 Docker stack is not healthy ❌"
-            echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..."
-            docker compose logs bw bw-scheduler
+            echo "🛡️ Showing logs ..."
+            docker compose -f "$compose_file" logs
             exit 1
         fi
     else
@@ -356,11 +370,12 @@ else
     sudo chmod -R 777 /etc/bunkerweb/plugins external bunkerweb
 fi
 
-tests="local multisite"
+tests="multisite upgrade"
+# tests="local multisite"
 
-if [ "$integration" == "docker" ] ; then
-    tests="$tests mariadb mysql postgres upgrade"
-fi
+# if [ "$integration" == "docker" ] ; then
+#     tests="$tests mariadb mysql postgres upgrade"
+# fi
 
 for test in $tests
 do
@@ -467,8 +482,8 @@ do
         older_version="$(curl -i https://github.com/bunkerity/bunkerweb/tags | grep -Po 'v[0-9]+\.[0-9]+\.[0-9]+' | uniq | sed -n 1p | cut -c 2-)"
         echo "💾 Running tests when upgrading from $older_version (older) to latest version ..."
         find . -type f -name 'docker-compose.*' -exec sed -i 's@DATABASE_URI: ".*"$@DATABASE_URI: "sqlite:////var/lib/bunkerweb/db.sqlite3"@' {} \;
-        sed -i 's@bunkerity/bunkerweb:.*$@bunkerity/bunkerweb:'"$older_version"'@' docker-compose.yml
-        sed -i 's@bunkerity/bunkerweb-scheduler:.*$@bunkerity/bunkerweb-scheduler:'"$older_version"'@' docker-compose.yml
+        sed -i 's@bunkerity/bunkerweb:.*$@bunkerity/bunkerweb:'"$older_version"'@' docker-compose.old.yml
+        sed -i 's@bunkerity/bunkerweb-scheduler:.*$@bunkerity/bunkerweb-scheduler:'"$older_version"'@' docker-compose.old.yml
 
         docker pull bunkerity/bunkerweb:"$older_version"
         # shellcheck disable=SC2181
@@ -484,12 +499,12 @@ do
             exit 1
         fi
 
-        starting_stack
+        starting_stack "old"
 
-        waiting_stack
+        waiting_stack "old"
 
         manual=1
-        cleanup_stack "1"
+        cleanup_stack "1" "old"
         manual=0
 
         sed -i 's@bunkerity/bunkerweb:.*$@bunkerity/bunkerweb:'"$release"'@' docker-compose.yml