ToolJet/.github/workflows/update-test-system.yml

name: Update test system (LTS and pre-release)

on:
  workflow_dispatch:
    inputs:
      branch_name:
        description: "Git branch to build from (required for deploy operations)"
        required: false
        default: "main"
      dockerfile_path:
        description: "Select Dockerfile (required for deploy operations)"
        required: false
        type: choice
        options:
          - ./docker/LTS/ee/ee-production.Dockerfile
          - ./docker/pre-release/ee/ee-production.Dockerfile
      docker_tag:
        description: "Docker tag suffix (e.g., pre-release-14). Leave blank if only managing env vars."
        required: false
        default: ""
      env_changes:
        description: "Environment changes (Format: ADD KEY=value, EDIT KEY=value, REMOVE KEY) - one per line. Leave blank if only deploying."
        required: false
        default: ""
      test_system:
        description: "Select test system"
        required: true
        type: choice
        options:
          - app-builder-3.16-lts
          - app-builder-pre-release
          - platform-3.16-lts
          - platform-pre-release
          - marketplace-3.16-lts
          - marketplace-pre-release
          - ai-3.16-lts
          - ai-pre-release

jobs:
  manage-environment:
    if: ${{ github.event.inputs.env_changes != '' }}
    runs-on: ubuntu-latest
    steps:
      
      - name: ✅ Check user authorization
        run: |
          allowed_users=(
            "${{ secrets.ALLOWED_USER1_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER2_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER3_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER4_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER5_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER6_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER7_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER8_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER9_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER10_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER11_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER12_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER13_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER14_TEST_SYSTEM }}"
          )
          current_user="${{ github.actor }}"
          authorized=false
          for user in "${allowed_users[@]}"; do
            if [[ "$current_user" == "$user" ]]; then
              authorized=true
              break
            fi
          done
          if [[ "$authorized" == "false" ]]; then
            echo "❌ User '$current_user' is not authorized to trigger this workflow."
            exit 1
          else
            echo "✅ User '$current_user' is authorized."
          fi          
      
      - name: Install SSH and JQ
        run: sudo apt-get update && sudo apt-get install -y jq openssh-client
      
      - name: Determine target host
        id: vmhost
        run: |
          test_system="${{ github.event.inputs.test_system }}"
          vm_host=$(echo '${{ secrets.VM_HOST_MAP_JSON }}' | jq -r --arg sys "$test_system" '.[$sys]')
          if [[ -z "$vm_host" || "$vm_host" == "null" ]]; then
            echo "VM mapping not found for $test_system"
            exit 1
          fi
          echo "host=$vm_host" >> $GITHUB_OUTPUT          
     
      - name: Update environment variables
        run: |
          echo "$SSH_KEY" > key.pem
          chmod 600 key.pem
          TARGET_SYSTEM="${{ github.event.inputs.test_system }}"
          ENV_CHANGES="${{ github.event.inputs.env_changes }}"
          ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR -i key.pem $SSH_USER@${{ steps.vmhost.outputs.host }} << EOF
          set -e
          TARGET_SYSTEM="$TARGET_SYSTEM"
          ENV_CHANGES="$ENV_CHANGES"
          cd ~
          echo "📁 Finding correct deployment directory"
          if [[ "\$TARGET_SYSTEM" == *-3.16-lts ]]; then
            echo "Detected LTS system: \$TARGET_SYSTEM"
            echo "🔍 Searching for LTS directories..."
            LTS_DIRS=\$(ls -1d ./*-lts 2>/dev/null | grep -E '[0-9]+\.[0-9]+' | sed 's|^\./||' | sort -V; \\
                       ls -1d ./*-lts 2>/dev/null | grep -Ev '[0-9]+\.[0-9]+' | sed 's|^\./||' | sort)
            if [[ -z "\$LTS_DIRS" ]]; then
              echo "❌ No LTS directories found!"
              echo "Available directories:"
              ls -la | grep "^d"
              exit 1
            fi
            echo "Available LTS directories:"
            echo "\$LTS_DIRS"
            SELECTED_LTS_DIR=\$(echo "\$LTS_DIRS" | head -n 1)
            echo "📂 Selected LTS directory: \$SELECTED_LTS_DIR"
            cd "\$SELECTED_LTS_DIR"
            echo "✅ Now in directory: \$(pwd)"
          else
            echo "Detected pre-release system: \$TARGET_SYSTEM"
            echo "📂 Working in home directory: \$(pwd)"
          fi
          echo ""
          echo "🔧 PROCESSING ENVIRONMENT VARIABLE CHANGES"
          BACKUP_FILE=".env.backup.\$(date +%s)"
          sudo cp .env "\$BACKUP_FILE"
          echo "✅ Backup created: \$BACKUP_FILE"
          PROTECTED_VARS="TOOLJET_HOST|LOCKBOX_MASTER_KEY|SECRET_KEY_BASE|ORM_LOGGING|PG_DB|PG_USER|PG_HOST|PG_PASS|TOOLJET_DB|TOOLJET_DB_USER|TOOLJET_DB_HOST|TOOLJET_DB_PASS|PGRST_DB_URI|PGRST_HOST|PGRST_JWT_SECRET|PGRST_SERVER_PORT|REDIS_HOST|REDIS_PORT|REDIS_USER|REDIS_PASSWORD|OLD_IMAGE|TOOLJET_IMAGE"
          ADD_SUCCESS=0
          ADD_FAIL=0
          EDIT_SUCCESS=0
          EDIT_FAIL=0
          REMOVE_SUCCESS=0
          REMOVE_FAIL=0
          while IFS= read -r line; do
            line=\$(echo "\$line" | xargs)
            [[ -z "\$line" || "\$line" =~ ^# ]] && continue
            if [[ "\$line" =~ ^ADD[[:space:]]+([^=]+)=(.*)$ ]]; then
              KEY="\${BASH_REMATCH[1]}"
              VALUE="\${BASH_REMATCH[2]}"
              if echo "\$KEY" | grep -qE "\$PROTECTED_VARS"; then
                echo "❌ FAILED: Cannot add protected variable '\$KEY'"
                ADD_FAIL=\$((ADD_FAIL + 1))
                continue
              fi
              if grep -q "^\${KEY}=" .env; then
                echo "⚠️  SKIPPED: Variable '\$KEY' already exists (use EDIT)"
                continue
              else
                if echo "\${KEY}=\${VALUE}" | sudo tee -a .env > /dev/null; then
                  echo "✅ SUCCESS: Added '\$KEY'"
                  ADD_SUCCESS=\$((ADD_SUCCESS + 1))
                else
                  echo "❌ FAILED: Could not add '\$KEY'"
                  ADD_FAIL=\$((ADD_FAIL + 1))
                fi
              fi
            elif [[ "\$line" =~ ^EDIT[[:space:]]+([^=]+)=(.*)$ ]]; then
              KEY="\${BASH_REMATCH[1]}"
              VALUE="\${BASH_REMATCH[2]}"
              if echo "\$KEY" | grep -qE "\$PROTECTED_VARS"; then
                echo "❌ FAILED: Cannot edit protected variable '\$KEY'"
                EDIT_FAIL=\$((EDIT_FAIL + 1))
                continue
              fi
              if grep -q "^\${KEY}=" .env; then
                if sudo sed -i "s|^\${KEY}=.*|\${KEY}=\${VALUE}|" .env; then
                  echo "✅ SUCCESS: Edited '\$KEY'"
                  EDIT_SUCCESS=\$((EDIT_SUCCESS + 1))
                else
                  echo "❌ FAILED: Could not edit '\$KEY'"
                  EDIT_FAIL=\$((EDIT_FAIL + 1))
                fi
              else
                echo "⚠️  SKIPPED: Variable '\$KEY' not found (use ADD)"
                continue
              fi
            elif [[ "\$line" =~ ^REMOVE[[:space:]]+([^=[:space:]]+)$ ]]; then
              KEY="\${BASH_REMATCH[1]}"
              if echo "\$KEY" | grep -qE "\$PROTECTED_VARS"; then
                echo "❌ FAILED: Cannot remove protected variable '\$KEY'"
                REMOVE_FAIL=\$((REMOVE_FAIL + 1))
                continue
              fi
              if grep -q "^\${KEY}=" .env; then
                if sudo sed -i "/^\${KEY}=/d" .env; then
                  echo "✅ SUCCESS: Removed '\$KEY'"
                  REMOVE_SUCCESS=\$((REMOVE_SUCCESS + 1))
                else
                  echo "❌ FAILED: Could not remove '\$KEY'"
                  REMOVE_FAIL=\$((REMOVE_FAIL + 1))
                fi
              else
                echo "⚠️  SKIPPED: Variable '\$KEY' not found"
                continue
              fi
            else
              echo "⚠️  INVALID FORMAT: \$line"
            fi
          done <<< "\$ENV_CHANGES"
          echo ""
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo "📊 SUMMARY"
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo "ADD:    ✅ \$ADD_SUCCESS succeeded | ❌ \$ADD_FAIL failed"
          echo "EDIT:   ✅ \$EDIT_SUCCESS succeeded | ❌ \$EDIT_FAIL failed"
          echo "REMOVE: ✅ \$REMOVE_SUCCESS succeeded | ❌ \$REMOVE_FAIL failed"
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          TOTAL_SUCCESS=\$((ADD_SUCCESS + EDIT_SUCCESS + REMOVE_SUCCESS))
          TOTAL_FAIL=\$((ADD_FAIL + EDIT_FAIL + REMOVE_FAIL))
          if [ \$TOTAL_SUCCESS -eq 0 ]; then
            echo "⚠️  No changes were applied"
            sudo cp "\$BACKUP_FILE" .env
            exit 1
          fi
          echo ""
          echo "🔄 Restarting containers..."
          sudo docker-compose down
          sudo docker-compose up -d
          echo "⏳ Waiting for containers (timeout: 120s)..."
          SUCCESS_FOUND=false
          TIMEOUT=120
          ELAPSED=0
          while [ \$ELAPSED -lt \$TIMEOUT ]; do
            if sudo docker-compose logs 2>/dev/null | grep -qE "🚀 TOOLJET APPLICATION STARTED SUCCESSFULLY|Ready to use at http://localhost:82 🚀|Ready to use at http://localhost:80"; then
              SUCCESS_FOUND=true
              break
            fi
            sleep 10
            ELAPSED=\$((ELAPSED + 10))
          done
          if [ "\$SUCCESS_FOUND" = false ]; then
            echo "❌ Container startup failed"
            echo "🔄 Rolling back..."
            sudo cp "\$BACKUP_FILE" .env
            sudo docker-compose down
            sudo docker-compose up -d
            echo "✅ Rollback completed"
            exit 1
          fi
          echo "✅ Environment variables updated successfully!"
          echo "🧹 Cleaning up old backups..."
          ls -t .env.backup.* 2>/dev/null | tail -n +2 | xargs -r sudo rm -f
          EOF          
        env:
          SSH_USER: ${{ secrets.AZURE_VM_USER }}
          SSH_KEY: ${{ secrets.AZURE_VM_KEY }}

  build-and-deploy:
    if: ${{ !cancelled() && github.event.inputs.docker_tag != '' }}
    needs: manage-environment
    runs-on: ubuntu-latest
    steps:
      - name: Validate required inputs
        run: |
          if [[ -z "${{ github.event.inputs.branch_name }}" ]]; then
            echo "❌ Error: branch_name is required"
            exit 1
          fi
          if [[ -z "${{ github.event.inputs.dockerfile_path }}" ]]; then
            echo "❌ Error: dockerfile_path is required"
            exit 1
          fi          
      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          sudo docker system prune -af
          sudo apt-get clean
          df -h          
      - name: ✅ Check user authorization
        run: |
          allowed_users=(
            "${{ secrets.ALLOWED_USER1_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER2_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER3_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER4_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER5_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER6_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER7_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER8_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER9_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER10_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER11_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER12_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER13_TEST_SYSTEM }}"
            "${{ secrets.ALLOWED_USER14_TEST_SYSTEM }}"
          )
          current_user="${{ github.actor }}"
          authorized=false
          for user in "${allowed_users[@]}"; do
            if [[ "$current_user" == "$user" ]]; then
              authorized=true
              break
            fi
          done
          if [[ "$authorized" == "false" ]]; then
            echo "❌ User '$current_user' is not authorized to trigger this workflow."
            exit 1
          else
            echo "✅ User '$current_user' is authorized."
          fi          
      
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.inputs.branch_name }}
          fetch-depth: 0
      
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      
      - name: Log in to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PAT }}
      
      - name: Generate full Docker tag
        id: taggen
        run: |
          input_tag="${{ github.event.inputs.docker_tag }}"
          if [[ "$input_tag" == *"/"* ]]; then
            echo "tag=$input_tag" >> $GITHUB_OUTPUT
          else
            echo "tag=tooljet/tj-osv:$input_tag" >> $GITHUB_OUTPUT
          fi          
      
      - name: Build and Push Docker image
        uses: docker/build-push-action@v4
        with:
          context: .
          file: ${{ github.event.inputs.dockerfile_path }}
          push: true
          tags: ${{ steps.taggen.outputs.tag }}
          platforms: linux/amd64
          build-args: |
            CUSTOM_GITHUB_TOKEN=${{ secrets.CUSTOM_GITHUB_TOKEN }}
            BRANCH_NAME=${{ github.event.inputs.branch_name }}            
      
      - name: Show the full Docker tag
        run: echo "✅ Docker image built and pushed:${{ steps.taggen.outputs.tag }}"
      
      - name: Install SSH and JQ
        run: sudo apt-get update && sudo apt-get install -y jq openssh-client
      
      - name: Determine target host
        id: vmhost
        run: |
          test_system="${{ github.event.inputs.test_system }}"
          vm_host=$(echo '${{ secrets.VM_HOST_MAP_JSON }}' | jq -r --arg sys "$test_system" '.[$sys]')
          if [[ -z "$vm_host" || "$vm_host" == "null" ]]; then
            echo "VM mapping not found for $test_system"
            exit 1
          fi
          echo "host=$vm_host" >> $GITHUB_OUTPUT          
      
      - name: Deploy to target environment
        run: |
          echo "$SSH_KEY" > key.pem
          chmod 600 key.pem
          IMAGE_TAG="${{ steps.taggen.outputs.tag }}"
          TARGET_SYSTEM="${{ github.event.inputs.test_system }}"
          ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR -i key.pem $SSH_USER@${{ steps.vmhost.outputs.host }} << EOF
          set -e
          IMAGE_TAG="$IMAGE_TAG"
          TARGET_SYSTEM="$TARGET_SYSTEM"
          cd ~
          echo "📁 Finding correct deployment directory"
          if [[ "\$TARGET_SYSTEM" == *-3.16-lts ]]; then
            echo "Detected LTS system: \$TARGET_SYSTEM"
            echo "🔍 Searching for LTS directories..."
            LTS_DIRS=\$(ls -1d ./*-lts 2>/dev/null | grep -E '[0-9]+\.[0-9]+' | sed 's|^\./||' | sort -V; \\
                       ls -1d ./*-lts 2>/dev/null | grep -Ev '[0-9]+\.[0-9]+' | sed 's|^\./||' | sort)
            if [[ -z "\$LTS_DIRS" ]]; then
              echo "❌ No LTS directories found!"
              echo "Available directories:"
              ls -la | grep "^d"
              exit 1
            fi
            echo "Available LTS directories:"
            echo "\$LTS_DIRS"
            SELECTED_LTS_DIR=\$(echo "\$LTS_DIRS" | head -n 1)
            echo "📂 Selected LTS directory: \$SELECTED_LTS_DIR"
            cd "\$SELECTED_LTS_DIR"
            echo "✅ Now in directory: \$(pwd)"
          else
            echo "Detected pre-release system: \$TARGET_SYSTEM"
            echo "📂 Moving to target directory: \$TARGET_SYSTEM"
            cd ~
            echo "✅ Now in directory: \$(pwd)"
          fi
          echo "🔐 Docker login"
          echo "${{ secrets.DOCKER_PAT }}" | sudo docker login --username "${{ secrets.DOCKER_USERNAME }}" --password-stdin
          echo "current image"
          cat .env | grep TOOLJET_IMAGE
          echo "📦 Reading current TOOLJET_IMAGE from .env"
          CURRENT_IMAGE=\$(grep '^TOOLJET_IMAGE=' .env | cut -d '=' -f2- | tr -d '"' | tr -d "'")
          echo "Found CURRENT_IMAGE: \$CURRENT_IMAGE"
          echo "🛑 Stopping containers"
          sudo docker-compose down
          echo "📝 Updating .env with new image"
          sudo sed -i "s|^TOOLJET_IMAGE=.*|TOOLJET_IMAGE=\$IMAGE_TAG|" .env
          echo "📥 Pulling new image: \$IMAGE_TAG"
          if [ -z "\$IMAGE_TAG" ]; then
            echo "❌ IMAGE_TAG is empty!"
            exit 1
          fi
          sudo docker pull "\$IMAGE_TAG"
          echo "🚀 Starting container in background"
          sudo docker-compose up -d
          echo "⏳ Waiting for ToolJet to start (timeout: 300 seconds)..."
          SUCCESS_FOUND=false
          TIMEOUT=300
          ELAPSED=0
          while [ \$ELAPSED -lt \$TIMEOUT ]; do
            if sudo docker-compose logs 2>/dev/null | grep -qE "🚀 TOOLJET APPLICATION STARTED SUCCESSFULLY|Ready to use at http://localhost:82 🚀|Ready to use at http://localhost:80"; then
              echo "✅ Found success message in logs!"
              SUCCESS_FOUND=true
              break
            fi
            echo "⏳ Still waiting... (\${ELAPSED}s elapsed)"
            sleep 10
            ELAPSED=\$((ELAPSED + 10))
          done
          if [ "\$SUCCESS_FOUND" = false ]; then
            echo "❌ Timeout reached without finding success logs"
            echo "📄 Showing current logs for troubleshooting..."
            sudo docker-compose logs --tail=50
            echo ""
            echo "=== CONTAINER STATUS ==="
            sudo docker-compose ps
            echo ""
            echo "🛑 Starting rollback process..."
            sudo docker-compose down
            echo "🔄 Reverting to previous image: \$CURRENT_IMAGE"
            sudo sed -i "s|^TOOLJET_IMAGE=.*|TOOLJET_IMAGE=\$CURRENT_IMAGE|" .env
            echo "🔄 Starting previous image..."
            sudo docker-compose up -d
            echo "✅ Rollback completed!"
            exit 1
          fi
          echo "✅ Deployment successful!"
          echo "📌 Storing successful deployment info in .env"
          sudo sed -i "/^OLD_IMAGE=/d" .env
          echo "OLD_IMAGE=\$CURRENT_IMAGE" | sudo tee -a .env
          echo "📄 Final application logs:"
          sudo docker-compose logs --tail=50
          echo "🧹 Pruning old Docker images"
          sudo docker image prune -a -f
          EOF          
        env:
          SSH_USER: ${{ secrets.AZURE_VM_USER }}
          SSH_KEY: ${{ secrets.AZURE_VM_KEY }}