name: Go Tests

on:
  push:
    branches:
      - main
      - patch-*
      - prepare-*
    paths:
      - '**.go'
      - 'go.mod'
      - 'go.sum'
      - '.github/workflows/test-go.yaml'
      - 'tools/osquery/in-a-box/docker-compose.yml'
      - 'tools/osquery/in-a-box/osquery/docker-compose.yml'
      - 'server/authz/policy.rego'
      - 'docker-compose.yml'
  pull_request:
    paths:
      - '**.go'
      - 'go.mod'
      - 'go.sum'
      - '.github/workflows/test-go.yaml'
      - 'tools/osquery/in-a-box/docker-compose.yml'
      - 'tools/osquery/in-a-box/osquery/docker-compose.yml'
      - 'server/authz/policy.rego'
      - 'docker-compose.yml'
  workflow_dispatch: # Manual
  schedule:
    - cron: '0 4 * * *'

# This allows a subsequently queued workflow run to interrupt previous runs
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id}}
  cancel-in-progress: true

defaults:
  run:
    # fail-fast using bash -eo pipefail. See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
    shell: bash

permissions:
  contents: read

jobs:
  test-go:
    strategy:
      matrix:
        suite: ["integration-core", "integration-enterprise", "integration-mdm", "fast", "fleetctl", "main", "mysql", "scripts", "service", "vuln"]
        os: [ubuntu-latest]
        mysql: ["mysql:8.0.32", "mysql:8.0.36", "mysql:8.4.6", "mysql:9.4.0"] # make sure to update supported versions docs when this changes
        isCron:
          - ${{ github.event_name == 'schedule' }}
        exclude:
          - isCron: false
            mysql: "mysql:8.4.6" # Run MySQL 8.4.X tests on cron schedule and not every time. We run MySQL 9.X tests every time since they are faster than 8.X tests.
          - isCron: false
            mysql: "mysql:8.0.32" # Run MySQL 8.0.32 tests on cron schedule and not every time.
          # The suites below do not need MySQL, so we exclude additional MySQL options from the above matrix.
          - suite: "fast"
            mysql: "mysql:8.0.32" # We must make sure that at least 1 instance of this suite will run, which is 8.0.36 in this case
          - suite: "fast"
            mysql: "mysql:8.4.6"
          - suite: "fast"
            mysql: "mysql:9.4.0"
          - suite: "scripts"
            mysql: "mysql:8.0.32"
          - suite: "scripts"
            mysql: "mysql:8.4.6"
          - suite: "scripts"
            mysql: "mysql:9.4.0"
    # Don't cancel other jobs if one test suite fails. Since our product teams are tightly coupled, we never want to see our tests fail due
    # to an unrelated issue in another product area.
    continue-on-error: true
    runs-on: ${{ matrix.os }}

    env:
      RACE_ENABLED: false
      GO_TEST_TIMEOUT: 20m
      DOCKER_COMMAND: docker compose -f docker-compose.yml -f docker-compose-redis-cluster.yml up -d mysql_test mysql_replica_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup minio saml_idp mailhog mailpit smtp4dev_test

    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
      with:
        egress-policy: audit

    - name: Configure job
      run: |
        echo "RUN_TESTS_ARG=" >> $GITHUB_ENV
        if [[ "${{ matrix.suite }}" == "main" ]]; then
          echo "CI_TEST_PKG=main" >> $GITHUB_ENV
          echo "NEED_DOCKER=1" >> $GITHUB_ENV
          echo "DOCKER_COMMAND=${{ env.DOCKER_COMMAND }} localstack" >> $GITHUB_ENV
        elif [[ "${{ matrix.suite }}" == "fast" ]]; then
          # DO NOT add any dependencies in this test suite.
          echo "CI_TEST_PKG=${{ matrix.suite }}" >> $GITHUB_ENV
        elif [[ "${{ matrix.suite }}" == "service" ]]; then
          echo "CI_TEST_PKG=service" >> $GITHUB_ENV
          echo "RUN_TESTS_ARG=-skip=^TestIntegrations" >> $GITHUB_ENV
          echo "NEED_DOCKER=1" >> $GITHUB_ENV
        elif [[ "${{ matrix.suite }}" == "integration-core" ]]; then
          echo "CI_TEST_PKG=service" >> $GITHUB_ENV
          echo "RUN_TESTS_ARG=-run=^TestIntegrations -skip '^(TestIntegrationsMDM|TestIntegrationsEnterprise)'" >> $GITHUB_ENV
          # We re-generate test schema just in case there is an issue with the schema. We only do this for one test.
          echo "GENERATE_TEST_SCHEMA=1" >> $GITHUB_ENV
          echo "NEED_DOCKER=1" >> $GITHUB_ENV
        elif [[ "${{ matrix.suite }}" == "integration-mdm" ]]; then
          echo "CI_TEST_PKG=service" >> $GITHUB_ENV
          echo "RUN_TESTS_ARG=-run=^TestIntegrationsMDM" >> $GITHUB_ENV
          echo "NEED_DOCKER=1" >> $GITHUB_ENV
        elif [[ "${{ matrix.suite }}" == "integration-enterprise" ]]; then
          echo "CI_TEST_PKG=service" >> $GITHUB_ENV
          echo "RUN_TESTS_ARG=-run=^TestIntegrationsEnterprise" >> $GITHUB_ENV
          echo "NEED_DOCKER=1" >> $GITHUB_ENV
        elif [[ "${{ matrix.suite }}" == "scripts" ]]; then
          echo "CI_TEST_PKG=${{ matrix.suite }}" >> $GITHUB_ENV
          echo "NEED_ZSH=1" >> $GITHUB_ENV
        else
          echo "CI_TEST_PKG=${{ matrix.suite }}" >> $GITHUB_ENV
          echo "NEED_DOCKER=1" >> $GITHUB_ENV
        fi

    - name: Set Go race setting on schedule
      if: github.event.schedule == '0 4 * * *'
      run: |
        echo "RACE_ENABLED=true" >> $GITHUB_ENV
        echo "GO_TEST_TIMEOUT=1h" >> $GITHUB_ENV

    - name: Checkout Code
      uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

    - name: Install Go
      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
      with:
        go-version-file: 'go.mod'

    # Pre-starting dependencies here means they are ready to go when we need them.
    - name: Start Infra Dependencies
      if: ${{ env.NEED_DOCKER }}
      # Use & to background this
      run: FLEET_MYSQL_IMAGE=${{ matrix.mysql }} $DOCKER_COMMAND &

    - name: Add TLS certificate for SMTP Tests
      if: ${{ env.NEED_DOCKER }}
      run: |
        sudo cp tools/smtp4dev/fleet.crt /usr/local/share/ca-certificates/
        sudo update-ca-certificates

    - name: Install ZSH
      if: ${{ env.NEED_ZSH }}
      run: sudo apt update && sudo apt install -y zsh

    - name: Generate static files
      run: |
        export PATH=$PATH:~/go/bin
        make generate-go

    - name: Wait for mysql
      if: ${{ env.NEED_DOCKER }}
      run: |
        # Function to wait for MySQL with timeout
        wait_for_mysql() {
          local container_name=$1
          local timeout_seconds=60  # 1 minute
          local start_time=$(date +%s)
          local attempt_logs=""
          
          echo "waiting for ${container_name}..."
          while true; do
            # Check if timeout has been reached
            current_time=$(date +%s)
            elapsed_time=$((current_time - start_time))
            if [ $elapsed_time -ge $timeout_seconds ]; then
              echo "Timeout reached (${timeout_seconds}s) while waiting for ${container_name}"
              echo "Connection attempt logs:"
              echo "$attempt_logs"
              # Dump MySQL container logs
              echo "Dumping ${container_name} logs:"
              docker compose logs ${container_name}
              return 1
            fi
            
            # Try to connect to MySQL
            output=$(docker compose exec -T $container_name sh -c "mysql -uroot -p\"\${MYSQL_ROOT_PASSWORD}\" -e \"SELECT 1=1\" fleet" 2>&1)
            exit_code=$?
            
            # Log the attempt
            timestamp=$(date "+%Y-%m-%d %H:%M:%S")
            attempt_logs="${attempt_logs}$(printf "\n%s - Exit code: %s - Output: %s" "$timestamp" "$exit_code" "$output")"
        
            # If connection successful, break the loop
            if [ $exit_code -eq 0 ]; then
              echo "${container_name} is ready"
              return 0
            fi
            
            echo "."
            sleep 1
          done
        }
        
        # Function to restart containers
        restart_containers() {
          echo "Stopping all containers..."
          docker compose down
          
          echo "Restarting containers..."
          FLEET_MYSQL_IMAGE=${{ matrix.mysql }} $DOCKER_COMMAND &
          
          # Give containers a moment to start
          sleep 10
        }
        
        # Try up to 5 times to connect to MySQL
        max_attempts=5
        attempt=1
        
        while [ $attempt -le $max_attempts ]; do
          echo "Attempt $attempt of $max_attempts"
          
          # Try to connect to MySQL
          if wait_for_mysql "mysql_test"; then
            # If MySQL is ready, try to connect to MySQL replica
            if wait_for_mysql "mysql_replica_test"; then
              # Both are ready, we're done
              echo "All MySQL connections successful"
              exit 0
            fi
          fi
          
          # If we get here, at least one connection failed
          echo "Failed to connect to MySQL on attempt $attempt"
          
          if [ $attempt -lt $max_attempts ]; then
            echo "Restarting containers and trying again..."
            restart_containers
          else
            echo "Maximum attempts reached. Failing the job."
            exit 1
          fi
          
          attempt=$((attempt + 1))
        done

    - name: Generate test schema
      if: ${{ env.GENERATE_TEST_SCHEMA }}
      run: make test-schema

    - name: Run Go Tests
      run: |
        GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT ${{ env.RUN_TESTS_ARG }}" \
          TEST_LOCK_FILE_PATH=$(pwd)/lock \
          TEST_CRON_NO_RECOVER=1 \
          NETWORK_TEST=1 \
          REDIS_TEST=1 \
          MYSQL_TEST=1 \
          MYSQL_REPLICA_TEST=1 \
          MINIO_STORAGE_TEST=1 \
          SAML_IDP_TEST=1 \
          MAIL_TEST=1 \
          AWS_ENDPOINT_URL="http://localhost:4566" \
          AWS_REGION=us-east-1 \
          NETWORK_TEST_GITHUB_TOKEN=${{ secrets.FLEET_RELEASE_GITHUB_PAT }} \
          CI_TEST_PKG="${{ env.CI_TEST_PKG }}" \
          make test-go 2>&1 | tee /tmp/gotest.log

    - name: Create mysql identifier without colon
      if: always()
      run: |
        echo "MATRIX_MYSQL_ID=$(echo ${{ matrix.mysql }} | tr -d ':')" >> $GITHUB_ENV

    - name: Save coverage
      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
      with:
        name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-coverage
        path: ./coverage.txt
        if-no-files-found: error

    - name: Generate summary of errors
      if: failure()
      run: |
        c1grep() { grep "$@" || test $? = 1; }
        c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
        c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
        c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
        c1grep -A 10 'panic: runtime error: ' /tmp/gotest.log >> /tmp/summary.txt
        c1grep ' FAIL\t' /tmp/gotest.log >> /tmp/summary.txt
        GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
        echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
        if [[ -z "$GO_FAIL_SUMMARY" ]]; then
          GO_FAIL_SUMMARY="unknown, please check the build URL"
        fi
        GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json

    - name: Slack Notification
      if: github.event.schedule == '0 4 * * *' && failure()
      uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
      with:
        payload-file-path: ./payload.json
      env:
        JOB_STATUS: ${{ job.status }}
        EVENT_URL: ${{ github.event.pull_request.html_url || github.event.head.html_url }}
        RUN_URL: https://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

    - name: Upload test log
      if: always()
      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
      with:
        name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-test-log
        path: /tmp/gotest.log
        if-no-files-found: error

    - name: Upload summary test log
      if: always()
      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  # v4.3.6
      with:
        name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-summary-test-log
        path: /tmp/summary.txt

    - name: Set test status
      if: always()
      run: |
        if [[ "${{ job.status }}" == "success" ]]; then
          echo "success" > /tmp/status
        else
          echo "fail" > /tmp/status
        fi

    - name: Upload status indicator
      if: always()
      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  # v4.3.6
      with:
        name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-status
        path: /tmp/status
        overwrite: true

  # Based on https://github.com/micromdm/nanomdm/blob/main/.github/workflows/on-push-pr.yml#L87
  test-go-nanomdm:
    runs-on: 'ubuntu-latest'
    services:
      mysql:
        image: mysql:8.0.36
        env:
          MYSQL_RANDOM_ROOT_PASSWORD: yes
          MYSQL_DATABASE: testdb
          MYSQL_USER: testuser
          MYSQL_PASSWORD: testpw
        ports:
          - 3800:3306
        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
    env:
      MYSQL_PWD: testpw
      PORT: 3800
      RACE_ENABLED: true
      GO_TEST_TIMEOUT: 20m
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - name: Checkout Code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

      - name: Install Go
        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
        with:
          go-version-file: 'go.mod'

      - name: verify mysql
        run: |
          while ! mysqladmin ping --host=localhost --port=$PORT --protocol=TCP --silent; do
            sleep 1
          done

      - name: mysql schema
        run: |
          mysql --version
          mysql --user=testuser --host=localhost --port=$PORT --protocol=TCP testdb < ./server/mdm/nanomdm/storage/mysql/schema.sql

      - name: set test dsn
        run: echo "NANOMDM_MYSQL_STORAGE_TEST_DSN=testuser:testpw@tcp(localhost:$PORT)/testdb" >> $GITHUB_ENV

      - name: Run Go tests
        run: |
          go test -v -parallel 8 -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT \
            -coverprofile=coverage.txt -covermode=atomic -coverpkg=github.com/fleetdm/fleet/v4/server/mdm/nanomdm/... \
          ./server/mdm/nanomdm/storage/mysql 2>&1 | tee /tmp/gotest.log

      - name: Save coverage
        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        with:
          name: nanomdm-coverage
          path: ./coverage.txt
          if-no-files-found: error

      - name: Generate summary of errors
        if: failure()
        run: |
          c1grep() { grep "$@" || test $? = 1; }
          c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
          c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
          c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
          c1grep -A 10 'panic: runtime error: ' /tmp/gotest.log >> /tmp/summary.txt
          c1grep ' FAIL\t' /tmp/gotest.log >> /tmp/summary.txt
          GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
          echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
          if [[ -z "$GO_FAIL_SUMMARY" ]]; then
            GO_FAIL_SUMMARY="unknown, please check the build URL"
          fi
          GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json

      - name: Slack Notification
        if: github.event.schedule == '0 4 * * *' && failure()
        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
        with:
          payload-file-path: ./payload.json
        env:
          JOB_STATUS: ${{ job.status }}
          EVENT_URL: ${{ github.event.pull_request.html_url || github.event.head.html_url }}
          RUN_URL: https://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

      - name: Upload test log
        if: always()
        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        with:
          name: nanomdm-test-log
          path: /tmp/gotest.log
          if-no-files-found: error

      - name: Upload summary test log
        if: always()
        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  # v4.3.6
        with:
          name: nanomdm-summary-test-log
          path: /tmp/summary.txt

  # We upload all backend coverage in one step so that we're less like to end up in a situation with a partial coverage report.
  upload-coverage:
    needs: [test-go, test-go-nanomdm]
    runs-on: ubuntu-latest
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - name: Checkout Code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
      - name: Download artifacts
        uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
        with:
          pattern: '*-coverage'
      - name: Upload to Codecov
        uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: backend

  # Our Go matrix test suites are run with continue-on-error: true, so they don't contribute to the workflow pass/fail.
  # This job explicitly checks if any Go matrix test suites have failed and marks the overall workflow with the proper pass/fail status.
  aggregate-result:
    needs: test-go
    runs-on: ubuntu-latest
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - name: Download artifacts
        uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
        with:
          pattern: '*-status'

      - name: Check for failures
        run: |
          failed_tests=""
          status_count=0
          # Find all status files (they are in directories like 'fleetctl-mysql8.0.36-status/status')
          for status_file in $(find ./ -type f -name 'status'); do
            status_count=$((status_count + 1))
            # Extract test name from parent directory (e.g., 'fleetctl-mysql8.0.36-status')
            test_dir=$(basename $(dirname "$status_file"))
            # Remove '-status' suffix to get the test name
            test_name="${test_dir%-status}"
            status_content=$(cat "$status_file")
            echo "Processing: $status_file (Test: $test_name) with status content: $status_content"
            if grep -q "fail" "$status_file"; then
              echo "  ❌ Test failed: $test_name"
              failed_tests="${failed_tests}${test_name}, "
            else
              echo "  ✅ Test passed: $test_name"
            fi
          done
          if [[ $status_count -eq 0 ]]; then
            echo "❌ ERROR: No status files found! This indicates a workflow issue."
            exit 1
          fi
          if [[ -n "$failed_tests" ]]; then
            echo "❌ One or more test jobs failed: ${failed_tests%, }"
            exit 1
          fi
          echo "✅ All test jobs succeeded."