fleet/.github/workflows/randokiller-go.yml

name: Stress Test Go Test
# A tool to test/debug/reproduce flaky tests. Update randokiller.json to point at a specific test, like ^TestIntegrationsMDM\$$/^TestVPPApps$

on:
  push:
    branches:
      - "**-randokiller"
    paths:
      - "**.go"
      - "go.mod"
      - "go.sum"
      - ".github/workflows/randokiller-go.yml"
      - "docker-compose.yml"
      - ".github/workflows/config/randokiller.json"
  workflow_dispatch: # manual

# This allows a subsequently queued workflow run to interrupt previous runs
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id}}
  cancel-in-progress: true

defaults:
  run:
    # fail-fast using bash -eo pipefail. See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
    shell: bash

permissions:
  contents: read

jobs:
  parse_config:
    runs-on: ubuntu-latest
    outputs:
      json: ${{steps.get_config_json.outputs.json}}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - name: Checkout Code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

      - name: Parse Config
        id: get_config_json
        run: echo "json=$(jq -c . < .github/workflows/config/randokiller.json)" >> $GITHUB_OUTPUT

  test-go:
    needs: parse_config
    strategy:
      matrix:
        os: [ubuntu-latest]
        mysql: ${{fromJson(needs.parse_config.outputs.json).mysql_matrix}}
    runs-on: ${{ matrix.os }}

    env:
      RACE_ENABLED: false
      GO_TEST_TIMEOUT: 20m

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - name: Checkout Code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

      - name: Install Go
        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
        with:
          go-version-file: "go.mod"

      # Pre-starting dependencies here means they are ready to go when we need them.
      - name: Start Infra Dependencies
        # Use & to background this
        run: FLEET_MYSQL_IMAGE=${{ matrix.mysql }} docker compose -f docker-compose.yml -f docker-compose-redis-cluster.yml up -d mysql_test mysql_replica_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup minio saml_idp mailhog mailpit smtp4dev_test &

      - name: Add TLS certificate for SMTP Tests
        run: |
          sudo cp tools/smtp4dev/fleet.crt /usr/local/share/ca-certificates/
          sudo update-ca-certificates          

      - name: Generate static files
        run: |
          export PATH=$PATH:~/go/bin
          make generate-go          

      - name: Install ZSH
        run: sudo apt update && sudo apt install -y zsh

      - name: Wait for mysql
        run: |
          echo "waiting for mysql..."
          until docker compose exec -T mysql_test sh -c "mysql -uroot -p\"\${MYSQL_ROOT_PASSWORD}\" -e \"SELECT 1=1\" fleet" &> /dev/null; do
              echo "."
              sleep 1
          done
          echo "mysql is ready"
          echo "waiting for mysql replica..."
          until docker compose exec -T mysql_replica_test sh -c "mysql -uroot -p\"\${MYSQL_ROOT_PASSWORD}\" -e \"SELECT 1=1\" fleet" &> /dev/null; do
              echo "."
              sleep 1
          done
          echo "mysql replica is ready"          

      - name: Run Go Tests
        run: |
          set +e
          counter=0
          NUM_TRIES=${{ fromJSON(needs.parse_config.outputs.json).num_tries }}
          while [ $counter -lt ${NUM_TRIES:-20} ]; do
            ((counter++))
            echo
            echo "----- TEST RUN #$counter -----"
            echo

            GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT" \
              TEST_LOCK_FILE_PATH=$(pwd)/lock \
              TEST_CRON_NO_RECOVER=1 \
              NETWORK_TEST_GITHUB_TOKEN=${{ secrets.FLEET_RELEASE_GITHUB_PAT }} \
              make run-go-tests TESTS_TO_RUN=${{ fromJSON(needs.parse_config.outputs.json).tests_to_run }} PKG_TO_TEST=${{ fromJSON(needs.parse_config.outputs.json).pkg_to_test }} 2>&1 | tee /tmp/gotest.log

            exit_code=$?

            if [ $exit_code -ne 0 ]; then
              echo "Test run #$counter failed with exit code $exit_code"
              exit $exit_code
            fi              
              
          done          

      - name: Create mysql identifier without colon
        if: always()
        run: |
          echo "MATRIX_MYSQL_ID=$(echo ${{ matrix.mysql }} | tr -d ':')" >> $GITHUB_ENV          

      - name: Generate summary of errors
        if: failure()
        run: |
          c1grep() { grep "$@" || test $? = 1; }
          c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
          c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
          c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
          c1grep -A 10 'panic: runtime error: ' /tmp/gotest.log >> /tmp/summary.txt
          c1grep ' FAIL\t' /tmp/gotest.log >> /tmp/summary.txt
          GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
          echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
          if [[ -z "$GO_FAIL_SUMMARY" ]]; then
            GO_FAIL_SUMMARY="unknown, please check the build URL"
          fi
          GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json