mirror of
https://github.com/fleetdm/fleet
synced 2026-04-21 21:47:20 +00:00
e4df954b0f
#23905
- Update with upstream nanomdm changes up to
825f2979a2
- Removed PostgeSQL folder from our nanomdm
- Added nanomdm MySQL test job to our CI
# Checklist for submitter
- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files)
for more information.
- [x] Added/updated tests
- [x] Manual QA for all new/changed functionality
307 lines
12 KiB
YAML
307 lines
12 KiB
YAML
name: Go Tests
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
- patch-*
|
|
- prepare-*
|
|
paths:
|
|
- '**.go'
|
|
- 'go.mod'
|
|
- 'go.sum'
|
|
- '.github/workflows/test-go.yaml'
|
|
- 'server/authz/policy.rego'
|
|
- 'docker-compose.yml'
|
|
pull_request:
|
|
paths:
|
|
- '**.go'
|
|
- 'go.mod'
|
|
- 'go.sum'
|
|
- '.github/workflows/test-go.yaml'
|
|
- 'server/authz/policy.rego'
|
|
- 'docker-compose.yml'
|
|
workflow_dispatch: # Manual
|
|
schedule:
|
|
- cron: '0 4 * * *'
|
|
|
|
# This allows a subsequently queued workflow run to interrupt previous runs
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id}}
|
|
cancel-in-progress: true
|
|
|
|
defaults:
|
|
run:
|
|
# fail-fast using bash -eo pipefail. See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
|
|
shell: bash
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
test-go:
|
|
strategy:
|
|
matrix:
|
|
suite: ["integration", "core"]
|
|
os: [ubuntu-latest]
|
|
mysql: ["mysql:8.0.36", "mysql:8.4.3"] # make sure to update supported versions docs when this changes
|
|
continue-on-error: ${{ matrix.suite == 'integration' }} # Since integration tests have a higher chance of failing, often for unrelated reasons, we don't want to fail the whole job if they fail
|
|
runs-on: ${{ matrix.os }}
|
|
|
|
env:
|
|
RACE_ENABLED: false
|
|
GO_TEST_TIMEOUT: 20m
|
|
|
|
steps:
|
|
- name: Harden Runner
|
|
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Checkout Code
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
|
|
|
- name: Install Go
|
|
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
|
|
# Pre-starting dependencies here means they are ready to go when we need them.
|
|
- name: Start Infra Dependencies
|
|
# Use & to background this
|
|
run: FLEET_MYSQL_IMAGE=${{ matrix.mysql }} docker compose -f docker-compose.yml -f docker-compose-redis-cluster.yml up -d mysql_test mysql_replica_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup minio saml_idp mailhog mailpit smtp4dev_test &
|
|
|
|
- name: Add TLS certificate for SMTP Tests
|
|
run: |
|
|
sudo cp tools/smtp4dev/fleet.crt /usr/local/share/ca-certificates/
|
|
sudo update-ca-certificates
|
|
|
|
# It seems faster not to cache Go dependencies
|
|
- name: Install Go Dependencies
|
|
run: make deps-go
|
|
|
|
- name: Install ZSH
|
|
run: sudo apt update && sudo apt install -y zsh
|
|
|
|
- name: Generate static files
|
|
run: |
|
|
export PATH=$PATH:~/go/bin
|
|
make generate-go
|
|
|
|
- name: Set Go race setting on schedule
|
|
if: github.event.schedule == '0 4 * * *'
|
|
run: |
|
|
echo "RACE_ENABLED=true" >> $GITHUB_ENV
|
|
echo "GO_TEST_TIMEOUT=1h" >> $GITHUB_ENV
|
|
|
|
- name: Wait for mysql
|
|
run: |
|
|
echo "waiting for mysql..."
|
|
until docker compose exec -T mysql_test sh -c "mysql -uroot -p\"\${MYSQL_ROOT_PASSWORD}\" -e \"SELECT 1=1\" fleet" &> /dev/null; do
|
|
echo "."
|
|
sleep 1
|
|
done
|
|
echo "mysql is ready"
|
|
echo "waiting for mysql replica..."
|
|
until docker compose exec -T mysql_replica_test sh -c "mysql -uroot -p\"\${MYSQL_ROOT_PASSWORD}\" -e \"SELECT 1=1\" fleet" &> /dev/null; do
|
|
echo "."
|
|
sleep 1
|
|
done
|
|
echo "mysql replica is ready"
|
|
|
|
- name: Run Go Tests
|
|
run: |
|
|
if [[ "${{ matrix.suite }}" == "core" ]]; then
|
|
RUN_TESTS_ARG='-skip=^TestIntegrations'
|
|
elif [[ "${{ matrix.suite }}" == "integration" ]]; then
|
|
RUN_TESTS_ARG='-run=^TestIntegrations'
|
|
else
|
|
RUN_TESTS_ARG=''
|
|
fi
|
|
GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT $RUN_TESTS_ARG" \
|
|
TEST_LOCK_FILE_PATH=$(pwd)/lock \
|
|
TEST_CRON_NO_RECOVER=1 \
|
|
NETWORK_TEST=1 \
|
|
REDIS_TEST=1 \
|
|
MYSQL_TEST=1 \
|
|
MYSQL_REPLICA_TEST=1 \
|
|
MINIO_STORAGE_TEST=1 \
|
|
SAML_IDP_TEST=1 \
|
|
MAIL_TEST=1 \
|
|
NETWORK_TEST_GITHUB_TOKEN=${{ secrets.FLEET_RELEASE_GITHUB_PAT }} \
|
|
make test-go 2>&1 | tee /tmp/gotest.log
|
|
|
|
- name: Create mysql identifier without colon
|
|
if: always()
|
|
run: |
|
|
echo "MATRIX_MYSQL_ID=$(echo ${{ matrix.mysql }} | tr -d ':')" >> $GITHUB_ENV
|
|
|
|
- name: Save coverage
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-coverage
|
|
path: ./coverage.txt
|
|
if-no-files-found: error
|
|
|
|
- name: Generate summary of errors
|
|
if: failure()
|
|
run: |
|
|
c1grep() { grep "$@" || test $? = 1; }
|
|
c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
|
|
c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
|
|
c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
|
|
c1grep -A 10 'panic: runtime error: ' /tmp/gotest.log >> /tmp/summary.txt
|
|
c1grep ' FAIL\t' /tmp/gotest.log >> /tmp/summary.txt
|
|
GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
|
|
echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
|
|
if [[ -z "$GO_FAIL_SUMMARY" ]]; then
|
|
GO_FAIL_SUMMARY="unknown, please check the build URL"
|
|
fi
|
|
GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json
|
|
|
|
- name: Slack Notification
|
|
if: github.event.schedule == '0 4 * * *' && failure()
|
|
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
|
|
with:
|
|
payload-file-path: ./payload.json
|
|
env:
|
|
JOB_STATUS: ${{ job.status }}
|
|
EVENT_URL: ${{ github.event.pull_request.html_url || github.event.head.html_url }}
|
|
RUN_URL: https://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}
|
|
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
|
|
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
|
|
|
|
- name: Upload test log
|
|
if: always()
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-test-log
|
|
path: /tmp/gotest.log
|
|
if-no-files-found: error
|
|
|
|
- name: Upload summary test log
|
|
if: always()
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-summary-test-log
|
|
path: /tmp/summary.txt
|
|
|
|
# Based on https://github.com/micromdm/nanomdm/blob/main/.github/workflows/on-push-pr.yml#L87
|
|
test-go-nanomdm:
|
|
runs-on: 'ubuntu-latest'
|
|
services:
|
|
mysql:
|
|
image: mysql:8.0.36
|
|
env:
|
|
MYSQL_RANDOM_ROOT_PASSWORD: yes
|
|
MYSQL_DATABASE: testdb
|
|
MYSQL_USER: testuser
|
|
MYSQL_PASSWORD: testpw
|
|
ports:
|
|
- 3800:3306
|
|
options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
|
|
env:
|
|
MYSQL_PWD: testpw
|
|
PORT: 3800
|
|
RACE_ENABLED: true
|
|
GO_TEST_TIMEOUT: 20m
|
|
steps:
|
|
- name: Harden Runner
|
|
uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Checkout Code
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
|
|
|
- name: Install Go
|
|
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
|
|
- name: verify mysql
|
|
run: |
|
|
while ! mysqladmin ping --host=localhost --port=$PORT --protocol=TCP --silent; do
|
|
sleep 1
|
|
done
|
|
|
|
- name: mysql schema
|
|
run: |
|
|
mysql --version
|
|
mysql --user=testuser --host=localhost --port=$PORT --protocol=TCP testdb < ./server/mdm/nanomdm/storage/mysql/schema.sql
|
|
|
|
- name: set test dsn
|
|
run: echo "NANOMDM_MYSQL_STORAGE_TEST_DSN=testuser:testpw@tcp(localhost:$PORT)/testdb" >> $GITHUB_ENV
|
|
|
|
- name: Run Go tests
|
|
run: |
|
|
go test -v -parallel 8 -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT \
|
|
-coverprofile=coverage.txt -covermode=atomic -coverpkg=github.com/fleetdm/fleet/v4/server/mdm/nanomdm/... \
|
|
./server/mdm/nanomdm/storage/mysql 2>&1 | tee /tmp/gotest.log
|
|
|
|
- name: Save coverage
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: nanomdm-coverage
|
|
path: ./coverage.txt
|
|
if-no-files-found: error
|
|
|
|
- name: Generate summary of errors
|
|
if: failure()
|
|
run: |
|
|
c1grep() { grep "$@" || test $? = 1; }
|
|
c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
|
|
c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
|
|
c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
|
|
c1grep -A 10 'panic: runtime error: ' /tmp/gotest.log >> /tmp/summary.txt
|
|
c1grep ' FAIL\t' /tmp/gotest.log >> /tmp/summary.txt
|
|
GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
|
|
echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
|
|
if [[ -z "$GO_FAIL_SUMMARY" ]]; then
|
|
GO_FAIL_SUMMARY="unknown, please check the build URL"
|
|
fi
|
|
GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json
|
|
|
|
- name: Slack Notification
|
|
if: github.event.schedule == '0 4 * * *' && failure()
|
|
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
|
|
with:
|
|
payload-file-path: ./payload.json
|
|
env:
|
|
JOB_STATUS: ${{ job.status }}
|
|
EVENT_URL: ${{ github.event.pull_request.html_url || github.event.head.html_url }}
|
|
RUN_URL: https://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}
|
|
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
|
|
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
|
|
|
|
- name: Upload test log
|
|
if: always()
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: nanomdm-test-log
|
|
path: /tmp/gotest.log
|
|
if-no-files-found: error
|
|
|
|
- name: Upload summary test log
|
|
if: always()
|
|
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
|
|
with:
|
|
name: nanomdm-summary-test-log
|
|
path: /tmp/summary.txt
|
|
|
|
# We upload all backend coverage in one step so that we're less like to end up in a situation with a partial coverage report.
|
|
upload-coverage:
|
|
needs: [test-go, test-go-nanomdm]
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
|
- name: Download artifacts
|
|
uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
|
|
with:
|
|
pattern: '*-coverage'
|
|
- name: Upload to Codecov
|
|
uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
flags: backend
|