diff --git a/.github/CI-TELEMETRY.md b/.github/CI-TELEMETRY.md index 46ab2d2e043..96e105a3f11 100644 --- a/.github/CI-TELEMETRY.md +++ b/.github/CI-TELEMETRY.md @@ -2,26 +2,39 @@ Pipeline: **GitHub Actions → Webhook → n8n → BigQuery** -## Standard Data Points +## Unified Payload Shape -All telemetry includes these fields for correlation: +All telemetry uses the same format: + +```json +{ + "timestamp": "2026-03-16T12:00:00.000Z", + "benchmark_name": "kafka-throughput-10n-10kb", + "git": { "sha": "abc12345", "branch": "master", "pr": null }, + "ci": { "runId": "123", "runUrl": "...", "job": "test", "workflow": "CI", "attempt": 1 }, + "runner": { "provider": "blacksmith", "cpuCores": 8, "memoryGb": 16.0 }, + "metrics": [ + { "metric_name": "exec-per-sec", "value": 12.4, "unit": "exec/s", "dimensions": { "trigger": "kafka", "nodes": 10 } } + ] +} +``` + +## Standard Context Fields ```typescript -// Git context git.sha // GITHUB_SHA (first 8 chars) git.branch // GITHUB_HEAD_REF ?? GITHUB_REF_NAME git.pr // PR number from GITHUB_REF -// CI context ci.runId // GITHUB_RUN_ID +ci.runUrl // https://github.com//actions/runs/ ci.job // GITHUB_JOB ci.workflow // GITHUB_WORKFLOW ci.attempt // GITHUB_RUN_ATTEMPT -// Runner detection -runner.provider // 'github' | 'blacksmith' | 'local' -runner.cpuCores // os.cpus().length -runner.memoryGb // os.totalmem() +runner.provider // 'github' | 'blacksmith' | 'local' +runner.cpuCores // os.cpus().length +runner.memoryGb // os.totalmem() ``` **Runner provider logic:** @@ -35,22 +48,67 @@ return 'blacksmith'; | Telemetry | Source | Metrics | |-----------|--------|---------| -| Build stats | `.github/scripts/send-build-stats.mjs` | Per-package build time, cache hits | -| Docker stats | `.github/scripts/send-docker-stats.mjs` | Image size, compiled artifact size, docker build time | -| Container stack | `packages/testing/containers/telemetry.ts` | E2E startup times | +| Playwright perf/benchmark | `packages/testing/playwright/reporters/metrics-reporter.ts` | Any metric attached via `attachMetric()` | +| Build stats | `.github/scripts/send-build-stats.mjs` | Per-package build duration, cache hit/miss, run total | +| Docker stats | `.github/scripts/send-docker-stats.mjs` | Image size per platform, docker build duration | +| Container stack | `packages/testing/containers/telemetry.ts` | E2E stack startup times per service | ## Secrets ``` -BUILD_STATS_WEBHOOK_URL -BUILD_STATS_WEBHOOK_USER -BUILD_STATS_WEBHOOK_PASSWORD # Alphanumeric + hyphens only (no $!#@) +QA_METRICS_WEBHOOK_URL +QA_METRICS_WEBHOOK_USER +QA_METRICS_WEBHOOK_PASSWORD +``` -DOCKER_STATS_WEBHOOK_URL +## BigQuery Table + +`qa_performance_metrics` — schema: + +```sql +timestamp TIMESTAMP NOT NULL +benchmark_name STRING +metric_name STRING NOT NULL +value FLOAT64 NOT NULL +unit STRING +dimensions JSON -- {"nodes": 10, "trigger": "kafka", "package": "@n8n/cli"} +git_sha STRING +git_branch STRING +git_pr INT64 +ci_run_id STRING +ci_run_url STRING +ci_job STRING +ci_workflow STRING +ci_attempt INT64 +runner_provider STRING +runner_cpu_cores INT64 +runner_memory_gb FLOAT64 +``` + +Query example: +```sql +-- Build duration trend by package (cache misses only) +SELECT DATE(timestamp), JSON_VALUE(dimensions, '$.package'), AVG(value) +FROM qa_performance_metrics +WHERE metric_name = 'build-duration' + AND JSON_VALUE(dimensions, '$.cache') = 'miss' +GROUP BY 1, 2 ORDER BY 1; ``` ## Adding New Telemetry -1. Copy data point helpers from `send-build-stats.mjs` -2. Create n8n workflow: Webhook (Basic Auth) → Code (flatten) → BigQuery -3. Add secrets to GitHub +**From a script:** +```javascript +import { sendMetrics, metric } from './send-metrics.mjs'; + +await sendMetrics([ + metric('my-metric', 42.0, 'ms', { context: 'value' }), +]); +``` + +**From a Playwright test:** +```typescript +import { attachMetric } from '../utils/performance-helper'; + +await attachMetric(testInfo, 'my-metric', 42.0, 'ms', { context: 'value' }); +``` diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index 4ce364af748..bf9ce773467 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -200,7 +200,6 @@ These only run if specific files changed: | Command | Workflow | Permissions | |--------------------|------------------------------|---------------------| -| `/build-unit-test` | `ci-manual-unit-tests.yml` | admin/write/maintain| | `/test-workflows` | `test-workflows-callable.yml`| admin/write/maintain| **Why:** Re-run tests without pushing commits. Useful for flaky test investigation. @@ -260,9 +259,6 @@ test-workflows-nightly.yml └──────────────────────────▶ test-workflows-callable.yml PR Comment Dispatchers (triggered by /command in PR comments): -build-unit-test-pr-comment.yml - └──────────────────────────▶ ci-manual-unit-tests.yml - test-workflows-pr-comment.yml └──────────────────────────▶ test-workflows-callable.yml ``` @@ -663,7 +659,7 @@ cosign verify-attestation --type openvex \ ### Redundancy Review -Comment triggers (`/build-unit-test`, `/test-workflows`) are workarounds. +Comment trigger (`/test-workflows`) is a workaround. Long-term: Main CI should be reliable enough to not need these. diff --git a/.github/scripts/send-build-stats.mjs b/.github/scripts/send-build-stats.mjs index 4eed624d852..4a7a06dc815 100644 --- a/.github/scripts/send-build-stats.mjs +++ b/.github/scripts/send-build-stats.mjs @@ -1,104 +1,67 @@ #!/usr/bin/env node /** - * Sends Turbo build summary JSON to a webhook + * Sends Turbo build stats to the unified QA metrics webhook. + * + * Reads the Turbo run summary from .turbo/runs/ and emits per-package + * build-duration metrics with {package, cache, task} dimensions, plus + * a run-level build-total-duration summary. * * Usage: node send-build-stats.mjs * - * Auto-detects summary from .turbo/runs/ directory. - * * Environment variables: - * BUILD_STATS_WEBHOOK_URL - Webhook URL (required to send) - * BUILD_STATS_WEBHOOK_USER - Basic auth username (required if URL set) - * BUILD_STATS_WEBHOOK_PASSWORD - Basic auth password (required if URL set) + * QA_METRICS_WEBHOOK_URL - Webhook URL (required to send) + * QA_METRICS_WEBHOOK_USER - Basic auth username + * QA_METRICS_WEBHOOK_PASSWORD - Basic auth password */ import { existsSync, readFileSync, readdirSync } from 'node:fs'; -import * as os from 'node:os'; import { join } from 'node:path'; +import { sendMetrics, metric } from './send-metrics.mjs'; + const runsDir = '.turbo/runs'; if (!existsSync(runsDir)) { console.log('No .turbo/runs directory found (turbo --summarize not used), skipping.'); process.exit(0); } -const files = readdirSync(runsDir).filter((f) => f.endsWith('.json')); -const summaryPath = files.length > 0 ? join(runsDir, files[0]) : null; -const webhookUrl = process.env.BUILD_STATS_WEBHOOK_URL; -const webhookUser = process.env.BUILD_STATS_WEBHOOK_USER; -const webhookPassword = process.env.BUILD_STATS_WEBHOOK_PASSWORD; - -if (!summaryPath) { +const files = readdirSync(runsDir) + .filter((f) => f.endsWith('.json')) + .sort(); +if (files.length === 0) { console.error('No summary file found in .turbo/runs/'); process.exit(1); } -if (!webhookUrl) { - console.log('BUILD_STATS_WEBHOOK_URL not set, skipping.'); - process.exit(0); +const summary = JSON.parse(readFileSync(join(runsDir, files.at(-1)), 'utf-8')); + +const metrics = []; + +for (const task of summary.tasks ?? []) { + if (task.execution?.exitCode !== 0) continue; + const durationMs = task.execution.durationMs ?? 0; + const cacheHit = task.cache?.status === 'HIT'; + // taskId format: "package-name#task-name" + const [pkg, taskName] = task.taskId?.split('#') ?? [task.package, task.task]; + + metrics.push( + metric('build-duration', durationMs / 1000, 's', { + package: pkg ?? 'unknown', + task: taskName ?? 'build', + cache: cacheHit ? 'hit' : 'miss', + }), + ); } -if (!webhookUser || !webhookPassword) { - console.error('BUILD_STATS_WEBHOOK_USER and BUILD_STATS_WEBHOOK_PASSWORD are required'); - process.exit(1); -} +const totalMs = summary.durationMs ?? 0; +const totalTasks = summary.tasks?.length ?? 0; +const cachedTasks = summary.tasks?.filter((t) => t.cache?.status === 'HIT').length ?? 0; -const basicAuth = Buffer.from(`${webhookUser}:${webhookPassword}`).toString('base64'); +metrics.push( + metric('build-total-duration', totalMs / 1000, 's', { + total_tasks: totalTasks, + cached_tasks: cachedTasks, + }), +); -const summary = JSON.parse(readFileSync(summaryPath, 'utf-8')); - -// Extract PR number from GITHUB_REF (refs/pull/123/merge) -const ref = process.env.GITHUB_REF ?? ''; -const prMatch = ref.match(/refs\/pull\/(\d+)/); - -// Detect runner provider (matches packages/testing/containers/telemetry.ts) -function getRunnerProvider() { - if (!process.env.CI) return 'local'; - if (process.env.RUNNER_ENVIRONMENT === 'github-hosted') return 'github'; - return 'blacksmith'; -} - -// Add git context (aligned with container telemetry) -summary.git = { - sha: process.env.GITHUB_SHA?.slice(0, 8) || null, - branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? null, - pr: prMatch ? parseInt(prMatch[1], 10) : null, -}; - -// Add CI context -summary.ci = { - runId: process.env.GITHUB_RUN_ID || null, - runUrl: process.env.GITHUB_RUN_ID - ? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` - : null, - job: process.env.GITHUB_JOB || null, - workflow: process.env.GITHUB_WORKFLOW || null, - attempt: process.env.GITHUB_RUN_ATTEMPT ? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) : null, -}; - -// Add runner info -summary.runner = { - provider: getRunnerProvider(), - cpuCores: os.cpus().length, - memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10, -}; - -const headers = { - 'Content-Type': 'application/json', - 'Authorization': `Basic ${basicAuth}`, -}; - -const response = await fetch(webhookUrl, { - method: 'POST', - headers, - body: JSON.stringify(summary), -}); - -if (!response.ok) { - console.error(`Webhook failed: ${response.status} ${response.statusText}`); - const body = await response.text(); - if (body) console.error(`Response: ${body}`); - process.exit(1); -} - -console.log(`Build stats sent: ${response.status}`); +await sendMetrics(metrics, 'build-stats'); diff --git a/.github/scripts/send-docker-stats.mjs b/.github/scripts/send-docker-stats.mjs index 2dfb8e02f00..3d89816ac0f 100644 --- a/.github/scripts/send-docker-stats.mjs +++ b/.github/scripts/send-docker-stats.mjs @@ -1,18 +1,22 @@ #!/usr/bin/env node /** - * Sends Docker build stats to a webhook for BigQuery ingestion. + * Sends Docker build stats to the unified QA metrics webhook. * - * Reads manifests produced by build-n8n.mjs and dockerize-n8n.mjs, - * enriches with git/CI/runner context, and POSTs to a webhook. + * Reads manifests produced by build-n8n.mjs and dockerize-n8n.mjs and emits + * per-image docker-image-size metrics and build duration metrics with + * {image, platform} dimensions. * * Usage: node send-docker-stats.mjs * * Environment variables: - * DOCKER_STATS_WEBHOOK_URL - Webhook URL (required to send) + * QA_METRICS_WEBHOOK_URL - Webhook URL (required to send) + * QA_METRICS_WEBHOOK_USER - Basic auth username + * QA_METRICS_WEBHOOK_PASSWORD - Basic auth password */ import { existsSync, readFileSync } from 'node:fs'; -import * as os from 'node:os'; + +import { sendMetrics, metric } from './send-metrics.mjs'; const buildManifestPath = 'compiled/build-manifest.json'; const dockerManifestPath = 'docker-build-manifest.json'; @@ -22,13 +26,6 @@ if (!existsSync(buildManifestPath) && !existsSync(dockerManifestPath)) { process.exit(0); } -const webhookUrl = process.env.DOCKER_STATS_WEBHOOK_URL; - -if (!webhookUrl) { - console.log('DOCKER_STATS_WEBHOOK_URL not set, skipping.'); - process.exit(0); -} - const buildManifest = existsSync(buildManifestPath) ? JSON.parse(readFileSync(buildManifestPath, 'utf-8')) : null; @@ -37,68 +34,41 @@ const dockerManifest = existsSync(dockerManifestPath) ? JSON.parse(readFileSync(dockerManifestPath, 'utf-8')) : null; -// Extract PR number from GITHUB_REF (refs/pull/123/merge) -const ref = process.env.GITHUB_REF ?? ''; -const prMatch = ref.match(/refs\/pull\/(\d+)/); +const metrics = []; -// Detect runner provider (matches packages/testing/containers/telemetry.ts) -function getRunnerProvider() { - if (!process.env.CI) return 'local'; - if (process.env.RUNNER_ENVIRONMENT === 'github-hosted') return 'github'; - return 'blacksmith'; +if (buildManifest) { + if (buildManifest.artifactSize != null) { + metrics.push(metric('artifact-size', buildManifest.artifactSize, 'bytes', { artifact: 'compiled' })); + } + if (buildManifest.buildDuration != null) { + metrics.push(metric('build-duration', buildManifest.buildDuration / 1000, 's', { artifact: 'compiled' })); + } } -const payload = { - build: buildManifest - ? { - artifactSize: buildManifest.artifactSize, - buildDuration: buildManifest.buildDuration, - } - : null, +if (dockerManifest) { + const platform = dockerManifest.platform ?? 'unknown'; - docker: dockerManifest - ? { - platform: dockerManifest.platform, - images: dockerManifest.images, - } - : null, + for (const image of dockerManifest.images ?? []) { + if (image.sizeBytes != null) { + metrics.push( + metric('docker-image-size', image.sizeBytes, 'bytes', { + image: image.name ?? 'unknown', + platform, + }), + ); + } + } - git: { - sha: process.env.GITHUB_SHA?.slice(0, 8) || null, - branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? null, - pr: prMatch ? parseInt(prMatch[1], 10) : null, - }, - - ci: { - runId: process.env.GITHUB_RUN_ID || null, - runUrl: process.env.GITHUB_RUN_ID - ? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` - : null, - job: process.env.GITHUB_JOB || null, - workflow: process.env.GITHUB_WORKFLOW || null, - attempt: process.env.GITHUB_RUN_ATTEMPT ? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) : null, - }, - - runner: { - provider: getRunnerProvider(), - cpuCores: os.cpus().length, - memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10, - }, -}; - -const response = await fetch(webhookUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), -}); - -if (!response.ok) { - console.error(`Webhook failed: ${response.status} ${response.statusText}`); - const body = await response.text(); - if (body) console.error(`Response: ${body}`); - process.exit(1); + if (dockerManifest.buildDurationMs != null) { + metrics.push( + metric('docker-build-duration', dockerManifest.buildDurationMs / 1000, 's', { platform }), + ); + } } -console.log(`Docker build stats sent: ${response.status}`); +if (metrics.length === 0) { + console.log('No metrics to send.'); + process.exit(0); +} + +await sendMetrics(metrics, 'docker-stats'); diff --git a/.github/scripts/send-metrics.mjs b/.github/scripts/send-metrics.mjs new file mode 100644 index 00000000000..c9f504ddce3 --- /dev/null +++ b/.github/scripts/send-metrics.mjs @@ -0,0 +1,94 @@ +#!/usr/bin/env node +/** + * Shared metrics sender for CI scripts. + * See .github/CI-TELEMETRY.md for payload shape and BigQuery schema. + * + * Usage: + * import { sendMetrics, metric } from './send-metrics.mjs'; + * await sendMetrics([metric('build-duration', 45.2, 's', { package: '@n8n/cli' })]); + * + * Env: QA_METRICS_WEBHOOK_URL, QA_METRICS_WEBHOOK_USER, QA_METRICS_WEBHOOK_PASSWORD + */ + +import * as os from 'node:os'; + +/** Build a single metric object. */ +export function metric(name, value, unit, dimensions = {}) { + return { metric_name: name, value, unit, dimensions }; +} + +/** Build git/ci/runner context from environment variables. */ +export function buildContext(benchmarkName = null) { + const ref = process.env.GITHUB_REF ?? ''; + const prMatch = ref.match(/refs\/pull\/(\d+)/); + const runId = process.env.GITHUB_RUN_ID ?? null; + + return { + timestamp: new Date().toISOString(), + benchmark_name: benchmarkName, + git: { + sha: process.env.GITHUB_SHA?.slice(0, 8) ?? null, + branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? null, + pr: prMatch ? parseInt(prMatch[1], 10) : null, + }, + ci: { + runId, + runUrl: + runId && process.env.GITHUB_REPOSITORY + ? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId}` + : null, + job: process.env.GITHUB_JOB ?? null, + workflow: process.env.GITHUB_WORKFLOW ?? null, + attempt: process.env.GITHUB_RUN_ATTEMPT + ? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) + : null, + }, + runner: { + provider: !process.env.CI + ? 'local' + : process.env.RUNNER_ENVIRONMENT === 'github-hosted' + ? 'github' + : 'blacksmith', + cpuCores: os.cpus().length, + memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10, + }, + }; +} + +export async function sendMetrics(metrics, benchmarkName = null) { + const webhookUrl = process.env.QA_METRICS_WEBHOOK_URL; + const webhookUser = process.env.QA_METRICS_WEBHOOK_USER; + const webhookPassword = process.env.QA_METRICS_WEBHOOK_PASSWORD; + + if (!webhookUrl) { + console.log('QA_METRICS_WEBHOOK_URL not set, skipping.'); + return; + } + + if (!webhookUser || !webhookPassword) { + console.log('QA_METRICS_WEBHOOK_USER/PASSWORD not set, skipping.'); + return; + } + + const payload = { ...buildContext(benchmarkName), metrics }; + const basicAuth = Buffer.from(`${webhookUser}:${webhookPassword}`).toString('base64'); + + const response = await fetch(webhookUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${basicAuth}`, + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(30_000), + }); + + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error( + `Webhook failed: ${response.status} ${response.statusText}${body ? `\n${body}` : ''}`, + ); + } + + console.log(`Sent ${metrics.length} metric(s): ${response.status}`); +} diff --git a/.github/workflows/build-unit-test-pr-comment.yml b/.github/workflows/build-unit-test-pr-comment.yml deleted file mode 100644 index 19068ce6497..00000000000 --- a/.github/workflows/build-unit-test-pr-comment.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: 'Build: Unit Test PR Comment' - -on: - issue_comment: - types: [created] - -permissions: - pull-requests: read - contents: read - actions: write - issues: write - -jobs: - validate_and_dispatch: - name: Validate user and dispatch CI workflow - if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/build-unit-test') - runs-on: ubuntu-latest - steps: - - name: Validate user permissions and collect PR data - id: check_permissions - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const commenter = context.actor; - const body = (context.payload.comment.body || '').trim(); - const isCommand = body.startsWith('/build-unit-test'); - const allowedPermissions = ['admin', 'write', 'maintain']; - const commentId = context.payload.comment.id; - const { owner, repo } = context.repo; - - async function react(content) { - try { - await github.rest.reactions.createForIssueComment({ - owner, - repo, - comment_id: commentId, - content, - }); - } catch (error) { - console.log(`Failed to add reaction '${content}': ${error.message}`); - } - } - - core.setOutput('proceed', 'false'); - core.setOutput('headSha', ''); - core.setOutput('prNumber', ''); - - if (!context.payload.issue.pull_request || !isCommand) { - console.log('Comment is not /build-unit-test on a pull request. Skipping.'); - return; - } - - let permission; - try { - const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ - owner, - repo, - username: commenter, - }); - permission = data.permission; - } catch (error) { - console.log(`Could not verify permissions for @${commenter}: ${error.message}`); - await react('confused'); - return; - } - - if (!allowedPermissions.includes(permission)) { - console.log(`User @${commenter} has '${permission}' permission; requires admin/write/maintain.`); - await react('-1'); - return; - } - - try { - const prNumber = context.issue.number; - const { data: pr } = await github.rest.pulls.get({ - owner, - repo, - pull_number: prNumber, - }); - await react('+1'); - core.setOutput('proceed', 'true'); - core.setOutput('headSha', pr.head.sha); - core.setOutput('prNumber', String(prNumber)); - } catch (error) { - console.log(`Failed to fetch PR details for PR #${context.issue.number}: ${error.message}`); - await react('confused'); - } - - - name: Dispatch build/unit test workflow - if: ${{ steps.check_permissions.outputs.proceed == 'true' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEAD_SHA: ${{ steps.check_permissions.outputs.headSha }} - PR_NUMBER: ${{ steps.check_permissions.outputs.prNumber }} - run: | - gh workflow run ci-manual-unit-tests.yml \ - --repo "${{ github.repository }}" \ - -f ref="${HEAD_SHA}" \ - -f pr_number="${PR_NUMBER}" diff --git a/.github/workflows/ci-manual-unit-tests.yml b/.github/workflows/ci-manual-unit-tests.yml deleted file mode 100644 index c5e2c8db2f0..00000000000 --- a/.github/workflows/ci-manual-unit-tests.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: 'CI: Manual Unit Tests' - -on: - workflow_dispatch: - inputs: - ref: - description: Commit SHA or ref to check out - required: true - pr_number: - description: PR number (optional, for check reporting) - required: false - type: string - -permissions: - contents: read - checks: write - -jobs: - create-check-run: - name: Create Check Run - runs-on: ubuntu-latest - if: inputs.pr_number != '' - outputs: - check_run_id: ${{ steps.create.outputs.check_run_id }} - steps: - - name: Create pending check run on PR - id: create - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { data: checkRun } = await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - name: 'Build & Unit Tests - Checks', - head_sha: '${{ inputs.ref }}', - status: 'in_progress', - output: { - title: 'Build & Unit Tests - Checks', - summary: 'Running build, unit tests, and lint...' - } - }); - - core.setOutput('check_run_id', checkRun.id); - console.log(`Created check run ${checkRun.id} on commit ${{ inputs.ref }}`); - - install-and-build: - name: Install & Build - runs-on: blacksmith-2vcpu-ubuntu-2204 - env: - NODE_OPTIONS: '--max-old-space-size=6144' - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ inputs.ref }} - - - name: Setup and Build - uses: ./.github/actions/setup-nodejs - - - name: Run format check - run: pnpm format:check - - - name: Run typecheck - run: pnpm typecheck - - unit-tests: - name: Unit tests - needs: install-and-build - uses: ./.github/workflows/test-unit-reusable.yml - with: - ref: ${{ inputs.ref }} - collectCoverage: true - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} - - lint: - name: Lint - needs: install-and-build - uses: ./.github/workflows/test-linting-reusable.yml - with: - ref: ${{ inputs.ref }} - - post-build-unit-tests: - name: Build & Unit Tests - Checks - runs-on: ubuntu-latest - needs: [create-check-run, install-and-build, unit-tests, lint] - if: always() - steps: - - name: Update check run on PR (if triggered from PR comment) - if: inputs.pr_number != '' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const checkRunId = '${{ needs.create-check-run.outputs.check_run_id }}'; - - if (!checkRunId) { - console.log('No check run ID found, skipping update'); - return; - } - - const buildResult = '${{ needs.install-and-build.result }}'; - const testResult = '${{ needs.unit-tests.result }}'; - const lintResult = '${{ needs.lint.result }}'; - - const conclusion = (buildResult === 'success' && testResult === 'success' && lintResult === 'success') - ? 'success' - : 'failure'; - - const summary = ` - **Build**: ${buildResult} - **Unit Tests**: ${testResult} - **Lint**: ${lintResult} - `; - - await github.rest.checks.update({ - owner: context.repo.owner, - repo: context.repo.repo, - check_run_id: parseInt(checkRunId), - status: 'completed', - conclusion: conclusion, - output: { - title: 'Build & Unit Tests - Checks', - summary: summary - } - }); - - console.log(`Updated check run ${checkRunId} with conclusion: ${conclusion}`); - - - name: Fail if any job failed - if: needs.install-and-build.result == 'failure' || needs.unit-tests.result == 'failure' || needs.lint.result == 'failure' - run: exit 1 diff --git a/.github/workflows/ci-master.yml b/.github/workflows/ci-master.yml index e903c619868..c00805e50d6 100644 --- a/.github/workflows/ci-master.yml +++ b/.github/workflows/ci-master.yml @@ -14,9 +14,9 @@ jobs: runs-on: ubuntu-latest env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup and Build @@ -33,11 +33,7 @@ jobs: ref: ${{ github.sha }} nodeVersion: ${{ matrix.node-version }} collectCoverage: ${{ matrix.node-version == '24.13.1' }} - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} + secrets: inherit lint: name: Lint diff --git a/.github/workflows/ci-pull-requests.yml b/.github/workflows/ci-pull-requests.yml index b036a8b5028..a5f91da8487 100644 --- a/.github/workflows/ci-pull-requests.yml +++ b/.github/workflows/ci-pull-requests.yml @@ -15,9 +15,9 @@ jobs: env: NODE_OPTIONS: '--max-old-space-size=6144' CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} outputs: ci: ${{ fromJSON(steps.ci-filter.outputs.results).ci == true }} unit: ${{ fromJSON(steps.ci-filter.outputs.results).unit == true }} @@ -98,11 +98,7 @@ jobs: with: ref: ${{ needs.install-and-build.outputs.commit_sha }} collectCoverage: true - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} + secrets: inherit typecheck: name: Typecheck diff --git a/.github/workflows/test-e2e-ci-reusable.yml b/.github/workflows/test-e2e-ci-reusable.yml index ac2a4875fe5..5a495dc9966 100644 --- a/.github/workflows/test-e2e-ci-reusable.yml +++ b/.github/workflows/test-e2e-ci-reusable.yml @@ -52,10 +52,9 @@ jobs: IMAGE_BASE_NAME: ghcr.io/${{ github.repository }} IMAGE_TAG: ci-${{ github.run_id }} RUNNERS_IMAGE_BASE_NAME: ghcr.io/${{ github.repository_owner }}/runners - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} - DOCKER_STATS_WEBHOOK_URL: ${{ secrets.DOCKER_STATS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} - name: Get changed files for impact analysis if: ${{ inputs.playwright-only }} diff --git a/.github/workflows/test-e2e-coverage-weekly.yml b/.github/workflows/test-e2e-coverage-weekly.yml index 56dabb91f94..bb0e161fbc5 100644 --- a/.github/workflows/test-e2e-coverage-weekly.yml +++ b/.github/workflows/test-e2e-coverage-weekly.yml @@ -41,9 +41,9 @@ jobs: BUILD_WITH_COVERAGE: 'true' CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }} CURRENTS_PROJECT_ID: 'LRxcNt' - QA_PERFORMANCE_METRICS_WEBHOOK_URL: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_URL }} - QA_PERFORMANCE_METRICS_WEBHOOK_USER: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_USER }} - QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} - name: Generate Coverage Report if: always() && steps.coverage-tests.outcome != 'skipped' diff --git a/.github/workflows/test-e2e-infrastructure-reusable.yml b/.github/workflows/test-e2e-infrastructure-reusable.yml index f6191402f2e..eba31148938 100644 --- a/.github/workflows/test-e2e-infrastructure-reusable.yml +++ b/.github/workflows/test-e2e-infrastructure-reusable.yml @@ -12,6 +12,10 @@ on: - 'packages/testing/containers/services/**' - '.github/workflows/test-e2e-infrastructure-reusable.yml' +concurrency: + group: e2e-infrastructure-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: benchmark: name: ${{ matrix.profile }} diff --git a/.github/workflows/test-e2e-reusable.yml b/.github/workflows/test-e2e-reusable.yml index f468338d1fa..74e4a55863d 100644 --- a/.github/workflows/test-e2e-reusable.yml +++ b/.github/workflows/test-e2e-reusable.yml @@ -63,25 +63,6 @@ on: default: '' type: string - secrets: - CURRENTS_RECORD_KEY: - required: false - CURRENTS_API_KEY: - required: false - QA_PERFORMANCE_METRICS_WEBHOOK_URL: - required: false - QA_PERFORMANCE_METRICS_WEBHOOK_USER: - required: false - QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD: - required: false - N8N_LICENSE_ACTIVATION_KEY: - required: false - N8N_LICENSE_CERT: - required: false - N8N_ENCRYPTION_KEY: - required: false - CONTAINER_TELEMETRY_WEBHOOK: - required: false env: NODE_OPTIONS: ${{ contains(inputs.runner, '2vcpu') && '--max-old-space-size=6144' || '' }} @@ -162,13 +143,12 @@ jobs: run: ${{ inputs.test-command }} --workers=${{ env.PLAYWRIGHT_WORKERS }} ${{ matrix.specs || format('--shard={0}/{1}', matrix.shard, strategy.job-total) }} env: CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }} - QA_PERFORMANCE_METRICS_WEBHOOK_URL: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_URL }} - QA_PERFORMANCE_METRICS_WEBHOOK_USER: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_USER }} - QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }} N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }} N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }} - CONTAINER_TELEMETRY_WEBHOOK: ${{secrets.CONTAINER_TELEMETRY_WEBHOOK}} - name: Upload Failure Artifacts if: ${{ failure() && inputs.upload-failure-artifacts }} diff --git a/.github/workflows/test-unit-reusable.yml b/.github/workflows/test-unit-reusable.yml index cfa40db3010..ed740bd28a5 100644 --- a/.github/workflows/test-unit-reusable.yml +++ b/.github/workflows/test-unit-reusable.yml @@ -17,20 +17,6 @@ on: required: false default: false type: boolean - secrets: - CODECOV_TOKEN: - description: 'Codecov upload token.' - required: false - BUILD_STATS_WEBHOOK_URL: - description: 'Webhook URL for test stats.' - required: false - BUILD_STATS_WEBHOOK_USER: - description: 'Basic auth username for test stats webhook.' - required: false - BUILD_STATS_WEBHOOK_PASSWORD: - description: 'Basic auth password for test stats webhook.' - required: false - env: NODE_OPTIONS: --max-old-space-size=7168 @@ -57,9 +43,9 @@ jobs: if: ${{ !cancelled() }} run: node .github/scripts/send-build-stats.mjs || true env: - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} - name: Upload test results to Codecov if: ${{ !cancelled() }} @@ -99,9 +85,9 @@ jobs: if: ${{ !cancelled() }} run: node .github/scripts/send-build-stats.mjs || true env: - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} - name: Upload test results to Codecov if: ${{ !cancelled() }} @@ -140,9 +126,9 @@ jobs: if: ${{ !cancelled() }} run: node .github/scripts/send-build-stats.mjs || true env: - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} - name: Upload test results to Codecov if: ${{ !cancelled() }} @@ -187,9 +173,9 @@ jobs: if: ${{ !cancelled() }} run: node .github/scripts/send-build-stats.mjs || true env: - BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }} - BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }} - BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} - name: Upload test results to Codecov if: ${{ !cancelled() }} diff --git a/packages/testing/containers/README.md b/packages/testing/containers/README.md index ebbf8d3502a..11291788910 100644 --- a/packages/testing/containers/README.md +++ b/packages/testing/containers/README.md @@ -393,7 +393,7 @@ Container stack telemetry tracks startup timing, configuration, and runner info. | Variable | Description | |----------|-------------| -| `CONTAINER_TELEMETRY_WEBHOOK` | POST telemetry JSON to this URL | +| `QA_METRICS_WEBHOOK_URL` | POST telemetry JSON to this URL | | `CONTAINER_TELEMETRY_VERBOSE` | Set to `1` for JSON output to console | ### What's Collected @@ -419,7 +419,7 @@ Container stack telemetry tracks startup timing, configuration, and runner info. CONTAINER_TELEMETRY_VERBOSE=1 pnpm stack # Send to webhook (CI) -CONTAINER_TELEMETRY_WEBHOOK=https://n8n.example.com/webhook/telemetry +QA_METRICS_WEBHOOK_URL=https://n8n.example.com/webhook/telemetry ``` ## Cleanup diff --git a/packages/testing/containers/telemetry.ts b/packages/testing/containers/telemetry.ts index 1936a56b079..db373646bf6 100644 --- a/packages/testing/containers/telemetry.ts +++ b/packages/testing/containers/telemetry.ts @@ -172,7 +172,7 @@ export class TelemetryRecorder { * Flush telemetry - outputs based on environment configuration * * Output modes: - * - CONTAINER_TELEMETRY_WEBHOOK: Send via detached process (non-blocking, survives parent exit) + * - QA_METRICS_WEBHOOK_URL: Send via detached process (non-blocking, survives parent exit) * - CONTAINER_TELEMETRY_VERBOSE=1: Full breakdown + elapsed logs (via utils.ts) * - Default: One-liner summary only */ @@ -180,7 +180,7 @@ export class TelemetryRecorder { const record = this.buildRecord(success, errorMessage); const isVerbose = process.env.CONTAINER_TELEMETRY_VERBOSE === '1'; - const webhookUrl = process.env.CONTAINER_TELEMETRY_WEBHOOK; + const webhookUrl = process.env.QA_METRICS_WEBHOOK_URL; if (webhookUrl) { this.sendToWebhook(record, webhookUrl); @@ -206,24 +206,102 @@ export class TelemetryRecorder { } } + private buildUnifiedPayload(record: StackTelemetryRecord): object { + const runId = record.ci.runId ?? null; + const repo = process.env.GITHUB_REPOSITORY ?? null; + + const metrics: Array<{ + metric_name: string; + value: number; + unit: string; + dimensions: Record; + }> = [ + { + metric_name: 'stack-startup-total', + value: record.timing.total, + unit: 'ms', + dimensions: { + stack_type: record.stack.type, + mains: record.stack.mains, + workers: record.stack.workers, + postgres: record.stack.postgres ? 'true' : 'false', + success: record.success ? 'true' : 'false', + }, + }, + { + metric_name: 'stack-startup-network', + value: record.timing.network, + unit: 'ms', + dimensions: { stack_type: record.stack.type }, + }, + { + metric_name: 'stack-startup-n8n', + value: record.timing.n8nStartup, + unit: 'ms', + dimensions: { stack_type: record.stack.type }, + }, + ...Object.entries(record.timing.services).map(([service, duration]) => ({ + metric_name: 'stack-startup-service', + value: duration, + unit: 'ms', + dimensions: { service, stack_type: record.stack.type }, + })), + { + metric_name: 'container-count', + value: record.containers.total, + unit: 'count', + dimensions: { stack_type: record.stack.type }, + }, + ]; + + return { + timestamp: record.timestamp, + benchmark_name: 'container-telemetry', + git: { + sha: record.git.sha, + branch: record.git.branch, + pr: record.git.pr ?? null, + }, + ci: { + runId, + runUrl: runId && repo ? `https://github.com/${repo}/actions/runs/${runId}` : null, + job: record.ci.job ?? null, + workflow: record.ci.workflow ?? null, + attempt: record.ci.attempt ?? null, + }, + runner: record.runner, + metrics, + }; + } + /** * Send telemetry via a detached child process. * The process runs independently and survives parent exit, ensuring delivery * even when the main process throws/exits immediately after flush(). */ private sendToWebhook(record: StackTelemetryRecord, webhookUrl: string): void { - const payload = JSON.stringify(record); + const webhookUser = process.env.QA_METRICS_WEBHOOK_USER; + const webhookPassword = process.env.QA_METRICS_WEBHOOK_PASSWORD; + + if (!webhookUser || !webhookPassword) { + console.log('QA_METRICS_WEBHOOK_USER/PASSWORD not set, skipping telemetry.'); + return; + } + + const payload = JSON.stringify(this.buildUnifiedPayload(record)); + const authHeader = `Basic ${Buffer.from(`${webhookUser}:${webhookPassword}`).toString('base64')}`; const script = ` - fetch(process.argv[1], { + fetch(process.argv[2], { method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: process.argv[2] + headers: { 'Content-Type': 'application/json', 'Authorization': process.env.WEBHOOK_AUTH }, + body: process.argv[3] }).catch(() => process.exit(1)); `; const child = spawn(process.execPath, ['-e', script, webhookUrl, payload], { detached: true, stdio: 'ignore', + env: { ...process.env, WEBHOOK_AUTH: authHeader }, }); child.unref(); } diff --git a/packages/testing/playwright/reporters/USAGE.md b/packages/testing/playwright/reporters/USAGE.md index 87c0948e893..381b5ee9304 100644 --- a/packages/testing/playwright/reporters/USAGE.md +++ b/packages/testing/playwright/reporters/USAGE.md @@ -1,66 +1,87 @@ -# Metrics Reporter Usage +# Metrics Reporter -Automatically collect performance metrics from Playwright tests and send them to a Webhook. +Collects `attachMetric()` calls from Playwright tests and sends them as a single batched +payload to the unified QA metrics webhook at the end of the run. + +See `.github/CI-TELEMETRY.md` for the full payload shape, BigQuery schema, and adding +new telemetry from scripts. ## Setup ```bash -export QA_PERFORMANCE_METRICS_WEBHOOK_URL=https://your-webhook-endpoint.com/metrics -export QA_PERFORMANCE_METRICS_WEBHOOK_USER=username -export QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD=password +export QA_METRICS_WEBHOOK_URL=https://your-webhook-endpoint.com/metrics +export QA_METRICS_WEBHOOK_USER=username +export QA_METRICS_WEBHOOK_PASSWORD=password ``` -## Attach Metrics in Tests +## Attaching Metrics -**Option 1: Helper function (recommended)** -```javascript +```typescript import { attachMetric } from '../../utils/performance-helper'; -await attachMetric(testInfo, 'memory-usage', 1234567, 'bytes'); -``` +// Simple metric +await attachMetric(testInfo, 'memory-rss', 368.04, 'MB'); -**Option 2: Direct attach** -```javascript -await testInfo.attach('metric:memory-usage', { - body: JSON.stringify({ value: 1234567, unit: 'bytes' }) +// With dimensions — separates the measurement from the scenario config +await attachMetric(testInfo, 'exec-per-sec', 245.3, 'exec/s', { + trigger: 'kafka', + nodes: 10, + output: '100KB', }); ``` -## What Gets Sent to BigQuery +`metric_name` is the measurement type. Config goes in `dimensions`, never baked into the name. -```json -{ - "test_name": "My performance test", - "metric_name": "memory-usage", - "metric_value": 1234567, - "metric_unit": "bytes", - "git_commit": "abc123...", - "git_branch": "main", - "timestamp": "2025-08-29T..." -} +## Benchmark Harnesses + +The harnesses derive dimensions automatically — prefer these over calling `attachMetric()` directly: + +```typescript +await runThroughputTest({ handle, api, services, testInfo, + trigger: 'kafka', nodeCount: 10, nodeOutputSize: '100KB', + messageCount: 5000, timeoutMs: 600_000, +}); + +await runWebhookThroughputTest({ handle, api, services, testInfo, baseUrl, + nodeCount: 10, nodeOutputSize: '100KB', + connections: 50, durationSeconds: 30, timeoutMs: 120_000, +}); ``` -## Data Pipeline +## Example Queries -**Playwright Test** → **n8n Webhook** → **BigQuery Table** +```sql +-- Throughput trend by node count +SELECT INT64(JSON_VALUE(dimensions, '$.nodes')) AS nodes, AVG(value) AS avg_exec_per_sec +FROM qa_performance_metrics +WHERE metric_name = 'exec-per-sec' + AND JSON_VALUE(dimensions, '$.trigger') = 'kafka' + AND git_branch = 'master' +GROUP BY nodes ORDER BY nodes; -The n8n workflow that processes the metrics is here: -https://internal.users.n8n.cloud/workflow/zSRjEwfBfCNjGXK8 - -## BigQuery Schema - -```json -{ - "fields": [ - {"name": "test_name", "type": "STRING", "mode": "REQUIRED"}, - {"name": "metric_name", "type": "STRING", "mode": "REQUIRED"}, - {"name": "metric_value", "type": "FLOAT", "mode": "REQUIRED"}, - {"name": "metric_unit", "type": "STRING", "mode": "REQUIRED"}, - {"name": "git_commit", "type": "STRING", "mode": "REQUIRED"}, - {"name": "git_branch", "type": "STRING", "mode": "REQUIRED"}, - {"name": "timestamp", "type": "TIMESTAMP", "mode": "REQUIRED"} - ] -} +-- PR vs 30-day master baseline +-- Note: dimensions is a JSON column; use JSON_VALUE() to extract fields for grouping/joining +WITH pr AS ( + SELECT metric_name, + JSON_VALUE(dimensions, '$.trigger') AS trigger, + JSON_VALUE(dimensions, '$.mode') AS mode, + AVG(value) AS value + FROM qa_performance_metrics WHERE git_pr = @pr_number + GROUP BY metric_name, trigger, mode +), +baseline AS ( + SELECT metric_name, + JSON_VALUE(dimensions, '$.trigger') AS trigger, + JSON_VALUE(dimensions, '$.mode') AS mode, + AVG(value) AS value + FROM qa_performance_metrics + WHERE git_branch = 'master' + AND timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY) + GROUP BY metric_name, trigger, mode +) +SELECT pr.metric_name, pr.trigger, pr.mode, + pr.value AS pr_value, baseline.value AS baseline_value, + ROUND((pr.value - baseline.value) / baseline.value * 100, 1) AS delta_pct +FROM pr JOIN baseline USING (metric_name, trigger, mode) +ORDER BY ABS(delta_pct) DESC; ``` - -That's it! Metrics are automatically collected and sent when you attach them to tests. diff --git a/packages/testing/playwright/reporters/benchmark-summary-reporter.ts b/packages/testing/playwright/reporters/benchmark-summary-reporter.ts index 1a9ebac5365..c2b61ff340a 100644 --- a/packages/testing/playwright/reporters/benchmark-summary-reporter.ts +++ b/packages/testing/playwright/reporters/benchmark-summary-reporter.ts @@ -76,11 +76,8 @@ function extractSuite(filePath: string): string { return 'other'; } -function extractMetricSuffix(metricName: string, scenario: string): string | null { - if (metricName.startsWith(`${scenario}-`)) { - return metricName.slice(scenario.length + 1); - } - return null; +function extractMetricSuffix(metricName: string): string { + return metricName; } class BenchmarkSummaryReporter implements Reporter { @@ -98,14 +95,12 @@ class BenchmarkSummaryReporter implements Reporter { for (const attachment of metricAttachments) { const fullName = attachment.name.replace('metric:', ''); - const suffix = extractMetricSuffix(fullName, scenario); - if (suffix) { - try { - const data = JSON.parse(attachment.body?.toString() ?? ''); - metrics.set(suffix, data.value); - } catch (error) { - console.warn(`[BenchmarkReporter] Malformed metric attachment "${fullName}":`, error); - } + const suffix = extractMetricSuffix(fullName); + try { + const data = JSON.parse(attachment.body?.toString() ?? ''); + metrics.set(suffix, data.value); + } catch (error) { + console.warn(`[BenchmarkReporter] Malformed metric attachment "${fullName}":`, error); } } diff --git a/packages/testing/playwright/reporters/metrics-reporter.ts b/packages/testing/playwright/reporters/metrics-reporter.ts index 29fe3ccdcb7..f115f0c7878 100644 --- a/packages/testing/playwright/reporters/metrics-reporter.ts +++ b/packages/testing/playwright/reporters/metrics-reporter.ts @@ -1,17 +1,20 @@ import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter'; -import { strict as assert } from 'assert'; -import { execSync } from 'child_process'; +import { execSync } from 'node:child_process'; +import * as os from 'node:os'; import { z } from 'zod'; const metricDataSchema = z.object({ value: z.number(), unit: z.string().optional(), + dimensions: z.record(z.union([z.string(), z.number()])).optional(), }); interface Metric { - name: string; + benchmark_name: string; + metric_name: string; value: number; unit: string | null; + dimensions: Record | null; } interface ReporterOptions { @@ -22,123 +25,157 @@ interface ReporterOptions { /** * Automatically collect performance metrics from Playwright tests and send them to a Webhook. - * If your test contains a testInfo.attach() call with a name starting with 'metric:', the metric will be collected and sent to the Webhook. + * If your test contains a testInfo.attach() call with a name starting with 'metric:', the metric + * will be collected and sent as a single batched payload at the end of the run. + * + * See utils/performance-helper.ts for the attachMetric() helper. */ class MetricsReporter implements Reporter { private webhookUrl: string | undefined; private webhookUser: string | undefined; private webhookPassword: string | undefined; - private pendingRequests: Array> = []; + private collectedMetrics: Metric[] = []; constructor(options: ReporterOptions = {}) { - this.webhookUrl = options.webhookUrl ?? process.env.QA_PERFORMANCE_METRICS_WEBHOOK_URL; - this.webhookUser = options.webhookUser ?? process.env.QA_PERFORMANCE_METRICS_WEBHOOK_USER; - this.webhookPassword = - options.webhookPassword ?? process.env.QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD; + this.webhookUrl = options.webhookUrl ?? process.env.QA_METRICS_WEBHOOK_URL; + this.webhookUser = options.webhookUser ?? process.env.QA_METRICS_WEBHOOK_USER; + this.webhookPassword = options.webhookPassword ?? process.env.QA_METRICS_WEBHOOK_PASSWORD; } - async onTestEnd(test: TestCase, result: TestResult): Promise { - if ( - !this.webhookUrl || - !this.webhookUser || - !this.webhookPassword || - result.status === 'skipped' - ) { + onTestEnd(test: TestCase, result: TestResult): void { + if (result.status === 'skipped') return; + const metrics = this.collectMetrics(test, result); + this.collectedMetrics.push(...metrics); + } + + async onEnd(): Promise { + const webhookUrl = this.webhookUrl; + if (!webhookUrl || this.collectedMetrics.length === 0) return; + if (!this.webhookUser || !this.webhookPassword) { + console.log('[MetricsReporter] QA_METRICS_WEBHOOK_USER/PASSWORD not set, skipping.'); return; } - const metrics = this.collectMetrics(result); - if (metrics.length > 0) { - const sendPromise = this.sendMetrics(test, metrics); - this.pendingRequests.push(sendPromise); - await sendPromise; + // Group by benchmark_name so each POST has a single top-level benchmark_name, + // consistent with how script-based sources (build-stats, docker-stats, etc.) send. + const byBenchmark = new Map(); + for (const m of this.collectedMetrics) { + const group = byBenchmark.get(m.benchmark_name) ?? []; + group.push(m); + byBenchmark.set(m.benchmark_name, group); } - } - private collectMetrics(result: TestResult): Metric[] { - const metrics: Metric[] = []; + const auth = Buffer.from(`${this.webhookUser}:${this.webhookPassword}`).toString('base64'); + const context = this.getContext(); - result.attachments.forEach((attachment) => { - if (attachment.name.startsWith('metric:')) { - const metricName = attachment.name.replace('metric:', ''); + const counts = await Promise.all( + Array.from(byBenchmark, async ([benchmarkName, metrics]) => { + const payload = { ...context, benchmark_name: benchmarkName, metrics }; try { - const parsedData = JSON.parse(attachment.body?.toString() ?? ''); - const data = metricDataSchema.parse(parsedData); - metrics.push({ - name: metricName, - value: data.value, - unit: data.unit ?? null, + const response = await fetch(webhookUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${auth}`, + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(30000), }); + + if (!response.ok) { + console.warn( + `[MetricsReporter] Webhook failed (${response.status}) for "${benchmarkName}": ${metrics.length} metrics dropped`, + ); + return 0; + } + return metrics.length; } catch (e) { console.warn( - `[MetricsReporter] Failed to parse metric ${metricName}: ${(e as Error).message}`, + `[MetricsReporter] Failed to send metrics for "${benchmarkName}": ${(e as Error).message}`, ); + return 0; } + }), + ); + + const sent = counts.reduce((total, n) => total + n, 0); + console.log( + `[MetricsReporter] Sent ${sent}/${this.collectedMetrics.length} metrics across ${byBenchmark.size} tests`, + ); + } + + private collectMetrics(test: TestCase, result: TestResult): Metric[] { + const metrics: Metric[] = []; + + for (const attachment of result.attachments) { + if (!attachment.name.startsWith('metric:')) continue; + const metricName = attachment.name.slice('metric:'.length); + try { + const parsed = metricDataSchema.parse(JSON.parse(attachment.body?.toString() ?? '')); + metrics.push({ + benchmark_name: test.title, + metric_name: metricName, + value: parsed.value, + unit: parsed.unit ?? null, + dimensions: parsed.dimensions ?? null, + }); + } catch (e) { + console.warn( + `[MetricsReporter] Failed to parse metric ${metricName}: ${(e as Error).message}`, + ); } - }); + } return metrics; } - private async sendMetrics(test: TestCase, metrics: Metric[]): Promise { - const gitInfo = this.getGitInfo(); + private getContext() { + const ref = process.env.GITHUB_REF ?? ''; + const prMatch = ref.match(/refs\/pull\/(\d+)/); + const runId = process.env.GITHUB_RUN_ID ?? null; - assert(gitInfo.commit, 'Git commit must be defined'); - assert(gitInfo.branch, 'Git branch must be defined'); - assert(gitInfo.author, 'Git author must be defined'); - - const payload = { - test_name: test.title, - git_commit: gitInfo.commit, - git_branch: gitInfo.branch, - git_author: gitInfo.author, + return { timestamp: new Date().toISOString(), - metrics: metrics.map((metric) => ({ - metric_name: metric.name, - metric_value: metric.value, - metric_unit: metric.unit, - })), + git: { + sha: (process.env.GITHUB_SHA ?? this.gitFallback('rev-parse HEAD'))?.slice(0, 8) ?? null, + branch: + process.env.GITHUB_HEAD_REF ?? + process.env.GITHUB_REF_NAME ?? + this.gitFallback('rev-parse --abbrev-ref HEAD'), + pr: prMatch ? parseInt(prMatch[1], 10) : null, + }, + ci: { + runId, + runUrl: + runId && process.env.GITHUB_REPOSITORY + ? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId}` + : null, + workflow: process.env.GITHUB_WORKFLOW ?? null, + job: process.env.GITHUB_JOB ?? null, + attempt: process.env.GITHUB_RUN_ATTEMPT + ? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) + : null, + }, + runner: { + provider: !process.env.CI + ? 'local' + : process.env.RUNNER_ENVIRONMENT === 'github-hosted' + ? 'github' + : 'blacksmith', + cpuCores: os.cpus().length, + memoryGb: Math.round((os.totalmem() / 1024 ** 3) * 10) / 10, + }, }; - - try { - const auth = Buffer.from(`${this.webhookUser}:${this.webhookPassword}`).toString('base64'); - - const response = await fetch(this.webhookUrl!, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Basic ${auth}`, - }, - body: JSON.stringify(payload), - signal: AbortSignal.timeout(10000), - }); - - if (!response.ok) { - console.warn(`[MetricsReporter] Webhook failed (${response.status}): ${test.title}`); - } - } catch (e) { - console.warn( - `[MetricsReporter] Failed to send metrics for test ${test.title}: ${(e as Error).message}`, - ); - } } - async onEnd(): Promise { - if (this.pendingRequests.length > 0) { - await Promise.allSettled(this.pendingRequests); - } - } - - private getGitInfo(): { commit: string | null; branch: string | null; author: string | null } { + private gitFallback(command: string): string | null { try { - return { - commit: execSync('git rev-parse HEAD', { encoding: 'utf8' }).trim(), - branch: execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf8' }).trim(), - author: execSync('git log -1 --pretty=format:"%an"', { encoding: 'utf8' }).trim(), - }; - } catch (e) { - console.error(`[MetricsReporter] Failed to get Git info: ${(e as Error).message}`); - return { commit: null, branch: null, author: null }; + return execSync(`git ${command}`, { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'ignore'], + }).trim(); + } catch { + return null; } } } diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/harness/load-harness.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/harness/load-harness.ts index 80538aff3b2..eb2777f38a5 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/harness/load-harness.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/harness/load-harness.ts @@ -14,11 +14,13 @@ import { formatDiagnosticValue, resolveMetricQuery, } from '../../../../utils/benchmark'; -import type { TriggerHandle, ExecutionMetrics } from '../../../../utils/benchmark'; - -export type LoadProfile = - | { type: 'steady'; ratePerSecond: number; durationSeconds: number } - | { type: 'preloaded'; count: number }; +import type { + TriggerHandle, + NodeOutputSize, + ExecutionMetrics, + TriggerType, + LoadProfile, +} from '../../../../utils/benchmark'; export interface LoadTestOptions { handle: TriggerHandle; @@ -27,6 +29,12 @@ export interface LoadTestOptions { testInfo: TestInfo; load: LoadProfile; timeoutMs: number; + /** Trigger type recorded as a dimension in BigQuery */ + trigger: TriggerType; + /** Node count recorded as a dimension in BigQuery */ + nodeCount: number; + /** Node output size recorded as a dimension in BigQuery */ + nodeOutputSize?: NodeOutputSize; /** PromQL metric to track workflow completions. Defaults to resolveMetricQuery(testInfo). */ metricQuery?: string; } @@ -40,10 +48,22 @@ export interface LoadTestOptions { * (direct mode: `n8n_workflow_success_total`, queue mode: `n8n_scaling_mode_queue_jobs_completed`). */ export async function runLoadTest(options: LoadTestOptions): Promise { - const { handle, api, services, testInfo, load, timeoutMs } = options; + const { handle, api, services, testInfo, load, timeoutMs, trigger } = options; const metricQuery = options.metricQuery ?? resolveMetricQuery(testInfo); testInfo.setTimeout(timeoutMs + 120_000); + const mode = testInfo.project.name.replace(':infrastructure', '').replace('benchmark-', ''); + + const dimensions: Record = { trigger, mode }; + if (options.nodeCount !== undefined) dimensions.nodes = options.nodeCount; + if (options.nodeOutputSize !== undefined) dimensions.output = options.nodeOutputSize; + if (load.type === 'steady') { + dimensions.rate = load.ratePerSecond; + dimensions.duration_s = load.durationSeconds; + } else { + dimensions.messages = load.count; + } + const obs = services.observability; const { workflowId, createdWorkflow } = await api.workflows.createWorkflowFromDefinition( @@ -117,11 +137,11 @@ export async function runLoadTest(options: LoadTestOptions): Promise 0 - ? ` Mode: queue (1 main + ${workers} workers)\n` + - ` Main: ${plan.memory}GB RAM, ${plan.cpu} CPU\n` + - ` Workers: ${wp.memory}GB RAM, ${wp.cpu} CPU each\n` + - ` Total: ${(plan.memory + wp.memory * workers).toFixed(1)}GB RAM, ${plan.cpu + wp.cpu * workers} CPU` - : ` Resources: ${plan.memory}GB RAM, ${plan.cpu} CPU`; - } - - return { name, workers, resourceSummary }; +function deriveMode(testInfo: TestInfo): string { + return testInfo.project.name.replace(':infrastructure', '').replace('benchmark-', ''); } /** @@ -74,7 +54,7 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise nodeCount, nodeOutputSize, timeoutMs, - pollIntervalMs, + trigger, plan, workerPlan, } = options; @@ -82,7 +62,31 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise testInfo.setTimeout(timeoutMs + 120_000); - const profile = deriveProfile(testInfo, plan, workerPlan); + const mode = deriveMode(testInfo); + const workers = + (testInfo.project.use as { containerConfig?: { workers?: number } }).containerConfig?.workers ?? + 0; + + const dimensions = { + trigger, + nodes: nodeCount, + output: nodeOutputSize, + messages: messageCount, + mode, + }; + + let resourceSummary = ''; + const wp = workerPlan ?? plan; + if (plan && wp) { + resourceSummary = + workers > 0 + ? ` Mode: queue (1 main + ${workers} workers)\n` + + ` Main: ${plan.memory}GB RAM, ${plan.cpu} CPU\n` + + ` Workers: ${wp.memory}GB RAM, ${wp.cpu} CPU each\n` + + ` Total: ${(plan.memory + wp.memory * workers).toFixed(1)}GB RAM, ${plan.cpu + wp.cpu * workers} CPU` + : ` Resources: ${plan.memory}GB RAM, ${plan.cpu} CPU`; + } + const obs = services.observability; const { workflowId, createdWorkflow } = await api.workflows.createWorkflowFromDefinition( @@ -93,7 +97,7 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise // Preload queue const publishResult = await handle.preload(messageCount); console.log( - `[BENCH-${profile.name}] Preloaded ${publishResult.totalPublished} messages in ${publishResult.publishDurationMs}ms`, + `[BENCH-${mode}] Preloaded ${publishResult.totalPublished} messages in ${publishResult.publishDurationMs}ms`, ); // Wait for VictoriaMetrics, then record baseline @@ -110,7 +114,7 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise // Measure throughput console.log( - `[BENCH-${profile.name}] Draining ${messageCount} messages through ${nodeCount}-node (${nodeOutputSize}) workflow (timeout: ${timeoutMs}ms)`, + `[BENCH-${mode}] Draining ${messageCount} messages through ${nodeCount}-node (${nodeOutputSize}) workflow (timeout: ${timeoutMs}ms)`, ); const result = await waitForThroughput(obs.metrics, { expectedCount: messageCount, @@ -118,49 +122,28 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise timeoutMs, baselineValue: baselineCounter, metricQuery, - pollIntervalMs, }); - // Attach results - await attachThroughputResults(testInfo, testInfo.title, result); + // Attach throughput results + await attachThroughputResults(testInfo, dimensions, result); // Execution duration sampling — provides p50/p95/p99 latency percentiles. // May be empty when EXECUTIONS_DATA_SAVE_ON_SUCCESS=none. const durations = await sampleExecutionDurations(api.workflows, workflowId); if (durations.length > 0) { const durationMetrics = buildMetrics(result.totalCompleted, 0, result.durationMs, durations); - await attachMetric( - testInfo, - `${testInfo.title}-duration-avg`, - durationMetrics.avgDurationMs, - 'ms', - ); - await attachMetric( - testInfo, - `${testInfo.title}-duration-p50`, - durationMetrics.p50DurationMs, - 'ms', - ); - await attachMetric( - testInfo, - `${testInfo.title}-duration-p95`, - durationMetrics.p95DurationMs, - 'ms', - ); - await attachMetric( - testInfo, - `${testInfo.title}-duration-p99`, - durationMetrics.p99DurationMs, - 'ms', - ); + await attachMetric(testInfo, 'duration-avg', durationMetrics.avgDurationMs, 'ms', dimensions); + await attachMetric(testInfo, 'duration-p50', durationMetrics.p50DurationMs, 'ms', dimensions); + await attachMetric(testInfo, 'duration-p95', durationMetrics.p95DurationMs, 'ms', dimensions); + await attachMetric(testInfo, 'duration-p99', durationMetrics.p99DurationMs, 'ms', dimensions); } // Diagnostics const diagnostics = await collectDiagnostics(obs.metrics, result.durationMs); - await attachDiagnostics(testInfo, testInfo.title, diagnostics); + await attachDiagnostics(testInfo, dimensions, diagnostics); const fmt = formatDiagnosticValue; console.log( - `[DIAG-${profile.name}] ${testInfo.title}\n` + + `[DIAG-${mode}] ${testInfo.title}\n` + ` Event Loop Lag: ${fmt(diagnostics.eventLoopLag, 's')}\n` + ` PG Transactions/s: ${fmt(diagnostics.pgTxRate, ' tx/s')}\n` + ` PG Rows Inserted/s: ${fmt(diagnostics.pgInsertRate, ' rows/s')}\n` + @@ -173,9 +156,9 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise // Summary console.log( - `[BENCH-${profile.name} RESULT] ${testInfo.title}\n` + - ` Profile: ${profile.name}\n` + - `${profile.resourceSummary}\n` + + `[BENCH-${mode} RESULT] ${testInfo.title}\n` + + ` Profile: ${mode}\n` + + `${resourceSummary}\n` + ` Nodes: ${nodeCount} (${nodeOutputSize}) | Messages: ${messageCount}\n` + ` Completed: ${result.totalCompleted}/${messageCount}\n` + ` Throughput: ${result.avgExecPerSec.toFixed(1)} exec/s | ${result.actionsPerSec.toFixed(1)} actions/s\n` + diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/harness/webhook-throughput-harness.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/harness/webhook-throughput-harness.ts index 11512250d91..1718bce1997 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/harness/webhook-throughput-harness.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/harness/webhook-throughput-harness.ts @@ -57,9 +57,18 @@ export async function runWebhookThroughputTest(options: WebhookThroughputOptions testInfo.setTimeout(timeoutMs + 120_000); - const profile = testInfo.project.name.replace(':infrastructure', '').replace('benchmark-', ''); + const mode = testInfo.project.name.replace(':infrastructure', '').replace('benchmark-', ''); const obs = services.observability; + const dimensions: Record = { + trigger: 'webhook', + nodes: nodeCount, + output: nodeOutputSize, + connections, + duration_s: durationSeconds, + mode, + }; + // Phase 1: Create + activate workflow // createWorkflowFromDefinition overwrites the webhook path and sets webhookId for proper registration const { workflowId, createdWorkflow, webhookPath } = @@ -116,50 +125,43 @@ export async function runWebhookThroughputTest(options: WebhookThroughputOptions const diagnostics = await collectDiagnostics(obs.metrics, throughputResult.durationMs); // Phase 6: Attach metrics — VictoriaMetrics throughput + autocannon HTTP stats + diagnostics - await attachThroughputResults(testInfo, testInfo.title, throughputResult); + await attachThroughputResults(testInfo, dimensions, throughputResult); + await attachMetric(testInfo, 'http-latency-p50', cannonResult.latency.p50, 'ms', dimensions); + await attachMetric(testInfo, 'http-latency-p99', cannonResult.latency.p99, 'ms', dimensions); await attachMetric( testInfo, - `${testInfo.title}-http-latency-p50`, - cannonResult.latency.p50, - 'ms', - ); - await attachMetric( - testInfo, - `${testInfo.title}-http-latency-p99`, - cannonResult.latency.p99, - 'ms', - ); - await attachMetric( - testInfo, - `${testInfo.title}-http-requests-total`, + 'http-requests-total', cannonResult.requests.total, 'count', + dimensions, ); await attachMetric( testInfo, - `${testInfo.title}-http-requests-avg`, + 'http-requests-avg', cannonResult.requests.average, 'req/s', + dimensions, ); await attachMetric( testInfo, - `${testInfo.title}-http-errors`, + 'http-errors', cannonResult.errors + cannonResult.non2xx, 'count', + dimensions, ); - await attachDiagnostics(testInfo, testInfo.title, diagnostics); + await attachDiagnostics(testInfo, dimensions, diagnostics); // Phase 7: Log summary const fmt = formatDiagnosticValue; console.log( - `[DIAG-${profile}] ${testInfo.title}\n` + + `[DIAG-${mode}] ${testInfo.title}\n` + ` Event Loop Lag: ${fmt(diagnostics.eventLoopLag, 's')}\n` + ` PG Transactions/s: ${fmt(diagnostics.pgTxRate, ' tx/s')}\n` + ` PG Active Connections: ${fmt(diagnostics.pgActiveConnections)}`, ); console.log( - `[WEBHOOK-${profile} RESULT] ${testInfo.title}\n` + + `[WEBHOOK-${mode} RESULT] ${testInfo.title}\n` + ` n8n Throughput: ${throughputResult.avgExecPerSec.toFixed(1)} exec/s | ` + `${throughputResult.actionsPerSec.toFixed(1)} actions/s\n` + ` Peak: ${throughputResult.peakExecPerSec.toFixed(1)} exec/s | ` + diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady-200.spec.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady-200.spec.ts index e3edd8c0bf4..c1d3910381e 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady-200.spec.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady-200.spec.ts @@ -22,6 +22,9 @@ test.describe( services, testInfo, load: { type: 'steady', ratePerSecond: 200, durationSeconds: 30 }, + trigger: 'kafka', + nodeCount: 30, + nodeOutputSize: '10KB', timeoutMs: 300_000, }); }); diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady-300.spec.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady-300.spec.ts index 25b52504e36..997e8e2109a 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady-300.spec.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady-300.spec.ts @@ -22,6 +22,9 @@ test.describe( services, testInfo, load: { type: 'steady', ratePerSecond: 300, durationSeconds: 30 }, + trigger: 'kafka', + nodeCount: 30, + nodeOutputSize: '10KB', timeoutMs: 300_000, }); }); diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady.spec.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady.spec.ts index 0dc0cc946f5..57df1f90f7f 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady.spec.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-30n-10kb-steady.spec.ts @@ -22,6 +22,9 @@ test.describe( services, testInfo, load: { type: 'steady', ratePerSecond: 100, durationSeconds: 30 }, + trigger: 'kafka', + nodeCount: 30, + nodeOutputSize: '10KB', timeoutMs: 300_000, }); }); diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-60n-1kb-burst.spec.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-60n-1kb-burst.spec.ts index 774a66b572b..94b7ca2610d 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-60n-1kb-burst.spec.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/load-60n-1kb-burst.spec.ts @@ -25,6 +25,8 @@ test.describe( services, testInfo, load: { type: 'preloaded', count: 10_000 }, + trigger: 'kafka', + nodeCount: 60, timeoutMs: 600_000, }); }); diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-10n-100kb.spec.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-10n-100kb.spec.ts index d8e49b2fd20..873b570259b 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-10n-100kb.spec.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-10n-100kb.spec.ts @@ -38,6 +38,7 @@ test.describe( messageCount: envMessages || 5_000, nodeCount: 10, nodeOutputSize: '100KB', + trigger: 'kafka', timeoutMs: 600_000, plan: BENCHMARK_MAIN_RESOURCES, workerPlan: BENCHMARK_WORKER_RESOURCES, diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-10n-10kb.spec.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-10n-10kb.spec.ts index 0e827483f1a..396d34a1189 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-10n-10kb.spec.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-10n-10kb.spec.ts @@ -33,6 +33,7 @@ test.describe( messageCount: envMessages || 5_000, nodeCount: 10, nodeOutputSize: '10KB', + trigger: 'kafka', timeoutMs: 300_000, plan: BENCHMARK_MAIN_RESOURCES, workerPlan: BENCHMARK_WORKER_RESOURCES, diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-30n-10kb.spec.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-30n-10kb.spec.ts index 99fba066454..2badd1fb287 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-30n-10kb.spec.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-30n-10kb.spec.ts @@ -33,6 +33,7 @@ test.describe( messageCount: envMessages || 5_000, nodeCount: 30, nodeOutputSize: '10KB', + trigger: 'kafka', timeoutMs: 300_000, plan: BENCHMARK_MAIN_RESOURCES, workerPlan: BENCHMARK_WORKER_RESOURCES, diff --git a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-60n-10kb.spec.ts b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-60n-10kb.spec.ts index 4cc009390c1..90e0e39fe4b 100644 --- a/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-60n-10kb.spec.ts +++ b/packages/testing/playwright/tests/infrastructure/benchmarks/kafka/throughput-60n-10kb.spec.ts @@ -33,6 +33,7 @@ test.describe( messageCount: envMessages || 5_000, nodeCount: 60, nodeOutputSize: '10KB', + trigger: 'kafka', timeoutMs: 600_000, plan: BENCHMARK_MAIN_RESOURCES, workerPlan: BENCHMARK_WORKER_RESOURCES, diff --git a/packages/testing/playwright/tests/performance/large-node-cloud.spec.ts b/packages/testing/playwright/tests/performance/large-node-cloud.spec.ts index 6df2e4ac120..4a6f3c8c011 100644 --- a/packages/testing/playwright/tests/performance/large-node-cloud.spec.ts +++ b/packages/testing/playwright/tests/performance/large-node-cloud.spec.ts @@ -51,8 +51,10 @@ test.describe( } const average = stats.reduce((a, b) => a + b, 0) / stats.length; - await attachMetric(testInfo, `open-node-${itemCount}`, average, 'ms'); - await attachMetric(testInfo, `trigger-workflow-${itemCount}`, triggerDuration, 'ms'); + await attachMetric(testInfo, 'open-node', average, 'ms', { item_count: itemCount }); + await attachMetric(testInfo, 'trigger-workflow', triggerDuration, 'ms', { + item_count: itemCount, + }); expect(average).toBeGreaterThan(0); expect(triggerDuration).toBeGreaterThan(0); diff --git a/packages/testing/playwright/utils/benchmark/diagnostics.ts b/packages/testing/playwright/utils/benchmark/diagnostics.ts index 0b323174471..bc9d8e91744 100644 --- a/packages/testing/playwright/utils/benchmark/diagnostics.ts +++ b/packages/testing/playwright/utils/benchmark/diagnostics.ts @@ -125,16 +125,16 @@ export async function collectDiagnostics( */ export async function attachDiagnostics( testInfo: TestInfo, - label: string, + dimensions: Record, diagnostics: DiagnosticsResult, ): Promise { if (diagnostics.eventLoopLag !== undefined) { - await attachMetric(testInfo, `${label}-event-loop-lag`, diagnostics.eventLoopLag, 's'); + await attachMetric(testInfo, 'event-loop-lag', diagnostics.eventLoopLag, 's', dimensions); } if (diagnostics.pgTxRate !== undefined) { - await attachMetric(testInfo, `${label}-pg-tx-rate`, diagnostics.pgTxRate, 'tx/s'); + await attachMetric(testInfo, 'pg-tx-rate', diagnostics.pgTxRate, 'tx/s', dimensions); } if (diagnostics.queueWaiting !== undefined) { - await attachMetric(testInfo, `${label}-queue-waiting`, diagnostics.queueWaiting, 'count'); + await attachMetric(testInfo, 'queue-waiting', diagnostics.queueWaiting, 'count', dimensions); } } diff --git a/packages/testing/playwright/utils/benchmark/execution-sampler.ts b/packages/testing/playwright/utils/benchmark/execution-sampler.ts index 83b2176b1b7..d2be9f53d62 100644 --- a/packages/testing/playwright/utils/benchmark/execution-sampler.ts +++ b/packages/testing/playwright/utils/benchmark/execution-sampler.ts @@ -82,20 +82,20 @@ export function buildMetrics( export async function attachLoadTestResults( testInfo: TestInfo, - label: string, + dimensions: Record, metrics: ExecutionMetrics, ): Promise { - await attachMetric(testInfo, `${label}-executions-completed`, metrics.totalCompleted, 'count'); - await attachMetric(testInfo, `${label}-executions-errors`, metrics.totalErrors, 'count'); - await attachMetric(testInfo, `${label}-throughput`, metrics.throughputPerSecond, 'exec/s'); - await attachMetric(testInfo, `${label}-total-duration`, metrics.durationMs, 'ms'); + await attachMetric(testInfo, 'executions-completed', metrics.totalCompleted, 'count', dimensions); + await attachMetric(testInfo, 'executions-errors', metrics.totalErrors, 'count', dimensions); + await attachMetric(testInfo, 'throughput', metrics.throughputPerSecond, 'exec/s', dimensions); + await attachMetric(testInfo, 'total-duration', metrics.durationMs, 'ms', dimensions); // Only attach duration percentiles when we have sampled data — otherwise the // reporter would show misleading "0ms" values (e.g. when EXECUTIONS_DATA_SAVE_ON_SUCCESS=none) if (metrics.executionDurations.length > 0) { - await attachMetric(testInfo, `${label}-duration-avg`, metrics.avgDurationMs, 'ms'); - await attachMetric(testInfo, `${label}-duration-p50`, metrics.p50DurationMs, 'ms'); - await attachMetric(testInfo, `${label}-duration-p95`, metrics.p95DurationMs, 'ms'); - await attachMetric(testInfo, `${label}-duration-p99`, metrics.p99DurationMs, 'ms'); + await attachMetric(testInfo, 'duration-avg', metrics.avgDurationMs, 'ms', dimensions); + await attachMetric(testInfo, 'duration-p50', metrics.p50DurationMs, 'ms', dimensions); + await attachMetric(testInfo, 'duration-p95', metrics.p95DurationMs, 'ms', dimensions); + await attachMetric(testInfo, 'duration-p99', metrics.p99DurationMs, 'ms', dimensions); } } diff --git a/packages/testing/playwright/utils/benchmark/throughput-measure.ts b/packages/testing/playwright/utils/benchmark/throughput-measure.ts index 481127f70de..69cae1121d4 100644 --- a/packages/testing/playwright/utils/benchmark/throughput-measure.ts +++ b/packages/testing/playwright/utils/benchmark/throughput-measure.ts @@ -210,18 +210,19 @@ function calculateThroughput( export async function attachThroughputResults( testInfo: TestInfo, - label: string, + dimensions: Record, result: ThroughputResult, ): Promise { - await attachMetric(testInfo, `${label}-exec-per-sec`, result.avgExecPerSec, 'exec/s'); - await attachMetric(testInfo, `${label}-actions-per-sec`, result.actionsPerSec, 'actions/s'); - await attachMetric(testInfo, `${label}-peak-exec-per-sec`, result.peakExecPerSec, 'exec/s'); + await attachMetric(testInfo, 'exec-per-sec', result.avgExecPerSec, 'exec/s', dimensions); + await attachMetric(testInfo, 'actions-per-sec', result.actionsPerSec, 'actions/s', dimensions); + await attachMetric(testInfo, 'peak-exec-per-sec', result.peakExecPerSec, 'exec/s', dimensions); await attachMetric( testInfo, - `${label}-peak-actions-per-sec`, + 'peak-actions-per-sec', result.peakActionsPerSec, 'actions/s', + dimensions, ); - await attachMetric(testInfo, `${label}-total-completed`, result.totalCompleted, 'count'); - await attachMetric(testInfo, `${label}-duration`, result.durationMs, 'ms'); + await attachMetric(testInfo, 'total-completed', result.totalCompleted, 'count', dimensions); + await attachMetric(testInfo, 'duration', result.durationMs, 'ms', dimensions); } diff --git a/packages/testing/playwright/utils/benchmark/types.ts b/packages/testing/playwright/utils/benchmark/types.ts index 2f61135651a..8ee00603c75 100644 --- a/packages/testing/playwright/utils/benchmark/types.ts +++ b/packages/testing/playwright/utils/benchmark/types.ts @@ -3,6 +3,23 @@ import type { IWorkflowBase } from 'n8n-workflow'; import type { ApiHelpers } from '../../services/api-helper'; +// --- Benchmark dimensions --- + +/** + * Structured dimensions attached to every benchmark metric. + * Stored as JSON in BigQuery — enables slicing by node count, payload, mode, etc. + * without encoding config into the metric name. + */ +export type BenchmarkDimensions = Record; + +/** Known trigger types. Must be set explicitly — no default to avoid silently wrong data. */ +export type TriggerType = 'kafka' | 'webhook'; + +/** Load profile for steady-rate or preloaded burst tests. */ +export type LoadProfile = + | { type: 'steady'; ratePerSecond: number; durationSeconds: number } + | { type: 'preloaded'; count: number }; + // --- Payload sizes --- export const PAYLOAD_PROFILES = { diff --git a/packages/testing/playwright/utils/performance-helper.ts b/packages/testing/playwright/utils/performance-helper.ts index 5af88ab44f7..4c0e78664af 100644 --- a/packages/testing/playwright/utils/performance-helper.ts +++ b/packages/testing/playwright/utils/performance-helper.ts @@ -36,9 +36,21 @@ export async function attachMetric( metricName: string, value: number, unit?: string, + dimensions?: Record, ): Promise { await testInfo.attach(`metric:${metricName}`, { - body: JSON.stringify({ value, unit }), + body: JSON.stringify({ value, unit, dimensions }), + }); + + // Currents native format — surfaces metrics in their analytics dashboard + testInfo.annotations.push({ + type: 'currents:metric', + description: JSON.stringify({ + name: metricName, + value, + type: Number.isInteger(value) ? 'integer' : 'float', + ...(unit && { unit: unit.toLowerCase() }), + }), }); }