mirror of
https://github.com/n8n-io/n8n
synced 2026-04-21 15:47:20 +00:00
ci: Unify QA metrics pipeline to single webhook, format, and BigQuery table (no-changelog) (#27111)
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Declan Carroll
GitHub
parent
dcd306bc5c
commit
7c7c70f142
36 changed files with 739 additions and 749 deletions
94
.github/CI-TELEMETRY.md
vendored
94
.github/CI-TELEMETRY.md
vendored
|
|
@ -2,26 +2,39 @@
|
|||
|
||||
Pipeline: **GitHub Actions → Webhook → n8n → BigQuery**
|
||||
|
||||
## Standard Data Points
|
||||
## Unified Payload Shape
|
||||
|
||||
All telemetry includes these fields for correlation:
|
||||
All telemetry uses the same format:
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": "2026-03-16T12:00:00.000Z",
|
||||
"benchmark_name": "kafka-throughput-10n-10kb",
|
||||
"git": { "sha": "abc12345", "branch": "master", "pr": null },
|
||||
"ci": { "runId": "123", "runUrl": "...", "job": "test", "workflow": "CI", "attempt": 1 },
|
||||
"runner": { "provider": "blacksmith", "cpuCores": 8, "memoryGb": 16.0 },
|
||||
"metrics": [
|
||||
{ "metric_name": "exec-per-sec", "value": 12.4, "unit": "exec/s", "dimensions": { "trigger": "kafka", "nodes": 10 } }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Standard Context Fields
|
||||
|
||||
```typescript
|
||||
// Git context
|
||||
git.sha // GITHUB_SHA (first 8 chars)
|
||||
git.branch // GITHUB_HEAD_REF ?? GITHUB_REF_NAME
|
||||
git.pr // PR number from GITHUB_REF
|
||||
|
||||
// CI context
|
||||
ci.runId // GITHUB_RUN_ID
|
||||
ci.runUrl // https://github.com/<repo>/actions/runs/<runId>
|
||||
ci.job // GITHUB_JOB
|
||||
ci.workflow // GITHUB_WORKFLOW
|
||||
ci.attempt // GITHUB_RUN_ATTEMPT
|
||||
|
||||
// Runner detection
|
||||
runner.provider // 'github' | 'blacksmith' | 'local'
|
||||
runner.cpuCores // os.cpus().length
|
||||
runner.memoryGb // os.totalmem()
|
||||
runner.provider // 'github' | 'blacksmith' | 'local'
|
||||
runner.cpuCores // os.cpus().length
|
||||
runner.memoryGb // os.totalmem()
|
||||
```
|
||||
|
||||
**Runner provider logic:**
|
||||
|
|
@ -35,22 +48,67 @@ return 'blacksmith';
|
|||
|
||||
| Telemetry | Source | Metrics |
|
||||
|-----------|--------|---------|
|
||||
| Build stats | `.github/scripts/send-build-stats.mjs` | Per-package build time, cache hits |
|
||||
| Docker stats | `.github/scripts/send-docker-stats.mjs` | Image size, compiled artifact size, docker build time |
|
||||
| Container stack | `packages/testing/containers/telemetry.ts` | E2E startup times |
|
||||
| Playwright perf/benchmark | `packages/testing/playwright/reporters/metrics-reporter.ts` | Any metric attached via `attachMetric()` |
|
||||
| Build stats | `.github/scripts/send-build-stats.mjs` | Per-package build duration, cache hit/miss, run total |
|
||||
| Docker stats | `.github/scripts/send-docker-stats.mjs` | Image size per platform, docker build duration |
|
||||
| Container stack | `packages/testing/containers/telemetry.ts` | E2E stack startup times per service |
|
||||
|
||||
## Secrets
|
||||
|
||||
```
|
||||
BUILD_STATS_WEBHOOK_URL
|
||||
BUILD_STATS_WEBHOOK_USER
|
||||
BUILD_STATS_WEBHOOK_PASSWORD # Alphanumeric + hyphens only (no $!#@)
|
||||
QA_METRICS_WEBHOOK_URL
|
||||
QA_METRICS_WEBHOOK_USER
|
||||
QA_METRICS_WEBHOOK_PASSWORD
|
||||
```
|
||||
|
||||
DOCKER_STATS_WEBHOOK_URL
|
||||
## BigQuery Table
|
||||
|
||||
`qa_performance_metrics` — schema:
|
||||
|
||||
```sql
|
||||
timestamp TIMESTAMP NOT NULL
|
||||
benchmark_name STRING
|
||||
metric_name STRING NOT NULL
|
||||
value FLOAT64 NOT NULL
|
||||
unit STRING
|
||||
dimensions JSON -- {"nodes": 10, "trigger": "kafka", "package": "@n8n/cli"}
|
||||
git_sha STRING
|
||||
git_branch STRING
|
||||
git_pr INT64
|
||||
ci_run_id STRING
|
||||
ci_run_url STRING
|
||||
ci_job STRING
|
||||
ci_workflow STRING
|
||||
ci_attempt INT64
|
||||
runner_provider STRING
|
||||
runner_cpu_cores INT64
|
||||
runner_memory_gb FLOAT64
|
||||
```
|
||||
|
||||
Query example:
|
||||
```sql
|
||||
-- Build duration trend by package (cache misses only)
|
||||
SELECT DATE(timestamp), JSON_VALUE(dimensions, '$.package'), AVG(value)
|
||||
FROM qa_performance_metrics
|
||||
WHERE metric_name = 'build-duration'
|
||||
AND JSON_VALUE(dimensions, '$.cache') = 'miss'
|
||||
GROUP BY 1, 2 ORDER BY 1;
|
||||
```
|
||||
|
||||
## Adding New Telemetry
|
||||
|
||||
1. Copy data point helpers from `send-build-stats.mjs`
|
||||
2. Create n8n workflow: Webhook (Basic Auth) → Code (flatten) → BigQuery
|
||||
3. Add secrets to GitHub
|
||||
**From a script:**
|
||||
```javascript
|
||||
import { sendMetrics, metric } from './send-metrics.mjs';
|
||||
|
||||
await sendMetrics([
|
||||
metric('my-metric', 42.0, 'ms', { context: 'value' }),
|
||||
]);
|
||||
```
|
||||
|
||||
**From a Playwright test:**
|
||||
```typescript
|
||||
import { attachMetric } from '../utils/performance-helper';
|
||||
|
||||
await attachMetric(testInfo, 'my-metric', 42.0, 'ms', { context: 'value' });
|
||||
```
|
||||
|
|
|
|||
6
.github/WORKFLOWS.md
vendored
6
.github/WORKFLOWS.md
vendored
|
|
@ -200,7 +200,6 @@ These only run if specific files changed:
|
|||
|
||||
| Command | Workflow | Permissions |
|
||||
|--------------------|------------------------------|---------------------|
|
||||
| `/build-unit-test` | `ci-manual-unit-tests.yml` | admin/write/maintain|
|
||||
| `/test-workflows` | `test-workflows-callable.yml`| admin/write/maintain|
|
||||
|
||||
**Why:** Re-run tests without pushing commits. Useful for flaky test investigation.
|
||||
|
|
@ -260,9 +259,6 @@ test-workflows-nightly.yml
|
|||
└──────────────────────────▶ test-workflows-callable.yml
|
||||
|
||||
PR Comment Dispatchers (triggered by /command in PR comments):
|
||||
build-unit-test-pr-comment.yml
|
||||
└──────────────────────────▶ ci-manual-unit-tests.yml
|
||||
|
||||
test-workflows-pr-comment.yml
|
||||
└──────────────────────────▶ test-workflows-callable.yml
|
||||
```
|
||||
|
|
@ -663,7 +659,7 @@ cosign verify-attestation --type openvex \
|
|||
|
||||
### Redundancy Review
|
||||
|
||||
Comment triggers (`/build-unit-test`, `/test-workflows`) are workarounds.
|
||||
Comment trigger (`/test-workflows`) is a workaround.
|
||||
|
||||
Long-term: Main CI should be reliable enough to not need these.
|
||||
|
||||
|
|
|
|||
121
.github/scripts/send-build-stats.mjs
vendored
121
.github/scripts/send-build-stats.mjs
vendored
|
|
@ -1,104 +1,67 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Sends Turbo build summary JSON to a webhook
|
||||
* Sends Turbo build stats to the unified QA metrics webhook.
|
||||
*
|
||||
* Reads the Turbo run summary from .turbo/runs/ and emits per-package
|
||||
* build-duration metrics with {package, cache, task} dimensions, plus
|
||||
* a run-level build-total-duration summary.
|
||||
*
|
||||
* Usage: node send-build-stats.mjs
|
||||
*
|
||||
* Auto-detects summary from .turbo/runs/ directory.
|
||||
*
|
||||
* Environment variables:
|
||||
* BUILD_STATS_WEBHOOK_URL - Webhook URL (required to send)
|
||||
* BUILD_STATS_WEBHOOK_USER - Basic auth username (required if URL set)
|
||||
* BUILD_STATS_WEBHOOK_PASSWORD - Basic auth password (required if URL set)
|
||||
* QA_METRICS_WEBHOOK_URL - Webhook URL (required to send)
|
||||
* QA_METRICS_WEBHOOK_USER - Basic auth username
|
||||
* QA_METRICS_WEBHOOK_PASSWORD - Basic auth password
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { sendMetrics, metric } from './send-metrics.mjs';
|
||||
|
||||
const runsDir = '.turbo/runs';
|
||||
if (!existsSync(runsDir)) {
|
||||
console.log('No .turbo/runs directory found (turbo --summarize not used), skipping.');
|
||||
process.exit(0);
|
||||
}
|
||||
const files = readdirSync(runsDir).filter((f) => f.endsWith('.json'));
|
||||
const summaryPath = files.length > 0 ? join(runsDir, files[0]) : null;
|
||||
|
||||
const webhookUrl = process.env.BUILD_STATS_WEBHOOK_URL;
|
||||
const webhookUser = process.env.BUILD_STATS_WEBHOOK_USER;
|
||||
const webhookPassword = process.env.BUILD_STATS_WEBHOOK_PASSWORD;
|
||||
|
||||
if (!summaryPath) {
|
||||
const files = readdirSync(runsDir)
|
||||
.filter((f) => f.endsWith('.json'))
|
||||
.sort();
|
||||
if (files.length === 0) {
|
||||
console.error('No summary file found in .turbo/runs/');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!webhookUrl) {
|
||||
console.log('BUILD_STATS_WEBHOOK_URL not set, skipping.');
|
||||
process.exit(0);
|
||||
const summary = JSON.parse(readFileSync(join(runsDir, files.at(-1)), 'utf-8'));
|
||||
|
||||
const metrics = [];
|
||||
|
||||
for (const task of summary.tasks ?? []) {
|
||||
if (task.execution?.exitCode !== 0) continue;
|
||||
const durationMs = task.execution.durationMs ?? 0;
|
||||
const cacheHit = task.cache?.status === 'HIT';
|
||||
// taskId format: "package-name#task-name"
|
||||
const [pkg, taskName] = task.taskId?.split('#') ?? [task.package, task.task];
|
||||
|
||||
metrics.push(
|
||||
metric('build-duration', durationMs / 1000, 's', {
|
||||
package: pkg ?? 'unknown',
|
||||
task: taskName ?? 'build',
|
||||
cache: cacheHit ? 'hit' : 'miss',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (!webhookUser || !webhookPassword) {
|
||||
console.error('BUILD_STATS_WEBHOOK_USER and BUILD_STATS_WEBHOOK_PASSWORD are required');
|
||||
process.exit(1);
|
||||
}
|
||||
const totalMs = summary.durationMs ?? 0;
|
||||
const totalTasks = summary.tasks?.length ?? 0;
|
||||
const cachedTasks = summary.tasks?.filter((t) => t.cache?.status === 'HIT').length ?? 0;
|
||||
|
||||
const basicAuth = Buffer.from(`${webhookUser}:${webhookPassword}`).toString('base64');
|
||||
metrics.push(
|
||||
metric('build-total-duration', totalMs / 1000, 's', {
|
||||
total_tasks: totalTasks,
|
||||
cached_tasks: cachedTasks,
|
||||
}),
|
||||
);
|
||||
|
||||
const summary = JSON.parse(readFileSync(summaryPath, 'utf-8'));
|
||||
|
||||
// Extract PR number from GITHUB_REF (refs/pull/123/merge)
|
||||
const ref = process.env.GITHUB_REF ?? '';
|
||||
const prMatch = ref.match(/refs\/pull\/(\d+)/);
|
||||
|
||||
// Detect runner provider (matches packages/testing/containers/telemetry.ts)
|
||||
function getRunnerProvider() {
|
||||
if (!process.env.CI) return 'local';
|
||||
if (process.env.RUNNER_ENVIRONMENT === 'github-hosted') return 'github';
|
||||
return 'blacksmith';
|
||||
}
|
||||
|
||||
// Add git context (aligned with container telemetry)
|
||||
summary.git = {
|
||||
sha: process.env.GITHUB_SHA?.slice(0, 8) || null,
|
||||
branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? null,
|
||||
pr: prMatch ? parseInt(prMatch[1], 10) : null,
|
||||
};
|
||||
|
||||
// Add CI context
|
||||
summary.ci = {
|
||||
runId: process.env.GITHUB_RUN_ID || null,
|
||||
runUrl: process.env.GITHUB_RUN_ID
|
||||
? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
|
||||
: null,
|
||||
job: process.env.GITHUB_JOB || null,
|
||||
workflow: process.env.GITHUB_WORKFLOW || null,
|
||||
attempt: process.env.GITHUB_RUN_ATTEMPT ? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) : null,
|
||||
};
|
||||
|
||||
// Add runner info
|
||||
summary.runner = {
|
||||
provider: getRunnerProvider(),
|
||||
cpuCores: os.cpus().length,
|
||||
memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10,
|
||||
};
|
||||
|
||||
const headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Basic ${basicAuth}`,
|
||||
};
|
||||
|
||||
const response = await fetch(webhookUrl, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(summary),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.error(`Webhook failed: ${response.status} ${response.statusText}`);
|
||||
const body = await response.text();
|
||||
if (body) console.error(`Response: ${body}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Build stats sent: ${response.status}`);
|
||||
await sendMetrics(metrics, 'build-stats');
|
||||
|
|
|
|||
110
.github/scripts/send-docker-stats.mjs
vendored
110
.github/scripts/send-docker-stats.mjs
vendored
|
|
@ -1,18 +1,22 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Sends Docker build stats to a webhook for BigQuery ingestion.
|
||||
* Sends Docker build stats to the unified QA metrics webhook.
|
||||
*
|
||||
* Reads manifests produced by build-n8n.mjs and dockerize-n8n.mjs,
|
||||
* enriches with git/CI/runner context, and POSTs to a webhook.
|
||||
* Reads manifests produced by build-n8n.mjs and dockerize-n8n.mjs and emits
|
||||
* per-image docker-image-size metrics and build duration metrics with
|
||||
* {image, platform} dimensions.
|
||||
*
|
||||
* Usage: node send-docker-stats.mjs
|
||||
*
|
||||
* Environment variables:
|
||||
* DOCKER_STATS_WEBHOOK_URL - Webhook URL (required to send)
|
||||
* QA_METRICS_WEBHOOK_URL - Webhook URL (required to send)
|
||||
* QA_METRICS_WEBHOOK_USER - Basic auth username
|
||||
* QA_METRICS_WEBHOOK_PASSWORD - Basic auth password
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
|
||||
import { sendMetrics, metric } from './send-metrics.mjs';
|
||||
|
||||
const buildManifestPath = 'compiled/build-manifest.json';
|
||||
const dockerManifestPath = 'docker-build-manifest.json';
|
||||
|
|
@ -22,13 +26,6 @@ if (!existsSync(buildManifestPath) && !existsSync(dockerManifestPath)) {
|
|||
process.exit(0);
|
||||
}
|
||||
|
||||
const webhookUrl = process.env.DOCKER_STATS_WEBHOOK_URL;
|
||||
|
||||
if (!webhookUrl) {
|
||||
console.log('DOCKER_STATS_WEBHOOK_URL not set, skipping.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const buildManifest = existsSync(buildManifestPath)
|
||||
? JSON.parse(readFileSync(buildManifestPath, 'utf-8'))
|
||||
: null;
|
||||
|
|
@ -37,68 +34,41 @@ const dockerManifest = existsSync(dockerManifestPath)
|
|||
? JSON.parse(readFileSync(dockerManifestPath, 'utf-8'))
|
||||
: null;
|
||||
|
||||
// Extract PR number from GITHUB_REF (refs/pull/123/merge)
|
||||
const ref = process.env.GITHUB_REF ?? '';
|
||||
const prMatch = ref.match(/refs\/pull\/(\d+)/);
|
||||
const metrics = [];
|
||||
|
||||
// Detect runner provider (matches packages/testing/containers/telemetry.ts)
|
||||
function getRunnerProvider() {
|
||||
if (!process.env.CI) return 'local';
|
||||
if (process.env.RUNNER_ENVIRONMENT === 'github-hosted') return 'github';
|
||||
return 'blacksmith';
|
||||
if (buildManifest) {
|
||||
if (buildManifest.artifactSize != null) {
|
||||
metrics.push(metric('artifact-size', buildManifest.artifactSize, 'bytes', { artifact: 'compiled' }));
|
||||
}
|
||||
if (buildManifest.buildDuration != null) {
|
||||
metrics.push(metric('build-duration', buildManifest.buildDuration / 1000, 's', { artifact: 'compiled' }));
|
||||
}
|
||||
}
|
||||
|
||||
const payload = {
|
||||
build: buildManifest
|
||||
? {
|
||||
artifactSize: buildManifest.artifactSize,
|
||||
buildDuration: buildManifest.buildDuration,
|
||||
}
|
||||
: null,
|
||||
if (dockerManifest) {
|
||||
const platform = dockerManifest.platform ?? 'unknown';
|
||||
|
||||
docker: dockerManifest
|
||||
? {
|
||||
platform: dockerManifest.platform,
|
||||
images: dockerManifest.images,
|
||||
}
|
||||
: null,
|
||||
for (const image of dockerManifest.images ?? []) {
|
||||
if (image.sizeBytes != null) {
|
||||
metrics.push(
|
||||
metric('docker-image-size', image.sizeBytes, 'bytes', {
|
||||
image: image.name ?? 'unknown',
|
||||
platform,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
git: {
|
||||
sha: process.env.GITHUB_SHA?.slice(0, 8) || null,
|
||||
branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? null,
|
||||
pr: prMatch ? parseInt(prMatch[1], 10) : null,
|
||||
},
|
||||
|
||||
ci: {
|
||||
runId: process.env.GITHUB_RUN_ID || null,
|
||||
runUrl: process.env.GITHUB_RUN_ID
|
||||
? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
|
||||
: null,
|
||||
job: process.env.GITHUB_JOB || null,
|
||||
workflow: process.env.GITHUB_WORKFLOW || null,
|
||||
attempt: process.env.GITHUB_RUN_ATTEMPT ? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) : null,
|
||||
},
|
||||
|
||||
runner: {
|
||||
provider: getRunnerProvider(),
|
||||
cpuCores: os.cpus().length,
|
||||
memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10,
|
||||
},
|
||||
};
|
||||
|
||||
const response = await fetch(webhookUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.error(`Webhook failed: ${response.status} ${response.statusText}`);
|
||||
const body = await response.text();
|
||||
if (body) console.error(`Response: ${body}`);
|
||||
process.exit(1);
|
||||
if (dockerManifest.buildDurationMs != null) {
|
||||
metrics.push(
|
||||
metric('docker-build-duration', dockerManifest.buildDurationMs / 1000, 's', { platform }),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Docker build stats sent: ${response.status}`);
|
||||
if (metrics.length === 0) {
|
||||
console.log('No metrics to send.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
await sendMetrics(metrics, 'docker-stats');
|
||||
|
|
|
|||
94
.github/scripts/send-metrics.mjs
vendored
Normal file
94
.github/scripts/send-metrics.mjs
vendored
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Shared metrics sender for CI scripts.
|
||||
* See .github/CI-TELEMETRY.md for payload shape and BigQuery schema.
|
||||
*
|
||||
* Usage:
|
||||
* import { sendMetrics, metric } from './send-metrics.mjs';
|
||||
* await sendMetrics([metric('build-duration', 45.2, 's', { package: '@n8n/cli' })]);
|
||||
*
|
||||
* Env: QA_METRICS_WEBHOOK_URL, QA_METRICS_WEBHOOK_USER, QA_METRICS_WEBHOOK_PASSWORD
|
||||
*/
|
||||
|
||||
import * as os from 'node:os';
|
||||
|
||||
/** Build a single metric object. */
|
||||
export function metric(name, value, unit, dimensions = {}) {
|
||||
return { metric_name: name, value, unit, dimensions };
|
||||
}
|
||||
|
||||
/** Build git/ci/runner context from environment variables. */
|
||||
export function buildContext(benchmarkName = null) {
|
||||
const ref = process.env.GITHUB_REF ?? '';
|
||||
const prMatch = ref.match(/refs\/pull\/(\d+)/);
|
||||
const runId = process.env.GITHUB_RUN_ID ?? null;
|
||||
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
benchmark_name: benchmarkName,
|
||||
git: {
|
||||
sha: process.env.GITHUB_SHA?.slice(0, 8) ?? null,
|
||||
branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? null,
|
||||
pr: prMatch ? parseInt(prMatch[1], 10) : null,
|
||||
},
|
||||
ci: {
|
||||
runId,
|
||||
runUrl:
|
||||
runId && process.env.GITHUB_REPOSITORY
|
||||
? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId}`
|
||||
: null,
|
||||
job: process.env.GITHUB_JOB ?? null,
|
||||
workflow: process.env.GITHUB_WORKFLOW ?? null,
|
||||
attempt: process.env.GITHUB_RUN_ATTEMPT
|
||||
? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10)
|
||||
: null,
|
||||
},
|
||||
runner: {
|
||||
provider: !process.env.CI
|
||||
? 'local'
|
||||
: process.env.RUNNER_ENVIRONMENT === 'github-hosted'
|
||||
? 'github'
|
||||
: 'blacksmith',
|
||||
cpuCores: os.cpus().length,
|
||||
memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function sendMetrics(metrics, benchmarkName = null) {
|
||||
const webhookUrl = process.env.QA_METRICS_WEBHOOK_URL;
|
||||
const webhookUser = process.env.QA_METRICS_WEBHOOK_USER;
|
||||
const webhookPassword = process.env.QA_METRICS_WEBHOOK_PASSWORD;
|
||||
|
||||
if (!webhookUrl) {
|
||||
console.log('QA_METRICS_WEBHOOK_URL not set, skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!webhookUser || !webhookPassword) {
|
||||
console.log('QA_METRICS_WEBHOOK_USER/PASSWORD not set, skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
const payload = { ...buildContext(benchmarkName), metrics };
|
||||
const basicAuth = Buffer.from(`${webhookUser}:${webhookPassword}`).toString('base64');
|
||||
|
||||
const response = await fetch(webhookUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Basic ${basicAuth}`,
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new Error(
|
||||
`Webhook failed: ${response.status} ${response.statusText}${body ? `\n${body}` : ''}`,
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`Sent ${metrics.length} metric(s): ${response.status}`);
|
||||
}
|
||||
100
.github/workflows/build-unit-test-pr-comment.yml
vendored
100
.github/workflows/build-unit-test-pr-comment.yml
vendored
|
|
@ -1,100 +0,0 @@
|
|||
name: 'Build: Unit Test PR Comment'
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
pull-requests: read
|
||||
contents: read
|
||||
actions: write
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
validate_and_dispatch:
|
||||
name: Validate user and dispatch CI workflow
|
||||
if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/build-unit-test')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Validate user permissions and collect PR data
|
||||
id: check_permissions
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const commenter = context.actor;
|
||||
const body = (context.payload.comment.body || '').trim();
|
||||
const isCommand = body.startsWith('/build-unit-test');
|
||||
const allowedPermissions = ['admin', 'write', 'maintain'];
|
||||
const commentId = context.payload.comment.id;
|
||||
const { owner, repo } = context.repo;
|
||||
|
||||
async function react(content) {
|
||||
try {
|
||||
await github.rest.reactions.createForIssueComment({
|
||||
owner,
|
||||
repo,
|
||||
comment_id: commentId,
|
||||
content,
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(`Failed to add reaction '${content}': ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
core.setOutput('proceed', 'false');
|
||||
core.setOutput('headSha', '');
|
||||
core.setOutput('prNumber', '');
|
||||
|
||||
if (!context.payload.issue.pull_request || !isCommand) {
|
||||
console.log('Comment is not /build-unit-test on a pull request. Skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
let permission;
|
||||
try {
|
||||
const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
|
||||
owner,
|
||||
repo,
|
||||
username: commenter,
|
||||
});
|
||||
permission = data.permission;
|
||||
} catch (error) {
|
||||
console.log(`Could not verify permissions for @${commenter}: ${error.message}`);
|
||||
await react('confused');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!allowedPermissions.includes(permission)) {
|
||||
console.log(`User @${commenter} has '${permission}' permission; requires admin/write/maintain.`);
|
||||
await react('-1');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const prNumber = context.issue.number;
|
||||
const { data: pr } = await github.rest.pulls.get({
|
||||
owner,
|
||||
repo,
|
||||
pull_number: prNumber,
|
||||
});
|
||||
await react('+1');
|
||||
core.setOutput('proceed', 'true');
|
||||
core.setOutput('headSha', pr.head.sha);
|
||||
core.setOutput('prNumber', String(prNumber));
|
||||
} catch (error) {
|
||||
console.log(`Failed to fetch PR details for PR #${context.issue.number}: ${error.message}`);
|
||||
await react('confused');
|
||||
}
|
||||
|
||||
- name: Dispatch build/unit test workflow
|
||||
if: ${{ steps.check_permissions.outputs.proceed == 'true' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
HEAD_SHA: ${{ steps.check_permissions.outputs.headSha }}
|
||||
PR_NUMBER: ${{ steps.check_permissions.outputs.prNumber }}
|
||||
run: |
|
||||
gh workflow run ci-manual-unit-tests.yml \
|
||||
--repo "${{ github.repository }}" \
|
||||
-f ref="${HEAD_SHA}" \
|
||||
-f pr_number="${PR_NUMBER}"
|
||||
135
.github/workflows/ci-manual-unit-tests.yml
vendored
135
.github/workflows/ci-manual-unit-tests.yml
vendored
|
|
@ -1,135 +0,0 @@
|
|||
name: 'CI: Manual Unit Tests'
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Commit SHA or ref to check out
|
||||
required: true
|
||||
pr_number:
|
||||
description: PR number (optional, for check reporting)
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
|
||||
jobs:
|
||||
create-check-run:
|
||||
name: Create Check Run
|
||||
runs-on: ubuntu-latest
|
||||
if: inputs.pr_number != ''
|
||||
outputs:
|
||||
check_run_id: ${{ steps.create.outputs.check_run_id }}
|
||||
steps:
|
||||
- name: Create pending check run on PR
|
||||
id: create
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const { data: checkRun } = await github.rest.checks.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: 'Build & Unit Tests - Checks',
|
||||
head_sha: '${{ inputs.ref }}',
|
||||
status: 'in_progress',
|
||||
output: {
|
||||
title: 'Build & Unit Tests - Checks',
|
||||
summary: 'Running build, unit tests, and lint...'
|
||||
}
|
||||
});
|
||||
|
||||
core.setOutput('check_run_id', checkRun.id);
|
||||
console.log(`Created check run ${checkRun.id} on commit ${{ inputs.ref }}`);
|
||||
|
||||
install-and-build:
|
||||
name: Install & Build
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
env:
|
||||
NODE_OPTIONS: '--max-old-space-size=6144'
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Setup and Build
|
||||
uses: ./.github/actions/setup-nodejs
|
||||
|
||||
- name: Run format check
|
||||
run: pnpm format:check
|
||||
|
||||
- name: Run typecheck
|
||||
run: pnpm typecheck
|
||||
|
||||
unit-tests:
|
||||
name: Unit tests
|
||||
needs: install-and-build
|
||||
uses: ./.github/workflows/test-unit-reusable.yml
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
collectCoverage: true
|
||||
secrets:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
|
||||
lint:
|
||||
name: Lint
|
||||
needs: install-and-build
|
||||
uses: ./.github/workflows/test-linting-reusable.yml
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
post-build-unit-tests:
|
||||
name: Build & Unit Tests - Checks
|
||||
runs-on: ubuntu-latest
|
||||
needs: [create-check-run, install-and-build, unit-tests, lint]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Update check run on PR (if triggered from PR comment)
|
||||
if: inputs.pr_number != ''
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const checkRunId = '${{ needs.create-check-run.outputs.check_run_id }}';
|
||||
|
||||
if (!checkRunId) {
|
||||
console.log('No check run ID found, skipping update');
|
||||
return;
|
||||
}
|
||||
|
||||
const buildResult = '${{ needs.install-and-build.result }}';
|
||||
const testResult = '${{ needs.unit-tests.result }}';
|
||||
const lintResult = '${{ needs.lint.result }}';
|
||||
|
||||
const conclusion = (buildResult === 'success' && testResult === 'success' && lintResult === 'success')
|
||||
? 'success'
|
||||
: 'failure';
|
||||
|
||||
const summary = `
|
||||
**Build**: ${buildResult}
|
||||
**Unit Tests**: ${testResult}
|
||||
**Lint**: ${lintResult}
|
||||
`;
|
||||
|
||||
await github.rest.checks.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
check_run_id: parseInt(checkRunId),
|
||||
status: 'completed',
|
||||
conclusion: conclusion,
|
||||
output: {
|
||||
title: 'Build & Unit Tests - Checks',
|
||||
summary: summary
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`Updated check run ${checkRunId} with conclusion: ${conclusion}`);
|
||||
|
||||
- name: Fail if any job failed
|
||||
if: needs.install-and-build.result == 'failure' || needs.unit-tests.result == 'failure' || needs.lint.result == 'failure'
|
||||
run: exit 1
|
||||
12
.github/workflows/ci-master.yml
vendored
12
.github/workflows/ci-master.yml
vendored
|
|
@ -14,9 +14,9 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Setup and Build
|
||||
|
|
@ -33,11 +33,7 @@ jobs:
|
|||
ref: ${{ github.sha }}
|
||||
nodeVersion: ${{ matrix.node-version }}
|
||||
collectCoverage: ${{ matrix.node-version == '24.13.1' }}
|
||||
secrets:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
secrets: inherit
|
||||
|
||||
lint:
|
||||
name: Lint
|
||||
|
|
|
|||
12
.github/workflows/ci-pull-requests.yml
vendored
12
.github/workflows/ci-pull-requests.yml
vendored
|
|
@ -15,9 +15,9 @@ jobs:
|
|||
env:
|
||||
NODE_OPTIONS: '--max-old-space-size=6144'
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
outputs:
|
||||
ci: ${{ fromJSON(steps.ci-filter.outputs.results).ci == true }}
|
||||
unit: ${{ fromJSON(steps.ci-filter.outputs.results).unit == true }}
|
||||
|
|
@ -98,11 +98,7 @@ jobs:
|
|||
with:
|
||||
ref: ${{ needs.install-and-build.outputs.commit_sha }}
|
||||
collectCoverage: true
|
||||
secrets:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
secrets: inherit
|
||||
|
||||
typecheck:
|
||||
name: Typecheck
|
||||
|
|
|
|||
7
.github/workflows/test-e2e-ci-reusable.yml
vendored
7
.github/workflows/test-e2e-ci-reusable.yml
vendored
|
|
@ -52,10 +52,9 @@ jobs:
|
|||
IMAGE_BASE_NAME: ghcr.io/${{ github.repository }}
|
||||
IMAGE_TAG: ci-${{ github.run_id }}
|
||||
RUNNERS_IMAGE_BASE_NAME: ghcr.io/${{ github.repository_owner }}/runners
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
DOCKER_STATS_WEBHOOK_URL: ${{ secrets.DOCKER_STATS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
|
||||
- name: Get changed files for impact analysis
|
||||
if: ${{ inputs.playwright-only }}
|
||||
|
|
|
|||
|
|
@ -41,9 +41,9 @@ jobs:
|
|||
BUILD_WITH_COVERAGE: 'true'
|
||||
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
|
||||
CURRENTS_PROJECT_ID: 'LRxcNt'
|
||||
QA_PERFORMANCE_METRICS_WEBHOOK_URL: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_URL }}
|
||||
QA_PERFORMANCE_METRICS_WEBHOOK_USER: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_USER }}
|
||||
QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
|
||||
- name: Generate Coverage Report
|
||||
if: always() && steps.coverage-tests.outcome != 'skipped'
|
||||
|
|
|
|||
|
|
@ -12,6 +12,10 @@ on:
|
|||
- 'packages/testing/containers/services/**'
|
||||
- '.github/workflows/test-e2e-infrastructure-reusable.yml'
|
||||
|
||||
concurrency:
|
||||
group: e2e-infrastructure-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
benchmark:
|
||||
name: ${{ matrix.profile }}
|
||||
|
|
|
|||
26
.github/workflows/test-e2e-reusable.yml
vendored
26
.github/workflows/test-e2e-reusable.yml
vendored
|
|
@ -63,25 +63,6 @@ on:
|
|||
default: ''
|
||||
type: string
|
||||
|
||||
secrets:
|
||||
CURRENTS_RECORD_KEY:
|
||||
required: false
|
||||
CURRENTS_API_KEY:
|
||||
required: false
|
||||
QA_PERFORMANCE_METRICS_WEBHOOK_URL:
|
||||
required: false
|
||||
QA_PERFORMANCE_METRICS_WEBHOOK_USER:
|
||||
required: false
|
||||
QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD:
|
||||
required: false
|
||||
N8N_LICENSE_ACTIVATION_KEY:
|
||||
required: false
|
||||
N8N_LICENSE_CERT:
|
||||
required: false
|
||||
N8N_ENCRYPTION_KEY:
|
||||
required: false
|
||||
CONTAINER_TELEMETRY_WEBHOOK:
|
||||
required: false
|
||||
|
||||
env:
|
||||
NODE_OPTIONS: ${{ contains(inputs.runner, '2vcpu') && '--max-old-space-size=6144' || '' }}
|
||||
|
|
@ -162,13 +143,12 @@ jobs:
|
|||
run: ${{ inputs.test-command }} --workers=${{ env.PLAYWRIGHT_WORKERS }} ${{ matrix.specs || format('--shard={0}/{1}', matrix.shard, strategy.job-total) }}
|
||||
env:
|
||||
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
|
||||
QA_PERFORMANCE_METRICS_WEBHOOK_URL: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_URL }}
|
||||
QA_PERFORMANCE_METRICS_WEBHOOK_USER: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_USER }}
|
||||
QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }}
|
||||
N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }}
|
||||
N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
|
||||
CONTAINER_TELEMETRY_WEBHOOK: ${{secrets.CONTAINER_TELEMETRY_WEBHOOK}}
|
||||
|
||||
- name: Upload Failure Artifacts
|
||||
if: ${{ failure() && inputs.upload-failure-artifacts }}
|
||||
|
|
|
|||
38
.github/workflows/test-unit-reusable.yml
vendored
38
.github/workflows/test-unit-reusable.yml
vendored
|
|
@ -17,20 +17,6 @@ on:
|
|||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
secrets:
|
||||
CODECOV_TOKEN:
|
||||
description: 'Codecov upload token.'
|
||||
required: false
|
||||
BUILD_STATS_WEBHOOK_URL:
|
||||
description: 'Webhook URL for test stats.'
|
||||
required: false
|
||||
BUILD_STATS_WEBHOOK_USER:
|
||||
description: 'Basic auth username for test stats webhook.'
|
||||
required: false
|
||||
BUILD_STATS_WEBHOOK_PASSWORD:
|
||||
description: 'Basic auth password for test stats webhook.'
|
||||
required: false
|
||||
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=7168
|
||||
|
||||
|
|
@ -57,9 +43,9 @@ jobs:
|
|||
if: ${{ !cancelled() }}
|
||||
run: node .github/scripts/send-build-stats.mjs || true
|
||||
env:
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
|
||||
- name: Upload test results to Codecov
|
||||
if: ${{ !cancelled() }}
|
||||
|
|
@ -99,9 +85,9 @@ jobs:
|
|||
if: ${{ !cancelled() }}
|
||||
run: node .github/scripts/send-build-stats.mjs || true
|
||||
env:
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
|
||||
- name: Upload test results to Codecov
|
||||
if: ${{ !cancelled() }}
|
||||
|
|
@ -140,9 +126,9 @@ jobs:
|
|||
if: ${{ !cancelled() }}
|
||||
run: node .github/scripts/send-build-stats.mjs || true
|
||||
env:
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
|
||||
- name: Upload test results to Codecov
|
||||
if: ${{ !cancelled() }}
|
||||
|
|
@ -187,9 +173,9 @@ jobs:
|
|||
if: ${{ !cancelled() }}
|
||||
run: node .github/scripts/send-build-stats.mjs || true
|
||||
env:
|
||||
BUILD_STATS_WEBHOOK_URL: ${{ secrets.BUILD_STATS_WEBHOOK_URL }}
|
||||
BUILD_STATS_WEBHOOK_USER: ${{ secrets.BUILD_STATS_WEBHOOK_USER }}
|
||||
BUILD_STATS_WEBHOOK_PASSWORD: ${{ secrets.BUILD_STATS_WEBHOOK_PASSWORD }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
|
||||
- name: Upload test results to Codecov
|
||||
if: ${{ !cancelled() }}
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ Container stack telemetry tracks startup timing, configuration, and runner info.
|
|||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `CONTAINER_TELEMETRY_WEBHOOK` | POST telemetry JSON to this URL |
|
||||
| `QA_METRICS_WEBHOOK_URL` | POST telemetry JSON to this URL |
|
||||
| `CONTAINER_TELEMETRY_VERBOSE` | Set to `1` for JSON output to console |
|
||||
|
||||
### What's Collected
|
||||
|
|
@ -419,7 +419,7 @@ Container stack telemetry tracks startup timing, configuration, and runner info.
|
|||
CONTAINER_TELEMETRY_VERBOSE=1 pnpm stack
|
||||
|
||||
# Send to webhook (CI)
|
||||
CONTAINER_TELEMETRY_WEBHOOK=https://n8n.example.com/webhook/telemetry
|
||||
QA_METRICS_WEBHOOK_URL=https://n8n.example.com/webhook/telemetry
|
||||
```
|
||||
|
||||
## Cleanup
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ export class TelemetryRecorder {
|
|||
* Flush telemetry - outputs based on environment configuration
|
||||
*
|
||||
* Output modes:
|
||||
* - CONTAINER_TELEMETRY_WEBHOOK: Send via detached process (non-blocking, survives parent exit)
|
||||
* - QA_METRICS_WEBHOOK_URL: Send via detached process (non-blocking, survives parent exit)
|
||||
* - CONTAINER_TELEMETRY_VERBOSE=1: Full breakdown + elapsed logs (via utils.ts)
|
||||
* - Default: One-liner summary only
|
||||
*/
|
||||
|
|
@ -180,7 +180,7 @@ export class TelemetryRecorder {
|
|||
const record = this.buildRecord(success, errorMessage);
|
||||
|
||||
const isVerbose = process.env.CONTAINER_TELEMETRY_VERBOSE === '1';
|
||||
const webhookUrl = process.env.CONTAINER_TELEMETRY_WEBHOOK;
|
||||
const webhookUrl = process.env.QA_METRICS_WEBHOOK_URL;
|
||||
|
||||
if (webhookUrl) {
|
||||
this.sendToWebhook(record, webhookUrl);
|
||||
|
|
@ -206,24 +206,102 @@ export class TelemetryRecorder {
|
|||
}
|
||||
}
|
||||
|
||||
private buildUnifiedPayload(record: StackTelemetryRecord): object {
|
||||
const runId = record.ci.runId ?? null;
|
||||
const repo = process.env.GITHUB_REPOSITORY ?? null;
|
||||
|
||||
const metrics: Array<{
|
||||
metric_name: string;
|
||||
value: number;
|
||||
unit: string;
|
||||
dimensions: Record<string, string | number>;
|
||||
}> = [
|
||||
{
|
||||
metric_name: 'stack-startup-total',
|
||||
value: record.timing.total,
|
||||
unit: 'ms',
|
||||
dimensions: {
|
||||
stack_type: record.stack.type,
|
||||
mains: record.stack.mains,
|
||||
workers: record.stack.workers,
|
||||
postgres: record.stack.postgres ? 'true' : 'false',
|
||||
success: record.success ? 'true' : 'false',
|
||||
},
|
||||
},
|
||||
{
|
||||
metric_name: 'stack-startup-network',
|
||||
value: record.timing.network,
|
||||
unit: 'ms',
|
||||
dimensions: { stack_type: record.stack.type },
|
||||
},
|
||||
{
|
||||
metric_name: 'stack-startup-n8n',
|
||||
value: record.timing.n8nStartup,
|
||||
unit: 'ms',
|
||||
dimensions: { stack_type: record.stack.type },
|
||||
},
|
||||
...Object.entries(record.timing.services).map(([service, duration]) => ({
|
||||
metric_name: 'stack-startup-service',
|
||||
value: duration,
|
||||
unit: 'ms',
|
||||
dimensions: { service, stack_type: record.stack.type },
|
||||
})),
|
||||
{
|
||||
metric_name: 'container-count',
|
||||
value: record.containers.total,
|
||||
unit: 'count',
|
||||
dimensions: { stack_type: record.stack.type },
|
||||
},
|
||||
];
|
||||
|
||||
return {
|
||||
timestamp: record.timestamp,
|
||||
benchmark_name: 'container-telemetry',
|
||||
git: {
|
||||
sha: record.git.sha,
|
||||
branch: record.git.branch,
|
||||
pr: record.git.pr ?? null,
|
||||
},
|
||||
ci: {
|
||||
runId,
|
||||
runUrl: runId && repo ? `https://github.com/${repo}/actions/runs/${runId}` : null,
|
||||
job: record.ci.job ?? null,
|
||||
workflow: record.ci.workflow ?? null,
|
||||
attempt: record.ci.attempt ?? null,
|
||||
},
|
||||
runner: record.runner,
|
||||
metrics,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Send telemetry via a detached child process.
|
||||
* The process runs independently and survives parent exit, ensuring delivery
|
||||
* even when the main process throws/exits immediately after flush().
|
||||
*/
|
||||
private sendToWebhook(record: StackTelemetryRecord, webhookUrl: string): void {
|
||||
const payload = JSON.stringify(record);
|
||||
const webhookUser = process.env.QA_METRICS_WEBHOOK_USER;
|
||||
const webhookPassword = process.env.QA_METRICS_WEBHOOK_PASSWORD;
|
||||
|
||||
if (!webhookUser || !webhookPassword) {
|
||||
console.log('QA_METRICS_WEBHOOK_USER/PASSWORD not set, skipping telemetry.');
|
||||
return;
|
||||
}
|
||||
|
||||
const payload = JSON.stringify(this.buildUnifiedPayload(record));
|
||||
const authHeader = `Basic ${Buffer.from(`${webhookUser}:${webhookPassword}`).toString('base64')}`;
|
||||
const script = `
|
||||
fetch(process.argv[1], {
|
||||
fetch(process.argv[2], {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: process.argv[2]
|
||||
headers: { 'Content-Type': 'application/json', 'Authorization': process.env.WEBHOOK_AUTH },
|
||||
body: process.argv[3]
|
||||
}).catch(() => process.exit(1));
|
||||
`;
|
||||
|
||||
const child = spawn(process.execPath, ['-e', script, webhookUrl, payload], {
|
||||
detached: true,
|
||||
stdio: 'ignore',
|
||||
env: { ...process.env, WEBHOOK_AUTH: authHeader },
|
||||
});
|
||||
child.unref();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,66 +1,87 @@
|
|||
# Metrics Reporter Usage
|
||||
# Metrics Reporter
|
||||
|
||||
Automatically collect performance metrics from Playwright tests and send them to a Webhook.
|
||||
Collects `attachMetric()` calls from Playwright tests and sends them as a single batched
|
||||
payload to the unified QA metrics webhook at the end of the run.
|
||||
|
||||
See `.github/CI-TELEMETRY.md` for the full payload shape, BigQuery schema, and adding
|
||||
new telemetry from scripts.
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
export QA_PERFORMANCE_METRICS_WEBHOOK_URL=https://your-webhook-endpoint.com/metrics
|
||||
export QA_PERFORMANCE_METRICS_WEBHOOK_USER=username
|
||||
export QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD=password
|
||||
export QA_METRICS_WEBHOOK_URL=https://your-webhook-endpoint.com/metrics
|
||||
export QA_METRICS_WEBHOOK_USER=username
|
||||
export QA_METRICS_WEBHOOK_PASSWORD=password
|
||||
```
|
||||
|
||||
## Attach Metrics in Tests
|
||||
## Attaching Metrics
|
||||
|
||||
**Option 1: Helper function (recommended)**
|
||||
```javascript
|
||||
```typescript
|
||||
import { attachMetric } from '../../utils/performance-helper';
|
||||
|
||||
await attachMetric(testInfo, 'memory-usage', 1234567, 'bytes');
|
||||
```
|
||||
// Simple metric
|
||||
await attachMetric(testInfo, 'memory-rss', 368.04, 'MB');
|
||||
|
||||
**Option 2: Direct attach**
|
||||
```javascript
|
||||
await testInfo.attach('metric:memory-usage', {
|
||||
body: JSON.stringify({ value: 1234567, unit: 'bytes' })
|
||||
// With dimensions — separates the measurement from the scenario config
|
||||
await attachMetric(testInfo, 'exec-per-sec', 245.3, 'exec/s', {
|
||||
trigger: 'kafka',
|
||||
nodes: 10,
|
||||
output: '100KB',
|
||||
});
|
||||
```
|
||||
|
||||
## What Gets Sent to BigQuery
|
||||
`metric_name` is the measurement type. Config goes in `dimensions`, never baked into the name.
|
||||
|
||||
```json
|
||||
{
|
||||
"test_name": "My performance test",
|
||||
"metric_name": "memory-usage",
|
||||
"metric_value": 1234567,
|
||||
"metric_unit": "bytes",
|
||||
"git_commit": "abc123...",
|
||||
"git_branch": "main",
|
||||
"timestamp": "2025-08-29T..."
|
||||
}
|
||||
## Benchmark Harnesses
|
||||
|
||||
The harnesses derive dimensions automatically — prefer these over calling `attachMetric()` directly:
|
||||
|
||||
```typescript
|
||||
await runThroughputTest({ handle, api, services, testInfo,
|
||||
trigger: 'kafka', nodeCount: 10, nodeOutputSize: '100KB',
|
||||
messageCount: 5000, timeoutMs: 600_000,
|
||||
});
|
||||
|
||||
await runWebhookThroughputTest({ handle, api, services, testInfo, baseUrl,
|
||||
nodeCount: 10, nodeOutputSize: '100KB',
|
||||
connections: 50, durationSeconds: 30, timeoutMs: 120_000,
|
||||
});
|
||||
```
|
||||
|
||||
## Data Pipeline
|
||||
## Example Queries
|
||||
|
||||
**Playwright Test** → **n8n Webhook** → **BigQuery Table**
|
||||
```sql
|
||||
-- Throughput trend by node count
|
||||
SELECT INT64(JSON_VALUE(dimensions, '$.nodes')) AS nodes, AVG(value) AS avg_exec_per_sec
|
||||
FROM qa_performance_metrics
|
||||
WHERE metric_name = 'exec-per-sec'
|
||||
AND JSON_VALUE(dimensions, '$.trigger') = 'kafka'
|
||||
AND git_branch = 'master'
|
||||
GROUP BY nodes ORDER BY nodes;
|
||||
|
||||
The n8n workflow that processes the metrics is here:
|
||||
https://internal.users.n8n.cloud/workflow/zSRjEwfBfCNjGXK8
|
||||
|
||||
## BigQuery Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"fields": [
|
||||
{"name": "test_name", "type": "STRING", "mode": "REQUIRED"},
|
||||
{"name": "metric_name", "type": "STRING", "mode": "REQUIRED"},
|
||||
{"name": "metric_value", "type": "FLOAT", "mode": "REQUIRED"},
|
||||
{"name": "metric_unit", "type": "STRING", "mode": "REQUIRED"},
|
||||
{"name": "git_commit", "type": "STRING", "mode": "REQUIRED"},
|
||||
{"name": "git_branch", "type": "STRING", "mode": "REQUIRED"},
|
||||
{"name": "timestamp", "type": "TIMESTAMP", "mode": "REQUIRED"}
|
||||
]
|
||||
}
|
||||
-- PR vs 30-day master baseline
|
||||
-- Note: dimensions is a JSON column; use JSON_VALUE() to extract fields for grouping/joining
|
||||
WITH pr AS (
|
||||
SELECT metric_name,
|
||||
JSON_VALUE(dimensions, '$.trigger') AS trigger,
|
||||
JSON_VALUE(dimensions, '$.mode') AS mode,
|
||||
AVG(value) AS value
|
||||
FROM qa_performance_metrics WHERE git_pr = @pr_number
|
||||
GROUP BY metric_name, trigger, mode
|
||||
),
|
||||
baseline AS (
|
||||
SELECT metric_name,
|
||||
JSON_VALUE(dimensions, '$.trigger') AS trigger,
|
||||
JSON_VALUE(dimensions, '$.mode') AS mode,
|
||||
AVG(value) AS value
|
||||
FROM qa_performance_metrics
|
||||
WHERE git_branch = 'master'
|
||||
AND timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY)
|
||||
GROUP BY metric_name, trigger, mode
|
||||
)
|
||||
SELECT pr.metric_name, pr.trigger, pr.mode,
|
||||
pr.value AS pr_value, baseline.value AS baseline_value,
|
||||
ROUND((pr.value - baseline.value) / baseline.value * 100, 1) AS delta_pct
|
||||
FROM pr JOIN baseline USING (metric_name, trigger, mode)
|
||||
ORDER BY ABS(delta_pct) DESC;
|
||||
```
|
||||
|
||||
That's it! Metrics are automatically collected and sent when you attach them to tests.
|
||||
|
|
|
|||
|
|
@ -76,11 +76,8 @@ function extractSuite(filePath: string): string {
|
|||
return 'other';
|
||||
}
|
||||
|
||||
function extractMetricSuffix(metricName: string, scenario: string): string | null {
|
||||
if (metricName.startsWith(`${scenario}-`)) {
|
||||
return metricName.slice(scenario.length + 1);
|
||||
}
|
||||
return null;
|
||||
function extractMetricSuffix(metricName: string): string {
|
||||
return metricName;
|
||||
}
|
||||
|
||||
class BenchmarkSummaryReporter implements Reporter {
|
||||
|
|
@ -98,14 +95,12 @@ class BenchmarkSummaryReporter implements Reporter {
|
|||
|
||||
for (const attachment of metricAttachments) {
|
||||
const fullName = attachment.name.replace('metric:', '');
|
||||
const suffix = extractMetricSuffix(fullName, scenario);
|
||||
if (suffix) {
|
||||
try {
|
||||
const data = JSON.parse(attachment.body?.toString() ?? '');
|
||||
metrics.set(suffix, data.value);
|
||||
} catch (error) {
|
||||
console.warn(`[BenchmarkReporter] Malformed metric attachment "${fullName}":`, error);
|
||||
}
|
||||
const suffix = extractMetricSuffix(fullName);
|
||||
try {
|
||||
const data = JSON.parse(attachment.body?.toString() ?? '');
|
||||
metrics.set(suffix, data.value);
|
||||
} catch (error) {
|
||||
console.warn(`[BenchmarkReporter] Malformed metric attachment "${fullName}":`, error);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,20 @@
|
|||
import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter';
|
||||
import { strict as assert } from 'assert';
|
||||
import { execSync } from 'child_process';
|
||||
import { execSync } from 'node:child_process';
|
||||
import * as os from 'node:os';
|
||||
import { z } from 'zod';
|
||||
|
||||
const metricDataSchema = z.object({
|
||||
value: z.number(),
|
||||
unit: z.string().optional(),
|
||||
dimensions: z.record(z.union([z.string(), z.number()])).optional(),
|
||||
});
|
||||
|
||||
interface Metric {
|
||||
name: string;
|
||||
benchmark_name: string;
|
||||
metric_name: string;
|
||||
value: number;
|
||||
unit: string | null;
|
||||
dimensions: Record<string, string | number> | null;
|
||||
}
|
||||
|
||||
interface ReporterOptions {
|
||||
|
|
@ -22,123 +25,157 @@ interface ReporterOptions {
|
|||
|
||||
/**
|
||||
* Automatically collect performance metrics from Playwright tests and send them to a Webhook.
|
||||
* If your test contains a testInfo.attach() call with a name starting with 'metric:', the metric will be collected and sent to the Webhook.
|
||||
* If your test contains a testInfo.attach() call with a name starting with 'metric:', the metric
|
||||
* will be collected and sent as a single batched payload at the end of the run.
|
||||
*
|
||||
* See utils/performance-helper.ts for the attachMetric() helper.
|
||||
*/
|
||||
class MetricsReporter implements Reporter {
|
||||
private webhookUrl: string | undefined;
|
||||
private webhookUser: string | undefined;
|
||||
private webhookPassword: string | undefined;
|
||||
private pendingRequests: Array<Promise<void>> = [];
|
||||
private collectedMetrics: Metric[] = [];
|
||||
|
||||
constructor(options: ReporterOptions = {}) {
|
||||
this.webhookUrl = options.webhookUrl ?? process.env.QA_PERFORMANCE_METRICS_WEBHOOK_URL;
|
||||
this.webhookUser = options.webhookUser ?? process.env.QA_PERFORMANCE_METRICS_WEBHOOK_USER;
|
||||
this.webhookPassword =
|
||||
options.webhookPassword ?? process.env.QA_PERFORMANCE_METRICS_WEBHOOK_PASSWORD;
|
||||
this.webhookUrl = options.webhookUrl ?? process.env.QA_METRICS_WEBHOOK_URL;
|
||||
this.webhookUser = options.webhookUser ?? process.env.QA_METRICS_WEBHOOK_USER;
|
||||
this.webhookPassword = options.webhookPassword ?? process.env.QA_METRICS_WEBHOOK_PASSWORD;
|
||||
}
|
||||
|
||||
async onTestEnd(test: TestCase, result: TestResult): Promise<void> {
|
||||
if (
|
||||
!this.webhookUrl ||
|
||||
!this.webhookUser ||
|
||||
!this.webhookPassword ||
|
||||
result.status === 'skipped'
|
||||
) {
|
||||
onTestEnd(test: TestCase, result: TestResult): void {
|
||||
if (result.status === 'skipped') return;
|
||||
const metrics = this.collectMetrics(test, result);
|
||||
this.collectedMetrics.push(...metrics);
|
||||
}
|
||||
|
||||
async onEnd(): Promise<void> {
|
||||
const webhookUrl = this.webhookUrl;
|
||||
if (!webhookUrl || this.collectedMetrics.length === 0) return;
|
||||
if (!this.webhookUser || !this.webhookPassword) {
|
||||
console.log('[MetricsReporter] QA_METRICS_WEBHOOK_USER/PASSWORD not set, skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
const metrics = this.collectMetrics(result);
|
||||
if (metrics.length > 0) {
|
||||
const sendPromise = this.sendMetrics(test, metrics);
|
||||
this.pendingRequests.push(sendPromise);
|
||||
await sendPromise;
|
||||
// Group by benchmark_name so each POST has a single top-level benchmark_name,
|
||||
// consistent with how script-based sources (build-stats, docker-stats, etc.) send.
|
||||
const byBenchmark = new Map<string, Metric[]>();
|
||||
for (const m of this.collectedMetrics) {
|
||||
const group = byBenchmark.get(m.benchmark_name) ?? [];
|
||||
group.push(m);
|
||||
byBenchmark.set(m.benchmark_name, group);
|
||||
}
|
||||
}
|
||||
|
||||
private collectMetrics(result: TestResult): Metric[] {
|
||||
const metrics: Metric[] = [];
|
||||
const auth = Buffer.from(`${this.webhookUser}:${this.webhookPassword}`).toString('base64');
|
||||
const context = this.getContext();
|
||||
|
||||
result.attachments.forEach((attachment) => {
|
||||
if (attachment.name.startsWith('metric:')) {
|
||||
const metricName = attachment.name.replace('metric:', '');
|
||||
const counts = await Promise.all(
|
||||
Array.from(byBenchmark, async ([benchmarkName, metrics]) => {
|
||||
const payload = { ...context, benchmark_name: benchmarkName, metrics };
|
||||
try {
|
||||
const parsedData = JSON.parse(attachment.body?.toString() ?? '');
|
||||
const data = metricDataSchema.parse(parsedData);
|
||||
metrics.push({
|
||||
name: metricName,
|
||||
value: data.value,
|
||||
unit: data.unit ?? null,
|
||||
const response = await fetch(webhookUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Basic ${auth}`,
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.warn(
|
||||
`[MetricsReporter] Webhook failed (${response.status}) for "${benchmarkName}": ${metrics.length} metrics dropped`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
return metrics.length;
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`[MetricsReporter] Failed to parse metric ${metricName}: ${(e as Error).message}`,
|
||||
`[MetricsReporter] Failed to send metrics for "${benchmarkName}": ${(e as Error).message}`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
const sent = counts.reduce((total, n) => total + n, 0);
|
||||
console.log(
|
||||
`[MetricsReporter] Sent ${sent}/${this.collectedMetrics.length} metrics across ${byBenchmark.size} tests`,
|
||||
);
|
||||
}
|
||||
|
||||
private collectMetrics(test: TestCase, result: TestResult): Metric[] {
|
||||
const metrics: Metric[] = [];
|
||||
|
||||
for (const attachment of result.attachments) {
|
||||
if (!attachment.name.startsWith('metric:')) continue;
|
||||
const metricName = attachment.name.slice('metric:'.length);
|
||||
try {
|
||||
const parsed = metricDataSchema.parse(JSON.parse(attachment.body?.toString() ?? ''));
|
||||
metrics.push({
|
||||
benchmark_name: test.title,
|
||||
metric_name: metricName,
|
||||
value: parsed.value,
|
||||
unit: parsed.unit ?? null,
|
||||
dimensions: parsed.dimensions ?? null,
|
||||
});
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`[MetricsReporter] Failed to parse metric ${metricName}: ${(e as Error).message}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private async sendMetrics(test: TestCase, metrics: Metric[]): Promise<void> {
|
||||
const gitInfo = this.getGitInfo();
|
||||
private getContext() {
|
||||
const ref = process.env.GITHUB_REF ?? '';
|
||||
const prMatch = ref.match(/refs\/pull\/(\d+)/);
|
||||
const runId = process.env.GITHUB_RUN_ID ?? null;
|
||||
|
||||
assert(gitInfo.commit, 'Git commit must be defined');
|
||||
assert(gitInfo.branch, 'Git branch must be defined');
|
||||
assert(gitInfo.author, 'Git author must be defined');
|
||||
|
||||
const payload = {
|
||||
test_name: test.title,
|
||||
git_commit: gitInfo.commit,
|
||||
git_branch: gitInfo.branch,
|
||||
git_author: gitInfo.author,
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
metrics: metrics.map((metric) => ({
|
||||
metric_name: metric.name,
|
||||
metric_value: metric.value,
|
||||
metric_unit: metric.unit,
|
||||
})),
|
||||
git: {
|
||||
sha: (process.env.GITHUB_SHA ?? this.gitFallback('rev-parse HEAD'))?.slice(0, 8) ?? null,
|
||||
branch:
|
||||
process.env.GITHUB_HEAD_REF ??
|
||||
process.env.GITHUB_REF_NAME ??
|
||||
this.gitFallback('rev-parse --abbrev-ref HEAD'),
|
||||
pr: prMatch ? parseInt(prMatch[1], 10) : null,
|
||||
},
|
||||
ci: {
|
||||
runId,
|
||||
runUrl:
|
||||
runId && process.env.GITHUB_REPOSITORY
|
||||
? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId}`
|
||||
: null,
|
||||
workflow: process.env.GITHUB_WORKFLOW ?? null,
|
||||
job: process.env.GITHUB_JOB ?? null,
|
||||
attempt: process.env.GITHUB_RUN_ATTEMPT
|
||||
? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10)
|
||||
: null,
|
||||
},
|
||||
runner: {
|
||||
provider: !process.env.CI
|
||||
? 'local'
|
||||
: process.env.RUNNER_ENVIRONMENT === 'github-hosted'
|
||||
? 'github'
|
||||
: 'blacksmith',
|
||||
cpuCores: os.cpus().length,
|
||||
memoryGb: Math.round((os.totalmem() / 1024 ** 3) * 10) / 10,
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
const auth = Buffer.from(`${this.webhookUser}:${this.webhookPassword}`).toString('base64');
|
||||
|
||||
const response = await fetch(this.webhookUrl!, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Basic ${auth}`,
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.warn(`[MetricsReporter] Webhook failed (${response.status}): ${test.title}`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`[MetricsReporter] Failed to send metrics for test ${test.title}: ${(e as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async onEnd(): Promise<void> {
|
||||
if (this.pendingRequests.length > 0) {
|
||||
await Promise.allSettled(this.pendingRequests);
|
||||
}
|
||||
}
|
||||
|
||||
private getGitInfo(): { commit: string | null; branch: string | null; author: string | null } {
|
||||
private gitFallback(command: string): string | null {
|
||||
try {
|
||||
return {
|
||||
commit: execSync('git rev-parse HEAD', { encoding: 'utf8' }).trim(),
|
||||
branch: execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf8' }).trim(),
|
||||
author: execSync('git log -1 --pretty=format:"%an"', { encoding: 'utf8' }).trim(),
|
||||
};
|
||||
} catch (e) {
|
||||
console.error(`[MetricsReporter] Failed to get Git info: ${(e as Error).message}`);
|
||||
return { commit: null, branch: null, author: null };
|
||||
return execSync(`git ${command}`, {
|
||||
encoding: 'utf8',
|
||||
stdio: ['pipe', 'pipe', 'ignore'],
|
||||
}).trim();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,11 +14,13 @@ import {
|
|||
formatDiagnosticValue,
|
||||
resolveMetricQuery,
|
||||
} from '../../../../utils/benchmark';
|
||||
import type { TriggerHandle, ExecutionMetrics } from '../../../../utils/benchmark';
|
||||
|
||||
export type LoadProfile =
|
||||
| { type: 'steady'; ratePerSecond: number; durationSeconds: number }
|
||||
| { type: 'preloaded'; count: number };
|
||||
import type {
|
||||
TriggerHandle,
|
||||
NodeOutputSize,
|
||||
ExecutionMetrics,
|
||||
TriggerType,
|
||||
LoadProfile,
|
||||
} from '../../../../utils/benchmark';
|
||||
|
||||
export interface LoadTestOptions {
|
||||
handle: TriggerHandle;
|
||||
|
|
@ -27,6 +29,12 @@ export interface LoadTestOptions {
|
|||
testInfo: TestInfo;
|
||||
load: LoadProfile;
|
||||
timeoutMs: number;
|
||||
/** Trigger type recorded as a dimension in BigQuery */
|
||||
trigger: TriggerType;
|
||||
/** Node count recorded as a dimension in BigQuery */
|
||||
nodeCount: number;
|
||||
/** Node output size recorded as a dimension in BigQuery */
|
||||
nodeOutputSize?: NodeOutputSize;
|
||||
/** PromQL metric to track workflow completions. Defaults to resolveMetricQuery(testInfo). */
|
||||
metricQuery?: string;
|
||||
}
|
||||
|
|
@ -40,10 +48,22 @@ export interface LoadTestOptions {
|
|||
* (direct mode: `n8n_workflow_success_total`, queue mode: `n8n_scaling_mode_queue_jobs_completed`).
|
||||
*/
|
||||
export async function runLoadTest(options: LoadTestOptions): Promise<ExecutionMetrics> {
|
||||
const { handle, api, services, testInfo, load, timeoutMs } = options;
|
||||
const { handle, api, services, testInfo, load, timeoutMs, trigger } = options;
|
||||
const metricQuery = options.metricQuery ?? resolveMetricQuery(testInfo);
|
||||
testInfo.setTimeout(timeoutMs + 120_000);
|
||||
|
||||
const mode = testInfo.project.name.replace(':infrastructure', '').replace('benchmark-', '');
|
||||
|
||||
const dimensions: Record<string, string | number> = { trigger, mode };
|
||||
if (options.nodeCount !== undefined) dimensions.nodes = options.nodeCount;
|
||||
if (options.nodeOutputSize !== undefined) dimensions.output = options.nodeOutputSize;
|
||||
if (load.type === 'steady') {
|
||||
dimensions.rate = load.ratePerSecond;
|
||||
dimensions.duration_s = load.durationSeconds;
|
||||
} else {
|
||||
dimensions.messages = load.count;
|
||||
}
|
||||
|
||||
const obs = services.observability;
|
||||
|
||||
const { workflowId, createdWorkflow } = await api.workflows.createWorkflowFromDefinition(
|
||||
|
|
@ -117,11 +137,11 @@ export async function runLoadTest(options: LoadTestOptions): Promise<ExecutionMe
|
|||
const durations = await sampleExecutionDurations(api.workflows, workflowId);
|
||||
const metrics = buildMetrics(throughputResult.totalCompleted, 0, totalDurationMs, durations);
|
||||
|
||||
await attachLoadTestResults(testInfo, testInfo.title, metrics);
|
||||
await attachLoadTestResults(testInfo, dimensions, metrics);
|
||||
|
||||
// Diagnostics
|
||||
const diagnostics = await collectDiagnostics(obs.metrics, totalDurationMs);
|
||||
await attachDiagnostics(testInfo, testInfo.title, diagnostics);
|
||||
await attachDiagnostics(testInfo, dimensions, diagnostics);
|
||||
const fmt = formatDiagnosticValue;
|
||||
console.log(
|
||||
`[DIAG] ${testInfo.title}\n` +
|
||||
|
|
|
|||
|
|
@ -4,17 +4,17 @@ import type { ServiceHelpers } from 'n8n-containers/services/types';
|
|||
|
||||
import type { ApiHelpers } from '../../../../services/api-helper';
|
||||
import {
|
||||
waitForThroughput,
|
||||
getBaselineCounter,
|
||||
attachThroughputResults,
|
||||
sampleExecutionDurations,
|
||||
buildMetrics,
|
||||
attachThroughputResults,
|
||||
waitForThroughput,
|
||||
getBaselineCounter,
|
||||
collectDiagnostics,
|
||||
attachDiagnostics,
|
||||
formatDiagnosticValue,
|
||||
resolveMetricQuery,
|
||||
} from '../../../../utils/benchmark';
|
||||
import type { TriggerHandle, NodeOutputSize } from '../../../../utils/benchmark';
|
||||
import type { TriggerHandle, NodeOutputSize, TriggerType } from '../../../../utils/benchmark';
|
||||
import { attachMetric } from '../../../../utils/performance-helper';
|
||||
|
||||
export interface ThroughputTestOptions {
|
||||
|
|
@ -26,36 +26,16 @@ export interface ThroughputTestOptions {
|
|||
nodeCount: number;
|
||||
nodeOutputSize: NodeOutputSize;
|
||||
timeoutMs: number;
|
||||
pollIntervalMs?: number;
|
||||
/** Trigger type recorded as a dimension in BigQuery */
|
||||
trigger: TriggerType;
|
||||
/** PromQL metric to track workflow completions. Defaults to resolveMetricQuery(testInfo). */
|
||||
metricQuery?: string;
|
||||
plan?: { memory: number; cpu: number };
|
||||
workerPlan?: { memory: number; cpu: number };
|
||||
}
|
||||
|
||||
function deriveProfile(
|
||||
testInfo: TestInfo,
|
||||
plan?: { memory: number; cpu: number },
|
||||
workerPlan?: { memory: number; cpu: number },
|
||||
) {
|
||||
const name = testInfo.project.name.replace(':infrastructure', '').replace('benchmark-', '');
|
||||
const workers =
|
||||
(testInfo.project.use as { containerConfig?: { workers?: number } }).containerConfig?.workers ??
|
||||
0;
|
||||
|
||||
const wp = workerPlan ?? plan;
|
||||
let resourceSummary = '';
|
||||
if (plan && wp) {
|
||||
resourceSummary =
|
||||
workers > 0
|
||||
? ` Mode: queue (1 main + ${workers} workers)\n` +
|
||||
` Main: ${plan.memory}GB RAM, ${plan.cpu} CPU\n` +
|
||||
` Workers: ${wp.memory}GB RAM, ${wp.cpu} CPU each\n` +
|
||||
` Total: ${(plan.memory + wp.memory * workers).toFixed(1)}GB RAM, ${plan.cpu + wp.cpu * workers} CPU`
|
||||
: ` Resources: ${plan.memory}GB RAM, ${plan.cpu} CPU`;
|
||||
}
|
||||
|
||||
return { name, workers, resourceSummary };
|
||||
function deriveMode(testInfo: TestInfo): string {
|
||||
return testInfo.project.name.replace(':infrastructure', '').replace('benchmark-', '');
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -74,7 +54,7 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise
|
|||
nodeCount,
|
||||
nodeOutputSize,
|
||||
timeoutMs,
|
||||
pollIntervalMs,
|
||||
trigger,
|
||||
plan,
|
||||
workerPlan,
|
||||
} = options;
|
||||
|
|
@ -82,7 +62,31 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise
|
|||
|
||||
testInfo.setTimeout(timeoutMs + 120_000);
|
||||
|
||||
const profile = deriveProfile(testInfo, plan, workerPlan);
|
||||
const mode = deriveMode(testInfo);
|
||||
const workers =
|
||||
(testInfo.project.use as { containerConfig?: { workers?: number } }).containerConfig?.workers ??
|
||||
0;
|
||||
|
||||
const dimensions = {
|
||||
trigger,
|
||||
nodes: nodeCount,
|
||||
output: nodeOutputSize,
|
||||
messages: messageCount,
|
||||
mode,
|
||||
};
|
||||
|
||||
let resourceSummary = '';
|
||||
const wp = workerPlan ?? plan;
|
||||
if (plan && wp) {
|
||||
resourceSummary =
|
||||
workers > 0
|
||||
? ` Mode: queue (1 main + ${workers} workers)\n` +
|
||||
` Main: ${plan.memory}GB RAM, ${plan.cpu} CPU\n` +
|
||||
` Workers: ${wp.memory}GB RAM, ${wp.cpu} CPU each\n` +
|
||||
` Total: ${(plan.memory + wp.memory * workers).toFixed(1)}GB RAM, ${plan.cpu + wp.cpu * workers} CPU`
|
||||
: ` Resources: ${plan.memory}GB RAM, ${plan.cpu} CPU`;
|
||||
}
|
||||
|
||||
const obs = services.observability;
|
||||
|
||||
const { workflowId, createdWorkflow } = await api.workflows.createWorkflowFromDefinition(
|
||||
|
|
@ -93,7 +97,7 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise
|
|||
// Preload queue
|
||||
const publishResult = await handle.preload(messageCount);
|
||||
console.log(
|
||||
`[BENCH-${profile.name}] Preloaded ${publishResult.totalPublished} messages in ${publishResult.publishDurationMs}ms`,
|
||||
`[BENCH-${mode}] Preloaded ${publishResult.totalPublished} messages in ${publishResult.publishDurationMs}ms`,
|
||||
);
|
||||
|
||||
// Wait for VictoriaMetrics, then record baseline
|
||||
|
|
@ -110,7 +114,7 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise
|
|||
|
||||
// Measure throughput
|
||||
console.log(
|
||||
`[BENCH-${profile.name}] Draining ${messageCount} messages through ${nodeCount}-node (${nodeOutputSize}) workflow (timeout: ${timeoutMs}ms)`,
|
||||
`[BENCH-${mode}] Draining ${messageCount} messages through ${nodeCount}-node (${nodeOutputSize}) workflow (timeout: ${timeoutMs}ms)`,
|
||||
);
|
||||
const result = await waitForThroughput(obs.metrics, {
|
||||
expectedCount: messageCount,
|
||||
|
|
@ -118,49 +122,28 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise
|
|||
timeoutMs,
|
||||
baselineValue: baselineCounter,
|
||||
metricQuery,
|
||||
pollIntervalMs,
|
||||
});
|
||||
|
||||
// Attach results
|
||||
await attachThroughputResults(testInfo, testInfo.title, result);
|
||||
// Attach throughput results
|
||||
await attachThroughputResults(testInfo, dimensions, result);
|
||||
|
||||
// Execution duration sampling — provides p50/p95/p99 latency percentiles.
|
||||
// May be empty when EXECUTIONS_DATA_SAVE_ON_SUCCESS=none.
|
||||
const durations = await sampleExecutionDurations(api.workflows, workflowId);
|
||||
if (durations.length > 0) {
|
||||
const durationMetrics = buildMetrics(result.totalCompleted, 0, result.durationMs, durations);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${testInfo.title}-duration-avg`,
|
||||
durationMetrics.avgDurationMs,
|
||||
'ms',
|
||||
);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${testInfo.title}-duration-p50`,
|
||||
durationMetrics.p50DurationMs,
|
||||
'ms',
|
||||
);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${testInfo.title}-duration-p95`,
|
||||
durationMetrics.p95DurationMs,
|
||||
'ms',
|
||||
);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${testInfo.title}-duration-p99`,
|
||||
durationMetrics.p99DurationMs,
|
||||
'ms',
|
||||
);
|
||||
await attachMetric(testInfo, 'duration-avg', durationMetrics.avgDurationMs, 'ms', dimensions);
|
||||
await attachMetric(testInfo, 'duration-p50', durationMetrics.p50DurationMs, 'ms', dimensions);
|
||||
await attachMetric(testInfo, 'duration-p95', durationMetrics.p95DurationMs, 'ms', dimensions);
|
||||
await attachMetric(testInfo, 'duration-p99', durationMetrics.p99DurationMs, 'ms', dimensions);
|
||||
}
|
||||
|
||||
// Diagnostics
|
||||
const diagnostics = await collectDiagnostics(obs.metrics, result.durationMs);
|
||||
await attachDiagnostics(testInfo, testInfo.title, diagnostics);
|
||||
await attachDiagnostics(testInfo, dimensions, diagnostics);
|
||||
const fmt = formatDiagnosticValue;
|
||||
console.log(
|
||||
`[DIAG-${profile.name}] ${testInfo.title}\n` +
|
||||
`[DIAG-${mode}] ${testInfo.title}\n` +
|
||||
` Event Loop Lag: ${fmt(diagnostics.eventLoopLag, 's')}\n` +
|
||||
` PG Transactions/s: ${fmt(diagnostics.pgTxRate, ' tx/s')}\n` +
|
||||
` PG Rows Inserted/s: ${fmt(diagnostics.pgInsertRate, ' rows/s')}\n` +
|
||||
|
|
@ -173,9 +156,9 @@ export async function runThroughputTest(options: ThroughputTestOptions): Promise
|
|||
|
||||
// Summary
|
||||
console.log(
|
||||
`[BENCH-${profile.name} RESULT] ${testInfo.title}\n` +
|
||||
` Profile: ${profile.name}\n` +
|
||||
`${profile.resourceSummary}\n` +
|
||||
`[BENCH-${mode} RESULT] ${testInfo.title}\n` +
|
||||
` Profile: ${mode}\n` +
|
||||
`${resourceSummary}\n` +
|
||||
` Nodes: ${nodeCount} (${nodeOutputSize}) | Messages: ${messageCount}\n` +
|
||||
` Completed: ${result.totalCompleted}/${messageCount}\n` +
|
||||
` Throughput: ${result.avgExecPerSec.toFixed(1)} exec/s | ${result.actionsPerSec.toFixed(1)} actions/s\n` +
|
||||
|
|
|
|||
|
|
@ -57,9 +57,18 @@ export async function runWebhookThroughputTest(options: WebhookThroughputOptions
|
|||
|
||||
testInfo.setTimeout(timeoutMs + 120_000);
|
||||
|
||||
const profile = testInfo.project.name.replace(':infrastructure', '').replace('benchmark-', '');
|
||||
const mode = testInfo.project.name.replace(':infrastructure', '').replace('benchmark-', '');
|
||||
const obs = services.observability;
|
||||
|
||||
const dimensions: Record<string, string | number> = {
|
||||
trigger: 'webhook',
|
||||
nodes: nodeCount,
|
||||
output: nodeOutputSize,
|
||||
connections,
|
||||
duration_s: durationSeconds,
|
||||
mode,
|
||||
};
|
||||
|
||||
// Phase 1: Create + activate workflow
|
||||
// createWorkflowFromDefinition overwrites the webhook path and sets webhookId for proper registration
|
||||
const { workflowId, createdWorkflow, webhookPath } =
|
||||
|
|
@ -116,50 +125,43 @@ export async function runWebhookThroughputTest(options: WebhookThroughputOptions
|
|||
const diagnostics = await collectDiagnostics(obs.metrics, throughputResult.durationMs);
|
||||
|
||||
// Phase 6: Attach metrics — VictoriaMetrics throughput + autocannon HTTP stats + diagnostics
|
||||
await attachThroughputResults(testInfo, testInfo.title, throughputResult);
|
||||
await attachThroughputResults(testInfo, dimensions, throughputResult);
|
||||
await attachMetric(testInfo, 'http-latency-p50', cannonResult.latency.p50, 'ms', dimensions);
|
||||
await attachMetric(testInfo, 'http-latency-p99', cannonResult.latency.p99, 'ms', dimensions);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${testInfo.title}-http-latency-p50`,
|
||||
cannonResult.latency.p50,
|
||||
'ms',
|
||||
);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${testInfo.title}-http-latency-p99`,
|
||||
cannonResult.latency.p99,
|
||||
'ms',
|
||||
);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${testInfo.title}-http-requests-total`,
|
||||
'http-requests-total',
|
||||
cannonResult.requests.total,
|
||||
'count',
|
||||
dimensions,
|
||||
);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${testInfo.title}-http-requests-avg`,
|
||||
'http-requests-avg',
|
||||
cannonResult.requests.average,
|
||||
'req/s',
|
||||
dimensions,
|
||||
);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${testInfo.title}-http-errors`,
|
||||
'http-errors',
|
||||
cannonResult.errors + cannonResult.non2xx,
|
||||
'count',
|
||||
dimensions,
|
||||
);
|
||||
await attachDiagnostics(testInfo, testInfo.title, diagnostics);
|
||||
await attachDiagnostics(testInfo, dimensions, diagnostics);
|
||||
|
||||
// Phase 7: Log summary
|
||||
const fmt = formatDiagnosticValue;
|
||||
console.log(
|
||||
`[DIAG-${profile}] ${testInfo.title}\n` +
|
||||
`[DIAG-${mode}] ${testInfo.title}\n` +
|
||||
` Event Loop Lag: ${fmt(diagnostics.eventLoopLag, 's')}\n` +
|
||||
` PG Transactions/s: ${fmt(diagnostics.pgTxRate, ' tx/s')}\n` +
|
||||
` PG Active Connections: ${fmt(diagnostics.pgActiveConnections)}`,
|
||||
);
|
||||
|
||||
console.log(
|
||||
`[WEBHOOK-${profile} RESULT] ${testInfo.title}\n` +
|
||||
`[WEBHOOK-${mode} RESULT] ${testInfo.title}\n` +
|
||||
` n8n Throughput: ${throughputResult.avgExecPerSec.toFixed(1)} exec/s | ` +
|
||||
`${throughputResult.actionsPerSec.toFixed(1)} actions/s\n` +
|
||||
` Peak: ${throughputResult.peakExecPerSec.toFixed(1)} exec/s | ` +
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ test.describe(
|
|||
services,
|
||||
testInfo,
|
||||
load: { type: 'steady', ratePerSecond: 200, durationSeconds: 30 },
|
||||
trigger: 'kafka',
|
||||
nodeCount: 30,
|
||||
nodeOutputSize: '10KB',
|
||||
timeoutMs: 300_000,
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ test.describe(
|
|||
services,
|
||||
testInfo,
|
||||
load: { type: 'steady', ratePerSecond: 300, durationSeconds: 30 },
|
||||
trigger: 'kafka',
|
||||
nodeCount: 30,
|
||||
nodeOutputSize: '10KB',
|
||||
timeoutMs: 300_000,
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ test.describe(
|
|||
services,
|
||||
testInfo,
|
||||
load: { type: 'steady', ratePerSecond: 100, durationSeconds: 30 },
|
||||
trigger: 'kafka',
|
||||
nodeCount: 30,
|
||||
nodeOutputSize: '10KB',
|
||||
timeoutMs: 300_000,
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ test.describe(
|
|||
services,
|
||||
testInfo,
|
||||
load: { type: 'preloaded', count: 10_000 },
|
||||
trigger: 'kafka',
|
||||
nodeCount: 60,
|
||||
timeoutMs: 600_000,
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ test.describe(
|
|||
messageCount: envMessages || 5_000,
|
||||
nodeCount: 10,
|
||||
nodeOutputSize: '100KB',
|
||||
trigger: 'kafka',
|
||||
timeoutMs: 600_000,
|
||||
plan: BENCHMARK_MAIN_RESOURCES,
|
||||
workerPlan: BENCHMARK_WORKER_RESOURCES,
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ test.describe(
|
|||
messageCount: envMessages || 5_000,
|
||||
nodeCount: 10,
|
||||
nodeOutputSize: '10KB',
|
||||
trigger: 'kafka',
|
||||
timeoutMs: 300_000,
|
||||
plan: BENCHMARK_MAIN_RESOURCES,
|
||||
workerPlan: BENCHMARK_WORKER_RESOURCES,
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ test.describe(
|
|||
messageCount: envMessages || 5_000,
|
||||
nodeCount: 30,
|
||||
nodeOutputSize: '10KB',
|
||||
trigger: 'kafka',
|
||||
timeoutMs: 300_000,
|
||||
plan: BENCHMARK_MAIN_RESOURCES,
|
||||
workerPlan: BENCHMARK_WORKER_RESOURCES,
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ test.describe(
|
|||
messageCount: envMessages || 5_000,
|
||||
nodeCount: 60,
|
||||
nodeOutputSize: '10KB',
|
||||
trigger: 'kafka',
|
||||
timeoutMs: 600_000,
|
||||
plan: BENCHMARK_MAIN_RESOURCES,
|
||||
workerPlan: BENCHMARK_WORKER_RESOURCES,
|
||||
|
|
|
|||
|
|
@ -51,8 +51,10 @@ test.describe(
|
|||
}
|
||||
const average = stats.reduce((a, b) => a + b, 0) / stats.length;
|
||||
|
||||
await attachMetric(testInfo, `open-node-${itemCount}`, average, 'ms');
|
||||
await attachMetric(testInfo, `trigger-workflow-${itemCount}`, triggerDuration, 'ms');
|
||||
await attachMetric(testInfo, 'open-node', average, 'ms', { item_count: itemCount });
|
||||
await attachMetric(testInfo, 'trigger-workflow', triggerDuration, 'ms', {
|
||||
item_count: itemCount,
|
||||
});
|
||||
|
||||
expect(average).toBeGreaterThan(0);
|
||||
expect(triggerDuration).toBeGreaterThan(0);
|
||||
|
|
|
|||
|
|
@ -125,16 +125,16 @@ export async function collectDiagnostics(
|
|||
*/
|
||||
export async function attachDiagnostics(
|
||||
testInfo: TestInfo,
|
||||
label: string,
|
||||
dimensions: Record<string, string | number>,
|
||||
diagnostics: DiagnosticsResult,
|
||||
): Promise<void> {
|
||||
if (diagnostics.eventLoopLag !== undefined) {
|
||||
await attachMetric(testInfo, `${label}-event-loop-lag`, diagnostics.eventLoopLag, 's');
|
||||
await attachMetric(testInfo, 'event-loop-lag', diagnostics.eventLoopLag, 's', dimensions);
|
||||
}
|
||||
if (diagnostics.pgTxRate !== undefined) {
|
||||
await attachMetric(testInfo, `${label}-pg-tx-rate`, diagnostics.pgTxRate, 'tx/s');
|
||||
await attachMetric(testInfo, 'pg-tx-rate', diagnostics.pgTxRate, 'tx/s', dimensions);
|
||||
}
|
||||
if (diagnostics.queueWaiting !== undefined) {
|
||||
await attachMetric(testInfo, `${label}-queue-waiting`, diagnostics.queueWaiting, 'count');
|
||||
await attachMetric(testInfo, 'queue-waiting', diagnostics.queueWaiting, 'count', dimensions);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,20 +82,20 @@ export function buildMetrics(
|
|||
|
||||
export async function attachLoadTestResults(
|
||||
testInfo: TestInfo,
|
||||
label: string,
|
||||
dimensions: Record<string, string | number>,
|
||||
metrics: ExecutionMetrics,
|
||||
): Promise<void> {
|
||||
await attachMetric(testInfo, `${label}-executions-completed`, metrics.totalCompleted, 'count');
|
||||
await attachMetric(testInfo, `${label}-executions-errors`, metrics.totalErrors, 'count');
|
||||
await attachMetric(testInfo, `${label}-throughput`, metrics.throughputPerSecond, 'exec/s');
|
||||
await attachMetric(testInfo, `${label}-total-duration`, metrics.durationMs, 'ms');
|
||||
await attachMetric(testInfo, 'executions-completed', metrics.totalCompleted, 'count', dimensions);
|
||||
await attachMetric(testInfo, 'executions-errors', metrics.totalErrors, 'count', dimensions);
|
||||
await attachMetric(testInfo, 'throughput', metrics.throughputPerSecond, 'exec/s', dimensions);
|
||||
await attachMetric(testInfo, 'total-duration', metrics.durationMs, 'ms', dimensions);
|
||||
|
||||
// Only attach duration percentiles when we have sampled data — otherwise the
|
||||
// reporter would show misleading "0ms" values (e.g. when EXECUTIONS_DATA_SAVE_ON_SUCCESS=none)
|
||||
if (metrics.executionDurations.length > 0) {
|
||||
await attachMetric(testInfo, `${label}-duration-avg`, metrics.avgDurationMs, 'ms');
|
||||
await attachMetric(testInfo, `${label}-duration-p50`, metrics.p50DurationMs, 'ms');
|
||||
await attachMetric(testInfo, `${label}-duration-p95`, metrics.p95DurationMs, 'ms');
|
||||
await attachMetric(testInfo, `${label}-duration-p99`, metrics.p99DurationMs, 'ms');
|
||||
await attachMetric(testInfo, 'duration-avg', metrics.avgDurationMs, 'ms', dimensions);
|
||||
await attachMetric(testInfo, 'duration-p50', metrics.p50DurationMs, 'ms', dimensions);
|
||||
await attachMetric(testInfo, 'duration-p95', metrics.p95DurationMs, 'ms', dimensions);
|
||||
await attachMetric(testInfo, 'duration-p99', metrics.p99DurationMs, 'ms', dimensions);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -210,18 +210,19 @@ function calculateThroughput(
|
|||
|
||||
export async function attachThroughputResults(
|
||||
testInfo: TestInfo,
|
||||
label: string,
|
||||
dimensions: Record<string, string | number>,
|
||||
result: ThroughputResult,
|
||||
): Promise<void> {
|
||||
await attachMetric(testInfo, `${label}-exec-per-sec`, result.avgExecPerSec, 'exec/s');
|
||||
await attachMetric(testInfo, `${label}-actions-per-sec`, result.actionsPerSec, 'actions/s');
|
||||
await attachMetric(testInfo, `${label}-peak-exec-per-sec`, result.peakExecPerSec, 'exec/s');
|
||||
await attachMetric(testInfo, 'exec-per-sec', result.avgExecPerSec, 'exec/s', dimensions);
|
||||
await attachMetric(testInfo, 'actions-per-sec', result.actionsPerSec, 'actions/s', dimensions);
|
||||
await attachMetric(testInfo, 'peak-exec-per-sec', result.peakExecPerSec, 'exec/s', dimensions);
|
||||
await attachMetric(
|
||||
testInfo,
|
||||
`${label}-peak-actions-per-sec`,
|
||||
'peak-actions-per-sec',
|
||||
result.peakActionsPerSec,
|
||||
'actions/s',
|
||||
dimensions,
|
||||
);
|
||||
await attachMetric(testInfo, `${label}-total-completed`, result.totalCompleted, 'count');
|
||||
await attachMetric(testInfo, `${label}-duration`, result.durationMs, 'ms');
|
||||
await attachMetric(testInfo, 'total-completed', result.totalCompleted, 'count', dimensions);
|
||||
await attachMetric(testInfo, 'duration', result.durationMs, 'ms', dimensions);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,23 @@ import type { IWorkflowBase } from 'n8n-workflow';
|
|||
|
||||
import type { ApiHelpers } from '../../services/api-helper';
|
||||
|
||||
// --- Benchmark dimensions ---
|
||||
|
||||
/**
|
||||
* Structured dimensions attached to every benchmark metric.
|
||||
* Stored as JSON in BigQuery — enables slicing by node count, payload, mode, etc.
|
||||
* without encoding config into the metric name.
|
||||
*/
|
||||
export type BenchmarkDimensions = Record<string, string | number>;
|
||||
|
||||
/** Known trigger types. Must be set explicitly — no default to avoid silently wrong data. */
|
||||
export type TriggerType = 'kafka' | 'webhook';
|
||||
|
||||
/** Load profile for steady-rate or preloaded burst tests. */
|
||||
export type LoadProfile =
|
||||
| { type: 'steady'; ratePerSecond: number; durationSeconds: number }
|
||||
| { type: 'preloaded'; count: number };
|
||||
|
||||
// --- Payload sizes ---
|
||||
|
||||
export const PAYLOAD_PROFILES = {
|
||||
|
|
|
|||
|
|
@ -36,9 +36,21 @@ export async function attachMetric(
|
|||
metricName: string,
|
||||
value: number,
|
||||
unit?: string,
|
||||
dimensions?: Record<string, string | number>,
|
||||
): Promise<void> {
|
||||
await testInfo.attach(`metric:${metricName}`, {
|
||||
body: JSON.stringify({ value, unit }),
|
||||
body: JSON.stringify({ value, unit, dimensions }),
|
||||
});
|
||||
|
||||
// Currents native format — surfaces metrics in their analytics dashboard
|
||||
testInfo.annotations.push({
|
||||
type: 'currents:metric',
|
||||
description: JSON.stringify({
|
||||
name: metricName,
|
||||
value,
|
||||
type: Number.isInteger(value) ? 'integer' : 'float',
|
||||
...(unit && { unit: unit.toLowerCase() }),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue