mirror of
https://github.com/n8n-io/n8n
synced 2026-04-21 15:47:20 +00:00
test: Rebalance (#23579)
This commit is contained in:
Declan Carroll
GitHub
parent
e9e480bb8e
commit
a087d36990
23 changed files with 739 additions and 559 deletions
631
.github/test-metrics/playwright.json
vendored
631
.github/test-metrics/playwright.json
vendored
|
|
@ -1,645 +1,670 @@
|
|||
{
|
||||
"updatedAt": "2025-12-22T14:46:56.990Z",
|
||||
"updatedAt": "2025-12-23T17:51:26.730Z",
|
||||
"source": "currents",
|
||||
"projectId": "I0yzoc",
|
||||
"specs": {
|
||||
"tests/e2e/workflows/list/workflows.spec.ts": {
|
||||
"avgDuration": 111282,
|
||||
"avgDuration": 113599,
|
||||
"testCount": 9,
|
||||
"flakyRate": 0.0024
|
||||
"flakyRate": 0.0066
|
||||
},
|
||||
"tests/e2e/workflows/templates/templates.spec.ts": {
|
||||
"avgDuration": 17326,
|
||||
"testCount": 8,
|
||||
"avgDuration": 19603,
|
||||
"testCount": 9,
|
||||
"flakyRate": 0.0016
|
||||
},
|
||||
"tests/e2e/workflows/editor/tags.spec.ts": {
|
||||
"avgDuration": 38949,
|
||||
"avgDuration": 38592,
|
||||
"testCount": 7,
|
||||
"flakyRate": 0.0015
|
||||
"flakyRate": 0.0014
|
||||
},
|
||||
"tests/e2e/chat-hub/chat-hub-workflow-agent.spec.ts": {
|
||||
"avgDuration": 45555,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0814
|
||||
},
|
||||
"tests/e2e/workflows/editor/workflow-actions/settings.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/workflows/editor/subworkflows/workflow-selector.spec.ts": {
|
||||
"avgDuration": 28402,
|
||||
"avgDuration": 28137,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0011
|
||||
"flakyRate": 0.0012
|
||||
},
|
||||
"tests/e2e/workflows/editor/workflow-actions/save.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/workflows/editor/workflow-actions/run.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/workflows/editor/workflow-actions/publish.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"testCount": 1,
|
||||
"avgDuration": 22021,
|
||||
"testCount": 8,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/workflows/checklist/production-checklist.spec.ts": {
|
||||
"avgDuration": 26868,
|
||||
"avgDuration": 27187,
|
||||
"testCount": 7,
|
||||
"flakyRate": 0.0013
|
||||
},
|
||||
"tests/e2e/workflows/executions/list.spec.ts": {
|
||||
"avgDuration": 53211,
|
||||
"testCount": 12,
|
||||
"flakyRate": 0.0022
|
||||
"avgDuration": 61008,
|
||||
"testCount": 11,
|
||||
"flakyRate": 0.0042
|
||||
},
|
||||
"tests/e2e/workflows/editor/workflow-actions/duplicate.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/workflows/editor/workflow-actions/copy-paste.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/ai/workflow-builder.spec.ts": {
|
||||
"avgDuration": 30233,
|
||||
"avgDuration": 32211,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0072
|
||||
"flakyRate": 0.0096
|
||||
},
|
||||
"tests/e2e/workflows/editor/workflow-actions/archive.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/settings/workers/workers.spec.ts": {
|
||||
"avgDuration": 5554,
|
||||
"avgDuration": 5662,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.001
|
||||
"flakyRate": 0.0011
|
||||
},
|
||||
"tests/e2e/nodes/webhook.spec.ts": {
|
||||
"avgDuration": 54803,
|
||||
"avgDuration": 55672,
|
||||
"testCount": 9,
|
||||
"flakyRate": 0.0019
|
||||
"flakyRate": 0.0021
|
||||
},
|
||||
"tests/e2e/api/webhook-isolation.spec.ts": {
|
||||
"avgDuration": 4609,
|
||||
"avgDuration": 5058,
|
||||
"testCount": 15,
|
||||
"flakyRate": 0.0002
|
||||
"flakyRate": 0.0005
|
||||
},
|
||||
"tests/e2e/app-config/versions.spec.ts": {
|
||||
"avgDuration": 5872,
|
||||
"avgDuration": 5875,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/settings/environments/variables.spec.ts": {
|
||||
"avgDuration": 15454,
|
||||
"avgDuration": 15854,
|
||||
"testCount": 7,
|
||||
"flakyRate": 0.0001
|
||||
"flakyRate": 0.0002
|
||||
},
|
||||
"tests/e2e/settings/users/users.spec.ts": {
|
||||
"avgDuration": 13556,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0093
|
||||
"avgDuration": 11978,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.015
|
||||
},
|
||||
"tests/e2e/building-blocks/user-service.spec.ts": {
|
||||
"avgDuration": 5468,
|
||||
"avgDuration": 6400,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0002
|
||||
"flakyRate": 0.0005
|
||||
},
|
||||
"tests/e2e/workflows/editor/canvas/undo-redo.spec.ts": {
|
||||
"avgDuration": 70056,
|
||||
"testCount": 14,
|
||||
"flakyRate": 0.0192
|
||||
"avgDuration": 65670,
|
||||
"testCount": 13,
|
||||
"flakyRate": 0.0193
|
||||
},
|
||||
"tests/e2e/building-blocks/workflow-entry-points.spec.ts": {
|
||||
"avgDuration": 11814,
|
||||
"avgDuration": 13573,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0003
|
||||
"flakyRate": 0.0006
|
||||
},
|
||||
"tests/e2e/chat-hub/chat-hub-tools.spec.ts": {
|
||||
"avgDuration": 11472,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0233
|
||||
},
|
||||
"tests/e2e/capabilities/task-runner.spec.ts": {
|
||||
"avgDuration": 53189,
|
||||
"avgDuration": 52322,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.1399
|
||||
"flakyRate": 0.1299
|
||||
},
|
||||
"tests/e2e/workflows/editor/subworkflows/debugging.spec.ts": {
|
||||
"avgDuration": 12969,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0004
|
||||
"avgDuration": 9686,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/workflows/editor/subworkflows/extraction.spec.ts": {
|
||||
"avgDuration": 35363,
|
||||
"avgDuration": 35747,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0012
|
||||
"flakyRate": 0.0011
|
||||
},
|
||||
"tests/e2e/settings/environments/source-control.spec.ts": {
|
||||
"avgDuration": 149562,
|
||||
"avgDuration": 142866,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.072
|
||||
"flakyRate": 0.1412
|
||||
},
|
||||
"tests/e2e/auth/signin.spec.ts": {
|
||||
"avgDuration": 89898,
|
||||
"avgDuration": 90584,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/chat-hub/chat-hub-settings.spec.ts": {
|
||||
"avgDuration": 29402,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0674
|
||||
},
|
||||
"tests/e2e/app-config/security-notifications.spec.ts": {
|
||||
"avgDuration": 13663,
|
||||
"avgDuration": 14479,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0002
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/schema-preview.spec.ts": {
|
||||
"avgDuration": 4888,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0012
|
||||
},
|
||||
"tests/e2e/nodes/schedule-trigger-node.spec.ts": {
|
||||
"avgDuration": 3451,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0008
|
||||
},
|
||||
"tests/e2e/regression/SUG-38-inline-expression-preview.spec.ts": {
|
||||
"avgDuration": 4726,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0008
|
||||
},
|
||||
"tests/e2e/regression/SUG-121-fields-reset-after-closing-ndv.spec.ts": {
|
||||
"avgDuration": 4118,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/workflows/editor/routing.spec.ts": {
|
||||
"avgDuration": 27704,
|
||||
"testCount": 6,
|
||||
"flakyRate": 0.0043
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/resource-mapper.spec.ts": {
|
||||
"avgDuration": 17622,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/schema-preview.spec.ts": {
|
||||
"avgDuration": 5002,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0021
|
||||
},
|
||||
"tests/e2e/nodes/schedule-trigger-node.spec.ts": {
|
||||
"avgDuration": 3494,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/regression/SUG-38-inline-expression-preview.spec.ts": {
|
||||
"avgDuration": 4768,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/regression/SUG-121-fields-reset-after-closing-ndv.spec.ts": {
|
||||
"avgDuration": 4232,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/workflows/editor/routing.spec.ts": {
|
||||
"avgDuration": 28147,
|
||||
"testCount": 6,
|
||||
"flakyRate": 0.0042
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/resource-mapper.spec.ts": {
|
||||
"avgDuration": 17808,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0005
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/resource-locator.spec.ts": {
|
||||
"avgDuration": 36071,
|
||||
"avgDuration": 36783,
|
||||
"testCount": 7,
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/ai/rag-callout.spec.ts": {
|
||||
"avgDuration": 6828,
|
||||
"avgDuration": 7145,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/source-control/push.spec.ts": {
|
||||
"avgDuration": 149821,
|
||||
"avgDuration": 161130,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.7013
|
||||
"flakyRate": 0.4348
|
||||
},
|
||||
"tests/e2e/source-control/pull.spec.ts": {
|
||||
"avgDuration": 71354,
|
||||
"avgDuration": 63320,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.4884
|
||||
"flakyRate": 0.2028
|
||||
},
|
||||
"tests/e2e/capabilities/proxy-server.spec.ts": {
|
||||
"avgDuration": 26812,
|
||||
"avgDuration": 27599,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.003
|
||||
"flakyRate": 0.0028
|
||||
},
|
||||
"tests/e2e/projects/project-settings.spec.ts": {
|
||||
"avgDuration": 49124,
|
||||
"avgDuration": 48766,
|
||||
"testCount": 8,
|
||||
"flakyRate": 0.0004
|
||||
"flakyRate": 0.0005
|
||||
},
|
||||
"tests/e2e/chat-hub/chat-hub-personal-agent.spec.ts": {
|
||||
"avgDuration": 117328,
|
||||
"avgDuration": 114815,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0638
|
||||
"flakyRate": 0.0448
|
||||
},
|
||||
"tests/e2e/settings/personal/personal.spec.ts": {
|
||||
"avgDuration": 40901,
|
||||
"avgDuration": 40996,
|
||||
"testCount": 9,
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/auth/password-reset.spec.ts": {
|
||||
"avgDuration": 17863,
|
||||
"avgDuration": 18625,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0008
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/workflows/editor/subworkflows/wait.spec.ts": {
|
||||
"avgDuration": 53816,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.2889
|
||||
"avgDuration": 40569,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.2677
|
||||
},
|
||||
"tests/e2e/nodes/pdf-node.spec.ts": {
|
||||
"avgDuration": 5411,
|
||||
"avgDuration": 5500,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0032
|
||||
"flakyRate": 0.003
|
||||
},
|
||||
"tests/e2e/auth/oidc.spec.ts": {
|
||||
"avgDuration": 43909,
|
||||
"avgDuration": 46170,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.004
|
||||
"flakyRate": 0.0033
|
||||
},
|
||||
"tests/e2e/credentials/oauth.spec.ts": {
|
||||
"avgDuration": 4171,
|
||||
"avgDuration": 4270,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/io-filter.spec.ts": {
|
||||
"avgDuration": 12837,
|
||||
"avgDuration": 12720,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.004
|
||||
"flakyRate": 0.0037
|
||||
},
|
||||
"tests/e2e/building-blocks/node-details-configuration.spec.ts": {
|
||||
"avgDuration": 32781,
|
||||
"avgDuration": 33860,
|
||||
"testCount": 7,
|
||||
"flakyRate": 0.0001
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/node-creator/workflows.spec.ts": {
|
||||
"avgDuration": 6382,
|
||||
"avgDuration": 6428,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/node-creator/vector-stores.spec.ts": {
|
||||
"avgDuration": 8933,
|
||||
"avgDuration": 9088,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0008
|
||||
},
|
||||
"tests/e2e/node-creator/special-nodes.spec.ts": {
|
||||
"avgDuration": 13528,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0008
|
||||
},
|
||||
"tests/e2e/node-creator/navigation.spec.ts": {
|
||||
"avgDuration": 16049,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/node-creator/categories.spec.ts": {
|
||||
"avgDuration": 16761,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0026
|
||||
},
|
||||
"tests/e2e/node-creator/actions.spec.ts": {
|
||||
"avgDuration": 12847,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/app-config/nps-survey.spec.ts": {
|
||||
"avgDuration": 35241,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0027
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/ndv-parameters.spec.ts": {
|
||||
"avgDuration": 53626,
|
||||
"testCount": 10,
|
||||
"flakyRate": 0.001
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/paired-item.spec.ts": {
|
||||
"avgDuration": 30544,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0003
|
||||
"tests/e2e/node-creator/special-nodes.spec.ts": {
|
||||
"avgDuration": 13503,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0012
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/ndv-floating-nodes.spec.ts": {
|
||||
"avgDuration": 24869,
|
||||
"tests/e2e/node-creator/navigation.spec.ts": {
|
||||
"avgDuration": 15911,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0045
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/ndv-data-display.spec.ts": {
|
||||
"avgDuration": 69364,
|
||||
"testCount": 11,
|
||||
"flakyRate": 0.0044
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/ndv-core.spec.ts": {
|
||||
"avgDuration": 57624,
|
||||
"testCount": 16,
|
||||
"flakyRate": 0.0002
|
||||
},
|
||||
"tests/e2e/node-creator/categories.spec.ts": {
|
||||
"avgDuration": 17552,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0027
|
||||
},
|
||||
"tests/e2e/node-creator/actions.spec.ts": {
|
||||
"avgDuration": 12991,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/app-config/nps-survey.spec.ts": {
|
||||
"avgDuration": 37040,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0025
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/ndv-parameters.spec.ts": {
|
||||
"avgDuration": 48876,
|
||||
"testCount": 9,
|
||||
"flakyRate": 0.0012
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/paired-item.spec.ts": {
|
||||
"avgDuration": 30863,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0006
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/ndv-floating-nodes.spec.ts": {
|
||||
"avgDuration": 25025,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0042
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/ndv-data-display.spec.ts": {
|
||||
"avgDuration": 63163,
|
||||
"testCount": 10,
|
||||
"flakyRate": 0.0047
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/ndv-core.spec.ts": {
|
||||
"avgDuration": 59650,
|
||||
"testCount": 16,
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/workflows/editor/execution/partial.spec.ts": {
|
||||
"avgDuration": 8226,
|
||||
"avgDuration": 8298,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/workflows/editor/execution/logs.spec.ts": {
|
||||
"avgDuration": 35288,
|
||||
"testCount": 8,
|
||||
"flakyRate": 0.0003
|
||||
"avgDuration": 39950,
|
||||
"testCount": 9,
|
||||
"flakyRate": 0.0002
|
||||
},
|
||||
"tests/e2e/settings/log-streaming/log-streaming-observability.spec.ts": {
|
||||
"avgDuration": 41746,
|
||||
"avgDuration": 41907,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0029
|
||||
"flakyRate": 0.0019
|
||||
},
|
||||
"tests/e2e/settings/log-streaming/log-streaming-ui-e2e.spec.ts": {
|
||||
"avgDuration": 29825,
|
||||
"avgDuration": 30182,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0059
|
||||
"flakyRate": 0.0039
|
||||
},
|
||||
"tests/e2e/settings/log-streaming/log-streaming.spec.ts": {
|
||||
"avgDuration": 13003,
|
||||
"avgDuration": 14143,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/ai/langchain-agents.spec.ts": {
|
||||
"avgDuration": 69209,
|
||||
"avgDuration": 69954,
|
||||
"testCount": 7,
|
||||
"flakyRate": 0.0017
|
||||
},
|
||||
"tests/e2e/ai/langchain-tools.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/ai/langchain-memory.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"testCount": 2,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/ai/langchain-chains.spec.ts": {
|
||||
"avgDuration": 41017,
|
||||
"avgDuration": 41803,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0015
|
||||
"flakyRate": 0.0014
|
||||
},
|
||||
"tests/e2e/ai/langchain-vectorstores.spec.ts": {
|
||||
"avgDuration": 33667,
|
||||
"avgDuration": 36841,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0035
|
||||
"flakyRate": 0.0153
|
||||
},
|
||||
"tests/e2e/workflows/editor/expressions/inline.spec.ts": {
|
||||
"avgDuration": 30405,
|
||||
"testCount": 7,
|
||||
"flakyRate": 0.0002
|
||||
"avgDuration": 36109,
|
||||
"testCount": 8,
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/workflows/editor/execution/inject-previous.spec.ts": {
|
||||
"avgDuration": 12583,
|
||||
"avgDuration": 12687,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/workflows/list/import.spec.ts": {
|
||||
"avgDuration": 15010,
|
||||
"avgDuration": 15225,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0002
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/nodes/if-node.spec.ts": {
|
||||
"avgDuration": 7297,
|
||||
"avgDuration": 7492,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/nodes/http-request-node.spec.ts": {
|
||||
"avgDuration": 8948,
|
||||
"avgDuration": 9072,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0004
|
||||
"flakyRate": 0.0011
|
||||
},
|
||||
"tests/e2e/regression/GHC-5776-ai-sessions-metadata-license-error.spec.ts": {
|
||||
"avgDuration": 2589,
|
||||
"avgDuration": 2602,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/nodes/form-trigger-node.spec.ts": {
|
||||
"avgDuration": 11639,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0
|
||||
"avgDuration": 27007,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/projects/folders-operations.spec.ts": {
|
||||
"avgDuration": 50221,
|
||||
"testCount": 14,
|
||||
"flakyRate": 0.0005
|
||||
"avgDuration": 46752,
|
||||
"testCount": 13,
|
||||
"flakyRate": 0.0006
|
||||
},
|
||||
"tests/e2e/projects/folders-basic.spec.ts": {
|
||||
"avgDuration": 24510,
|
||||
"testCount": 10,
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/projects/folders-advanced.spec.ts": {
|
||||
"avgDuration": 22155,
|
||||
"testCount": 6,
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/workflows/editor/canvas/focus-panel.spec.ts": {
|
||||
"avgDuration": 4090,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/api/webhook-external.spec.ts": {
|
||||
"avgDuration": 8903,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0643
|
||||
},
|
||||
"tests/e2e/workflows/editor/expressions/modal.spec.ts": {
|
||||
"avgDuration": 41513,
|
||||
"testCount": 6,
|
||||
"avgDuration": 28139,
|
||||
"testCount": 11,
|
||||
"flakyRate": 0.0005
|
||||
},
|
||||
"tests/e2e/projects/folders-advanced.spec.ts": {
|
||||
"avgDuration": 20134,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/workflows/editor/canvas/focus-panel.spec.ts": {
|
||||
"avgDuration": 4056,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/chat-hub/chat-hub-attachment.spec.ts": {
|
||||
"avgDuration": 44171,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0147
|
||||
},
|
||||
"tests/e2e/api/webhook-external.spec.ts": {
|
||||
"avgDuration": 9425,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0594
|
||||
},
|
||||
"tests/e2e/workflows/editor/expressions/modal.spec.ts": {
|
||||
"avgDuration": 41609,
|
||||
"testCount": 6,
|
||||
"flakyRate": 0.0006
|
||||
},
|
||||
"tests/e2e/workflows/editor/execution/execution.spec.ts": {
|
||||
"avgDuration": 47249,
|
||||
"avgDuration": 47322,
|
||||
"testCount": 14,
|
||||
"flakyRate": 0
|
||||
"flakyRate": 0.0001
|
||||
},
|
||||
"tests/e2e/workflows/editor/execution/previous-nodes.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/ai/evaluations.spec.ts": {
|
||||
"avgDuration": 30000,
|
||||
"avgDuration": 60000,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/app-config/env-feature-flags.spec.ts": {
|
||||
"avgDuration": 1148,
|
||||
"avgDuration": 1359,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/nodes/email-send-node.spec.ts": {
|
||||
"avgDuration": 21579,
|
||||
"avgDuration": 22418,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0058
|
||||
"flakyRate": 0.0054
|
||||
},
|
||||
"tests/e2e/workflows/editor/code/editors.spec.ts": {
|
||||
"avgDuration": 48793,
|
||||
"avgDuration": 49237,
|
||||
"testCount": 11,
|
||||
"flakyRate": 0.0001
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/workflows/editor/editor-after-route-changes.spec.ts": {
|
||||
"avgDuration": 15941,
|
||||
"avgDuration": 16091,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0229
|
||||
"flakyRate": 0.0211
|
||||
},
|
||||
"tests/e2e/app-config/demo.spec.ts": {
|
||||
"avgDuration": 14563,
|
||||
"avgDuration": 14718,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0167
|
||||
"flakyRate": 0.0176
|
||||
},
|
||||
"tests/e2e/workflows/editor/execution/debug.spec.ts": {
|
||||
"avgDuration": 46347,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0038
|
||||
"avgDuration": 34255,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0037
|
||||
},
|
||||
"tests/e2e/workflows/editor/expressions/transformation.spec.ts": {
|
||||
"avgDuration": 25043,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0002
|
||||
"avgDuration": 30134,
|
||||
"testCount": 6,
|
||||
"flakyRate": 0.0001
|
||||
},
|
||||
"tests/e2e/workflows/editor/ndv/pinning.spec.ts": {
|
||||
"avgDuration": 36248,
|
||||
"avgDuration": 36652,
|
||||
"testCount": 10,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/data-tables/tables.spec.ts": {
|
||||
"avgDuration": 70760,
|
||||
"avgDuration": 71055,
|
||||
"testCount": 7,
|
||||
"flakyRate": 0.0006
|
||||
"flakyRate": 0.0011
|
||||
},
|
||||
"tests/e2e/data-tables/details.spec.ts": {
|
||||
"avgDuration": 71609,
|
||||
"avgDuration": 72323,
|
||||
"testCount": 11,
|
||||
"flakyRate": 0.0001
|
||||
},
|
||||
"tests/e2e/workflows/editor/expressions/mapping.spec.ts": {
|
||||
"avgDuration": 42611,
|
||||
"testCount": 10,
|
||||
"flakyRate": 0.005
|
||||
},
|
||||
"tests/e2e/credentials/crud.spec.ts": {
|
||||
"avgDuration": 73865,
|
||||
"testCount": 14,
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/workflows/editor/expressions/mapping.spec.ts": {
|
||||
"avgDuration": 42523,
|
||||
"testCount": 10,
|
||||
"flakyRate": 0.0047
|
||||
},
|
||||
"tests/e2e/credentials/crud.spec.ts": {
|
||||
"avgDuration": 80126,
|
||||
"testCount": 15,
|
||||
"flakyRate": 0.0009
|
||||
},
|
||||
"tests/e2e/building-blocks/credentials.spec.ts": {
|
||||
"avgDuration": 28596,
|
||||
"avgDuration": 28608,
|
||||
"testCount": 6,
|
||||
"flakyRate": 0.0006
|
||||
},
|
||||
"tests/e2e/credentials/api-operations.spec.ts": {
|
||||
"avgDuration": 1628,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/settings/community-nodes/community-nodes.spec.ts": {
|
||||
"avgDuration": 4058,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0008
|
||||
},
|
||||
"tests/e2e/credentials/api-operations.spec.ts": {
|
||||
"avgDuration": 1602,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/settings/community-nodes/community-nodes.spec.ts": {
|
||||
"avgDuration": 4091,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/nodes/community-nodes.spec.ts": {
|
||||
"avgDuration": 13865,
|
||||
"avgDuration": 13999,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0005
|
||||
},
|
||||
"tests/e2e/workflows/editor/code/code-node.spec.ts": {
|
||||
"avgDuration": 73038,
|
||||
"avgDuration": 74990,
|
||||
"testCount": 12,
|
||||
"flakyRate": 0.0224
|
||||
"flakyRate": 0.0226
|
||||
},
|
||||
"tests/e2e/chat-hub/chat-hub-chat-user.spec.ts": {
|
||||
"avgDuration": 9556,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/ai/chat-session.spec.ts": {
|
||||
"avgDuration": 5264,
|
||||
"avgDuration": 6490,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0077
|
||||
"flakyRate": 0.01
|
||||
},
|
||||
"tests/e2e/workflows/editor/canvas/canvas-zoom.spec.ts": {
|
||||
"avgDuration": 54125,
|
||||
"avgDuration": 54599,
|
||||
"testCount": 12,
|
||||
"flakyRate": 0.0003
|
||||
"flakyRate": 0.0005
|
||||
},
|
||||
"tests/e2e/workflows/editor/canvas/canvas-nodes.spec.ts": {
|
||||
"avgDuration": 53821,
|
||||
"testCount": 7,
|
||||
"flakyRate": 0.0036
|
||||
"avgDuration": 65556,
|
||||
"testCount": 8,
|
||||
"flakyRate": 0.0042
|
||||
},
|
||||
"tests/e2e/building-blocks/canvas-actions.spec.ts": {
|
||||
"avgDuration": 31661,
|
||||
"avgDuration": 31578,
|
||||
"testCount": 9,
|
||||
"flakyRate": 0.0003
|
||||
"flakyRate": 0.0006
|
||||
},
|
||||
"tests/e2e/workflows/editor/canvas/actions.spec.ts": {
|
||||
"avgDuration": 78851,
|
||||
"testCount": 21,
|
||||
"flakyRate": 0.0004
|
||||
"avgDuration": 80903,
|
||||
"testCount": 20,
|
||||
"flakyRate": 0.0005
|
||||
},
|
||||
"tests/e2e/workflows/editor/canvas/stickies.spec.ts": {
|
||||
"avgDuration": 2919,
|
||||
"avgDuration": 2962,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0.0008
|
||||
"flakyRate": 0.0015
|
||||
},
|
||||
"tests/e2e/regression/CAT-726-canvas-node-connectors-not-rendered-when-nodes-inserted.spec.ts": {
|
||||
"avgDuration": 4896,
|
||||
"avgDuration": 4951,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/app-config/become-creator.spec.ts": {
|
||||
"avgDuration": 4615,
|
||||
"avgDuration": 5583,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/chat-hub/chat-hub-basic.spec.ts": {
|
||||
"avgDuration": 104021,
|
||||
"avgDuration": 99260,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0351
|
||||
"flakyRate": 0.0319
|
||||
},
|
||||
"tests/e2e/auth/authenticated.spec.ts": {
|
||||
"avgDuration": 14158,
|
||||
"avgDuration": 13964,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/auth/admin-smoke.spec.ts": {
|
||||
"avgDuration": 2227,
|
||||
"avgDuration": 2237,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/regression/AI-812-partial-execs-broken-when-using-chat-trigger.spec.ts": {
|
||||
"avgDuration": 8134,
|
||||
"avgDuration": 8215,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0004
|
||||
"flakyRate": 0.0011
|
||||
},
|
||||
"tests/e2e/regression/AI-716-correctly-set-up-agent-model-shows-error.spec.ts": {
|
||||
"avgDuration": 4593,
|
||||
"avgDuration": 4666,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/regression/AI-1401-sub-nodes-input-panel.spec.ts": {
|
||||
"avgDuration": 4871,
|
||||
"avgDuration": 4941,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/ai/assistant-basic.spec.ts": {
|
||||
"avgDuration": 54391,
|
||||
"avgDuration": 55527,
|
||||
"testCount": 11,
|
||||
"flakyRate": 0.0001
|
||||
},
|
||||
"tests/e2e/ai/assistant-support-chat.spec.ts": {
|
||||
"avgDuration": 11897,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0003
|
||||
},
|
||||
"tests/e2e/ai/assistant-support-chat.spec.ts": {
|
||||
"avgDuration": 13201,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0.0002
|
||||
},
|
||||
"tests/e2e/ai/assistant-credential-help.spec.ts": {
|
||||
"avgDuration": 17717,
|
||||
"avgDuration": 18891,
|
||||
"testCount": 4,
|
||||
"flakyRate": 0.0008
|
||||
"flakyRate": 0.0007
|
||||
},
|
||||
"tests/e2e/ai/assistant-code-help.spec.ts": {
|
||||
"avgDuration": 10551,
|
||||
"avgDuration": 12021,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/regression/ADO-2929-can-load-old-switch-node-workflows.spec.ts": {
|
||||
"avgDuration": 4492,
|
||||
"avgDuration": 4430,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/regression/ADO-2372-prevent-clipping-params.spec.ts": {
|
||||
"avgDuration": 8223,
|
||||
"avgDuration": 8217,
|
||||
"testCount": 2,
|
||||
"flakyRate": 0.0004
|
||||
},
|
||||
"tests/e2e/regression/ADO-2270-opening-webhook-ndv-marks-workflow-as-unsaved.spec.ts": {
|
||||
"avgDuration": 3757,
|
||||
"avgDuration": 3801,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
"flakyRate": 0.0015
|
||||
},
|
||||
"tests/e2e/regression/ADO-2230-ndv-reset-data-pagination.spec.ts": {
|
||||
"avgDuration": 3744,
|
||||
"avgDuration": 3794,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
},
|
||||
"tests/e2e/regression/ADO-1338-ndv-missing-input-panel.spec.ts": {
|
||||
"avgDuration": 9737,
|
||||
"avgDuration": 9604,
|
||||
"testCount": 1,
|
||||
"flakyRate": 0
|
||||
}
|
||||
|
|
|
|||
2
.github/workflows/playwright-test-ci.yml
vendored
2
.github/workflows/playwright-test-ci.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
branch: ${{ inputs.branch }}
|
||||
test-mode: docker-build
|
||||
test-command: pnpm --filter=n8n-playwright test:container:multi-main:e2e
|
||||
shards: 13
|
||||
shards: 14
|
||||
runner: blacksmith-2vcpu-ubuntu-2204
|
||||
workers: '1'
|
||||
use-custom-orchestration: true
|
||||
|
|
|
|||
13
.github/workflows/playwright-test-reusable.yml
vendored
13
.github/workflows/playwright-test-reusable.yml
vendored
|
|
@ -71,16 +71,19 @@ env:
|
|||
|
||||
jobs:
|
||||
matrix:
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
outputs:
|
||||
matrix: ${{ steps.generate.outputs.matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github/test-metrics
|
||||
packages/testing/playwright/scripts
|
||||
sparse-checkout-cone-mode: false
|
||||
ref: ${{ inputs.branch || github.ref }}
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Environment
|
||||
uses: ./.github/actions/setup-nodejs
|
||||
with:
|
||||
build-command: ''
|
||||
|
||||
- name: Generate shard matrix
|
||||
id: generate
|
||||
|
|
|
|||
|
|
@ -1,190 +1,102 @@
|
|||
# Custom Test Orchestration
|
||||
|
||||
Duration-based test distribution across CI shards, using committed metrics for deterministic runs.
|
||||
Capability-aware test distribution across CI shards.
|
||||
|
||||
## Overview
|
||||
## How It Works
|
||||
|
||||
Instead of Playwright's built-in sharding (which distributes by file count), this approach distributes specs by **estimated duration** using a greedy bin-packing algorithm. This results in more balanced shard execution times.
|
||||
| Step | What Happens |
|
||||
|------|--------------|
|
||||
| 1. Discovery | `pnpm playwright test --list --project="multi-main:e2e"` |
|
||||
| 2. Metrics | Get `avgDuration` per spec from Currents (last 30 days) |
|
||||
| 3. Default | Missing specs get **60s** default (accounts for container startup) |
|
||||
| 4. Group | Group specs by `@capability:xxx` tag for worker reuse |
|
||||
| 5. Effective Duration | Calculate actual time accounting for container reuse within groups |
|
||||
| 6. Split | If a group exceeds **5 min**, split into sub-groups |
|
||||
| 7. Bin Pack | Greedy assign groups + standard specs to lightest shard |
|
||||
|
||||
**Key properties:**
|
||||
- **Deterministic** - Same commit always produces same distribution
|
||||
- **Re-run safe** - Failed jobs re-run the same specs (no full suite re-run)
|
||||
- **Community PR friendly** - No secrets needed at runtime
|
||||
- **Source agnostic** - Metrics can come from any test analytics provider
|
||||
### Why Group by Capability?
|
||||
|
||||
## Scripts
|
||||
Tests requiring containers (proxy, email, etc.) include ~20s startup overhead. When grouped on the same shard, only the first test pays this cost - the rest reuse the worker.
|
||||
|
||||
Located in `packages/testing/playwright/scripts/`:
|
||||
**Example:** 15 proxy tests across 8 shards = 8 container starts (160s). Grouped on 2 shards = 2 starts (40s). **Saves 120s.**
|
||||
|
||||
| Script | Purpose |
|
||||
|--------|---------|
|
||||
| `distribute-tests.mjs` | Assigns specs to shards using bin-packing |
|
||||
| `fetch-currents-metrics.mjs` | Fetches duration data from Currents API |
|
||||
### Self-Balancing
|
||||
|
||||
## Metrics File
|
||||
Metrics auto-correct over time. As grouped tests run, they report actual execution time (not startup overhead), so future distributions become more accurate.
|
||||
|
||||
Committed at `.github/test-metrics/playwright.json`
|
||||
## Writing Tests with Capabilities
|
||||
|
||||
### Schema
|
||||
### 1. Import shared capabilities (required for worker reuse)
|
||||
|
||||
```json
|
||||
{
|
||||
"updatedAt": "2025-01-15T00:00:00.000Z",
|
||||
"source": "currents | playwright | manual | <provider>",
|
||||
"specs": {
|
||||
"tests/e2e/path/to/spec.ts": {
|
||||
"avgDuration": 45000,
|
||||
"testCount": 5,
|
||||
"flakyRate": 0.02
|
||||
}
|
||||
}
|
||||
}
|
||||
```typescript
|
||||
// fixtures/capabilities.ts has shared objects
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
|
||||
// CORRECT - same object reference enables worker reuse
|
||||
test.use({ addContainerCapability: capabilities.proxy });
|
||||
|
||||
// WRONG - inline object breaks worker reuse
|
||||
test.use({ addContainerCapability: { proxyServerEnabled: true } });
|
||||
```
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `updatedAt` | ISO 8601 | When metrics were last refreshed |
|
||||
| `source` | string | Where metrics originated |
|
||||
| `specs` | object | Map of spec path to metrics |
|
||||
| `specs[path].avgDuration` | number | Average duration in milliseconds |
|
||||
| `specs[path].testCount` | number | Number of tests in spec |
|
||||
| `specs[path].flakyRate` | number | Flakiness rate (0-1) |
|
||||
### 2. Add @capability tag (required for orchestration grouping)
|
||||
|
||||
Only `avgDuration` is required for distribution. Other fields are informational.
|
||||
```typescript
|
||||
test('My feature @capability:proxy', async ({ page }) => {
|
||||
// This test will be grouped with other proxy tests
|
||||
});
|
||||
|
||||
## CI Usage
|
||||
|
||||
### Enabling Custom Orchestration
|
||||
|
||||
In the workflow call to `playwright-test-reusable.yml`:
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
e2e-tests:
|
||||
uses: ./.github/workflows/playwright-test-reusable.yml
|
||||
with:
|
||||
test-command: pnpm --filter=n8n-playwright test:local
|
||||
shards: 8
|
||||
use-custom-orchestration: true # Enable duration-based distribution
|
||||
secrets: inherit
|
||||
// Or at describe level:
|
||||
test.describe('Feature @capability:email', () => {
|
||||
// All tests inherit the tag
|
||||
});
|
||||
```
|
||||
|
||||
### How It Works
|
||||
### Available Capabilities
|
||||
|
||||
When `use-custom-orchestration: true`:
|
||||
|
||||
```bash
|
||||
# Each shard runs:
|
||||
SPECS=$(node packages/testing/playwright/scripts/distribute-tests.mjs $TOTAL_SHARDS $SHARD_INDEX)
|
||||
pnpm test:local --workers=2 $SPECS
|
||||
```
|
||||
|
||||
The distribute script:
|
||||
1. Reads committed metrics from `.github/test-metrics/playwright.json`
|
||||
2. Sorts specs by duration (descending)
|
||||
3. Assigns each spec to the lightest shard (greedy bin-packing)
|
||||
4. Outputs space-separated spec paths for the requested shard
|
||||
|
||||
### Distribution Algorithm
|
||||
|
||||
```
|
||||
Input: specs sorted by duration [100s, 80s, 60s, 40s, 30s, 20s]
|
||||
Shards: 3
|
||||
|
||||
Step 1: 100s → Shard 0 (lightest) → [100, 0, 0]
|
||||
Step 2: 80s → Shard 1 (lightest) → [100, 80, 0]
|
||||
Step 3: 60s → Shard 2 (lightest) → [100, 80, 60]
|
||||
Step 4: 40s → Shard 2 (lightest) → [100, 80, 100]
|
||||
Step 5: 30s → Shard 1 (lightest) → [100, 110, 100]
|
||||
Step 6: 20s → Shard 0 (lightest) → [120, 110, 100]
|
||||
|
||||
Result: Balanced ~110s per shard instead of uneven distribution
|
||||
```
|
||||
| Import | Tag | Container |
|
||||
|--------|-----|-----------|
|
||||
| `capabilities.proxy` | `@capability:proxy` | Proxy server |
|
||||
| `capabilities.email` | `@capability:email` | Mailpit |
|
||||
| `capabilities.sourceControl` | `@capability:source-control` | Git server |
|
||||
| `capabilities.taskRunner` | `@capability:task-runner` | Task runner |
|
||||
| `capabilities.oidc` | `@capability:oidc` | OIDC provider |
|
||||
| `capabilities.observability` | `@capability:observability` | VictoriaLogs |
|
||||
|
||||
## Refreshing Metrics
|
||||
|
||||
### Using Currents API
|
||||
|
||||
```bash
|
||||
CURRENTS_API_KEY=<key> node packages/testing/playwright/scripts/fetch-currents-metrics.mjs --project=<id>
|
||||
```
|
||||
|
||||
The script:
|
||||
1. Fetches test durations from Currents API (last 30 days)
|
||||
2. Aggregates by spec file
|
||||
3. Validates against `pnpm playwright test --list --project="standard:e2e"`
|
||||
4. Reports drift (stale specs removed, new specs added with 30s default)
|
||||
5. Writes to `.github/test-metrics/playwright.json`
|
||||
This fetches the last 30 days of test durations from Currents, aggregates by spec, and writes to `.github/test-metrics/playwright.json`.
|
||||
|
||||
### Using Other Sources
|
||||
**When to refresh:**
|
||||
- Weekly (recommended)
|
||||
- After significant test changes
|
||||
- When adding new specs (optional - they get 60s default)
|
||||
|
||||
Create a script that outputs the same JSON schema. The distribution only requires:
|
||||
## Scripts
|
||||
|
||||
```json
|
||||
{
|
||||
"specs": {
|
||||
"tests/e2e/example.spec.ts": { "avgDuration": 45000 }
|
||||
}
|
||||
}
|
||||
| Script | Purpose |
|
||||
|--------|---------|
|
||||
| `scripts/distribute-tests.mjs` | Distributes specs across shards |
|
||||
| `scripts/fetch-currents-metrics.mjs` | Fetches metrics from Currents API |
|
||||
|
||||
### Testing Locally
|
||||
|
||||
```bash
|
||||
# See distribution for 14 shards
|
||||
node scripts/distribute-tests.mjs --matrix 14 --orchestrate
|
||||
|
||||
# Get specs for shard 0
|
||||
node scripts/distribute-tests.mjs 14 0
|
||||
```
|
||||
|
||||
### When to Refresh
|
||||
|
||||
- When new spec files are added (they get 30s default until refreshed)
|
||||
- When specs are deleted/renamed (stale entries are filtered out)
|
||||
- Periodically to capture duration changes (weekly recommended)
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Detecting Drift
|
||||
|
||||
Run the fetch script - it reports mismatches:
|
||||
|
||||
```
|
||||
Stale specs (in metrics but not Playwright):
|
||||
- tests/e2e/deleted/old.spec.ts
|
||||
|
||||
New specs (in Playwright but not metrics, using 30s default):
|
||||
- tests/e2e/new/feature.spec.ts
|
||||
```
|
||||
|
||||
### Manual Metrics Entry
|
||||
|
||||
For new specs before CI data exists:
|
||||
|
||||
```json
|
||||
{
|
||||
"specs": {
|
||||
"tests/e2e/new/feature.spec.ts": {
|
||||
"avgDuration": 45000,
|
||||
"testCount": 3,
|
||||
"flakyRate": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Estimate duration based on similar specs, or use 30000 (30s) as default.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Specs not running
|
||||
|
||||
Check that the spec path in `playwright.json` matches exactly what Playwright outputs:
|
||||
```bash
|
||||
pnpm --filter=n8n-playwright playwright test --list --project="standard:e2e"
|
||||
```
|
||||
|
||||
### Unbalanced shards
|
||||
|
||||
Refresh metrics - durations may have changed significantly since last update.
|
||||
|
||||
### Script errors
|
||||
|
||||
```bash
|
||||
# Test distribution locally
|
||||
node packages/testing/playwright/scripts/distribute-tests.mjs 8 0
|
||||
|
||||
# Validate metrics file
|
||||
cat .github/test-metrics/playwright.json | jq '.specs | length'
|
||||
```
|
||||
| Problem | Solution |
|
||||
|---------|----------|
|
||||
| Specs not running | Check path matches `playwright test --list` output |
|
||||
| Unbalanced shards | Refresh metrics - durations may have drifted |
|
||||
| Worker not reused | Use imported `capabilities.xxx`, not inline objects |
|
||||
|
|
|
|||
34
packages/testing/playwright/fixtures/capabilities.ts
Normal file
34
packages/testing/playwright/fixtures/capabilities.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
/**
|
||||
* Shared container capability configurations.
|
||||
*
|
||||
* IMPORTANT: Import these instead of defining inline objects in test.use().
|
||||
* Using the same object reference enables Playwright worker reuse across
|
||||
* test files with identical configurations, avoiding redundant container creation.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* import { capabilities } from '../fixtures/capabilities';
|
||||
* test.use({ addContainerCapability: capabilities.email });
|
||||
* ```
|
||||
*/
|
||||
export const capabilities = {
|
||||
/** Email testing with Mailpit SMTP server */
|
||||
email: { email: true },
|
||||
|
||||
/** Mock HTTP server for external API testing */
|
||||
proxy: { proxyServerEnabled: true },
|
||||
|
||||
/** Git-based source control testing */
|
||||
sourceControl: { sourceControl: true },
|
||||
|
||||
/** External task runner container */
|
||||
taskRunner: { taskRunner: true },
|
||||
|
||||
/** OIDC/SSO testing with Keycloak (includes postgres) */
|
||||
oidc: { oidc: true },
|
||||
|
||||
/** Observability stack (VictoriaLogs + VictoriaMetrics) */
|
||||
observability: { observability: true },
|
||||
} as const;
|
||||
|
||||
export type CapabilityName = keyof typeof capabilities;
|
||||
|
|
@ -2,7 +2,16 @@
|
|||
// @ts-check
|
||||
|
||||
/**
|
||||
* Distributes Playwright specs across shards using greedy bin-packing.
|
||||
* Distributes Playwright specs across shards using capability-aware bin-packing.
|
||||
*
|
||||
* Algorithm:
|
||||
* 1. Group specs by capability (proxy, source-control, email, etc.)
|
||||
* 2. Calculate effective duration (accounts for container startup reuse)
|
||||
* 3. Split large groups (>5 min) into smaller sub-groups
|
||||
* 4. Assign groups + standard specs using greedy bin-packing
|
||||
*
|
||||
* This minimizes container startup overhead by keeping tests that need the same
|
||||
* container on the same shard, enabling Playwright worker reuse.
|
||||
*
|
||||
* Usage:
|
||||
* node distribute-tests.mjs <shards> <index> # Output specs for single shard
|
||||
|
|
@ -10,6 +19,7 @@
|
|||
* node distribute-tests.mjs --matrix <shards> --orchestrate # Output JSON matrix with distributed specs
|
||||
*/
|
||||
|
||||
import { execSync } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
|
@ -18,7 +28,10 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|||
const ROOT_DIR = path.resolve(__dirname, '../../../..');
|
||||
const METRICS_PATH = path.join(ROOT_DIR, '.github/test-metrics/playwright.json');
|
||||
const PLAYWRIGHT_DIR = path.resolve(__dirname, '..');
|
||||
const DEFAULT_DURATION = 30000;
|
||||
const DEFAULT_DURATION = 60000; // 1 minute default (accounts for container startup)
|
||||
const E2E_PROJECT = 'multi-main:e2e';
|
||||
const CONTAINER_STARTUP_TIME = 20000; // 20 seconds per container type
|
||||
const MAX_GROUP_DURATION = 5 * 60 * 1000; // 5 minutes - split groups larger than this
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const matrixMode = args.includes('--matrix');
|
||||
|
|
@ -32,31 +45,246 @@ if (!shards || shards < 1) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Distribute specs using greedy bin-packing
|
||||
* Get spec files and their capabilities from Playwright test --list output
|
||||
* @returns {{path: string, capabilities: string[]}[]} Array of spec info
|
||||
*/
|
||||
function getSpecsFromPlaywright() {
|
||||
const output = execSync(`pnpm playwright test --list --project="${E2E_PROJECT}"`, {
|
||||
cwd: PLAYWRIGHT_DIR,
|
||||
encoding: 'utf-8',
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
// Parse output: "[multi-main:e2e] › tests/e2e/path/spec.ts:line:col › describe @capability:xxx › test"
|
||||
/** @type {Map<string, Set<string>>} */
|
||||
const specCapabilities = new Map();
|
||||
|
||||
for (const line of output.split('\n')) {
|
||||
const specMatch = line.match(/› (tests\/e2e\/[^:]+\.spec\.ts):/);
|
||||
if (specMatch) {
|
||||
const specPath = specMatch[1];
|
||||
if (!specCapabilities.has(specPath)) {
|
||||
specCapabilities.set(specPath, new Set());
|
||||
}
|
||||
|
||||
// Extract @capability:xxx tags from the test description
|
||||
const capMatches = line.matchAll(/@capability:(\w+[-\w]*)/g);
|
||||
for (const match of capMatches) {
|
||||
specCapabilities.get(specPath)?.add(match[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...specCapabilities.entries()].map(([path, caps]) => ({
|
||||
path,
|
||||
capabilities: [...caps],
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Distribute specs using capability-aware bin-packing
|
||||
*
|
||||
* Key insight: Reported test durations include container startup overhead.
|
||||
* When tests are grouped (same capability on same shard), only ONE test pays
|
||||
* the startup cost - the rest reuse the worker. So we:
|
||||
*
|
||||
* 1. Calculate "effective duration" for capability groups (removes redundant startup)
|
||||
* 2. Treat each capability group as an atomic unit for bin-packing
|
||||
* 3. Use greedy bin-packing on groups + standard specs together
|
||||
*
|
||||
* @param {number} numShards
|
||||
*/
|
||||
function distributeCapabilityAware(numShards) {
|
||||
const metrics = JSON.parse(fs.readFileSync(METRICS_PATH, 'utf-8'));
|
||||
const allSpecs = getSpecsFromPlaywright();
|
||||
|
||||
if (allSpecs.length === 0) {
|
||||
console.error('Error: No spec files found. Check Playwright config and project name.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Add duration info
|
||||
const specsWithDuration = allSpecs.map((spec) => ({
|
||||
...spec,
|
||||
duration: metrics.specs?.[spec.path]?.avgDuration || DEFAULT_DURATION,
|
||||
}));
|
||||
|
||||
// Group specs by capability (specs can only have one capability in practice)
|
||||
/** @type {Map<string, typeof specsWithDuration>} */
|
||||
const capabilityGroups = new Map();
|
||||
/** @type {typeof specsWithDuration} */
|
||||
const standardSpecs = [];
|
||||
|
||||
for (const spec of specsWithDuration) {
|
||||
if (spec.capabilities.length > 0) {
|
||||
// Use first capability as the grouping key
|
||||
const cap = spec.capabilities[0];
|
||||
if (!capabilityGroups.has(cap)) {
|
||||
capabilityGroups.set(cap, []);
|
||||
}
|
||||
capabilityGroups.get(cap)?.push(spec);
|
||||
} else {
|
||||
standardSpecs.push(spec);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate effective durations for capability groups
|
||||
// When grouped, only first test pays container startup - rest reuse worker
|
||||
// If a group exceeds MAX_GROUP_DURATION, split it into smaller sub-groups
|
||||
/** @type {Array<{type: string, capability: string, specs: string[], reportedDuration: number, effectiveDuration: number, startupSavings: number, subGroup?: number}>} */
|
||||
const capabilityItems = [];
|
||||
|
||||
for (const [capability, specs] of capabilityGroups.entries()) {
|
||||
// Sort specs by duration (largest first) for better splitting
|
||||
specs.sort((a, b) => b.duration - a.duration);
|
||||
|
||||
const reportedDuration = specs.reduce((sum, s) => sum + s.duration, 0);
|
||||
// Calculate effective duration: each spec's actual time (minus startup) + one startup
|
||||
const actualTestTime = reportedDuration - specs.length * CONTAINER_STARTUP_TIME;
|
||||
const effectiveDuration = actualTestTime + CONTAINER_STARTUP_TIME;
|
||||
|
||||
// Check if we need to split this group
|
||||
if (effectiveDuration > MAX_GROUP_DURATION && specs.length > 1) {
|
||||
// Calculate how many sub-groups we need
|
||||
const numSubGroups = Math.ceil(effectiveDuration / MAX_GROUP_DURATION);
|
||||
const targetPerSubGroup = effectiveDuration / numSubGroups;
|
||||
|
||||
// Greedy split: fill each sub-group up to target
|
||||
let currentSubGroup = 0;
|
||||
let currentTotal = 0;
|
||||
/** @type {typeof specs[]} */
|
||||
const subGroups = [[]];
|
||||
|
||||
for (const spec of specs) {
|
||||
const specActualTime = spec.duration - CONTAINER_STARTUP_TIME;
|
||||
|
||||
// Start new sub-group if current is full (unless it's empty)
|
||||
if (currentTotal + specActualTime > targetPerSubGroup && subGroups[currentSubGroup].length > 0) {
|
||||
currentSubGroup++;
|
||||
subGroups[currentSubGroup] = [];
|
||||
currentTotal = 0;
|
||||
}
|
||||
|
||||
subGroups[currentSubGroup].push(spec);
|
||||
currentTotal += specActualTime;
|
||||
}
|
||||
|
||||
// Create items for each sub-group
|
||||
for (let i = 0; i < subGroups.length; i++) {
|
||||
const subGroupSpecs = subGroups[i];
|
||||
const subReported = subGroupSpecs.reduce((sum, s) => sum + s.duration, 0);
|
||||
const subActual = subReported - subGroupSpecs.length * CONTAINER_STARTUP_TIME;
|
||||
const subEffective = subActual + CONTAINER_STARTUP_TIME; // Each sub-group pays one startup
|
||||
|
||||
capabilityItems.push({
|
||||
type: 'capability',
|
||||
capability,
|
||||
specs: subGroupSpecs.map((s) => s.path),
|
||||
reportedDuration: subReported,
|
||||
effectiveDuration: subEffective,
|
||||
startupSavings: (subGroupSpecs.length - 1) * CONTAINER_STARTUP_TIME,
|
||||
subGroup: i + 1,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Keep as single group
|
||||
capabilityItems.push({
|
||||
type: 'capability',
|
||||
capability,
|
||||
specs: specs.map((s) => s.path),
|
||||
reportedDuration,
|
||||
effectiveDuration,
|
||||
startupSavings: (specs.length - 1) * CONTAINER_STARTUP_TIME,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Standard specs keep their reported duration (no grouping benefit)
|
||||
const standardItems = standardSpecs.map((spec) => ({
|
||||
type: 'standard',
|
||||
capability: null,
|
||||
specs: [spec.path],
|
||||
reportedDuration: spec.duration,
|
||||
effectiveDuration: spec.duration,
|
||||
startupSavings: 0,
|
||||
}));
|
||||
|
||||
// Combine and sort by effective duration (largest first for greedy packing)
|
||||
const allItems = [...capabilityItems, ...standardItems].sort(
|
||||
(a, b) => b.effectiveDuration - a.effectiveDuration,
|
||||
);
|
||||
|
||||
// Calculate totals for reporting
|
||||
const totalEffective = allItems.reduce((sum, item) => sum + item.effectiveDuration, 0);
|
||||
const totalReported = allItems.reduce((sum, item) => sum + item.reportedDuration, 0);
|
||||
const totalSavings = capabilityItems.reduce((sum, item) => sum + item.startupSavings, 0);
|
||||
const targetPerShard = totalEffective / numShards;
|
||||
|
||||
console.error('\n📦 Capability-Aware Distribution:');
|
||||
console.error(` Reported total: ${(totalReported / 60000).toFixed(1)} min`);
|
||||
console.error(` Effective total: ${(totalEffective / 60000).toFixed(1)} min (after grouping)`);
|
||||
console.error(` Worker reuse savings: ${(totalSavings / 1000).toFixed(0)}s`);
|
||||
console.error(` Target per shard: ${(targetPerShard / 60000).toFixed(1)} min\n`);
|
||||
|
||||
// Report capability groups
|
||||
console.error(' Capability groups (treated as atomic units):');
|
||||
for (const item of capabilityItems) {
|
||||
const reported = (item.reportedDuration / 60000).toFixed(1);
|
||||
const effective = (item.effectiveDuration / 60000).toFixed(1);
|
||||
const saved = (item.startupSavings / 1000).toFixed(0);
|
||||
console.error(` ${item.capability}: ${item.specs.length} specs, ${reported} min → ${effective} min (saved ${saved}s)`);
|
||||
}
|
||||
console.error(` Standard specs: ${standardItems.length} specs\n`);
|
||||
|
||||
// Initialize buckets
|
||||
/** @type {Array<{specs: string[]; total: number; capabilities: Set<string>}>} */
|
||||
const buckets = Array.from({ length: numShards }, () => ({
|
||||
specs: [],
|
||||
total: 0,
|
||||
capabilities: new Set(),
|
||||
}));
|
||||
|
||||
// Greedy bin-packing: assign each item to the lightest bucket
|
||||
for (const item of allItems) {
|
||||
const lightest = buckets.reduce((min, b) => (b.total < min.total ? b : min));
|
||||
|
||||
// Add all specs from this item to the bucket
|
||||
for (const specPath of item.specs) {
|
||||
lightest.specs.push(specPath);
|
||||
}
|
||||
lightest.total += item.effectiveDuration;
|
||||
|
||||
if (item.capability) {
|
||||
lightest.capabilities.add(item.capability);
|
||||
}
|
||||
}
|
||||
|
||||
// Report container optimization
|
||||
const simpleContainers = capabilityItems.reduce((sum, item) => {
|
||||
// Estimate: with simple distribution, specs spread across ~70% of shards
|
||||
return sum + Math.min(numShards, Math.ceil(item.specs.length * 0.7));
|
||||
}, 0);
|
||||
|
||||
const optimizedContainers = capabilityItems.reduce((sum, item) => {
|
||||
// Count actual shards with this capability
|
||||
return sum + buckets.filter((b) => b.capabilities.has(item.capability)).length;
|
||||
}, 0);
|
||||
|
||||
console.error(' Container optimization:');
|
||||
console.error(` Simple distribution: ~${simpleContainers} container starts`);
|
||||
console.error(` Capability-aware: ${optimizedContainers} container starts`);
|
||||
console.error(` Containers saved: ~${simpleContainers - optimizedContainers}`);
|
||||
console.error(` Total time saved: ~${((totalSavings + (simpleContainers - optimizedContainers) * CONTAINER_STARTUP_TIME) / 1000).toFixed(0)}s\n`);
|
||||
|
||||
return buckets;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main distribution function
|
||||
* @param {number} numShards
|
||||
*/
|
||||
function distribute(numShards) {
|
||||
const metrics = JSON.parse(fs.readFileSync(METRICS_PATH, 'utf-8'));
|
||||
const pattern = path.join(PLAYWRIGHT_DIR, 'tests/**/*.spec.ts');
|
||||
const allSpecs = fs.globSync(pattern).map((fullPath) => path.relative(PLAYWRIGHT_DIR, fullPath));
|
||||
|
||||
const specs = allSpecs
|
||||
.map((specPath) => ({
|
||||
path: specPath,
|
||||
duration: metrics[specPath]?.avgDuration || DEFAULT_DURATION,
|
||||
}))
|
||||
.sort((a, b) => b.duration - a.duration);
|
||||
|
||||
/**
|
||||
* @type Array<{specs: string[]; total:number}>
|
||||
*/
|
||||
const buckets = Array.from({ length: numShards }, () => ({ specs: [], total: 0 }));
|
||||
for (const spec of specs) {
|
||||
const lightest = buckets.reduce((min, b) => (b.total < min.total ? b : min));
|
||||
lightest.specs.push(spec.path);
|
||||
lightest.total += spec.duration;
|
||||
}
|
||||
return buckets;
|
||||
return distributeCapabilityAware(numShards);
|
||||
}
|
||||
|
||||
if (matrixMode) {
|
||||
|
|
@ -65,6 +293,18 @@ if (matrixMode) {
|
|||
shard: i + 1,
|
||||
specs: orchestrate ? (buckets?.[i].specs.join(' ') ?? '') : '',
|
||||
}));
|
||||
|
||||
if (orchestrate && buckets) {
|
||||
console.error('\n📊 Shard Distribution:');
|
||||
for (let i = 0; i < buckets.length; i++) {
|
||||
const mins = (buckets[i].total / 60000).toFixed(1);
|
||||
const caps = buckets[i].capabilities.size > 0 ? ` [${[...buckets[i].capabilities].join(', ')}]` : '';
|
||||
console.error(` Shard ${i + 1}: ${buckets[i].specs.length} specs, ~${mins} min${caps}`);
|
||||
}
|
||||
const totalMins = (buckets.reduce((sum, b) => sum + b.total, 0) / 60000).toFixed(1);
|
||||
console.error(` Total: ${totalMins} min across ${shards} shards\n`);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(matrix));
|
||||
} else {
|
||||
const index = parseInt(args[1]);
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ const PLAYWRIGHT_DIR = path.join(ROOT_DIR, 'packages', 'testing', 'playwright');
|
|||
const OUTPUT_PATH = path.join(ROOT_DIR, '.github', 'test-metrics', 'playwright.json');
|
||||
|
||||
const CURRENTS_API = 'https://api.currents.dev/v1';
|
||||
const DEFAULT_DURATION = 30000;
|
||||
const DEFAULT_DURATION = 60000; // 1 minute default for new specs (accounts for container startup)
|
||||
|
||||
const PROJECT_ID = process.argv.find((a) => a.startsWith('--project='))?.split('=')[1];
|
||||
if (!PROJECT_ID) {
|
||||
|
|
@ -85,7 +85,7 @@ function aggregateBySpec(tests) {
|
|||
function getPlaywrightSpecs() {
|
||||
console.log('Getting specs from Playwright...');
|
||||
try {
|
||||
const output = execSync('pnpm playwright test --list --project="standard:e2e"', {
|
||||
const output = execSync('pnpm playwright test --list --project="multi-main:e2e"', {
|
||||
cwd: PLAYWRIGHT_DIR,
|
||||
encoding: 'utf-8',
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
import { expect, test } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
proxyServerEnabled: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.proxy });
|
||||
|
||||
test.describe('Evaluations @capability:proxy', () => {
|
||||
test.beforeEach(async ({ n8n, proxyServer }) => {
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
SCHEDULE_TRIGGER_NODE_NAME,
|
||||
} from '../../../config/constants';
|
||||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
import type { n8nPage } from '../../../pages/n8nPage';
|
||||
|
||||
// Helper functions for common operations
|
||||
|
|
@ -74,11 +75,7 @@ async function setupBasicAgentWorkflow(n8n: n8nPage, additionalNodes: string[] =
|
|||
await addOpenAILanguageModelWithCredentials(n8n, AGENT_NODE_NAME);
|
||||
}
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
proxyServerEnabled: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.proxy });
|
||||
test.describe('Langchain Integration @capability:proxy', () => {
|
||||
test.beforeEach(async ({ n8n, proxyServer }) => {
|
||||
await proxyServer.clearAllExpectations();
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
MANUAL_CHAT_TRIGGER_NODE_NAME,
|
||||
} from '../../../config/constants';
|
||||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
import type { n8nPage } from '../../../pages/n8nPage';
|
||||
|
||||
// Helper functions for common operations
|
||||
|
|
@ -38,11 +39,7 @@ async function executeChatAndWaitForResponse(n8n: n8nPage, message: string) {
|
|||
await waitForWorkflowSuccess(n8n);
|
||||
}
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
proxyServerEnabled: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.proxy });
|
||||
test.describe('Langchain Integration @capability:proxy', () => {
|
||||
test.beforeEach(async ({ n8n, proxyServer }) => {
|
||||
await proxyServer.clearAllExpectations();
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
MANUAL_CHAT_TRIGGER_NODE_NAME,
|
||||
} from '../../../config/constants';
|
||||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
import type { n8nPage } from '../../../pages/n8nPage';
|
||||
|
||||
// Helper functions for common operations
|
||||
|
|
@ -48,11 +49,7 @@ async function verifyChatMessages(n8n: n8nPage, expectedCount: number, inputMess
|
|||
return messages;
|
||||
}
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
proxyServerEnabled: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.proxy });
|
||||
test.describe('Langchain Integration @capability:proxy', () => {
|
||||
test.beforeEach(async ({ n8n, proxyServer }) => {
|
||||
await proxyServer.clearAllExpectations();
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
MANUAL_CHAT_TRIGGER_NODE_NAME,
|
||||
} from '../../../config/constants';
|
||||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
import type { n8nPage } from '../../../pages/n8nPage';
|
||||
|
||||
// Helper functions for common operations
|
||||
|
|
@ -51,11 +52,7 @@ async function setupBasicAgentWorkflow(n8n: n8nPage, additionalNodes: string[] =
|
|||
await addOpenAILanguageModelWithCredentials(n8n, AGENT_NODE_NAME);
|
||||
}
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
proxyServerEnabled: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.proxy });
|
||||
test.describe('Langchain Integration @capability:proxy', () => {
|
||||
test.beforeEach(async ({ n8n, proxyServer }) => {
|
||||
await proxyServer.clearAllExpectations();
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
import type { n8nPage } from '../../../pages/n8nPage';
|
||||
|
||||
// Helper functions for common operations
|
||||
|
|
@ -8,11 +9,7 @@ async function waitForWorkflowSuccess(n8n: n8nPage, timeout = 10000) {
|
|||
});
|
||||
}
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
proxyServerEnabled: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.proxy });
|
||||
test.describe('Langchain Integration @capability:proxy', () => {
|
||||
test.beforeEach(async ({ n8n, proxyServer }) => {
|
||||
await proxyServer.clearAllExpectations();
|
||||
|
|
|
|||
|
|
@ -6,11 +6,10 @@ import {
|
|||
} from 'n8n-containers';
|
||||
|
||||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
oidc: true,
|
||||
},
|
||||
addContainerCapability: capabilities.oidc,
|
||||
ignoreHTTPSErrors: true, // Keycloak uses self-signed certs
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
|
||||
test.use({ addContainerCapability: { email: true } });
|
||||
test.use({ addContainerCapability: capabilities.email });
|
||||
|
||||
test('Password reset email is delivered @capability:email', async ({ api, chaos }) => {
|
||||
const ownerEmail = 'nathan@n8n.io';
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
import assert from 'node:assert';
|
||||
|
||||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
proxyServerEnabled: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.proxy });
|
||||
// @capability:proxy tag ensures that test suite is only run when proxy is available
|
||||
test.describe('Proxy server @capability:proxy', () => {
|
||||
test.beforeEach(async ({ proxyServer }) => {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
import { CODE_NODE_NAME, MANUAL_TRIGGER_NODE_NAME } from '../../../config/constants';
|
||||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
taskRunner: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.taskRunner });
|
||||
|
||||
/**
|
||||
* Task Runner Capability Tests
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
|
||||
test.use({ addContainerCapability: { email: true } });
|
||||
test.use({ addContainerCapability: capabilities.email });
|
||||
|
||||
test('EmailSend node sends via SMTP @capability:email', async ({ api, n8n, chaos }) => {
|
||||
// Sign in to use internal APIs for creating credentials and workflows
|
||||
|
|
|
|||
|
|
@ -12,13 +12,10 @@
|
|||
import { SYSLOG_DEFAULTS, ObservabilityHelper } from 'n8n-containers';
|
||||
|
||||
import { test, expect } from '../../../../fixtures/base';
|
||||
import { capabilities } from '../../../../fixtures/capabilities';
|
||||
|
||||
// Worker-scoped fixtures must be at top level
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
observability: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.observability });
|
||||
|
||||
test.describe('Log Streaming to VictoriaLogs @capability:observability', () => {
|
||||
test.beforeEach(async ({ n8n }) => {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,10 @@
|
|||
import { MANUAL_TRIGGER_NODE_NAME } from '../../../config/constants';
|
||||
import { expect, test } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
import type { n8nPage } from '../../../pages/n8nPage';
|
||||
import { setupGitRepo } from '../../../utils/source-control-helper';
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
sourceControl: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.sourceControl });
|
||||
|
||||
async function expectPullSuccess(n8n: n8nPage) {
|
||||
expect(
|
||||
|
|
|
|||
|
|
@ -1,13 +1,10 @@
|
|||
import { MANUAL_TRIGGER_NODE_NAME } from '../../../config/constants';
|
||||
import { expect, test } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
import type { n8nPage } from '../../../pages/n8nPage';
|
||||
import { setupGitRepo } from '../../../utils/source-control-helper';
|
||||
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
sourceControl: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.sourceControl });
|
||||
|
||||
async function expectPushSuccess(n8n: n8nPage) {
|
||||
expect(
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
import { test, expect } from '../../../fixtures/base';
|
||||
import { capabilities } from '../../../fixtures/capabilities';
|
||||
|
||||
// Enable observability to use VictoriaLogs for log queries
|
||||
test.use({
|
||||
addContainerCapability: {
|
||||
observability: true,
|
||||
},
|
||||
});
|
||||
test.use({ addContainerCapability: capabilities.observability });
|
||||
|
||||
test('Leader election @mode:multi-main @chaostest @capability:observability', async ({ chaos }) => {
|
||||
// First get the container (try main 1 first)
|
||||
|
|
|
|||
|
|
@ -227,8 +227,9 @@ async function buildDockerImage({ name, dockerfilePath, fullImageName }) {
|
|||
${config.buildContext}`;
|
||||
echo(stdout);
|
||||
} else {
|
||||
// use docker command by default since most other engines have compatibility layers for it.
|
||||
const { stdout } = await $`docker build \
|
||||
// Use docker buildx build to leverage Blacksmith's layer caching when running in CI.
|
||||
// The setup-docker-builder action creates a buildx builder with sticky disk cache.
|
||||
const { stdout } = await $`docker buildx build \
|
||||
--platform ${platform} \
|
||||
--build-arg TARGETPLATFORM=${platform} \
|
||||
-t ${fullImageName} \
|
||||
|
|
|
|||
Loading…
Reference in a new issue