chore: update e2e test configs to better work with CI (#1458)

* Also adds a helpful test:e2e:ci script to test playwright tests inside of a docker image with a clean build to more closely resemble running in CI * Upgrades playwright version
2026-04-21 13:37:15 +00:00 · 2025-12-10 15:13:00 -05:00 · 2025-12-10 15:13:00 -05:00 · 8b153c076c
commit 8b153c076c
parent 776e392777
7 changed files with 100 additions and 30 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -103,7 +103,7 @@ jobs:
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    container:
-      image: mcr.microsoft.com/playwright:v1.55.0-jammy
+      image: mcr.microsoft.com/playwright:v1.57.0-jammy
    permissions:
      contents: read
      pull-requests: write
--- a/agent_docs/development.md
+++ b/agent_docs/development.md
@ -12,10 +12,14 @@ yarn dev

 ## Key Development Scripts

- `yarn app:dev`: Start API, frontend, alerts task, and common-utils in watch mode
+- `yarn app:dev`: Start API, frontend, alerts task, and common-utils in watch
+  mode
 - `yarn lint`: Run linting across all packages
 - `yarn dev:int`: Run integration tests in watch mode
 - `yarn dev:unit`: Run unit tests in watch mode (per package)
+- `yarn test:e2e`: Run Playwright E2E tests (in `packages/app`)
+- `yarn test:e2e:ci`: Run Playwright E2E tests in CI Docker environment (in
+  `packages/app`)

 ## Environment Configuration

@ -30,12 +34,14 @@ yarn dev
 - **Unit Tests**: Jest with TypeScript support
 - **Integration Tests**: Jest with database fixtures
 - **Frontend Testing**: React Testing Library + Jest
- **E2E Testing**: Custom smoke tests with BATS
+- **E2E Testing**: Playwright (frontend) and Custom smoke tests with BATS
+  (ingestion)

 ### Testing Patterns

 - **TDD Approach**: Write tests before implementation for new features
- **Test organization**: Tests co-located with source files in `__tests__/` directories
+- **Test organization**: Tests co-located with source files in `__tests__/`
+  directories
 - **Mocking**: MSW for API mocking in frontend tests
 - **Database testing**: Isolated test databases with fixtures

@ -52,6 +58,7 @@ yarn dev:int
 ```

 **CI Testing Notes:**
+
 - Uses separate Docker Compose configuration optimized for CI
 - Isolated test environment with `-p int` project name
 - Includes all necessary services (ClickHouse, MongoDB, OTel Collector)
@ -79,6 +86,7 @@ yarn dev:int
 ### Pre-commit Hooks

 The project uses Husky + lint-staged to automatically run:
+
 - Prettier for formatting
 - ESLint for linting
 - API doc generation (for external API changes)
@ -101,11 +109,11 @@ yarn run lint

 ## File Locations Quick Reference

- **Config**: `packages/api/src/config.ts`, `packages/app/next.config.mjs`, `docker-compose.dev.yml`
+- **Config**: `packages/api/src/config.ts`, `packages/app/next.config.mjs`,
+  `docker-compose.dev.yml`
 - **Models**: `packages/api/src/models/`
 - **API Routes**: `packages/api/src/routers/`
 - **Controllers**: `packages/api/src/controllers/`
 - **Pages**: `packages/app/pages/`
 - **Components**: `packages/app/src/`
 - **Shared Utils**: `packages/common-utils/src/`
-
--- a/packages/app/package.json
+++ b/packages/app/package.json
@ -20,6 +20,7 @@
    "test:e2e": "playwright test",
    "test:e2e:ui": "playwright test --ui",
    "test:e2e:debug": "playwright test --debug",
+    "test:e2e:ci": "../../scripts/test-e2e-ci.sh",
    "storybook": "storybook dev -p 6006",
    "storybook:build": "storybook build",
    "knip": "knip"
@ -107,7 +108,7 @@
    "@chromatic-com/storybook": "^4.1.3",
    "@hookform/devtools": "^4.3.1",
    "@jedmao/location": "^3.0.0",
-    "@playwright/test": "^1.47.0",
+    "@playwright/test": "^1.57.0",
    "@storybook/addon-docs": "^10.1.4",
    "@storybook/addon-links": "^10.1.4",
    "@storybook/addon-styling-webpack": "^3.0.0",
--- a/packages/app/playwright.config.ts
+++ b/packages/app/playwright.config.ts
@ -19,7 +19,9 @@ export default defineConfig({
  reporter: [
    ['html'],
    ['json', { outputFile: 'test-results/results.json' }],
-    ...(process.env.CI ? [['github', {}] as const] : []),
+    ...(process.env.CI
+      ? [['github'] as const, ['list'] as const]
+      : [['list'] as const]),
  ],
  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
  use: {
@ -34,7 +36,7 @@ export default defineConfig({
  },

  /* Global test timeout - CI needs more time than local */
-  timeout: 45 * 1000,
+  timeout: 60 * 1000,

  /* Configure projects for different test environments */
  projects: [
@ -48,8 +50,9 @@ export default defineConfig({

  /* Run your local dev server before starting the tests */
  webServer: {
-    command:
-      'NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_TELEMETRY_DISABLED=1 PORT=8081 yarn run dev',
+    command: process.env.CI
+      ? 'NEXT_PUBLIC_IS_LOCAL_MODE=true yarn build && NEXT_PUBLIC_IS_LOCAL_MODE=true PORT=8081 yarn start'
+      : 'NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_TELEMETRY_DISABLED=1 PORT=8081 yarn run dev',
    port: 8081,
    reuseExistingServer: !process.env.CI,
    timeout: 180 * 1000,
--- a/packages/app/tests/e2e/README.md
+++ b/packages/app/tests/e2e/README.md
@ -1,6 +1,8 @@
 # End-to-End Testing

-This directory contains Playwright-based end-to-end tests for the HyperDX application. The tests are organized into core functionality and feature-specific test suites.
+This directory contains Playwright-based end-to-end tests for the HyperDX
+application. The tests are organized into core functionality and
+feature-specific test suites.

 ## Prerequisites

@ -47,7 +49,9 @@ cd packages/app && yarn test:e2e --grep "@dashboard"

 ### Local Mode vs Full Server

-Tests tagged with `@local-mode` can run against the local development server without external dependencies. The test configuration automatically starts a local development server with `NEXT_PUBLIC_IS_LOCAL_MODE=true`.
+Tests tagged with `@local-mode` can run against the local development server
+without external dependencies. The test configuration automatically starts a
+local development server with `NEXT_PUBLIC_IS_LOCAL_MODE=true`.

 ## Test Organization

@ -87,6 +91,15 @@ Run tests in debug mode with browser developer tools:
 yarn test:e2e:debug
 ```

+### CI Mode
+
+Run tests in ci mode, which runs it in a docker container and environment
+similar to how it runs inside of Github Actions
+
+```bash
+yarn test:e2e:ci
+```
+
 ### Single Test Debugging

 To debug a specific test file:
@ -114,6 +127,7 @@ yarn playwright show-report
 ```

 The report includes:
+
 - Test execution timeline
 - Screenshots of failures
 - Video recordings of failed tests
@ -122,6 +136,7 @@ The report includes:
 ### Test Results

 Test artifacts are stored in:
+
 - `test-results/` - Screenshots, videos, and traces for failed tests
 - `playwright-report/` - HTML report files

@ -138,7 +153,8 @@ yarn playwright show-trace test-results/[test-name]/trace.zip
 The test configuration is defined in `playwright.config.ts`:

 - **Base URL**: `http://localhost:8080` (configurable via `PLAYWRIGHT_BASE_URL`)
- **Test Timeout**: 60 seconds (increased from default 30s to reduce flaky test failures)
+- **Test Timeout**: 60 seconds (increased from default 30s to reduce flaky test
+  failures)
 - **Retries**: 1 retry locally, 2 on CI
 - **Workers**: Undefined (uses Playwright defaults)
 - **Screenshots**: Captured on failure only
@ -152,6 +168,7 @@ The test configuration is defined in `playwright.config.ts`:
 ### Writing Tests

 Tests use the extended base test from `utils/base-test.ts` which provides:
+
 - Automatic handling of connection/sources
 - Tanstack Query devtools management
 - Network idle waiting after navigation
@ -169,6 +186,7 @@ Tests use the extended base test from `utils/base-test.ts` which provides:
 ### Server Connection Issues

 If tests fail with connection errors:
+
 1. Ensure no other services are running on port 8080
 2. Check that the development server starts successfully
 3. Verify environment variables are properly configured
@ -176,14 +194,17 @@ If tests fail with connection errors:
 ### Flaky Tests

 For intermittent failures:
+
 1. Check the HTML report for timing issues
 2. Review network logs for failed requests
-3. Consider if individual test steps need longer wait times (global timeout is now 60s)
+3. Consider if individual test steps need longer wait times (global timeout is
+   now 60s)
 4. Use the trace viewer to analyze test execution

 ### CI/CD Integration

 Tests are configured to run in CI environments with:
+
 - 60-second test timeout (same as local)
 - Multiple retry attempts (2 retries on CI vs 1 locally)
 - Artifact collection for failed tests
--- a/scripts/test-e2e-ci.sh
+++ b/scripts/test-e2e-ci.sh
@ -0,0 +1,37 @@
+#!/bin/bash
+# Run E2E tests in Docker with exact CI environment
+# This replicates the GitHub Actions CI environment locally for debugging
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+echo "Running E2E tests in CI Docker environment..."
+echo "Repository root: $REPO_ROOT"
+
+docker run --rm \
+  -v "$REPO_ROOT:/workspace" \
+  -w /workspace \
+  -e CI=true \
+  mcr.microsoft.com/playwright:v1.57.0-jammy \
+  bash -c '
+    # Clean all build artifacts and dependencies
+    rm -rf packages/app/.next packages/common-utils/dist node_modules packages/*/node_modules .yarn/cache
+    
+    # Fresh install
+    corepack enable
+    yarn install
+    
+    # Build in production mode
+    npx nx run-many -t ci:build
+    
+    # Run tests
+    cd packages/app
+    yarn test:e2e
+    
+    # Fix permissions so host can read results
+    chmod -R 777 test-results playwright-report 2>/dev/null || true
+  '
+
+echo "Done! Check packages/app/test-results/ for results"
--- a/yarn.lock
+++ b/yarn.lock
@ -4268,7 +4268,7 @@ __metadata:
    "@mantine/notifications": "npm:^7.17.8"
    "@mantine/spotlight": "npm:^7.17.8"
    "@microsoft/fetch-event-source": "npm:^2.0.1"
-    "@playwright/test": "npm:^1.47.0"
+    "@playwright/test": "npm:^1.57.0"
    "@storybook/addon-docs": "npm:^10.1.4"
    "@storybook/addon-links": "npm:^10.1.4"
    "@storybook/addon-styling-webpack": "npm:^3.0.0"
@ -7719,14 +7719,14 @@ __metadata:
  languageName: node
  linkType: hard

-"@playwright/test@npm:^1.47.0":
-  version: 1.55.0
-  resolution: "@playwright/test@npm:1.55.0"
+"@playwright/test@npm:^1.57.0":
+  version: 1.57.0
+  resolution: "@playwright/test@npm:1.57.0"
  dependencies:
-    playwright: "npm:1.55.0"
+    playwright: "npm:1.57.0"
  bin:
    playwright: cli.js
-  checksum: 10c0/e68b59cd8271f1b57c0649fc0562ab2d5f6bba8c3653dd7bd52ca1338dc380fde34588d0254e3cd3f0f2b20af04a80dfb080419ceb7475990bb2fc4d8c474984
+  checksum: 10c0/35ba4b28be72bf0a53e33dbb11c6cff848fb9a37f49e893ce63a90675b5291ec29a1ba82c8a3b043abaead129400f0589623e9ace2e6a1c8eaa409721ecc3774
  languageName: node
  linkType: hard

@ -22637,27 +22637,27 @@ __metadata:
  languageName: node
  linkType: hard

-"playwright-core@npm:1.55.0":
-  version: 1.55.0
-  resolution: "playwright-core@npm:1.55.0"
+"playwright-core@npm:1.57.0":
+  version: 1.57.0
+  resolution: "playwright-core@npm:1.57.0"
  bin:
    playwright-core: cli.js
-  checksum: 10c0/c39d6aa30e7a4e73965942ca5e13405ae05c9cb49f755a35f04248c864c0b24cf662d9767f1797b3ec48d1cf4e54774dce4a19c16534bd5cfd2aa3da81c9dc3a
+  checksum: 10c0/798e35d83bf48419a8c73de20bb94d68be5dde68de23f95d80a0ebe401e3b83e29e3e84aea7894d67fa6c79d2d3d40cc5bcde3e166f657ce50987aaa2421b6a9
  languageName: node
  linkType: hard

-"playwright@npm:1.55.0":
-  version: 1.55.0
-  resolution: "playwright@npm:1.55.0"
+"playwright@npm:1.57.0":
+  version: 1.57.0
+  resolution: "playwright@npm:1.57.0"
  dependencies:
    fsevents: "npm:2.3.2"
-    playwright-core: "npm:1.55.0"
+    playwright-core: "npm:1.57.0"
  dependenciesMeta:
    fsevents:
      optional: true
  bin:
    playwright: cli.js
-  checksum: 10c0/51605b7e57a5650e57972c5fdfc09d7a9934cca1cbee5beacca716fa801e25cb5bb7c1663de90c22b300fde884e5545a2b13a0505a93270b660687791c478304
+  checksum: 10c0/ab03c99a67b835bdea9059f516ad3b6e42c21025f9adaa161a4ef6bc7ca716dcba476d287140bb240d06126eb23f889a8933b8f5f1f1a56b80659d92d1358899
  languageName: node
  linkType: hard