diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95a7cbca..dc5e0269 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -103,7 +103,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 container: - image: mcr.microsoft.com/playwright:v1.55.0-jammy + image: mcr.microsoft.com/playwright:v1.57.0-jammy permissions: contents: read pull-requests: write diff --git a/agent_docs/development.md b/agent_docs/development.md index 4d97692f..5c062dac 100644 --- a/agent_docs/development.md +++ b/agent_docs/development.md @@ -12,10 +12,14 @@ yarn dev ## Key Development Scripts -- `yarn app:dev`: Start API, frontend, alerts task, and common-utils in watch mode +- `yarn app:dev`: Start API, frontend, alerts task, and common-utils in watch + mode - `yarn lint`: Run linting across all packages - `yarn dev:int`: Run integration tests in watch mode - `yarn dev:unit`: Run unit tests in watch mode (per package) +- `yarn test:e2e`: Run Playwright E2E tests (in `packages/app`) +- `yarn test:e2e:ci`: Run Playwright E2E tests in CI Docker environment (in + `packages/app`) ## Environment Configuration @@ -30,12 +34,14 @@ yarn dev - **Unit Tests**: Jest with TypeScript support - **Integration Tests**: Jest with database fixtures - **Frontend Testing**: React Testing Library + Jest -- **E2E Testing**: Custom smoke tests with BATS +- **E2E Testing**: Playwright (frontend) and Custom smoke tests with BATS + (ingestion) ### Testing Patterns - **TDD Approach**: Write tests before implementation for new features -- **Test organization**: Tests co-located with source files in `__tests__/` directories +- **Test organization**: Tests co-located with source files in `__tests__/` + directories - **Mocking**: MSW for API mocking in frontend tests - **Database testing**: Isolated test databases with fixtures @@ -52,6 +58,7 @@ yarn dev:int ``` **CI Testing Notes:** + - Uses separate Docker Compose configuration optimized for CI - Isolated test environment with `-p int` project name - Includes all necessary services (ClickHouse, MongoDB, OTel Collector) @@ -79,6 +86,7 @@ yarn dev:int ### Pre-commit Hooks The project uses Husky + lint-staged to automatically run: + - Prettier for formatting - ESLint for linting - API doc generation (for external API changes) @@ -101,11 +109,11 @@ yarn run lint ## File Locations Quick Reference -- **Config**: `packages/api/src/config.ts`, `packages/app/next.config.mjs`, `docker-compose.dev.yml` +- **Config**: `packages/api/src/config.ts`, `packages/app/next.config.mjs`, + `docker-compose.dev.yml` - **Models**: `packages/api/src/models/` - **API Routes**: `packages/api/src/routers/` - **Controllers**: `packages/api/src/controllers/` - **Pages**: `packages/app/pages/` - **Components**: `packages/app/src/` - **Shared Utils**: `packages/common-utils/src/` - diff --git a/packages/app/package.json b/packages/app/package.json index d12f55ab..bdd0f1f9 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -20,6 +20,7 @@ "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", "test:e2e:debug": "playwright test --debug", + "test:e2e:ci": "../../scripts/test-e2e-ci.sh", "storybook": "storybook dev -p 6006", "storybook:build": "storybook build", "knip": "knip" @@ -107,7 +108,7 @@ "@chromatic-com/storybook": "^4.1.3", "@hookform/devtools": "^4.3.1", "@jedmao/location": "^3.0.0", - "@playwright/test": "^1.47.0", + "@playwright/test": "^1.57.0", "@storybook/addon-docs": "^10.1.4", "@storybook/addon-links": "^10.1.4", "@storybook/addon-styling-webpack": "^3.0.0", diff --git a/packages/app/playwright.config.ts b/packages/app/playwright.config.ts index 123d2c4a..58ed8d08 100644 --- a/packages/app/playwright.config.ts +++ b/packages/app/playwright.config.ts @@ -19,7 +19,9 @@ export default defineConfig({ reporter: [ ['html'], ['json', { outputFile: 'test-results/results.json' }], - ...(process.env.CI ? [['github', {}] as const] : []), + ...(process.env.CI + ? [['github'] as const, ['list'] as const] + : [['list'] as const]), ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { @@ -34,7 +36,7 @@ export default defineConfig({ }, /* Global test timeout - CI needs more time than local */ - timeout: 45 * 1000, + timeout: 60 * 1000, /* Configure projects for different test environments */ projects: [ @@ -48,8 +50,9 @@ export default defineConfig({ /* Run your local dev server before starting the tests */ webServer: { - command: - 'NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_TELEMETRY_DISABLED=1 PORT=8081 yarn run dev', + command: process.env.CI + ? 'NEXT_PUBLIC_IS_LOCAL_MODE=true yarn build && NEXT_PUBLIC_IS_LOCAL_MODE=true PORT=8081 yarn start' + : 'NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_TELEMETRY_DISABLED=1 PORT=8081 yarn run dev', port: 8081, reuseExistingServer: !process.env.CI, timeout: 180 * 1000, diff --git a/packages/app/tests/e2e/README.md b/packages/app/tests/e2e/README.md index a270b17b..71a5a141 100644 --- a/packages/app/tests/e2e/README.md +++ b/packages/app/tests/e2e/README.md @@ -1,6 +1,8 @@ # End-to-End Testing -This directory contains Playwright-based end-to-end tests for the HyperDX application. The tests are organized into core functionality and feature-specific test suites. +This directory contains Playwright-based end-to-end tests for the HyperDX +application. The tests are organized into core functionality and +feature-specific test suites. ## Prerequisites @@ -47,7 +49,9 @@ cd packages/app && yarn test:e2e --grep "@dashboard" ### Local Mode vs Full Server -Tests tagged with `@local-mode` can run against the local development server without external dependencies. The test configuration automatically starts a local development server with `NEXT_PUBLIC_IS_LOCAL_MODE=true`. +Tests tagged with `@local-mode` can run against the local development server +without external dependencies. The test configuration automatically starts a +local development server with `NEXT_PUBLIC_IS_LOCAL_MODE=true`. ## Test Organization @@ -87,6 +91,15 @@ Run tests in debug mode with browser developer tools: yarn test:e2e:debug ``` +### CI Mode + +Run tests in ci mode, which runs it in a docker container and environment +similar to how it runs inside of Github Actions + +```bash +yarn test:e2e:ci +``` + ### Single Test Debugging To debug a specific test file: @@ -114,6 +127,7 @@ yarn playwright show-report ``` The report includes: + - Test execution timeline - Screenshots of failures - Video recordings of failed tests @@ -122,6 +136,7 @@ The report includes: ### Test Results Test artifacts are stored in: + - `test-results/` - Screenshots, videos, and traces for failed tests - `playwright-report/` - HTML report files @@ -138,7 +153,8 @@ yarn playwright show-trace test-results/[test-name]/trace.zip The test configuration is defined in `playwright.config.ts`: - **Base URL**: `http://localhost:8080` (configurable via `PLAYWRIGHT_BASE_URL`) -- **Test Timeout**: 60 seconds (increased from default 30s to reduce flaky test failures) +- **Test Timeout**: 60 seconds (increased from default 30s to reduce flaky test + failures) - **Retries**: 1 retry locally, 2 on CI - **Workers**: Undefined (uses Playwright defaults) - **Screenshots**: Captured on failure only @@ -152,6 +168,7 @@ The test configuration is defined in `playwright.config.ts`: ### Writing Tests Tests use the extended base test from `utils/base-test.ts` which provides: + - Automatic handling of connection/sources - Tanstack Query devtools management - Network idle waiting after navigation @@ -169,6 +186,7 @@ Tests use the extended base test from `utils/base-test.ts` which provides: ### Server Connection Issues If tests fail with connection errors: + 1. Ensure no other services are running on port 8080 2. Check that the development server starts successfully 3. Verify environment variables are properly configured @@ -176,14 +194,17 @@ If tests fail with connection errors: ### Flaky Tests For intermittent failures: + 1. Check the HTML report for timing issues 2. Review network logs for failed requests -3. Consider if individual test steps need longer wait times (global timeout is now 60s) +3. Consider if individual test steps need longer wait times (global timeout is + now 60s) 4. Use the trace viewer to analyze test execution ### CI/CD Integration Tests are configured to run in CI environments with: + - 60-second test timeout (same as local) - Multiple retry attempts (2 retries on CI vs 1 locally) - Artifact collection for failed tests diff --git a/scripts/test-e2e-ci.sh b/scripts/test-e2e-ci.sh new file mode 100755 index 00000000..7d2eaed0 --- /dev/null +++ b/scripts/test-e2e-ci.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Run E2E tests in Docker with exact CI environment +# This replicates the GitHub Actions CI environment locally for debugging + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +echo "Running E2E tests in CI Docker environment..." +echo "Repository root: $REPO_ROOT" + +docker run --rm \ + -v "$REPO_ROOT:/workspace" \ + -w /workspace \ + -e CI=true \ + mcr.microsoft.com/playwright:v1.57.0-jammy \ + bash -c ' + # Clean all build artifacts and dependencies + rm -rf packages/app/.next packages/common-utils/dist node_modules packages/*/node_modules .yarn/cache + + # Fresh install + corepack enable + yarn install + + # Build in production mode + npx nx run-many -t ci:build + + # Run tests + cd packages/app + yarn test:e2e + + # Fix permissions so host can read results + chmod -R 777 test-results playwright-report 2>/dev/null || true + ' + +echo "Done! Check packages/app/test-results/ for results" \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 408a4e3b..1b2a2da5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4268,7 +4268,7 @@ __metadata: "@mantine/notifications": "npm:^7.17.8" "@mantine/spotlight": "npm:^7.17.8" "@microsoft/fetch-event-source": "npm:^2.0.1" - "@playwright/test": "npm:^1.47.0" + "@playwright/test": "npm:^1.57.0" "@storybook/addon-docs": "npm:^10.1.4" "@storybook/addon-links": "npm:^10.1.4" "@storybook/addon-styling-webpack": "npm:^3.0.0" @@ -7719,14 +7719,14 @@ __metadata: languageName: node linkType: hard -"@playwright/test@npm:^1.47.0": - version: 1.55.0 - resolution: "@playwright/test@npm:1.55.0" +"@playwright/test@npm:^1.57.0": + version: 1.57.0 + resolution: "@playwright/test@npm:1.57.0" dependencies: - playwright: "npm:1.55.0" + playwright: "npm:1.57.0" bin: playwright: cli.js - checksum: 10c0/e68b59cd8271f1b57c0649fc0562ab2d5f6bba8c3653dd7bd52ca1338dc380fde34588d0254e3cd3f0f2b20af04a80dfb080419ceb7475990bb2fc4d8c474984 + checksum: 10c0/35ba4b28be72bf0a53e33dbb11c6cff848fb9a37f49e893ce63a90675b5291ec29a1ba82c8a3b043abaead129400f0589623e9ace2e6a1c8eaa409721ecc3774 languageName: node linkType: hard @@ -22637,27 +22637,27 @@ __metadata: languageName: node linkType: hard -"playwright-core@npm:1.55.0": - version: 1.55.0 - resolution: "playwright-core@npm:1.55.0" +"playwright-core@npm:1.57.0": + version: 1.57.0 + resolution: "playwright-core@npm:1.57.0" bin: playwright-core: cli.js - checksum: 10c0/c39d6aa30e7a4e73965942ca5e13405ae05c9cb49f755a35f04248c864c0b24cf662d9767f1797b3ec48d1cf4e54774dce4a19c16534bd5cfd2aa3da81c9dc3a + checksum: 10c0/798e35d83bf48419a8c73de20bb94d68be5dde68de23f95d80a0ebe401e3b83e29e3e84aea7894d67fa6c79d2d3d40cc5bcde3e166f657ce50987aaa2421b6a9 languageName: node linkType: hard -"playwright@npm:1.55.0": - version: 1.55.0 - resolution: "playwright@npm:1.55.0" +"playwright@npm:1.57.0": + version: 1.57.0 + resolution: "playwright@npm:1.57.0" dependencies: fsevents: "npm:2.3.2" - playwright-core: "npm:1.55.0" + playwright-core: "npm:1.57.0" dependenciesMeta: fsevents: optional: true bin: playwright: cli.js - checksum: 10c0/51605b7e57a5650e57972c5fdfc09d7a9934cca1cbee5beacca716fa801e25cb5bb7c1663de90c22b300fde884e5545a2b13a0505a93270b660687791c478304 + checksum: 10c0/ab03c99a67b835bdea9059f516ad3b6e42c21025f9adaa161a4ef6bc7ca716dcba476d287140bb240d06126eb23f889a8933b8f5f1f1a56b80659d92d1358899 languageName: node linkType: hard