Enable parallel integration testing across multiple worktrees (#1917)

## Summary - Enable multiple agents/developers to run `make dev-int` simultaneously from different git worktrees without Docker port conflicts - Compute a deterministic port offset (0-99) from the worktree directory name via `cksum`, giving each worktree its own isolated Docker Compose project and port range - Switch `.env.test` files to use `${HDX_CI_*:-default}` variable expansion (powered by `dotenv-expand`) so test processes connect to the correct dynamic ports ## How it works Each worktree gets a unique **slot** derived from its directory name. All service ports are offset by that slot: | Service | Base port | Example (slot 68) | |-----------------|-----------|-------------------| | ClickHouse HTTP | 18123 | 18191 | | MongoDB | 39999 | 40067 | | API test server | 19000 | 19068 | | OpAMP | 14320 | 14388 | Docker Compose project names are also unique (`int-<slot>`), isolating containers and networks. Backward compatible — when no `HDX_CI_*` env vars are set, all ports fall back to their original defaults. ## Changes - **Makefile**: Added `HDX_CI_SLOT` computation and dynamic project names/ports for all `dev-int` targets - **docker-compose.ci.yml**: Ports use `${HDX_CI_*:-default}` env vars; removed unused OTel collector published port; removed hardcoded network name (auto-generated from project name) - **packages/api/.env.test** / **packages/common-utils/.env.test**: Ports use `${HDX_CI_*:-default}` expansion syntax - **packages/api/jest.config.js** / **packages/common-utils/jest.int.config.js**: Switched from `dotenv/config` to `dotenv-expand/config` to enable variable expansion - **packages/api/package.json** / **packages/common-utils/package.json**: Added `dotenv-expand` devDependency - **agent_docs/development.md**: Documented multi-agent worktree support ## Testing Ran full Alert integration test suite (`make dev-int FILE=alerts`) — **6 test suites, 150 tests passed** on slot 68 with dynamic ports.
2026-04-21 13:37:15 +00:00 · 2026-03-16 12:42:08 -07:00 · 2026-03-16 12:42:08 -07:00 · b9c9682972
commit b9c9682972
parent 81fe186e8a
7 changed files with 76 additions and 32 deletions
--- a/38
+++ b/38
@ -3,6 +3,28 @@ BUILD_PLATFORMS = linux/arm64,linux/amd64

 include .env

+# ---------------------------------------------------------------------------
+# Multi-agent / worktree isolation
+# ---------------------------------------------------------------------------
+# Compute a deterministic port offset (0-99) from the working directory name
+# so that multiple worktrees can run integration tests in parallel without
+# port conflicts.  Override HDX_CI_SLOT manually if you need a specific slot.
+#
+# Port mapping (base + slot):
+#   ClickHouse HTTP : 18123 + slot
+#   MongoDB         : 39999 + slot
+#   API test server : 19000 + slot
+#   OpAMP           : 14320 + slot
+# ---------------------------------------------------------------------------
+HDX_CI_SLOT      ?= $(shell printf '%s' "$(notdir $(CURDIR))" | cksum | awk '{print $$1 % 100}')
+HDX_CI_PROJECT   := int-$(HDX_CI_SLOT)
+HDX_CI_CH_PORT   := $(shell echo $$((18123 + $(HDX_CI_SLOT))))
+HDX_CI_MONGO_PORT:= $(shell echo $$((39999 + $(HDX_CI_SLOT))))
+HDX_CI_API_PORT  := $(shell echo $$((19000 + $(HDX_CI_SLOT))))
+HDX_CI_OPAMP_PORT:= $(shell echo $$((14320 + $(HDX_CI_SLOT))))
+
+export HDX_CI_CH_PORT HDX_CI_MONGO_PORT HDX_CI_API_PORT HDX_CI_OPAMP_PORT
+
 .PHONY: all
 all: install-tools

@ -38,26 +60,28 @@ ci-lint:
 .PHONY: dev-int-build
 dev-int-build:
 	npx nx run-many -t ci:build
-	docker compose -p int -f ./docker-compose.ci.yml build
+	docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml build

 .PHONY: dev-int
 dev-int:
-	docker compose -p int -f ./docker-compose.ci.yml up -d
+	@echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT))"
+	docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d
 	npx nx run @hyperdx/api:dev:int $(FILE); ret=$$?; \
-	docker compose -p int -f ./docker-compose.ci.yml down; \
+	docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down; \
 	exit $$ret

 .PHONY: dev-int-common-utils
 dev-int-common-utils:
-	docker compose -p int -f ./docker-compose.ci.yml up -d
+	@echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT))"
+	docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d
 	npx nx run @hyperdx/common-utils:dev:int $(FILE)
-	docker compose -p int -f ./docker-compose.ci.yml down
+	docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down

 .PHONY: ci-int
 ci-int:
-	docker compose -p int -f ./docker-compose.ci.yml up -d --quiet-pull
+	docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull
 	npx nx run-many -t ci:int --parallel=false
-	docker compose -p int -f ./docker-compose.ci.yml down
+	docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down

 .PHONY: dev-unit
 dev-unit:
--- a/agent_docs/development.md
+++ b/agent_docs/development.md
@ -45,22 +45,45 @@ yarn dev
 - **Mocking**: MSW for API mocking in frontend tests
 - **Database testing**: Isolated test databases with fixtures

-### CI Testing
+### CI / Integration Testing

-For integration testing in CI environments:
+For integration testing:

 ```bash
-# Start CI testing stack (ClickHouse, MongoDB, etc.)
-docker compose -p int -f ./docker-compose.ci.yml up -d
+# Build dependencies (run once before first test run)
+make dev-int-build

-# Run integration tests
-yarn dev:int
+# Run API integration tests (spins up Docker services, runs tests, tears down)
+make dev-int FILE=<TEST_FILE_NAME>
+
+# Run common-utils integration tests
+make dev-int-common-utils FILE=<TEST_FILE_NAME>
 ```

+**Multi-agent / worktree support:**
+
+The `make dev-int` command automatically assigns unique Docker ports per
+worktree directory, so multiple agents can run integration tests in parallel
+without port conflicts.
+
+- A deterministic slot (0-99) is computed from the worktree directory name
+- Each slot gets its own Docker Compose project name and port range
+- Override the slot manually: `make dev-int HDX_CI_SLOT=5 FILE=alerts`
+- The slot and assigned ports are printed when `dev-int` starts
+
+Port mapping (base + slot):
+
+| Service         | Default port (slot 0) | Variable          |
+| --------------- | --------------------- | ----------------- |
+| ClickHouse HTTP | 18123                 | HDX_CI_CH_PORT    |
+| MongoDB         | 39999                 | HDX_CI_MONGO_PORT |
+| API test server | 19000                 | HDX_CI_API_PORT   |
+| OpAMP           | 14320                 | HDX_CI_OPAMP_PORT |
+
 **CI Testing Notes:**

- Uses separate Docker Compose configuration optimized for CI
- Isolated test environment with `-p int` project name
+- Uses separate Docker Compose configuration (`docker-compose.ci.yml`)
+- Isolated test environment with unique `-p int-<slot>` project name
 - Includes all necessary services (ClickHouse, MongoDB, OTel Collector)
 - Tests run against real database instances for accurate integration testing

--- a/docker-compose.ci.yml
+++ b/docker-compose.ci.yml
@ -13,12 +13,6 @@ services:
      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
    volumes:
      - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
-    ports:
-      - '23133:13133' # health_check extension
-    #   - '24225:24225' # fluentd receiver
-    #   - '4317:4317' # OTLP gRPC receiver
-    #   - '4318:4318' # OTLP http receiver
-    #   - '8888:8888' # metrics extension
    restart: on-failure
    networks:
      - internal
@ -34,17 +28,15 @@ services:
      - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
    restart: on-failure
    ports:
-      - 18123:8123 # http api
-    #   - 9000:9000 # native
+      - '${HDX_CI_CH_PORT:-18123}:8123' # http api
    networks:
      - internal
  db:
    image: mongo:5.0.32-focal
    command: --port 29999
    ports:
-      - 39999:29999
+      - '${HDX_CI_MONGO_PORT:-39999}:29999'
    networks:
      - internal
 networks:
  internal:
-    name: 'hyperdx-ci-internal-network'
--- a/packages/api/.env.test
+++ b/packages/api/.env.test
@ -1,13 +1,16 @@
 # Environment configuration for Integration tests, corresponding to ports defined in docker-compose.ci.yml
-CLICKHOUSE_HOST=http://localhost:18123
+# For multi-agent worktree support, ports are configured via HDX_CI_* env vars.
+# Set HDX_CI_CH_PORT, HDX_CI_MONGO_PORT, HDX_CI_API_PORT, HDX_CI_OPAMP_PORT
+# before running tests to use non-default ports (see Makefile dev-int target).
+CLICKHOUSE_HOST=http://localhost:${HDX_CI_CH_PORT:-18123}
 CLICKHOUSE_PASSWORD=api
 CLICKHOUSE_USER=api
 RUN_SCHEDULED_TASKS_EXTERNALLY=true
 EXPRESS_SESSION_SECRET="hyperdx is cool 👋"
 FRONTEND_URL=http://app:8080
-MONGO_URI=mongodb://localhost:39999/hyperdx-test
+MONGO_URI=mongodb://localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test
 NODE_ENV=test
-PORT=19000 # (API port)
-OPAMP_PORT=14320
+PORT=${HDX_CI_API_PORT:-19000}
+OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320}
 # Default to only logging warnings/errors. Adjust if you need more verbosity
 HYPERDX_LOG_LEVEL=warn
--- a/packages/api/jest.config.js
+++ b/packages/api/jest.config.js
@ -1,7 +1,7 @@
 /** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
 module.exports = {
  setupFilesAfterEnv: ['<rootDir>/../jest.setup.ts'],
-  setupFiles: ['dotenv/config'],
+  setupFiles: ['dotenv-expand/config'],
  preset: 'ts-jest',
  testEnvironment: 'node',
  verbose: true,
--- a/packages/common-utils/.env.test
+++ b/packages/common-utils/.env.test
@ -1,5 +1,7 @@
 # Environment configuration for Integration tests, corresponding to ports defined in docker-compose.ci.yml
-CLICKHOUSE_HOST=http://localhost:18123
+# For multi-agent worktree support, set HDX_CI_CH_PORT env var before running
+# tests to use a non-default ClickHouse port (see Makefile dev-int target).
+CLICKHOUSE_HOST=http://localhost:${HDX_CI_CH_PORT:-18123}
 CLICKHOUSE_PASSWORD=
 CLICKHOUSE_USER=default
 NODE_ENV=test
--- a/packages/common-utils/jest.int.config.js
+++ b/packages/common-utils/jest.int.config.js
@ -1,6 +1,6 @@
 /** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
 module.exports = {
-  setupFiles: ['dotenv/config'],
+  setupFiles: ['dotenv-expand/config'],
  preset: 'ts-jest',
  testEnvironment: 'node',
  setupFilesAfterEnv: ['<rootDir>/../jest.setup.ts'],