feat: single app image (#519)

.env

@@ -1,13 +1,12 @@
 # Used by docker-compose.yml
 IMAGE_NAME=ghcr.io/hyperdxio/hyperdx
+IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx
 LOCAL_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-local
 LOCAL_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-local
-IMAGE_VERSION=1.9.0
-V2_BETA_IMAGE_VERSION=2-beta
+IMAGE_VERSION=2-beta.6
 
 # Set up domain URLs
-HYPERDX_API_PORT=8000
-HYPERDX_API_URL=http://localhost
+HYPERDX_API_PORT=8000 #optional (should not be taken by other services)
 HYPERDX_APP_PORT=8080
 HYPERDX_APP_URL=http://localhost
 HYPERDX_LOG_LEVEL=debug

Makefile

@@ -66,31 +66,12 @@ dev-migrate-db:
 	@echo "Migrating ClickHouse db...\n"
 	npx nx run @hyperdx/api:dev:migrate-ch
 
-.PHONY: build-local
-build-local:
-	docker build ./docker/hostmetrics -t ${IMAGE_NAME}:${LATEST_VERSION}-hostmetrics --target prod &
-	docker build ./docker/ingestor -t ${IMAGE_NAME}:${LATEST_VERSION}-ingestor --target prod &
-	docker build ./docker/otel-collector -t ${IMAGE_NAME}:${LATEST_VERSION}-otel-collector --target prod &
-	docker build --build-arg CODE_VERSION=${LATEST_VERSION} . -f ./packages/miner/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-miner --target prod &
-	docker build --build-arg CODE_VERSION=${LATEST_VERSION} . -f ./packages/go-parser/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-go-parser &
-	docker build \
-		--build-arg CODE_VERSION=${LATEST_VERSION} \
-		--build-arg PORT=${HYPERDX_API_PORT} \
-		. -f ./packages/api/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-api --target prod &
-	docker build \
-		--build-arg CODE_VERSION=${LATEST_VERSION} \
-		--build-arg OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} \
-		--build-arg OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME} \
-		--build-arg PORT=${HYPERDX_APP_PORT} \
-		--build-arg SERVER_URL=${HYPERDX_API_URL}:${HYPERDX_API_PORT} \
-		. -f ./packages/app/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-app --target prod
-
 .PHONY: version
 version:
 	sh ./version.sh
 
-.PHONY: release-v2-beta
-release-v2-beta:
+.PHONY: release-local
+release-local:
 	docker buildx build --squash . -f ./docker/local/Dockerfile \
 		--build-context clickhouse=./docker/clickhouse \
 		--build-context otel-collector=./docker/otel-collector \
@@ -98,35 +79,30 @@ release-v2-beta:
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
 		--platform ${BUILD_PLATFORMS} \
-		-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${V2_BETA_IMAGE_VERSION} \
-		-t ${LOCAL_IMAGE_NAME}:${V2_BETA_IMAGE_VERSION} --push
+		-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \
+		-t ${LOCAL_IMAGE_NAME}:${IMAGE_VERSION} --push
+
+.PHONY: release-ui
+release-ui:
+	docker buildx build . -f ./packages/app/Dockerfile \
+		--build-arg IS_LOCAL_MODE=true \
+		--build-arg PORT=${HYPERDX_APP_PORT} \
+		--target prod \
+		--platform ${BUILD_PLATFORMS} \
+		-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}-ui \
+		-t ${LOCAL_IMAGE_NAME}:${IMAGE_VERSION}-ui --push
 
 .PHONY: release
 release:
-	docker buildx build --platform ${BUILD_PLATFORMS} ./docker/hostmetrics -t ${IMAGE_NAME}:${LATEST_VERSION}-hostmetrics --target prod --push &
-	docker buildx build --platform ${BUILD_PLATFORMS} ./docker/ingestor -t ${IMAGE_NAME}:${LATEST_VERSION}-ingestor --target prod --push &
-	docker buildx build --platform ${BUILD_PLATFORMS} ./docker/otel-collector -t ${IMAGE_NAME}:${LATEST_VERSION}-otel-collector --target prod --push &
-	docker buildx build --build-arg CODE_VERSION=${LATEST_VERSION} --platform ${BUILD_PLATFORMS} . -f ./packages/miner/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-miner --target prod --push &
-	docker buildx build --build-arg CODE_VERSION=${LATEST_VERSION} --platform ${BUILD_PLATFORMS} . -f ./packages/go-parser/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-go-parser --push &
-	docker buildx build \
-		--build-arg CODE_VERSION=${LATEST_VERSION} \
-		--build-arg PORT=${HYPERDX_API_PORT} \
-		--platform ${BUILD_PLATFORMS} . -f ./packages/api/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-api --target prod --push &
-	docker buildx build \
-		--build-arg CODE_VERSION=${LATEST_VERSION} \
-		--build-arg OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} \
-		--build-arg OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME} \
-		--build-arg PORT=${HYPERDX_APP_PORT} \
-		--build-arg SERVER_URL=${HYPERDX_API_URL}:${HYPERDX_API_PORT} \
-		--platform ${BUILD_PLATFORMS} . -f ./packages/app/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-app --target prod --push &
-	docker buildx build \
-		--squash . -f ./docker/local/Dockerfile \
-		--build-context clickhouse=./docker/clickhouse \
-		--build-context otel-collector=./docker/otel-collector \
-		--build-context ingestor=./docker/ingestor \
-		--build-context local=./docker/local \
+	docker buildx build --platform ${BUILD_PLATFORMS} ./docker/otel-collector \
+		-t ${IMAGE_NAME}:${IMAGE_VERSION}-otel-collector \
+		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}-otel-collector \
+		--target prod --push &
+	docker buildx build --squash . -f ./docker/fullstack/Dockerfile \
+		--build-context fullstack=./docker/fullstack \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
 		--platform ${BUILD_PLATFORMS} \
-		-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:latest -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${LATEST_VERSION} \
-		-t ${LOCAL_IMAGE_NAME}:latest -t ${LOCAL_IMAGE_NAME}:${LATEST_VERSION} --push
+		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}-app \
+		-t ${IMAGE_NAME}:${IMAGE_VERSION}-app \
+		--target prod --push

docker-compose.ci.yml

@@ -1,7 +1,6 @@
-version: '3'
+name: hdx-ci
 services:
   otel-collector:
-    container_name: hdx-ci-otel-collector
     build:
       context: ./docker/otel-collector
       target: dev
@@ -23,7 +22,6 @@ services:
     depends_on:
       - ch-server
   ch-server:
-    container_name: hdx-ci-ch-server
     image: clickhouse/clickhouse-server:23.8.8-alpine
     environment:
       # default settings
@@ -38,7 +36,6 @@ services:
     networks:
       - internal
   db:
-    container_name: hdx-ci-db
     image: mongo:5.0.14-focal
     command: --port 29999
     # ports:
@@ -46,7 +43,6 @@ services:
     networks:
       - internal
   redis:
-    container_name: hdx-ci-redis
     image: redis:7.0.11-alpine
     # ports:
     #   - 6379:6379
@@ -57,7 +53,6 @@ services:
       context: .
       dockerfile: ./packages/api/Dockerfile
       target: dev
-    container_name: hdx-ci-api
     image: hyperdx/ci/api
     # ports:
     #   - 9000:9000
@@ -73,7 +68,6 @@ services:
       NODE_ENV: ci
       PORT: 9000
       REDIS_URL: redis://redis:6379
-      SERVER_URL: http://localhost:9000
     volumes:
       - ./packages/api/src:/app/src
     networks:

docker-compose.dev.yml

@@ -1,4 +1,4 @@
-version: '3'
+name: hdx-oss-dev
 x-hyperdx-logging: &hyperdx-logging
   driver: fluentd
   options:
@@ -6,7 +6,6 @@ x-hyperdx-logging: &hyperdx-logging
     labels: 'service.name'
 services:
   # miner:
-  #   container_name: hdx-oss-dev-miner
   #   build:
   #     context: .
   #     dockerfile: ./packages/miner/Dockerfile
@@ -29,7 +28,6 @@ services:
     labels:
       service.name: 'hdx-oss-dev-redis'
     image: redis:7.0.11-alpine
-    container_name: hdx-oss-dev-redis
     volumes:
       - .volumes/redis:/data
     ports:
@@ -43,7 +41,6 @@ services:
     labels:
       service.name: 'hdx-oss-dev-db'
     image: mongo:5.0.14-focal
-    container_name: hdx-oss-dev-db
     volumes:
       - .volumes/db:/data/db
     ports:
@@ -73,7 +70,6 @@ services:
     depends_on:
       - ch-server
   # task-check-alerts:
-  #   container_name: hdx-oss-dev-task-check-alerts
   #   build:
   #     context: .
   #     dockerfile: ./packages/api/Dockerfile
@@ -110,13 +106,12 @@ services:
   #     - db
   #     - redis
   api:
-    container_name: hdx-oss-dev-api
     build:
       context: .
       dockerfile: ./packages/api/Dockerfile
       target: dev
     ports:
-      - 8000:8000
+      - ${HYPERDX_API_PORT}:${HYPERDX_API_PORT}
     environment:
       AGGREGATOR_API_URL: 'http://aggregator:8001'
       APP_TYPE: 'api'
@@ -125,22 +120,20 @@ services:
       CLICKHOUSE_PASSWORD: api
       CLICKHOUSE_USER: api
       EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
-      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      FRONTEND_URL: 'http://localhost:${HYPERDX_APP_PORT}' # need to be localhost (CORS)
       HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
       HDX_NODE_BETA_MODE: 1
       HDX_NODE_CONSOLE_CAPTURE: 1
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      SENTRY_DSN: ${SENTRY_DSN}
       INGESTOR_API_URL: 'http://ingestor:8002'
       MINER_API_URL: 'http://miner:5123'
       MONGO_URI: 'mongodb://db:27017/hyperdx'
       NODE_ENV: development
       OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
       OTEL_SERVICE_NAME: 'hdx-oss-dev-api'
-      PORT: 8000
+      PORT: ${HYPERDX_API_PORT}
       REDIS_URL: redis://redis:6379
-      SERVER_URL: 'http://localhost:8000'
       USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-false}
     volumes:
       - ./packages/api/src:/app/src
@@ -151,35 +144,35 @@ services:
       - db
       - redis
   app:
-    container_name: hdx-oss-dev-app
     build:
       context: .
       dockerfile: ./packages/app/Dockerfile
       target: dev
     ports:
-      - 8080:8080
+      - ${HYPERDX_APP_PORT}:${HYPERDX_APP_PORT}
     environment:
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       NEXT_PUBLIC_HDX_COLLECTOR_URL: 'http://localhost:4318'
       NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-dev-app'
-      NEXT_PUBLIC_SERVER_URL: 'http://localhost:8000' # need to be localhost (CORS)
+      NEXT_PUBLIC_SERVER_URL: 'http://api:${HYPERDX_API_PORT}'
       NODE_ENV: development
-      PORT: 8080
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
+      OTEL_SERVICE_NAME: 'hdx-oss-dev-app'
+      PORT: ${HYPERDX_APP_PORT}
     volumes:
+      - ./packages/app/mdx.d.ts:/app/mdx.d.ts
+      - ./packages/app/next-env.d.ts:/app/next-env.d.ts
+      - ./packages/app/next.config.js:/app/next.config.js
       - ./packages/app/pages:/app/pages
       - ./packages/app/public:/app/public
       - ./packages/app/src:/app/src
       - ./packages/app/styles:/app/styles
-      - ./packages/app/mdx.d.ts:/app/mdx.d.ts
-      - ./packages/app/next-env.d.ts:/app/next-env.d.ts
-      - ./packages/app/next.config.js:/app/next.config.js
     networks:
       - internal
     depends_on:
       - api
   ch-server:
     image: clickhouse/clickhouse-server:head-alpine
-    container_name: hdx-oss-dev-ch-server
     ports:
       - 8123:8123 # http api
       - 9000:9000 # native

docker-compose.yml

@@ -1,66 +1,43 @@
-# TODO: migrate to V2
-version: '3'
+name: hdx-oss
 services:
-  go-parser:
-    image: ${IMAGE_NAME}:${IMAGE_VERSION}-go-parser
-    container_name: hdx-oss-go-parser
-    environment:
-      AGGREGATOR_API_URL: 'http://aggregator:8001'
-      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
-      OTEL_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      OTEL_SERVICE_NAME: hdx-oss-go-parser
-      PORT: 7777
-    ports:
-      - 7777:7777
-    networks:
-      - internal
-  miner:
-    image: ${IMAGE_NAME}:${IMAGE_VERSION}-miner
-    container_name: hdx-oss-miner
-    environment:
-      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1
-      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
-      OTEL_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      OTEL_SERVICE_NAME: hdx-oss-miner
-    ports:
-      - 5123:5123
-    networks:
-      - internal
-  hostmetrics:
-    image: ${IMAGE_NAME}:${IMAGE_VERSION}-hostmetrics
-    container_name: hdx-oss-hostmetrics
-    environment:
-      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      OTEL_SERVICE_NAME: hostmetrics
-    restart: always
-    networks:
-      - internal
-  ingestor:
-    image: ${IMAGE_NAME}:${IMAGE_VERSION}-ingestor
-    container_name: hdx-oss-ingestor
-    volumes:
-      - .volumes/ingestor_data:/var/lib/vector
-    ports:
-      - 8002:8002 # http-generic
-      - 8686:8686 # healthcheck
-    environment:
-      AGGREGATOR_API_URL: 'http://aggregator:8001'
-      ENABLE_GO_PARSER: 'true'
-      GO_PARSER_API_URL: 'http://go-parser:7777'
-      RUST_BACKTRACE: full
-      VECTOR_LOG: ${HYPERDX_LOG_LEVEL}
-      VECTOR_OPENSSL_LEGACY_PROVIDER: 'false'
-    restart: always
-    networks:
-      - internal
+  # go-parser:
+  #   image: ${IMAGE_NAME}:${IMAGE_VERSION}-go-parser
+  #   environment:
+  #     AGGREGATOR_API_URL: 'http://aggregator:8001'
+  #     HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+  #     HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+  #     OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+  #     OTEL_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+  #     OTEL_SERVICE_NAME: hdx-oss-go-parser
+  #     PORT: 7777
+  #   ports:
+  #     - 7777:7777
+  #   networks:
+  #     - internal
+  # miner:
+  #   image: ${IMAGE_NAME}:${IMAGE_VERSION}-miner
+  #   environment:
+  #     HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+  #     HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1
+  #     HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+  #     OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+  #     OTEL_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+  #     OTEL_SERVICE_NAME: hdx-oss-miner
+  #   ports:
+  #     - 5123:5123
+  #   networks:
+  #     - internal
+  # hostmetrics:
+  #   image: ${IMAGE_NAME}:${IMAGE_VERSION}-hostmetrics
+  #   environment:
+  #     HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+  #     HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+  #     OTEL_SERVICE_NAME: hostmetrics
+  #   restart: always
+  #   networks:
+  #     - internal
   redis:
     image: redis:7.0.11-alpine
-    container_name: hdx-oss-redis
     volumes:
       - .volumes/redis:/data
     ports:
@@ -69,7 +46,6 @@ services:
       - internal
   db:
     image: mongo:5.0.14-focal
-    container_name: hdx-oss-db
     volumes:
       - .volumes/db:/data/db
     ports:
@@ -78,8 +54,8 @@ services:
       - internal
   otel-collector:
     image: ${IMAGE_NAME}:${IMAGE_VERSION}-otel-collector
-    container_name: hdx-oss-otel-collector
     environment:
+      CLICKHOUSE_SERVER_ENDPOINT: 'ch-server:9000'
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
       INGESTOR_API_URL: 'http://ingestor:8002'
     ports:
@@ -91,90 +67,68 @@ services:
     restart: always
     networks:
       - internal
-  aggregator:
-    image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
-    container_name: hdx-oss-aggregator
-    ports:
-      - 8001:8001
-    environment:
-      AGGREGATOR_PAYLOAD_SIZE_LIMIT: '144mb'
-      APP_TYPE: 'aggregator'
-      CLICKHOUSE_HOST: http://ch-server:8123
-      CLICKHOUSE_PASSWORD: aggregator
-      CLICKHOUSE_USER: aggregator
-      FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT} # need to be localhost (CORS)
-      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      MONGO_URI: 'mongodb://db:27017/hyperdx'
-      NODE_ENV: development
-      PORT: 8001
-      REDIS_URL: redis://redis:6379
-      SERVER_URL: ${HYPERDX_API_URL}:${HYPERDX_API_PORT}
-    networks:
-      - internal
-    depends_on:
-      - db
-      - redis
-      - ch-server
-  task-check-alerts:
-    image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
-    container_name: hdx-oss-task-check-alerts
-    entrypoint: 'node'
-    command: './build/tasks/index.js check-alerts'
-    environment:
-      APP_TYPE: 'scheduled-task'
-      CLICKHOUSE_HOST: http://ch-server:8123
-      CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      CLICKHOUSE_PASSWORD: worker
-      CLICKHOUSE_USER: worker
-      EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
-      FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT} # need to be localhost (CORS)
-      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
-      HDX_NODE_BETA_MODE: 0
-      HDX_NODE_CONSOLE_CAPTURE: 1
-      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      INGESTOR_API_URL: 'http://ingestor:8002'
-      MINER_API_URL: 'http://miner:5123'
-      MONGO_URI: 'mongodb://db:27017/hyperdx'
-      NODE_ENV: development
-      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
-      OTEL_SERVICE_NAME: 'hdx-oss-task-check-alerts'
-      REDIS_URL: redis://redis:6379
-    restart: always
-    networks:
-      - internal
     depends_on:
       - ch-server
-      - db
-      - redis
-  api:
-    image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
-    container_name: hdx-oss-api
+  # task-check-alerts:
+  #   image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
+  #   entrypoint: 'node'
+  #   command: './build/tasks/index.js check-alerts'
+  #   environment:
+  #     APP_TYPE: 'scheduled-task'
+  #     CLICKHOUSE_HOST: http://ch-server:8123
+  #     CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+  #     CLICKHOUSE_PASSWORD: worker
+  #     CLICKHOUSE_USER: worker
+  #     EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
+  #     FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT} # need to be localhost (CORS)
+  #     HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
+  #     HDX_NODE_BETA_MODE: 0
+  #     HDX_NODE_CONSOLE_CAPTURE: 1
+  #     HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+  #     HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+  #     INGESTOR_API_URL: 'http://ingestor:8002'
+  #     MINER_API_URL: 'http://miner:5123'
+  #     MONGO_URI: 'mongodb://db:27017/hyperdx'
+  #     NODE_ENV: development
+  #     OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
+  #     OTEL_SERVICE_NAME: 'hdx-oss-task-check-alerts'
+  #     REDIS_URL: redis://redis:6379
+  #   restart: always
+  #   networks:
+  #     - internal
+  #   depends_on:
+  #     - ch-server
+  #     - db
+  #     - redis
+  app:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-app
     ports:
       - ${HYPERDX_API_PORT}:${HYPERDX_API_PORT}
+      - ${HYPERDX_APP_PORT}:${HYPERDX_APP_PORT}
     environment:
-      AGGREGATOR_API_URL: 'http://aggregator:8001'
       APP_TYPE: 'api'
       CLICKHOUSE_HOST: http://ch-server:8123
       CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
       CLICKHOUSE_PASSWORD: api
       CLICKHOUSE_USER: api
       EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
-      FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT} # need to be localhost (CORS)
+      FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT}
       HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
       HDX_NODE_BETA_MODE: 1
       HDX_NODE_CONSOLE_CAPTURE: 1
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_API_PORT: ${HYPERDX_API_PORT}
+      HYPERDX_APP_PORT: ${HYPERDX_APP_PORT}
+      HYPERDX_APP_URL: ${HYPERDX_APP_URL}
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
       INGESTOR_API_URL: 'http://ingestor:8002'
       MINER_API_URL: 'http://miner:5123'
       MONGO_URI: 'mongodb://db:27017/hyperdx'
-      NODE_ENV: development
+      NEXT_PUBLIC_CLICKHOUSE_HOST: http://ch-server:8123
+      NEXT_PUBLIC_SERVER_URL: http://127.0.0.1:${HYPERDX_API_PORT}
       OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
       OTEL_SERVICE_NAME: 'hdx-oss-api'
-      PORT: ${HYPERDX_API_PORT}
       REDIS_URL: redis://redis:6379
-      SERVER_URL: ${HYPERDX_API_URL}:${HYPERDX_API_PORT}
       USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-true}
     networks:
       - internal
@@ -182,23 +136,8 @@ services:
       - ch-server
       - db
       - redis
-  app:
-    image: ${IMAGE_NAME}:${IMAGE_VERSION}-app
-    container_name: hdx-oss-app
-    ports:
-      - ${HYPERDX_APP_PORT}:${HYPERDX_APP_PORT}
-    environment:
-      NEXT_PUBLIC_API_SERVER_URL: 'http://localhost:8000'
-      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      NODE_ENV: development
-      PORT: ${HYPERDX_APP_PORT}
-    networks:
-      - internal
-    depends_on:
-      - api
   ch-server:
-    image: clickhouse/clickhouse-server:23.8.8-alpine
-    container_name: hdx-oss-ch-server
+    image: clickhouse/clickhouse-server:head-alpine
     ports:
       - 8123:8123 # http api
       - 9000:9000 # native

docker/fullstack/Dockerfile

@@ -0,0 +1,75 @@
+# Starts several services in a single container for local use
+# - API (Node)
+# - App (Frontend)
+
+ARG NODE_VERSION=18.20.3
+
+# base #############################################################################################
+FROM node:${NODE_VERSION}-alpine AS base
+
+WORKDIR /app/api
+
+COPY ./yarn.lock ./.yarnrc.yml ./
+COPY ./.yarn ./.yarn
+COPY --from=api ./package.json .
+RUN yarn install && yarn cache clean
+
+WORKDIR /app/app
+
+COPY ./yarn.lock ./.yarnrc.yml ./
+COPY ./.yarn ./.yarn
+COPY --from=app ./package.json ./
+
+RUN yarn install && yarn cache clean
+
+
+## API Builder Image ###############################################################################
+FROM base AS api_builder
+
+WORKDIR /app/api
+
+COPY --from=api ./tsconfig.json ./
+COPY --from=api ./src ./src
+RUN yarn build && rm -rf node_modules && yarn workspaces focus --production
+
+
+# APP Builder Image ###############################################################################
+FROM base AS app_builder
+
+WORKDIR /app/app
+
+COPY --from=app ./.eslintrc.js ./next.config.js ./tsconfig.json ./next.config.js ./mdx.d.ts ./.eslintrc.js ./
+COPY --from=app ./src ./src 
+COPY --from=app ./pages ./pages 
+COPY --from=app ./public ./public 
+COPY --from=app ./styles ./styles
+
+ENV NEXT_TELEMETRY_DISABLED 1
+ENV NEXT_OUTPUT_STANDALONE false
+ENV NEXT_PUBLIC_IS_LOCAL_MODE false
+RUN yarn build && rm -rf node_modules && yarn workspaces focus --production
+
+
+# prod ############################################################################################
+FROM node:${NODE_VERSION}-alpine AS prod
+
+ENV NODE_ENV production
+
+USER node
+
+# Set up API
+WORKDIR /app/api
+COPY --chown=node:node --from=api_builder ./app/api/build ./build
+COPY --chown=node:node --from=api_builder ./app/api/node_modules ./node_modules
+
+# Set up App
+WORKDIR /app/app
+COPY --from=app_builder /app/app/next.config.js ./
+COPY --chown=node:node --from=app_builder /app/app/public ./public
+COPY --chown=node:node --from=app_builder /app/app/.next ./.next
+COPY --from=app_builder /app/app/node_modules ./node_modules
+COPY --from=app_builder /app/app/package.json ./package.json
+
+# Set up start script
+COPY --chown=node:node --from=fullstack ./entry.sh /etc/local/entry.sh
+CMD sh /etc/local/entry.sh

docker/fullstack/entry.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Use concurrently to run both the API and App servers
+npx concurrently \
+  "--kill-others" \
+  "--names=API,APP" \
+  "APP_TYPE=api PORT=${HYPERDX_API_PORT:-8000} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \
+  "/app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}"

docker/ingestor/Dockerfile

@@ -1,33 +0,0 @@
-## base #############################################################################################
-FROM timberio/vector:0.39.0-alpine AS base
-
-RUN mkdir -p /var/lib/vector
-VOLUME ["/var/lib/vector"]
-
-WORKDIR /app
-
-COPY ./*.toml ./
-
-
-## dev #############################################################################################
-FROM base as dev
-
-EXPOSE 8002 8686
-
-ENTRYPOINT ["vector", \
-  "-c", "sources.toml", \
-  "-c", "core.toml", \
-  "-c", "http-sinks.toml", \
-  "--require-healthy", "true"]
-
-
-## prod #############################################################################################
-FROM base as prod
-
-EXPOSE 8002 8686
-
-ENTRYPOINT ["vector", \
-  "-c", "sources.toml", \
-  "-c", "core.toml", \
-  "-c", "http-sinks.toml", \
-  "--require-healthy", "true"]

docker/ingestor/core.toml

@@ -1,980 +0,0 @@
-# Set global options
-data_dir = "/var/lib/vector"
-acknowledgements.enabled = true
-
-# --------------------------------------------------------------------------------
-# ------------------------------ Sources -----------------------------------------
-# --------------------------------------------------------------------------------
-[api]
-  enabled = true
-  address = "0.0.0.0:8686"
-
-[sources.vector_logs]
-  type = "internal_logs"
-
-# [sinks.console]
-#   type = "console"
-#   inputs = ["internal_logs"]
-#   encoding.codec = "json"
-# --------------------------------------------------------------------------------
-
-
-# --------------------------------------------------------------------------------
-# ------------------------------ Middleware --------------------------------------
-# --------------------------------------------------------------------------------
-[transforms.internal_logs]
-type = "remap"
-inputs = ["vector_logs"]
-source = '''
-  . = merge(.,
-    {
-      "hdx_token": "${HYPERDX_API_KEY-tacocat}",
-      "hdx_platform": "vector-internal"
-    }
-  )
-'''
-# WARNING: used for logs and spans only
-[transforms.process_headers_n_params]
-type = "remap"
-inputs = ["http_server"]
-source = '''
-  .hdx_content_type = del(."Content-Type")
-  .hdx_trace_id = split(del(.Traceparent), "-")[1] ?? null
-  tmp_sentry_auth = del(."X-Sentry-Auth")
-
-  # Hack sentry 😎
-  # for client like ruby
-  if is_string(tmp_sentry_auth) {
-    # input example: Sentry sentry_version=7, sentry_client=sentry-ruby/5.18.1, sentry_timestamp=1720998946, sentry_key=blabla
-    tmp_sentry_auth_headers = parse_key_value(tmp_sentry_auth, field_delimiter:",") ?? null
-    if is_object(tmp_sentry_auth_headers) {
-      .sentry_version = tmp_sentry_auth_headers."Sentry sentry_version"
-      .sentry_client = tmp_sentry_auth_headers.sentry_client
-      .sentry_key = tmp_sentry_auth_headers.sentry_key
-      tmp_timestamp = tmp_sentry_auth_headers.sentry_timestamp
-      if !is_nullish(tmp_timestamp) {
-        # override timestamp so its type is consistent
-        .b.timestamp = tmp_timestamp
-      }
-    }
-  }
-  if is_string(.sentry_key) && length(to_string(.sentry_key) ?? "") == 32 { 
-    .hdx_content_type = "application/json"
-    .hdx_platform = "sentry"
-    .hdx_token = (slice(.sentry_key, start: 0, end: 8) + "-" + slice(.sentry_key, start: 8, end: 12) + "-" + slice(.sentry_key, start: 12, end: 16) + "-" + slice(.sentry_key, start: 16, end: 20) + "-" + slice(.sentry_key, start: 20, end: 32)) ?? null
-  }
-'''
-[transforms.unnest_jsons]
-type = "remap"
-inputs = ["process_headers_n_params"]
-source = '''
-  if !includes(["otel-metrics"], .hdx_platform) && .hdx_content_type == "application/json" {
-    structured, err = parse_json(.message)
-    if err != null {
-      log("Unable to parse JSON: " + err, level: "warn")
-    } else {
-      .message = structured
-      if is_array(.message) {
-        ., err = unnest(.message)
-        if err != null {
-          log("unnest failed: " + err, level: "error")
-        }
-      }
-    }
-  }
-'''
-[transforms.extract_token]
-type = "remap"
-inputs = ["unnest_jsons"]
-source = '''
-  if is_nullish(.hdx_token) {
-    if includes(["otel-traces"], .hdx_platform) {
-      .hdx_token = del(.message.JaegerTag.__HDX_API_KEY)
-    } else {
-      .hdx_token = split(del(.authorization), " ")[1] ?? null
-      if is_nullish(.hdx_token) {
-        .hdx_token = del(.message.__HDX_API_KEY)
-      }
-    }
-    # TODO: support metrics
-  }
-
-  # attach a fake token for local mode
-  if is_nullish(.hdx_token) && "${IS_LOCAL_APP_MODE:-false}" == "true" {
-    .hdx_token = "00000000-0000-4000-a000-000000000000"
-  }
-
-  # check if token is in uuid format
-  if "${IS_LOCAL_APP_MODE:-false}" == "false" && !match(to_string(.hdx_token) ?? "", r'^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$') {
-    log("Invalid token: " + (to_string(.hdx_token) ?? ""), level: "warn")
-    .hdx_token = null
-  }
-
-  if !is_nullish(.hdx_token) {
-    .hdx_token_hash = md5(.hdx_token)
-  }
-'''
-[transforms.pre_filter]
-type = "remap"
-inputs = ["extract_token"]
-source = '''
-  hdx_message_size = strlen(encode_json(.))
-  if hdx_message_size > 6000000 {
-    token = to_string(.hdx_token) ?? ""
-    log("Message size too large: " + to_string(hdx_message_size) + " bytes, token = " + token, level: "warn")
-    # HACK: so the downstream filter can drop it
-    .message = {}
-    .hdx_token = null
-  }
-'''
-# --------------------------------------------------------------------------------
-
-
-# --------------------------------------------------------------------------------
-# ------------------------------ Logs Transform ----------------------------------
-# --------------------------------------------------------------------------------
-[transforms.filter_logs]
-type = "filter"
-inputs = ["pre_filter"]
-condition = '''
-  !includes(["otel-traces", "otel-metrics"], .hdx_platform) && !is_nullish(.hdx_token)
-'''
-
-[transforms.logs]
-type = "remap"
-inputs = ["filter_logs", "internal_logs"]
-drop_on_abort = true
-drop_on_error = true
-reroute_dropped = true
-source = '''
-.r = del(.message)
-if .hdx_platform == "vector-internal" {
-  .ts = to_unix_timestamp(parse_timestamp(del(.timestamp), format: "%+") ?? now(), unit: "nanoseconds")
-  .b = .
-  .b.message = .r
-  .b._hdx_body = .r
-  .h = .b.host
-  .st = downcase(.b.metadata.level) ?? null
-  .sv = .hdx_platform
-  del(.b.r)
-  del(.b.hdx_platform)
-  del(.b.hdx_token)
-  .r = .b
-} else if .hdx_platform == "sentry" {
-  .b = .r
-  if (to_int(.sentry_version) ?? 0) >= 7 && is_object(.b.contexts) {
-    .ts = (to_float(.b.timestamp) ?? 0.0) * 1000000000
-    .b._hdx_body = .b.message
-    .h = .b.server_name
-    .st = downcase(.b.level) ?? "fatal"
-
-    # extract service name
-    if exists(.b.contexts.hyperdx.serviceName) {
-      .sv = .b.contexts.hyperdx.serviceName
-    } else {
-      .sv = .b.platform
-    }
-
-    # extract request metadata
-    if is_object(.b.request) {
-      .b.span.kind = "server" # only for UI
-      .b.http.method = upcase(.b.request.method) ?? ""
-      .b.http.url = .b.request.url
-      .b.http.request.header = .b.request.headers
-      del(.b.request)
-    }
-
-    # extract body message
-    if is_nullish(.b._hdx_body) {
-      if is_array(.b.exception.values) {
-        exception_type = .b.exception.values[0].type
-        exception_value = .b.exception.values[0].value
-        if is_string(.b.transaction) {
-          exception_value = .b.transaction
-        } 
-        .b._hdx_body = (exception_type + ": " + exception_value) ?? ""
-      } else {
-        .b._hdx_body = "<unlabeled event>"
-      }
-    }
-
-    # user tags
-    if is_object(.b.user) {
-      .b.userEmail = del(.b.user.email)
-      .b.userId = del(.b.user.id)
-      .b.userName = del(.b.user.username)
-    }
-
-    # promote span id and trace id
-    .b.span_id = .b.contexts.trace.span_id
-    .b.trace_id = .hdx_trace_id # use "Traceparent" header
-    if is_nullish(.b.trace_id) {
-      .b.trace_id = .b.contexts.trace.trace_id
-    }
-  }
-} else if .hdx_content_type == "application/logplex-1" {
-  tmp_raw_msg = string(.r) ?? ""
-  tmp_attrs = parse_regex(tmp_raw_msg, r'^(\d+) <(\d+)>(\d+) (\S+) (\S+) (\S+) (\S+) - (.*)', numeric_groups: true) ?? null
-  if !is_object(tmp_attrs) {
-    log("Unable to parse logplex: '" + tmp_raw_msg + "', token: " + to_string(.hdx_token) ?? "", level: "warn")
-    del(.hdx_token)
-    del(.hdx_platform)
-  } else {
-    # convert .r into an object
-    .r.message = tmp_raw_msg
-
-    tmp_priority = to_int(tmp_attrs."2") ?? null
-    if is_integer(tmp_priority) {
-      .st = to_syslog_level(mod(tmp_priority, 8) ?? 6) ?? null
-    }
-    .h = tmp_attrs."5"
-    .sv = (tmp_attrs."6" + "-" + tmp_attrs."7") ?? ""
-    tmp_body = tmp_attrs."8"
-
-    .b = {}
-    .b._hdx_body = tmp_body
-    .b.heroku = {}
-    .b.heroku.app = del(.heroku_app)
-    .b.heroku.dyno = tmp_attrs."7"
-    .b.heroku.source = tmp_attrs."6"
-    .ts = to_unix_timestamp(parse_timestamp(del(tmp_attrs."4"), format: "%+") ?? now(), unit: "nanoseconds")
-
-    if contains(.sv, "heroku") {
-      structured = parse_key_value(tmp_body, accept_standalone_key: false) ?? null
-      if is_object(structured) {
-        .b = merge(.b, structured, deep: true) ?? .b
-        .h = .b.host
-      }
-      if exists(.b.at) {
-        .st = downcase(.b.at) ?? null
-      }
-
-      # parse integer fields
-      tmp_int_fields = [
-        "connect",
-        "service",
-        "status",
-        "bytes"
-      ]
-      for_each(tmp_int_fields) -> |_index, field| {
-        tmp_value = get(value: .b, path: [field]) ?? null
-        tmp_parsed_values = parse_regex(tmp_value, r'(\d+)', numeric_groups: true) ?? null
-        if is_object(tmp_parsed_values) {
-          tmp_parsed_value = to_int(tmp_parsed_values."0") ?? null
-          if is_integer(tmp_parsed_value) {
-            .b = set(value: .b, path: [field], data: tmp_parsed_value) ?? .b
-          }
-        }
-      }
-
-      for_each(.b) -> |key, value| {
-        if starts_with(key, "sample#") {
-          tmp_parsed_values = parse_regex(value, r'[-+]?(?:\d*\.*\d+)', numeric_groups: true) ?? null
-          if is_object(tmp_parsed_values) {
-            tmp_parsed_value = to_float(tmp_parsed_values."0") ?? null
-            if is_float(tmp_parsed_value) {
-              .b = set(value: .b, path: [key], data: tmp_parsed_value) ?? .b
-            }
-          }
-        }
-      }
-    } else {
-      parsed = false
-
-      # ruby logs
-      if !parsed {
-        structured = parse_regex(tmp_body, r'\[(.*) #(\d+)\]\s*(\S+) -- : (.*)', numeric_groups: true) ?? null
-        if is_object(structured) {
-          parsed = true
-          .b.pid = structured."2"
-          .b._hdx_body = structured."4"
-          .st = downcase(structured."3") ?? null
-        }
-      }
-
-      # JSON
-      if !parsed {
-        structured = parse_json(tmp_body) ?? null
-        if is_object(structured) {
-          parsed = true
-          .b = merge(.b, structured, deep: true) ?? .b
-          if !is_nullish(.b.message) {
-            .b._hdx_body = .b.message
-          }
-          if !is_nullish(.b.level) {
-            .st = downcase(.b.level) ?? null
-          }
-        }
-      }
-    }
-  }
-} else if contains(to_string(.hdx_content_type) ?? "", "text/plain", case_sensitive: false) {
-  .b = parse_json(.r) ?? .r
-  if is_object(.b) {
-    .r = .b
-    if .hdx_platform == "go" {
-      .b._hdx_body = .b.message
-      .st = downcase(.b.level) ?? null
-      .sv = del(.b.__hdx_sv)
-      .h = del(.b.__hdx_h)
-      .ts = to_unix_timestamp(parse_timestamp(.b.ts, format: "%+") ?? now(), unit: "nanoseconds")
-    }
-  }
-} else if .hdx_content_type == "application/json" {
-  .b = .r
-  if is_object(.b) {
-    if .hdx_platform == "nodejs" {
-      tmp_hdx = del(.b.__hdx)
-      .r = .b
-      .b._hdx_body = tmp_hdx.b
-      .h = tmp_hdx.h
-      .st = downcase(tmp_hdx.st) ?? null
-      .sv = tmp_hdx.sv
-      .ts = to_unix_timestamp(parse_timestamp(tmp_hdx.ts, format: "%+") ?? now(), unit: "nanoseconds")
-    } else if .hdx_platform == "elixir" {
-      .h = del(.b.__hdx_h)
-      .st = downcase(.b.erl_level) ?? downcase(.b.level) ?? null
-      .sv = del(.b.__hdx_sv)
-      .ts = to_unix_timestamp(parse_timestamp(.b.timestamp, format: "%+") ?? now(), unit: "nanoseconds")
-      .b._hdx_body = .b.message
-      structured = parse_json(.b.message) ?? null
-      if is_object(structured) {
-        .b = merge(.b, structured, deep: true) ?? .b
-      }
-    } else if .hdx_platform == "flyio" {
-      .h = .b.host
-      tmp_level = downcase(.b.log.level) ?? downcase(.b.log.severity) ?? "info"
-      if tmp_level != "info" {
-        .st = tmp_level
-      }
-      .ts = to_unix_timestamp(parse_timestamp(.b.timestamp, format: "%+") ?? now(), unit: "nanoseconds")
-      .b._hdx_body = .b.message
-      structured = parse_json(.b.message) ?? null
-      if is_object(structured) {
-        .b = merge(.b, structured, deep: true) ?? .b
-      }
-
-      # use user-specifed service name by default
-      .sv = .b."service.name"
-      if is_nullish(.sv) {
-        .sv = .b.fly.app.name
-      }
-
-      # TODO: maybe move this to post_logs
-      if !is_nullish(.b.message) {
-        .b._hdx_body = .b.message
-      }
-    } else if .hdx_platform == "vercel" {
-      .h = .b.host
-      .st = downcase(.b.level) ?? downcase(.b.severity) ?? null
-      .sv = .b.projectName
-      .ts = (to_int(.b.timestamp) ?? 0) * 1000000
-
-      # default to message
-      .b._hdx_body = .b.message
-
-      # build up custom message (apache http log format)
-      if exists(.b.proxy) {
-        # set status code
-        tmp_status = to_int(.b.proxy.statusCode) ?? 200
-        if tmp_status >= 500 {
-          .st = "error"
-        }
- 
-        .b._hdx_body = .b.proxy.clientIp + " - " + "\"" + (join([
-          .b.proxy.method,
-          .b.proxy.path,
-        ], separator: " ") ?? "") + "\"" + " " + to_string(.b.proxy.statusCode) ?? ""
-      }
-
-      # attach trace id
-      .b.trace_id = .b.requestId
-
-      # extract more props
-      if .b.source == "lambda" {
-        .b.Duration = to_float(parse_regex(.b.message, r'Duration: (?P<d>.*?) ms').d ?? null) ?? null
-        .b.BilledDuration = to_int(parse_regex(.b.message, r'Billed Duration: (?P<d>.*?) ms').d ?? null) ?? null
-        .b.InitDuration = to_float(parse_regex(.b.message, r'Init Duration: (?P<d>.*?) ms').d ?? null) ?? null
-        .b.MemorySize = to_int(parse_regex(.b.message, r'Memory Size: (?P<d>.*?) MB').d ?? null) ?? null
-        .b.MaxMemoryUsed = to_int(parse_regex(.b.message, r'Max Memory Used: (?P<d>.*?) MB').d ?? null) ?? null
-
-        tmp_splits = split(.b.message, "\n") ?? []
-        tmp_logs = []
-
-        for_each(tmp_splits) -> |_index, value| {
-          if !is_nullish(value) {
-            tmp_cur_log = {}
-            tmp_cur_log._hdx_body = value
-
-            tmp_msg_splits = split(value, "\t")
-            for_each(tmp_msg_splits) -> |__index, tmp_msg_split| {
-              if starts_with(tmp_msg_split, "{") && ends_with(tmp_msg_split, "}") {
-                _structured = parse_json(tmp_msg_split) ?? null
-                if is_object(_structured) {
-                  tmp_cur_log = merge(tmp_cur_log, _structured, deep: false) ?? tmp_cur_log
-                }
-              }
-            }
-
-            tmp_logs = push(tmp_logs, tmp_cur_log)
-          }
-        }
-
-        if length(tmp_logs) > 0 {
-          # will be split into multiple logs
-          .__hdx_logs = tmp_logs
-        }
-      }
-    } else if .hdx_platform == "aws-lambda" {
-      tmp_timestamp = to_int(.b."@timestamp") ?? 0
-      .b._hdx_body = .b.message
-      .ts = to_unix_timestamp(from_unix_timestamp(tmp_timestamp, unit: "milliseconds") ?? now(), unit: "nanoseconds")
-
-      structured = parse_json(.b.message) ?? null
-      if is_object(structured) {
-        .b = merge(.b, structured, deep: true) ?? .b
-      } else {
-        # BETA: extract json from message
-        # TODO: move this to post_logs if it's performant enough
-        if ends_with(string(.b.message) ?? "", "}") {
-          _SEARCH_CHARS_LENGTH = 64
-          left_most_bracket_index = find(slice(.b.message, start:0, end:_SEARCH_CHARS_LENGTH) ?? "", "{") ?? -1
-          if left_most_bracket_index != -1 {
-            tmp_json = slice(.b.message, start:left_most_bracket_index) ?? ""
-            structured = parse_json(tmp_json) ?? null
-            if is_object(structured) {
-              .b = merge(.b, structured, deep: true) ?? .b
-            }
-          }
-        }
-      }
-
-      # use user-specifed service name by default
-      tmp_type = del(.b.type)
-      .sv = .b."service.name"
-      if is_nullish(.sv) {
-        .sv = tmp_type
-        if is_nullish(.sv) {
-          .sv = .b.logGroup
-        }
-      }
-    } else if .hdx_platform == "aws-sns" {
-      .st = "ok"
-      .b._hdx_body = .b.Message
-      .ts = to_unix_timestamp(parse_timestamp(.b.Timestamp, format: "%+") ?? now(), unit: "nanoseconds")
-      structured = parse_json(.b.Message) ?? null
-      if is_object(structured) {
-        .b = merge(.b, structured, deep: true) ?? .b
-      }
-    } else if .hdx_platform == "otel-logs" {
-      del(.r."@timestamp")
-      tmp_timestamp = del(.b."@timestamp")
-      
-      # FIX: need to fix python SDK instead
-      if .b."telemetry.sdk.language" == "python" {
-        if .b."service.name" == "unknown_service" {
-          .b."service.name" = del(.b.otelServiceName)
-        }
-        if is_nullish(.b.span_id) {
-          .b.span_id = del(.b.otelSpanID)
-        }
-        if is_nullish(.b.trace_id) {
-          .b.trace_id = del(.b.otelTraceID)
-        }
-      } 
-
-      .h = .b.host
-      .sv = .b."service.name"
-      .ts = to_unix_timestamp(from_unix_timestamp(tmp_timestamp, unit: "milliseconds") ?? now(), unit: "nanoseconds")
-      .b._hdx_body = .b.message
-      structured = parse_json(.b.message) ?? null
-      if is_object(structured) {
-        .b = merge(.b, structured, deep: true) ?? .b
-      }
-
-      # extract k8s event metadata
-      # TODO: should we check "k8s.event.name" instead?
-      if .b."k8s.resource.name" == "events" && .b.object.kind == "Event" {
-        # set severity
-        if is_nullish(.b.level) && (.b.object.type == "Warning" || .b.object.type == "Normal") {
-          .b.level = .b.object.type
-        }
-
-        # check event format (cluster legacy v1 vs v1 events)
-        _targetObject = {}
-        if .b.object.apiVersion == "events.k8s.io/v1" && is_object(.b.object.regarding) {
-          _targetObject = .b.object.regarding
-          .b._hdx_body = .b.object.note
-        } else if .b.object.apiVersion == "v1" && is_object(.b.object.involvedObject) {
-          _targetObject = .b.object.involvedObject
-          .b._hdx_body = .b.object.message
-        }
-
-        # transform the attributes so that the log events use the k8s.* semantic conventions 
-        # ref: https://docs.honeycomb.io/integrations/kubernetes/kubernetes-events/
-        if _targetObject.kind == "Pod" {
-          .b."k8s.pod.name" = _targetObject.name
-          .b."k8s.pod.uid" = _targetObject.uid
-          .b."k8s.namespace.name" = _targetObject.namespace
-        } else if _targetObject.kind == "Node" {
-          .b."k8s.node.name" = _targetObject.name
-          .b."k8s.node.uid" = _targetObject.uid
-        } else if _targetObject.kind == "Job" {
-          .b."k8s.job.name" = _targetObject.name
-          .b."k8s.job.uid" = _targetObject.uid
-          .b."k8s.namespace.name" = _targetObject.namespace
-        } else if _targetObject.kind == "CronJob" {
-          .b."k8s.cronjob.name" = _targetObject.name
-          .b."k8s.cronjob.uid" = _targetObject.uid
-          .b."k8s.namespace.name" = _targetObject.namespace
-        }
-      }
-
-      # set severity after merging structured message (to avoid conflict)
-      .st = downcase(.b.level) ?? null
-
-      if exists(.b."rr-web.event") {
-        .hdx_platform = "rrweb"
-        temp_msg = .b.message
-        temp_msg_json = parse_json(temp_msg) ?? null
-        temp_rum_session_id = .b."rum.sessionId"
-        temp_rum_script_instance = .b."rum.scriptInstance"
-        temp_rrweb = {
-          "event": .b."rr-web.event",
-          "offset": .b."rr-web.offset",
-          "chunk": .b."rr-web.chunk",
-          "total-chunks": .b."rr-web.total-chunks"
-        }
-        .b = {}
-        .b._hdx_body = temp_msg
-        if is_object(temp_msg_json) {
-          .b.type = temp_msg_json.type
-        }
-        .b."rum.sessionId" = temp_rum_session_id
-        .b."rum.scriptInstance" = temp_rum_script_instance
-        .b."rr-web" = temp_rrweb
-      }
-    }
-  } else {
-    del(.b)
-  }
-} 
-del(.source_type)
-del(.timestamp)
-del(.authorization)
-del(.hdx_content_type)
-del(.hdx_trace_id)
-del(.sentry_client)
-del(.sentry_key)
-del(.sentry_version)
-'''
-
-[transforms.post_logs_unnest]
-type = "remap"
-inputs = ["logs"]
-drop_on_abort = true
-drop_on_error = true
-reroute_dropped = true
-source = '''
-if is_array(.__hdx_logs) {
-  ., err = unnest(.__hdx_logs)
-  if err != null {
-    log("unnest failed: " + err, level: "error")
-  }
-}
-'''
-
-[transforms.post_logs]
-type = "remap"
-inputs = ["post_logs_unnest"]
-drop_on_abort = true
-drop_on_error = true
-reroute_dropped = true
-source = '''
-# extract shared fields
-if is_object(.b) {
-  # extract span/trace id
-  .s_id = string(.b.span_id) ?? string(.b.spanID) ?? null
-  .t_id = string(.b.trace_id) ?? string(.b.traceID) ?? null
-  del(.b.spanID)
-  del(.b.span_id)
-  del(.b.traceID)
-  del(.b.trace_id)
-
-  .tso = to_unix_timestamp(now(), unit: "nanoseconds")
-
-  if is_nullish(.st) {
-    .st = downcase(.b.level) ?? downcase(.b.severity) ?? downcase(.b.LEVEL) ?? downcase(.b.SEVERITY) ?? null
-  }
-
-  # address .b.level and .st conflict
-  if !is_nullish(.b.level) && .b.level != .st {
-    .b.level = .st
-  }
-
-  # merge vercel logs
-  if is_object(.__hdx_logs) {
-    tmp_b_size = strlen(encode_json(.b))
-    tmp_hdx_logs_size = strlen(encode_json(.__hdx_logs))
-    tmp_total_size = tmp_b_size + tmp_hdx_logs_size
-
-    # Max expanded log size 16MB
-    if tmp_total_size > 16000000 {
-      log("__hdx_logs + body size too large: " + to_string(tmp_total_size) + " bytes, token = " + to_string(.hdx_token) ?? "", level: "warn")
-      del(.__hdx_logs)
-    } else {
-      .b = merge(.b, del(.__hdx_logs), deep: false) ?? .b
-    }
-
-    .b.message = .b._hdx_body
-    if is_object(.r) {
-      .r.message = .b._hdx_body
-    }
-  }
-
-  # add _hdx_body to raw logs (make it searchable)
-  if is_object(.r) {
-    if !is_nullish(.b._hdx_body) && !contains(encode_json(.r), to_string(.b._hdx_body) ?? "", case_sensitive: false) {
-      .r._hdx_body = .b._hdx_body
-    }
-  }
-}
-
-# validate timestamp and set it to observed timestamp if it is invalid
-if exists(.ts) {
-  a_day_from_now = to_unix_timestamp(now(), unit: "nanoseconds") + 86400000000000
-  nighty_days_ago = to_unix_timestamp(now(), unit: "nanoseconds") - 90 * 86400000000000
-  if (.ts > a_day_from_now ?? false) || (.ts < nighty_days_ago ?? false) {
-    .ts = .tso
-  }
-}
-
-# infer log level for raw logs
-if is_nullish(.st) || !is_string(.st) {
-  header = ""
-  if is_object(.r) && !is_nullish(.b._hdx_body) {
-    header = slice(to_string(.b._hdx_body) ?? "", start: 0, end: 256) ?? ""
-  } else {
-    header = slice(to_string(.r) ?? "", start: 0, end: 256) ?? ""
-  }
-
-  # should infer level by the order of severity
-  if contains(header, "emerg", case_sensitive: false) {
-    .st = "emergency"
-  } else if contains(header, "alert", case_sensitive: false) {
-    .st = "alert"
-  } else if contains(header, "crit", case_sensitive: false) {
-    .st = "critical"
-  } else if contains(header, "fatal", case_sensitive: false) {
-    .st = "fatal"
-  } else if contains(header, "error", case_sensitive: false) {
-    .st = "error"
-  } else if contains(header, "warn", case_sensitive: false) {
-    .st = "warn"
-  } else if contains(header, "notice", case_sensitive: false) {
-    .st = "notice"
-  } else if contains(header, "info", case_sensitive: false) {
-    .st = "info"
-  } else if contains(header, "debug", case_sensitive: false) {
-    .st = "debug"
-  } else if contains(header, "trace", case_sensitive: false) {
-    .st = "trace"
-  } else {
-    .st = "info"
-  }
-}
-
-# TODO: compute sn ?
-.sn = 0
-'''
-# --------------------------------------------------------------------------------
-
-
-# --------------------------------------------------------------------------------
-# ----------------------------- Spans Transform ----------------------------------
-# --------------------------------------------------------------------------------
-[transforms.filter_spans]
-type = "filter"
-inputs = ["pre_filter"]
-condition = '''
-  includes(["otel-traces"], .hdx_platform) && !is_nullish(.hdx_token)
-'''
-
-[transforms.spans]
-type = "remap"
-inputs = ["filter_spans"]
-drop_on_abort = true
-drop_on_error = true
-reroute_dropped = true
-source = '''
-.r = del(.message)
-if is_object(.r) {
-  tmp_JaegerTag = map_keys(object(.r.JaegerTag) ?? {}, recursive: true) -> |key| { replace(key,"@",".") }
-  tmp_process = map_keys(object(.r.process) ?? {}, recursive: true) -> |key| { replace(key,"@",".") } 
-  tmp_timestamp = del(.r."@timestamp")
-  .r.timestamp = tmp_timestamp
-  .r.JaegerTag = tmp_JaegerTag
-  .r.process = tmp_process
-  .s_n = .r.operationName
-  .s_id = .r.spanID
-  .t_id = .r.traceID
-  .p_id = null
-  ref = .r.references[0]
-  if (ref != null && ref.refType == "CHILD_OF") {
-    .p_id = ref.spanID
-  }
-  .b = tmp_JaegerTag
-  .b.process = tmp_process
-  .b.__events = .r.logs
-
-  # TODO: we want to delete the redundant .b.process.tag.x fields eventually
-  # TODO: maybe we want to move "tag" to the root level
-  # extract k8s tags
-  if !is_nullish(.b.process.tag."k8s.pod.name") {
-    .b."k8s.pod.name" = .b.process.tag."k8s.pod.name"
-  }
-  if !is_nullish(.b.process.tag."k8s.pod.uid") {
-    .b."k8s.pod.uid" = .b.process.tag."k8s.pod.uid"
-  }
-  if !is_nullish(.b.process.tag."k8s.namespace.name") {
-    .b."k8s.namespace.name" = .b.process.tag."k8s.namespace.name"
-  }
-  if !is_nullish(.b.process.tag."k8s.node.name") {
-    .b."k8s.node.name" = .b.process.tag."k8s.node.name"
-  }
-  if !is_nullish(.b.process.tag."k8s.deployment.name") {
-    .b."k8s.deployment.name" = .b.process.tag."k8s.deployment.name"
-  }
-
-  # copy common resource attributes to the top level
-  if is_nullish(.b."deployment.environment") && !is_nullish(.b.process.tag."deployment.environment") {
-    .b."deployment.environment" = .b.process.tag."deployment.environment"
-  }
-
-  if (.b."span.kind" == "server") {
-    if (exists(.b."http.status_code") && exists(.b."http.method") && exists(.b."http.route")) {
-      .b._hdx_body = join([
-        to_string(.b."http.status_code") ?? "",
-        .b."http.method",
-        .b."http.route",
-      ], separator: " ") ?? .s_n
-    }
-  } else if (.b."span.kind" == "client") {
-    if (exists(.b."http.status_code") && exists(.b."http.method") && exists(.b."http.url")) {
-      .b._hdx_body = join([
-        to_string(.b."http.status_code") ?? "",
-        .b."http.method",
-        .b."http.url",
-      ], separator: " ") ?? .s_n
-    }
-  } else if (.b."span.kind" == "internal") {
-    if .b.component == "console" {
-      .b._hdx_body = .b.message
-      .st = .b.level
-    }
-  }
-
-  if ((to_int(.b.error) ?? 0) == 1) {
-    .st = "error"
-    if !is_nullish(.b."otel.status_description") {
-      .b._hdx_body = .b."otel.status_description"
-    } else if !is_nullish(.b."error.message") {
-      .b._hdx_body = .b."error.message"
-    }
-  } else if is_array(.b.__events) {
-    for_each(array(.b.__events) ?? []) -> |_index, value| {
-      if is_object(value) {
-        if (value.fields[0].key == "event" && value.fields[0].value == "exception") {
-          .st = "error"
-        }
-      }
-    }
-  }
-
-  # set default values
-  if is_nullish(.st) {
-    .st = "ok"
-  }
-  if is_nullish(.b._hdx_body) {
-    .b._hdx_body = .s_n
-  }
-
-  # RN instrumentation
-  if exists(.b."process.serviceName") {
-    .sv = .b."process.serviceName"
-  } else {
-    .sv = .r.process.serviceName
-  }
-
-  .ts = to_unix_timestamp(from_unix_timestamp(.r.startTimeMillis, unit: "milliseconds") ?? now(), unit: "nanoseconds")
-  .et = .ts + .r.duration * 1000 ?? 0
-  .tso = to_unix_timestamp(now(), unit: "nanoseconds")
-
-  # TODO: move this to post_spans
-  # add _hdx_body to raw log
-  if !is_nullish(.b._hdx_body) && !contains(encode_json(.r), to_string(.b._hdx_body) ?? "", case_sensitive: false) {
-    .r._hdx_body = .b._hdx_body
-  }
-
-  del(.source_type)
-  del(.timestamp)
-  del(.authorization)
-  del(.hdx_content_type)
-}
-'''
-
-[transforms.post_spans]
-type = "filter"
-inputs = ["spans"]
-condition = '''
-  ("${ENABLE_GO_PARSER:-false}" == "true" && is_nullish(.b."db.statement")) || "${ENABLE_GO_PARSER:-false}" == "false"
-'''
-
-[transforms.go_spans]
-type = "filter"
-inputs = ["spans"]
-condition = '''
-  "${ENABLE_GO_PARSER:-false}" == "true" && !is_nullish(.b."db.statement")
-'''
-# --------------------------------------------------------------------------------
-
-
-# --------------------------------------------------------------------------------
-# ---------------------------- Metrics Transform ---------------------------------
-# --------------------------------------------------------------------------------
-[transforms.filter_metrics]
-type = "filter"
-inputs = ["http_server"]
-condition = '''
-  includes(["otel-metrics"], .hdx_platform)
-'''
-
-[transforms.pre_metrics]
-type = "remap"
-inputs = ["filter_metrics"]
-source = '''
-  del(.path)
-  del(.source_type)
-  del(.timestamp)
-  del(.authorization)
-  del(.hdx_content_type)
-  tmp_msg = del(.message)
-  .message = split(tmp_msg, "}}") ?? []
-  . = unnest(.message)
-'''
-
-[transforms.metrics]
-type = "remap"
-inputs = ["pre_metrics"]
-drop_on_abort = true
-drop_on_error = true
-reroute_dropped = true
-source = '''
-  tmp = (del(.message) + "}}") ?? null
-  structured, err = parse_json(tmp)
-  if err == null && structured.event == "metric" {
-    # TODO: do this at extract_token
-    .hdx_token = del(structured.fields.__HDX_API_KEY)
-    .at = to_int(del(structured.fields.metric_aggregation_temporality)) ?? 0
-    .dt = del(structured.fields.metric_type)
-    .im = to_bool(del(structured.fields.metric_is_monotonic)) ?? null
-    .u = del(structured.fields.metric_unit)
-
-    filtered_keys = []
-    for_each(object(structured.fields) ?? {})-> |key, value| {
-      if is_integer(value) || is_float(value) {
-        filtered_keys = push(filtered_keys, key)
-        .n = replace(key, "metric_name:", "")
-        .v = value
-      }
-    }
-    .tso = to_unix_timestamp(now(), unit: "nanoseconds")
-    .ts = to_int(structured.time * 1000000000 ?? null) ?? .tso
-    .b = filter(object(structured.fields) ?? {})-> |key, value| {
-      !includes(filtered_keys, key)
-    }
-    .b.host = structured.host
-
-    # Extra K8s tags
-    if .n == "k8s.pod.phase" {
-      tmp_metrics = []
-      tmp_metrics = push(tmp_metrics, .)
-      tmp_states = [
-        "pending",
-        "running",
-        "succeeded",
-        "failed",
-        "unknown"
-      ]
-      # Create new metrics "k8s.pod.status_phase"
-      for_each(tmp_states) -> |_index, value| {
-        _tmp_metric = .
-        _tmp_metric.n = "k8s.pod.status_phase"
-        _tmp_metric.v = 0
-        _tmp_metric.b.state = value
-        if .v == 1 && value == "pending" {
-          _tmp_metric.v = 1
-        } else if .v == 2 && value == "running" {
-          _tmp_metric.v = 1
-        } else if .v == 3 && value == "succeeded" {
-          _tmp_metric.v = 1
-        } else if .v == 4 && value == "failed" {
-          _tmp_metric.v = 1
-        } else if .v == 5 && value == "unknown" {
-          _tmp_metric.v = 1
-        }
-        tmp_metrics = push(tmp_metrics, _tmp_metric)
-      }
-      .__hdx_metrics = tmp_metrics
-    }
-  }
-'''
-
-[transforms.post_metrics_unnest]
-type = "remap"
-inputs = ["metrics"]
-drop_on_abort = true
-drop_on_error = true
-reroute_dropped = true
-source = '''
-if is_array(.__hdx_metrics) {
-  ., err = unnest(.__hdx_metrics)
-  if err != null {
-    log("unnest failed: " + err, level: "error")
-  }
-}
-'''
-
-[transforms.post_metrics]
-type = "remap"
-inputs = ["post_metrics_unnest"]
-drop_on_abort = true
-drop_on_error = true
-reroute_dropped = true
-source = '''
-if is_object(.__hdx_metrics) {
-  tmp = .__hdx_metrics
-  . = tmp
-}
-'''
-# --------------------------------------------------------------------------------
-
-
-# --------------------------------------------------------------------------------
-# --------------------------------- Debug ----------------------------------------
-# --------------------------------------------------------------------------------
-[transforms.debug_dropped]
-type = "remap"
-inputs = [
-  "logs.dropped",
-  "post_logs_unnest.dropped",
-  "post_logs.dropped",
-  "spans.dropped",
-  "metrics.dropped",
-  "post_metrics_unnest.dropped",
-  "post_metrics.dropped"
-]
-source = '''
-  log(., level: "error")
-'''
-# --------------------------------------------------------------------------------

docker/ingestor/http-sinks.toml

@@ -1,36 +0,0 @@
-# --------------------------------------------------------------------------------
-# --------------------------------- Sinks ----------------------------------------
-# --------------------------------------------------------------------------------
-
-[sinks.go_parser]
-type = "http"
-uri = "${GO_PARSER_API_URL}"
-inputs = ["go_spans"] # only send spans for now
-compression = "gzip"
-encoding.codec = "json"
-batch.max_bytes = 10485760 # 10MB, required for rrweb payloads 
-batch.max_events = 100
-batch.timeout_secs = 1
-
-
-[sinks.dev_hdx_aggregator]
-type = "http"
-uri = "${AGGREGATOR_API_URL}"
-inputs = ["post_spans", "post_logs"]
-compression = "gzip"
-encoding.codec = "json"
-batch.max_bytes = 10485760 # 10MB, required for rrweb payloads 
-batch.max_events = 100
-batch.timeout_secs = 1
-
-
-[sinks.dev_hdx_metrics_aggregator]
-type = "http"
-uri = "${AGGREGATOR_API_URL}?telemetry=metric"
-inputs = ["metrics"]
-compression = "gzip"
-encoding.codec = "json"
-batch.max_bytes = 100000
-batch.max_events = 100
-batch.timeout_secs = 1
-# --------------------------------------------------------------------------------

docker/ingestor/run_linting.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-directory="./docker/ingestor"
-
-export AGGREGATOR_API_URL="http://aggregator:8001"
-export GO_PARSER_API_URL="http://go-parser:7777"
-
-vector validate --no-environment $directory/*.toml

docker/ingestor/sources.toml

@@ -1,13 +0,0 @@
-[sources.http_server]
-type = "http_server"
-address = "0.0.0.0:8002"
-headers = ["authorization", "Content-Type", "Traceparent", "X-Sentry-Auth"]
-strict_path = false
-path = ""
-query_parameters = [
-  "hdx_platform",
-  "hdx_token",
-  "sentry_client",
-  "sentry_key",
-  "sentry_version"
-]

docker/local/Dockerfile

@@ -51,7 +51,7 @@ COPY --from=app ./styles ./styles
 ENV NEXT_TELEMETRY_DISABLED 1
 ENV NEXT_OUTPUT_STANDALONE true
 ENV NEXT_PUBLIC_IS_LOCAL_MODE true
-RUN yarn build
+RUN yarn build && rm -rf node_modules && yarn workspaces focus --production
 
 # == Clickhouse/Base Image ==
 FROM clickhouse/clickhouse-server:${CLICKHOUSE_VERSION}-alpine AS clickhouse_base
@@ -118,9 +118,11 @@ COPY --from=otel-collector ./config.local.yaml /etc/otelcol-contrib/config.yaml
 
 # Set up App
 WORKDIR /app/app
-COPY --from=app_builder ./app/app/public .
-COPY --from=app_builder ./app/app/.next/standalone .
-COPY --from=app_builder ./app/app/.next/static ./.next/static
+COPY --from=app_builder /app/app/next.config.js ./
+COPY --from=app_builder /app/app/public ./public
+COPY --from=app_builder /app/app/.next ./.next
+COPY --from=app_builder /app/app/node_modules ./node_modules
+COPY --from=app_builder /app/app/package.json ./package.json
 
 # Expose ports
 EXPOSE 8000 8080 4317 4318 13133 8123 9000

docker/local/entry.sh

@@ -7,7 +7,7 @@ export CLICKHOUSE_LOG_LEVEL="error"
 # User can specify either an entire SERVER_URL, or override slectively the 
 # HYPERDX_API_URL or HYPERDX_API_PORT from the defaults
 # Same applies to the frontend/app
-export SERVER_URL="${SERVER_URL:-${HYPERDX_API_URL:-http://localhost}:${HYPERDX_API_PORT:-8000}}"
+export SERVER_URL="http://127.0.0.1:${HYPERDX_API_PORT:-8000}"
 export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}"
 
 # Internal Services
@@ -60,9 +60,8 @@ otelcol-contrib --config /etc/otelcol-contrib/config.yaml &
 
 # App
 NODE_ENV=production \
-PORT=8080 \
 NEXT_PUBLIC_SERVER_URL="${SERVER_URL}" \
-node /app/app/server.js > /var/log/app.log 2>&1 &
+/app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080} > /var/log/app.log 2>&1 &
 
 # Wait for any process to exit
 wait -n

packages/api/Dockerfile

@@ -1,5 +1,5 @@
 ## base #############################################################################################
-FROM 597726328204.dkr.ecr.us-east-2.amazonaws.com/ecr-public/docker/library/node:18.20.3-alpine AS base
+FROM node:18.20.3-alpine AS base
 
 WORKDIR /app
 
@@ -28,7 +28,7 @@ RUN yarn run build
 
 ## prod ############################################################################################
 
-FROM 597726328204.dkr.ecr.us-east-2.amazonaws.com/ecr-public/docker/library/node:18.20.3-alpine AS prod
+FROM node:18.20.3-alpine AS prod
 
 ARG CODE_VERSION
 

packages/api/package.json

@@ -11,7 +11,6 @@
     "@hyperdx/lucene": "^3.1.1",
     "@hyperdx/node-opentelemetry": "^0.8.1",
     "@opentelemetry/api": "^1.8.0",
-    "@sentry/node": "^7.85.0",
     "@slack/webhook": "^6.1.0",
     "axios": "^1.6.2",
     "compression": "^1.7.4",

packages/api/src/api-app.ts

@@ -1,9 +1,7 @@
-import * as Sentry from '@sentry/node';
 import compression from 'compression';
 import MongoStore from 'connect-mongo';
 import express from 'express';
 import session from 'express-session';
-import { createProxyMiddleware } from 'http-proxy-middleware';
 import ms from 'ms';
 import onHeaders from 'on-headers';
 
@@ -23,22 +21,6 @@ import passport from './utils/passport';
 
 const app: express.Application = express();
 
-if (config.SENTRY_DSN) {
-  Sentry.init({
-    dsn: config.SENTRY_DSN,
-    environment: config.NODE_ENV,
-    release: config.CODE_VERSION,
-  });
-
-  Sentry.setContext('hyperdx', {
-    serviceName: config.OTEL_SERVICE_NAME,
-  });
-}
-
-// RequestHandler creates a separate execution context using domains, so that every
-// transaction/span/breadcrumb is attached to its own Hub instance
-app.use(Sentry.Handlers.requestHandler());
-
 const sess: session.SessionOptions & { cookie: session.CookieOptions } = {
   resave: false,
   saveUninitialized: false,
@@ -51,10 +33,13 @@ const sess: session.SessionOptions & { cookie: session.CookieOptions } = {
   store: new MongoStore({ mongoUrl: config.MONGO_URI }),
 };
 
-if (config.IS_PROD) {
-  app.set('trust proxy', 1); // Super important or cookies don't get set in prod
-  sess.cookie.secure = true;
-  sess.cookie.domain = config.COOKIE_DOMAIN;
+app.set('trust proxy', 1);
+if (config.FRONTEND_URL) {
+  const feUrl = new URL(config.FRONTEND_URL);
+  sess.cookie.domain = feUrl.hostname;
+  if (feUrl.protocol === 'https:') {
+    sess.cookie.secure = true;
+  }
 }
 
 app.disable('x-powered-by');
@@ -124,10 +109,6 @@ app.use('/clickhouse-proxy', isUserAuthenticated, clickhouseProxyRouter);
 // API v1
 app.use('/api/v1', externalRoutersV1);
 
-// ---------------------------------------------------------------------
-// The error handler must be before any other error middleware and after all controllers
-app.use(Sentry.Handlers.errorHandler());
-
 // error handling
 app.use(appErrorHandler);
 

packages/api/src/config.ts

@@ -9,13 +9,11 @@ export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string;
 export const CLICKHOUSE_PASSWORD = env.CLICKHOUSE_PASSWORD as string;
 export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string;
 export const CODE_VERSION = env.CODE_VERSION as string;
-export const COOKIE_DOMAIN = env.COOKIE_DOMAIN as string; // prod ONLY
 export const EXPRESS_SESSION_SECRET = env.EXPRESS_SESSION_SECRET as string;
 export const FRONTEND_URL = env.FRONTEND_URL as string;
 export const HYPERDX_API_KEY = env.HYPERDX_API_KEY as string;
 export const HYPERDX_LOG_LEVEL = env.HYPERDX_LOG_LEVEL as string;
 export const INGESTOR_API_URL = env.INGESTOR_API_URL as string;
-export const SENTRY_DSN = env.SENTRY_DSN as string;
 export const IS_CI = NODE_ENV === 'ci';
 export const IS_DEV = NODE_ENV === 'development';
 export const IS_PROD = NODE_ENV === 'production';
@@ -26,7 +24,6 @@ export const OTEL_EXPORTER_OTLP_ENDPOINT =
 export const OTEL_SERVICE_NAME = env.OTEL_SERVICE_NAME as string;
 export const PORT = Number.parseInt(env.PORT as string);
 export const REDIS_URL = env.REDIS_URL as string;
-export const SERVER_URL = env.SERVER_URL as string;
 export const USAGE_STATS_ENABLED = env.USAGE_STATS_ENABLED !== 'false';
 
 // Only for single container local deployments, disable authentication

packages/api/src/middleware/error.ts

@@ -1,8 +1,8 @@
+import { recordException } from '@hyperdx/node-opentelemetry';
 import type { NextFunction, Request, Response } from 'express';
-import { serializeError } from 'serialize-error';
 
+import { IS_PROD } from '@/config';
 import { BaseError, isOperationalError, StatusCode } from '@/utils/errors';
-import logger from '@/utils/logger';
 
 // WARNING: need to keep the 4th arg for express to identify it as an error-handling middleware function
 export const appErrorHandler = (
@@ -11,18 +11,24 @@ export const appErrorHandler = (
   res: Response,
   next: NextFunction,
 ) => {
-  logger.error({
-    location: 'appErrorHandler',
-    error: serializeError(err),
-  });
+  if (!IS_PROD) {
+    console.error(err);
+  }
 
   const userFacingErrorMessage = isOperationalError(err)
-    ? err.message
+    ? err.name || err.message
     : 'Something went wrong :(';
 
+  void recordException(err, {
+    mechanism: {
+      type: 'generic',
+      handled: userFacingErrorMessage ? true : false,
+    },
+  });
+
   if (!res.headersSent) {
-    res
-      .status(err.statusCode ?? StatusCode.INTERNAL_SERVER)
-      .send(userFacingErrorMessage);
+    res.status(err.statusCode ?? StatusCode.INTERNAL_SERVER).json({
+      message: userFacingErrorMessage,
+    });
   }
 };

packages/api/src/routers/api/__tests__/team.test.ts

@@ -27,8 +27,7 @@ describe('team router', () => {
     expect(_.omit(resp.body, ['_id', 'apiKey'])).toMatchInlineSnapshot(`
 Object {
   "allowedAuthMethods": Array [],
-  "name": "fake@deploysentinel.com's Team",
-  "sentryDSN": "",
+  "name": "fake@deploysentinel.com's Team"
 }
 `);
   });

packages/api/src/routers/api/team.ts

@@ -1,7 +1,6 @@
 import crypto from 'crypto';
 import express from 'express';
 import pick from 'lodash/pick';
-import { serializeError } from 'serialize-error';
 import { z } from 'zod';
 import { validateRequest } from 'zod-express-middleware';
 
@@ -13,25 +12,10 @@ import {
   findUsersByTeam,
 } from '@/controllers/user';
 import TeamInvite from '@/models/teamInvite';
-import logger from '@/utils/logger';
 import { objectIdSchema } from '@/utils/zod';
 
 const router = express.Router();
 
-const getSentryDSN = (apiKey: string, ingestorApiUrl: string) => {
-  try {
-    const url = new URL(ingestorApiUrl);
-    url.username = apiKey.replaceAll('-', '');
-    url.pathname = '0';
-    // TODO: Set up hostname from env variable
-    url.hostname = 'localhost';
-    return url.toString();
-  } catch (e) {
-    logger.error(serializeError(e));
-    return '';
-  }
-};
-
 router.get('/', async (req, res, next) => {
   try {
     const teamId = req.user?.team;
@@ -56,10 +40,7 @@ router.get('/', async (req, res, next) => {
       throw new Error(`Team ${teamId} not found for user ${userId}`);
     }
 
-    res.json({
-      ...team.toJSON(),
-      sentryDSN: getSentryDSN(team.apiKey, config.INGESTOR_API_URL),
-    });
+    res.json(team.toJSON());
   } catch (e) {
     next(e);
   }

packages/api/src/server.ts

@@ -2,7 +2,6 @@ import http from 'http';
 import gracefulShutdown from 'http-graceful-shutdown';
 import { serializeError } from 'serialize-error';
 
-import * as clickhouse from './clickhouse';
 import * as config from './config';
 import { connectDB, mongooseConnection } from './models';
 import logger from './utils/logger';
@@ -35,12 +34,10 @@ export default class Server {
   protected async shutdown(signal?: string) {
     let hasError = false;
     logger.info('Closing all db clients...');
-    const [redisCloseResult, mongoCloseResult, clickhouseCloseResult] =
-      await Promise.allSettled([
-        redisClient.disconnect(),
-        mongooseConnection.close(false),
-        clickhouse.client.close(),
-      ]);
+    const [redisCloseResult, mongoCloseResult] = await Promise.allSettled([
+      redisClient.disconnect(),
+      mongooseConnection.close(false),
+    ]);
 
     if (redisCloseResult.status === 'rejected') {
       hasError = true;
@@ -56,13 +53,6 @@ export default class Server {
       logger.info('MongoDB client closed.');
     }
 
-    if (clickhouseCloseResult.status === 'rejected') {
-      hasError = true;
-      logger.error(serializeError(clickhouseCloseResult.reason));
-    } else {
-      logger.info('Clickhouse client closed.');
-    }
-
     if (hasError) {
       throw new Error('Failed to close all clients.');
     }
@@ -95,10 +85,6 @@ export default class Server {
       });
     }
 
-    await Promise.all([
-      connectDB(),
-      redisClient.connect(),
-      clickhouse.connect(),
-    ]);
+    await Promise.all([connectDB(), redisClient.connect()]);
   }
 }

packages/app/Dockerfile

@@ -1,5 +1,5 @@
 ## base #############################################################################################
-FROM 597726328204.dkr.ecr.us-east-2.amazonaws.com/ecr-public/docker/library/node:18.20.3-alpine AS base
+FROM node:18.20.3-alpine AS base
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk add --no-cache libc6-compat
 WORKDIR /app
@@ -26,10 +26,10 @@ FROM base AS builder
 # doc: https://nextjs.org/docs/pages/building-your-application/configuring/environment-variables#bundling-environment-variables-for-the-browser
 ARG OTEL_EXPORTER_OTLP_ENDPOINT
 ARG OTEL_SERVICE_NAME
-ARG SERVER_URL
+ARG IS_LOCAL_MODE
 ENV NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT $OTEL_EXPORTER_OTLP_ENDPOINT
 ENV NEXT_PUBLIC_OTEL_SERVICE_NAME $OTEL_SERVICE_NAME
-ENV NEXT_PUBLIC_SERVER_URL $SERVER_URL
+ENV NEXT_PUBLIC_IS_LOCAL_MODE $IS_LOCAL_MODE
 
 COPY ./packages/app/tsconfig.json ./packages/app/next.config.js ./packages/app/mdx.d.ts ./packages/app/.eslintrc.js ./
 COPY ./packages/app/src ./src 
@@ -37,11 +37,11 @@ COPY ./packages/app/pages ./pages
 COPY ./packages/app/public ./public 
 COPY ./packages/app/styles ./styles
 COPY --from=base /app/node_modules ./node_modules
-RUN yarn build && yarn install --production --ignore-scripts --prefer-offline
+RUN yarn build && rm -rf node_modules && yarn workspaces focus --production
 
 
 ## prod ############################################################################################
-FROM 597726328204.dkr.ecr.us-east-2.amazonaws.com/ecr-public/docker/library/node:18.20.3-alpine AS prod
+FROM node:18.20.3-alpine AS prod
 WORKDIR /app
 
 ENV NODE_ENV production
@@ -50,8 +50,8 @@ RUN addgroup -g 1001 -S nodejs
 RUN adduser -S nextjs -u 1001
 
 # You only need to copy next.config.js if you are NOT using the default configuration
-# COPY --from=builder /app/next.config.js ./
-COPY --from=builder /app/public ./public
+COPY --from=builder /app/next.config.js ./
+COPY --from=builder --chown=nextjs:nodejs /app/public ./public
 COPY --from=builder --chown=nextjs:nodejs /app/.next ./.next
 COPY --from=builder /app/node_modules ./node_modules
 COPY --from=builder /app/package.json ./package.json

packages/app/next.config.js

@@ -8,29 +8,45 @@ const withNextra = require('nextra')({
   themeConfig: './src/nextra.config.tsx',
 });
 
-module.exports = withNextra({
-  async headers() {
-    return [
-      {
-        source: '/(.*)?', // Matches all pages
-        headers: [
-          {
-            key: 'X-Frame-Options',
-            value: 'DENY',
-          },
-        ],
-      },
-    ];
+module.exports = {
+  experimental: {
+    instrumentationHook: true,
   },
-  // This slows down builds by 2x for some reason...
-  swcMinify: false,
-  publicRuntimeConfig: {
-    version,
+  // Ignore otel pkgs warnings
+  // https://github.com/open-telemetry/opentelemetry-js/issues/4173#issuecomment-1822938936
+  webpack: (
+    config,
+    { buildId, dev, isServer, defaultLoaders, nextRuntime, webpack },
+  ) => {
+    if (isServer) {
+      config.ignoreWarnings = [{ module: /opentelemetry/ }];
+    }
+    return config;
   },
-  productionBrowserSourceMaps: false,
-  ...(process.env.NEXT_OUTPUT_STANDALONE === 'true'
-    ? {
-        output: 'standalone',
-      }
-    : {}),
-});
+  ...withNextra({
+    async headers() {
+      return [
+        {
+          source: '/(.*)?', // Matches all pages
+          headers: [
+            {
+              key: 'X-Frame-Options',
+              value: 'DENY',
+            },
+          ],
+        },
+      ];
+    },
+    // This slows down builds by 2x for some reason...
+    swcMinify: false,
+    publicRuntimeConfig: {
+      version,
+    },
+    productionBrowserSourceMaps: false,
+    ...(process.env.NEXT_OUTPUT_STANDALONE === 'true'
+      ? {
+          output: 'standalone',
+        }
+      : {}),
+  }),
+};

packages/app/package.json

@@ -7,7 +7,7 @@
     "node": ">=18.12.0"
   },
   "scripts": {
-    "dev": "NEXT_PUBLIC_SERVER_URL=http://localhost:8000 next dev -p 8080",
+    "dev": "next dev",
     "dev:local": "NEXT_PUBLIC_CLICKHOUSE_HOST=http://localhost:8123 NEXT_PUBLIC_IS_LOCAL_MODE=true next dev -p 8080",
     "build": "next build",
     "start": "next start",
@@ -28,6 +28,7 @@
     "@hookform/resolvers": "^3.9.0",
     "@hyperdx/browser": "^0.21.1",
     "@hyperdx/lucene": "^3.1.1",
+    "@hyperdx/node-opentelemetry": "^0.8.1",
     "@lezer/highlight": "^1.2.0",
     "@mantine/core": "7.9.2",
     "@mantine/dates": "^7.11.2",
@@ -51,6 +52,7 @@
     "date-fns": "^2.28.0",
     "date-fns-tz": "^2.0.0",
     "fuse.js": "^6.6.2",
+    "http-proxy-middleware": "^3.0.3",
     "immer": "^9.0.21",
     "jotai": "^2.5.1",
     "ky": "^0.30.0",

packages/app/pages/_app.tsx

@@ -16,7 +16,6 @@ import {
 } from '@tanstack/react-query';
 import { ReactQueryDevtools } from '@tanstack/react-query-devtools';
 
-import { apiConfigs } from '@/api';
 import { IS_LOCAL_MODE } from '@/config';
 import { ThemeWrapper } from '@/ThemeWrapper';
 import { useConfirmModal } from '@/useConfirm';
@@ -62,16 +61,6 @@ export default function MyApp({ Component, pageProps }: AppPropsWithLayout) {
     fetch('/api/config')
       .then(res => res.json())
       .then(_jsonData => {
-        // Set API url dynamically for users who aren't rebuilding
-        try {
-          const url = new URL(_jsonData.apiServerUrl);
-          if (url != null) {
-            apiConfigs.prefixUrl = url.toString().replace(/\/$/, '');
-          }
-        } catch (err) {
-          // ignore
-        }
-
         if (_jsonData?.apiKey) {
           let hostname;
           try {
@@ -86,6 +75,7 @@ export default function MyApp({ Component, pageProps }: AppPropsWithLayout) {
             maskAllInputs: true,
             maskAllText: true,
             service: _jsonData.serviceName,
+            disableReplay: true,
             // tracePropagationTargets: [new RegExp(hostname ?? 'localhost', 'i')],
             url: _jsonData.collectorUrl,
           });

packages/app/pages/api/[...all].ts

@@ -0,0 +1,38 @@
+import { NextApiRequest, NextApiResponse } from 'next';
+import { createProxyMiddleware, fixRequestBody } from 'http-proxy-middleware';
+
+import { IS_DEV } from '@/config';
+
+export const config = {
+  api: {
+    externalResolver: true,
+    bodyParser: true,
+  },
+};
+
+export default (req: NextApiRequest, res: NextApiResponse) => {
+  const proxy = createProxyMiddleware({
+    changeOrigin: true,
+    // logger: console, // DEBUG
+    pathRewrite: { '^/api': '' },
+    target: process.env.NEXT_PUBLIC_SERVER_URL,
+    autoRewrite: true,
+    /**
+     * Fix bodyParser
+     **/
+    on: {
+      proxyReq: fixRequestBody,
+    },
+    ...(IS_DEV && {
+      logger: console,
+    }),
+  });
+  return proxy(req, res, error => {
+    if (error) {
+      console.error(error);
+      res.status(500).send('API proxy error');
+      return;
+    }
+    res.status(404).send('Not found');
+  });
+};

packages/app/pages/api/config.ts

@@ -1,11 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 
-import {
-  HDX_API_KEY,
-  HDX_COLLECTOR_URL,
-  HDX_SERVICE_NAME,
-  SERVER_URL,
-} from '@/config';
+import { HDX_API_KEY, HDX_COLLECTOR_URL, HDX_SERVICE_NAME } from '@/config';
 import type { NextApiConfigResponseData } from '@/types';
 
 export default function handler(
@@ -14,7 +9,6 @@ export default function handler(
 ) {
   res.status(200).json({
     apiKey: HDX_API_KEY,
-    apiServerUrl: SERVER_URL,
     collectorUrl: HDX_COLLECTOR_URL,
     serviceName: HDX_SERVICE_NAME,
   });

packages/app/src/AppNav.tsx

@@ -41,7 +41,7 @@ import {
   AppNavUserMenu,
 } from './AppNav.components';
 import AuthLoadingBlocker from './AuthLoadingBlocker';
-import { IS_LOCAL_MODE, SERVER_URL } from './config';
+import { IS_LOCAL_MODE } from './config';
 import Icon from './Icon';
 import Logo from './Logo';
 import { useSavedSearches, useUpdateSavedSearch } from './savedSearch';
@@ -1106,7 +1106,7 @@ export default function AppNav({ fixed = false }: { fixed?: boolean }) {
             teamName={meData?.team?.name}
             isCollapsed={isCollapsed}
             onClickUserPreferences={openUserPreferences}
-            logoutUrl={IS_LOCAL_MODE ? null : `${SERVER_URL}/logout`}
+            logoutUrl={IS_LOCAL_MODE ? null : `/api/logout`}
           />
         </div>
       </div>

packages/app/src/AuthPage.tsx

@@ -16,7 +16,6 @@ import {
 } from '@mantine/core';
 
 import api from './api';
-import { SERVER_URL } from './config';
 import * as config from './config';
 import LandingHeader from './LandingHeader';
 import { CheckOrX, PasswordCheck } from './PasswordCheck';
@@ -112,7 +111,7 @@ export default function AuthPage({ action }: { action: 'register' | 'login' }) {
       }
     : {
         controller: {
-          action: `${SERVER_URL}/login/password`,
+          action: `/api/login/password`,
           method: 'POST',
         },
         email: { name: 'email' },

packages/app/src/JoinTeamPage.tsx

@@ -3,8 +3,6 @@ import { NextSeo } from 'next-seo';
 import { Button, Form } from 'react-bootstrap';
 import { Paper } from '@mantine/core';
 
-import { SERVER_URL } from './config';
-
 export default function JoinTeam() {
   const router = useRouter();
   const { err, token } = router.query;
@@ -21,7 +19,7 @@ export default function JoinTeam() {
             <div className="text-center">
               <Form
                 className="text-start"
-                action={`${SERVER_URL}/team/setup/${token}`}
+                action={`/api/team/setup/${token}`}
                 method="POST"
               >
                 <Form.Label

packages/app/src/api.ts

@@ -10,7 +10,7 @@ import type {
 } from '@tanstack/react-query';
 import { useInfiniteQuery, useMutation, useQuery } from '@tanstack/react-query';
 
-import { IS_LOCAL_MODE, SERVER_URL } from './config';
+import { IS_LOCAL_MODE } from './config';
 import type {
   AlertChannel,
   AlertInterval,
@@ -116,12 +116,8 @@ export function loginHook(request: Request, options: any, response: Response) {
   }
 }
 
-export const apiConfigs = {
-  prefixUrl: SERVER_URL,
-};
-
 export const server = ky.create({
-  prefixUrl: SERVER_URL,
+  prefixUrl: '/api',
   credentials: 'include',
   hooks: {
     afterResponse: [loginHook],
@@ -134,7 +130,6 @@ export const hdxServer = (
   options?: Options | undefined,
 ): ResponsePromise => {
   return server(url, {
-    ...apiConfigs,
     ...options,
   });
 };

packages/app/src/clickhouse.ts

@@ -5,7 +5,7 @@ import { useQuery, UseQueryOptions } from '@tanstack/react-query';
 
 import { loginHook } from '@/api';
 import { timeBucketByGranularity } from '@/ChartUtils';
-import { CLICKHOUSE_HOST, IS_LOCAL_MODE, SERVER_URL } from '@/config';
+import { CLICKHOUSE_HOST, IS_LOCAL_MODE } from '@/config';
 import { getLocalConnections } from '@/connection';
 import { SQLInterval } from '@/sqlTypes';
 import { hashCode } from '@/utils';

packages/app/src/config.ts

@@ -1,11 +1,6 @@
 import { env } from 'next-runtime-env';
 
-export const SERVER_URL =
-  process.env.NEXT_PUBLIC_SERVER_URL ?? 'http://localhost:8000'; // NEXT_PUBLIC_SERVER_URL can be empty string
-
-export const USE_CLICKHOUSE_PROXY = !process.env.NEXT_PUBLIC_CLICKHOUSE_HOST;
-export const CLICKHOUSE_HOST =
-  process.env.NEXT_PUBLIC_CLICKHOUSE_HOST ?? `${SERVER_URL}/clickhouse-proxy`;
+export const CLICKHOUSE_HOST = '/api/clickhouse-proxy';
 
 // ONLY USED IN LOCAL MODE
 // ex: NEXT_PUBLIC_HDX_LOCAL_DEFAULT_CONNECTIONS='[{"id":"local","name":"Demo","host":"https://demo-ch.hyperdx.io","username":"demo","password":"demo"}]' NEXT_PUBLIC_HDX_LOCAL_DEFAULT_SOURCES='[{"id":"l701179602","kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"default","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"l-758211293","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage"},{"id":"l-758211293","kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"default","tableName":"otel_logs"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","traceSourceId":"l701179602"}]' yarn dev:local
@@ -16,12 +11,16 @@ export const HDX_LOCAL_DEFAULT_SOURCES = env(
   'NEXT_PUBLIC_HDX_LOCAL_DEFAULT_SOURCES',
 );
 
+export const NODE_ENV = process.env.NODE_ENV as string;
 export const HDX_API_KEY = process.env.HYPERDX_API_KEY as string; // for nextjs server
 export const HDX_SERVICE_NAME =
   process.env.NEXT_PUBLIC_OTEL_SERVICE_NAME ?? 'hdx-oss-dev-app';
 export const HDX_COLLECTOR_URL =
   process.env.NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT ??
   'http://localhost:4318';
+export const IS_CI = NODE_ENV === 'ci';
+export const IS_DEV = NODE_ENV === 'development';
+export const IS_PROD = NODE_ENV === 'production';
 
 export const IS_OSS = process.env.NEXT_PUBLIC_IS_OSS ?? 'true' === 'true';
 export const IS_LOCAL_MODE = //true;

packages/app/src/instrumentation.ts

@@ -0,0 +1,9 @@
+export async function register() {
+  if (process.env.NEXT_RUNTIME === 'nodejs') {
+    const { init } = await import('@hyperdx/node-opentelemetry');
+    init({
+      apiKey: process.env.HYPERDX_API_KEY,
+      additionalInstrumentations: [], // optional, default: []
+    });
+  }
+}

packages/app/src/types.ts

@@ -311,7 +311,6 @@ export enum WebhookService {
 
 export type NextApiConfigResponseData = {
   apiKey: string;
-  apiServerUrl: string;
   collectorUrl: string;
   serviceName: string;
 };

yarn.lock

@@ -4019,7 +4019,6 @@ __metadata:
     "@hyperdx/lucene": "npm:^3.1.1"
     "@hyperdx/node-opentelemetry": "npm:^0.8.1"
     "@opentelemetry/api": "npm:^1.8.0"
-    "@sentry/node": "npm:^7.85.0"
     "@slack/types": "npm:^2.8.0"
     "@slack/webhook": "npm:^6.1.0"
     "@types/airbnb__node-memwatch": "npm:^2.0.0"
@@ -4104,6 +4103,7 @@ __metadata:
     "@hookform/resolvers": "npm:^3.9.0"
     "@hyperdx/browser": "npm:^0.21.1"
     "@hyperdx/lucene": "npm:^3.1.1"
+    "@hyperdx/node-opentelemetry": "npm:^0.8.1"
     "@jedmao/location": "npm:^3.0.0"
     "@lezer/highlight": "npm:^1.2.0"
     "@mantine/core": "npm:7.9.2"
@@ -4154,6 +4154,7 @@ __metadata:
     date-fns: "npm:^2.28.0"
     date-fns-tz: "npm:^2.0.0"
     fuse.js: "npm:^6.6.2"
+    http-proxy-middleware: "npm:^3.0.3"
     immer: "npm:^9.0.21"
     jest: "npm:^28.1.3"
     jest-environment-jsdom: "npm:^29.7.0"
@@ -7543,27 +7544,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@sentry-internal/tracing@npm:7.85.0":
-  version: 7.85.0
-  resolution: "@sentry-internal/tracing@npm:7.85.0"
-  dependencies:
-    "@sentry/core": "npm:7.85.0"
-    "@sentry/types": "npm:7.85.0"
-    "@sentry/utils": "npm:7.85.0"
-  checksum: 10c0/3ae23d1d4d5dcee0328a875492cbc1c7335949a4f57a0fd6d15a899488011e6acead0a7ed66de3387af33775f35df9daa941c7992fcea406773041b652b6891e
-  languageName: node
-  linkType: hard
-
-"@sentry/core@npm:7.85.0":
-  version: 7.85.0
-  resolution: "@sentry/core@npm:7.85.0"
-  dependencies:
-    "@sentry/types": "npm:7.85.0"
-    "@sentry/utils": "npm:7.85.0"
-  checksum: 10c0/16b88d9a38a7e32fc8c1a34d38baae17d99fabd242c7d79487d6c0bcb56d670112d7dbd5a3a0d5bb99f7cae9ef78f4e72afcd4633335cfb1dd9a4e38debf0c40
-  languageName: node
-  linkType: hard
-
 "@sentry/core@npm:^8.7.0":
   version: 8.13.0
   resolution: "@sentry/core@npm:8.13.0"
@@ -7574,26 +7554,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@sentry/node@npm:^7.85.0":
-  version: 7.85.0
-  resolution: "@sentry/node@npm:7.85.0"
-  dependencies:
-    "@sentry-internal/tracing": "npm:7.85.0"
-    "@sentry/core": "npm:7.85.0"
-    "@sentry/types": "npm:7.85.0"
-    "@sentry/utils": "npm:7.85.0"
-    https-proxy-agent: "npm:^5.0.0"
-  checksum: 10c0/b9a8a8e6927125068d48c4662ac2faeeb8780f2ea117980806dee24934e93ff73a1f3051a1c4f56dde3993e4eff1ab5d3c76e2acd45a482bd4a11b0dc9189601
-  languageName: node
-  linkType: hard
-
-"@sentry/types@npm:7.85.0":
-  version: 7.85.0
-  resolution: "@sentry/types@npm:7.85.0"
-  checksum: 10c0/d2af8270e06e88f840935b56669a35ca2a13b42de7f4159e8c256449a9721a9d89c3c92e9da44f7f83bc1d55d0715e56cc344661ebd5a2ed681c1c36d9644137
-  languageName: node
-  linkType: hard
-
 "@sentry/types@npm:8.13.0, @sentry/types@npm:^8.7.0":
   version: 8.13.0
   resolution: "@sentry/types@npm:8.13.0"
@@ -7601,15 +7561,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@sentry/utils@npm:7.85.0":
-  version: 7.85.0
-  resolution: "@sentry/utils@npm:7.85.0"
-  dependencies:
-    "@sentry/types": "npm:7.85.0"
-  checksum: 10c0/71071412e2982ecedb8101198c4632d52526322f3f81cef61fa37b4c3cda3409049fc5112b77e1caf0ed184550e99094ae012e0c80e16e1a3906402f11f93b40
-  languageName: node
-  linkType: hard
-
 "@sentry/utils@npm:8.13.0, @sentry/utils@npm:^8.7.0":
   version: 8.13.0
   resolution: "@sentry/utils@npm:8.13.0"
@@ -16773,6 +16724,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"http-proxy-middleware@npm:^3.0.3":
+  version: 3.0.3
+  resolution: "http-proxy-middleware@npm:3.0.3"
+  dependencies:
+    "@types/http-proxy": "npm:^1.17.15"
+    debug: "npm:^4.3.6"
+    http-proxy: "npm:^1.18.1"
+    is-glob: "npm:^4.0.3"
+    is-plain-object: "npm:^5.0.0"
+    micromatch: "npm:^4.0.8"
+  checksum: 10c0/c4d68a10d8d42f02e59f7dc8249c98d1ac03aecee177b42c2d8b6a0cb6b71c6688e759e5387f4cdb570150070ca1c6808b38010cbdf67f4500a2e75671a36e05
+  languageName: node
+  linkType: hard
+
 "http-proxy@npm:^1.18.1":
   version: 1.18.1
   resolution: "http-proxy@npm:1.18.1"
@@ -16791,7 +16756,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"https-proxy-agent@npm:^5.0.0, https-proxy-agent@npm:^5.0.1":
+"https-proxy-agent@npm:^5.0.1":
   version: 5.0.1
   resolution: "https-proxy-agent@npm:5.0.1"
   dependencies: