DX: standalone app docker image (#532)

For better self-hosting experience, users should be able to run ``` docker run -e MONGO_URI=xxx -p 8080:8080 hyperdx/hyperdx:2-beta ``` to spin up the project that includes the server components
2026-04-21 13:37:15 +00:00 · 2024-12-16 15:13:16 -08:00 · 2024-12-16 15:13:16 -08:00 · 9993fb2097
commit 9993fb2097
parent 74767e820e
19 changed files with 72 additions and 108 deletions
--- a/.env
+++ b/.env
@ -4,7 +4,9 @@ IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx
 IMAGE_NAME_HDX=docker.hyperdx.io/hyperdx/hyperdx
 LOCAL_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-local
 LOCAL_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-local
-IMAGE_VERSION_SUB_TAG=.7
+OTEL_COLLECTOR_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-otel-collector
+OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-otel-collector
+IMAGE_VERSION_SUB_TAG=.8
 IMAGE_VERSION=2-beta

 # Set up domain URLs
@ -12,7 +14,6 @@ HYPERDX_API_PORT=8000 #optional (should not be taken by other services)
 HYPERDX_APP_PORT=8080
 HYPERDX_APP_URL=http://localhost
 HYPERDX_LOG_LEVEL=debug
-OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 # port is fixed

 # TEMPORARY: local development
 HYPERDX_API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
--- a/18
+++ b/18
@ -98,18 +98,18 @@ release-local:
 .PHONY: release
 release:
 	docker buildx build --platform ${BUILD_PLATFORMS} ./docker/otel-collector \
-		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-otel-collector \
-		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}-otel-collector \
-		-t ${IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-otel-collector \
-		-t ${IMAGE_NAME}:${IMAGE_VERSION}-otel-collector \
-		--target prod --push &
+		-t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
+		-t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \
+		-t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
+		-t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_VERSION} \
+		--target prod --push & \
 	docker buildx build --squash . -f ./docker/fullstack/Dockerfile \
 		--build-context fullstack=./docker/fullstack \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
 		--platform ${BUILD_PLATFORMS} \
-		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-app \
-		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}-app \
-		-t ${IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-app \
-		-t ${IMAGE_NAME}:${IMAGE_VERSION}-app \
+		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
+		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \
+		-t ${IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
+		-t ${IMAGE_NAME}:${IMAGE_VERSION} \
 		--target prod --push
--- a/docker-compose.ci.yml
+++ b/docker-compose.ci.yml
@ -56,10 +56,6 @@ services:
    # ports:
    #   - 9000:9000
    environment:
-      APP_TYPE: 'api'
-      CLICKHOUSE_HOST: http://ch-server:8123
-      CLICKHOUSE_PASSWORD: api
-      CLICKHOUSE_USER: api
      EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
      MONGO_URI: 'mongodb://db:29999/hyperdx-test'
      NODE_ENV: ci
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@ -78,10 +78,6 @@ services:
  #   command: 'dev:task check-alerts'
  #   environment:
  #     APP_TYPE: 'scheduled-task'
-  #     CLICKHOUSE_HOST: http://ch-server:8123
-  #     CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-  #     CLICKHOUSE_PASSWORD: worker
-  #     CLICKHOUSE_USER: worker
  #     EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
  #     FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
  #     HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
@ -113,11 +109,6 @@ services:
    ports:
      - ${HYPERDX_API_PORT}:${HYPERDX_API_PORT}
    environment:
-      APP_TYPE: 'api'
-      CLICKHOUSE_HOST: http://ch-server:8123
-      CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      CLICKHOUSE_PASSWORD: api
-      CLICKHOUSE_USER: api
      EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
      FRONTEND_URL: 'http://localhost:${HYPERDX_APP_PORT}' # need to be localhost (CORS)
      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
@ -125,11 +116,9 @@ services:
      HDX_NODE_CONSOLE_CAPTURE: 1
      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      INGESTOR_API_URL: 'http://ingestor:8002'
      MINER_API_URL: 'http://miner:5123'
      MONGO_URI: 'mongodb://db:27017/hyperdx'
      NODE_ENV: development
-      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
      OTEL_SERVICE_NAME: 'hdx-oss-dev-api'
      PORT: ${HYPERDX_API_PORT}
      REDIS_URL: redis://redis:6379
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -53,7 +53,7 @@ services:
    networks:
      - internal
  otel-collector:
-    image: ${IMAGE_NAME_HDX}:${IMAGE_VERSION}-otel-collector
+    image: ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_VERSION}
    environment:
      CLICKHOUSE_SERVER_ENDPOINT: 'ch-server:9000'
      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
@ -75,10 +75,7 @@ services:
  #   command: './build/tasks/index.js check-alerts'
  #   environment:
  #     APP_TYPE: 'scheduled-task'
-  #     CLICKHOUSE_HOST: http://ch-server:8123
  #     CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-  #     CLICKHOUSE_PASSWORD: worker
-  #     CLICKHOUSE_USER: worker
  #     EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
  #     FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT} # need to be localhost (CORS)
  #     HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
@ -101,32 +98,20 @@ services:
  #     - db
  #     - redis
  app:
-    image: ${IMAGE_NAME_HDX}:${IMAGE_VERSION}-app
+    image: ${IMAGE_NAME_HDX}:${IMAGE_VERSION}
    ports:
      - ${HYPERDX_API_PORT}:${HYPERDX_API_PORT}
      - ${HYPERDX_APP_PORT}:${HYPERDX_APP_PORT}
    environment:
-      APP_TYPE: 'api'
-      CLICKHOUSE_HOST: http://ch-server:8123
-      CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      CLICKHOUSE_PASSWORD: api
-      CLICKHOUSE_USER: api
-      EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
      FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT}
-      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
-      HDX_NODE_BETA_MODE: 1
-      HDX_NODE_CONSOLE_CAPTURE: 1
      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
      HYPERDX_API_PORT: ${HYPERDX_API_PORT}
      HYPERDX_APP_PORT: ${HYPERDX_APP_PORT}
      HYPERDX_APP_URL: ${HYPERDX_APP_URL}
      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      INGESTOR_API_URL: 'http://ingestor:8002'
      MINER_API_URL: 'http://miner:5123'
      MONGO_URI: 'mongodb://db:27017/hyperdx'
-      NEXT_PUBLIC_CLICKHOUSE_HOST: http://ch-server:8123
      NEXT_PUBLIC_SERVER_URL: http://127.0.0.1:${HYPERDX_API_PORT}
-      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
      OTEL_SERVICE_NAME: 'hdx-oss-api'
      REDIS_URL: redis://redis:6379
      USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-true}
--- a/docker/fullstack/Dockerfile
+++ b/docker/fullstack/Dockerfile
@ -55,6 +55,9 @@ FROM node:${NODE_VERSION}-alpine AS prod

 ENV NODE_ENV production

+# Install libs used for the start script
+RUN npm install -g concurrently@9.1.0
+
 USER node

 # Set up API
@ -72,4 +75,4 @@ COPY --from=app_builder /app/app/package.json ./package.json

 # Set up start script
 COPY --chown=node:node --from=fullstack ./entry.sh /etc/local/entry.sh
-CMD sh /etc/local/entry.sh
+ENTRYPOINT ["sh", "/etc/local/entry.sh"]
--- a/docker/fullstack/entry.sh
+++ b/docker/fullstack/entry.sh
@ -1,8 +1,13 @@
 #!/bin/bash

+export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}"
+
+echo "Visit the HyperDX UI at $FRONTEND_URL"
+echo ""
+
 # Use concurrently to run both the API and App servers
 npx concurrently \
  "--kill-others" \
  "--names=API,APP" \
-  "APP_TYPE=api PORT=${HYPERDX_API_PORT:-8000} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \
-  "/app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}"
+  "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \
+  "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}"
--- a/docker/local/entry.sh
+++ b/docker/local/entry.sh
@ -11,7 +11,6 @@ export SERVER_URL="http://127.0.0.1:${HYPERDX_API_PORT:-8000}"
 export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}"

 # Internal Services
-export CLICKHOUSE_HOST="http://ch-server:8123"
 export CLICKHOUSE_SERVER_ENDPOINT="ch-server:9000"
 export MONGO_URI="mongodb://db:27017/hyperdx"
 export REDIS_URI="redis://redis:6379"
@ -52,7 +51,6 @@ done
 otelcol-contrib --config /etc/otelcol-contrib/config.yaml &

 # Api
-# APP_TYPE=api \
 # CLICKHOUSE_USER=api \
 # CLICKHOUSE_PASSWORD=api \
 # PORT=8000 \
--- a/packages/api/src/api-app.ts
+++ b/packages/api/src/api-app.ts
@ -34,7 +34,7 @@ const sess: session.SessionOptions & { cookie: session.CookieOptions } = {
 };

 app.set('trust proxy', 1);
-if (config.FRONTEND_URL) {
+if (config.FRONTEND_URL && !config.IS_CI) {
  const feUrl = new URL(config.FRONTEND_URL);
  sess.cookie.domain = feUrl.hostname;
  if (feUrl.protocol === 'https:') {
--- a/packages/api/src/clickhouse/index.ts
+++ b/packages/api/src/clickhouse/index.ts
@ -162,11 +162,10 @@ export class CHLogger implements _CHLogger {
  }
 }

-// TODO: move this to somewhere else
+// TODO: TO BE DEPRECATED
 export const client = createClient({
-  host: config.CLICKHOUSE_HOST,
-  username: config.CLICKHOUSE_USER,
-  password: config.CLICKHOUSE_PASSWORD,
+  host: 'http://localhost:8123',
+  username: 'default',
  request_timeout: ms('1m'),
  compression: {
    request: false,
--- a/packages/api/src/config.ts
+++ b/packages/api/src/config.ts
@ -1,26 +1,30 @@
 const env = process.env;
+
+// DEFAULTS
+const DEFAULT_APP_TYPE = 'api';
+const DEFAULT_EXPRESS_SESSION = 'hyperdx is cool 👋';
+const DEFAULT_FRONTEND_URL = `http://localhost:${env.HYPERDX_APP_PORT}`;
+
 export const NODE_ENV = env.NODE_ENV as string;

-export const APP_TYPE = env.APP_TYPE as 'api' | 'scheduled-task';
-export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string;
-export const CLICKHOUSE_PASSWORD = env.CLICKHOUSE_PASSWORD as string;
-export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string;
+export const APP_TYPE = (env.APP_TYPE || DEFAULT_APP_TYPE) as
+  | 'api'
+  | 'scheduled-task';
 export const CODE_VERSION = env.CODE_VERSION as string;
-export const EXPRESS_SESSION_SECRET = env.EXPRESS_SESSION_SECRET as string;
-export const FRONTEND_URL = env.FRONTEND_URL as string;
+export const EXPRESS_SESSION_SECRET = (env.EXPRESS_SESSION_SECRET ||
+  DEFAULT_EXPRESS_SESSION) as string;
+export const FRONTEND_URL = (env.FRONTEND_URL ||
+  DEFAULT_FRONTEND_URL) as string;
 export const HYPERDX_API_KEY = env.HYPERDX_API_KEY as string;
 export const HYPERDX_LOG_LEVEL = env.HYPERDX_LOG_LEVEL as string;
-export const INGESTOR_API_URL = env.INGESTOR_API_URL as string;
 export const IS_CI = NODE_ENV === 'ci';
 export const IS_DEV = NODE_ENV === 'development';
 export const IS_PROD = NODE_ENV === 'production';
 export const MINER_API_URL = env.MINER_API_URL as string;
-export const MONGO_URI = env.MONGO_URI as string;
-export const OTEL_EXPORTER_OTLP_ENDPOINT =
-  env.OTEL_EXPORTER_OTLP_ENDPOINT as string;
+export const MONGO_URI = env.MONGO_URI;
 export const OTEL_SERVICE_NAME = env.OTEL_SERVICE_NAME as string;
 export const PORT = Number.parseInt(env.PORT as string);
-export const REDIS_URL = env.REDIS_URL as string;
+export const REDIS_URL = env.REDIS_URL;
 export const USAGE_STATS_ENABLED = env.USAGE_STATS_ENABLED !== 'false';

 // Only for single container local deployments, disable authentication
--- a/packages/api/src/fixtures.ts
+++ b/packages/api/src/fixtures.ts
@ -32,6 +32,9 @@ export const connectDB = async () => {
  if (!config.IS_CI) {
    throw new Error('ONLY execute this in CI env 😈 !!!');
  }
+  if (config.MONGO_URI == null) {
+    throw new Error('MONGO_URI is not set');
+  }
  await mongoose.connect(config.MONGO_URI);
 };

--- a/packages/api/src/models/index.ts
+++ b/packages/api/src/models/index.ts
@ -35,6 +35,9 @@ mongoose.connection.on('reconnectFailed', () => {
 });

 export const connectDB = async () => {
+  if (config.MONGO_URI == null) {
+    throw new Error('MONGO_URI is not set');
+  }
  await mongoose.connect(config.MONGO_URI, {
    heartbeatFrequencyMS: 10000, // retry failed heartbeats
    maxPoolSize: 100, // 5 nodes -> max 1000 connections
--- a/packages/api/src/server.ts
+++ b/packages/api/src/server.ts
@ -2,28 +2,19 @@ import http from 'http';
 import gracefulShutdown from 'http-graceful-shutdown';
 import { serializeError } from 'serialize-error';

+import apiServer from './api-app';
 import * as config from './config';
 import { connectDB, mongooseConnection } from './models';
 import logger from './utils/logger';
 import redisClient from './utils/redis';

 export default class Server {
-  protected readonly appType = config.APP_TYPE;
-
  protected shouldHandleGracefulShutdown = true;

  protected httpServer!: http.Server;

  private async createServer() {
-    switch (this.appType) {
-      case 'api':
-        return http.createServer(
-          // eslint-disable-next-line n/no-unsupported-features/es-syntax
-          (await import('./api-app').then(m => m.default)) as any,
-        );
-      default:
-        throw new Error(`Invalid APP_TYPE: ${config.APP_TYPE}`);
-    }
+    return http.createServer(apiServer);
  }

  protected async shutdown(signal?: string) {
--- a/packages/api/src/tasks/usageStats.ts
+++ b/packages/api/src/tasks/usageStats.ts
@ -51,35 +51,10 @@ const getClickhouseTableSize = async () => {
  return result.data;
 };

-const healthChecks = async () => {
-  const ping = async (url: string) => {
-    try {
-      const res = await fetch(url);
-      return res.status === 200;
-    } catch (err) {
-      return false;
-    }
-  };
-
-  const otelCollectorUrl = new URL(config.OTEL_EXPORTER_OTLP_ENDPOINT ?? '');
-
-  const [pingOtelCollector, pingCH] = await Promise.all([
-    otelCollectorUrl.hostname && otelCollectorUrl.protocol
-      ? ping(`${otelCollectorUrl.protocol}//${otelCollectorUrl.hostname}:13133`)
-      : Promise.resolve(null),
-    ping(`${config.CLICKHOUSE_HOST}/ping`),
-  ]);
-
-  return {
-    pingOtelCollector,
-    pingCH,
-  };
-};
-
 export default async () => {
  try {
    const nowInMs = Date.now();
-    const [userCounts, team, chTables, servicesHealth] = await Promise.all([
+    const [userCounts, team, chTables] = await Promise.all([
      User.countDocuments(),
      Team.find(
        {},
@ -88,7 +63,6 @@ export default async () => {
        },
      ).limit(1),
      getClickhouseTableSize(),
-      healthChecks(),
    ]);
    const clusterId = team[0]?._id;
    logger.info({
@ -96,7 +70,6 @@ export default async () => {
      clusterId,
      version: config.CODE_VERSION,
      userCounts,
-      servicesHealth,
      os: {
        arch: os.arch(),
        freemem: os.freemem(),
--- a/packages/api/src/utils/logger.ts
+++ b/packages/api/src/utils/logger.ts
@ -6,7 +6,6 @@ import {
  APP_TYPE,
  HYPERDX_API_KEY,
  HYPERDX_LOG_LEVEL,
-  INGESTOR_API_URL,
  IS_PROD,
 } from '@/config';

@ -30,7 +29,6 @@ const DEFAULT_FORMAT = winston.format.combine(
 const hyperdxTransport = HYPERDX_API_KEY
  ? getWinstonTransport(MAX_LEVEL, {
      bufferSize: APP_TYPE === 'scheduled-task' ? 1 : 100,
-      ...(INGESTOR_API_URL && { baseUrl: INGESTOR_API_URL }),
    })
  : null;

--- a/packages/api/src/utils/redis.ts
+++ b/packages/api/src/utils/redis.ts
@ -8,8 +8,16 @@ const client = createClient({
  url: config.REDIS_URL,
 });

+// check if client is initialized
+if (client == null) {
+  logger.warn('Redis client is not initialized');
+  // IMPLEMENT: use local in-memory cache
+}
+
 client.on('error', (err: any) => {
-  logger.error('Redis error: ', serializeError(err));
+  if (config.REDIS_URL) {
+    logger.error('Redis error: ', serializeError(err));
+  }
 });

 // TODO: add tests
@ -22,6 +30,9 @@ class SimpleCache<T> {
  ) {}

  async refresh() {
+    if (client == null) {
+      throw new Error('Redis client is not initialized');
+    }
    const dt = Date.now();
    const result = await this.fetcher();
    if (this.shouldRefreshOnResult(result)) {
@ -38,6 +49,9 @@ class SimpleCache<T> {
  }

  async get(): Promise<T> {
+    if (client == null) {
+      throw new Error('Redis client is not initialized');
+    }
    const cached = await client.get(this.key);
    if (cached != null) {
      logger.info({
--- a/packages/app/package.json
+++ b/packages/app/package.json
@ -8,7 +8,7 @@
  },
  "scripts": {
    "dev": "next dev",
-    "dev:local": "NEXT_PUBLIC_CLICKHOUSE_HOST=http://localhost:8123 NEXT_PUBLIC_IS_LOCAL_MODE=true next dev -p 8080",
+    "dev:local": "NEXT_PUBLIC_IS_LOCAL_MODE=true next dev -p 8080",
    "build": "next build",
    "start": "next start",
    "lint": "eslint --quiet . --ext .ts,.tsx",
--- a/packages/app/pages/api/[...all].ts
+++ b/packages/app/pages/api/[...all].ts
@ -3,6 +3,8 @@ import { createProxyMiddleware, fixRequestBody } from 'http-proxy-middleware';

 import { IS_DEV } from '@/config';

+const DEFAULT_SERVER_URL = `http://127.0.0.1:${process.env.HYPERDX_API_PORT}`;
+
 export const config = {
  api: {
    externalResolver: true,
@ -15,7 +17,7 @@ export default (req: NextApiRequest, res: NextApiResponse) => {
    changeOrigin: true,
    // logger: console, // DEBUG
    pathRewrite: { '^/api': '' },
-    target: process.env.NEXT_PUBLIC_SERVER_URL,
+    target: process.env.NEXT_PUBLIC_SERVER_URL || DEFAULT_SERVER_URL,
    autoRewrite: true,
    /**
     * Fix bodyParser