feat: introduce usage-stats service

2026-04-21 13:37:15 +00:00 · 2023-09-12 21:25:01 -07:00 · 2023-09-12 21:25:01 -07:00 · f2396340b4
commit f2396340b4
parent 773ce91264
20 changed files with 567 additions and 156 deletions
--- a/.env
+++ b/.env
@ -0,0 +1,2 @@
+IMAGE_NAME=ghcr.io/hyperdxio/hyperdx
+IMAGE_VERSION=1.0.3
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -28,7 +28,7 @@ jobs:
          ~/.vector/bin/vector --version
          ln -s ~/.vector/bin/vector /usr/local/bin/vector
      - name: Run lint + type check
-        run: yarn ci:lint
+        run: make ci-lint
  integration:
    timeout-minutes: 8
    runs-on: ubuntu-20.04
@ -46,4 +46,4 @@ jobs:
          docker buildx create --use --driver=docker-container
          docker buildx bake -f ./docker-compose.ci.yml --set *.cache-to="type=gha" --set *.cache-from="type=gha" --load
      - name: Run integration tests
-        run: yarn ci:int
+        run: make ci-int
--- a/.gitignore
+++ b/.gitignore
@ -11,11 +11,12 @@
 **/lerna-debug.log*

 # dotenv environment variable files
-**/.env
 **/.env.development.local
 **/.env.test.local
 **/.env.production.local
 **/.env.local
+**/.dockerhub.env
+**/.ghcr.env

 # Next.js build output
 packages/app/.next
--- a/52
+++ b/52
@ -0,0 +1,52 @@
+LATEST_VERSION := $$(sed -n 's/.*"version": "\([^"]*\)".*/\1/p' package.json)
+BUILD_PLATFORMS = linux/arm64/v8,linux/amd64
+
+include .env
+
+.PHONY: all
+all: install-tools
+
+.PHONY: install-tools
+install-tools:
+	yarn install
+	@echo "All tools installed"
+
+.PHONY: dev-build
+dev-build:
+	docker compose -f docker-compose.dev.yml build
+
+.PHONY: dev-up
+dev-up:
+	docker compose -f docker-compose.dev.yml up -d
+
+.PHONY: dev-down
+dev-down:
+	docker compose -f docker-compose.dev.yml down
+
+.PHONY: dev-lint
+dev-lint:
+	./docker/ingestor/run_linting.sh && yarn workspaces run lint
+
+.PHONY: ci-lint
+ci-lint:
+	./docker/ingestor/run_linting.sh && yarn workspaces run ci:lint
+
+.PHONY: dev-int
+dev-int:
+	docker compose -p int -f ./docker-compose.ci.yml run --rm api dev:int
+
+.PHONY: ci-int
+ci-int:
+	docker compose -p int -f ./docker-compose.ci.yml run --rm api ci:int
+
+
+.PHONY: build-and-push-ghcr
+build-and-push-ghcr:
+	docker buildx build --platform ${BUILD_PLATFORMS} ./docker/hostmetrics -t ${IMAGE_NAME}:${LATEST_VERSION}-hostmetrics  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} ./docker/ingestor -t ${IMAGE_NAME}:${LATEST_VERSION}-ingestor  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} ./docker/otel-collector -t ${IMAGE_NAME}:${LATEST_VERSION}-otel-collector  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} . -f ./packages/miner/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-miner  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} . -f ./packages/api/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-api  --target dev --push &
+	docker buildx build --platform ${BUILD_PLATFORMS} . -f ./packages/app/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-app  --target prod --push
+
+
--- a/README.md
+++ b/README.md
@ -54,7 +54,7 @@ You can get started by deploying a complete stack via Docker Compose. After
 cloning this repository, simply start the stack with:

 ```bash
-docker compose up
+docker compose up -d
 ```

 Afterwards, you can visit http://localhost:8080 to access the HyperDX UI.
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@ -0,0 +1,235 @@
+version: '3'
+services:
+  miner:
+    container_name: hdx-oss-dev-miner
+    build:
+      context: .
+      dockerfile: ./packages/miner/Dockerfile
+      target: dev
+    environment:
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+      OTEL_LOG_LEVEL: ERROR
+      OTEL_SERVICE_NAME: hdx-oss-dev-miner
+    volumes:
+      - ./packages/miner/src:/app/src
+    ports:
+      - 5123:5123
+    networks:
+      - internal
+  hostmetrics:
+    container_name: hdx-oss-dev-hostmetrics
+    build:
+      context: ./docker/hostmetrics
+      target: dev
+    volumes:
+      - ./docker/hostmetrics/config.dev.yaml:/etc/otelcol-contrib/config.yaml
+    environment:
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      OTEL_SERVICE_NAME: hostmetrics
+    restart: always
+    networks:
+      - internal
+  ingestor:
+    container_name: hdx-oss-dev-ingestor
+    build:
+      context: ./docker/ingestor
+      target: dev
+    volumes:
+      - ./docker/ingestor:/app
+      - .volumes/ingestor_data:/var/lib/vector
+    ports:
+      - 8002:8002 # http-generic
+      - 8686:8686 # healthcheck
+    environment:
+      RUST_BACKTRACE: full
+      VECTOR_LOG: debug
+      VECTOR_OPENSSL_LEGACY_PROVIDER: false
+    restart: always
+    networks:
+      - internal
+  redis:
+    image: redis:7.0.11-alpine
+    container_name: hdx-oss-dev-redis
+    volumes:
+      - .volumes/redis:/data
+    ports:
+      - 6379:6379
+    networks:
+      - internal
+  db:
+    image: mongo:5.0.14-focal
+    container_name: hdx-oss-dev-db
+    volumes:
+      - .volumes/db:/data/db
+    ports:
+      - 27017:27017
+    networks:
+      - internal
+  otel-collector:
+    container_name: hdx-oss-dev-otel-collector
+    build:
+      context: ./docker/otel-collector
+      target: dev
+    volumes:
+      - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
+    ports:
+      - '13133:13133' # health_check extension
+      - '1888:1888' # pprof extension
+      - '24225:24225' # fluentd receiver
+      - '4317:4317' # OTLP gRPC receiver
+      - '4318:4318' # OTLP http receiver
+      - '55679:55679' # zpages extension
+      - '8888:8888' # metrics extension
+      - '9411:9411' # zipkin
+    restart: always
+    networks:
+      - internal
+  aggregator:
+    container_name: hdx-oss-dev-aggregator
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+    ports:
+      - 8001:8001
+    environment:
+      APP_TYPE: 'aggregator'
+      CLICKHOUSE_HOST: http://ch-server:8123
+      CLICKHOUSE_PASSWORD: aggregator
+      CLICKHOUSE_USER: aggregator
+      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      MONGO_URI: 'mongodb://db:27017/hyperdx'
+      NODE_ENV: development
+      PORT: 8001
+      REDIS_URL: redis://redis:6379
+      SERVER_URL: 'http://localhost:8000'
+    volumes:
+      - ./packages/api/src:/app/src
+    networks:
+      - internal
+    depends_on:
+      - db
+      - redis
+      - ch-server
+  task-check-alerts:
+    container_name: hdx-oss-dev-task-check-alerts
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+    entrypoint: 'yarn'
+    command: 'dev:task check-alerts'
+    environment:
+      APP_TYPE: 'scheduled-task'
+      CLICKHOUSE_HOST: http://ch-server:8123
+      CLICKHOUSE_LOG_LEVEL: trace
+      CLICKHOUSE_PASSWORD: worker
+      CLICKHOUSE_USER: worker
+      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
+      HDX_NODE_BETA_MODE: 0
+      HDX_NODE_CONSOLE_CAPTURE: 1
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      INGESTOR_API_URL: 'http://ingestor:8002'
+      MINER_API_URL: 'http://miner:5123'
+      MONGO_URI: 'mongodb://db:27017/hyperdx'
+      NODE_ENV: development
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
+      OTEL_SERVICE_NAME: 'hdx-oss-dev-task-check-alerts'
+      REDIS_URL: redis://redis:6379
+    volumes:
+      - ./packages/api/src:/app/src
+    restart: always
+    networks:
+      - internal
+    depends_on:
+      - ch-server
+      - db
+      - redis
+  api:
+    container_name: hdx-oss-dev-api
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+    ports:
+      - 8000:8000
+    environment:
+      AGGREGATOR_API_URL: 'http://aggregator:8001'
+      APP_TYPE: 'api'
+      CLICKHOUSE_HOST: http://ch-server:8123
+      CLICKHOUSE_LOG_LEVEL: trace
+      CLICKHOUSE_PASSWORD: api
+      CLICKHOUSE_USER: api
+      EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
+      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
+      HDX_NODE_BETA_MODE: 1
+      HDX_NODE_CONSOLE_CAPTURE: 1
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      INGESTOR_API_URL: 'http://ingestor:8002'
+      MINER_API_URL: 'http://miner:5123'
+      MONGO_URI: 'mongodb://db:27017/hyperdx'
+      NODE_ENV: development
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
+      OTEL_SERVICE_NAME: 'hdx-oss-dev-api'
+      PORT: 8000
+      REDIS_URL: redis://redis:6379
+      SERVER_URL: 'http://localhost:8000'
+      USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-false}
+    volumes:
+      - ./packages/api/src:/app/src
+    networks:
+      - internal
+    depends_on:
+      - ch-server
+      - db
+      - redis
+  app:
+    container_name: hdx-oss-dev-app
+    build:
+      context: .
+      dockerfile: ./packages/app/Dockerfile
+      target: dev
+    ports:
+      - 8080:8080
+    environment:
+      NEXT_PUBLIC_API_SERVER_URL: 'http://localhost:8000' # need to be localhost (CORS)
+      NEXT_PUBLIC_HDX_API_KEY: ${HYPERDX_API_KEY}
+      NEXT_PUBLIC_HDX_COLLECTOR_URL: 'http://localhost:4318'
+      NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-dev-app'
+      NODE_ENV: development
+      PORT: 8080
+    volumes:
+      - ./packages/app/pages:/app/pages
+      - ./packages/app/public:/app/public
+      - ./packages/app/src:/app/src
+      - ./packages/app/styles:/app/styles
+      - ./packages/app/mdx.d.ts:/app/mdx.d.ts
+      - ./packages/app/next-env.d.ts:/app/next-env.d.ts
+      - ./packages/app/next.config.js:/app/next.config.js
+    networks:
+      - internal
+    depends_on:
+      - api
+  ch-server:
+    image: clickhouse/clickhouse-server:23.7.1-alpine
+    container_name: hdx-oss-dev-ch-server
+    ports:
+      - 8123:8123 # http api
+      - 9000:9000 # native
+    environment:
+      # default settings
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    volumes:
+      - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
+      - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
+      - .volumes/ch_data:/var/lib/clickhouse
+      - .volumes/ch_logs:/var/log/clickhouse-server
+    restart: on-failure
+    networks:
+      - internal
+networks:
+  internal:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,17 +1,14 @@
 version: '3'
 services:
  miner:
-    container_name: hdx-oss-dev-miner
-    build:
-      context: .
-      dockerfile: ./packages/miner/Dockerfile
-      target: dev
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-miner
+    container_name: hdx-oss-miner
    environment:
      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
      HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1
-      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
      OTEL_LOG_LEVEL: ERROR
-      OTEL_SERVICE_NAME: hdx-oss-dev-miner
+      OTEL_SERVICE_NAME: hdx-oss-miner
    volumes:
      - ./packages/miner/src:/app/src
    ports:
@ -19,10 +16,8 @@ services:
    networks:
      - internal
  hostmetrics:
-    container_name: hdx-oss-dev-hostmetrics
-    build:
-      context: ./docker/hostmetrics
-      target: dev
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-hostmetrics
+    container_name: hdx-oss-hostmetrics
    volumes:
      - ./docker/hostmetrics/config.dev.yaml:/etc/otelcol-contrib/config.yaml
    environment:
@ -32,10 +27,8 @@ services:
    networks:
      - internal
  ingestor:
-    container_name: hdx-oss-dev-ingestor
-    build:
-      context: ./docker/ingestor
-      target: dev
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-ingestor
+    container_name: hdx-oss-ingestor
    volumes:
      - ./docker/ingestor:/app
      - .volumes/ingestor_data:/var/lib/vector
@ -50,8 +43,8 @@ services:
    networks:
      - internal
  redis:
-    container_name: hdx-oss-dev-redis
    image: redis:7.0.11-alpine
+    container_name: hdx-oss-redis
    volumes:
      - .volumes/redis:/data
    ports:
@ -59,19 +52,17 @@ services:
    networks:
      - internal
  db:
-    container_name: hdx-oss-dev-db
    image: mongo:5.0.14-focal
+    container_name: hdx-oss-db
    volumes:
      - .volumes/db:/data/db
    ports:
      - 27017:27017
    networks:
      - internal
-  otel:
-    container_name: hdx-oss-dev-otel
-    build:
-      context: ./docker/otel-collector
-      target: dev
+  otel-collector:
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-otel-collector
+    container_name: hdx-oss-otel-collector
    volumes:
      - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
    ports:
@ -87,12 +78,8 @@ services:
    networks:
      - internal
  aggregator:
-    container_name: hdx-oss-dev-aggregator
-    build:
-      context: .
-      dockerfile: ./packages/api/Dockerfile
-      target: dev
-    image: hyperdx/dev/api
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
+    container_name: hdx-oss-aggregator
    ports:
      - 8001:8001
    environment:
@ -115,12 +102,8 @@ services:
      - redis
      - ch-server
  task-check-alerts:
-    container_name: hdx-oss-dev-task-check-alerts
-    build:
-      context: .
-      dockerfile: ./packages/api/Dockerfile
-      target: dev
-    image: hyperdx/dev/api
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
+    container_name: hdx-oss-task-check-alerts
    entrypoint: 'yarn'
    command: 'dev:task check-alerts'
    environment:
@ -134,12 +117,12 @@ services:
      HDX_NODE_BETA_MODE: 0
      HDX_NODE_CONSOLE_CAPTURE: 1
      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      HYPERDX_INGESTOR_ENDPOINT: 'http://ingestor:8002'
+      INGESTOR_API_URL: 'http://ingestor:8002'
      MINER_API_URL: 'http://miner:5123'
      MONGO_URI: 'mongodb://db:27017/hyperdx'
      NODE_ENV: development
-      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel:4318'
-      OTEL_SERVICE_NAME: 'hdx-oss-dev-task-check-alerts'
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
+      OTEL_SERVICE_NAME: 'hdx-oss-task-check-alerts'
      REDIS_URL: redis://redis:6379
    volumes:
      - ./packages/api/src:/app/src
@ -151,15 +134,12 @@ services:
      - db
      - redis
  api:
-    container_name: hdx-oss-dev-api
-    build:
-      context: .
-      dockerfile: ./packages/api/Dockerfile
-      target: dev
-    image: hyperdx/dev/api
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-api
+    container_name: hdx-oss-api
    ports:
      - 8000:8000
    environment:
+      AGGREGATOR_API_URL: 'http://aggregator:8001'
      APP_TYPE: 'api'
      CLICKHOUSE_HOST: http://ch-server:8123
      CLICKHOUSE_LOG_LEVEL: trace
@ -171,15 +151,16 @@ services:
      HDX_NODE_BETA_MODE: 1
      HDX_NODE_CONSOLE_CAPTURE: 1
      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      HYPERDX_INGESTOR_ENDPOINT: 'http://ingestor:8002'
+      INGESTOR_API_URL: 'http://ingestor:8002'
      MINER_API_URL: 'http://miner:5123'
      MONGO_URI: 'mongodb://db:27017/hyperdx'
      NODE_ENV: development
-      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel:4318'
-      OTEL_SERVICE_NAME: 'hdx-oss-dev-api'
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
+      OTEL_SERVICE_NAME: 'hdx-oss-api'
      PORT: 8000
      REDIS_URL: redis://redis:6379
      SERVER_URL: 'http://localhost:8000'
+      USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-true}
    volumes:
      - ./packages/api/src:/app/src
    networks:
@ -189,19 +170,15 @@ services:
      - db
      - redis
  app:
-    container_name: hdx-oss-dev-app
-    build:
-      context: .
-      dockerfile: ./packages/app/Dockerfile
-      target: dev
-    image: hyperdx/dev/app
+    image: ${IMAGE_NAME}:${IMAGE_VERSION}-app
+    container_name: hdx-oss-app
    ports:
      - 8080:8080
    environment:
      NEXT_PUBLIC_API_SERVER_URL: 'http://localhost:8000' # need to be localhost (CORS)
      NEXT_PUBLIC_HDX_API_KEY: ${HYPERDX_API_KEY}
      NEXT_PUBLIC_HDX_COLLECTOR_URL: 'http://localhost:4318'
-      NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-dev-app'
+      NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-app'
      NODE_ENV: development
      PORT: 8080
    volumes:
@ -217,8 +194,8 @@ services:
    depends_on:
      - api
  ch-server:
-    container_name: hdx-oss-dev-ch-server
    image: clickhouse/clickhouse-server:23.7.1-alpine
+    container_name: hdx-oss-ch-server
    ports:
      - 8123:8123 # http api
      - 9000:9000 # native
--- a/docker/hostmetrics/config.dev.yaml
+++ b/docker/hostmetrics/config.dev.yaml
@ -23,7 +23,7 @@ exporters:
  logging:
    loglevel: debug
  otlphttp:
-    endpoint: 'http://otel:4318'
+    endpoint: 'http://otel-collector:4318'
    headers:
      authorization: ${HYPERDX_API_KEY}
    compression: gzip
--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
  "private": true,
-  "version": "0.0.0",
+  "version": "1.0.3",
  "license": "MIT",
  "workspaces": [
    "packages/*"
@ -13,11 +13,7 @@
    "prettier": "2.8.4"
  },
  "scripts": {
-    "prepare": "husky install",
-    "dev:lint": "./docker/ingestor/run_linting.sh && yarn workspaces run ci:lint",
-    "ci:lint": "./docker/ingestor/run_linting.sh && yarn workspaces run ci:lint",
-    "dev:int": "docker compose -p int -f ./docker-compose.ci.yml run --rm api dev:int",
-    "ci:int": "docker compose -p int -f ./docker-compose.ci.yml run --rm api ci:int"
+    "prepare": "husky install"
  },
  "lint-staged": {
    "**/*": "prettier --write --ignore-unknown"
--- a/packages/api/package.json
+++ b/packages/api/package.json
@ -1,6 +1,6 @@
 {
  "name": "@hyperdx/api",
-  "version": "1.0.0",
+  "version": "1.0.3",
  "license": "MIT",
  "private": true,
  "engines": {
@ -9,6 +9,7 @@
  "dependencies": {
    "@clickhouse/client": "^0.1.1",
    "@hyperdx/lucene": "^3.1.1",
+    "@hyperdx/node-logger": "^0.2.7",
    "@hyperdx/node-opentelemetry": "^0.2.2",
    "@slack/webhook": "^6.1.0",
    "compression": "^1.7.4",
--- a/packages/api/src/api-app.ts
+++ b/packages/api/src/api-app.ts
@ -1,13 +1,15 @@
 import MongoStore from 'connect-mongo';
 import compression from 'compression';
 import express from 'express';
-import session from 'express-session';
+import ms from 'ms';
 import onHeaders from 'on-headers';
+import session from 'express-session';

 import * as config from './config';
 import defaultCors from './middleware/cors';
 import passport from './utils/passport';
 import routers from './routers/api';
+import usageStats from './tasks/usageStats';
 import { appErrorHandler } from './middleware/error';
 import { expressLogger } from './utils/logger';

@ -55,6 +57,17 @@ app.use(function (req, res, next) {
 });
 app.use(defaultCors);

+// ---------------------------------------------------------------------
+// ----------------------- Background Jobs -----------------------------
+// ---------------------------------------------------------------------
+if (config.USAGE_STATS_ENABLED) {
+  void usageStats();
+  setInterval(() => {
+    void usageStats();
+  }, ms('4h'));
+}
+// ---------------------------------------------------------------------
+
 // ---------------------------------------------------------------------
 // ----------------------- Internal Routers ----------------------------
 // ---------------------------------------------------------------------
--- a/packages/api/src/config.ts
+++ b/packages/api/src/config.ts
@ -1,35 +1,28 @@
 import { version } from '../package.json';

 const env = process.env;
-
-export const CODE_VERSION = version;
-
-export const APP_TYPE = env.APP_TYPE as 'api' | 'aggregator' | 'scheduled-task';
-
 export const NODE_ENV = env.NODE_ENV as string;
-export const IS_PROD = NODE_ENV === 'production';
-export const IS_DEV = NODE_ENV === 'development';
-export const IS_CI = NODE_ENV === 'ci';
-
-export const PORT = Number.parseInt(env.PORT as string);
-export const SERVER_URL = env.SERVER_URL as string;
-export const FRONTEND_URL = env.FRONTEND_URL as string;
-export const COOKIE_DOMAIN = env.COOKIE_DOMAIN as string; // prod ONLY
-
-export const MONGO_URI = env.MONGO_URI as string;

+export const AGGREGATOR_API_URL = env.AGGREGATOR_API_URL as string;
+export const APP_TYPE = env.APP_TYPE as 'api' | 'aggregator' | 'scheduled-task';
 export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string;
 export const CLICKHOUSE_PASSWORD = env.CLICKHOUSE_PASSWORD as string;
 export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string;
-
-export const HYPERDX_API_KEY = env.HYPERDX_API_KEY as string;
-export const HYPERDX_INGESTOR_ENDPOINT =
-  env.HYPERDX_INGESTOR_ENDPOINT as string;
-
+export const CODE_VERSION = version;
+export const COOKIE_DOMAIN = env.COOKIE_DOMAIN as string; // prod ONLY
 export const EXPRESS_SESSION_SECRET = env.EXPRESS_SESSION_SECRET as string;
-
-export const REDIS_URL = env.REDIS_URL as string;
-
+export const FRONTEND_URL = env.FRONTEND_URL as string;
+export const HYPERDX_API_KEY = env.HYPERDX_API_KEY as string;
+export const INGESTOR_API_URL = env.INGESTOR_API_URL as string;
+export const IS_CI = NODE_ENV === 'ci';
+export const IS_DEV = NODE_ENV === 'development';
+export const IS_PROD = NODE_ENV === 'production';
 export const MINER_API_URL = env.MINER_API_URL as string;
-
+export const MONGO_URI = env.MONGO_URI as string;
+export const OTEL_EXPORTER_OTLP_ENDPOINT =
+  env.OTEL_EXPORTER_OTLP_ENDPOINT as string;
 export const OTEL_SERVICE_NAME = env.OTEL_SERVICE_NAME as string;
+export const PORT = Number.parseInt(env.PORT as string);
+export const REDIS_URL = env.REDIS_URL as string;
+export const SERVER_URL = env.SERVER_URL as string;
+export const USAGE_STATS_ENABLED = env.USAGE_STATS_ENABLED !== 'false';
--- a/packages/api/src/tasks/usageStats.ts
+++ b/packages/api/src/tasks/usageStats.ts
@ -0,0 +1,147 @@
+import os from 'os';
+import url from 'url';
+
+import winston from 'winston';
+import { HyperDXWinston } from '@hyperdx/node-logger';
+
+import * as clickhouse from '../clickhouse';
+import * as config from '../config';
+import Team from '../models/team';
+import User from '../models/user';
+
+import type { ResponseJSON } from '@clickhouse/client';
+
+const hyperdxTransport = new HyperDXWinston({
+  apiKey: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key
+  baseUrl: 'https://in.hyperdx.io',
+  maxLevel: 'info',
+  service: 'hyperdx-oss-usage-stats',
+});
+
+const logger = winston.createLogger({
+  level: 'info',
+  format: winston.format.json(),
+  transports: [hyperdxTransport],
+});
+
+const getClickhouseTableSize = async () => {
+  const rows = await clickhouse.client.query({
+    query: `
+      SELECT
+          table,
+          sum(bytes) AS size,
+          sum(rows) AS rows,
+          min(min_time) AS min_time,
+          max(max_time) AS max_time,
+          max(modification_time) AS latestModification,
+          toUInt32((max_time - min_time) / 86400) AS days,
+          size / ((max_time - min_time) / 86400) AS avgDaySize
+      FROM system.parts
+      WHERE active
+      AND database = 'default'
+      AND (table = {table1: String} OR table = {table2: String} OR table = {table3: String})
+      GROUP BY table
+      ORDER BY rows DESC
+    `,
+    format: 'JSON',
+    query_params: {
+      table1: clickhouse.TableName.LogStream,
+      table2: clickhouse.TableName.Rrweb,
+      table3: clickhouse.TableName.Metric,
+    },
+  });
+  const result = await rows.json<ResponseJSON<any>>();
+  return result.data;
+};
+
+const healthChecks = async () => {
+  const ping = async (url: string) => {
+    try {
+      const res = await fetch(url);
+      return res.status === 200;
+    } catch (err) {
+      return false;
+    }
+  };
+
+  const ingestorUrl = url.parse(config.INGESTOR_API_URL ?? '');
+  const otelCollectorUrl = url.parse(config.OTEL_EXPORTER_OTLP_ENDPOINT ?? '');
+  const aggregatorUrl = url.parse(config.AGGREGATOR_API_URL ?? '');
+
+  const [pingIngestor, pingOtelCollector, pingAggregator, pingMiner, pingCH] =
+    await Promise.all([
+      ingestorUrl.hostname && ingestorUrl.protocol
+        ? ping(`${ingestorUrl.protocol}//${ingestorUrl.hostname}:8686/health`)
+        : Promise.resolve(null),
+      otelCollectorUrl.hostname && otelCollectorUrl.protocol
+        ? ping(
+            `${otelCollectorUrl.protocol}//${otelCollectorUrl.hostname}:13133`,
+          )
+        : Promise.resolve(null),
+      aggregatorUrl.href
+        ? ping(`${aggregatorUrl.href}health`)
+        : Promise.resolve(null),
+      ping(`${config.MINER_API_URL}/health`),
+      ping(`${config.CLICKHOUSE_HOST}/ping`),
+    ]);
+
+  return {
+    pingIngestor,
+    pingOtelCollector,
+    pingAggregator,
+    pingMiner,
+    pingCH,
+  };
+};
+
+export default async () => {
+  try {
+    const nowInMs = Date.now();
+    const [userCounts, team, chTables, servicesHealth] = await Promise.all([
+      User.countDocuments(),
+      Team.find(
+        {},
+        {
+          _id: 1,
+        },
+      ).limit(1),
+      getClickhouseTableSize(),
+      healthChecks(),
+    ]);
+    const clusterId = team[0]?._id;
+    logger.info({
+      message: 'track-hyperdx-oss-usage-stats',
+      clusterId,
+      version: config.CODE_VERSION,
+      userCounts,
+      servicesHealth,
+      os: {
+        arch: os.arch(),
+        freemem: os.freemem(),
+        uptime: os.uptime(),
+      },
+      chStats: {
+        tables: chTables.reduce(
+          (acc, curr) => ({
+            ...acc,
+            [curr.table]: {
+              avgDaySize: parseInt(curr.avgDaySize),
+              days: parseInt(curr.days),
+              lastModified: new Date(curr.latestModification).getTime(),
+              maxTime: new Date(curr.max_time).getTime(),
+              minTime: new Date(curr.min_time).getTime(),
+              rows: parseInt(curr.rows),
+              size: parseInt(curr.size),
+            },
+          }),
+          {},
+        ),
+        rows: chTables.reduce((acc, curr) => acc + parseInt(curr.rows), 0),
+        size: chTables.reduce((acc, curr) => acc + parseInt(curr.size), 0),
+      },
+      timestamp: nowInMs,
+    });
+  } catch (err) {
+    // ignore
+  }
+};
--- a/packages/api/src/utils/logger.ts
+++ b/packages/api/src/utils/logger.ts
@ -6,7 +6,7 @@ import { getWinsonTransport } from '@hyperdx/node-opentelemetry';
 import {
  APP_TYPE,
  HYPERDX_API_KEY,
-  HYPERDX_INGESTOR_ENDPOINT,
+  INGESTOR_API_URL,
  IS_DEV,
  IS_PROD,
 } from '../config';
@ -25,24 +25,15 @@ addColors({
 });

 const MAX_LEVEL = IS_PROD ? 'debug' : 'debug';
-const DEFAULT_FORMAT = IS_DEV
-  ? winston.format.combine(
-      winston.format.errors({ stack: true }),
-      winston.format.colorize({ all: true }),
-      winston.format.timestamp({ format: 'MM/DD/YY HH:MM:SS' }),
-      winston.format.printf(
-        info => `[${info.level}] ${info.timestamp} ${info.message}`,
-      ),
-    )
-  : winston.format.combine(
-      winston.format.errors({ stack: true }),
-      winston.format.json(),
-    );
+const DEFAULT_FORMAT = winston.format.combine(
+  winston.format.errors({ stack: true }),
+  winston.format.json(),
+);

 const hyperdxTransport = HYPERDX_API_KEY
  ? getWinsonTransport(MAX_LEVEL, {
      bufferSize: APP_TYPE === 'scheduled-task' ? 1 : 100,
-      ...(HYPERDX_INGESTOR_ENDPOINT && { baseUrl: HYPERDX_INGESTOR_ENDPOINT }),
+      ...(INGESTOR_API_URL && { baseUrl: INGESTOR_API_URL }),
    })
  : null;

--- a/packages/app/Dockerfile
+++ b/packages/app/Dockerfile
@ -31,7 +31,6 @@ COPY ./packages/app/src ./src
 COPY ./packages/app/pages ./pages 
 COPY ./packages/app/public ./public 
 COPY ./packages/app/styles ./styles
-COPY ./packages/app/json ./json
 COPY --from=base /app/node_modules ./node_modules
 RUN yarn build && yarn install --production --ignore-scripts --prefer-offline

@ -48,7 +47,6 @@ RUN adduser -S nextjs -u 1001
 # You only need to copy next.config.js if you are NOT using the default configuration
 # COPY --from=builder /app/next.config.js ./
 COPY --from=builder /app/public ./public
-COPY --from=builder /app/json ./json
 COPY --from=builder --chown=nextjs:nodejs /app/.next ./.next
 COPY --from=builder /app/node_modules ./node_modules
 COPY --from=builder /app/package.json ./package.json
--- a/packages/app/package.json
+++ b/packages/app/package.json
@ -1,6 +1,6 @@
 {
  "name": "@hyperdx/app",
-  "version": "1.0.0",
+  "version": "1.0.3",
  "private": true,
  "license": "MIT",
  "engines": {
--- a/packages/app/pages/_app.tsx
+++ b/packages/app/pages/_app.tsx
@ -1,5 +1,5 @@
 import Head from 'next/head';
-import React from 'react';
+import React, { useEffect } from 'react';
 import SSRProvider from 'react-bootstrap/SSRProvider';
 import type { AppProps } from 'next/app';
 import { QueryClient, QueryClientProvider } from 'react-query';
@ -22,23 +22,28 @@ const queryClient = new QueryClient();

 import HyperDX from '@hyperdx/browser';

-if (config.HDX_API_KEY != null) {
-  HyperDX.init({
-    ...(config.HDX_COLLECTOR_URL != null
-      ? {
-          url: config.HDX_COLLECTOR_URL,
-        }
-      : {}),
-    apiKey: config.HDX_API_KEY,
-    consoleCapture: true,
-    maskAllInputs: true,
-    maskAllText: true,
-    service: config.HDX_SERVICE_NAME,
-    tracePropagationTargets: [/localhost/i, /hyperdx\.io/i],
-  });
-}
-
 export default function MyApp({ Component, pageProps }: AppProps) {
+  // port to react query ? (needs to wrap with QueryClientProvider)
+  useEffect(() => {
+    fetch('/api/config')
+      .then(res => res.json())
+      .then(_jsonData => {
+        if (_jsonData?.apiKey) {
+          HyperDX.init({
+            apiKey: _jsonData.apiKey,
+            consoleCapture: true,
+            maskAllInputs: true,
+            maskAllText: true,
+            service: _jsonData.serviceName,
+            tracePropagationTargets: [/localhost/i, /hyperdx\.io/i],
+            url: _jsonData.collectorUrl,
+          });
+        }
+      })
+      .catch(err => {
+        // ignore
+      });
+  });
  return (
    <React.Fragment>
      <Head>
--- a/packages/app/pages/api/config.ts
+++ b/packages/app/pages/api/config.ts
@ -0,0 +1,24 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+
+import {
+  HDX_API_KEY,
+  HDX_COLLECTOR_URL,
+  HDX_SERVICE_NAME,
+} from '../../src/config';
+
+type ResponseData = {
+  apiKey: string;
+  collectorUrl: string;
+  serviceName: string;
+};
+
+export default function handler(
+  req: NextApiRequest,
+  res: NextApiResponse<ResponseData>,
+) {
+  res.status(200).json({
+    apiKey: HDX_API_KEY,
+    collectorUrl: HDX_COLLECTOR_URL,
+    serviceName: HDX_SERVICE_NAME,
+  });
+}
--- a/packages/app/src/config.ts
+++ b/packages/app/src/config.ts
@ -1,10 +1,11 @@
 export const API_SERVER_URL =
-  process.env.NEXT_PUBLIC_API_SERVER_URL ?? 'http://localhost:8000';
+  process.env.NEXT_PUBLIC_API_SERVER_URL || 'http://localhost:8000'; // NEXT_PUBLIC_API_SERVER_URL can be empty string

-export const HDX_API_KEY = process.env.NEXT_PUBLIC_HDX_API_KEY as string;
+export const HDX_API_KEY = (process.env.NEXT_PUBLIC_HDX_API_KEY ||
+  process.env.HYPERDX_API_KEY) as string;
 export const HDX_SERVICE_NAME =
-  process.env.NEXT_PUBLIC_HDX_SERVICE_NAME ?? 'hdx-oss-dev-app';
-export const HDX_COLLECTOR_URL = process.env
-  .NEXT_PUBLIC_HDX_COLLECTOR_URL as string;
+  process.env.NEXT_PUBLIC_HDX_SERVICE_NAME || 'hdx-oss-dev-app';
+export const HDX_COLLECTOR_URL =
+  process.env.NEXT_PUBLIC_HDX_COLLECTOR_URL || 'http://localhost:4318';

 export const IS_OSS = process.env.NEXT_PUBLIC_IS_OSS ?? 'true' === 'true';
--- a/yarn.lock
+++ b/yarn.lock
@ -4429,13 +4429,6 @@
  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.0.tgz#4668bc392bb6938637b47e98b1f2ed5426f33316"
  integrity sha512-BsAaKhB+7X+H4GnSjGhJG9Qi8Tw+inU9nJDwmD5CgOmBLEI6ArdhikpLX7DjbjDRDTbqZzU2LSQNZg8WGPiSZQ==

-"@types/oauth@*":
-  version "0.9.1"
-  resolved "https://registry.yarnpkg.com/@types/oauth/-/oauth-0.9.1.tgz#e17221e7f7936b0459ae7d006255dff61adca305"
-  integrity sha512-a1iY62/a3yhZ7qH7cNUsxoI3U/0Fe9+RnuFrpTKr+0WVOzbKlSLojShCKe20aOD1Sppv+i8Zlq0pLDuTJnwS4A==
-  dependencies:
-    "@types/node" "*"
-
 "@types/object-hash@^2.2.1":
  version "2.2.1"
  resolved "https://registry.yarnpkg.com/@types/object-hash/-/object-hash-2.2.1.tgz#67c169f8f033e0b62abbf81df2d00f4598d540b9"
@ -4451,15 +4444,6 @@
  resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.3.tgz#705bb349e789efa06f43f128cef51240753424cb"
  integrity sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==

-"@types/passport-google-oauth20@^2.0.11":
-  version "2.0.11"
-  resolved "https://registry.yarnpkg.com/@types/passport-google-oauth20/-/passport-google-oauth20-2.0.11.tgz#271ec71de3030a3e1c004b24e633e4b298ccba97"
-  integrity sha512-9XMT1GfwhZL7UQEiCepLef55RNPHkbrCtsU7rsWPTEOsmu5qVIW8nSemtB4p+P24CuOhA+IKkv8LsPThYghGww==
-  dependencies:
-    "@types/express" "*"
-    "@types/passport" "*"
-    "@types/passport-oauth2" "*"
-
 "@types/passport-http-bearer@^1.0.37":
  version "1.0.37"
  resolved "https://registry.yarnpkg.com/@types/passport-http-bearer/-/passport-http-bearer-1.0.37.tgz#6882825a46717725f952731d17e1bb0a698155a4"
@ -4478,15 +4462,6 @@
    "@types/passport" "*"
    "@types/passport-strategy" "*"

-"@types/passport-oauth2@*":
-  version "1.4.12"
-  resolved "https://registry.yarnpkg.com/@types/passport-oauth2/-/passport-oauth2-1.4.12.tgz#c2ae0ee3b16646188d8b0b6cdbc6880a0247dc5f"
-  integrity sha512-RZg6cYTyEGinrZn/7REYQds6zrTxoBorX1/fdaz5UHzkG8xdFE7QQxkJagCr2ETzGII58FAFDmnmbTUVMrltNA==
-  dependencies:
-    "@types/express" "*"
-    "@types/oauth" "*"
-    "@types/passport" "*"
-
 "@types/passport-strategy@*":
  version "0.2.35"
  resolved "https://registry.yarnpkg.com/@types/passport-strategy/-/passport-strategy-0.2.35.tgz#e52f5212279ea73f02d9b06af67efe9cefce2d0c"