DX: running api + app + task concurrently with npm script (dev) (#567)

1. Test with `common-utils` easily 2. Faster hot-reloads TL;DR run `npm run dev` or `make dev-up` to run HyperDX fullstack locally
2026-04-21 13:37:15 +00:00 · 2025-01-23 09:31:25 -08:00 · 2025-01-23 09:31:25 -08:00 · af4faa4611
commit af4faa4611
parent 7c03397fe9
12 changed files with 161 additions and 125 deletions
--- a/2
+++ b/2
@ -17,7 +17,7 @@ dev-build:

 .PHONY: dev-up
 dev-up:
-	docker compose -f docker-compose.dev.yml up -d
+	npm run dev

 .PHONY: dev-down
 dev-down:
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@ -99,70 +99,6 @@ services:
  #     - ch-server
  #     - db
  #     - redis
-  api:
-    build:
-      context: .
-      dockerfile: ./packages/api/Dockerfile
-      target: dev
-    ports:
-      - ${HYPERDX_API_PORT}:${HYPERDX_API_PORT}
-    environment:
-      EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
-      FRONTEND_URL: 'http://localhost:${HYPERDX_APP_PORT}' # need to be localhost (CORS)
-      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
-      HDX_NODE_BETA_MODE: 1
-      HDX_NODE_CONSOLE_CAPTURE: 1
-      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      MINER_API_URL: 'http://miner:5123'
-      MONGO_URI: 'mongodb://db:27017/hyperdx'
-      NODE_ENV: development
-      OTEL_SERVICE_NAME: 'hdx-oss-dev-api'
-      PORT: ${HYPERDX_API_PORT}
-      REDIS_URL: redis://redis:6379
-      USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-false}
-      # Event deltas create large SQL queries so we need to increase
-      # the max header size for proxied SQL requests
-      NODE_OPTIONS: '--max-http-header-size=131072'
-    volumes:
-      - ./packages/api/src:/app/src
-    networks:
-      - internal
-    depends_on:
-      - ch-server
-      - db
-      - redis
-  app:
-    build:
-      context: .
-      dockerfile: ./packages/app/Dockerfile
-      target: dev
-    ports:
-      - ${HYPERDX_APP_PORT}:${HYPERDX_APP_PORT}
-    environment:
-      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      NEXT_PUBLIC_HDX_COLLECTOR_URL: 'http://localhost:4318'
-      NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-dev-app'
-      NEXT_PUBLIC_SERVER_URL: 'http://api:${HYPERDX_API_PORT}'
-      NODE_ENV: development
-      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
-      OTEL_SERVICE_NAME: 'hdx-oss-dev-app'
-      PORT: ${HYPERDX_APP_PORT}
-      # Event deltas create large SQL queries so we need to increase
-      # the max header size for proxied SQL requests
-      NODE_OPTIONS: '--max-http-header-size=131072'
-    volumes:
-      - ./packages/app/mdx.d.ts:/app/mdx.d.ts
-      - ./packages/app/next-env.d.ts:/app/next-env.d.ts
-      - ./packages/app/next.config.js:/app/next.config.js
-      - ./packages/app/pages:/app/pages
-      - ./packages/app/public:/app/public
-      - ./packages/app/src:/app/src
-      - ./packages/app/styles:/app/styles
-    networks:
-      - internal
-    depends_on:
-      - api
  ch-server:
    image: clickhouse/clickhouse-server:head-alpine
    ports:
--- a/package.json
+++ b/package.json
@ -11,7 +11,10 @@
    "@nx/workspace": "16.8.1",
    "@typescript-eslint/eslint-plugin": "^8.7.0",
    "@typescript-eslint/parser": "^8.7.0",
+    "concurrently": "^9.1.2",
    "dotenv": "^16.4.7",
+    "dotenv-cli": "^8.0.0",
+    "dotenv-expand": "^12.0.1",
    "eslint": "^8.57.0",
    "eslint-config-next": "13",
    "eslint-config-prettier": "^9.1.0",
@ -26,11 +29,14 @@
  },
  "scripts": {
    "prepare": "husky install",
+    "app:dev": "npx concurrently -k -n 'API,APP,ALERTS-TASK' -c 'green.bold,blue.bold,yellow.bold' 'nx run @hyperdx/api:dev' 'nx run @hyperdx/app:dev' 'nx run @hyperdx/api:dev-task check-alerts'",
    "app:lint": "nx run @hyperdx/app:ci:lint",
+    "predev": "docker compose -f docker-compose.dev.yml up -d",
+    "dev": "yarn app:dev",
+    "postdev": "docker compose -f docker-compose.dev.yml down",
    "lint": "npx nx run-many -t ci:lint",
    "version": "npx changeset version --ignore @hyperdx/api --ignore @hyperdx/app",
-    "release": "npx nx run-many --target=build --projects=@hyperdx/common-utils && npx changeset publish",
-    "dev": "docker compose -f docker-compose.dev.yml up"
+    "release": "npx nx run-many --target=build --projects=@hyperdx/common-utils && npx changeset publish"
  },
  "lint-staged": {
    "**/*.{ts,tsx}": [
--- a/packages/api/.env.development
+++ b/packages/api/.env.development
@ -0,0 +1,20 @@
+HYPERDX_API_KEY="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+HYPERDX_API_PORT=8000
+HYPERDX_APP_PORT=8080
+HYPERDX_LOG_LEVEL=debug
+EXPRESS_SESSION_SECRET="hyperdx is cool 👋"
+FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}"
+HDX_NODE_ADVANCED_NETWORK_CAPTURE=1
+HDX_NODE_BETA_MODE=1
+HDX_NODE_CONSOLE_CAPTURE=1
+HYPERDX_API_KEY=${HYPERDX_API_KEY}
+HYPERDX_LOG_LEVEL=${HYPERDX_LOG_LEVEL}
+MINER_API_URL="http://localhost:5123"
+MONGO_URI="mongodb://localhost:27017/hyperdx"
+NODE_ENV=development
+OTEL_SERVICE_NAME="hdx-oss-dev-api"
+OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
+PORT=${HYPERDX_API_PORT}
+REDIS_URL=redis://localhost:6379
+USAGE_STATS_ENABLED=false
+NODE_OPTIONS="--max-http-header-size=131072"
--- a/packages/api/package.json
+++ b/packages/api/package.json
@ -17,6 +17,7 @@
    "compression": "^1.7.4",
    "connect-mongo": "^4.6.0",
    "cors": "^2.8.5",
+    "cron": "^3.5.0",
    "date-fns": "^2.28.0",
    "date-fns-tz": "^2.0.0",
    "express": "^4.19.2",
@ -33,7 +34,6 @@
    "mongodb": "^6.3.0",
    "mongoose": "^6.12.0",
    "ms": "^2.1.3",
-    "node-schedule": "^2.1.1",
    "object-hash": "^3.0.0",
    "on-headers": "^1.0.2",
    "passport": "^0.6.0",
@ -85,8 +85,8 @@
  },
  "scripts": {
    "start": "node ./build/index.js",
-    "dev": "nodemon --signal SIGTERM -e ts,json --exec 'ts-node' --transpile-only -r tsconfig-paths/register -r '@hyperdx/node-opentelemetry/build/src/tracing' ./src/index.ts",
-    "dev:task": "ts-node -r tsconfig-paths/register ./src/tasks/index.ts",
+    "dev": "DOTENV_CONFIG_PATH=.env.development nodemon --signal SIGTERM -e ts,json --exec 'ts-node' --transpile-only -r tsconfig-paths/register -r dotenv-expand/config -r '@hyperdx/node-opentelemetry/build/src/tracing' ./src/index.ts",
+    "dev-task": "DOTENV_CONFIG_PATH=.env.development nodemon --signal SIGTERM -e ts,json --exec 'ts-node' --transpile-only -r tsconfig-paths/register -r dotenv-expand/config -r '@hyperdx/node-opentelemetry/build/src/tracing' ./src/tasks/index.ts",
    "build": "rimraf ./build && tsc && tsc-alias",
    "lint": "eslint --quiet . --ext .ts",
    "lint:fix": "eslint . --ext .ts --fix",
--- a/packages/api/src/routers/api/clickhouseProxy.ts
+++ b/packages/api/src/routers/api/clickhouseProxy.ts
@ -108,9 +108,9 @@ router.get(
            console.error(err);
          },
        },
-        ...(config.IS_DEV && {
-          logger: console,
-        }),
+        // ...(config.IS_DEV && {
+        //   logger: console,
+        // }),
      })(req, res, next);
    } catch (e) {
      next(e);
--- a/packages/api/src/tasks/index.ts
+++ b/packages/api/src/tasks/index.ts
@ -1,5 +1,5 @@
+import { CronJob } from 'cron';
 import minimist from 'minimist';
-import schedule from 'node-schedule';
 import { performance } from 'perf_hooks';
 import { serializeError } from 'serialize-error';

@ -11,10 +11,7 @@ import redisClient from '@/utils/redis';
 import checkAlerts from './checkAlerts';
 import refreshPropertyTypeMappings from './refreshPropertyTypeMappings';

-const main = async () => {
-  const argv = minimist(process.argv.slice(2));
-  const taskName = argv._[0];
-
+const main = async (taskName: string) => {
  // connect dbs + redis
  await Promise.all([connectDB(), redisClient.connect()]);

@ -57,10 +54,24 @@ const main = async () => {
  await Promise.all([redisClient.disconnect(), mongooseConnection.close()]);
 };

+// Entry point
+const argv = minimist(process.argv.slice(2));
+const taskName = argv._[0];
+// WARNING: the cron job will be enabled only in development mode
 if (IS_DEV) {
-  schedule.scheduleJob('*/1 * * * *', main);
+  // run cron job every 15 seconds
+  const job = CronJob.from({
+    cronTime: '*/15 * * * * *',
+    waitForCompletion: true,
+    onTick: async () => main(taskName),
+    errorHandler: err => {
+      console.error(err);
+    },
+    start: true,
+    timeZone: 'UTC',
+  });
 } else {
-  main()
+  main(taskName)
    .then(() => {
      process.exit(0);
    })
--- a/packages/app/.env.development
+++ b/packages/app/.env.development
@ -0,0 +1,9 @@
+HYPERDX_API_KEY="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+HYPERDX_API_PORT=8000
+HYPERDX_APP_PORT=8080
+NEXT_PUBLIC_SERVER_URL="http://localhost:${HYPERDX_API_PORT}"
+NODE_ENV=development
+OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
+OTEL_SERVICE_NAME="hdx-oss-dev-app"
+PORT=${HYPERDX_APP_PORT}
+NODE_OPTIONS="--max-http-header-size=131072"
--- a/packages/app/package.json
+++ b/packages/app/package.json
@ -7,8 +7,8 @@
    "node": ">=18.12.0"
  },
  "scripts": {
-    "dev": "next dev",
-    "dev:local": "NEXT_PUBLIC_IS_LOCAL_MODE=true next dev -p 8080",
+    "dev": "npx dotenv -e .env.development -- next dev",
+    "dev:local": "NEXT_PUBLIC_IS_LOCAL_MODE=true npx dotenv -e .env.development -- next dev",
    "build": "next build",
    "start": "next start",
    "lint": "eslint --quiet . --ext .ts,.tsx",
--- a/packages/app/pages/api/[...all].ts
+++ b/packages/app/pages/api/[...all].ts
@ -25,9 +25,9 @@ export default (req: NextApiRequest, res: NextApiResponse) => {
    on: {
      proxyReq: fixRequestBody,
    },
-    ...(IS_DEV && {
-      logger: console,
-    }),
+    // ...(IS_DEV && {
+    //   logger: console,
+    // }),
  });
  return proxy(req, res, error => {
    if (error) {
--- a/packages/common-utils/README.md
+++ b/packages/common-utils/README.md
@ -1,5 +1,5 @@
-## How to test the package with HyperDX app (browser) locally
+## How to test the package with HyperDX app/api 

-1. Run `yarn dev` in the root directory of this project (packages/common-utils)
-2. Run `yarn dev:local` in the root directory of app project (packages/app)
+1. Run `yarn build` in the root directory of this project (packages/common-utils)
+2. Run `yarn dev` in the root directory of app/api project (packages/app,api)
 3. You should be able to test utils with the app
--- a/yarn.lock
+++ b/yarn.lock
@ -4245,6 +4245,7 @@ __metadata:
    compression: "npm:^1.7.4"
    connect-mongo: "npm:^4.6.0"
    cors: "npm:^2.8.5"
+    cron: "npm:^3.5.0"
    date-fns: "npm:^2.28.0"
    date-fns-tz: "npm:^2.0.0"
    express: "npm:^4.19.2"
@ -4263,7 +4264,6 @@ __metadata:
    mongodb: "npm:^6.3.0"
    mongoose: "npm:^6.12.0"
    ms: "npm:^2.1.3"
-    node-schedule: "npm:^2.1.1"
    nodemon: "npm:^2.0.20"
    object-hash: "npm:^3.0.0"
    on-headers: "npm:^1.0.2"
@ -9852,6 +9852,13 @@ __metadata:
  languageName: node
  linkType: hard

+"@types/luxon@npm:~3.4.0":
+  version: 3.4.2
+  resolution: "@types/luxon@npm:3.4.2"
+  checksum: 10c0/d835467de3daf7e17ba78b50bb5a14efd94272439ca067990d71332a54b311544459c69623eddd243b511b28d70194c9591a9ee8cf9c038962c965f991affd7e
+  languageName: node
+  linkType: hard
+
 "@types/mdast@npm:^3.0.0":
  version: 3.0.10
  resolution: "@types/mdast@npm:3.0.10"
@ -13104,6 +13111,24 @@ __metadata:
  languageName: node
  linkType: hard

+"concurrently@npm:^9.1.2":
+  version: 9.1.2
+  resolution: "concurrently@npm:9.1.2"
+  dependencies:
+    chalk: "npm:^4.1.2"
+    lodash: "npm:^4.17.21"
+    rxjs: "npm:^7.8.1"
+    shell-quote: "npm:^1.8.1"
+    supports-color: "npm:^8.1.1"
+    tree-kill: "npm:^1.2.2"
+    yargs: "npm:^17.7.2"
+  bin:
+    conc: dist/bin/concurrently.js
+    concurrently: dist/bin/concurrently.js
+  checksum: 10c0/88e00269366aa885ca2b97fd53b04e7af2b0f31774d991bfc0e88c0de61cdebdf115ddacc9c897fbd1f1b90369014637fa77045a171d072a75693332b36dcc70
+  languageName: node
+  linkType: hard
+
 "connect-mongo@npm:^4.6.0":
  version: 4.6.0
  resolution: "connect-mongo@npm:4.6.0"
@ -13325,12 +13350,13 @@ __metadata:
  languageName: node
  linkType: hard

-"cron-parser@npm:^4.2.0":
-  version: 4.9.0
-  resolution: "cron-parser@npm:4.9.0"
+"cron@npm:^3.5.0":
+  version: 3.5.0
+  resolution: "cron@npm:3.5.0"
  dependencies:
-    luxon: "npm:^3.2.1"
-  checksum: 10c0/348622bdcd1a15695b61fc33af8a60133e5913a85cf99f6344367579e7002896514ba3b0a9d6bb569b02667d6b06836722bf2295fcd101b3de378f71d37bed0b
+    "@types/luxon": "npm:~3.4.0"
+    luxon: "npm:~3.5.0"
+  checksum: 10c0/ca5fbfb0d54a77ff4a293ad7bd0e67872915985d739fa4b72ecfaae72db1918f542469a1a6638af4c5a280753443ba308dd04382fe8dcc07d29bd7922856ea94
  languageName: node
  linkType: hard

@ -13356,6 +13382,17 @@ __metadata:
  languageName: node
  linkType: hard

+"cross-spawn@npm:^7.0.6":
+  version: 7.0.6
+  resolution: "cross-spawn@npm:7.0.6"
+  dependencies:
+    path-key: "npm:^3.1.0"
+    shebang-command: "npm:^2.0.0"
+    which: "npm:^2.0.1"
+  checksum: 10c0/053ea8b2135caff68a9e81470e845613e374e7309a47731e81639de3eaeb90c3d01af0e0b44d2ab9d50b43467223b88567dfeb3262db942dc063b9976718ffc1
+  languageName: node
+  linkType: hard
+
 "crypto-browserify@npm:^3.12.0":
  version: 3.12.0
  resolution: "crypto-browserify@npm:3.12.0"
@ -14188,6 +14225,20 @@ __metadata:
  languageName: node
  linkType: hard

+"dotenv-cli@npm:^8.0.0":
+  version: 8.0.0
+  resolution: "dotenv-cli@npm:8.0.0"
+  dependencies:
+    cross-spawn: "npm:^7.0.6"
+    dotenv: "npm:^16.3.0"
+    dotenv-expand: "npm:^10.0.0"
+    minimist: "npm:^1.2.6"
+  bin:
+    dotenv: cli.js
+  checksum: 10c0/000469632758b7b44aaaa80cbbbd7f0c94dc170ec02e51aa8d8280341a0108fb7407954c23054257b77235b064033efdb8745836633eb6fd1586924953cf0528
+  languageName: node
+  linkType: hard
+
 "dotenv-expand@npm:^10.0.0, dotenv-expand@npm:~10.0.0":
  version: 10.0.0
  resolution: "dotenv-expand@npm:10.0.0"
@ -14195,6 +14246,15 @@ __metadata:
  languageName: node
  linkType: hard

+"dotenv-expand@npm:^12.0.1":
+  version: 12.0.1
+  resolution: "dotenv-expand@npm:12.0.1"
+  dependencies:
+    dotenv: "npm:^16.4.5"
+  checksum: 10c0/51996bfa670073d7a441b8fbed26ac991026fba2c05e9a937a898ce7d2a2e7166f7b6ec4eb8879e576defb5d1ad399ed1300db8f803d6977577fea55b4d82dac
+  languageName: node
+  linkType: hard
+
 "dotenv@npm:^16.0.0":
  version: 16.4.5
  resolution: "dotenv@npm:16.4.5"
@ -14202,7 +14262,7 @@ __metadata:
  languageName: node
  linkType: hard

-"dotenv@npm:^16.4.7":
+"dotenv@npm:^16.3.0, dotenv@npm:^16.4.5, dotenv@npm:^16.4.7":
  version: 16.4.7
  resolution: "dotenv@npm:16.4.7"
  checksum: 10c0/be9f597e36a8daf834452daa1f4cc30e5375a5968f98f46d89b16b983c567398a330580c88395069a77473943c06b877d1ca25b4afafcdd6d4adb549e8293462
@ -17350,7 +17410,10 @@ __metadata:
    "@nx/workspace": "npm:16.8.1"
    "@typescript-eslint/eslint-plugin": "npm:^8.7.0"
    "@typescript-eslint/parser": "npm:^8.7.0"
+    concurrently: "npm:^9.1.2"
    dotenv: "npm:^16.4.7"
+    dotenv-cli: "npm:^8.0.0"
+    dotenv-expand: "npm:^12.0.1"
    eslint: "npm:^8.57.0"
    eslint-config-next: "npm:13"
    eslint-config-prettier: "npm:^9.1.0"
@ -19584,13 +19647,6 @@ __metadata:
  languageName: node
  linkType: hard

-"long-timeout@npm:0.1.1":
-  version: 0.1.1
-  resolution: "long-timeout@npm:0.1.1"
-  checksum: 10c0/a62240cc8f449d7a00081e817ae543fb1ded4d9fc05492e9fa8d6868cb33b2c9d5d71176a6f8be4473df7ba4b208460b3073b0e05069c3ec286122f3e4b5747f
-  languageName: node
-  linkType: hard
-
 "long@npm:^4.0.0":
  version: 4.0.0
  resolution: "long@npm:4.0.0"
@ -19690,10 +19746,10 @@ __metadata:
  languageName: node
  linkType: hard

-"luxon@npm:^3.2.1":
-  version: 3.4.3
-  resolution: "luxon@npm:3.4.3"
-  checksum: 10c0/65ad727684d367af9c8fcbde05ad7989ef2436a4496b62d993059baaa641e90a5fa74e98f34ee2ff29f7af7f2b27238ca78b87834f9068624e4133d786f87bc7
+"luxon@npm:~3.5.0":
+  version: 3.5.0
+  resolution: "luxon@npm:3.5.0"
+  checksum: 10c0/335789bba95077db831ef99894edadeb23023b3eb2137a1b56acd0d290082b691cf793143d69e30bc069ec95f0b49f36419f48e951c68014f19ffe12045e3494
  languageName: node
  linkType: hard

@ -21548,17 +21604,6 @@ __metadata:
  languageName: node
  linkType: hard

-"node-schedule@npm:^2.1.1":
-  version: 2.1.1
-  resolution: "node-schedule@npm:2.1.1"
-  dependencies:
-    cron-parser: "npm:^4.2.0"
-    long-timeout: "npm:0.1.1"
-    sorted-array-functions: "npm:^1.3.0"
-  checksum: 10c0/6ec51b34b9e676740ac25298e4ced5ee46053379f0d3aad533e51d7e083bc24ced045df1772a95bf9d9cfdb81299340bbf551549a7c5eb6e4d2dc6468c85c70e
-  languageName: node
-  linkType: hard
-
 "node-sql-parser@npm:^5.3.5":
  version: 5.3.5
  resolution: "node-sql-parser@npm:5.3.5"
@ -24856,6 +24901,15 @@ __metadata:
  languageName: node
  linkType: hard

+"rxjs@npm:^7.8.1":
+  version: 7.8.1
+  resolution: "rxjs@npm:7.8.1"
+  dependencies:
+    tslib: "npm:^2.1.0"
+  checksum: 10c0/3c49c1ecd66170b175c9cacf5cef67f8914dcbc7cd0162855538d365c83fea631167cacb644b3ce533b2ea0e9a4d0b12175186985f89d75abe73dbd8f7f06f68
+  languageName: node
+  linkType: hard
+
 "sade@npm:^1.7.3":
  version: 1.8.1
  resolution: "sade@npm:1.8.1"
@ -25361,6 +25415,13 @@ __metadata:
  languageName: node
  linkType: hard

+"shell-quote@npm:^1.8.1":
+  version: 1.8.2
+  resolution: "shell-quote@npm:1.8.2"
+  checksum: 10c0/85fdd44f2ad76e723d34eb72c753f04d847ab64e9f1f10677e3f518d0e5b0752a176fd805297b30bb8c3a1556ebe6e77d2288dbd7b7b0110c7e941e9e9c20ce1
+  languageName: node
+  linkType: hard
+
 "shiki@npm:0.10.1":
  version: 0.10.1
  resolution: "shiki@npm:0.10.1"
@ -25555,13 +25616,6 @@ __metadata:
  languageName: node
  linkType: hard

-"sorted-array-functions@npm:^1.3.0":
-  version: 1.3.0
-  resolution: "sorted-array-functions@npm:1.3.0"
-  checksum: 10c0/d94e3401a2bc1689dc913f56939621c892a3ff1288e984e85689a6c6e46b0ec16f65edc8b47d46b0f09d06857f67ca245553b462da597619102b9fad270476d9
-  languageName: node
-  linkType: hard
-
 "source-map-js@npm:>=0.6.2 <2.0.0, source-map-js@npm:^1.0.2":
  version: 1.0.2
  resolution: "source-map-js@npm:1.0.2"
@ -26385,7 +26439,7 @@ __metadata:
  languageName: node
  linkType: hard

-"supports-color@npm:^8.0.0":
+"supports-color@npm:^8.0.0, supports-color@npm:^8.1.1":
  version: 8.1.1
  resolution: "supports-color@npm:8.1.1"
  dependencies: