feat: introduce RUN_SCHEDULED_TASKS_EXTERNALLY (#733)

1. new env var RUN_SCHEDULED_TASKS_EXTERNALLY to opt out in-app task process
2. introduce new `prod-extended` build that includes mongodb process
3. GA k8s dashboard (only picking the connection)
4. bake check-alert task into fullstack app build

.changeset/twelve-ladybugs-confess.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+feat: introduce RUN_SCHEDULED_TASKS_EXTERNALLY + enable in-app task

Makefile

@@ -130,3 +130,16 @@ release-app:
 		--push \
 	 	--cache-from=type=gha \
 		--cache-to=type=gha,mode=max
+
+.PHONY: release-extended-app
+release-app:
+	docker buildx build --squash . -f ./docker/fullstack/Dockerfile \
+		--build-context fullstack=./docker/fullstack \
+		--build-context api=./packages/api \
+		--build-context app=./packages/app \
+		--platform ${BUILD_PLATFORMS} \
+		-t ${IMAGE_NAME_DOCKERHUB}:latest
+		--target prod-extended \
+		--push \
+	 	--cache-from=type=gha \
+		--cache-to=type=gha,mode=max

docker-compose.dev.yml

@@ -70,37 +70,6 @@ services:
     depends_on:
       ch-server:
         condition: service_healthy
-  # task-check-alerts:
-  #   build:
-  #     context: .
-  #     dockerfile: ./packages/api/Dockerfile
-  #     target: dev
-  #   entrypoint: 'yarn'
-  #   command: 'dev:task check-alerts'
-  #   environment:
-  #     APP_TYPE: 'scheduled-task'
-  #     EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
-  #     FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
-  #     HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
-  #     HDX_NODE_BETA_MODE: 0
-  #     HDX_NODE_CONSOLE_CAPTURE: 1
-  #     HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-  #     HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-  #     MINER_API_URL: 'http://miner:5123'
-  #     MONGO_URI: 'mongodb://db:27017/hyperdx'
-  #     NODE_ENV: development
-  #     OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
-  #     OTEL_SERVICE_NAME: 'hdx-oss-dev-task-check-alerts'
-  #     REDIS_URL: redis://redis:6379
-  #   volumes:
-  #     - ./packages/api/src:/app/src
-  #   restart: always
-  #   networks:
-  #     - internal
-  #   depends_on:
-  #     - ch-server
-  #     - db
-  #     - redis
   ch-server:
     image: clickhouse/clickhouse-server:24-alpine
     ports:

docker-compose.yml

@@ -82,28 +82,6 @@ services:
       - internal
     depends_on:
       - ch-server
-  task-check-alerts:
-    image: ${IMAGE_NAME_HDX}:${IMAGE_VERSION}
-    entrypoint: 'node'
-    command: '/app/api/build/tasks/index.js check-alerts'
-    environment:
-      APP_TYPE: 'scheduled-task'
-      FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT}
-      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
-      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      MINER_API_URL: 'http://miner:5123'
-      MONGO_URI: 'mongodb://db:27017/hyperdx'
-      NODE_ENV: 'development' # so task will CronJob library
-      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
-      OTEL_SERVICE_NAME: 'hdx-oss-task-check-alerts'
-      REDIS_URL: redis://redis:6379
-    restart: always
-    networks:
-      - internal
-    depends_on:
-      - ch-server
-      - db
-      - redis
   app:
     image: ${IMAGE_NAME_HDX}:${IMAGE_VERSION}
     ports:

docker/fullstack/Dockerfile

@@ -76,3 +76,23 @@ COPY --from=app_builder /app/app/package.json ./package.json
 # Set up start script
 COPY --chown=node:node --from=fullstack ./entry.sh /etc/local/entry.sh
 ENTRYPOINT ["sh", "/etc/local/entry.sh"]
+
+# prod-extended ############################################################################################
+FROM prod AS prod-extended
+
+USER root
+
+# == Install MongoDB v4 Deps ==
+RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/main' >> /etc/apk/repositories
+RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/community' >> /etc/apk/repositories
+RUN apk update
+RUN apk add mongodb yaml-cpp=0.6.2-r2
+
+# Set up Mongo directories with proper permissions
+RUN mkdir -p /data/db && \
+  chown -R node:node /data/db && \
+  chown -R node:node /var/log
+
+USER node
+
+ENTRYPOINT ["sh", "/etc/local/entry.sh"]

docker/fullstack/entry.sh

@@ -2,12 +2,24 @@
 
 export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}"
 
+# Internal Services
+export MONGO_URI="${MONGO_URI:-mongodb://127.0.0.1:27017/hyperdx}"
+
 echo "Visit the HyperDX UI at $FRONTEND_URL"
 echo ""
 
+# Start Mongo Server if it exists (will run in prod-extended but not in prod)
+if command -v mongod &> /dev/null; then
+  echo "Starting MongoDB server..."
+  mongod --quiet --dbpath /data/db > /var/log/mongod.log 2>&1 &
+  # Wait for MongoDB to start
+  sleep 2
+fi
+
 # Use concurrently to run both the API and App servers
 npx concurrently \
   "--kill-others" \
-  "--names=API,APP" \
-  "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \
-  "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}"
+  "--names=API,APP,ALERT-TASK" \
+  "MONGO_URI=${MONGO_URI} PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \
+  "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}" \
+  "MONGO_URI=${MONGO_URI} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/tasks/index.js check-alerts"

packages/api/.env.test

@@ -1,6 +1,7 @@
 CLICKHOUSE_HOST=http://localhost:8123
 CLICKHOUSE_PASSWORD=api
 CLICKHOUSE_USER=api
+CRON_IN_APP_DISABLED=true
 EXPRESS_SESSION_SECRET="hyperdx is cool 👋"
 FRONTEND_URL=http://app:8080
 MONGO_URI=mongodb://localhost:29999/hyperdx-test

packages/api/src/config.ts

@@ -26,6 +26,8 @@ export const OTEL_SERVICE_NAME = env.OTEL_SERVICE_NAME as string;
 export const PORT = Number.parseInt(env.PORT as string);
 export const REDIS_URL = env.REDIS_URL;
 export const USAGE_STATS_ENABLED = env.USAGE_STATS_ENABLED !== 'false';
+export const RUN_SCHEDULED_TASKS_EXTERNALLY =
+  env.RUN_SCHEDULED_TASKS_EXTERNALLY === 'true';
 
 // Only for single container local deployments, disable authentication
 export const IS_LOCAL_APP_MODE =

packages/api/src/tasks/index.ts

@@ -3,20 +3,18 @@ import minimist from 'minimist';
 import { performance } from 'perf_hooks';
 import { serializeError } from 'serialize-error';
 
-import { IS_DEV } from '@/config';
+import { RUN_SCHEDULED_TASKS_EXTERNALLY } from '@/config';
 import { connectDB, mongooseConnection } from '@/models';
 import logger from '@/utils/logger';
-import redisClient from '@/utils/redis';
 
 import checkAlerts from './checkAlerts';
 import refreshPropertyTypeMappings from './refreshPropertyTypeMappings';
 
-const shutdown = async () =>
-  Promise.all([redisClient.disconnect(), mongooseConnection.close()]);
+const shutdown = async () => Promise.all([mongooseConnection.close()]);
 
 const main = async (taskName: string) => {
-  // connect dbs + redis
-  await Promise.all([connectDB(), redisClient.connect()]);
+  // connect dbs
+  await Promise.all([connectDB()]);
 
   const t0 = performance.now();
   logger.info(`Task [${taskName}] started at ${new Date()}`);
@@ -60,10 +58,11 @@ const main = async (taskName: string) => {
 const argv = minimist(process.argv.slice(2));
 const taskName = argv._[0];
 // WARNING: the cron job will be enabled only in development mode
-if (IS_DEV) {
-  // run cron job every 15 seconds
+if (!RUN_SCHEDULED_TASKS_EXTERNALLY) {
+  logger.info('In-app cron job is enabled');
+  // run cron job every 1 minute
   const job = CronJob.from({
-    cronTime: '*/15 * * * * *',
+    cronTime: '0 * * * * *',
     waitForCompletion: true,
     onTick: async () => main(taskName),
     errorHandler: async err => {
@@ -74,6 +73,7 @@ if (IS_DEV) {
     timeZone: 'UTC',
   });
 } else {
+  logger.warn('In-app cron job is disabled');
   main(taskName)
     .then(() => {
       process.exit(0);

packages/app/src/config.ts

@@ -26,7 +26,7 @@ export const IS_LOCAL_MODE = //true;
   (process.env.NEXT_PUBLIC_IS_LOCAL_MODE ?? 'false') === 'true';
 
 // Features in development
-export const IS_K8S_DASHBOARD_ENABLED = false || IS_DEV;
+export const IS_K8S_DASHBOARD_ENABLED = true;
 export const IS_METRICS_ENABLED = true;
 export const IS_MTVIEWS_ENABLED = false;
 export const IS_SESSIONS_ENABLED = true;