diff --git a/.env b/.env new file mode 100644 index 00000000..4d3928a6 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +IMAGE_NAME=ghcr.io/hyperdxio/hyperdx +IMAGE_VERSION=1.0.3 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac2891e5..3b7ead51 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: ~/.vector/bin/vector --version ln -s ~/.vector/bin/vector /usr/local/bin/vector - name: Run lint + type check - run: yarn ci:lint + run: make ci-lint integration: timeout-minutes: 8 runs-on: ubuntu-20.04 @@ -46,4 +46,4 @@ jobs: docker buildx create --use --driver=docker-container docker buildx bake -f ./docker-compose.ci.yml --set *.cache-to="type=gha" --set *.cache-from="type=gha" --load - name: Run integration tests - run: yarn ci:int + run: make ci-int diff --git a/.gitignore b/.gitignore index 33ea49e8..f0970df6 100644 --- a/.gitignore +++ b/.gitignore @@ -11,11 +11,12 @@ **/lerna-debug.log* # dotenv environment variable files -**/.env **/.env.development.local **/.env.test.local **/.env.production.local **/.env.local +**/.dockerhub.env +**/.ghcr.env # Next.js build output packages/app/.next diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..5a4f2386 --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ +LATEST_VERSION := $$(sed -n 's/.*"version": "\([^"]*\)".*/\1/p' package.json) +BUILD_PLATFORMS = linux/arm64/v8,linux/amd64 + +include .env + +.PHONY: all +all: install-tools + +.PHONY: install-tools +install-tools: + yarn install + @echo "All tools installed" + +.PHONY: dev-build +dev-build: + docker compose -f docker-compose.dev.yml build + +.PHONY: dev-up +dev-up: + docker compose -f docker-compose.dev.yml up -d + +.PHONY: dev-down +dev-down: + docker compose -f docker-compose.dev.yml down + +.PHONY: dev-lint +dev-lint: + ./docker/ingestor/run_linting.sh && yarn workspaces run lint + +.PHONY: ci-lint +ci-lint: + ./docker/ingestor/run_linting.sh && yarn workspaces run ci:lint + +.PHONY: dev-int +dev-int: + docker compose -p int -f ./docker-compose.ci.yml run --rm api dev:int + +.PHONY: ci-int +ci-int: + docker compose -p int -f ./docker-compose.ci.yml run --rm api ci:int + + +.PHONY: build-and-push-ghcr +build-and-push-ghcr: + docker buildx build --platform ${BUILD_PLATFORMS} ./docker/hostmetrics -t ${IMAGE_NAME}:${LATEST_VERSION}-hostmetrics --target dev --push & + docker buildx build --platform ${BUILD_PLATFORMS} ./docker/ingestor -t ${IMAGE_NAME}:${LATEST_VERSION}-ingestor --target dev --push & + docker buildx build --platform ${BUILD_PLATFORMS} ./docker/otel-collector -t ${IMAGE_NAME}:${LATEST_VERSION}-otel-collector --target dev --push & + docker buildx build --platform ${BUILD_PLATFORMS} . -f ./packages/miner/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-miner --target dev --push & + docker buildx build --platform ${BUILD_PLATFORMS} . -f ./packages/api/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-api --target dev --push & + docker buildx build --platform ${BUILD_PLATFORMS} . -f ./packages/app/Dockerfile -t ${IMAGE_NAME}:${LATEST_VERSION}-app --target prod --push + + diff --git a/README.md b/README.md index 9a132bb5..50e4ff6d 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ You can get started by deploying a complete stack via Docker Compose. After cloning this repository, simply start the stack with: ```bash -docker compose up +docker compose up -d ``` Afterwards, you can visit http://localhost:8080 to access the HyperDX UI. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..daa7c429 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,235 @@ +version: '3' +services: + miner: + container_name: hdx-oss-dev-miner + build: + context: . + dockerfile: ./packages/miner/Dockerfile + target: dev + environment: + HYPERDX_API_KEY: ${HYPERDX_API_KEY} + HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1 + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_LOG_LEVEL: ERROR + OTEL_SERVICE_NAME: hdx-oss-dev-miner + volumes: + - ./packages/miner/src:/app/src + ports: + - 5123:5123 + networks: + - internal + hostmetrics: + container_name: hdx-oss-dev-hostmetrics + build: + context: ./docker/hostmetrics + target: dev + volumes: + - ./docker/hostmetrics/config.dev.yaml:/etc/otelcol-contrib/config.yaml + environment: + HYPERDX_API_KEY: ${HYPERDX_API_KEY} + OTEL_SERVICE_NAME: hostmetrics + restart: always + networks: + - internal + ingestor: + container_name: hdx-oss-dev-ingestor + build: + context: ./docker/ingestor + target: dev + volumes: + - ./docker/ingestor:/app + - .volumes/ingestor_data:/var/lib/vector + ports: + - 8002:8002 # http-generic + - 8686:8686 # healthcheck + environment: + RUST_BACKTRACE: full + VECTOR_LOG: debug + VECTOR_OPENSSL_LEGACY_PROVIDER: false + restart: always + networks: + - internal + redis: + image: redis:7.0.11-alpine + container_name: hdx-oss-dev-redis + volumes: + - .volumes/redis:/data + ports: + - 6379:6379 + networks: + - internal + db: + image: mongo:5.0.14-focal + container_name: hdx-oss-dev-db + volumes: + - .volumes/db:/data/db + ports: + - 27017:27017 + networks: + - internal + otel-collector: + container_name: hdx-oss-dev-otel-collector + build: + context: ./docker/otel-collector + target: dev + volumes: + - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - '13133:13133' # health_check extension + - '1888:1888' # pprof extension + - '24225:24225' # fluentd receiver + - '4317:4317' # OTLP gRPC receiver + - '4318:4318' # OTLP http receiver + - '55679:55679' # zpages extension + - '8888:8888' # metrics extension + - '9411:9411' # zipkin + restart: always + networks: + - internal + aggregator: + container_name: hdx-oss-dev-aggregator + build: + context: . + dockerfile: ./packages/api/Dockerfile + target: dev + ports: + - 8001:8001 + environment: + APP_TYPE: 'aggregator' + CLICKHOUSE_HOST: http://ch-server:8123 + CLICKHOUSE_PASSWORD: aggregator + CLICKHOUSE_USER: aggregator + FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS) + MONGO_URI: 'mongodb://db:27017/hyperdx' + NODE_ENV: development + PORT: 8001 + REDIS_URL: redis://redis:6379 + SERVER_URL: 'http://localhost:8000' + volumes: + - ./packages/api/src:/app/src + networks: + - internal + depends_on: + - db + - redis + - ch-server + task-check-alerts: + container_name: hdx-oss-dev-task-check-alerts + build: + context: . + dockerfile: ./packages/api/Dockerfile + target: dev + entrypoint: 'yarn' + command: 'dev:task check-alerts' + environment: + APP_TYPE: 'scheduled-task' + CLICKHOUSE_HOST: http://ch-server:8123 + CLICKHOUSE_LOG_LEVEL: trace + CLICKHOUSE_PASSWORD: worker + CLICKHOUSE_USER: worker + FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS) + HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1 + HDX_NODE_BETA_MODE: 0 + HDX_NODE_CONSOLE_CAPTURE: 1 + HYPERDX_API_KEY: ${HYPERDX_API_KEY} + INGESTOR_API_URL: 'http://ingestor:8002' + MINER_API_URL: 'http://miner:5123' + MONGO_URI: 'mongodb://db:27017/hyperdx' + NODE_ENV: development + OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318' + OTEL_SERVICE_NAME: 'hdx-oss-dev-task-check-alerts' + REDIS_URL: redis://redis:6379 + volumes: + - ./packages/api/src:/app/src + restart: always + networks: + - internal + depends_on: + - ch-server + - db + - redis + api: + container_name: hdx-oss-dev-api + build: + context: . + dockerfile: ./packages/api/Dockerfile + target: dev + ports: + - 8000:8000 + environment: + AGGREGATOR_API_URL: 'http://aggregator:8001' + APP_TYPE: 'api' + CLICKHOUSE_HOST: http://ch-server:8123 + CLICKHOUSE_LOG_LEVEL: trace + CLICKHOUSE_PASSWORD: api + CLICKHOUSE_USER: api + EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋' + FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS) + HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1 + HDX_NODE_BETA_MODE: 1 + HDX_NODE_CONSOLE_CAPTURE: 1 + HYPERDX_API_KEY: ${HYPERDX_API_KEY} + INGESTOR_API_URL: 'http://ingestor:8002' + MINER_API_URL: 'http://miner:5123' + MONGO_URI: 'mongodb://db:27017/hyperdx' + NODE_ENV: development + OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318' + OTEL_SERVICE_NAME: 'hdx-oss-dev-api' + PORT: 8000 + REDIS_URL: redis://redis:6379 + SERVER_URL: 'http://localhost:8000' + USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-false} + volumes: + - ./packages/api/src:/app/src + networks: + - internal + depends_on: + - ch-server + - db + - redis + app: + container_name: hdx-oss-dev-app + build: + context: . + dockerfile: ./packages/app/Dockerfile + target: dev + ports: + - 8080:8080 + environment: + NEXT_PUBLIC_API_SERVER_URL: 'http://localhost:8000' # need to be localhost (CORS) + NEXT_PUBLIC_HDX_API_KEY: ${HYPERDX_API_KEY} + NEXT_PUBLIC_HDX_COLLECTOR_URL: 'http://localhost:4318' + NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-dev-app' + NODE_ENV: development + PORT: 8080 + volumes: + - ./packages/app/pages:/app/pages + - ./packages/app/public:/app/public + - ./packages/app/src:/app/src + - ./packages/app/styles:/app/styles + - ./packages/app/mdx.d.ts:/app/mdx.d.ts + - ./packages/app/next-env.d.ts:/app/next-env.d.ts + - ./packages/app/next.config.js:/app/next.config.js + networks: + - internal + depends_on: + - api + ch-server: + image: clickhouse/clickhouse-server:23.7.1-alpine + container_name: hdx-oss-dev-ch-server + ports: + - 8123:8123 # http api + - 9000:9000 # native + environment: + # default settings + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + volumes: + - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml + - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml + - .volumes/ch_data:/var/lib/clickhouse + - .volumes/ch_logs:/var/log/clickhouse-server + restart: on-failure + networks: + - internal +networks: + internal: diff --git a/docker-compose.yml b/docker-compose.yml index bf15adb7..30390821 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,14 @@ version: '3' services: miner: - container_name: hdx-oss-dev-miner - build: - context: . - dockerfile: ./packages/miner/Dockerfile - target: dev + image: ${IMAGE_NAME}:${IMAGE_VERSION}-miner + container_name: hdx-oss-miner environment: HYPERDX_API_KEY: ${HYPERDX_API_KEY} HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1 - OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318 + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 OTEL_LOG_LEVEL: ERROR - OTEL_SERVICE_NAME: hdx-oss-dev-miner + OTEL_SERVICE_NAME: hdx-oss-miner volumes: - ./packages/miner/src:/app/src ports: @@ -19,10 +16,8 @@ services: networks: - internal hostmetrics: - container_name: hdx-oss-dev-hostmetrics - build: - context: ./docker/hostmetrics - target: dev + image: ${IMAGE_NAME}:${IMAGE_VERSION}-hostmetrics + container_name: hdx-oss-hostmetrics volumes: - ./docker/hostmetrics/config.dev.yaml:/etc/otelcol-contrib/config.yaml environment: @@ -32,10 +27,8 @@ services: networks: - internal ingestor: - container_name: hdx-oss-dev-ingestor - build: - context: ./docker/ingestor - target: dev + image: ${IMAGE_NAME}:${IMAGE_VERSION}-ingestor + container_name: hdx-oss-ingestor volumes: - ./docker/ingestor:/app - .volumes/ingestor_data:/var/lib/vector @@ -50,8 +43,8 @@ services: networks: - internal redis: - container_name: hdx-oss-dev-redis image: redis:7.0.11-alpine + container_name: hdx-oss-redis volumes: - .volumes/redis:/data ports: @@ -59,19 +52,17 @@ services: networks: - internal db: - container_name: hdx-oss-dev-db image: mongo:5.0.14-focal + container_name: hdx-oss-db volumes: - .volumes/db:/data/db ports: - 27017:27017 networks: - internal - otel: - container_name: hdx-oss-dev-otel - build: - context: ./docker/otel-collector - target: dev + otel-collector: + image: ${IMAGE_NAME}:${IMAGE_VERSION}-otel-collector + container_name: hdx-oss-otel-collector volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml ports: @@ -87,12 +78,8 @@ services: networks: - internal aggregator: - container_name: hdx-oss-dev-aggregator - build: - context: . - dockerfile: ./packages/api/Dockerfile - target: dev - image: hyperdx/dev/api + image: ${IMAGE_NAME}:${IMAGE_VERSION}-api + container_name: hdx-oss-aggregator ports: - 8001:8001 environment: @@ -115,12 +102,8 @@ services: - redis - ch-server task-check-alerts: - container_name: hdx-oss-dev-task-check-alerts - build: - context: . - dockerfile: ./packages/api/Dockerfile - target: dev - image: hyperdx/dev/api + image: ${IMAGE_NAME}:${IMAGE_VERSION}-api + container_name: hdx-oss-task-check-alerts entrypoint: 'yarn' command: 'dev:task check-alerts' environment: @@ -134,12 +117,12 @@ services: HDX_NODE_BETA_MODE: 0 HDX_NODE_CONSOLE_CAPTURE: 1 HYPERDX_API_KEY: ${HYPERDX_API_KEY} - HYPERDX_INGESTOR_ENDPOINT: 'http://ingestor:8002' + INGESTOR_API_URL: 'http://ingestor:8002' MINER_API_URL: 'http://miner:5123' MONGO_URI: 'mongodb://db:27017/hyperdx' NODE_ENV: development - OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel:4318' - OTEL_SERVICE_NAME: 'hdx-oss-dev-task-check-alerts' + OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318' + OTEL_SERVICE_NAME: 'hdx-oss-task-check-alerts' REDIS_URL: redis://redis:6379 volumes: - ./packages/api/src:/app/src @@ -151,15 +134,12 @@ services: - db - redis api: - container_name: hdx-oss-dev-api - build: - context: . - dockerfile: ./packages/api/Dockerfile - target: dev - image: hyperdx/dev/api + image: ${IMAGE_NAME}:${IMAGE_VERSION}-api + container_name: hdx-oss-api ports: - 8000:8000 environment: + AGGREGATOR_API_URL: 'http://aggregator:8001' APP_TYPE: 'api' CLICKHOUSE_HOST: http://ch-server:8123 CLICKHOUSE_LOG_LEVEL: trace @@ -171,15 +151,16 @@ services: HDX_NODE_BETA_MODE: 1 HDX_NODE_CONSOLE_CAPTURE: 1 HYPERDX_API_KEY: ${HYPERDX_API_KEY} - HYPERDX_INGESTOR_ENDPOINT: 'http://ingestor:8002' + INGESTOR_API_URL: 'http://ingestor:8002' MINER_API_URL: 'http://miner:5123' MONGO_URI: 'mongodb://db:27017/hyperdx' NODE_ENV: development - OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel:4318' - OTEL_SERVICE_NAME: 'hdx-oss-dev-api' + OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318' + OTEL_SERVICE_NAME: 'hdx-oss-api' PORT: 8000 REDIS_URL: redis://redis:6379 SERVER_URL: 'http://localhost:8000' + USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-true} volumes: - ./packages/api/src:/app/src networks: @@ -189,19 +170,15 @@ services: - db - redis app: - container_name: hdx-oss-dev-app - build: - context: . - dockerfile: ./packages/app/Dockerfile - target: dev - image: hyperdx/dev/app + image: ${IMAGE_NAME}:${IMAGE_VERSION}-app + container_name: hdx-oss-app ports: - 8080:8080 environment: NEXT_PUBLIC_API_SERVER_URL: 'http://localhost:8000' # need to be localhost (CORS) NEXT_PUBLIC_HDX_API_KEY: ${HYPERDX_API_KEY} NEXT_PUBLIC_HDX_COLLECTOR_URL: 'http://localhost:4318' - NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-dev-app' + NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-app' NODE_ENV: development PORT: 8080 volumes: @@ -217,8 +194,8 @@ services: depends_on: - api ch-server: - container_name: hdx-oss-dev-ch-server image: clickhouse/clickhouse-server:23.7.1-alpine + container_name: hdx-oss-ch-server ports: - 8123:8123 # http api - 9000:9000 # native diff --git a/docker/hostmetrics/config.dev.yaml b/docker/hostmetrics/config.dev.yaml index 57ca596f..4528f76e 100644 --- a/docker/hostmetrics/config.dev.yaml +++ b/docker/hostmetrics/config.dev.yaml @@ -23,7 +23,7 @@ exporters: logging: loglevel: debug otlphttp: - endpoint: 'http://otel:4318' + endpoint: 'http://otel-collector:4318' headers: authorization: ${HYPERDX_API_KEY} compression: gzip diff --git a/package.json b/package.json index de1c7c57..657074de 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "0.0.0", + "version": "1.0.3", "license": "MIT", "workspaces": [ "packages/*" @@ -13,11 +13,7 @@ "prettier": "2.8.4" }, "scripts": { - "prepare": "husky install", - "dev:lint": "./docker/ingestor/run_linting.sh && yarn workspaces run ci:lint", - "ci:lint": "./docker/ingestor/run_linting.sh && yarn workspaces run ci:lint", - "dev:int": "docker compose -p int -f ./docker-compose.ci.yml run --rm api dev:int", - "ci:int": "docker compose -p int -f ./docker-compose.ci.yml run --rm api ci:int" + "prepare": "husky install" }, "lint-staged": { "**/*": "prettier --write --ignore-unknown" diff --git a/packages/api/package.json b/packages/api/package.json index 895a8334..db26dd41 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/api", - "version": "1.0.0", + "version": "1.0.3", "license": "MIT", "private": true, "engines": { @@ -9,6 +9,7 @@ "dependencies": { "@clickhouse/client": "^0.1.1", "@hyperdx/lucene": "^3.1.1", + "@hyperdx/node-logger": "^0.2.7", "@hyperdx/node-opentelemetry": "^0.2.2", "@slack/webhook": "^6.1.0", "compression": "^1.7.4", diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index e263f36f..1b4d4b45 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -1,13 +1,15 @@ import MongoStore from 'connect-mongo'; import compression from 'compression'; import express from 'express'; -import session from 'express-session'; +import ms from 'ms'; import onHeaders from 'on-headers'; +import session from 'express-session'; import * as config from './config'; import defaultCors from './middleware/cors'; import passport from './utils/passport'; import routers from './routers/api'; +import usageStats from './tasks/usageStats'; import { appErrorHandler } from './middleware/error'; import { expressLogger } from './utils/logger'; @@ -55,6 +57,17 @@ app.use(function (req, res, next) { }); app.use(defaultCors); +// --------------------------------------------------------------------- +// ----------------------- Background Jobs ----------------------------- +// --------------------------------------------------------------------- +if (config.USAGE_STATS_ENABLED) { + void usageStats(); + setInterval(() => { + void usageStats(); + }, ms('4h')); +} +// --------------------------------------------------------------------- + // --------------------------------------------------------------------- // ----------------------- Internal Routers ---------------------------- // --------------------------------------------------------------------- diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 1ebb7a41..2423e59e 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -1,35 +1,28 @@ import { version } from '../package.json'; const env = process.env; - -export const CODE_VERSION = version; - -export const APP_TYPE = env.APP_TYPE as 'api' | 'aggregator' | 'scheduled-task'; - export const NODE_ENV = env.NODE_ENV as string; -export const IS_PROD = NODE_ENV === 'production'; -export const IS_DEV = NODE_ENV === 'development'; -export const IS_CI = NODE_ENV === 'ci'; - -export const PORT = Number.parseInt(env.PORT as string); -export const SERVER_URL = env.SERVER_URL as string; -export const FRONTEND_URL = env.FRONTEND_URL as string; -export const COOKIE_DOMAIN = env.COOKIE_DOMAIN as string; // prod ONLY - -export const MONGO_URI = env.MONGO_URI as string; +export const AGGREGATOR_API_URL = env.AGGREGATOR_API_URL as string; +export const APP_TYPE = env.APP_TYPE as 'api' | 'aggregator' | 'scheduled-task'; export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; export const CLICKHOUSE_PASSWORD = env.CLICKHOUSE_PASSWORD as string; export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string; - -export const HYPERDX_API_KEY = env.HYPERDX_API_KEY as string; -export const HYPERDX_INGESTOR_ENDPOINT = - env.HYPERDX_INGESTOR_ENDPOINT as string; - +export const CODE_VERSION = version; +export const COOKIE_DOMAIN = env.COOKIE_DOMAIN as string; // prod ONLY export const EXPRESS_SESSION_SECRET = env.EXPRESS_SESSION_SECRET as string; - -export const REDIS_URL = env.REDIS_URL as string; - +export const FRONTEND_URL = env.FRONTEND_URL as string; +export const HYPERDX_API_KEY = env.HYPERDX_API_KEY as string; +export const INGESTOR_API_URL = env.INGESTOR_API_URL as string; +export const IS_CI = NODE_ENV === 'ci'; +export const IS_DEV = NODE_ENV === 'development'; +export const IS_PROD = NODE_ENV === 'production'; export const MINER_API_URL = env.MINER_API_URL as string; - +export const MONGO_URI = env.MONGO_URI as string; +export const OTEL_EXPORTER_OTLP_ENDPOINT = + env.OTEL_EXPORTER_OTLP_ENDPOINT as string; export const OTEL_SERVICE_NAME = env.OTEL_SERVICE_NAME as string; +export const PORT = Number.parseInt(env.PORT as string); +export const REDIS_URL = env.REDIS_URL as string; +export const SERVER_URL = env.SERVER_URL as string; +export const USAGE_STATS_ENABLED = env.USAGE_STATS_ENABLED !== 'false'; diff --git a/packages/api/src/tasks/usageStats.ts b/packages/api/src/tasks/usageStats.ts new file mode 100644 index 00000000..3a43210e --- /dev/null +++ b/packages/api/src/tasks/usageStats.ts @@ -0,0 +1,147 @@ +import os from 'os'; +import url from 'url'; + +import winston from 'winston'; +import { HyperDXWinston } from '@hyperdx/node-logger'; + +import * as clickhouse from '../clickhouse'; +import * as config from '../config'; +import Team from '../models/team'; +import User from '../models/user'; + +import type { ResponseJSON } from '@clickhouse/client'; + +const hyperdxTransport = new HyperDXWinston({ + apiKey: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key + baseUrl: 'https://in.hyperdx.io', + maxLevel: 'info', + service: 'hyperdx-oss-usage-stats', +}); + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.json(), + transports: [hyperdxTransport], +}); + +const getClickhouseTableSize = async () => { + const rows = await clickhouse.client.query({ + query: ` + SELECT + table, + sum(bytes) AS size, + sum(rows) AS rows, + min(min_time) AS min_time, + max(max_time) AS max_time, + max(modification_time) AS latestModification, + toUInt32((max_time - min_time) / 86400) AS days, + size / ((max_time - min_time) / 86400) AS avgDaySize + FROM system.parts + WHERE active + AND database = 'default' + AND (table = {table1: String} OR table = {table2: String} OR table = {table3: String}) + GROUP BY table + ORDER BY rows DESC + `, + format: 'JSON', + query_params: { + table1: clickhouse.TableName.LogStream, + table2: clickhouse.TableName.Rrweb, + table3: clickhouse.TableName.Metric, + }, + }); + const result = await rows.json>(); + return result.data; +}; + +const healthChecks = async () => { + const ping = async (url: string) => { + try { + const res = await fetch(url); + return res.status === 200; + } catch (err) { + return false; + } + }; + + const ingestorUrl = url.parse(config.INGESTOR_API_URL ?? ''); + const otelCollectorUrl = url.parse(config.OTEL_EXPORTER_OTLP_ENDPOINT ?? ''); + const aggregatorUrl = url.parse(config.AGGREGATOR_API_URL ?? ''); + + const [pingIngestor, pingOtelCollector, pingAggregator, pingMiner, pingCH] = + await Promise.all([ + ingestorUrl.hostname && ingestorUrl.protocol + ? ping(`${ingestorUrl.protocol}//${ingestorUrl.hostname}:8686/health`) + : Promise.resolve(null), + otelCollectorUrl.hostname && otelCollectorUrl.protocol + ? ping( + `${otelCollectorUrl.protocol}//${otelCollectorUrl.hostname}:13133`, + ) + : Promise.resolve(null), + aggregatorUrl.href + ? ping(`${aggregatorUrl.href}health`) + : Promise.resolve(null), + ping(`${config.MINER_API_URL}/health`), + ping(`${config.CLICKHOUSE_HOST}/ping`), + ]); + + return { + pingIngestor, + pingOtelCollector, + pingAggregator, + pingMiner, + pingCH, + }; +}; + +export default async () => { + try { + const nowInMs = Date.now(); + const [userCounts, team, chTables, servicesHealth] = await Promise.all([ + User.countDocuments(), + Team.find( + {}, + { + _id: 1, + }, + ).limit(1), + getClickhouseTableSize(), + healthChecks(), + ]); + const clusterId = team[0]?._id; + logger.info({ + message: 'track-hyperdx-oss-usage-stats', + clusterId, + version: config.CODE_VERSION, + userCounts, + servicesHealth, + os: { + arch: os.arch(), + freemem: os.freemem(), + uptime: os.uptime(), + }, + chStats: { + tables: chTables.reduce( + (acc, curr) => ({ + ...acc, + [curr.table]: { + avgDaySize: parseInt(curr.avgDaySize), + days: parseInt(curr.days), + lastModified: new Date(curr.latestModification).getTime(), + maxTime: new Date(curr.max_time).getTime(), + minTime: new Date(curr.min_time).getTime(), + rows: parseInt(curr.rows), + size: parseInt(curr.size), + }, + }), + {}, + ), + rows: chTables.reduce((acc, curr) => acc + parseInt(curr.rows), 0), + size: chTables.reduce((acc, curr) => acc + parseInt(curr.size), 0), + }, + timestamp: nowInMs, + }); + } catch (err) { + // ignore + } +}; diff --git a/packages/api/src/utils/logger.ts b/packages/api/src/utils/logger.ts index 8b15153c..8f7cb931 100644 --- a/packages/api/src/utils/logger.ts +++ b/packages/api/src/utils/logger.ts @@ -6,7 +6,7 @@ import { getWinsonTransport } from '@hyperdx/node-opentelemetry'; import { APP_TYPE, HYPERDX_API_KEY, - HYPERDX_INGESTOR_ENDPOINT, + INGESTOR_API_URL, IS_DEV, IS_PROD, } from '../config'; @@ -25,24 +25,15 @@ addColors({ }); const MAX_LEVEL = IS_PROD ? 'debug' : 'debug'; -const DEFAULT_FORMAT = IS_DEV - ? winston.format.combine( - winston.format.errors({ stack: true }), - winston.format.colorize({ all: true }), - winston.format.timestamp({ format: 'MM/DD/YY HH:MM:SS' }), - winston.format.printf( - info => `[${info.level}] ${info.timestamp} ${info.message}`, - ), - ) - : winston.format.combine( - winston.format.errors({ stack: true }), - winston.format.json(), - ); +const DEFAULT_FORMAT = winston.format.combine( + winston.format.errors({ stack: true }), + winston.format.json(), +); const hyperdxTransport = HYPERDX_API_KEY ? getWinsonTransport(MAX_LEVEL, { bufferSize: APP_TYPE === 'scheduled-task' ? 1 : 100, - ...(HYPERDX_INGESTOR_ENDPOINT && { baseUrl: HYPERDX_INGESTOR_ENDPOINT }), + ...(INGESTOR_API_URL && { baseUrl: INGESTOR_API_URL }), }) : null; diff --git a/packages/app/Dockerfile b/packages/app/Dockerfile index d78bc662..10fc05d7 100644 --- a/packages/app/Dockerfile +++ b/packages/app/Dockerfile @@ -31,7 +31,6 @@ COPY ./packages/app/src ./src COPY ./packages/app/pages ./pages COPY ./packages/app/public ./public COPY ./packages/app/styles ./styles -COPY ./packages/app/json ./json COPY --from=base /app/node_modules ./node_modules RUN yarn build && yarn install --production --ignore-scripts --prefer-offline @@ -48,7 +47,6 @@ RUN adduser -S nextjs -u 1001 # You only need to copy next.config.js if you are NOT using the default configuration # COPY --from=builder /app/next.config.js ./ COPY --from=builder /app/public ./public -COPY --from=builder /app/json ./json COPY --from=builder --chown=nextjs:nodejs /app/.next ./.next COPY --from=builder /app/node_modules ./node_modules COPY --from=builder /app/package.json ./package.json diff --git a/packages/app/package.json b/packages/app/package.json index 8310ec13..9e6ca6fe 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/app", - "version": "1.0.0", + "version": "1.0.3", "private": true, "license": "MIT", "engines": { diff --git a/packages/app/pages/_app.tsx b/packages/app/pages/_app.tsx index 32817c6e..7a41cf57 100644 --- a/packages/app/pages/_app.tsx +++ b/packages/app/pages/_app.tsx @@ -1,5 +1,5 @@ import Head from 'next/head'; -import React from 'react'; +import React, { useEffect } from 'react'; import SSRProvider from 'react-bootstrap/SSRProvider'; import type { AppProps } from 'next/app'; import { QueryClient, QueryClientProvider } from 'react-query'; @@ -22,23 +22,28 @@ const queryClient = new QueryClient(); import HyperDX from '@hyperdx/browser'; -if (config.HDX_API_KEY != null) { - HyperDX.init({ - ...(config.HDX_COLLECTOR_URL != null - ? { - url: config.HDX_COLLECTOR_URL, - } - : {}), - apiKey: config.HDX_API_KEY, - consoleCapture: true, - maskAllInputs: true, - maskAllText: true, - service: config.HDX_SERVICE_NAME, - tracePropagationTargets: [/localhost/i, /hyperdx\.io/i], - }); -} - export default function MyApp({ Component, pageProps }: AppProps) { + // port to react query ? (needs to wrap with QueryClientProvider) + useEffect(() => { + fetch('/api/config') + .then(res => res.json()) + .then(_jsonData => { + if (_jsonData?.apiKey) { + HyperDX.init({ + apiKey: _jsonData.apiKey, + consoleCapture: true, + maskAllInputs: true, + maskAllText: true, + service: _jsonData.serviceName, + tracePropagationTargets: [/localhost/i, /hyperdx\.io/i], + url: _jsonData.collectorUrl, + }); + } + }) + .catch(err => { + // ignore + }); + }); return ( diff --git a/packages/app/pages/api/config.ts b/packages/app/pages/api/config.ts new file mode 100644 index 00000000..7c88fa2e --- /dev/null +++ b/packages/app/pages/api/config.ts @@ -0,0 +1,24 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; + +import { + HDX_API_KEY, + HDX_COLLECTOR_URL, + HDX_SERVICE_NAME, +} from '../../src/config'; + +type ResponseData = { + apiKey: string; + collectorUrl: string; + serviceName: string; +}; + +export default function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + res.status(200).json({ + apiKey: HDX_API_KEY, + collectorUrl: HDX_COLLECTOR_URL, + serviceName: HDX_SERVICE_NAME, + }); +} diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index 05d144b0..8aef38df 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -1,10 +1,11 @@ export const API_SERVER_URL = - process.env.NEXT_PUBLIC_API_SERVER_URL ?? 'http://localhost:8000'; + process.env.NEXT_PUBLIC_API_SERVER_URL || 'http://localhost:8000'; // NEXT_PUBLIC_API_SERVER_URL can be empty string -export const HDX_API_KEY = process.env.NEXT_PUBLIC_HDX_API_KEY as string; +export const HDX_API_KEY = (process.env.NEXT_PUBLIC_HDX_API_KEY || + process.env.HYPERDX_API_KEY) as string; export const HDX_SERVICE_NAME = - process.env.NEXT_PUBLIC_HDX_SERVICE_NAME ?? 'hdx-oss-dev-app'; -export const HDX_COLLECTOR_URL = process.env - .NEXT_PUBLIC_HDX_COLLECTOR_URL as string; + process.env.NEXT_PUBLIC_HDX_SERVICE_NAME || 'hdx-oss-dev-app'; +export const HDX_COLLECTOR_URL = + process.env.NEXT_PUBLIC_HDX_COLLECTOR_URL || 'http://localhost:4318'; export const IS_OSS = process.env.NEXT_PUBLIC_IS_OSS ?? 'true' === 'true'; diff --git a/yarn.lock b/yarn.lock index ccb61e8d..8ee690da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4429,13 +4429,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.0.tgz#4668bc392bb6938637b47e98b1f2ed5426f33316" integrity sha512-BsAaKhB+7X+H4GnSjGhJG9Qi8Tw+inU9nJDwmD5CgOmBLEI6ArdhikpLX7DjbjDRDTbqZzU2LSQNZg8WGPiSZQ== -"@types/oauth@*": - version "0.9.1" - resolved "https://registry.yarnpkg.com/@types/oauth/-/oauth-0.9.1.tgz#e17221e7f7936b0459ae7d006255dff61adca305" - integrity sha512-a1iY62/a3yhZ7qH7cNUsxoI3U/0Fe9+RnuFrpTKr+0WVOzbKlSLojShCKe20aOD1Sppv+i8Zlq0pLDuTJnwS4A== - dependencies: - "@types/node" "*" - "@types/object-hash@^2.2.1": version "2.2.1" resolved "https://registry.yarnpkg.com/@types/object-hash/-/object-hash-2.2.1.tgz#67c169f8f033e0b62abbf81df2d00f4598d540b9" @@ -4451,15 +4444,6 @@ resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.3.tgz#705bb349e789efa06f43f128cef51240753424cb" integrity sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g== -"@types/passport-google-oauth20@^2.0.11": - version "2.0.11" - resolved "https://registry.yarnpkg.com/@types/passport-google-oauth20/-/passport-google-oauth20-2.0.11.tgz#271ec71de3030a3e1c004b24e633e4b298ccba97" - integrity sha512-9XMT1GfwhZL7UQEiCepLef55RNPHkbrCtsU7rsWPTEOsmu5qVIW8nSemtB4p+P24CuOhA+IKkv8LsPThYghGww== - dependencies: - "@types/express" "*" - "@types/passport" "*" - "@types/passport-oauth2" "*" - "@types/passport-http-bearer@^1.0.37": version "1.0.37" resolved "https://registry.yarnpkg.com/@types/passport-http-bearer/-/passport-http-bearer-1.0.37.tgz#6882825a46717725f952731d17e1bb0a698155a4" @@ -4478,15 +4462,6 @@ "@types/passport" "*" "@types/passport-strategy" "*" -"@types/passport-oauth2@*": - version "1.4.12" - resolved "https://registry.yarnpkg.com/@types/passport-oauth2/-/passport-oauth2-1.4.12.tgz#c2ae0ee3b16646188d8b0b6cdbc6880a0247dc5f" - integrity sha512-RZg6cYTyEGinrZn/7REYQds6zrTxoBorX1/fdaz5UHzkG8xdFE7QQxkJagCr2ETzGII58FAFDmnmbTUVMrltNA== - dependencies: - "@types/express" "*" - "@types/oauth" "*" - "@types/passport" "*" - "@types/passport-strategy@*": version "0.2.35" resolved "https://registry.yarnpkg.com/@types/passport-strategy/-/passport-strategy-0.2.35.tgz#e52f5212279ea73f02d9b06af67efe9cefce2d0c"