diff --git a/.changeset/honest-taxis-deny.md b/.changeset/honest-taxis-deny.md new file mode 100644 index 00000000..289007d1 --- /dev/null +++ b/.changeset/honest-taxis-deny.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +bring usage stats up to date diff --git a/.env b/.env index af590e39..bf9d30fb 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ ALL_IN_ONE_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-all-in-one ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-all-in-one OTEL_COLLECTOR_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-otel-collector OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-otel-collector -CHANGESET_TAG=2.0.0-beta.16 +CODE_VERSION=2.0.0-beta.16 IMAGE_VERSION_SUB_TAG=.16 IMAGE_VERSION=2-beta IMAGE_NIGHTLY_TAG=2-nightly diff --git a/Makefile b/Makefile index 6e1c4b16..41b6dc3f 100644 --- a/Makefile +++ b/Makefile @@ -87,6 +87,7 @@ build-local: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ --target all-in-one-noauth @@ -98,6 +99,7 @@ build-all-in-one: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ -t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ --target all-in-one-auth @@ -107,6 +109,7 @@ build-app: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ -t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ --target prod @@ -122,6 +125,7 @@ build-app-nightly: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ -t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ --target prod @@ -133,6 +137,7 @@ build-local-nightly: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ --target all-in-one-noauth @@ -144,6 +149,7 @@ build-all-in-one-nightly: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ -t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ --target all-in-one-auth @@ -169,6 +175,7 @@ release-local: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ --platform ${BUILD_PLATFORMS} \ -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \ -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ @@ -187,6 +194,7 @@ release-all-in-one: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ --platform ${BUILD_PLATFORMS} \ -t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \ -t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ @@ -203,6 +211,7 @@ release-app: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ --platform ${BUILD_PLATFORMS} \ -t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ -t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \ @@ -228,6 +237,7 @@ release-app-nightly: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ --platform ${BUILD_PLATFORMS} \ -t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ --target prod \ @@ -243,6 +253,7 @@ release-local-nightly: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ --platform ${BUILD_PLATFORMS} \ -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ --target all-in-one-noauth \ @@ -258,6 +269,7 @@ release-all-in-one-nightly: --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ + --build-arg CODE_VERSION=${CODE_VERSION} \ --platform ${BUILD_PLATFORMS} \ -t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ --target all-in-one-auth \ diff --git a/README.md b/README.md index 96079390..1fd4f30c 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,15 @@ Here's a high-level list of support we're working on delivering as part of v2: - [ ] v1 Migration Tooling - [ ] Public API +## HyperDX Usage Data + +HyperDX collects anonymized usage data for open source deployments. This data +supports our mission for observability to be available to any team and helps +support our open source product run in a variety of different environments. +While we hope you will continue to support our mission in this way, you may opt +out of usage data collection by setting the `USAGE_STATS_ENABLED` environment +variable to `false`. Thank you for supporting the development of HyperDX! + ## License [MIT](/LICENSE) diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 239e5069..29de32de 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -56,6 +56,9 @@ RUN rm -rf node_modules && yarn workspaces focus @hyperdx/api @hyperdx/app --pro # prod ############################################################################################ FROM node:${NODE_VERSION}-alpine AS prod +ARG CODE_VERSION + +ENV CODE_VERSION=$CODE_VERSION ENV NODE_ENV production # Install libs used for the start script @@ -86,6 +89,9 @@ ENTRYPOINT ["sh", "/etc/local/entry.sh"] # all-in-one base ############################################################################################ FROM scratch AS all-in-one-base +ARG CODE_VERSION + +ENV CODE_VERSION=$CODE_VERSION # Copy from clickhouse and otel collector bases COPY --from=clickhouse_base / / COPY --from=otel_collector_base --chmod=755 /otelcol-contrib /otelcontribcol diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index 18bb523a..cf4d7095 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -2,7 +2,6 @@ import compression from 'compression'; import MongoStore from 'connect-mongo'; import express from 'express'; import session from 'express-session'; -import ms from 'ms'; import onHeaders from 'on-headers'; import * as config from './config'; @@ -72,10 +71,7 @@ app.use(defaultCors); // ----------------------- Background Jobs ----------------------------- // --------------------------------------------------------------------- if (config.USAGE_STATS_ENABLED) { - void usageStats(); - setInterval(() => { - void usageStats(); - }, ms('4h')); + usageStats(); } // --------------------------------------------------------------------- diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 657383f2..94896b46 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -10,7 +10,7 @@ export const NODE_ENV = env.NODE_ENV as string; export const APP_TYPE = (env.APP_TYPE || DEFAULT_APP_TYPE) as | 'api' | 'scheduled-task'; -export const CODE_VERSION = env.CODE_VERSION as string; +export const CODE_VERSION = env.CODE_VERSION ?? ''; export const EXPRESS_SESSION_SECRET = (env.EXPRESS_SESSION_SECRET || DEFAULT_EXPRESS_SESSION) as string; export const FRONTEND_URL = (env.FRONTEND_URL || diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 8384e653..078e63b3 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -12,6 +12,8 @@ export interface ISource extends Omit { connection: ObjectId | string; } +export type SourceDocument = mongoose.HydratedDocument; + export const Source = mongoose.model( 'Source', new Schema( diff --git a/packages/api/src/routers/api/clickhouseProxy.ts b/packages/api/src/routers/api/clickhouseProxy.ts index ece7d7ae..efa95379 100644 --- a/packages/api/src/routers/api/clickhouseProxy.ts +++ b/packages/api/src/routers/api/clickhouseProxy.ts @@ -3,6 +3,7 @@ import { createProxyMiddleware } from 'http-proxy-middleware'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; +import { CODE_VERSION } from '@/config'; import { getConnectionById } from '@/controllers/connection'; import { getNonNullUserWithTeam } from '@/middleware/auth'; import { validateRequestHeaders } from '@/middleware/validation'; @@ -120,7 +121,9 @@ const proxyMiddleware: RequestHandler = }, on: { proxyReq: (proxyReq, _req) => { - const newPath = _req.params[0]; + // set user-agent to the hyperdx version identifier + proxyReq.setHeader('user-agent', `hyperdx ${CODE_VERSION}`); + // @ts-expect-error _req.query is type ParamQs, which doesn't play nicely with URLSearchParams. TODO: Replace with getting query params from _req.url eventually const qparams = new URLSearchParams(_req.query); @@ -139,6 +142,7 @@ const proxyMiddleware: RequestHandler = // TODO: Use fixRequestBody after this issue is resolved: https://github.com/chimurai/http-proxy-middleware/issues/1102 proxyReq.write(_req.body); } + const newPath = _req.params[0]; proxyReq.path = `/${newPath}?${qparams}`; }, proxyRes: (proxyRes, _req, res) => { diff --git a/packages/api/src/routers/api/me.ts b/packages/api/src/routers/api/me.ts index 698dd198..ccff590b 100644 --- a/packages/api/src/routers/api/me.ts +++ b/packages/api/src/routers/api/me.ts @@ -1,5 +1,6 @@ import express from 'express'; +import { USAGE_STATS_ENABLED } from '@/config'; import { getTeam } from '@/controllers/team'; import { Api404Error } from '@/utils/errors'; @@ -29,6 +30,7 @@ router.get('/', async (req, res, next) => { id, name, team, + usageStatsEnabled: USAGE_STATS_ENABLED, }); } catch (e) { next(e); diff --git a/packages/api/src/tasks/usageStats.ts b/packages/api/src/tasks/usageStats.ts index 08144bc9..a91f693b 100644 --- a/packages/api/src/tasks/usageStats.ts +++ b/packages/api/src/tasks/usageStats.ts @@ -1,10 +1,14 @@ import type { ResponseJSON } from '@clickhouse/client'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse'; +import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types'; import * as HyperDX from '@hyperdx/node-opentelemetry'; +import ms from 'ms'; import os from 'os'; import winston from 'winston'; -import * as clickhouse from '@/clickhouse'; import * as config from '@/config'; +import Connection from '@/models/connection'; +import { Source, SourceDocument } from '@/models/source'; import Team from '@/models/team'; import User from '@/models/user'; @@ -13,45 +17,104 @@ const logger = winston.createLogger({ format: winston.format.json(), transports: [ HyperDX.getWinstonTransport('info', { - apiKey: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key + headers: { + Authorization: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key + }, baseUrl: 'https://in-otel.hyperdx.io/v1/logs', maxLevel: 'info', service: 'hyperdx-oss-usage-stats', - } as any), + }), ], }); +function extractTableNames(source: SourceDocument): string[] { + const tables: string[] = []; + if (source.kind === SourceKind.Metric) { + for (const key of Object.values(MetricsDataType)) { + const metricTable = source.metricTables?.[key]; + if (!metricTable) continue; + tables.push(metricTable); + } + } else { + tables.push(source.from.tableName); + } + return tables; +} + const getClickhouseTableSize = async () => { - const rows = await clickhouse.client.query({ - query: ` - SELECT - table, - sum(bytes) AS size, - sum(rows) AS rows, - min(min_time) AS min_time, - max(max_time) AS max_time, - max(modification_time) AS latestModification, - toUInt32((max_time - min_time) / 86400) AS days, - size / ((max_time - min_time) / 86400) AS avgDaySize - FROM system.parts - WHERE active - AND database = 'default' - AND (table = {table1: String} OR table = {table2: String} OR table = {table3: String}) - GROUP BY table - ORDER BY rows DESC - `, - format: 'JSON', - query_params: { - table1: clickhouse.TableName.LogStream, - table2: clickhouse.TableName.Rrweb, - table3: clickhouse.TableName.Metric, - }, - }); - const result = await rows.json>(); - return result.data; + // fetch mongo data + const connections = await Connection.find(); + const sources = await Source.find(); + + // build map for each db instance + const distributedTableMap = new Map(); + for (const source of sources) { + const key = `${source.connection.toString()},${source.from.databaseName}`; + if (distributedTableMap.has(key)) { + distributedTableMap.get(key)?.push(...extractTableNames(source)); + } else { + distributedTableMap.set(key, extractTableNames(source)); + } + } + + // fetch usage data + const results: any[] = []; + for (const [key, tables] of distributedTableMap) { + const [connectionId, dbName] = key.split(','); + const tableListString = tables + .map((_, idx) => `table = {table${idx}: String}`) + .join(' OR '); + const query_params = tables.reduce( + (acc, table, idx) => { + acc[`table${idx}`] = table; + return acc; + }, + {} as { [key: string]: string }, + ); + query_params.dbName = dbName; + + // find connection + const connection = connections.find(c => c.id === connectionId); + if (!connection) continue; + + // query clickhouse + try { + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + const _rows = await clickhouseClient.query({ + query: ` + SELECT + table, + sum(bytes) AS size, + sum(rows) AS rows, + min(min_time) AS min_time, + max(max_time) AS max_time, + max(modification_time) AS latestModification, + toUInt32((max_time - min_time) / 86400) AS days, + size / ((max_time - min_time) / 86400) AS avgDaySize + FROM system.parts + WHERE active + AND database = {dbName: String} + AND (${tableListString}) + GROUP BY table + ORDER BY rows DESC + `, + format: 'JSON', + query_params, + }); + const res = await _rows.json>(); + results.push(...res.data); + } catch (error) { + // ignore + } + } + return results; }; -export default async () => { +async function getUsageStats() { try { const nowInMs = Date.now(); const [userCounts, team, chTables] = await Promise.all([ @@ -99,4 +162,11 @@ export default async () => { } catch (err) { // ignore } -}; +} + +export default function () { + void getUsageStats(); + setInterval(() => { + void getUsageStats(); + }, ms('4h')); +} diff --git a/packages/app/src/AppNav.tsx b/packages/app/src/AppNav.tsx index 37624d55..855c7bd3 100644 --- a/packages/app/src/AppNav.tsx +++ b/packages/app/src/AppNav.tsx @@ -860,6 +860,12 @@ export default function AppNav({ fixed = false }: { fixed?: boolean }) { onClickUserPreferences={openUserPreferences} logoutUrl={IS_LOCAL_MODE ? null : `/api/logout`} /> + {meData && meData.usageStatsEnabled && ( + + )}