usage info (#846)

Ref: HDX-1692

.changeset/honest-taxis-deny.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+bring usage stats up to date

.env

@@ -8,7 +8,7 @@ ALL_IN_ONE_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-all-in-one
 ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-all-in-one
 OTEL_COLLECTOR_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-otel-collector
 OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-otel-collector
-CHANGESET_TAG=2.0.0-beta.16
+CODE_VERSION=2.0.0-beta.16
 IMAGE_VERSION_SUB_TAG=.16
 IMAGE_VERSION=2-beta
 IMAGE_NIGHTLY_TAG=2-nightly

Makefile

@@ -87,6 +87,7 @@ build-local:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
 		--target all-in-one-noauth
 
@@ -98,6 +99,7 @@ build-all-in-one:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
 		--target all-in-one-auth
 
@@ -107,6 +109,7 @@ build-app:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
 		--target prod
 
@@ -122,6 +125,7 @@ build-app-nightly:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
 		--target prod
 
@@ -133,6 +137,7 @@ build-local-nightly:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
 		--target all-in-one-noauth
 
@@ -144,6 +149,7 @@ build-all-in-one-nightly:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
 		--target all-in-one-auth
 
@@ -169,6 +175,7 @@ release-local:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		--platform ${BUILD_PLATFORMS} \
 		-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \
 		-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
@@ -187,6 +194,7 @@ release-all-in-one:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		--platform ${BUILD_PLATFORMS} \
 		-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \
 		-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
@@ -203,6 +211,7 @@ release-app:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		--platform ${BUILD_PLATFORMS} \
 		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
 		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \
@@ -228,6 +237,7 @@ release-app-nightly:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		--platform ${BUILD_PLATFORMS} \
 		-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
 		--target prod \
@@ -243,6 +253,7 @@ release-local-nightly:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		--platform ${BUILD_PLATFORMS} \
 		-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
 		--target all-in-one-noauth \
@@ -258,6 +269,7 @@ release-all-in-one-nightly:
 		--build-context hyperdx=./docker/hyperdx \
 		--build-context api=./packages/api \
 		--build-context app=./packages/app \
+		--build-arg CODE_VERSION=${CODE_VERSION} \
 		--platform ${BUILD_PLATFORMS} \
 		-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
 		--target all-in-one-auth \

README.md

@@ -205,6 +205,15 @@ Here's a high-level list of support we're working on delivering as part of v2:
 - [ ] v1 Migration Tooling
 - [ ] Public API
 
+## HyperDX Usage Data
+
+HyperDX collects anonymized usage data for open source deployments. This data
+supports our mission for observability to be available to any team and helps
+support our open source product run in a variety of different environments.
+While we hope you will continue to support our mission in this way, you may opt
+out of usage data collection by setting the `USAGE_STATS_ENABLED` environment
+variable to `false`. Thank you for supporting the development of HyperDX!
+
 ## License
 
 [MIT](/LICENSE)

docker/hyperdx/Dockerfile

@@ -56,6 +56,9 @@ RUN rm -rf node_modules && yarn workspaces focus @hyperdx/api @hyperdx/app --pro
 # prod ############################################################################################
 FROM node:${NODE_VERSION}-alpine AS prod
 
+ARG CODE_VERSION
+
+ENV CODE_VERSION=$CODE_VERSION
 ENV NODE_ENV production
 
 # Install libs used for the start script
@@ -86,6 +89,9 @@ ENTRYPOINT ["sh", "/etc/local/entry.sh"]
 # all-in-one base ############################################################################################
 FROM scratch AS all-in-one-base
 
+ARG CODE_VERSION
+
+ENV CODE_VERSION=$CODE_VERSION
 # Copy from clickhouse and otel collector bases
 COPY --from=clickhouse_base / /
 COPY --from=otel_collector_base --chmod=755 /otelcol-contrib /otelcontribcol

packages/api/src/api-app.ts

@@ -2,7 +2,6 @@ import compression from 'compression';
 import MongoStore from 'connect-mongo';
 import express from 'express';
 import session from 'express-session';
-import ms from 'ms';
 import onHeaders from 'on-headers';
 
 import * as config from './config';
@@ -72,10 +71,7 @@ app.use(defaultCors);
 // ----------------------- Background Jobs -----------------------------
 // ---------------------------------------------------------------------
 if (config.USAGE_STATS_ENABLED) {
-  void usageStats();
-  setInterval(() => {
-    void usageStats();
-  }, ms('4h'));
+  usageStats();
 }
 // ---------------------------------------------------------------------
 

packages/api/src/config.ts

@@ -10,7 +10,7 @@ export const NODE_ENV = env.NODE_ENV as string;
 export const APP_TYPE = (env.APP_TYPE || DEFAULT_APP_TYPE) as
   | 'api'
   | 'scheduled-task';
-export const CODE_VERSION = env.CODE_VERSION as string;
+export const CODE_VERSION = env.CODE_VERSION ?? '';
 export const EXPRESS_SESSION_SECRET = (env.EXPRESS_SESSION_SECRET ||
   DEFAULT_EXPRESS_SESSION) as string;
 export const FRONTEND_URL = (env.FRONTEND_URL ||

packages/api/src/models/source.ts

@@ -12,6 +12,8 @@ export interface ISource extends Omit<TSource, 'connection'> {
   connection: ObjectId | string;
 }
 
+export type SourceDocument = mongoose.HydratedDocument<ISource>;
+
 export const Source = mongoose.model<ISource>(
   'Source',
   new Schema<ISource>(

packages/api/src/routers/api/clickhouseProxy.ts

@@ -3,6 +3,7 @@ import { createProxyMiddleware } from 'http-proxy-middleware';
 import { z } from 'zod';
 import { validateRequest } from 'zod-express-middleware';
 
+import { CODE_VERSION } from '@/config';
 import { getConnectionById } from '@/controllers/connection';
 import { getNonNullUserWithTeam } from '@/middleware/auth';
 import { validateRequestHeaders } from '@/middleware/validation';
@@ -120,7 +121,9 @@ const proxyMiddleware: RequestHandler =
     },
     on: {
       proxyReq: (proxyReq, _req) => {
-        const newPath = _req.params[0];
+        // set user-agent to the hyperdx version identifier
+        proxyReq.setHeader('user-agent', `hyperdx ${CODE_VERSION}`);
+
         // @ts-expect-error _req.query is type ParamQs, which doesn't play nicely with URLSearchParams. TODO: Replace with getting query params from _req.url eventually
         const qparams = new URLSearchParams(_req.query);
 
@@ -139,6 +142,7 @@ const proxyMiddleware: RequestHandler =
           // TODO: Use fixRequestBody after this issue is resolved: https://github.com/chimurai/http-proxy-middleware/issues/1102
           proxyReq.write(_req.body);
         }
+        const newPath = _req.params[0];
         proxyReq.path = `/${newPath}?${qparams}`;
       },
       proxyRes: (proxyRes, _req, res) => {

packages/api/src/routers/api/me.ts

@@ -1,5 +1,6 @@
 import express from 'express';
 
+import { USAGE_STATS_ENABLED } from '@/config';
 import { getTeam } from '@/controllers/team';
 import { Api404Error } from '@/utils/errors';
 
@@ -29,6 +30,7 @@ router.get('/', async (req, res, next) => {
       id,
       name,
       team,
+      usageStatsEnabled: USAGE_STATS_ENABLED,
     });
   } catch (e) {
     next(e);

packages/api/src/tasks/usageStats.ts

@@ -1,10 +1,14 @@
 import type { ResponseJSON } from '@clickhouse/client';
+import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse';
+import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types';
 import * as HyperDX from '@hyperdx/node-opentelemetry';
+import ms from 'ms';
 import os from 'os';
 import winston from 'winston';
 
-import * as clickhouse from '@/clickhouse';
 import * as config from '@/config';
+import Connection from '@/models/connection';
+import { Source, SourceDocument } from '@/models/source';
 import Team from '@/models/team';
 import User from '@/models/user';
 
@@ -13,45 +17,104 @@ const logger = winston.createLogger({
   format: winston.format.json(),
   transports: [
     HyperDX.getWinstonTransport('info', {
-      apiKey: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key
+      headers: {
+        Authorization: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key
+      },
       baseUrl: 'https://in-otel.hyperdx.io/v1/logs',
       maxLevel: 'info',
       service: 'hyperdx-oss-usage-stats',
-    } as any),
+    }),
   ],
 });
 
+function extractTableNames(source: SourceDocument): string[] {
+  const tables: string[] = [];
+  if (source.kind === SourceKind.Metric) {
+    for (const key of Object.values(MetricsDataType)) {
+      const metricTable = source.metricTables?.[key];
+      if (!metricTable) continue;
+      tables.push(metricTable);
+    }
+  } else {
+    tables.push(source.from.tableName);
+  }
+  return tables;
+}
+
 const getClickhouseTableSize = async () => {
-  const rows = await clickhouse.client.query({
-    query: `
-      SELECT
-          table,
-          sum(bytes) AS size,
-          sum(rows) AS rows,
-          min(min_time) AS min_time,
-          max(max_time) AS max_time,
-          max(modification_time) AS latestModification,
-          toUInt32((max_time - min_time) / 86400) AS days,
-          size / ((max_time - min_time) / 86400) AS avgDaySize
-      FROM system.parts
-      WHERE active
-      AND database = 'default'
-      AND (table = {table1: String} OR table = {table2: String} OR table = {table3: String})
-      GROUP BY table
-      ORDER BY rows DESC
-    `,
-    format: 'JSON',
-    query_params: {
-      table1: clickhouse.TableName.LogStream,
-      table2: clickhouse.TableName.Rrweb,
-      table3: clickhouse.TableName.Metric,
-    },
-  });
-  const result = await rows.json<ResponseJSON<any>>();
-  return result.data;
+  // fetch mongo data
+  const connections = await Connection.find();
+  const sources = await Source.find();
+
+  // build map for each db instance
+  const distributedTableMap = new Map<string, string[]>();
+  for (const source of sources) {
+    const key = `${source.connection.toString()},${source.from.databaseName}`;
+    if (distributedTableMap.has(key)) {
+      distributedTableMap.get(key)?.push(...extractTableNames(source));
+    } else {
+      distributedTableMap.set(key, extractTableNames(source));
+    }
+  }
+
+  // fetch usage data
+  const results: any[] = [];
+  for (const [key, tables] of distributedTableMap) {
+    const [connectionId, dbName] = key.split(',');
+    const tableListString = tables
+      .map((_, idx) => `table = {table${idx}: String}`)
+      .join(' OR ');
+    const query_params = tables.reduce(
+      (acc, table, idx) => {
+        acc[`table${idx}`] = table;
+        return acc;
+      },
+      {} as { [key: string]: string },
+    );
+    query_params.dbName = dbName;
+
+    // find connection
+    const connection = connections.find(c => c.id === connectionId);
+    if (!connection) continue;
+
+    // query clickhouse
+    try {
+      const clickhouseClient = new ClickhouseClient({
+        host: connection.host,
+        username: connection.username,
+        password: connection.password,
+      });
+      const _rows = await clickhouseClient.query({
+        query: `
+        SELECT
+            table,
+            sum(bytes) AS size,
+            sum(rows) AS rows,
+            min(min_time) AS min_time,
+            max(max_time) AS max_time,
+            max(modification_time) AS latestModification,
+            toUInt32((max_time - min_time) / 86400) AS days,
+            size / ((max_time - min_time) / 86400) AS avgDaySize
+        FROM system.parts
+        WHERE active
+        AND database = {dbName: String}
+        AND (${tableListString})
+        GROUP BY table
+        ORDER BY rows DESC
+      `,
+        format: 'JSON',
+        query_params,
+      });
+      const res = await _rows.json<ResponseJSON<any>>();
+      results.push(...res.data);
+    } catch (error) {
+      // ignore
+    }
+  }
+  return results;
 };
 
-export default async () => {
+async function getUsageStats() {
   try {
     const nowInMs = Date.now();
     const [userCounts, team, chTables] = await Promise.all([
@@ -99,4 +162,11 @@ export default async () => {
   } catch (err) {
     // ignore
   }
-};
+}
+
+export default function () {
+  void getUsageStats();
+  setInterval(() => {
+    void getUsageStats();
+  }, ms('4h'));
+}

packages/app/src/AppNav.tsx

@@ -860,6 +860,12 @@ export default function AppNav({ fixed = false }: { fixed?: boolean }) {
             onClickUserPreferences={openUserPreferences}
             logoutUrl={IS_LOCAL_MODE ? null : `/api/logout`}
           />
+          {meData && meData.usageStatsEnabled && (
+            <img
+              referrerPolicy="no-referrer-when-downgrade"
+              src="https://static.scarf.sh/a.png?x-pxid=bbc99c42-7a75-4eee-9fb9-2b161fc4acd6"
+            />
+          )}
         </div>
       </div>
       <UserPreferencesModal

version.sh

@@ -21,8 +21,8 @@ sed -i '' 's/\("version":\s*"\)[^"]*/\"$API_LATEST_VERSION\"/' package.json
 echo "Updated root package.json version to $API_LATEST_VERSION"
 
 # update tags in .env 
-sed -i '' -e "s/CHANGESET_TAG=.*/CHANGESET_TAG=$API_LATEST_VERSION/g" ./.env
-echo "Updated .env CHANGESET_TAG to $API_LATEST_VERSION"
+sed -i '' -e "s/CODE_VERSION=.*/CODE_VERSION=$API_LATEST_VERSION/g" ./.env
+echo "Updated .env CODE_VERSION to $API_LATEST_VERSION"
 
 sed -i '' -e "s/IMAGE_VERSION_SUB_TAG=.*/IMAGE_VERSION_SUB_TAG=${API_LATEST_VERSION##*-beta}/g" ./.env
 echo "Updated .env IMAGE_VERSION_SUB_TAG to ${API_LATEST_VERSION##*-beta}"