perf + fix: single clickhouse proxy middleware instance (#687)

Ref: HDX-1423

create clickhouse proxy middleware once to prevent memory leaks issue

.changeset/rotten-cheetahs-argue.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/common-utils": patch
+---
+
+feat: export 'Connection' type

.changeset/wet-elephants-battle.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+perf + fix: single clickhouse proxy middleware instance

packages/api/package.json

@@ -12,6 +12,8 @@
     "@hyperdx/lucene": "^3.1.1",
     "@hyperdx/node-opentelemetry": "^0.8.1",
     "@opentelemetry/api": "^1.8.0",
+    "@opentelemetry/host-metrics": "^0.35.5",
+    "@opentelemetry/sdk-metrics": "^1.30.1",
     "@slack/webhook": "^6.1.0",
     "axios": "^1.6.2",
     "compression": "^1.7.4",
@@ -27,7 +29,7 @@
     "extract-domain": "^2.4.1",
     "handlebars": "^4.7.8",
     "http-graceful-shutdown": "^3.1.13",
-    "http-proxy-middleware": "^3.0.2",
+    "http-proxy-middleware": "^3.0.3",
     "jsonwebtoken": "^9.0.0",
     "lodash": "^4.17.21",
     "minimist": "^1.2.7",

packages/api/src/index.ts

@@ -1,3 +1,6 @@
+import { getHyperDXMetricReader } from '@hyperdx/node-opentelemetry/build/src/metrics';
+import { HostMetrics } from '@opentelemetry/host-metrics';
+import { MeterProvider, MetricReader } from '@opentelemetry/sdk-metrics';
 import { serializeError } from 'serialize-error';
 
 import * as config from '@/config';
@@ -5,6 +8,16 @@ import Server from '@/server';
 import { isOperationalError } from '@/utils/errors';
 import logger from '@/utils/logger';
 
+if (config.IS_DEV) {
+  // Start collecting host metrics
+  const meterProvider = new MeterProvider({
+    // FIXME: missing selectCardinalityLimit property
+    readers: [getHyperDXMetricReader() as unknown as MetricReader],
+  });
+  const hostMetrics = new HostMetrics({ meterProvider });
+  hostMetrics.start();
+}
+
 const server = new Server();
 
 process.on('uncaughtException', (err: Error) => {

packages/api/src/middleware/auth.ts

@@ -1,3 +1,4 @@
+import { Connection } from '@hyperdx/common-utils/dist/types';
 import { setTraceAttributes } from '@hyperdx/node-opentelemetry';
 import type { NextFunction, Request, Response } from 'express';
 import { serializeError } from 'serialize-error';
@@ -11,6 +12,11 @@ declare global {
   namespace Express {
     interface User extends UserDocument {}
   }
+  namespace Express {
+    interface Request {
+      _hdx_connection?: Connection;
+    }
+  }
 }
 
 declare module 'express-session' {

packages/api/src/routers/api/clickhouseProxy.ts

@@ -1,9 +1,8 @@
-import express from 'express';
+import express, { Request, Response } from 'express';
 import { createProxyMiddleware } from 'http-proxy-middleware';
 import { z } from 'zod';
 import { validateRequest } from 'zod-express-middleware';
 
-import * as config from '@/config';
 import { getConnectionById } from '@/controllers/connection';
 import { getNonNullUserWithTeam } from '@/middleware/auth';
 import { objectIdSchema } from '@/utils/zod';
@@ -82,76 +81,86 @@ router.get(
         return;
       }
 
-      const newPath = req.params[0];
-      const qparams = new URLSearchParams(req.query);
-      qparams.delete('hyperdx_connection_id');
-
-      return createProxyMiddleware({
-        target: connection.host,
-        changeOrigin: true,
-        pathFilter: (path, req) => {
-          // TODO: allow other methods
-          return req.method === 'GET';
-        },
-        pathRewrite: {
-          '^/clickhouse-proxy': '',
-        },
-        headers: {
-          ...(connection.username
-            ? { 'X-ClickHouse-User': connection.username }
-            : {}),
-          ...(connection.password
-            ? { 'X-ClickHouse-Key': connection.password }
-            : {}),
-        },
-        on: {
-          proxyReq: (proxyReq, req) => {
-            proxyReq.path = `/${newPath}?${qparams.toString()}`;
-          },
-          proxyRes: (proxyRes, req, res) => {
-            // since clickhouse v24, the cors headers * will be attached to the response by default
-            // which will cause the browser to block the response
-            if (req.headers['access-control-request-method']) {
-              proxyRes.headers['access-control-allow-methods'] =
-                req.headers['access-control-request-method'];
-            }
-
-            if (req.headers['access-control-request-headers']) {
-              proxyRes.headers['access-control-allow-headers'] =
-                req.headers['access-control-request-headers'];
-            }
-
-            if (req.headers.origin) {
-              proxyRes.headers['access-control-allow-origin'] =
-                req.headers.origin;
-              proxyRes.headers['access-control-allow-credentials'] = 'true';
-            }
-          },
-          error: (err, _req, _res) => {
-            console.error('Proxy error:', err);
-            res.writeHead(500, {
-              'Content-Type': 'application/json',
-            });
-            res.end(
-              JSON.stringify({
-                success: false,
-                error: err.message || 'Failed to connect to ClickHouse server',
-              }),
-            );
-          },
-        },
-        // ...(config.IS_DEV && {
-        //   logger: console,
-        // }),
-      })(req, res, next);
+      req._hdx_connection = {
+        host: connection.host,
+        id: connection.id,
+        name: connection.name,
+        password: connection.password,
+        username: connection.username,
+      };
+      next();
     } catch (e) {
-      console.error('Router error:', e);
-      res.status(500).json({
-        success: false,
-        error: e instanceof Error ? e.message : 'Internal server error',
-      });
+      console.error('Error fetching connection info:', e);
+      next(e);
     }
   },
+  createProxyMiddleware({
+    target: '', // doesn't matter. it should be overridden by the router
+    changeOrigin: true,
+    pathFilter: (path, _req) => {
+      // TODO: allow other methods
+      return _req.method === 'GET';
+    },
+    pathRewrite: {
+      '^/clickhouse-proxy': '',
+    },
+    router: _req => {
+      if (!_req._hdx_connection?.host) {
+        throw new Error('[createProxyMiddleware] Connection not found');
+      }
+      return _req._hdx_connection.host;
+    },
+    on: {
+      proxyReq: (proxyReq, _req) => {
+        const newPath = _req.params[0];
+        const qparams = new URLSearchParams(_req.query);
+        qparams.delete('hyperdx_connection_id');
+        if (_req._hdx_connection?.username && _req._hdx_connection?.password) {
+          proxyReq.setHeader(
+            'X-ClickHouse-User',
+            _req._hdx_connection.username,
+          );
+          proxyReq.setHeader('X-ClickHouse-Key', _req._hdx_connection.password);
+        }
+        proxyReq.path = `/${newPath}?${qparams.toString()}`;
+      },
+      proxyRes: (proxyRes, _req, res) => {
+        // since clickhouse v24, the cors headers * will be attached to the response by default
+        // which will cause the browser to block the response
+        if (_req.headers['access-control-request-method']) {
+          proxyRes.headers['access-control-allow-methods'] =
+            _req.headers['access-control-request-method'];
+        }
+
+        if (_req.headers['access-control-request-headers']) {
+          proxyRes.headers['access-control-allow-headers'] =
+            _req.headers['access-control-request-headers'];
+        }
+
+        if (_req.headers.origin) {
+          proxyRes.headers['access-control-allow-origin'] = _req.headers.origin;
+          proxyRes.headers['access-control-allow-credentials'] = 'true';
+        }
+      },
+      error: (err, _req, _res) => {
+        console.error('Proxy error:', err);
+        if (_res instanceof Response) {
+          (_res as Response).writeHead(500, {
+            'Content-Type': 'application/json',
+          });
+          _res.end(
+            JSON.stringify({
+              success: false,
+              error: err.message || 'Failed to connect to ClickHouse server',
+            }),
+          );
+        }
+      },
+    },
+    // ...(config.IS_DEV && {
+    //   logger: console,
+    // }),
+  }),
 );
 
 export default router;

packages/common-utils/src/types.ts

@@ -435,6 +435,8 @@ export const ConnectionSchema = z.object({
   password: z.string().optional(),
 });
 
+export type Connection = z.infer<typeof ConnectionSchema>;
+
 // --------------------------
 // TABLE SOURCES
 // --------------------------

yarn.lock

@@ -4220,6 +4220,8 @@ __metadata:
     "@hyperdx/lucene": "npm:^3.1.1"
     "@hyperdx/node-opentelemetry": "npm:^0.8.1"
     "@opentelemetry/api": "npm:^1.8.0"
+    "@opentelemetry/host-metrics": "npm:^0.35.5"
+    "@opentelemetry/sdk-metrics": "npm:^1.30.1"
     "@slack/types": "npm:^2.8.0"
     "@slack/webhook": "npm:^6.1.0"
     "@types/airbnb__node-memwatch": "npm:^2.0.0"
@@ -4255,7 +4257,7 @@ __metadata:
     extract-domain: "npm:^2.4.1"
     handlebars: "npm:^4.7.8"
     http-graceful-shutdown: "npm:^3.1.13"
-    http-proxy-middleware: "npm:^3.0.2"
+    http-proxy-middleware: "npm:^3.0.3"
     jest: "npm:^28.1.3"
     jsonwebtoken: "npm:^9.0.0"
     lodash: "npm:^4.17.21"
@@ -6072,6 +6074,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/core@npm:1.30.1":
+  version: 1.30.1
+  resolution: "@opentelemetry/core@npm:1.30.1"
+  dependencies:
+    "@opentelemetry/semantic-conventions": "npm:1.28.0"
+  peerDependencies:
+    "@opentelemetry/api": ">=1.0.0 <1.10.0"
+  checksum: 10c0/4c25ba50a6137c2ba9ca563fb269378f3c9ca6fd1b3f15dbb6eff78eebf5656f281997cbb7be8e51c01649fd6ad091083fcd8a42dd9b5dfac907dc06d7cfa092
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/core@npm:^1.1.0":
   version: 1.15.2
   resolution: "@opentelemetry/core@npm:1.15.2"
@@ -6191,6 +6204,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/host-metrics@npm:^0.35.5":
+  version: 0.35.5
+  resolution: "@opentelemetry/host-metrics@npm:0.35.5"
+  dependencies:
+    systeminformation: "npm:5.23.8"
+  peerDependencies:
+    "@opentelemetry/api": ^1.3.0
+  checksum: 10c0/3065ab9f8b2f02e26d12298d9a8171cf8370b0af0fd2dd08f7b14685a2587093e31aa46e4a04563c9833914990dcaa7ab858cb7d8f2a0d129f2d41a604df9243
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/instrumentation-amqplib@npm:^0.37.0":
   version: 0.37.0
   resolution: "@opentelemetry/instrumentation-amqplib@npm:0.37.0"
@@ -6985,6 +7009,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/resources@npm:1.30.1":
+  version: 1.30.1
+  resolution: "@opentelemetry/resources@npm:1.30.1"
+  dependencies:
+    "@opentelemetry/core": "npm:1.30.1"
+    "@opentelemetry/semantic-conventions": "npm:1.28.0"
+  peerDependencies:
+    "@opentelemetry/api": ">=1.0.0 <1.10.0"
+  checksum: 10c0/688e73258283c80662bfa9a858aaf73bf3b832a18d96e546d0dddfa6dcec556cdfa087a1d0df643435293406009e4122d7fb7eeea69aa87b539d3bab756fba74
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/resources@npm:^1.0.0":
   version: 1.13.0
   resolution: "@opentelemetry/resources@npm:1.13.0"
@@ -7036,6 +7072,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/sdk-metrics@npm:^1.30.1":
+  version: 1.30.1
+  resolution: "@opentelemetry/sdk-metrics@npm:1.30.1"
+  dependencies:
+    "@opentelemetry/core": "npm:1.30.1"
+    "@opentelemetry/resources": "npm:1.30.1"
+  peerDependencies:
+    "@opentelemetry/api": ">=1.3.0 <1.10.0"
+  checksum: 10c0/7e60178e61eaf49db5d74f6c3701706762d71d370044253c72bb5668dba3a435030ed6847605ee55d0e1b8908ad123a2517b5f00415a2fb3d98468a0a318e3c0
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/sdk-metrics@npm:^1.9.1":
   version: 1.15.0
   resolution: "@opentelemetry/sdk-metrics@npm:1.15.0"
@@ -7231,6 +7279,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/semantic-conventions@npm:1.28.0":
+  version: 1.28.0
+  resolution: "@opentelemetry/semantic-conventions@npm:1.28.0"
+  checksum: 10c0/deb8a0f744198071e70fea27143cf7c9f7ecb7e4d7b619488c917834ea09b31543c1c2bcea4ec5f3cf68797f0ef3549609c14e859013d9376400ac1499c2b9cb
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/semantic-conventions@npm:^1.0.0":
   version: 1.11.0
   resolution: "@opentelemetry/semantic-conventions@npm:1.11.0"
@@ -17354,20 +17409,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"http-proxy-middleware@npm:^3.0.2":
-  version: 3.0.2
-  resolution: "http-proxy-middleware@npm:3.0.2"
-  dependencies:
-    "@types/http-proxy": "npm:^1.17.15"
-    debug: "npm:^4.3.6"
-    http-proxy: "npm:^1.18.1"
-    is-glob: "npm:^4.0.3"
-    is-plain-object: "npm:^5.0.0"
-    micromatch: "npm:^4.0.8"
-  checksum: 10c0/74afb03174e5203afc41c7a1bd4b0d8b16f229da8187c47496d3817a3a5844da31ca8a8e1a230fd32ffd74211fbf13342d4dbe129d7e4b4b46f3174f9852c268
-  languageName: node
-  linkType: hard
-
 "http-proxy-middleware@npm:^3.0.3":
   version: 3.0.3
   resolution: "http-proxy-middleware@npm:3.0.3"
@@ -26600,6 +26641,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"systeminformation@npm:5.23.8":
+  version: 5.23.8
+  resolution: "systeminformation@npm:5.23.8"
+  bin:
+    systeminformation: lib/cli.js
+  checksum: 10c0/d4d750d82345081a6a12200ec8f559ff65a8c28d9797d4368c246682ee02131ee08a4227e4401b6680839f0f0e1a72758071fd84eae2f0584a89e948d583703f
+  conditions: (os=darwin | os=linux | os=win32 | os=freebsd | os=openbsd | os=netbsd | os=sunos | os=android)
+  languageName: node
+  linkType: hard
+
 "tabbable@npm:^6.0.0":
   version: 6.2.0
   resolution: "tabbable@npm:6.2.0"