fix: add whereLanguage to tile alerts (#1842)

## Summary

- Fix tile alerts to support `groupBy` for Gauge/Sum metrics — each group-by value appears as its own column in the response
- Add missing `whereLanguage` to tile alert config so Lucene WHERE conditions are parsed correctly
- Replace stale fixture-based ClickHouse schema with otel-collector's canonical schema in integration tests

Ref: HDX-3576

.changeset/lemon-knives-double.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+fix: add whereLanguage to tile alerts

Makefile

@@ -42,8 +42,9 @@ dev-int-build:
 .PHONY: dev-int
 dev-int:
 	docker compose -p int -f ./docker-compose.ci.yml up -d
-	npx nx run @hyperdx/api:dev:int $(FILE)
-	docker compose -p int -f ./docker-compose.ci.yml down
+	npx nx run @hyperdx/api:dev:int $(FILE); ret=$$?; \
+	docker compose -p int -f ./docker-compose.ci.yml down; \
+	exit $$ret
 
 .PHONY: dev-int-common-utils
 dev-int-common-utils:

docker-compose.ci.yml

@@ -8,6 +8,7 @@ services:
     environment:
       CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
       HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
+      HYPERDX_OTEL_EXPORTER_TABLES_TTL: '87600h'
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
     volumes:
@@ -18,6 +19,7 @@ services:
     #   - '4317:4317' # OTLP gRPC receiver
     #   - '4318:4318' # OTLP http receiver
     #   - '8888:8888' # metrics extension
+    restart: on-failure
     networks:
       - internal
     depends_on:

packages/api/package.json

@@ -89,7 +89,7 @@
     "lint:fix": "npx eslint . --ext .ts --fix",
     "ci:lint": "yarn lint && yarn tsc --noEmit",
     "ci:int": "DOTENV_CONFIG_PATH=.env.test jest --runInBand --ci --forceExit --coverage",
-    "dev:int": "DOTENV_CONFIG_PATH=.env.test node --inspect-brk ../../node_modules/.bin/jest --runInBand --ci --forceExit --coverage",
+    "dev:int": "DOTENV_CONFIG_PATH=.env.test jest --runInBand --forceExit --coverage",
     "dev:migrate-db-create": "ts-node node_modules/.bin/migrate-mongo create -f migrate-mongo-config.ts",
     "dev:migrate-db": "ts-node node_modules/.bin/migrate-mongo up -f migrate-mongo-config.ts",
     "dev:migrate-ch-create": "migrate create -ext sql -dir ./migrations/ch -seq",

packages/api/src/fixtures.ts

@@ -70,185 +70,49 @@ const healthCheck = async () => {
   }
 };
 
-const connectClickhouse = async () => {
-  // health check
+const REQUIRED_TABLES = [
+  DEFAULT_LOGS_TABLE,
+  DEFAULT_TRACES_TABLE,
+  DEFAULT_METRICS_TABLE.GAUGE,
+  DEFAULT_METRICS_TABLE.SUM,
+  DEFAULT_METRICS_TABLE.HISTOGRAM,
+  DEFAULT_METRICS_TABLE.SUMMARY,
+  DEFAULT_METRICS_TABLE.EXPONENTIAL_HISTOGRAM,
+];
+
+const waitForClickhouseSchema = async () => {
   await healthCheck();
 
   const client = await getTestFixtureClickHouseClient();
-  await client.command({
-    query: `
-      CREATE TABLE IF NOT EXISTS ${DEFAULT_DATABASE}.${DEFAULT_LOGS_TABLE}
-      (
-        Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
-        TimestampTime DateTime DEFAULT toDateTime(Timestamp),
-        TraceId String CODEC(ZSTD(1)),
-        SpanId String CODEC(ZSTD(1)),
-        TraceFlags UInt8,
-        SeverityText LowCardinality(String) CODEC(ZSTD(1)),
-        SeverityNumber UInt8,
-        ServiceName LowCardinality(String) CODEC(ZSTD(1)),
-        Body String CODEC(ZSTD(1)),
-        ResourceSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
-        ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-        ScopeSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
-        ScopeName String CODEC(ZSTD(1)),
-        ScopeVersion LowCardinality(String) CODEC(ZSTD(1)),
-        ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-        LogAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-        INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-        INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-        INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-        INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-        INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-        INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-        INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-        INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
-      )
-      ENGINE = MergeTree
-      PARTITION BY toDate(TimestampTime)
-      PRIMARY KEY (ServiceName, TimestampTime)
-      ORDER BY (ServiceName, TimestampTime, Timestamp)
-      SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
-    `,
-    // Recommended for cluster usage to avoid situations
-    // where a query processing error occurred after the response code
-    // and HTTP headers were sent to the client.
-    // See https://clickhouse.com/docs/en/interfaces/http/#response-buffering
-    clickhouse_settings: {
-      wait_end_of_query: 1,
-    },
-  });
+  const maxWaitMs = 30_000;
+  const pollIntervalMs = 500;
+  const start = Date.now();
 
-  await client.command({
-    query: `
-      CREATE TABLE IF NOT EXISTS ${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.GAUGE}
-        (
-          ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-          ResourceSchemaUrl String CODEC(ZSTD(1)),
-          ScopeName String CODEC(ZSTD(1)),
-          ScopeVersion String CODEC(ZSTD(1)),
-          ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-          ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),
-          ScopeSchemaUrl String CODEC(ZSTD(1)),
-          ServiceName LowCardinality(String) CODEC(ZSTD(1)),
-          MetricName String CODEC(ZSTD(1)),
-          MetricDescription String CODEC(ZSTD(1)),
-          MetricUnit String CODEC(ZSTD(1)),
-          Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-          StartTimeUnix DateTime64(9) CODEC(Delta(8), ZSTD(1)),
-          TimeUnix DateTime64(9) CODEC(Delta(8), ZSTD(1)),
-          Value Float64 CODEC(ZSTD(1)),
-          Flags UInt32 CODEC(ZSTD(1)),
-          INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
-      )
-      ENGINE = MergeTree
-      PARTITION BY toDate(TimeUnix)
-      ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
-      SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
-    `,
-    // Recommended for cluster usage to avoid situations
-    // where a query processing error occurred after the response code
-    // and HTTP headers were sent to the client.
-    // See https://clickhouse.com/docs/en/interfaces/http/#response-buffering
-    clickhouse_settings: {
-      wait_end_of_query: 1,
-    },
-  });
+  while (Date.now() - start < maxWaitMs) {
+    const result = await client
+      .query({
+        query: `SELECT name FROM system.tables WHERE database = '${DEFAULT_DATABASE}'`,
+        format: 'JSONEachRow',
+      })
+      .then((res: any) => res.json());
 
-  await client.command({
-    query: `
-      CREATE TABLE IF NOT EXISTS ${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.SUM}
-      (
-          ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-          ResourceSchemaUrl String CODEC(ZSTD(1)),
-          ScopeName String CODEC(ZSTD(1)),
-          ScopeVersion String CODEC(ZSTD(1)),
-          ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-          ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),
-          ScopeSchemaUrl String CODEC(ZSTD(1)),
-          ServiceName LowCardinality(String) CODEC(ZSTD(1)),
-          MetricName String CODEC(ZSTD(1)),
-          MetricDescription String CODEC(ZSTD(1)),
-          MetricUnit String CODEC(ZSTD(1)),
-          Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-          StartTimeUnix DateTime64(9) CODEC(Delta(8), ZSTD(1)),
-          TimeUnix DateTime64(9) CODEC(Delta(8), ZSTD(1)),
-          Value Float64 CODEC(ZSTD(1)),
-          Flags UInt32 CODEC(ZSTD(1)),
-          AggregationTemporality Int32 CODEC(ZSTD(1)),
-          IsMonotonic Bool CODEC(Delta(1), ZSTD(1)),
-          INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
-      )
-      ENGINE = MergeTree
-      PARTITION BY toDate(TimeUnix)
-      ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
-      SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
-    `,
-    // Recommended for cluster usage to avoid situations
-    // where a query processing error occurred after the response code
-    // and HTTP headers were sent to the client.
-    // See https://clickhouse.com/docs/en/interfaces/http/#response-buffering
-    clickhouse_settings: {
-      wait_end_of_query: 1,
-    },
-  });
+    const existingTables = new Set(result.map((row: any) => row.name));
+    const missing = REQUIRED_TABLES.filter(t => !existingTables.has(t));
 
-  await client.command({
-    query: `
-      CREATE TABLE IF NOT EXISTS ${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.HISTOGRAM}
-      (
-          ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-          ResourceSchemaUrl String CODEC(ZSTD(1)),
-          ScopeName String CODEC(ZSTD(1)),
-          ScopeVersion String CODEC(ZSTD(1)),
-          ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-          ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),
-          ScopeSchemaUrl String CODEC(ZSTD(1)),
-          ServiceName LowCardinality(String) CODEC(ZSTD(1)),
-          MetricName String CODEC(ZSTD(1)),
-          MetricDescription String CODEC(ZSTD(1)),
-          MetricUnit String CODEC(ZSTD(1)),
-          Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-          StartTimeUnix DateTime64(9) CODEC(Delta(8), ZSTD(1)),
-          TimeUnix DateTime64(9) CODEC(Delta(8), ZSTD(1)),
-          Count UInt64 CODEC(Delta(8), ZSTD(1)),
-          Sum Float64 CODEC(ZSTD(1)),
-          BucketCounts Array(UInt64) CODEC(ZSTD(1)),
-          ExplicitBounds Array(Float64) CODEC(ZSTD(1)),
-          Flags UInt32 CODEC(ZSTD(1)),
-          Min Float64 CODEC(ZSTD(1)),
-          Max Float64 CODEC(ZSTD(1)),
-          AggregationTemporality Int32 CODEC(ZSTD(1)),
-          INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-          INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
-      )
-      ENGINE = MergeTree
-      PARTITION BY toDate(TimeUnix)
-      ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
-      SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
-    `,
-    // Recommended for cluster usage to avoid situations
-    // where a query processing error occurred after the response code
-    // and HTTP headers were sent to the client.
-    // See https://clickhouse.com/docs/en/interfaces/http/#response-buffering
-    clickhouse_settings: {
-      wait_end_of_query: 1,
-    },
-  });
+    if (missing.length === 0) {
+      logger.info('All required ClickHouse tables are ready');
+      return;
+    }
+
+    logger.info(
+      `Waiting for ClickHouse tables: ${missing.join(', ')} (${Math.round((Date.now() - start) / 1000)}s elapsed)`,
+    );
+    await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
+  }
+
+  throw new Error(
+    `Timed out waiting for ClickHouse tables after ${maxWaitMs / 1000}s`,
+  );
 };
 
 export const connectDB = async () => {
@@ -287,7 +151,7 @@ export const initCiEnvs = async () => {
   }
 
   // Populate fake persistent data here...
-  await connectClickhouse();
+  await waitForClickhouseSchema();
 };
 
 class MockServer extends Server {
@@ -388,6 +252,8 @@ export const clearClickhouseTables = async () => {
     `${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.GAUGE}`,
     `${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.SUM}`,
     `${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.HISTOGRAM}`,
+    `${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.SUMMARY}`,
+    `${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.EXPONENTIAL_HISTOGRAM}`,
   ];
 
   const promises: any = [];

packages/api/src/tasks/checkAlerts/index.ts

@@ -332,6 +332,7 @@ const getChartConfigFromAlert = (
         select: tile.config.select,
         timestampValueExpression: source.timestampValueExpression,
         where: tile.config.where,
+        whereLanguage: tile.config.whereLanguage,
         seriesReturnType: tile.config.seriesReturnType,
       };
     }