diff --git a/.changeset/add-alerts-page.md b/.changeset/add-alerts-page.md new file mode 100644 index 00000000..0a60b697 --- /dev/null +++ b/.changeset/add-alerts-page.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/cli": patch +--- + +Add alerts page (Shift+A) with overview and recent trigger history diff --git a/.changeset/add-cli-pattern-mining.md b/.changeset/add-cli-pattern-mining.md new file mode 100644 index 00000000..3c8f3b3e --- /dev/null +++ b/.changeset/add-cli-pattern-mining.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/cli": patch +--- + +Add event pattern mining view (Shift+P) with sampled estimation and drill-down diff --git a/.changeset/add-drain-library.md b/.changeset/add-drain-library.md new file mode 100644 index 00000000..63506a59 --- /dev/null +++ b/.changeset/add-drain-library.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/common-utils": patch +--- + +Add Drain log template mining library (ported from browser-drain) diff --git a/.changeset/bump-otel-collector-to-0149.md b/.changeset/bump-otel-collector-to-0149.md new file mode 100644 index 00000000..b78297e3 --- /dev/null +++ b/.changeset/bump-otel-collector-to-0149.md @@ -0,0 +1,8 @@ +--- +'@hyperdx/otel-collector': minor +--- + +feat: Bump OTel Collector from 0.147.0 to 0.149.0 + +Upgrade the OpenTelemetry Collector and all its components from v0.147.0 to +v0.149.0 (core providers from v1.53.0 to v1.55.0). diff --git a/.changeset/cli-app-url-migration.md b/.changeset/cli-app-url-migration.md new file mode 100644 index 00000000..54012d64 --- /dev/null +++ b/.changeset/cli-app-url-migration.md @@ -0,0 +1,10 @@ +--- +"@hyperdx/cli": minor +--- + +**Breaking:** Replace `-s`/`--server` flag with `-a`/`--app-url` across all CLI commands (except `upload-sourcemaps`). Users should now provide the HyperDX app URL instead of the API URL — the CLI derives the API URL by appending `/api`. + +- `hdx auth login` now prompts interactively for login method, app URL, and credentials (no flags required) +- Expired/missing sessions prompt for re-login with the last URL autofilled instead of printing an error +- Add URL input validation and post-login session verification +- Existing saved sessions are auto-migrated from `apiUrl` to `appUrl` diff --git a/.changeset/cool-pants-train.md b/.changeset/cool-pants-train.md new file mode 100644 index 00000000..423a4e45 --- /dev/null +++ b/.changeset/cool-pants-train.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": minor +--- + +Add an MCP (Model Context Protocol) server to the HyperDX API, enabling AI assistants (Claude, Cursor, OpenCode, etc.) to query observability data, manage dashboards, and explore data sources directly via standardized tool calls. \ No newline at end of file diff --git a/.changeset/deprecate-clickhouse-json-feature-gate.md b/.changeset/deprecate-clickhouse-json-feature-gate.md new file mode 100644 index 00000000..2b7aef6f --- /dev/null +++ b/.changeset/deprecate-clickhouse-json-feature-gate.md @@ -0,0 +1,11 @@ +--- +'@hyperdx/otel-collector': patch +--- + +refactor: Deprecate clickhouse.json feature gate in favor of per-exporter json config + +Replace the upstream-deprecated `--feature-gates=clickhouse.json` CLI flag with +the per-exporter `json: true` config option controlled by +`HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE`. The old +`OTEL_AGENT_FEATURE_GATE_ARG` is still supported for backward compatibility but +prints a deprecation warning when `clickhouse.json` is detected. diff --git a/.changeset/early-ducks-grow.md b/.changeset/early-ducks-grow.md new file mode 100644 index 00000000..c66ffa3c --- /dev/null +++ b/.changeset/early-ducks-grow.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Support alerts on Raw SQL Number Charts diff --git a/.changeset/fix-copy-row-json-button.md b/.changeset/fix-copy-row-json-button.md new file mode 100644 index 00000000..2dfc9db7 --- /dev/null +++ b/.changeset/fix-copy-row-json-button.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix: Fix "Copy entire row as JSON" button crashing on rows with non-string values diff --git a/.changeset/hdx-3908-validation-toast-dedupe.md b/.changeset/hdx-3908-validation-toast-dedupe.md new file mode 100644 index 00000000..94716165 --- /dev/null +++ b/.changeset/hdx-3908-validation-toast-dedupe.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +Dedupe source validation issue toasts so repeated source refetches update a single notification instead of stacking duplicates. diff --git a/.changeset/healthy-eyes-kiss.md b/.changeset/healthy-eyes-kiss.md new file mode 100644 index 00000000..d1709ce7 --- /dev/null +++ b/.changeset/healthy-eyes-kiss.md @@ -0,0 +1,9 @@ +--- +"@hyperdx/otel-collector": patch +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +"@hyperdx/cli": patch +--- + +feat: Add between-type alert thresholds diff --git a/.changeset/migrate-otel-collector-to-ocb.md b/.changeset/migrate-otel-collector-to-ocb.md new file mode 100644 index 00000000..b9f7c0b2 --- /dev/null +++ b/.changeset/migrate-otel-collector-to-ocb.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/otel-collector': minor +--- + +feat: Migrate OTel Collector build to use OCB (OpenTelemetry Collector Builder) + +Replace the pre-built otel/opentelemetry-collector-contrib image with a custom +binary built via OCB. This enables adding custom receiver/processor components +in the future while including only the components HyperDX needs. The collector +version is now centralized in `.env` via `OTEL_COLLECTOR_VERSION` and +`OTEL_COLLECTOR_CORE_VERSION`, with `builder-config.yaml` using templatized +placeholders substituted at Docker build time. diff --git a/.changeset/open-trace-in-browser.md b/.changeset/open-trace-in-browser.md new file mode 100644 index 00000000..90ac0659 --- /dev/null +++ b/.changeset/open-trace-in-browser.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/cli": patch +--- + +Add `o` keybinding to open the current trace/span in the HyperDX web app from the TUI. Deep-links to the exact view with side panel, tab, and span selection preserved. Works for both trace and log sources. diff --git a/.changeset/optimize-trace-waterfall.md b/.changeset/optimize-trace-waterfall.md new file mode 100644 index 00000000..ab8f7e1b --- /dev/null +++ b/.changeset/optimize-trace-waterfall.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/cli": patch +--- + +Optimize event detail and trace waterfall queries; add trace detail page and waterfall scrolling diff --git a/.changeset/otel-collector-add-core-extensions.md b/.changeset/otel-collector-add-core-extensions.md new file mode 100644 index 00000000..5bb6904f --- /dev/null +++ b/.changeset/otel-collector-add-core-extensions.md @@ -0,0 +1,13 @@ +--- +'@hyperdx/otel-collector': minor +--- + +feat: Add missing core extensions, commonly-used contrib processors/receivers, and filestorage extension + +Add the two missing core extensions (memorylimiterextension, zpagesextension), +12 commonly-used contrib processors (attributes, filter, resource, k8sattributes, +tailsampling, probabilisticsampler, span, groupbyattrs, redaction, logdedup, +metricstransform, cumulativetodelta), 4 commonly-used contrib receivers +(filelog, dockerstats, k8scluster, kubeletstats), and the filestorage extension +(used for persistent sending queue in the clickhouse exporter) to +builder-config.yaml. diff --git a/.changeset/polite-grapes-cross.md b/.changeset/polite-grapes-cross.md new file mode 100644 index 00000000..e3781c6d --- /dev/null +++ b/.changeset/polite-grapes-cross.md @@ -0,0 +1,8 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +"@hyperdx/cli": patch +--- + +feat: Add additional alert threshold types diff --git a/.changeset/serious-chicken-hammer.md b/.changeset/serious-chicken-hammer.md new file mode 100644 index 00000000..bb3d3471 --- /dev/null +++ b/.changeset/serious-chicken-hammer.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/api": minor +"@hyperdx/app": minor +--- + +Introduces Shared Filters, enabling teams to pin and surface common filters across all members. \ No newline at end of file diff --git a/.changeset/shaggy-tigers-tan.md b/.changeset/shaggy-tigers-tan.md new file mode 100644 index 00000000..6b117e3b --- /dev/null +++ b/.changeset/shaggy-tigers-tan.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +feat: Add Python Runtime Metrics dashboard template diff --git a/.changeset/sharp-emus-reflect.md b/.changeset/sharp-emus-reflect.md new file mode 100644 index 00000000..46d3165a --- /dev/null +++ b/.changeset/sharp-emus-reflect.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/common-utils": patch +--- + +fix: Skip rendering empty SQL dashboard filter diff --git a/.changeset/short-badgers-applaud.md b/.changeset/short-badgers-applaud.md new file mode 100644 index 00000000..6e47923d --- /dev/null +++ b/.changeset/short-badgers-applaud.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Implement alerting for Raw SQL-based dashboard tiles diff --git a/.changeset/short-tools-sleep.md b/.changeset/short-tools-sleep.md new file mode 100644 index 00000000..fd2fd4c0 --- /dev/null +++ b/.changeset/short-tools-sleep.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix: time selector always resets to 00:00 diff --git a/.changeset/silly-toes-cough.md b/.changeset/silly-toes-cough.md new file mode 100644 index 00000000..6c9dfdc9 --- /dev/null +++ b/.changeset/silly-toes-cough.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Add alert history + ack to alert editor diff --git a/.changeset/thirty-students-exist.md b/.changeset/thirty-students-exist.md new file mode 100644 index 00000000..0237fb0b --- /dev/null +++ b/.changeset/thirty-students-exist.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +Allow manually constructed /trace URLs to land in the existing search experience with the trace viewer opened from URL state. This keeps trace deep links user-friendly while reusing the search page for source selection, not-found handling, and trace inspection. diff --git a/.changeset/upgrade-mantine-v9.md b/.changeset/upgrade-mantine-v9.md new file mode 100644 index 00000000..d73466e6 --- /dev/null +++ b/.changeset/upgrade-mantine-v9.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': minor +--- + +Upgrade Mantine from v7 to v9 and remove react-hook-form-mantine dependency diff --git a/.claude/skills/playwright/SKILL.md b/.claude/skills/playwright/SKILL.md index ba616c20..070d6780 100644 --- a/.claude/skills/playwright/SKILL.md +++ b/.claude/skills/playwright/SKILL.md @@ -23,6 +23,8 @@ Delegate to the **`playwright-test-generator`** agent (via the Agent tool). Pass The agent will drive a real browser, execute the steps live, and produce spec code that follows HyperDX conventions. Review the output before proceeding. +NOTE: When there is an existing spec file covering the feature, add new tests to the existing file instead of creating a new one. This keeps related tests together and avoids fragmentation. + ### 2. Test Execution After the generator agent writes the file, run the test: diff --git a/.env b/.env index 7d0a5bb5..b52fe0f0 100644 --- a/.env +++ b/.env @@ -8,8 +8,8 @@ NEXT_ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=clickhouse/clickstack-all-in-one ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-all-in-one NEXT_OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=clickhouse/clickstack-otel-collector OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-otel-collector -CODE_VERSION=2.23.0 -IMAGE_VERSION_SUB_TAG=.23.0 +CODE_VERSION=2.23.2 +IMAGE_VERSION_SUB_TAG=.23.2 IMAGE_VERSION=2 IMAGE_NIGHTLY_TAG=2-nightly IMAGE_LATEST_TAG=latest @@ -38,5 +38,11 @@ HDX_DEV_OTEL_HTTP_PORT=4318 HDX_DEV_OTEL_METRICS_PORT=8888 HDX_DEV_OTEL_JSON_HTTP_PORT=14318 +# Otel Collector version (used as Docker build arg for image tags and component versions) +# When bumping, look up the core version from the upstream manifest: +# https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml +OTEL_COLLECTOR_VERSION=0.149.0 +OTEL_COLLECTOR_CORE_VERSION=1.55.0 + # Otel/Clickhouse config HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default diff --git a/.github/scripts/__tests__/pr-triage-classify.test.js b/.github/scripts/__tests__/pr-triage-classify.test.js new file mode 100644 index 00000000..8b03d3af --- /dev/null +++ b/.github/scripts/__tests__/pr-triage-classify.test.js @@ -0,0 +1,569 @@ +'use strict'; + +// Tests for the pure classification functions in pr-triage-classify.js. +// Uses Node's built-in test runner (no extra dependencies required). +// Run with: node --test .github/scripts/__tests__/pr-triage-classify.test.js + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); + +const { + isTestFile, isTrivialFile, isCriticalFile, + computeSignals, determineTier, buildTierComment, +} = require('../pr-triage-classify'); + +// ── Test helpers ───────────────────────────────────────────────────────────── + +/** Minimal PR object matching the shape returned by the GitHub API */ +function makePR(login, ref) { + return { user: { login }, head: { ref } }; +} + +/** Minimal file entry matching the shape returned by pulls.listFiles */ +function makeFile(filename, additions = 10, deletions = 5) { + return { filename, additions, deletions }; +} + +/** Classify a PR end-to-end from raw inputs (the common test path) */ +function classify(login, ref, files) { + return determineTier(computeSignals(makePR(login, ref), files)); +} + +// ── File classification helpers ────────────────────────────────────────────── + +describe('isTestFile', () => { + it('matches __tests__ directory', () => { + assert.ok(isTestFile('packages/api/src/__tests__/foo.test.ts')); + assert.ok(isTestFile('packages/app/src/components/__tests__/Foo.test.tsx')); + }); + + it('matches .test.* and .spec.* extensions', () => { + assert.ok(isTestFile('packages/app/src/Foo.test.tsx')); + assert.ok(isTestFile('packages/app/src/Foo.spec.js')); + assert.ok(isTestFile('packages/api/src/bar.test.ts')); + }); + + it('matches packages/app/tests/ prefix', () => { + assert.ok(isTestFile('packages/app/tests/e2e/navigation.ts')); + }); + + it('does not match regular source files', () => { + assert.ok(!isTestFile('packages/api/src/routers/foo.ts')); + assert.ok(!isTestFile('packages/app/src/App.tsx')); + }); +}); + +describe('isTrivialFile', () => { + it('matches docs and images', () => { + assert.ok(isTrivialFile('README.md')); + assert.ok(isTrivialFile('docs/setup.txt')); + assert.ok(isTrivialFile('assets/logo.png')); + assert.ok(isTrivialFile('assets/icon.svg')); + }); + + it('matches lock files and yarn config', () => { + assert.ok(isTrivialFile('yarn.lock')); + assert.ok(isTrivialFile('package-lock.json')); + assert.ok(isTrivialFile('.yarnrc.yml')); + }); + + it('matches .changeset/ files', () => { + assert.ok(isTrivialFile('.changeset/some-change.md')); + assert.ok(isTrivialFile('.changeset/fancy-bears-dance.md')); + }); + + it('matches .env.example and .github/images/', () => { + assert.ok(isTrivialFile('.env.example')); + assert.ok(isTrivialFile('.github/images/screenshot.png')); + }); + + it('matches .github/scripts/ files', () => { + assert.ok(isTrivialFile('.github/scripts/pr-triage.js')); + assert.ok(isTrivialFile('.github/scripts/pr-triage-classify.js')); + }); + + it('matches .github/workflows/ files', () => { + assert.ok(isTrivialFile('.github/workflows/pr-triage.yml')); + assert.ok(isTrivialFile('.github/workflows/knip.yml')); + // main.yml and release.yml are also trivial per isTrivialFile, but they are + // caught first by isCriticalFile in computeSignals, so they still → Tier 4 + assert.ok(isTrivialFile('.github/workflows/main.yml')); + }); + + it('does not match production source files', () => { + assert.ok(!isTrivialFile('packages/app/src/App.tsx')); + assert.ok(!isTrivialFile('packages/api/src/routers/logs.ts')); + assert.ok(!isTrivialFile('Makefile')); + assert.ok(!isTrivialFile('knip.json')); + }); +}); + +describe('isCriticalFile', () => { + it('matches auth middleware', () => { + assert.ok(isCriticalFile('packages/api/src/middleware/auth.ts')); + assert.ok(isCriticalFile('packages/api/src/middleware/auth/index.ts')); + }); + + it('matches sensitive API routes', () => { + assert.ok(isCriticalFile('packages/api/src/routers/api/me.ts')); + assert.ok(isCriticalFile('packages/api/src/routers/api/team.ts')); + assert.ok(isCriticalFile('packages/api/src/routers/external-api/logs.ts')); + }); + + it('matches core data models', () => { + assert.ok(isCriticalFile('packages/api/src/models/user.ts')); + assert.ok(isCriticalFile('packages/api/src/models/team.ts')); + assert.ok(isCriticalFile('packages/api/src/models/teamInvite.ts')); + }); + + it('matches config, tasks, otel, clickhouse, and core CI workflows', () => { + assert.ok(isCriticalFile('packages/api/src/config.ts')); + assert.ok(isCriticalFile('packages/api/src/tasks/alertChecker.ts')); + assert.ok(isCriticalFile('packages/otel-collector/config.yaml')); + assert.ok(isCriticalFile('docker/clickhouse/config.xml')); + assert.ok(isCriticalFile('.github/workflows/main.yml')); + assert.ok(isCriticalFile('.github/workflows/release.yml')); + }); + + it('does NOT flag non-core workflow files as critical', () => { + assert.ok(!isCriticalFile('.github/workflows/pr-triage.yml')); + assert.ok(!isCriticalFile('.github/workflows/knip.yml')); + assert.ok(!isCriticalFile('.github/workflows/claude.yml')); + }); + + it('matches docker/hyperdx/', () => { + assert.ok(isCriticalFile('docker/hyperdx/Dockerfile')); + }); + + it('does NOT match non-critical API models', () => { + assert.ok(!isCriticalFile('packages/api/src/models/alert.ts')); + assert.ok(!isCriticalFile('packages/api/src/models/dashboard.ts')); + }); + + it('does NOT match regular app and API files', () => { + assert.ok(!isCriticalFile('packages/app/src/App.tsx')); + assert.ok(!isCriticalFile('packages/api/src/routers/logs.ts')); + }); + + // Note: isCriticalFile DOES return true for test files under critical paths + // (e.g. packages/api/src/tasks/tests/util.test.ts). The exclusion happens in + // computeSignals, which filters test files out before building criticalFiles. + it('returns true for test files under critical paths (exclusion is in computeSignals)', () => { + assert.ok(isCriticalFile('packages/api/src/tasks/tests/util.test.ts')); + }); +}); + +// ── computeSignals ─────────────────────────────────────────────────────────── + +describe('computeSignals', () => { + it('separates prod, test, and trivial file line counts', () => { + const pr = makePR('alice', 'feature/foo'); + const files = [ + makeFile('packages/app/src/Foo.tsx', 20, 5), // prod: 25 lines + makeFile('packages/app/src/__tests__/Foo.test.tsx', 50, 0), // test: 50 lines + makeFile('README.md', 2, 1), // trivial: excluded + ]; + const s = computeSignals(pr, files); + assert.equal(s.prodFiles.length, 1); + assert.equal(s.prodLines, 25); + assert.equal(s.testLines, 50); + }); + + it('excludes changeset files from prod counts', () => { + const pr = makePR('alice', 'feature/foo'); + const files = [ + makeFile('packages/app/src/Foo.tsx', 20, 5), + makeFile('.changeset/witty-foxes-run.md', 5, 0), // trivial + ]; + const s = computeSignals(pr, files); + assert.equal(s.prodFiles.length, 1); + assert.equal(s.prodLines, 25); + }); + + it('detects agent branches by prefix', () => { + for (const prefix of ['claude/', 'agent/', 'ai/']) { + const s = computeSignals(makePR('alice', `${prefix}fix-thing`), []); + assert.ok(s.isAgentBranch, `expected isAgentBranch for prefix "${prefix}"`); + } + assert.ok(!computeSignals(makePR('alice', 'feature/normal'), []).isAgentBranch); + }); + + it('detects bot authors', () => { + assert.ok(computeSignals(makePR('dependabot[bot]', 'dependabot/npm/foo'), []).isBotAuthor); + assert.ok(!computeSignals(makePR('alice', 'feature/foo'), []).isBotAuthor); + }); + + it('sets allFilesTrivial when every file is trivial', () => { + const files = [makeFile('README.md'), makeFile('yarn.lock')]; + assert.ok(computeSignals(makePR('alice', 'docs/update'), files).allFilesTrivial); + }); + + it('does not set allFilesTrivial for mixed files', () => { + const files = [makeFile('README.md'), makeFile('packages/app/src/Foo.tsx')]; + assert.ok(!computeSignals(makePR('alice', 'feat/foo'), files).allFilesTrivial); + }); + + it('detects cross-layer changes (frontend + backend)', () => { + const files = [ + makeFile('packages/app/src/NewFeature.tsx'), // frontend + makeFile('packages/api/src/services/newFeature.ts'), // backend (not models/routers) + ]; + const s = computeSignals(makePR('alice', 'feat/new'), files); + assert.ok(s.isCrossLayer); + assert.ok(s.touchesFrontend); + assert.ok(s.touchesBackend); + }); + + it('detects cross-layer changes (backend + shared-utils)', () => { + const files = [ + makeFile('packages/api/src/services/foo.ts'), + makeFile('packages/common-utils/src/queryParser.ts'), + ]; + const s = computeSignals(makePR('alice', 'feat/foo'), files); + assert.ok(s.isCrossLayer); + assert.ok(s.touchesSharedUtils); + }); + + it('does not flag single-package changes as cross-layer', () => { + const files = [ + makeFile('packages/app/src/Foo.tsx'), + makeFile('packages/app/src/Bar.tsx'), + ]; + assert.ok(!computeSignals(makePR('alice', 'feat/foo'), files).isCrossLayer); + }); + + it('blocks agent branch from Tier 2 when prod lines exceed threshold', () => { + // 60 prod lines > AGENT_TIER2_MAX_LINES (50) + const s = computeSignals(makePR('alice', 'claude/feature'), [ + makeFile('packages/app/src/Foo.tsx', 60, 0), + ]); + assert.ok(s.agentBlocksTier2); + }); + + it('blocks agent branch from Tier 2 when prod file count exceeds threshold', () => { + // 5 prod files > AGENT_TIER2_MAX_PROD_FILES (3) + const files = Array.from({ length: 5 }, (_, i) => + makeFile(`packages/app/src/File${i}.tsx`, 5, 2) + ); + const s = computeSignals(makePR('alice', 'claude/feature'), files); + assert.ok(s.agentBlocksTier2); + }); + + it('does NOT block agent branch when change is small and focused', () => { + // 16 prod lines, 1 prod file — well under both thresholds + const s = computeSignals(makePR('mikeshi', 'claude/fix-mobile-nav'), [ + makeFile('packages/app/src/AppNav.tsx', 11, 5), + ]); + assert.ok(!s.agentBlocksTier2); + }); +}); + +// ── determineTier ──────────────────────────────────────────────────────────── + +describe('determineTier', () => { + describe('Tier 1', () => { + it('bot author', () => { + assert.equal(classify('dependabot[bot]', 'dependabot/npm/foo', [ + makeFile('package.json', 5, 3), + ]), 1); + }); + + // package.json is not in TIER1_PATTERNS (it's a production file), but bot + // author short-circuits to Tier 1 before the trivial-file check fires. + it('bot author with package.json (non-trivial file) is still Tier 1', () => { + assert.equal(classify('dependabot[bot]', 'dependabot/npm/lodash', [ + makeFile('package.json', 5, 3), + makeFile('packages/api/package.json', 2, 2), + ]), 1); + }); + + it('all trivial files (docs + lock)', () => { + assert.equal(classify('alice', 'docs/update-readme', [ + makeFile('README.md', 10, 2), + makeFile('docs/setup.md', 5, 0), + makeFile('yarn.lock', 100, 80), + ]), 1); + }); + + it('changeset-only PR', () => { + assert.equal(classify('alice', 'release/v2.1', [ + makeFile('.changeset/witty-foxes-run.md', 4, 0), + ]), 1); + }); + }); + + describe('Tier 4', () => { + it('touches auth middleware', () => { + assert.equal(classify('alice', 'fix/auth-bug', [ + makeFile('packages/api/src/middleware/auth.ts', 20, 5), + ]), 4); + }); + + it('touches ClickHouse docker config', () => { + assert.equal(classify('alice', 'infra/clickhouse-update', [ + makeFile('docker/clickhouse/config.xml', 10, 2), + ]), 4); + }); + + it('touches main.yml or release.yml', () => { + assert.equal(classify('alice', 'ci/add-step', [ + makeFile('.github/workflows/main.yml', 15, 3), + ]), 4); + assert.equal(classify('alice', 'ci/release-fix', [ + makeFile('.github/workflows/release.yml', 8, 2), + ]), 4); + }); + + it('non-critical workflow-only changes are Tier 1 (workflow files are trivial)', () => { + assert.equal(classify('alice', 'ci/add-triage-step', [ + makeFile('.github/workflows/pr-triage.yml', 10, 2), + ]), 1); + }); + + it('does NOT flag test files under critical paths as Tier 4', () => { + // e.g. packages/api/src/tasks/tests/util.test.ts should not be critical + assert.equal(classify('alice', 'feat/alert-tests', [ + makeFile('packages/api/src/tasks/tests/util.test.ts', 40, 0), + makeFile('packages/api/src/tasks/checkAlerts/tests/checkAlerts.test.ts', 80, 0), + ]), 2); + }); + + it('touches core user/team models', () => { + assert.equal(classify('alice', 'feat/user-fields', [ + makeFile('packages/api/src/models/user.ts', 10, 2), + ]), 4); + }); + + it('escalates Tier 3 human branch past 1000 prod lines', () => { + assert.equal(classify('alice', 'feat/huge-refactor', [ + makeFile('packages/app/src/BigComponent.tsx', 600, 450), // 1050 lines + ]), 4); + }); + + it('escalates Tier 3 agent branch past 400 prod lines (stricter threshold)', () => { + assert.equal(classify('alice', 'claude/large-feature', [ + makeFile('packages/app/src/BigFeature.tsx', 300, 120), // 420 lines + ]), 4); + }); + }); + + describe('Tier 2', () => { + it('small single-layer frontend change', () => { + assert.equal(classify('alice', 'fix/button-style', [ + makeFile('packages/app/src/components/Button.tsx', 20, 10), + ]), 2); + }); + + it('small single-layer backend change (not models/routers)', () => { + assert.equal(classify('alice', 'fix/service-bug', [ + makeFile('packages/api/src/services/logs.ts', 30, 15), + ]), 2); + }); + + it('agent branch small enough to qualify (PR #1431 pattern: 1 file, 16 lines)', () => { + assert.equal(classify('mikeshi', 'claude/fix-mobile-nav', [ + makeFile('packages/app/src/AppNav.tsx', 11, 5), + ]), 2); + }); + + it('agent branch exactly at file limit (3 prod files, small lines)', () => { + const files = Array.from({ length: 3 }, (_, i) => + makeFile(`packages/app/src/File${i}.tsx`, 10, 5) + ); + assert.equal(classify('alice', 'claude/small-multi', files), 2); + }); + + it('human branch at 149 prod lines (just under threshold)', () => { + assert.equal(classify('alice', 'fix/component', [ + makeFile('packages/app/src/Foo.tsx', 100, 49), // 149 lines + ]), 2); + }); + + it('agent branch at exactly 49 prod lines qualifies for Tier 2', () => { + assert.equal(classify('alice', 'claude/fix', [ + makeFile('packages/app/src/Foo.tsx', 49, 0), + ]), 2); + }); + }); + + describe('Tier 3', () => { + it('cross-layer change (frontend + backend)', () => { + assert.equal(classify('alice', 'feat/new-feature', [ + makeFile('packages/app/src/NewFeature.tsx', 30, 5), + makeFile('packages/api/src/services/newFeature.ts', 40, 10), + ]), 3); + }); + + it('touches API routes (non-critical)', () => { + assert.equal(classify('alice', 'feat/new-route', [ + makeFile('packages/api/src/routers/logs.ts', 30, 5), + ]), 3); + }); + + it('touches API models (non-critical)', () => { + assert.equal(classify('alice', 'feat/model-field', [ + makeFile('packages/api/src/models/alert.ts', 20, 3), + ]), 3); + }); + + it('agent branch at exactly 50 prod lines is blocked from Tier 2', () => { + assert.equal(classify('alice', 'claude/feature', [ + makeFile('packages/app/src/Foo.tsx', 50, 0), // exactly AGENT_TIER2_MAX_LINES — >= blocks it + ]), 3); + }); + + it('agent branch over prod-line threshold (60 > 50) → Tier 3, not Tier 2', () => { + assert.equal(classify('alice', 'claude/medium-feature', [ + makeFile('packages/app/src/Foo.tsx', 60, 0), + ]), 3); + }); + + it('agent branch over file count threshold (4 files) → Tier 3', () => { + const files = Array.from({ length: 4 }, (_, i) => + makeFile(`packages/app/src/File${i}.tsx`, 10, 5) + ); + assert.equal(classify('alice', 'claude/big-feature', files), 3); + }); + + it('does NOT escalate agent branch at exactly 400 lines (threshold is exclusive)', () => { + // prodLines > threshold, not >=, so 400 stays at Tier 3 + assert.equal(classify('alice', 'claude/medium-large', [ + makeFile('packages/app/src/Feature.tsx', 200, 200), // exactly 400 + ]), 3); + }); + + it('large test additions with small prod change stay Tier 3 (PR #2122 pattern)', () => { + // Alert threshold PR: 1300 total adds but ~1100 are tests + const files = [ + makeFile('packages/api/src/services/checkAlerts.ts', 180, 70), // prod: 250 lines + makeFile('packages/api/src/__tests__/checkAlerts.test.ts', 1100, 0), // test: excluded + ]; + // 250 prod lines > TIER2_MAX_LINES (150) → Tier 3, not Tier 4 + assert.equal(classify('alice', 'feat/alert-thresholds', files), 3); + }); + + it('human branch at exactly 150 prod lines is Tier 3, not Tier 2', () => { + assert.equal(classify('alice', 'fix/component', [ + makeFile('packages/app/src/Foo.tsx', 100, 50), // exactly TIER2_MAX_LINES — < is exclusive + ]), 3); + }); + + it('does NOT escalate human branch at exactly 1000 prod lines', () => { + assert.equal(classify('alice', 'feat/medium-large', [ + makeFile('packages/app/src/Feature.tsx', 500, 500), // exactly 1000 + ]), 3); + }); + }); +}); + +// ── buildTierComment ───────────────────────────────────────────────────────── + +describe('buildTierComment', () => { + /** Build a signal object with sensible defaults, overrideable per test */ + function makeSignals(overrides = {}) { + return { + author: 'alice', + branchName: 'feature/foo', + prodFiles: [makeFile('packages/app/src/Foo.tsx')], + prodLines: 50, + testLines: 0, + criticalFiles: [], + isAgentBranch: false, + isBotAuthor: false, + allFilesTrivial: false, + touchesApiModels: false, + touchesFrontend: true, + touchesBackend: false, + touchesSharedUtils: false, + isCrossLayer: false, + agentBlocksTier2: false, + ...overrides, + }; + } + + it('always includes the pr-triage sentinel marker', () => { + assert.ok(buildTierComment(2, makeSignals()).includes('')); + }); + + it('includes the correct headline for each tier', () => { + assert.ok(buildTierComment(1, makeSignals()).includes('Tier 1')); + assert.ok(buildTierComment(2, makeSignals()).includes('Tier 2')); + assert.ok(buildTierComment(3, makeSignals()).includes('Tier 3')); + assert.ok(buildTierComment(4, makeSignals()).includes('Tier 4')); + }); + + it('includes override instructions with the correct tier label', () => { + const body = buildTierComment(3, makeSignals()); + assert.ok(body.includes('review/tier-3')); + assert.ok(body.includes('Manual overrides are preserved')); + }); + + it('lists critical files when present', () => { + const signals = makeSignals({ + criticalFiles: [makeFile('packages/api/src/middleware/auth.ts')], + }); + const body = buildTierComment(4, signals); + assert.ok(body.includes('Critical-path files')); + assert.ok(body.includes('auth.ts')); + }); + + it('explains cross-layer trigger with which layers are involved', () => { + const signals = makeSignals({ + isCrossLayer: true, + touchesFrontend: true, + touchesBackend: true, + touchesSharedUtils: false, + }); + const body = buildTierComment(3, signals); + assert.ok(body.includes('Cross-layer change')); + assert.ok(body.includes('packages/app')); + assert.ok(body.includes('packages/api')); + }); + + it('explains API model/route trigger', () => { + const body = buildTierComment(3, makeSignals({ touchesApiModels: true })); + assert.ok(body.includes('API routes or data models')); + }); + + it('explains agent branch bump to Tier 3', () => { + const signals = makeSignals({ + isAgentBranch: true, + agentBlocksTier2: true, + branchName: 'claude/big-feature', + prodLines: 80, + prodFiles: Array.from({ length: 5 }, (_, i) => makeFile(`packages/app/src/File${i}.tsx`)), + }); + const body = buildTierComment(3, signals); + assert.ok(body.includes('bumped to Tier 3')); + }); + + it('notes when agent branch is small enough for Tier 2', () => { + const signals = makeSignals({ + isAgentBranch: true, + agentBlocksTier2: false, + branchName: 'claude/tiny-fix', + }); + const body = buildTierComment(2, signals); + assert.ok(body.includes('small enough to qualify for Tier 2')); + }); + + it('shows test line count in stats when non-zero', () => { + const body = buildTierComment(2, makeSignals({ testLines: 200 })); + assert.ok(body.includes('200 in test files')); + }); + + it('omits test line note when testLines is 0', () => { + const body = buildTierComment(2, makeSignals({ testLines: 0 })); + assert.ok(!body.includes('test files')); + }); + + it('includes a catch-all trigger for standard Tier 3 PRs with no specific signals', () => { + const body = buildTierComment(3, makeSignals()); + assert.ok(body.includes('Standard feature/fix')); + }); + + it('includes bot-author trigger for Tier 1 bot PRs', () => { + const body = buildTierComment(1, makeSignals({ isBotAuthor: true, author: 'dependabot[bot]' })); + assert.ok(body.includes('Bot author')); + }); +}); diff --git a/.github/scripts/pr-triage-classify.js b/.github/scripts/pr-triage-classify.js new file mode 100644 index 00000000..5d501901 --- /dev/null +++ b/.github/scripts/pr-triage-classify.js @@ -0,0 +1,257 @@ +'use strict'; + +// ── File classification patterns ───────────────────────────────────────────── +const TIER4_PATTERNS = [ + /^packages\/api\/src\/middleware\/auth/, + /^packages\/api\/src\/routers\/api\/me\./, + /^packages\/api\/src\/routers\/api\/team\./, + /^packages\/api\/src\/routers\/external-api\//, + /^packages\/api\/src\/models\/(user|team|teamInvite)\./, + /^packages\/api\/src\/config\./, + /^packages\/api\/src\/tasks\//, + /^packages\/otel-collector\//, + /^docker\/otel-collector\//, + /^docker\/clickhouse\//, + /^docker\/hyperdx\//, + /^\.github\/workflows\/(main|release)\.yml$/, +]; + +const TIER1_PATTERNS = [ + /\.(md|txt|png|jpg|jpeg|gif|svg|ico)$/i, + /^yarn\.lock$/, + /^package-lock\.json$/, + /^\.yarnrc\.yml$/, + /^\.github\/images\//, + /^\.env\.example$/, + /^\.changeset\//, // version-bump config files; no functional code + /^\.github\/scripts\//, // GitHub Actions scripts; not application code + /^\.github\/workflows\//, // workflow files (main.yml/release.yml still caught by TIER4_PATTERNS) +]; + +const TEST_FILE_PATTERNS = [ + /\/__tests__\//, + /\.test\.[jt]sx?$/, + /\.spec\.[jt]sx?$/, + /^packages\/app\/tests\//, +]; + +// ── Thresholds (all line counts exclude test and trivial files) ─────────────── +const TIER2_MAX_LINES = 150; // max prod lines eligible for Tier 2 +const TIER4_ESCALATION_HUMAN = 1000; // Tier 3 → 4 for human branches +const TIER4_ESCALATION_AGENT = 400; // Tier 3 → 4 for agent branches (stricter) + +// Agent branches can reach Tier 2 only for very small, focused changes +const AGENT_TIER2_MAX_LINES = 50; +const AGENT_TIER2_MAX_PROD_FILES = 3; + +// ── Other constants ────────────────────────────────────────────────────────── +const BOT_AUTHORS = ['dependabot', 'dependabot[bot]']; +const AGENT_BRANCH_PREFIXES = ['claude/', 'agent/', 'ai/']; + +const TIER_LABELS = { + 1: { name: 'review/tier-1', color: '0E8A16', description: 'Trivial — auto-merge candidate once CI passes' }, + 2: { name: 'review/tier-2', color: '1D76DB', description: 'Low risk — AI review + quick human skim' }, + 3: { name: 'review/tier-3', color: 'E4E669', description: 'Standard — full human review required' }, + 4: { name: 'review/tier-4', color: 'B60205', description: 'Critical — deep review + domain expert sign-off' }, +}; + +const TIER_INFO = { + 1: { + emoji: '🟢', + headline: 'Tier 1 — Trivial', + detail: 'Docs, images, lock files, or a dependency bump. No functional code changes detected.', + process: 'Auto-merge once CI passes. No human review required.', + sla: 'Resolves automatically.', + }, + 2: { + emoji: '🔵', + headline: 'Tier 2 — Low Risk', + detail: 'Small, isolated change with no API route or data model modifications.', + process: 'AI review + quick human skim (target: 5–15 min). Reviewer validates AI assessment and checks for domain-specific concerns.', + sla: 'Resolve within 4 business hours.', + }, + 3: { + emoji: '🟡', + headline: 'Tier 3 — Standard', + detail: 'Introduces new logic, modifies core functionality, or touches areas with non-trivial risk.', + process: 'Full human review — logic, architecture, edge cases.', + sla: 'First-pass feedback within 1 business day.', + }, + 4: { + emoji: '🔴', + headline: 'Tier 4 — Critical', + detail: 'Touches auth, data models, config, tasks, OTel pipeline, ClickHouse, or CI/CD.', + process: 'Deep review from a domain expert. Synchronous walkthrough may be required.', + sla: 'Schedule synchronous review within 2 business days.', + }, +}; + +// ── File classification helpers ────────────────────────────────────────────── +const isTestFile = f => TEST_FILE_PATTERNS.some(p => p.test(f)); +const isTrivialFile = f => TIER1_PATTERNS.some(p => p.test(f)); +const isCriticalFile = f => TIER4_PATTERNS.some(p => p.test(f)); + +// ── Signal computation ─────────────────────────────────────────────────────── +// Returns a flat object of all facts needed for tier determination and comment +// generation. All derived from PR metadata + file list — no GitHub API calls. +// +// @param {object} pr - GitHub PR object: { user: { login }, head: { ref } } +// @param {Array} filesRes - GitHub files array: [{ filename, additions, deletions }] +function computeSignals(pr, filesRes) { + const author = pr.user.login; + const branchName = pr.head.ref; + + const testFiles = filesRes.filter(f => isTestFile(f.filename)); + const prodFiles = filesRes.filter(f => !isTestFile(f.filename) && !isTrivialFile(f.filename)); + const criticalFiles = filesRes.filter(f => isCriticalFile(f.filename) && !isTestFile(f.filename)); + + const prodLines = prodFiles.reduce((sum, f) => sum + f.additions + f.deletions, 0); + const testLines = testFiles.reduce((sum, f) => sum + f.additions + f.deletions, 0); + + const isAgentBranch = AGENT_BRANCH_PREFIXES.some(p => branchName.startsWith(p)); + const isBotAuthor = BOT_AUTHORS.includes(author); + const allFilesTrivial = filesRes.length > 0 && filesRes.every(f => isTrivialFile(f.filename)); + + // Blocks Tier 2 — API models and routes carry implicit cross-cutting risk + const touchesApiModels = prodFiles.some(f => + f.filename.startsWith('packages/api/src/models/') || + f.filename.startsWith('packages/api/src/routers/') + ); + + // Cross-layer: production changes spanning multiple monorepo packages + const touchesFrontend = prodFiles.some(f => f.filename.startsWith('packages/app/')); + const touchesBackend = prodFiles.some(f => f.filename.startsWith('packages/api/')); + const touchesSharedUtils = prodFiles.some(f => f.filename.startsWith('packages/common-utils/')); + const isCrossLayer = [touchesFrontend, touchesBackend, touchesSharedUtils].filter(Boolean).length >= 2; + + // Agent branches can reach Tier 2 only when the change is very small and focused + const agentBlocksTier2 = isAgentBranch && + (prodLines >= AGENT_TIER2_MAX_LINES || prodFiles.length > AGENT_TIER2_MAX_PROD_FILES); + + return { + author, branchName, + prodFiles, prodLines, testLines, criticalFiles, + isAgentBranch, isBotAuthor, allFilesTrivial, + touchesApiModels, touchesFrontend, touchesBackend, touchesSharedUtils, + isCrossLayer, agentBlocksTier2, + }; +} + +// ── Tier determination ─────────────────────────────────────────────────────── +// @param {object} signals - output of computeSignals() +// @returns {number} tier - 1 | 2 | 3 | 4 +function determineTier(signals) { + const { + criticalFiles, isBotAuthor, allFilesTrivial, + prodLines, touchesApiModels, isCrossLayer, agentBlocksTier2, isAgentBranch, + } = signals; + + // Tier 4: touches critical infrastructure (auth, config, pipeline, CI/CD) + if (criticalFiles.length > 0) return 4; + + // Tier 1: bot-authored or only docs/images/lock files changed + if (isBotAuthor || allFilesTrivial) return 1; + + // Tier 2: small, isolated, single-layer change + // Agent branches qualify when the change is very small and focused + // (agentBlocksTier2 is false when under AGENT_TIER2_MAX_LINES / MAX_PROD_FILES) + const qualifiesForTier2 = + prodLines < TIER2_MAX_LINES && + !touchesApiModels && + !isCrossLayer && + !agentBlocksTier2; + if (qualifiesForTier2) return 2; + + // Tier 3: everything else — escalate very large diffs to Tier 4 + const sizeThreshold = isAgentBranch ? TIER4_ESCALATION_AGENT : TIER4_ESCALATION_HUMAN; + return prodLines > sizeThreshold ? 4 : 3; +} + +// ── Comment generation ─────────────────────────────────────────────────────── +// @param {number} tier - 1 | 2 | 3 | 4 +// @param {object} signals - output of computeSignals() +// @returns {string} - Markdown comment body +function buildTierComment(tier, signals) { + const { + author, branchName, + prodFiles, prodLines, testLines, criticalFiles, + isAgentBranch, isBotAuthor, allFilesTrivial, + touchesApiModels, touchesFrontend, touchesBackend, touchesSharedUtils, + isCrossLayer, agentBlocksTier2, + } = signals; + + const info = TIER_INFO[tier]; + const sizeThreshold = isAgentBranch ? TIER4_ESCALATION_AGENT : TIER4_ESCALATION_HUMAN; + + // Primary triggers — the specific reasons this tier was assigned + const triggers = []; + if (criticalFiles.length > 0) { + triggers.push(`**Critical-path files** (${criticalFiles.length}):\n${criticalFiles.map(f => ` - \`${f.filename}\``).join('\n')}`); + } + if (tier === 4 && prodLines > sizeThreshold && criticalFiles.length === 0) { + triggers.push(`**Large diff**: ${prodLines} production lines changed (threshold: ${sizeThreshold})`); + } + if (isBotAuthor) triggers.push(`**Bot author**: \`${author}\``); + if (allFilesTrivial && !isBotAuthor) triggers.push('**All files are docs / images / lock files**'); + if (isCrossLayer) { + const layers = [ + touchesFrontend && 'frontend (`packages/app`)', + touchesBackend && 'backend (`packages/api`)', + touchesSharedUtils && 'shared utils (`packages/common-utils`)', + ].filter(Boolean); + triggers.push(`**Cross-layer change**: touches ${layers.join(' + ')}`); + } + if (touchesApiModels && criticalFiles.length === 0) { + triggers.push('**Touches API routes or data models** — hidden complexity risk'); + } + if (isAgentBranch && agentBlocksTier2 && tier <= 3) { + triggers.push(`**Agent-generated branch** (\`${branchName}\`) with ${prodLines} prod lines across ${prodFiles.length} files — bumped to Tier 3 for mandatory human review`); + } + if (triggers.length === 0) { + triggers.push('**Standard feature/fix** — introduces new logic or modifies core functionality'); + } + + // Informational context — didn't drive the tier on their own + const contextSignals = []; + if (isAgentBranch && !agentBlocksTier2 && tier === 2) { + contextSignals.push(`agent branch (\`${branchName}\`) — change small enough to qualify for Tier 2`); + } else if (isAgentBranch && tier === 4) { + contextSignals.push(`agent branch (\`${branchName}\`)`); + } + + const triggerSection = `\n**Why this tier:**\n${triggers.map(t => `- ${t}`).join('\n')}`; + const contextSection = contextSignals.length > 0 + ? `\n**Additional context:** ${contextSignals.join(', ')}` + : ''; + + return [ + '', + `## ${info.emoji} ${info.headline}`, + '', + info.detail, + triggerSection, + contextSection, + '', + `**Review process**: ${info.process}`, + `**SLA**: ${info.sla}`, + '', + '
Stats', + '', + `- Production files changed: ${prodFiles.length}`, + `- Production lines changed: ${prodLines}${testLines > 0 ? ` (+ ${testLines} in test files, excluded from tier calculation)` : ''}`, + `- Branch: \`${branchName}\``, + `- Author: ${author}`, + '', + '
', + '', + `> To override this classification, remove the \`${TIER_LABELS[tier].name}\` label and apply a different \`review/tier-*\` label. Manual overrides are preserved on subsequent pushes.`, + ].join('\n'); +} + +module.exports = { + // Constants needed by the orchestration script + TIER_LABELS, TIER_INFO, + // Pure functions + isTestFile, isTrivialFile, isCriticalFile, + computeSignals, determineTier, buildTierComment, +}; diff --git a/.github/scripts/pr-triage.js b/.github/scripts/pr-triage.js new file mode 100644 index 00000000..ac570a5a --- /dev/null +++ b/.github/scripts/pr-triage.js @@ -0,0 +1,123 @@ +'use strict'; + +// Entry point for actions/github-script@v7 via script-path. +// Pure classification logic lives in pr-triage-classify.js so it can be +// unit-tested without GitHub API machinery. + +const { + TIER_LABELS, + computeSignals, determineTier, buildTierComment, +} = require('./pr-triage-classify'); + +module.exports = async ({ github, context }) => { + const owner = context.repo.owner; + const repo = context.repo.repo; + + // ── Determine which PRs to process ────────────────────────────────────── + let prNumbers; + if (context.eventName === 'workflow_dispatch') { + // Use context.payload.inputs to avoid script-injection via template interpolation + const input = (context.payload.inputs?.pr_number ?? '').trim(); + if (input !== '') { + const num = Number(input); + if (!Number.isInteger(num) || num <= 0) { + throw new Error(`Invalid PR number: "${input}"`); + } + prNumbers = [num]; + } else { + const openPRs = await github.paginate( + github.rest.pulls.list, + { owner, repo, state: 'open', per_page: 100 } + ); + prNumbers = openPRs.map(pr => pr.number); + console.log(`Bulk triage: found ${prNumbers.length} open PRs`); + } + } else { + prNumbers = [context.payload.pull_request.number]; + } + + // ── Ensure tier labels exist (once, before the loop) ──────────────────── + const repoLabels = await github.paginate( + github.rest.issues.listLabelsForRepo, + { owner, repo, per_page: 100 } + ); + const repoLabelNames = new Set(repoLabels.map(l => l.name)); + for (const label of Object.values(TIER_LABELS)) { + if (!repoLabelNames.has(label.name)) { + await github.rest.issues.createLabel({ owner, repo, ...label }); + repoLabelNames.add(label.name); + } + } + + // ── Classify a single PR ───────────────────────────────────────────────── + async function classifyPR(prNumber) { + const filesRes = await github.paginate( + github.rest.pulls.listFiles, + { owner, repo, pull_number: prNumber, per_page: 100 } + ); + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); + const { data: currentLabels } = await github.rest.issues.listLabelsOnIssue({ owner, repo, issue_number: prNumber }); + const currentLabelNames = new Set(currentLabels.map(l => l.name)); + + // Skip drafts (bulk mode; PR events already filter these via the job condition) + if (pr.draft) { + console.log(`PR #${prNumber}: skipping draft`); + return; + } + + // Respect manual tier overrides — don't overwrite labels applied by humans + const existingTierLabel = currentLabels.find(l => l.name.startsWith('review/tier-')); + if (existingTierLabel) { + const events = await github.paginate( + github.rest.issues.listEvents, + { owner, repo, issue_number: prNumber, per_page: 100 } + ); + const lastLabelEvent = events + .filter(e => e.event === 'labeled' && e.label?.name === existingTierLabel.name) + .pop(); + if (lastLabelEvent && lastLabelEvent.actor.type !== 'Bot') { + console.log(`PR #${prNumber}: tier manually set to ${existingTierLabel.name} by ${lastLabelEvent.actor.login} — skipping`); + return; + } + } + + const signals = computeSignals(pr, filesRes); + const tier = determineTier(signals); + const body = buildTierComment(tier, signals); + + // Apply the tier label (remove any stale tier label first) + for (const label of currentLabels) { + if (label.name.startsWith('review/tier-') && label.name !== TIER_LABELS[tier].name) { + await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name: label.name }); + } + } + if (!currentLabelNames.has(TIER_LABELS[tier].name)) { + await github.rest.issues.addLabels({ owner, repo, issue_number: prNumber, labels: [TIER_LABELS[tier].name] }); + } + + // Post or update the triage comment + const comments = await github.paginate( + github.rest.issues.listComments, + { owner, repo, issue_number: prNumber, per_page: 100 } + ); + const existingComment = comments.find( + c => c.user.login === 'github-actions[bot]' && c.body.includes('') + ); + if (existingComment) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existingComment.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body }); + } + + console.log(`PR #${prNumber}: Tier ${tier} (${signals.prodLines} prod lines, ${signals.prodFiles.length} prod files, ${signals.testLines} test lines)`); + } + + // ── Process all target PRs ─────────────────────────────────────────────── + for (const prNumber of prNumbers) { + try { + await classifyPR(prNumber); + } catch (err) { + console.error(`PR #${prNumber}: classification failed — ${err.message}`); + } + } +}; diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 28f3013e..355d70d7 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -51,4 +51,4 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} use_sticky_comment: 'true' include_fix_links: 'true' - claude_args: '--max-turns 20' + claude_args: '--max-turns 60' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b60478a2..f7b7bec8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,7 +47,7 @@ jobs: - name: Run unit tests run: make ci-unit integration: - timeout-minutes: 8 + timeout-minutes: 16 runs-on: ubuntu-24.04 steps: - name: Checkout @@ -93,7 +93,7 @@ jobs: working-directory: ./packages/otel-collector run: go test ./... otel-smoke-test: - timeout-minutes: 8 + timeout-minutes: 16 runs-on: ubuntu-24.04 steps: - name: Checkout diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml index 3f697080..1ed6dd72 100644 --- a/.github/workflows/pr-triage.yml +++ b/.github/workflows/pr-triage.yml @@ -2,6 +2,7 @@ name: PR Triage on: pull_request: + branches: [main] types: [opened, synchronize, reopened, ready_for_review] workflow_dispatch: inputs: @@ -16,246 +17,39 @@ permissions: pull-requests: write issues: write +concurrency: + group: + ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id + }} + cancel-in-progress: true + jobs: + test: + name: Test triage logic + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + - run: node --test .github/scripts/__tests__/pr-triage-classify.test.js + classify: name: Classify PR risk tier + needs: test runs-on: ubuntu-24.04 + timeout-minutes: 8 # For pull_request events skip drafts; workflow_dispatch always runs if: ${{ github.event_name == 'workflow_dispatch' || !github.event.pull_request.draft }} steps: + - uses: actions/checkout@v4 - name: Classify and label PR(s) uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const owner = context.repo.owner; - const repo = context.repo.repo; - - // ── Determine which PRs to process ────────────────────────────── - let prNumbers; - if (context.eventName === 'workflow_dispatch') { - // Use context.payload.inputs to avoid script-injection via template interpolation - const input = (context.payload.inputs?.pr_number ?? '').trim(); - if (input && input !== '') { - prNumbers = [Number(input)]; - } else { - const openPRs = await github.paginate( - github.rest.pulls.list, - { owner, repo, state: 'open', per_page: 100 } - ); - prNumbers = openPRs.map(pr => pr.number); - console.log(`Bulk triage: found ${prNumbers.length} open PRs`); - } - } else { - prNumbers = [context.payload.pull_request.number]; - } - - // ── Shared constants ───────────────────────────────────────────── - const TIER4_PATTERNS = [ - /^packages\/api\/src\/middleware\/auth/, - /^packages\/api\/src\/routers\/api\/me\./, - /^packages\/api\/src\/routers\/api\/team\./, - /^packages\/api\/src\/routers\/external-api\//, - /^packages\/api\/src\/models\/(user|team|teamInvite)\./, - /^packages\/api\/src\/config\./, - /^packages\/api\/src\/tasks\//, - /^packages\/otel-collector\//, - /^docker\/otel-collector\//, - /^docker\/clickhouse\//, - /^\.github\/workflows\//, - ]; - - const TIER1_PATTERNS = [ - /\.(md|txt|png|jpg|jpeg|gif|svg|ico)$/i, - /^yarn\.lock$/, - /^package-lock\.json$/, - /^\.yarnrc\.yml$/, - /^\.github\/images\//, - /^\.env\.example$/, - ]; - - const BOT_AUTHORS = ['dependabot', 'dependabot[bot]']; - const AGENT_BRANCH_PREFIXES = ['claude/', 'agent/', 'ai/']; - - const TIER_LABELS = { - 1: { name: 'review/tier-1', color: '0E8A16', description: 'Trivial — auto-merge candidate once CI passes' }, - 2: { name: 'review/tier-2', color: '1D76DB', description: 'Low risk — AI review + quick human skim' }, - 3: { name: 'review/tier-3', color: 'E4E669', description: 'Standard — full human review required' }, - 4: { name: 'review/tier-4', color: 'B60205', description: 'Critical — deep review + domain expert sign-off' }, - }; - - const TIER_INFO = { - 1: { - emoji: '🟢', - headline: 'Tier 1 — Trivial', - detail: 'Docs, images, lock files, or a dependency bump. No functional code changes detected.', - process: 'Auto-merge once CI passes. No human review required.', - sla: 'Resolves automatically.', - }, - 2: { - emoji: '🔵', - headline: 'Tier 2 — Low Risk', - detail: 'Small, isolated change with no API route or data model modifications.', - process: 'AI review + quick human skim (target: 5–15 min). Reviewer validates AI assessment and checks for domain-specific concerns.', - sla: 'Resolve within 4 business hours.', - }, - 3: { - emoji: '🟡', - headline: 'Tier 3 — Standard', - detail: 'Introduces new logic, modifies core functionality, or touches areas with non-trivial risk.', - process: 'Full human review — logic, architecture, edge cases.', - sla: 'First-pass feedback within 1 business day.', - }, - 4: { - emoji: '🔴', - headline: 'Tier 4 — Critical', - detail: 'Touches auth, data models, config, tasks, OTel pipeline, ClickHouse, or CI/CD.', - process: 'Deep review from a domain expert. Synchronous walkthrough may be required.', - sla: 'Schedule synchronous review within 2 business days.', - }, - }; - - // ── Ensure tier labels exist (once, before the loop) ───────────── - const repoLabels = await github.paginate( - github.rest.issues.listLabelsForRepo, - { owner, repo, per_page: 100 } - ); - const repoLabelNames = new Set(repoLabels.map(l => l.name)); - for (const label of Object.values(TIER_LABELS)) { - if (!repoLabelNames.has(label.name)) { - await github.rest.issues.createLabel({ owner, repo, ...label }); - repoLabelNames.add(label.name); - } - } - - // ── Classify a single PR ───────────────────────────────────────── - async function classifyPR(prNumber) { - // Fetch changed files - const filesRes = await github.paginate( - github.rest.pulls.listFiles, - { owner, repo, pull_number: prNumber, per_page: 100 } - ); - const files = filesRes.map(f => f.filename); - const linesChanged = filesRes.reduce((sum, f) => sum + f.additions + f.deletions, 0); - - // Fetch PR metadata - const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); - const author = pr.user.login; - const branchName = pr.head.ref; - - // Skip drafts when running in bulk mode - if (pr.draft) { - console.log(`Skipping PR #${prNumber}: draft`); - return; - } - - // Check for manual tier override — if a human last applied the label, respect it - const { data: currentLabels } = await github.rest.issues.listLabelsOnIssue({ owner, repo, issue_number: prNumber }); - const existingTierLabel = currentLabels.find(l => l.name.startsWith('review/tier-')); - if (existingTierLabel) { - const events = await github.paginate( - github.rest.issues.listEvents, - { owner, repo, issue_number: prNumber, per_page: 100 } - ); - const lastLabelEvent = events - .filter(e => e.event === 'labeled' && e.label?.name === existingTierLabel.name) - .pop(); - if (lastLabelEvent && lastLabelEvent.actor.type !== 'Bot') { - console.log(`PR #${prNumber}: tier manually set to ${existingTierLabel.name} by ${lastLabelEvent.actor.login} — skipping`); - return; - } - } - - // Classify - const isTier4 = files.some(f => TIER4_PATTERNS.some(p => p.test(f))); - const isTrivialAuthor = BOT_AUTHORS.includes(author); - const allFilesTrivial = files.length > 0 && files.every(f => TIER1_PATTERNS.some(p => p.test(f))); - const isTier1 = isTrivialAuthor || allFilesTrivial; - const isAgentBranch = AGENT_BRANCH_PREFIXES.some(p => branchName.startsWith(p)); - const touchesApiModels = files.some(f => - f.startsWith('packages/api/src/models/') || f.startsWith('packages/api/src/routers/') - ); - const isSmallDiff = linesChanged < 100; - // Agent branches are bumped to Tier 3 regardless of size to ensure human review - const isTier2 = !isTier4 && !isTier1 && isSmallDiff && !touchesApiModels && !isAgentBranch; - - let tier; - if (isTier4) tier = 4; - else if (isTier1) tier = 1; - else if (isTier2) tier = 2; - else tier = 3; - - // Escalate very large non-critical PRs to Tier 4; this also applies to agent - // branches that were bumped to Tier 3 above — a 400+ line agent-generated change - // warrants deep review regardless of which files it touches. - if (tier === 3 && linesChanged > 400) tier = 4; - - // Apply label - for (const existing of currentLabels) { - if (existing.name.startsWith('review/tier-') && existing.name !== TIER_LABELS[tier].name) { - await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name: existing.name }); - } - } - if (!currentLabels.find(l => l.name === TIER_LABELS[tier].name)) { - await github.rest.issues.addLabels({ owner, repo, issue_number: prNumber, labels: [TIER_LABELS[tier].name] }); - } - - // Build comment body - const info = TIER_INFO[tier]; - const signals = []; - if (isTier4) signals.push('critical-path files detected'); - if (isAgentBranch) signals.push(`agent branch (\`${branchName}\`)`); - if (linesChanged > 400) signals.push(`large diff (${linesChanged} lines changed)`); - if (isTrivialAuthor) signals.push(`bot author (${author})`); - if (allFilesTrivial && !isTrivialAuthor) signals.push('all files are docs/images/lock files'); - if (touchesApiModels) signals.push('API routes or data models changed'); - - const signalList = signals.length > 0 ? `\n**Signals**: ${signals.join(', ')}` : ''; - - const body = [ - '', - `## ${info.emoji} ${info.headline}`, - '', - info.detail, - signalList, - '', - `**Review process**: ${info.process}`, - `**SLA**: ${info.sla}`, - '', - `
Stats`, - '', - `- Files changed: ${files.length}`, - `- Lines changed: ${linesChanged}`, - `- Branch: \`${branchName}\``, - `- Author: ${author}`, - '', - '
', - '', - `> To override this classification, remove the \`${TIER_LABELS[tier].name}\` label and apply a different \`review/tier-*\` label. Manual overrides are preserved on subsequent pushes.`, - ].join('\n'); - - // Post or update the single triage comment - const comments = await github.paginate( - github.rest.issues.listComments, - { owner, repo, issue_number: prNumber, per_page: 100 } - ); - const existing = comments.find(c => c.user.login === 'github-actions[bot]' && c.body.includes('')); - if (existing) { - await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); - } else { - await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body }); - } - - console.log(`PR #${prNumber}: Tier ${tier} (${linesChanged} lines, ${files.length} files)`); - } - - // ── Process all target PRs ─────────────────────────────────────── - for (const prNumber of prNumbers) { - try { - await classifyPR(prNumber); - } catch (err) { - console.error(`PR #${prNumber}: classification failed — ${err.message}`); - } - } + const path = require('path'); + const run = require(path.join(process.env.GITHUB_WORKSPACE, '.github/scripts/pr-triage.js')); + await run({ github, context }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc147716..3e79c784 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -418,6 +418,106 @@ jobs: "${IMAGE}:${VERSION}-arm64" done + # --------------------------------------------------------------------------- + # CLI – compile standalone binaries and upload as GitHub Release assets + # npm publishing is handled by changesets in the check_changesets job above. + # This job only compiles platform-specific binaries and creates a GH Release. + # --------------------------------------------------------------------------- + release-cli: + name: Release CLI Binaries + needs: [check_changesets, check_version] + if: needs.check_version.outputs.should_release == 'true' + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache-dependency-path: 'yarn.lock' + cache: 'yarn' + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.11' + - name: Install dependencies + run: yarn install + - name: Build common-utils + run: make ci-build + - name: Get CLI version + id: cli_version + run: | + CLI_VERSION=$(node -p "require('./packages/cli/package.json').version") + echo "version=${CLI_VERSION}" >> $GITHUB_OUTPUT + echo "CLI version: ${CLI_VERSION}" + - name: Check if CLI release already exists + id: check_cli_release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if gh release view "cli-v${{ steps.cli_version.outputs.version }}" > /dev/null 2>&1; then + echo "Release cli-v${{ steps.cli_version.outputs.version }} already exists. Skipping." + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "Release does not exist. Proceeding." + echo "exists=false" >> $GITHUB_OUTPUT + fi + - name: Compile CLI binaries + if: steps.check_cli_release.outputs.exists == 'false' + working-directory: packages/cli + run: | + yarn compile:linux + yarn compile:macos + yarn compile:macos-x64 + - name: Create GitHub Release + if: steps.check_cli_release.outputs.exists == 'false' + uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 + with: + tag_name: cli-v${{ steps.cli_version.outputs.version }} + name: '@hyperdx/cli v${{ steps.cli_version.outputs.version }}' + body: | + ## @hyperdx/cli v${{ steps.cli_version.outputs.version }} + + ### Installation + + **npm (recommended):** + ```bash + npm install -g @hyperdx/cli + ``` + + **Or run directly with npx:** + ```bash + npx @hyperdx/cli tui -s + ``` + + **Manual download (standalone binary, no Node.js required):** + ```bash + # macOS Apple Silicon + curl -L https://github.com/hyperdxio/hyperdx/releases/download/cli-v${{ steps.cli_version.outputs.version }}/hdx-darwin-arm64 -o hdx + # macOS Intel + curl -L https://github.com/hyperdxio/hyperdx/releases/download/cli-v${{ steps.cli_version.outputs.version }}/hdx-darwin-x64 -o hdx + # Linux x64 + curl -L https://github.com/hyperdxio/hyperdx/releases/download/cli-v${{ steps.cli_version.outputs.version }}/hdx-linux-x64 -o hdx + + chmod +x hdx && sudo mv hdx /usr/local/bin/ + ``` + + ### Usage + + ```bash + hdx auth login -s + hdx tui + ``` + draft: false + prerelease: false + files: | + packages/cli/dist/hdx-linux-x64 + packages/cli/dist/hdx-darwin-arm64 + packages/cli/dist/hdx-darwin-x64 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # --------------------------------------------------------------------------- # Downstream notifications # --------------------------------------------------------------------------- @@ -553,6 +653,7 @@ jobs: publish-otel-collector, publish-local, publish-all-in-one, + release-cli, notify_helm_charts, notify_ch, notify_clickhouse_clickstack, diff --git a/AGENTS.md b/AGENTS.md index 9b32d055..a3396c37 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -229,6 +229,49 @@ formatting checks pass. Fix any issues before creating the commit. manual intervention rather than guessing. A wrong guess silently breaks things; asking is always cheaper than debugging later. +## Cursor Cloud specific instructions + +### Docker requirement + +Docker must be installed and running before starting the dev stack or running +integration/E2E tests. The VM update script handles `yarn install` and +`yarn build:common-utils`, but Docker daemon startup is a prerequisite that must +already be available. + +### Starting the dev stack + +`yarn dev` uses `sh -c` to source `scripts/dev-env.sh`, which contains +bash-specific syntax (`BASH_SOURCE`). On systems where `/bin/sh` is `dash` +(e.g. Ubuntu), this fails with "Bad substitution". Work around it by running +with bash directly: + +```bash +bash -c 'export PATH="/workspace/node_modules/.bin:$PATH" && source ./scripts/dev-env.sh && yarn build:common-utils && dotenvx run --convention=nextjs -- docker compose -p "$HDX_DEV_PROJECT" -f docker-compose.dev.yml up -d && yarn app:dev' +``` + +Port isolation assigns a slot based on the worktree directory name. In the +default `/workspace` directory, the slot is **76**, so services are at: + +- **App**: http://localhost:30276 +- **API**: http://localhost:30176 +- **ClickHouse**: http://localhost:30576 +- **MongoDB**: localhost:30476 + +### Key commands reference + +See `AGENTS.md` above and `agent_docs/development.md` for the full command +reference. Quick summary: + +- `make ci-lint` — lint + TypeScript type check +- `make ci-unit` — unit tests (all packages) +- `make dev-int FILE=` — integration tests (spins up Docker services) +- `make dev-e2e FILE=` — E2E tests (Playwright) + +### First-time registration + +When the dev stack starts fresh (empty MongoDB), the app shows a registration +page. Create any account to get started — no external auth provider is needed. + --- _Need more details? Check the `agent_docs/` directory or ask which documentation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d445729..9341914e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -125,7 +125,11 @@ yarn dev:unit ## AI-Assisted Development -The repo ships with configuration for AI coding assistants that enables interactive browser-based E2E test generation and debugging via the [Playwright MCP server](https://github.com/microsoft/playwright-mcp). +HyperDX includes an [MCP server](https://modelcontextprotocol.io/) that lets AI assistants query observability data, manage dashboards, and +explore data sources. See [MCP.md](/MCP.md) for setup instructions. + +The repo also ships with configuration for AI coding assistants that enables interactive browser-based E2E test generation and debugging via +the [Playwright MCP server](https://github.com/microsoft/playwright-mcp). ### Claude Code diff --git a/MCP.md b/MCP.md new file mode 100644 index 00000000..104b457f --- /dev/null +++ b/MCP.md @@ -0,0 +1,89 @@ +# HyperDX MCP Server + +HyperDX exposes a [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) server that lets AI assistants query your observability +data, manage dashboards, and explore data sources directly. + +## Prerequisites + +- A running HyperDX instance (see [CONTRIBUTING.md](/CONTRIBUTING.md) for local development setup, or [DEPLOY.md](/DEPLOY.md) for + self-hosted deployment) +- A **Personal API Access Key** — find yours in the HyperDX UI under **Team Settings > API Keys > Personal API Access Key** + +## Endpoint + +The MCP server is available at the `/api/mcp` path on your HyperDX instance. For local development this is: + +``` +http://localhost:8080/api/mcp +``` + +Replace `localhost:8080` with your instance's host and port if you've customized the defaults. + +## Connecting an MCP Client + +The MCP server uses the **Streamable HTTP** transport with Bearer token authentication. In the examples below, replace `` +with your instance URL (e.g. `http://localhost:8080`). + +### Claude Code + +```bash +claude mcp add --transport http hyperdx /api/mcp \ + --header "Authorization: Bearer " +``` + +### OpenCode + +```bash +opencode mcp add --transport http hyperdx /api/mcp \ + --header "Authorization: Bearer " +``` + +### Cursor + +Add the following to `.cursor/mcp.json` in your project (or your global Cursor settings): + +```json +{ + "mcpServers": { + "hyperdx": { + "url": "/api/mcp", + "headers": { + "Authorization": "Bearer " + } + } + } +} +``` + +### MCP Inspector + +The MCP Inspector is useful for interactively testing and debugging the server. + +```bash +cd packages/api && yarn dev:mcp +``` + +Then configure the inspector: + +1. **Transport Type:** Streamable HTTP +2. **URL:** `/api/mcp` +3. **Authentication:** Header `Authorization` with value `Bearer ` +4. Click **Connect** + +### Other Clients + +Any MCP client that supports Streamable HTTP transport can connect. Configure it with: + +- **URL:** `/api/mcp` +- **Header:** `Authorization: Bearer ` + +## Available Tools + +| Tool | Description | +| -------------------------- | -------------------------------------------------------------------------------------------- | +| `hyperdx_list_sources` | List all data sources and database connections, including column schemas and attribute keys | +| `hyperdx_query` | Query observability data (logs, metrics, traces) using builder mode, search mode, or raw SQL | +| `hyperdx_get_dashboard` | List all dashboards or get full detail for a specific dashboard | +| `hyperdx_save_dashboard` | Create or update a dashboard with tiles (charts, tables, numbers, search, markdown) | +| `hyperdx_delete_dashboard` | Permanently delete a dashboard and its attached alerts | +| `hyperdx_query_tile` | Execute the query for a specific dashboard tile to validate results | diff --git a/Makefile b/Makefile index 258fb04d..413696d3 100644 --- a/Makefile +++ b/Makefile @@ -187,6 +187,10 @@ dev-unit: ci-unit: npx nx run-many -t ci:unit +.PHONY: ci-triage +ci-triage: + node --test .github/scripts/__tests__/pr-triage-classify.test.js + # --------------------------------------------------------------------------- # E2E tests — port isolation is handled by scripts/test-e2e.sh # --------------------------------------------------------------------------- diff --git a/agent_docs/code_style.md b/agent_docs/code_style.md index 535ca9f8..fd1c3c92 100644 --- a/agent_docs/code_style.md +++ b/agent_docs/code_style.md @@ -93,6 +93,34 @@ The project uses Mantine UI with **custom variants** defined in `packages/app/sr This pattern cannot be enforced by ESLint and requires manual code review. +### EmptyState Component (REQUIRED) + +**Use `EmptyState` (`@/components/EmptyState`) for all empty/no-data states.** Do not create ad-hoc inline empty states. + +| Prop | Type | Default | Description | +|------|------|---------|-------------| +| `icon` | `ReactNode` | — | Icon in the theme circle (hidden if not provided) | +| `title` | `string` | — | Heading text (headline style — no trailing period) | +| `description` | `ReactNode` | — | Subtext below the title | +| `children` | `ReactNode` | — | Actions (buttons, links) below description | +| `variant` | `"default" \| "card"` | `"default"` | `"card"` wraps in a bordered Paper | + +```tsx +// ❌ BAD - ad-hoc inline empty states +
No data
+Nothing here + +// ✅ GOOD - use the EmptyState component +} + title="No alerts created yet" + description="Create alerts from dashboard charts or saved searches." + variant="card" +/> +``` + +**Title copy**: Treat `title` as a short headline (like `Title` in the UI). Do **not** end it with a period. Use `description` for full sentences, which should use normal punctuation including a trailing period when appropriate. Match listing pages (e.g. dashboards and saved searches use parallel phrasing such as “No matching … yet” / “No … yet” without dots). + ## Refactoring - Edit files directly - don't create `component-v2.tsx` copies diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 42a35111..525209c8 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -5,6 +5,9 @@ services: context: . dockerfile: docker/otel-collector/Dockerfile target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 35e640a0..dd2e5747 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -26,6 +26,9 @@ services: context: . dockerfile: docker/otel-collector/Dockerfile target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363' @@ -63,6 +66,9 @@ services: context: . dockerfile: docker/otel-collector/Dockerfile target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363' @@ -74,9 +80,9 @@ services: CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml' # Uncomment to enable stdout logging for the OTel collector OTEL_SUPERVISOR_LOGS: 'true' - # Uncomment to enable JSON schema in ClickHouse + # Enable JSON schema in the ClickHouse exporter (per-exporter config) # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server - OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json' + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE: 'true' volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl @@ -103,7 +109,7 @@ services: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE} # Set to 'true' to allow for proper OTel JSON Schema creation - # Be sure to also set the OTEL_AGENT_FEATURE_GATE_ARG env in otel-collector + # Be sure to also set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE in otel-collector # BETA_CH_OTEL_JSON_SCHEMA_ENABLED: 'true' volumes: - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 0447dda7..d7ffaea7 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -9,15 +9,31 @@ ARG NODE_VERSION=22.22 ARG CLICKHOUSE_VERSION=26.1 -ARG OTEL_COLLECTOR_VERSION=0.147.0 -ARG OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION=0.147.0 +ARG OTEL_COLLECTOR_VERSION=0.149.0 +ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0 # base ############################################################################################# # == Otel Collector Image == -FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_collector_base -FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION} AS otel_collector_opampsupervisor_base +FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS otel_collector_opampsupervisor_base FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose +# Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder). +# This replaces the pre-built otel/opentelemetry-collector-contrib image so we can +# include custom receiver/processor components alongside the standard contrib ones. +# Note: The official OCB image may ship an older Go than the contrib modules require, +# so we copy the ocb binary into a golang base with a newer toolchain. +FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin +FROM golang:1.26-alpine AS ocb-builder +ARG OTEL_COLLECTOR_VERSION +ARG OTEL_COLLECTOR_CORE_VERSION +COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb +WORKDIR /build +COPY packages/otel-collector/builder-config.yaml . +RUN --mount=type=cache,target=/go/pkg/mod,id=ocb-gomod \ + --mount=type=cache,target=/root/.cache/go-build,id=ocb-gobuild \ + sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \ + CGO_ENABLED=0 ocb --config=builder-config.yaml + # Build the Go migration tool with full TLS support for ClickHouse FROM golang:1.26-alpine AS migrate-builder WORKDIR /build @@ -132,7 +148,7 @@ ARG USER_GID=10001 ENV CODE_VERSION=$CODE_VERSION ENV OTEL_RESOURCE_ATTRIBUTES="service.version=$CODE_VERSION" # Copy from otel collector bases -COPY --from=otel_collector_base --chmod=755 /otelcol-contrib /otelcontribcol +COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol COPY --from=otel_collector_opampsupervisor_base --chmod=755 /usr/local/bin/opampsupervisor /usr/local/bin/opampsupervisor # Copy Node.js runtime from node base diff --git a/docker/otel-collector/Dockerfile b/docker/otel-collector/Dockerfile index b5ab2906..1d69a089 100644 --- a/docker/otel-collector/Dockerfile +++ b/docker/otel-collector/Dockerfile @@ -1,12 +1,31 @@ ## base ############################################################################################# -FROM otel/opentelemetry-collector-contrib:0.147.0 AS col -FROM otel/opentelemetry-collector-opampsupervisor:0.147.0 AS supervisor +ARG OTEL_COLLECTOR_VERSION=0.149.0 +ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0 + +FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS supervisor FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose -# Build the Go migration tool with full TLS support for ClickHouse +# Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder). +# This replaces the pre-built otel/opentelemetry-collector-contrib image so we can +# include custom receiver/processor components alongside the standard contrib ones. # Note: Build context must be repo root (use: docker build -f docker/otel-collector/Dockerfile .) -FROM golang:1.25-alpine AS migrate-builder +# Note: The official OCB image may ship an older Go than the contrib modules require, +# so we copy the ocb binary into a golang base with a newer toolchain. +FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin +FROM golang:1.26-alpine AS ocb-builder +ARG OTEL_COLLECTOR_VERSION +ARG OTEL_COLLECTOR_CORE_VERSION +COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb +WORKDIR /build +COPY packages/otel-collector/builder-config.yaml . +RUN --mount=type=cache,target=/go/pkg/mod,id=ocb-gomod \ + --mount=type=cache,target=/root/.cache/go-build,id=ocb-gobuild \ + sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \ + CGO_ENABLED=0 ocb --config=builder-config.yaml + +# Build the Go migration tool with full TLS support for ClickHouse +FROM golang:1.26-alpine AS migrate-builder WORKDIR /build COPY packages/otel-collector/go.mod packages/otel-collector/go.sum ./ RUN go mod download @@ -38,7 +57,7 @@ COPY --from=migrate-builder /migrate /usr/local/bin/migrate USER ${USER_UID}:${USER_GID} COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor -COPY --from=col --chmod=755 /otelcol-contrib /otelcontribcol +COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol # Copy entrypoint and log tail wrapper scripts COPY --chmod=755 docker/otel-collector/entrypoint.sh /entrypoint.sh diff --git a/docker/otel-collector/config.standalone.yaml b/docker/otel-collector/config.standalone.yaml index f43189f2..46e3958b 100644 --- a/docker/otel-collector/config.standalone.yaml +++ b/docker/otel-collector/config.standalone.yaml @@ -33,6 +33,7 @@ exporters: logs_table_name: hyperdx_sessions timeout: 5s create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false} + json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false} retry_on_failure: enabled: true initial_interval: 5s @@ -46,6 +47,7 @@ exporters: ttl: 720h timeout: 5s create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false} + json: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false} retry_on_failure: enabled: true initial_interval: 5s diff --git a/docker/otel-collector/entrypoint.sh b/docker/otel-collector/entrypoint.sh index 3aae225c..272473e4 100644 --- a/docker/otel-collector/entrypoint.sh +++ b/docker/otel-collector/entrypoint.sh @@ -1,8 +1,26 @@ #!/bin/sh set -e -# Fall back to legacy schema when the ClickHouse JSON feature gate is enabled +# DEPRECATED: The clickhouse.json feature gate has been removed upstream. +# When OTEL_AGENT_FEATURE_GATE_ARG contains clickhouse.json, strip it and +# map it to HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE instead. Other feature gates +# are preserved and passed through to the collector. if echo "$OTEL_AGENT_FEATURE_GATE_ARG" | grep -q "clickhouse.json"; then + echo "WARNING: '--feature-gates=clickhouse.json' is deprecated and no longer supported by the collector." + echo "WARNING: Use HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true instead. This flag will be removed in a future release." + export HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE=true + + # Strip clickhouse.json from the feature gates, keeping any other gates + REMAINING_GATES=$(echo "$OTEL_AGENT_FEATURE_GATE_ARG" | sed 's/--feature-gates=//' | tr ',' '\n' | grep -v 'clickhouse.json' | tr '\n' ',' | sed 's/,$//') + if [ -n "$REMAINING_GATES" ]; then + export OTEL_AGENT_FEATURE_GATE_ARG="--feature-gates=$REMAINING_GATES" + else + unset OTEL_AGENT_FEATURE_GATE_ARG + fi +fi + +# Fall back to legacy schema when ClickHouse JSON exporter mode is enabled +if [ "$HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE" = "true" ]; then export HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA=true fi @@ -39,7 +57,7 @@ if [ -z "$OPAMP_SERVER_URL" ]; then COLLECTOR_ARGS="$COLLECTOR_ARGS --config $CUSTOM_OTELCOL_CONFIG_FILE" fi - # Pass feature gates to the collector in standalone mode + # Pass remaining feature gates to the collector in standalone mode if [ -n "$OTEL_AGENT_FEATURE_GATE_ARG" ]; then COLLECTOR_ARGS="$COLLECTOR_ARGS $OTEL_AGENT_FEATURE_GATE_ARG" fi diff --git a/docker/otel-collector/supervisor_docker.yaml.tmpl b/docker/otel-collector/supervisor_docker.yaml.tmpl index 3ee29c02..f91e3cc5 100644 --- a/docker/otel-collector/supervisor_docker.yaml.tmpl +++ b/docker/otel-collector/supervisor_docker.yaml.tmpl @@ -23,8 +23,8 @@ agent: {{- if getenv "CUSTOM_OTELCOL_CONFIG_FILE" }} - {{ getenv "CUSTOM_OTELCOL_CONFIG_FILE" }} {{- end }} - args: {{- if getenv "OTEL_AGENT_FEATURE_GATE_ARG" }} + args: - {{ getenv "OTEL_AGENT_FEATURE_GATE_ARG" }} {{- end }} diff --git a/knip.json b/knip.json index 9bb451db..1337d9a0 100644 --- a/knip.json +++ b/knip.json @@ -27,7 +27,7 @@ "project": ["src/**/*.ts"] } }, - "ignore": ["scripts/dev-portal/**"], + "ignore": ["scripts/dev-portal/**", ".github/scripts/**"], "ignoreBinaries": ["make", "migrate", "playwright"], "ignoreDependencies": [ "@dotenvx/dotenvx", diff --git a/package.json b/package.json index ee1611fc..0d2e0827 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "run:clickhouse": "nx run @hyperdx/app:run:clickhouse", "dev": "sh -c '. ./scripts/dev-env.sh && yarn build:common-utils && dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml up -d && yarn app:dev; dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down'", "dev:local": "IS_LOCAL_APP_MODE='DANGEROUSLY_is_local_app_mode💀' yarn dev", + "cli:dev": "yarn workspace @hyperdx/cli dev", "dev:down": "sh -c '. ./scripts/dev-env.sh && docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down && sh ./scripts/dev-kill-ports.sh'", "dev:compose": "sh -c '. ./scripts/dev-env.sh && docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml'", "knip": "knip", diff --git a/packages/api/.env.test b/packages/api/.env.test index dd8476a7..2bc5fa80 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -12,5 +12,7 @@ MONGO_URI=mongodb://localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test NODE_ENV=test PORT=${HDX_CI_API_PORT:-19000} OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320} -# Default to only logging warnings/errors. Adjust if you need more verbosity -HYPERDX_LOG_LEVEL=warn \ No newline at end of file +# Default to only logging errors. Adjust if you need more verbosity. +# Note: the logger module is mocked in jest.setup.ts to suppress expected +# operational noise (validation errors, MCP tool errors, etc.) during tests. +HYPERDX_LOG_LEVEL=error \ No newline at end of file diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index 65a1ff7b..d333121e 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,15 @@ # @hyperdx/api +## 2.23.2 + +## 2.23.1 + +### Patch Changes + +- f8d2edde: feat: Show created/updated metadata for saved searches and dashboards +- Updated dependencies [24767c58] + - @hyperdx/common-utils@0.17.1 + ## 2.23.0 ### Minor Changes diff --git a/packages/api/jest.config.js b/packages/api/jest.config.js index b9dca9a7..8fc45f0f 100644 --- a/packages/api/jest.config.js +++ b/packages/api/jest.config.js @@ -1,8 +1,12 @@ -/** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */ +const { createJsWithTsPreset } = require('ts-jest'); + +const tsJestTransformCfg = createJsWithTsPreset(); + +/** @type {import("jest").Config} **/ module.exports = { + ...tsJestTransformCfg, setupFilesAfterEnv: ['/../jest.setup.ts'], setupFiles: ['dotenv-expand/config'], - preset: 'ts-jest', testEnvironment: 'node', verbose: true, rootDir: './src', diff --git a/packages/api/jest.setup.ts b/packages/api/jest.setup.ts index 610e85cc..7549dd08 100644 --- a/packages/api/jest.setup.ts +++ b/packages/api/jest.setup.ts @@ -1,12 +1,11 @@ -// @eslint-disable @typescript-eslint/no-var-requires jest.retryTimes(1, { logErrorsBeforeRetry: true }); -global.console = { - ...console, - // Turn off noisy console logs in tests - debug: jest.fn(), - info: jest.fn(), -}; +// Suppress noisy console output during test runs. +// - debug/info: ClickHouse query logging, server startup messages +// - warn: expected column-not-found warnings from renderChartConfig on CTE tables +jest.spyOn(console, 'debug').mockImplementation(() => {}); +jest.spyOn(console, 'info').mockImplementation(() => {}); +jest.spyOn(console, 'warn').mockImplementation(() => {}); // Mock alert notification functions to prevent HTTP calls during tests jest.mock('@/utils/slack', () => ({ diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 5ff15b61..531322b3 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -70,7 +70,13 @@ "type": "string", "enum": [ "above", - "below" + "below", + "above_exclusive", + "below_or_equal", + "equal", + "not_equal", + "between", + "not_between" ], "description": "Threshold comparison direction." }, @@ -162,7 +168,7 @@ }, "tileId": { "type": "string", - "description": "Tile ID for tile-based alerts. May not be a Raw-SQL-based tile.", + "description": "Tile ID for tile-based alerts. Must be a line, stacked bar, or number type tile.", "nullable": true, "example": "65f5e4a3b9e77c001a901234" }, @@ -180,9 +186,15 @@ }, "threshold": { "type": "number", - "description": "Threshold value for triggering the alert.", + "description": "Threshold value for triggering the alert. For between and not_between threshold types, this is the lower bound.", "example": 100 }, + "thresholdMax": { + "type": "number", + "nullable": true, + "description": "Upper bound for between and not_between threshold types. Required when thresholdType is between or not_between, must be >= threshold.", + "example": 500 + }, "interval": { "$ref": "#/components/schemas/AlertInterval", "description": "Evaluation interval for the alert.", diff --git a/packages/api/package.json b/packages/api/package.json index 23dc3d1b..2000c642 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/api", - "version": "2.23.0", + "version": "2.23.2", "license": "MIT", "private": true, "engines": { @@ -10,9 +10,10 @@ "@ai-sdk/anthropic": "^3.0.58", "@ai-sdk/openai": "^3.0.47", "@esm2cjs/p-queue": "^7.3.0", - "@hyperdx/common-utils": "^0.17.0", + "@hyperdx/common-utils": "^0.17.1", "@hyperdx/node-opentelemetry": "^0.9.0", "@hyperdx/passport-local-mongoose": "^9.0.1", + "@modelcontextprotocol/sdk": "^1.27.1", "@opentelemetry/api": "^1.8.0", "@opentelemetry/host-metrics": "^0.35.5", "@opentelemetry/sdk-metrics": "^1.30.1", @@ -57,7 +58,7 @@ "@types/cors": "^2.8.14", "@types/express": "^4.17.13", "@types/express-session": "^1.17.7", - "@types/jest": "^28.1.1", + "@types/jest": "^29.5.14", "@types/lodash": "^4.14.198", "@types/minimist": "^1.2.2", "@types/ms": "^0.7.31", @@ -65,7 +66,7 @@ "@types/supertest": "^2.0.12", "@types/swagger-jsdoc": "^6", "@types/uuid": "^8.3.4", - "jest": "^28.1.3", + "jest": "^30.2.0", "migrate-mongo": "^11.0.0", "nodemon": "^2.0.20", "pino-pretty": "^13.1.1", @@ -82,13 +83,14 @@ "scripts": { "start": "node ./build/index.js", "dev": "DOTENV_CONFIG_PATH=.env.development nodemon --exec 'ts-node' --transpile-only -r tsconfig-paths/register -r dotenv-expand/config -r '@hyperdx/node-opentelemetry/build/src/tracing' ./src/index.ts", + "dev:mcp": "npx @modelcontextprotocol/inspector", "dev-task": "DOTENV_CONFIG_PATH=.env.development nodemon --exec 'ts-node' --transpile-only -r tsconfig-paths/register -r dotenv-expand/config -r '@hyperdx/node-opentelemetry/build/src/tracing' ./src/tasks/index.ts", "build": "rimraf ./build && tsc && tsc-alias && cp -r ./src/opamp/proto ./build/opamp/", "lint": "npx eslint --quiet . --ext .ts", "lint:fix": "npx eslint . --ext .ts --fix", "ci:lint": "yarn lint && yarn tsc --noEmit && yarn lint:openapi", "ci:int": "DOTENV_CONFIG_PATH=.env.test DOTENV_CONFIG_OVERRIDE=true jest --runInBand --ci --forceExit --coverage", - "dev:int": "DOTENV_CONFIG_PATH=.env.test DOTENV_CONFIG_OVERRIDE=true jest --runInBand --forceExit --coverage", + "dev:int": "DOTENV_CONFIG_PATH=.env.test DOTENV_CONFIG_OVERRIDE=true jest --runInBand --coverage", "dev:migrate-db-create": "ts-node node_modules/.bin/migrate-mongo create -f migrate-mongo-config.ts", "dev:migrate-db": "ts-node node_modules/.bin/migrate-mongo up -f migrate-mongo-config.ts", "dev:migrate-ch-create": "migrate create -ext sql -dir ./migrations/ch -seq", diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index 7102d313..bec663c7 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -5,6 +5,7 @@ import session from 'express-session'; import onHeaders from 'on-headers'; import * as config from './config'; +import mcpRouter from './mcp/app'; import { isUserAuthenticated } from './middleware/auth'; import defaultCors from './middleware/cors'; import { appErrorHandler } from './middleware/error'; @@ -12,6 +13,7 @@ import routers from './routers/api'; import clickhouseProxyRouter from './routers/api/clickhouseProxy'; import connectionsRouter from './routers/api/connections'; import favoritesRouter from './routers/api/favorites'; +import pinnedFiltersRouter from './routers/api/pinnedFilters'; import savedSearchRouter from './routers/api/savedSearch'; import sourcesRouter from './routers/api/sources'; import externalRoutersV2 from './routers/external-api/v2'; @@ -79,7 +81,7 @@ app.use(defaultCors); // --------------------------------------------------------------------- // ----------------------- Background Jobs ----------------------------- // --------------------------------------------------------------------- -if (config.USAGE_STATS_ENABLED) { +if (config.USAGE_STATS_ENABLED && !config.IS_CI) { usageStats(); } // --------------------------------------------------------------------- @@ -90,6 +92,9 @@ if (config.USAGE_STATS_ENABLED) { // PUBLIC ROUTES app.use('/', routers.rootRouter); +// SELF-AUTHENTICATED ROUTES (validated via access key, not session middleware) +app.use('/mcp', mcpRouter); + // PRIVATE ROUTES app.use('/ai', isUserAuthenticated, routers.aiRouter); app.use('/alerts', isUserAuthenticated, routers.alertsRouter); @@ -101,6 +106,7 @@ app.use('/connections', isUserAuthenticated, connectionsRouter); app.use('/sources', isUserAuthenticated, sourcesRouter); app.use('/saved-search', isUserAuthenticated, savedSearchRouter); app.use('/favorites', isUserAuthenticated, favoritesRouter); +app.use('/pinned-filters', isUserAuthenticated, pinnedFiltersRouter); app.use('/clickhouse-proxy', isUserAuthenticated, clickhouseProxyRouter); // --------------------------------------------------------------------- diff --git a/packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap b/packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap index d613b118..e40f178c 100644 --- a/packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap +++ b/packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap @@ -1,33 +1,33 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing exports[`renderChartConfig K8s Semantic Convention Migrations with metricNameSql should handle gauge metric with metricNameSql and groupBy 1`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "arrayElement(ResourceAttributes, 'k8s.pod.name')": "test-pod", "avg(toFloat64OrDefault(toString(LastValue)))": 45, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:01:00Z", "arrayElement(ResourceAttributes, 'k8s.pod.name')": "test-pod", "avg(toFloat64OrDefault(toString(LastValue)))": 50, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:02:00Z", "arrayElement(ResourceAttributes, 'k8s.pod.name')": "test-pod", "avg(toFloat64OrDefault(toString(LastValue)))": 55, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:03:00Z", "arrayElement(ResourceAttributes, 'k8s.pod.name')": "test-pod", "avg(toFloat64OrDefault(toString(LastValue)))": 60, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:04:00Z", "arrayElement(ResourceAttributes, 'k8s.pod.name')": "test-pod", "avg(toFloat64OrDefault(toString(LastValue)))": 65, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "arrayElement(ResourceAttributes, 'k8s.pod.name')": "test-pod", "avg(toFloat64OrDefault(toString(LastValue)))": 70, @@ -36,16 +36,16 @@ Array [ `; exports[`renderChartConfig K8s Semantic Convention Migrations with metricNameSql should handle metrics without metricNameSql (backward compatibility) 1`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 45, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:01:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 50, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:02:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 55, }, @@ -53,28 +53,28 @@ Array [ `; exports[`renderChartConfig K8s Semantic Convention Migrations with metricNameSql should query k8s.pod.cpu.utilization gauge metric using metricNameSql to handle both old and new conventions 1`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 45, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:01:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 50, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:02:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 55, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:03:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 60, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:04:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 65, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 70, }, @@ -82,12 +82,12 @@ Array [ `; exports[`renderChartConfig Query Events - Logs simple select + group by query logs 1`] = ` -Array [ - Object { +[ + { "ServiceName": "app", "count": "1", }, - Object { + { "ServiceName": "api", "count": "1", }, @@ -95,31 +95,31 @@ Array [ `; exports[`renderChartConfig Query Events - Logs simple select + where query logs 1`] = ` -Array [ - Object { +[ + { "Body": "Oh no! Something went wrong!", }, ] `; exports[`renderChartConfig Query Metrics - Gauge single avg gauge with group-by 1`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "arrayElement(ResourceAttributes, 'host')": "host2", "avg(toFloat64OrDefault(toString(LastValue)))": 4, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "arrayElement(ResourceAttributes, 'host')": "host1", "avg(toFloat64OrDefault(toString(LastValue)))": 6.25, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "arrayElement(ResourceAttributes, 'host')": "host2", "avg(toFloat64OrDefault(toString(LastValue)))": 4, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "arrayElement(ResourceAttributes, 'host')": "host1", "avg(toFloat64OrDefault(toString(LastValue)))": 80, @@ -128,12 +128,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Gauge single avg gauge with where 1`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 6.25, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 80, }, @@ -141,12 +141,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Gauge single max gauge with delta 1`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "max(toFloat64OrDefault(toString(LastValue)))": 5, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "max(toFloat64OrDefault(toString(LastValue)))": -1.6666666666666667, }, @@ -154,23 +154,23 @@ Array [ `; exports[`renderChartConfig Query Metrics - Gauge single max gauge with delta and group by 1`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "arrayElement(ResourceAttributes, 'host')": "host2", "max(toFloat64OrDefault(toString(LastValue)))": 5, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "arrayElement(ResourceAttributes, 'host')": "host1", "max(toFloat64OrDefault(toString(LastValue)))": -72.91666666666667, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "arrayElement(ResourceAttributes, 'host')": "host2", "max(toFloat64OrDefault(toString(LastValue)))": -1.6666666666666667, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "arrayElement(ResourceAttributes, 'host')": "host1", "max(toFloat64OrDefault(toString(LastValue)))": -33.333333333333336, @@ -179,12 +179,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Gauge single max/avg/sum gauge 1`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 5.125, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "avg(toFloat64OrDefault(toString(LastValue)))": 42, }, @@ -192,12 +192,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Gauge single max/avg/sum gauge 2`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "max(toFloat64OrDefault(toString(LastValue)))": 6.25, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "max(toFloat64OrDefault(toString(LastValue)))": 80, }, @@ -205,12 +205,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Gauge single max/avg/sum gauge 3`] = ` -Array [ - Object { +[ + { "__hdx_time_bucket": "2022-01-05T00:00:00Z", "sum(toFloat64OrDefault(toString(LastValue)))": 10.25, }, - Object { + { "__hdx_time_bucket": "2022-01-05T00:05:00Z", "sum(toFloat64OrDefault(toString(LastValue)))": 84, }, @@ -218,32 +218,32 @@ Array [ `; exports[`renderChartConfig Query Metrics - Histogram should bucket correctly when grouping by a single attribute 1`] = ` -Array [ - Object { +[ + { "Value": 3.5714285714285716, "__hdx_time_bucket": "2022-01-05T00:00:00Z", - "group": Array [ + "group": [ "host-b", ], }, - Object { + { "Value": 8.382352941176471, "__hdx_time_bucket": "2022-01-05T00:00:00Z", - "group": Array [ + "group": [ "host-a", ], }, - Object { + { "Value": 3.5, "__hdx_time_bucket": "2022-01-05T00:02:00Z", - "group": Array [ + "group": [ "host-b", ], }, - Object { + { "Value": 4.95, "__hdx_time_bucket": "2022-01-05T00:02:00Z", - "group": Array [ + "group": [ "host-a", ], }, @@ -251,99 +251,99 @@ Array [ `; exports[`renderChartConfig Query Metrics - Histogram should bucket correctly when grouping by multiple attributes 1`] = ` -Array [ - Object { +[ + { "Value": 2.916666666666667, "__hdx_time_bucket": "2022-01-05T00:00:00Z", - "group": Array [ + "group": [ "host-b", "service-2", ], }, - Object { + { "Value": 4.852941176470588, "__hdx_time_bucket": "2022-01-05T00:00:00Z", - "group": Array [ + "group": [ "host-a", "service-2", ], }, - Object { + { "Value": 8.75, "__hdx_time_bucket": "2022-01-05T00:00:00Z", - "group": Array [ + "group": [ "host-a", "service-1", ], }, - Object { + { "Value": 3.1578947368421053, "__hdx_time_bucket": "2022-01-05T00:00:00Z", - "group": Array [ + "group": [ "host-b", "service-1", ], }, - Object { + { "Value": 58.33333333333333, "__hdx_time_bucket": "2022-01-05T00:00:00Z", - "group": Array [ + "group": [ "host-a", "service-3", ], }, - Object { + { "Value": 6.25, "__hdx_time_bucket": "2022-01-05T00:00:00Z", - "group": Array [ + "group": [ "host-b", "service-3", ], }, - Object { + { "Value": 3.4090909090909087, "__hdx_time_bucket": "2022-01-05T00:02:00Z", - "group": Array [ + "group": [ "host-b", "service-1", ], }, - Object { + { "Value": 7.916666666666667, "__hdx_time_bucket": "2022-01-05T00:02:00Z", - "group": Array [ + "group": [ "host-a", "service-1", ], }, - Object { + { "Value": 3.25, "__hdx_time_bucket": "2022-01-05T00:02:00Z", - "group": Array [ + "group": [ "host-b", "service-3", ], }, - Object { + { "Value": 4.25, "__hdx_time_bucket": "2022-01-05T00:02:00Z", - "group": Array [ + "group": [ "host-a", "service-3", ], }, - Object { + { "Value": 4.75, "__hdx_time_bucket": "2022-01-05T00:02:00Z", - "group": Array [ + "group": [ "host-a", "service-2", ], }, - Object { + { "Value": 3.888888888888889, "__hdx_time_bucket": "2022-01-05T00:02:00Z", - "group": Array [ + "group": [ "host-b", "service-2", ], @@ -352,12 +352,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Histogram should bucket correctly when no grouping is defined 1`] = ` -Array [ - Object { +[ + { "Value": 5.241935483870968, "__hdx_time_bucket": "2022-01-05T00:00:00Z", }, - Object { + { "Value": 4.40625, "__hdx_time_bucket": "2022-01-05T00:02:00Z", }, @@ -365,8 +365,8 @@ Array [ `; exports[`renderChartConfig Query Metrics - Histogram two_timestamps_bounded histogram (p25) 1`] = ` -Array [ - Object { +[ + { "Value": 7.5, "__hdx_time_bucket": "2022-01-05T00:01:00Z", }, @@ -374,8 +374,8 @@ Array [ `; exports[`renderChartConfig Query Metrics - Histogram two_timestamps_bounded histogram (p50) 1`] = ` -Array [ - Object { +[ + { "Value": 20, "__hdx_time_bucket": "2022-01-05T00:01:00Z", }, @@ -383,8 +383,8 @@ Array [ `; exports[`renderChartConfig Query Metrics - Histogram two_timestamps_bounded histogram (p90) 1`] = ` -Array [ - Object { +[ + { "Value": 30, "__hdx_time_bucket": "2022-01-05T00:01:00Z", }, @@ -392,8 +392,8 @@ Array [ `; exports[`renderChartConfig Query Metrics - Histogram two_timestamps_lower_bound_inf histogram (p50) 1`] = ` -Array [ - Object { +[ + { "Value": 0.5, "__hdx_time_bucket": "2022-01-05T00:01:00Z", }, @@ -401,12 +401,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Sum calculates min_rate/max_rate correctly for sum metrics: maxSum 1`] = ` -Array [ - Object { +[ + { "Value": 24, "__hdx_time_bucket": "2022-01-05T00:00:00Z", }, - Object { + { "Value": 134, "__hdx_time_bucket": "2022-01-05T00:10:00Z", }, @@ -414,12 +414,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Sum calculates min_rate/max_rate correctly for sum metrics: minSum 1`] = ` -Array [ - Object { +[ + { "Value": 15, "__hdx_time_bucket": "2022-01-05T00:00:00Z", }, - Object { + { "Value": 52, "__hdx_time_bucket": "2022-01-05T00:10:00Z", }, @@ -427,12 +427,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Sum handles counter resets correctly for sum metrics 1`] = ` -Array [ - Object { +[ + { "Value": 15, "__hdx_time_bucket": "2022-01-05T00:00:00Z", }, - Object { + { "Value": 52, "__hdx_time_bucket": "2022-01-05T00:10:00Z", }, @@ -440,20 +440,20 @@ Array [ `; exports[`renderChartConfig Query Metrics - Sum single sum rate 1`] = ` -Array [ - Object { +[ + { "Value": 19, "__hdx_time_bucket": "2022-01-05T00:00:00Z", }, - Object { + { "Value": 79, "__hdx_time_bucket": "2022-01-05T00:05:00Z", }, - Object { + { "Value": 5813, "__hdx_time_bucket": "2022-01-05T00:10:00Z", }, - Object { + { "Value": 78754, "__hdx_time_bucket": "2022-01-05T00:15:00Z", }, @@ -461,12 +461,12 @@ Array [ `; exports[`renderChartConfig Query Metrics - Sum sum values as without rate computation 1`] = ` -Array [ - Object { +[ + { "Value": 950400, "__hdx_time_bucket": "2022-01-05T00:00:00Z", }, - Object { + { "Value": 1641600, "__hdx_time_bucket": "2022-01-05T00:10:00Z", }, @@ -474,19 +474,19 @@ Array [ `; exports[`renderChartConfig Query settings handles the the query settings 1`] = ` -Array [ - Object { +[ + { "Body": "Oh no! Something went wrong!", }, - Object { + { "Body": "This is a test message.", }, ] `; exports[`renderChartConfig aggFn numeric agg functions should handle numeric values as strings 1`] = ` -Array [ - Object { +[ + { "AVG(toFloat64OrDefault(toString(strVal)))": 0.5, "MAX(toFloat64OrDefault(toString(strVal)))": 3, "MIN(toFloat64OrDefault(toString(strVal)))": -1.1, @@ -497,8 +497,8 @@ Array [ `; exports[`renderChartConfig aggFn numeric agg functions should use default values for other types 1`] = ` -Array [ - Object { +[ + { "AVG(toFloat64OrDefault(toString(strVal)))": 0, "MAX(toFloat64OrDefault(toString(strVal)))": 0, "MIN(toFloat64OrDefault(toString(strVal)))": 0, diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index c5ef1fc3..1712ed92 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -1,3 +1,7 @@ +import { + displayTypeSupportsRawSqlAlerts, + validateRawSqlForAlert, +} from '@hyperdx/common-utils/dist/core/utils'; import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { sign, verify } from 'jsonwebtoken'; import { groupBy } from 'lodash'; @@ -5,13 +9,7 @@ import ms from 'ms'; import { z } from 'zod'; import type { ObjectId } from '@/models'; -import Alert, { - AlertChannel, - AlertInterval, - AlertSource, - AlertThresholdType, - IAlert, -} from '@/models/alert'; +import Alert, { AlertSource, IAlert } from '@/models/alert'; import Dashboard, { IDashboard } from '@/models/dashboard'; import { ISavedSearch, SavedSearch } from '@/models/savedSearch'; import { IUser } from '@/models/user'; @@ -20,34 +18,23 @@ import { Api400Error } from '@/utils/errors'; import logger from '@/utils/logger'; import { alertSchema, objectIdSchema } from '@/utils/zod'; -export type AlertInput = { +export type AlertInput = Omit< + IAlert, + | 'id' + | 'scheduleStartAt' + | 'savedSearchId' + | 'createdAt' + | 'createdBy' + | 'updatedAt' + | 'team' + | 'state' +> & { id?: string; - source?: AlertSource; - channel: AlertChannel; - interval: AlertInterval; - scheduleOffsetMinutes?: number; + // Replace the Date-type fields from IAlert scheduleStartAt?: string | null; - thresholdType: AlertThresholdType; - threshold: number; - - // Message template - name?: string | null; - message?: string | null; - - // Log alerts - groupBy?: string; + // Replace the ObjectId-type fields from IAlert savedSearchId?: string; - - // Chart alerts dashboardId?: string; - tileId?: string; - - // Silenced - silenced?: { - by?: ObjectId; - at: Date; - until: Date; - }; }; const validateObjectId = (id: string | undefined, message: string) => { @@ -82,7 +69,18 @@ export const validateAlertInput = async ( } if (tile.config != null && isRawSqlSavedChartConfig(tile.config)) { - throw new Api400Error('Cannot create an alert on a raw SQL tile'); + if (!displayTypeSupportsRawSqlAlerts(tile.config.displayType)) { + throw new Api400Error( + 'Alerts on Raw SQL tiles are only supported for Line, Stacked Bar, or Number display types', + ); + } + + const { errors } = validateRawSqlForAlert(tile.config); + if (errors.length > 0) { + throw new Api400Error( + `Raw SQL alert query is invalid: ${errors.join(', ')}`, + ); + } } } @@ -140,6 +138,7 @@ const makeAlert = (alert: AlertInput, userId?: ObjectId): Partial => { }), source: alert.source, threshold: alert.threshold, + thresholdMax: alert.thresholdMax, thresholdType: alert.thresholdType, ...(userId && { createdBy: userId }), @@ -287,6 +286,20 @@ export const getAlertsEnhanced = async (teamId: ObjectId) => { }>(['savedSearch', 'dashboard', 'createdBy', 'silenced.by']); }; +export const getAlertEnhanced = async ( + alertId: ObjectId | string, + teamId: ObjectId, +) => { + return Alert.findOne({ _id: alertId, team: teamId }).populate<{ + savedSearch: ISavedSearch; + dashboard: IDashboard; + createdBy?: IUser; + silenced?: IAlert['silenced'] & { + by: IUser; + }; + }>(['savedSearch', 'dashboard', 'createdBy', 'silenced.by']); +}; + export const deleteAlert = async (id: string, teamId: ObjectId) => { return Alert.deleteOne({ _id: id, diff --git a/packages/api/src/controllers/dashboard.ts b/packages/api/src/controllers/dashboard.ts index 4449ad15..92a93e4c 100644 --- a/packages/api/src/controllers/dashboard.ts +++ b/packages/api/src/controllers/dashboard.ts @@ -1,7 +1,6 @@ -import { isBuilderSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { - BuilderSavedChartConfig, DashboardWithoutIdSchema, + SavedChartConfig, Tile, } from '@hyperdx/common-utils/dist/types'; import { map, partition, uniq } from 'lodash'; @@ -19,7 +18,7 @@ import Dashboard from '@/models/dashboard'; function pickAlertsByTile(tiles: Tile[]) { return tiles.reduce((acc, tile) => { - if (isBuilderSavedChartConfig(tile.config) && tile.config.alert) { + if (tile.config.alert) { acc[tile.id] = tile.config.alert; } return acc; @@ -27,9 +26,7 @@ function pickAlertsByTile(tiles: Tile[]) { } type TileForAlertSync = Pick & { - config?: - | Pick - | { alert?: IAlert | AlertDocument }; + config?: Pick | { alert?: IAlert | AlertDocument }; }; function extractTileAlertData(tiles: TileForAlertSync[]): { @@ -55,9 +52,7 @@ async function syncDashboardAlerts( const newTilesForAlertSync: TileForAlertSync[] = newTiles.map(t => ({ id: t.id, - config: isBuilderSavedChartConfig(t.config) - ? { alert: t.config.alert } - : {}, + config: { alert: t.config.alert }, })); const { tileIds: newTileIds, tileIdsWithAlerts: newTileIdsWithAlerts } = extractTileAlertData(newTilesForAlertSync); @@ -95,7 +90,9 @@ async function syncDashboardAlerts( export async function getDashboards(teamId: ObjectId) { const [_dashboards, alerts] = await Promise.all([ - Dashboard.find({ team: teamId }), + Dashboard.find({ team: teamId }) + .populate('createdBy', 'email name') + .populate('updatedBy', 'email name'), getTeamDashboardAlertsByDashboardAndTile(teamId), ]); @@ -117,12 +114,14 @@ export async function getDashboards(teamId: ObjectId) { export async function getDashboard(dashboardId: string, teamId: ObjectId) { const [_dashboard, alerts] = await Promise.all([ - Dashboard.findOne({ _id: dashboardId, team: teamId }), + Dashboard.findOne({ _id: dashboardId, team: teamId }) + .populate('createdBy', 'email name') + .populate('updatedBy', 'email name'), getDashboardAlertsByTile(teamId, dashboardId), ]); return { - ..._dashboard, + ..._dashboard?.toJSON(), tiles: _dashboard?.tiles.map(t => ({ ...t, config: { ...t.config, alert: alerts[t.id]?.[0] }, @@ -138,6 +137,8 @@ export async function createDashboard( const newDashboard = await new Dashboard({ ...dashboard, team: teamId, + createdBy: userId, + updatedBy: userId, }).save(); await createOrUpdateDashboardAlerts( @@ -180,6 +181,7 @@ export async function updateDashboard( { ...updates, tags: updates.tags && uniq(updates.tags), + updatedBy: userId, }, { new: true }, ); diff --git a/packages/api/src/controllers/pinnedFilter.ts b/packages/api/src/controllers/pinnedFilter.ts new file mode 100644 index 00000000..706f8671 --- /dev/null +++ b/packages/api/src/controllers/pinnedFilter.ts @@ -0,0 +1,42 @@ +import type { PinnedFiltersValue } from '@hyperdx/common-utils/dist/types'; +import mongoose from 'mongoose'; + +import type { ObjectId } from '@/models'; +import PinnedFilterModel from '@/models/pinnedFilter'; + +/** + * Get team-level pinned filters for a team+source combination. + */ +export async function getPinnedFilters( + teamId: string | ObjectId, + sourceId: string | ObjectId, +) { + return PinnedFilterModel.findOne({ + team: new mongoose.Types.ObjectId(teamId), + source: new mongoose.Types.ObjectId(sourceId), + }); +} + +/** + * Upsert team-level pinned filters for a team+source. + */ +export async function updatePinnedFilters( + teamId: string | ObjectId, + sourceId: string | ObjectId, + data: { fields: string[]; filters: PinnedFiltersValue }, +) { + const filter = { + team: new mongoose.Types.ObjectId(teamId), + source: new mongoose.Types.ObjectId(sourceId), + }; + + return PinnedFilterModel.findOneAndUpdate( + filter, + { + ...filter, + fields: data.fields, + filters: data.filters, + }, + { upsert: true, new: true }, + ); +} diff --git a/packages/api/src/controllers/savedSearch.ts b/packages/api/src/controllers/savedSearch.ts index 8e68beaa..203b86a5 100644 --- a/packages/api/src/controllers/savedSearch.ts +++ b/packages/api/src/controllers/savedSearch.ts @@ -1,16 +1,22 @@ -import { SavedSearchSchema } from '@hyperdx/common-utils/dist/types'; -import { groupBy, pick } from 'lodash'; +import { + SavedSearchListApiResponse, + SavedSearchSchema, +} from '@hyperdx/common-utils/dist/types'; +import { groupBy } from 'lodash'; import { z } from 'zod'; import { deleteSavedSearchAlerts } from '@/controllers/alerts'; import Alert from '@/models/alert'; import { SavedSearch } from '@/models/savedSearch'; -import type { IUser } from '@/models/user'; type SavedSearchWithoutId = Omit, 'id'>; -export async function getSavedSearches(teamId: string) { - const savedSearches = await SavedSearch.find({ team: teamId }); +export async function getSavedSearches( + teamId: string, +): Promise { + const savedSearches = await SavedSearch.find({ team: teamId }) + .populate('createdBy', 'email name') + .populate('updatedBy', 'email name'); const alerts = await Alert.find( { team: teamId, savedSearch: { $exists: true, $ne: null } }, { __v: 0 }, @@ -27,26 +33,36 @@ export async function getSavedSearches(teamId: string) { } export function getSavedSearch(teamId: string, savedSearchId: string) { - return SavedSearch.findOne({ _id: savedSearchId, team: teamId }); + return SavedSearch.findOne({ _id: savedSearchId, team: teamId }) + .populate('createdBy', 'email name') + .populate('updatedBy', 'email name'); } export function createSavedSearch( teamId: string, savedSearch: SavedSearchWithoutId, + userId?: string, ) { - return SavedSearch.create({ ...savedSearch, team: teamId }); + return SavedSearch.create({ + ...savedSearch, + team: teamId, + createdBy: userId, + updatedBy: userId, + }); } export function updateSavedSearch( teamId: string, savedSearchId: string, savedSearch: SavedSearchWithoutId, + userId?: string, ) { return SavedSearch.findOneAndUpdate( { _id: savedSearchId, team: teamId }, { ...savedSearch, team: teamId, + updatedBy: userId, }, { new: true }, ); diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 55e24926..ffe01378 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -1,5 +1,6 @@ import { createNativeClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { + AlertThresholdType, BuilderSavedChartConfig, DisplayType, RawSqlSavedChartConfig, @@ -15,7 +16,7 @@ import { AlertInput } from '@/controllers/alerts'; import { getTeam } from '@/controllers/team'; import { findUserByEmail } from '@/controllers/user'; import { mongooseConnection } from '@/models'; -import { AlertInterval, AlertSource, AlertThresholdType } from '@/models/alert'; +import { AlertInterval, AlertSource } from '@/models/alert'; import Server from '@/server'; import logger from '@/utils/logger'; import { MetricModel } from '@/utils/logParser'; @@ -63,6 +64,13 @@ export const getTestFixtureClickHouseClient = async () => { return clickhouseClient; }; +export const closeTestFixtureClickHouseClient = async () => { + if (clickhouseClient) { + await clickhouseClient.close(); + clickhouseClient = null; + } +}; + const healthCheck = async () => { const client = await getTestFixtureClickHouseClient(); const result = await client.ping(); @@ -132,6 +140,7 @@ export const closeDB = async () => { throw new Error('ONLY execute this in CI env 😈 !!!'); } await mongooseConnection.dropDatabase(); + await mongoose.disconnect(); }; export const clearDBCollections = async () => { @@ -175,8 +184,8 @@ class MockServer extends Server { } } - stop() { - return new Promise((resolve, reject) => { + async stop() { + await new Promise((resolve, reject) => { this.appServer.close(err => { if (err) { reject(err); @@ -187,13 +196,12 @@ class MockServer extends Server { reject(err); return; } - super - .shutdown() - .then(() => resolve()) - .catch(err => reject(err)); + resolve(); }); }); }); + await closeTestFixtureClickHouseClient(); + await super.shutdown(); } clearDBs() { @@ -501,7 +509,39 @@ export const makeExternalTile = (opts?: { }, }); -export const makeRawSqlTile = (opts?: { id?: string }): Tile => ({ +export const makeRawSqlTile = (opts?: { + id?: string; + displayType?: DisplayType; + sqlTemplate?: string; + connectionId?: string; +}): Tile => ({ + id: opts?.id ?? randomMongoId(), + x: 1, + y: 1, + w: 1, + h: 1, + config: { + configType: 'sql', + displayType: opts?.displayType ?? DisplayType.Line, + sqlTemplate: opts?.sqlTemplate ?? 'SELECT 1', + connection: opts?.connectionId ?? 'test-connection', + } satisfies RawSqlSavedChartConfig, +}); + +export const RAW_SQL_ALERT_TEMPLATE = [ + 'SELECT toStartOfInterval(Timestamp, INTERVAL {intervalSeconds:Int64} second) AS ts,', + ' count() AS cnt', + ' FROM default.otel_logs', + ' WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})', + ' AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})', + ' GROUP BY ts ORDER BY ts', +].join(''); + +export const makeRawSqlAlertTile = (opts?: { + id?: string; + connectionId?: string; + sqlTemplate?: string; +}): Tile => ({ id: opts?.id ?? randomMongoId(), x: 1, y: 1, @@ -510,8 +550,33 @@ export const makeRawSqlTile = (opts?: { id?: string }): Tile => ({ config: { configType: 'sql', displayType: DisplayType.Line, - sqlTemplate: 'SELECT 1', - connection: 'test-connection', + sqlTemplate: opts?.sqlTemplate ?? RAW_SQL_ALERT_TEMPLATE, + connection: opts?.connectionId ?? 'test-connection', + } satisfies RawSqlSavedChartConfig, +}); + +export const RAW_SQL_NUMBER_ALERT_TEMPLATE = [ + 'SELECT count() AS cnt', + ' FROM default.otel_logs', + ' WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})', + ' AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})', +].join(''); + +export const makeRawSqlNumberAlertTile = (opts?: { + id?: string; + connectionId?: string; + sqlTemplate?: string; +}): Tile => ({ + id: opts?.id ?? randomMongoId(), + x: 1, + y: 1, + w: 1, + h: 1, + config: { + configType: 'sql', + displayType: DisplayType.Number, + sqlTemplate: opts?.sqlTemplate ?? RAW_SQL_NUMBER_ALERT_TEMPLATE, + connection: opts?.connectionId ?? 'test-connection', } satisfies RawSqlSavedChartConfig, }); diff --git a/packages/api/src/mcp/__tests__/dashboards.test.ts b/packages/api/src/mcp/__tests__/dashboards.test.ts new file mode 100644 index 00000000..99d7bde3 --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards.test.ts @@ -0,0 +1,545 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; + +import * as config from '@/config'; +import { + DEFAULT_DATABASE, + DEFAULT_TRACES_TABLE, + getLoggedInAgent, + getServer, +} from '@/fixtures'; +import Connection from '@/models/connection'; +import Dashboard from '@/models/dashboard'; +import { Source } from '@/models/source'; + +import { McpContext } from '../tools/types'; +import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; + +describe('MCP Dashboard Tools', () => { + const server = getServer(); + let team: any; + let user: any; + let traceSource: any; + let connection: any; + let client: Client; + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const result = await getLoggedInAgent(server); + team = result.team; + user = result.user; + + connection = await Connection.create({ + team: team._id, + name: 'Default', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + traceSource = await Source.create({ + kind: SourceKind.Trace, + team: team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_TRACES_TABLE, + }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Traces', + }); + + const context: McpContext = { + teamId: team._id.toString(), + userId: user._id.toString(), + }; + client = await createTestClient(context); + }); + + afterEach(async () => { + await client.close(); + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + describe('hyperdx_list_sources', () => { + it('should list available sources and connections', async () => { + const result = await callTool(client, 'hyperdx_list_sources'); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + + const output = JSON.parse(getFirstText(result)); + expect(output.sources).toHaveLength(1); + expect(output.sources[0]).toMatchObject({ + id: traceSource._id.toString(), + name: 'Traces', + kind: SourceKind.Trace, + }); + + expect(output.connections).toHaveLength(1); + expect(output.connections[0]).toMatchObject({ + id: connection._id.toString(), + name: 'Default', + }); + + expect(output.usage).toBeDefined(); + }); + + it('should include column schema for sources', async () => { + const result = await callTool(client, 'hyperdx_list_sources'); + const output = JSON.parse(getFirstText(result)); + const source = output.sources[0]; + + expect(source.columns).toBeDefined(); + expect(Array.isArray(source.columns)).toBe(true); + expect(source.columns.length).toBeGreaterThan(0); + // Each column should have name, type, and jsType + expect(source.columns[0]).toHaveProperty('name'); + expect(source.columns[0]).toHaveProperty('type'); + expect(source.columns[0]).toHaveProperty('jsType'); + }); + + it('should return empty sources for a team with no sources', async () => { + // Clear everything and re-register with new team + await client.close(); + await server.clearDBs(); + const result2 = await getLoggedInAgent(server); + const context2: McpContext = { + teamId: result2.team._id.toString(), + }; + const client2 = await createTestClient(context2); + + const result = await callTool(client2, 'hyperdx_list_sources'); + const output = JSON.parse(getFirstText(result)); + + expect(output.sources).toHaveLength(0); + expect(output.connections).toHaveLength(0); + + await client2.close(); + }); + }); + + describe('hyperdx_get_dashboard', () => { + it('should list all dashboards when no id provided', async () => { + await new Dashboard({ + name: 'Dashboard 1', + tiles: [], + team: team._id, + tags: ['tag1'], + }).save(); + await new Dashboard({ + name: 'Dashboard 2', + tiles: [], + team: team._id, + tags: ['tag2'], + }).save(); + + const result = await callTool(client, 'hyperdx_get_dashboard', {}); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveLength(2); + expect(output[0]).toHaveProperty('id'); + expect(output[0]).toHaveProperty('name'); + expect(output[0]).toHaveProperty('tags'); + }); + + it('should get dashboard detail when id is provided', async () => { + const dashboard = await new Dashboard({ + name: 'My Dashboard', + tiles: [], + team: team._id, + tags: ['test'], + }).save(); + + const result = await callTool(client, 'hyperdx_get_dashboard', { + id: dashboard._id.toString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.id).toBe(dashboard._id.toString()); + expect(output.name).toBe('My Dashboard'); + expect(output.tags).toEqual(['test']); + expect(output.tiles).toEqual([]); + }); + + it('should return error for non-existent dashboard id', async () => { + const fakeId = '000000000000000000000000'; + const result = await callTool(client, 'hyperdx_get_dashboard', { + id: fakeId, + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + }); + + describe('hyperdx_save_dashboard', () => { + it('should create a new dashboard with tiles', async () => { + const sourceId = traceSource._id.toString(); + const result = await callTool(client, 'hyperdx_save_dashboard', { + name: 'New MCP Dashboard', + tiles: [ + { + name: 'Line Chart', + x: 0, + y: 0, + w: 12, + h: 4, + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count', where: '' }], + }, + }, + ], + tags: ['mcp-test'], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.id).toBeDefined(); + expect(output.name).toBe('New MCP Dashboard'); + expect(output.tiles).toHaveLength(1); + expect(output.tiles[0].config.displayType).toBe('line'); + expect(output.tags).toEqual(['mcp-test']); + + // Verify in database + const dashboard = await Dashboard.findById(output.id); + expect(dashboard).not.toBeNull(); + expect(dashboard?.name).toBe('New MCP Dashboard'); + }); + + it('should create a dashboard with a markdown tile', async () => { + const result = await callTool(client, 'hyperdx_save_dashboard', { + name: 'Markdown Dashboard', + tiles: [ + { + name: 'Notes', + config: { + displayType: 'markdown', + markdown: '# Hello World', + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles).toHaveLength(1); + expect(output.tiles[0].config.displayType).toBe('markdown'); + }); + + it('should update an existing dashboard', async () => { + const sourceId = traceSource._id.toString(); + + // Create first + const createResult = await callTool(client, 'hyperdx_save_dashboard', { + name: 'Original Name', + tiles: [ + { + name: 'Tile 1', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + const created = JSON.parse(getFirstText(createResult)); + + // Update + const updateResult = await callTool(client, 'hyperdx_save_dashboard', { + id: created.id, + name: 'Updated Name', + tiles: [ + { + name: 'Updated Tile', + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + tags: ['updated'], + }); + + expect(updateResult.isError).toBeFalsy(); + const updated = JSON.parse(getFirstText(updateResult)); + expect(updated.id).toBe(created.id); + expect(updated.name).toBe('Updated Name'); + expect(updated.tiles).toHaveLength(1); + expect(updated.tiles[0].name).toBe('Updated Tile'); + expect(updated.tiles[0].config.displayType).toBe('table'); + }); + + it('should return error for missing source ID', async () => { + const fakeSourceId = '000000000000000000000000'; + const result = await callTool(client, 'hyperdx_save_dashboard', { + name: 'Bad Dashboard', + tiles: [ + { + name: 'Bad Tile', + config: { + displayType: 'line', + sourceId: fakeSourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('source'); + }); + + it('should return error when updating non-existent dashboard', async () => { + const sourceId = traceSource._id.toString(); + const result = await callTool(client, 'hyperdx_save_dashboard', { + id: '000000000000000000000000', + name: 'Ghost Dashboard', + tiles: [ + { + name: 'Tile', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + + it('should create a dashboard with multiple tile types', async () => { + const sourceId = traceSource._id.toString(); + const result = await callTool(client, 'hyperdx_save_dashboard', { + name: 'Multi-tile Dashboard', + tiles: [ + { + name: 'Line', + x: 0, + y: 0, + w: 12, + h: 4, + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Table', + x: 0, + y: 4, + w: 12, + h: 4, + config: { + displayType: 'table', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Number', + x: 0, + y: 8, + w: 6, + h: 3, + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Pie', + x: 6, + y: 8, + w: 6, + h: 3, + config: { + displayType: 'pie', + sourceId, + select: [{ aggFn: 'count' }], + groupBy: 'SpanName', + }, + }, + { + name: 'Notes', + x: 0, + y: 11, + w: 12, + h: 2, + config: { displayType: 'markdown', markdown: '# Dashboard Notes' }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles).toHaveLength(5); + }); + + it('should create a dashboard with a raw SQL tile', async () => { + const connectionId = connection._id.toString(); + const result = await callTool(client, 'hyperdx_save_dashboard', { + name: 'SQL Dashboard', + tiles: [ + { + name: 'Raw SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId, + sqlTemplate: 'SELECT 1 AS value LIMIT 1', + }, + }, + ], + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.tiles).toHaveLength(1); + }); + }); + + describe('hyperdx_delete_dashboard', () => { + it('should delete an existing dashboard', async () => { + const dashboard = await new Dashboard({ + name: 'To Delete', + tiles: [], + team: team._id, + }).save(); + + const result = await callTool(client, 'hyperdx_delete_dashboard', { + id: dashboard._id.toString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.deleted).toBe(true); + expect(output.id).toBe(dashboard._id.toString()); + + // Verify deleted from database + const found = await Dashboard.findById(dashboard._id); + expect(found).toBeNull(); + }); + + it('should return error for non-existent dashboard', async () => { + const result = await callTool(client, 'hyperdx_delete_dashboard', { + id: '000000000000000000000000', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + }); + + describe('hyperdx_query_tile', () => { + it('should return error for non-existent dashboard', async () => { + const result = await callTool(client, 'hyperdx_query_tile', { + dashboardId: '000000000000000000000000', + tileId: 'some-tile-id', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + + it('should return error for non-existent tile', async () => { + const sourceId = traceSource._id.toString(); + const createResult = await callTool(client, 'hyperdx_save_dashboard', { + name: 'Tile Query Test', + tiles: [ + { + name: 'My Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + const dashboard = JSON.parse(getFirstText(createResult)); + + const result = await callTool(client, 'hyperdx_query_tile', { + dashboardId: dashboard.id, + tileId: 'non-existent-tile-id', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Tile not found'); + }); + + it('should return error for invalid time range', async () => { + const sourceId = traceSource._id.toString(); + const createResult = await callTool(client, 'hyperdx_save_dashboard', { + name: 'Time Range Test', + tiles: [ + { + name: 'Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + const dashboard = JSON.parse(getFirstText(createResult)); + + const result = await callTool(client, 'hyperdx_query_tile', { + dashboardId: dashboard.id, + tileId: dashboard.tiles[0].id, + startTime: 'not-a-date', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Invalid'); + }); + + it('should execute query for a valid tile', async () => { + const sourceId = traceSource._id.toString(); + const createResult = await callTool(client, 'hyperdx_save_dashboard', { + name: 'Query Tile Test', + tiles: [ + { + name: 'Count Tile', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + const dashboard = JSON.parse(getFirstText(createResult)); + + const result = await callTool(client, 'hyperdx_query_tile', { + dashboardId: dashboard.id, + tileId: dashboard.tiles[0].id, + startTime: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + // Should succeed (may have empty results since no data inserted) + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + }); +}); diff --git a/packages/api/src/mcp/__tests__/mcpTestUtils.ts b/packages/api/src/mcp/__tests__/mcpTestUtils.ts new file mode 100644 index 00000000..82550a98 --- /dev/null +++ b/packages/api/src/mcp/__tests__/mcpTestUtils.ts @@ -0,0 +1,53 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { + type CallToolResult, + CallToolResultSchema, +} from '@modelcontextprotocol/sdk/types.js'; + +import { createServer } from '../mcpServer'; +import { McpContext } from '../tools/types'; + +/** + * Connect an MCP server to an in-process Client via InMemoryTransport and + * return the client. This is the officially supported way to test MCP servers + * without accessing private SDK internals. + */ +export async function createTestClient(context: McpContext): Promise { + const mcpServer = createServer(context); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await mcpServer.connect(serverTransport); + const client = new Client({ name: 'test-client', version: '1.0.0' }); + await client.connect(clientTransport); + return client; +} + +/** + * Call a named MCP tool and return a properly-typed result. + * + * The SDK's `Client.callTool()` return type carries an index signature + * `[x: string]: unknown` that widens all property accesses to `unknown`. + * Re-parsing through `CallToolResultSchema` gives the concrete named type + * needed for clean test assertions. + */ +export async function callTool( + c: Client, + name: string, + args: Record = {}, +): Promise { + const raw = await c.callTool({ name, arguments: args }); + return CallToolResultSchema.parse(raw); +} + +/** + * Extract the text from the first content item of a tool result. + * Throws if the item is not a text block. + */ +export function getFirstText(result: CallToolResult): string { + const item = result.content[0]; + if (!item || item.type !== 'text') { + throw new Error(`Expected text content, got: ${JSON.stringify(item)}`); + } + return item.text; +} diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts new file mode 100644 index 00000000..6f6f31b5 --- /dev/null +++ b/packages/api/src/mcp/__tests__/query.test.ts @@ -0,0 +1,74 @@ +import { parseTimeRange } from '../tools/query/helpers'; + +describe('parseTimeRange', () => { + it('should return default range (last 15 minutes) when no arguments provided', () => { + const before = Date.now(); + const result = parseTimeRange(); + const after = Date.now(); + + expect(result).not.toHaveProperty('error'); + if ('error' in result) return; + + // endDate should be approximately now + expect(result.endDate.getTime()).toBeGreaterThanOrEqual(before); + expect(result.endDate.getTime()).toBeLessThanOrEqual(after); + // startDate should be ~15 minutes before endDate + const diffMs = result.endDate.getTime() - result.startDate.getTime(); + expect(diffMs).toBe(15 * 60 * 1000); + }); + + it('should use provided startTime and endTime', () => { + const result = parseTimeRange( + '2025-01-01T00:00:00Z', + '2025-01-02T00:00:00Z', + ); + expect(result).not.toHaveProperty('error'); + if ('error' in result) return; + + expect(result.startDate.toISOString()).toBe('2025-01-01T00:00:00.000Z'); + expect(result.endDate.toISOString()).toBe('2025-01-02T00:00:00.000Z'); + }); + + it('should default startTime to 15 minutes before endTime', () => { + const result = parseTimeRange(undefined, '2025-06-15T10:00:00Z'); + expect(result).not.toHaveProperty('error'); + if ('error' in result) return; + + expect(result.endDate.toISOString()).toBe('2025-06-15T10:00:00.000Z'); + expect(result.startDate.toISOString()).toBe('2025-06-15T09:45:00.000Z'); + }); + + it('should default endTime to now', () => { + const before = Date.now(); + const result = parseTimeRange('2025-06-15T11:00:00Z'); + const after = Date.now(); + + expect(result).not.toHaveProperty('error'); + if ('error' in result) return; + + expect(result.startDate.toISOString()).toBe('2025-06-15T11:00:00.000Z'); + expect(result.endDate.getTime()).toBeGreaterThanOrEqual(before); + expect(result.endDate.getTime()).toBeLessThanOrEqual(after); + }); + + it('should return error for invalid startTime', () => { + const result = parseTimeRange('not-a-date', '2025-01-01T00:00:00Z'); + expect(result).toHaveProperty('error'); + if (!('error' in result)) return; + + expect(result.error).toContain('Invalid'); + }); + + it('should return error for invalid endTime', () => { + const result = parseTimeRange('2025-01-01T00:00:00Z', 'garbage'); + expect(result).toHaveProperty('error'); + if (!('error' in result)) return; + + expect(result.error).toContain('Invalid'); + }); + + it('should return error when both times are invalid', () => { + const result = parseTimeRange('bad', 'also-bad'); + expect(result).toHaveProperty('error'); + }); +}); diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts new file mode 100644 index 00000000..dcbacc33 --- /dev/null +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -0,0 +1,233 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; + +import * as config from '@/config'; +import { + DEFAULT_DATABASE, + DEFAULT_TRACES_TABLE, + getLoggedInAgent, + getServer, +} from '@/fixtures'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; + +import { McpContext } from '../tools/types'; +import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; + +describe('MCP Query Tool', () => { + const server = getServer(); + let team: any; + let user: any; + let traceSource: any; + let connection: any; + let client: Client; + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const result = await getLoggedInAgent(server); + team = result.team; + user = result.user; + + connection = await Connection.create({ + team: team._id, + name: 'Default', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + traceSource = await Source.create({ + kind: SourceKind.Trace, + team: team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_TRACES_TABLE, + }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Traces', + }); + + const context: McpContext = { + teamId: team._id.toString(), + userId: user._id.toString(), + }; + client = await createTestClient(context); + }); + + afterEach(async () => { + await client.close(); + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + describe('builder queries', () => { + it('should execute a number query', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'number', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveProperty('result'); + }); + + it('should execute a line chart query', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'line', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should execute a table query', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'table', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + groupBy: 'SpanName', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should execute a pie query', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'pie', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + groupBy: 'SpanName', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should execute a stacked_bar query', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'stacked_bar', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should use default time range when not provided', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'number', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should return result for query with no matching data', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'number', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count', where: 'SpanName:z_impossible_value_xyz' }], + startTime: new Date(Date.now() - 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + }); + + describe('search queries', () => { + it('should execute a search query', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'search', + sourceId: traceSource._id.toString(), + where: '', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should respect maxResults parameter', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'search', + sourceId: traceSource._id.toString(), + maxResults: 10, + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + }); + }); + + describe('SQL queries', () => { + it('should execute a raw SQL query', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'sql', + connectionId: connection._id.toString(), + sql: 'SELECT 1 AS value', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should execute SQL with time macros', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'sql', + connectionId: connection._id.toString(), + sql: `SELECT count() AS cnt FROM ${DEFAULT_DATABASE}.${DEFAULT_TRACES_TABLE} WHERE $__timeFilter(Timestamp) LIMIT 10`, + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + }); + + describe('error handling', () => { + it('should return error for invalid time range', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'number', + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + startTime: 'invalid-date', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Invalid'); + }); + }); +}); diff --git a/packages/api/src/mcp/__tests__/tracing.test.ts b/packages/api/src/mcp/__tests__/tracing.test.ts new file mode 100644 index 00000000..2fda2191 --- /dev/null +++ b/packages/api/src/mcp/__tests__/tracing.test.ts @@ -0,0 +1,160 @@ +// Mock OpenTelemetry and all modules that transitively import it +// These must be declared before any imports + +const mockSpan = { + setAttribute: jest.fn(), + setStatus: jest.fn(), + recordException: jest.fn(), + end: jest.fn(), +}; + +const mockTracer = { + startActiveSpan: ( + _name: string, + fn: (span: typeof mockSpan) => Promise, + ) => fn(mockSpan), +}; + +jest.mock('@opentelemetry/api', () => ({ + __esModule: true, + default: { + trace: { + getTracer: () => mockTracer, + }, + }, + SpanStatusCode: { + OK: 1, + ERROR: 2, + }, +})); + +jest.mock('@/config', () => ({ + CODE_VERSION: 'test-version', +})); + +jest.mock('@/utils/logger', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +import { withToolTracing } from '../utils/tracing'; + +describe('withToolTracing', () => { + const context = { teamId: 'team-123', userId: 'user-456' }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should call the handler and return its result', async () => { + const handler = jest.fn().mockResolvedValue({ + content: [{ type: 'text', text: 'hello' }], + }); + + const traced = withToolTracing('test_tool', context, handler); + const result = await traced({ some: 'args' }); + + expect(handler).toHaveBeenCalledWith({ some: 'args' }); + expect(result).toEqual({ + content: [{ type: 'text', text: 'hello' }], + }); + }); + + it('should set span attributes for tool name, team, and user', async () => { + const handler = jest.fn().mockResolvedValue({ + content: [{ type: 'text', text: 'ok' }], + }); + + const traced = withToolTracing('my_tool', context, handler); + await traced({}); + + expect(mockSpan.setAttribute).toHaveBeenCalledWith( + 'mcp.tool.name', + 'my_tool', + ); + expect(mockSpan.setAttribute).toHaveBeenCalledWith( + 'mcp.team.id', + 'team-123', + ); + expect(mockSpan.setAttribute).toHaveBeenCalledWith( + 'mcp.user.id', + 'user-456', + ); + }); + + it('should not set user id attribute when userId is undefined', async () => { + const noUserContext = { teamId: 'team-123' }; + const handler = jest.fn().mockResolvedValue({ + content: [{ type: 'text', text: 'ok' }], + }); + + const traced = withToolTracing('my_tool', noUserContext, handler); + await traced({}); + + expect(mockSpan.setAttribute).not.toHaveBeenCalledWith( + 'mcp.user.id', + expect.anything(), + ); + }); + + it('should set OK status for successful results', async () => { + const handler = jest.fn().mockResolvedValue({ + content: [{ type: 'text', text: 'ok' }], + }); + + const traced = withToolTracing('my_tool', context, handler); + await traced({}); + + expect(mockSpan.setStatus).toHaveBeenCalledWith({ code: 1 }); // SpanStatusCode.OK + expect(mockSpan.end).toHaveBeenCalled(); + }); + + it('should set ERROR status for isError results', async () => { + const handler = jest.fn().mockResolvedValue({ + isError: true, + content: [{ type: 'text', text: 'something went wrong' }], + }); + + const traced = withToolTracing('my_tool', context, handler); + await traced({}); + + expect(mockSpan.setStatus).toHaveBeenCalledWith({ code: 2 }); // SpanStatusCode.ERROR + expect(mockSpan.setAttribute).toHaveBeenCalledWith('mcp.tool.error', true); + expect(mockSpan.end).toHaveBeenCalled(); + }); + + it('should set ERROR status and re-throw on handler exception', async () => { + const error = new Error('boom'); + const handler = jest.fn().mockRejectedValue(error); + + const traced = withToolTracing('my_tool', context, handler); + + await expect(traced({})).rejects.toThrow('boom'); + + expect(mockSpan.setStatus).toHaveBeenCalledWith({ + code: 2, + message: 'boom', + }); + expect(mockSpan.recordException).toHaveBeenCalledWith(error); + expect(mockSpan.end).toHaveBeenCalled(); + }); + + it('should record duration on the span', async () => { + const handler = jest.fn().mockResolvedValue({ + content: [{ type: 'text', text: 'ok' }], + }); + + const traced = withToolTracing('my_tool', context, handler); + await traced({}); + + expect(mockSpan.setAttribute).toHaveBeenCalledWith( + 'mcp.tool.duration_ms', + expect.any(Number), + ); + }); +}); diff --git a/packages/api/src/mcp/app.ts b/packages/api/src/mcp/app.ts new file mode 100644 index 00000000..970bbe0d --- /dev/null +++ b/packages/api/src/mcp/app.ts @@ -0,0 +1,58 @@ +import { setTraceAttributes } from '@hyperdx/node-opentelemetry'; +import { createMcpExpressApp } from '@modelcontextprotocol/sdk/server/express.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; + +import { validateUserAccessKey } from '../middleware/auth'; +import logger from '../utils/logger'; +import rateLimiter, { rateLimiterKeyGenerator } from '../utils/rateLimiter'; +import { createServer } from './mcpServer'; +import { McpContext } from './tools/types'; + +const app = createMcpExpressApp(); + +const mcpRateLimiter = rateLimiter({ + windowMs: 60 * 1000, // 1 minute + max: 100, + standardHeaders: true, + legacyHeaders: false, + keyGenerator: rateLimiterKeyGenerator, +}); + +app.all('/', mcpRateLimiter, validateUserAccessKey, async (req, res) => { + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, // stateless + }); + + const teamId = req.user?.team; + + if (!teamId) { + logger.warn('MCP request rejected: no teamId'); + res.sendStatus(403); + return; + } + + const userId = req.user?._id?.toString(); + const context: McpContext = { + teamId: teamId.toString(), + userId, + }; + + setTraceAttributes({ + 'mcp.team.id': context.teamId, + ...(userId && { 'mcp.user.id': userId }), + }); + + logger.info({ teamId: context.teamId, userId }, 'MCP request received'); + + const server = createServer(context); + + try { + await server.connect(transport); + await transport.handleRequest(req, res, req.body); + } finally { + await server.close(); + await transport.close(); + } +}); + +export default app; diff --git a/packages/api/src/mcp/mcpServer.ts b/packages/api/src/mcp/mcpServer.ts new file mode 100644 index 00000000..df9592a1 --- /dev/null +++ b/packages/api/src/mcp/mcpServer.ts @@ -0,0 +1,21 @@ +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; + +import { CODE_VERSION } from '@/config'; + +import dashboardPrompts from './prompts/dashboards/index'; +import dashboardsTools from './tools/dashboards/index'; +import queryTools from './tools/query/index'; +import { McpContext } from './tools/types'; + +export function createServer(context: McpContext) { + const server = new McpServer({ + name: 'hyperdx', + version: `${CODE_VERSION}-beta`, + }); + + dashboardsTools(server, context); + queryTools(server, context); + dashboardPrompts(server, context); + + return server; +} diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts new file mode 100644 index 00000000..0015b232 --- /dev/null +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -0,0 +1,719 @@ +// ─── Prompt content builders ────────────────────────────────────────────────── +// Each function returns a plain string that is injected as a prompt message. + +export function buildCreateDashboardPrompt( + sourceSummary: string, + traceSourceId: string, + logSourceId: string, + description?: string, +): string { + const userContext = description + ? `\nThe user wants to create a dashboard for: ${description}\nTailor the dashboard tiles to match this goal.\n` + : ''; + + return `You are an expert at creating HyperDX observability dashboards. +${userContext} +${sourceSummary} + +IMPORTANT: Call hyperdx_list_sources first to get the full column schema and attribute keys for each source. The source IDs above are correct, but you need the schema details to write accurate queries. + +== WORKFLOW == + +1. Call hyperdx_list_sources — get source IDs, column schemas, and attribute keys +2. Design tiles — pick tile types that match the monitoring goal +3. Call hyperdx_save_dashboard — create the dashboard with all tiles +4. Call hyperdx_query_tile on each tile — validate queries return data + +== TILE TYPE GUIDE == + +Use BUILDER tiles (with sourceId) for most cases: + line — Time-series trends (error rate, request volume, latency over time) + stacked_bar — Compare categories over time (requests by service, errors by status code) + number — Single KPI metric (total requests, current error rate, p99 latency) + table — Ranked lists (top endpoints by latency, error counts by service) + pie — Proportional breakdowns (traffic share by service, errors by type) + search — Browse raw log/event rows (error logs, recent traces) + markdown — Dashboard notes, section headers, or documentation + +Use RAW SQL tiles (with connectionId) only for advanced queries: + Requires configType: "sql" plus a displayType (line, stacked_bar, table, number, pie) + Use when you need JOINs, sub-queries, CTEs, or queries the builder cannot express + +== COLUMN NAMING == + +- Top-level columns use PascalCase by default: Duration, StatusCode, SpanName, Body, SeverityText, ServiceName + NOTE: These are defaults for the standard HyperDX schema. Custom sources may use different names. + Always call hyperdx_list_sources to get the real column names and keyColumns for each source. +- Map-type columns use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name'] + NEVER use dot notation for Map columns (SpanAttributes.http.method) — always use brackets. +- JSON-type columns use dot notation: JsonColumn.key.subkey + Check the jsType returned by hyperdx_list_sources to determine whether a column is Map or JSON. +- Call hyperdx_list_sources to discover the exact column names, types, and attribute keys + +== LAYOUT GRID == + +The dashboard grid is 24 columns wide. Tiles are positioned with (x, y, w, h): + - Number tiles: w=6, h=4 — fit 4 across in a row + - Line/Bar charts: w=12, h=4 — fit 2 side-by-side + - Tables: w=24, h=6 — full width + - Search tiles: w=24, h=6 — full width + - Markdown: w=24, h=2 — full width section header + +Recommended layout pattern (top to bottom): + Row 0: KPI number tiles across the top (y=0) + Row 1: Time-series charts (y=4) + Row 2: Tables or search tiles (y=8) + +== FILTER SYNTAX (Lucene) == + +Simple match: level:error +AND: service.name:api AND http.status_code:>=500 +OR: level:error OR level:fatal +Wildcards: service.name:front* +Negation: NOT level:debug +Exists: _exists_:http.route +Range: Duration:>1000000000 +Phrase: Body:"connection refused" +Grouped: (level:error OR level:fatal) AND service.name:api + +== COMPLETE EXAMPLE == + +Here is a full dashboard creation call with properly structured tiles: + +hyperdx_save_dashboard({ + name: "Service Overview", + tags: ["overview"], + tiles: [ + { + name: "Total Requests", + x: 0, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "count" }] + } + }, + { + name: "Error Count", + x: 6, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "count", where: "StatusCode:STATUS_CODE_ERROR" }] + } + }, + { + name: "P95 Latency (ms)", + x: 12, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "quantile", valueExpression: "Duration", level: 0.95 }] + } + }, + { + name: "Request Rate by Service", + x: 0, y: 4, w: 12, h: 4, + config: { + displayType: "line", + sourceId: "${traceSourceId}", + select: [{ aggFn: "count" }], + groupBy: "ResourceAttributes['service.name']" + } + }, + { + name: "Error Rate Over Time", + x: 12, y: 4, w: 12, h: 4, + config: { + displayType: "line", + sourceId: "${traceSourceId}", + select: [ + { aggFn: "count", where: "StatusCode:STATUS_CODE_ERROR", alias: "Errors" }, + { aggFn: "count", alias: "Total" } + ], + asRatio: true + } + }, + { + name: "Top Endpoints by Request Count", + x: 0, y: 8, w: 24, h: 6, + config: { + displayType: "table", + sourceId: "${traceSourceId}", + groupBy: "SpanName", + select: [ + { aggFn: "count", alias: "Requests" }, + { aggFn: "avg", valueExpression: "Duration", alias: "Avg Duration" }, + { aggFn: "quantile", valueExpression: "Duration", level: 0.95, alias: "P95 Duration" } + ] + } + } + ] +}) + +== STATUS CODE & SEVERITY VALUES == + +IMPORTANT: The exact values for StatusCode and SeverityText vary by deployment. +Do NOT assume values like "STATUS_CODE_ERROR", "Ok", "error", or "fatal". +Always call hyperdx_list_sources first and inspect the keyValues / mapAttributeKeys +returned for each source to discover the real values used in your data. + +== COMMON MISTAKES TO AVOID == + +- Using valueExpression with aggFn "count" — count does not take a valueExpression +- Forgetting valueExpression for non-count aggFns — avg, sum, min, max, quantile all require it +- Using dot notation for Map-type attributes — always use SpanAttributes['key'] bracket syntax for Map columns +- Not calling hyperdx_list_sources first — you need real source IDs, not placeholders +- Not validating with hyperdx_query_tile after saving — tiles can silently fail +- Number and Pie tiles accept exactly 1 select item — not multiple +- Missing level for quantile aggFn — must specify 0.5, 0.9, 0.95, or 0.99 +- Assuming StatusCode or SeverityText values — always inspect the source first`; +} + +export function buildDashboardExamplesPrompt( + traceSourceId: string, + logSourceId: string, + connectionId: string, + pattern?: string, +): string { + const examples: Record = {}; + + examples['service_overview'] = ` +== SERVICE HEALTH OVERVIEW == + +A high-level view of service health with KPIs, trends, and endpoint details. + +{ + name: "Service Health Overview", + tags: ["overview", "service"], + tiles: [ + { + name: "Total Requests", + x: 0, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "count" }] + } + }, + { + name: "Error Count", + x: 6, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "count", where: "StatusCode:STATUS_CODE_ERROR" }] + } + }, + { + name: "Avg Latency", + x: 12, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "avg", valueExpression: "Duration" }] + } + }, + { + name: "P99 Latency", + x: 18, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "quantile", valueExpression: "Duration", level: 0.99 }] + } + }, + { + name: "Request Volume Over Time", + x: 0, y: 4, w: 12, h: 4, + config: { + displayType: "line", + sourceId: "${traceSourceId}", + select: [{ aggFn: "count" }], + groupBy: "ResourceAttributes['service.name']" + } + }, + { + name: "Error Rate Over Time", + x: 12, y: 4, w: 12, h: 4, + config: { + displayType: "line", + sourceId: "${traceSourceId}", + select: [ + { aggFn: "count", where: "StatusCode:STATUS_CODE_ERROR", alias: "Errors" }, + { aggFn: "count", alias: "Total" } + ], + asRatio: true + } + }, + { + name: "Top Endpoints", + x: 0, y: 8, w: 24, h: 6, + config: { + displayType: "table", + sourceId: "${traceSourceId}", + groupBy: "SpanName", + select: [ + { aggFn: "count", alias: "Requests" }, + { aggFn: "avg", valueExpression: "Duration", alias: "Avg Duration" }, + { aggFn: "quantile", valueExpression: "Duration", level: 0.95, alias: "P95" }, + { aggFn: "count", where: "StatusCode:STATUS_CODE_ERROR", alias: "Errors" } + ] + } + } + ] +}`; + + examples['error_tracking'] = ` +== ERROR TRACKING == + +Focus on errors: volume, distribution, and raw error logs. + +{ + name: "Error Tracking", + tags: ["errors"], + tiles: [ + { + name: "Total Errors", + x: 0, y: 0, w: 8, h: 4, + config: { + displayType: "number", + sourceId: "${logSourceId}", + select: [{ aggFn: "count", where: "SeverityText:error OR SeverityText:fatal" }] + } + }, + { + name: "Errors Over Time by Service", + x: 0, y: 4, w: 12, h: 4, + config: { + displayType: "line", + sourceId: "${logSourceId}", + select: [{ aggFn: "count", where: "SeverityText:error OR SeverityText:fatal" }], + groupBy: "ResourceAttributes['service.name']" + } + }, + { + name: "Error Breakdown by Service", + x: 12, y: 4, w: 12, h: 4, + config: { + displayType: "pie", + sourceId: "${logSourceId}", + select: [{ aggFn: "count", where: "SeverityText:error OR SeverityText:fatal" }], + groupBy: "ResourceAttributes['service.name']" + } + }, + { + name: "Error Logs", + x: 0, y: 8, w: 24, h: 6, + config: { + displayType: "search", + sourceId: "${logSourceId}", + where: "SeverityText:error OR SeverityText:fatal" + } + } + ] +}`; + + examples['latency'] = ` +== LATENCY MONITORING == + +Track response times with percentile breakdowns and slow endpoint identification. + +{ + name: "Latency Monitoring", + tags: ["latency", "performance"], + tiles: [ + { + name: "P50 Latency", + x: 0, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "quantile", valueExpression: "Duration", level: 0.5 }] + } + }, + { + name: "P95 Latency", + x: 6, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "quantile", valueExpression: "Duration", level: 0.95 }] + } + }, + { + name: "P99 Latency", + x: 12, y: 0, w: 6, h: 4, + config: { + displayType: "number", + sourceId: "${traceSourceId}", + select: [{ aggFn: "quantile", valueExpression: "Duration", level: 0.99 }] + } + }, + { + name: "Latency Percentiles Over Time", + x: 0, y: 4, w: 24, h: 4, + config: { + displayType: "line", + sourceId: "${traceSourceId}", + select: [ + { aggFn: "quantile", valueExpression: "Duration", level: 0.5, alias: "P50" }, + { aggFn: "quantile", valueExpression: "Duration", level: 0.95, alias: "P95" }, + { aggFn: "quantile", valueExpression: "Duration", level: 0.99, alias: "P99" } + ] + } + }, + { + name: "Latency by Service", + x: 0, y: 8, w: 12, h: 4, + config: { + displayType: "stacked_bar", + sourceId: "${traceSourceId}", + select: [{ aggFn: "avg", valueExpression: "Duration" }], + groupBy: "ResourceAttributes['service.name']" + } + }, + { + name: "Slowest Endpoints", + x: 12, y: 8, w: 12, h: 6, + config: { + displayType: "table", + sourceId: "${traceSourceId}", + groupBy: "SpanName", + select: [ + { aggFn: "quantile", valueExpression: "Duration", level: 0.95, alias: "P95 Duration" }, + { aggFn: "avg", valueExpression: "Duration", alias: "Avg Duration" }, + { aggFn: "count", alias: "Request Count" } + ] + } + } + ] +}`; + + examples['log_analysis'] = ` +== LOG ANALYSIS == + +Analyze log volume, severity distribution, and browse log events. + +{ + name: "Log Analysis", + tags: ["logs"], + tiles: [ + { + name: "Total Log Events", + x: 0, y: 0, w: 8, h: 4, + config: { + displayType: "number", + sourceId: "${logSourceId}", + select: [{ aggFn: "count" }] + } + }, + { + name: "Log Volume by Severity", + x: 0, y: 4, w: 12, h: 4, + config: { + displayType: "stacked_bar", + sourceId: "${logSourceId}", + select: [{ aggFn: "count" }], + groupBy: "SeverityText" + } + }, + { + name: "Severity Breakdown", + x: 12, y: 4, w: 12, h: 4, + config: { + displayType: "pie", + sourceId: "${logSourceId}", + select: [{ aggFn: "count" }], + groupBy: "SeverityText" + } + }, + { + name: "Top Services by Log Volume", + x: 0, y: 8, w: 12, h: 6, + config: { + displayType: "table", + sourceId: "${logSourceId}", + groupBy: "ResourceAttributes['service.name']", + select: [ + { aggFn: "count", alias: "Log Count" }, + { aggFn: "count", where: "SeverityText:error OR SeverityText:fatal", alias: "Error Count" } + ] + } + }, + { + name: "Recent Logs", + x: 12, y: 8, w: 12, h: 6, + config: { + displayType: "search", + sourceId: "${logSourceId}" + } + } + ] +}`; + + examples['infrastructure_sql'] = ` +== INFRASTRUCTURE MONITORING (Raw SQL) == + +Advanced dashboard using raw SQL tiles for custom ClickHouse queries. +Use this pattern when you need JOINs, CTEs, or queries the builder cannot express. + +{ + name: "Infrastructure (SQL)", + tags: ["infrastructure", "sql"], + tiles: [ + { + name: "Log Ingestion Rate Over Time", + x: 0, y: 0, w: 12, h: 4, + config: { + configType: "sql", + displayType: "line", + connectionId: "${connectionId}", + sqlTemplate: "SELECT $__timeInterval(Timestamp) AS ts, count() AS logs_per_interval FROM otel_logs WHERE $__timeFilter(Timestamp) GROUP BY ts ORDER BY ts" + } + }, + { + name: "Top 20 Services by Span Count", + x: 12, y: 0, w: 12, h: 4, + config: { + configType: "sql", + displayType: "table", + connectionId: "${connectionId}", + sqlTemplate: "SELECT ServiceName, count() AS span_count, avg(Duration) AS avg_duration FROM otel_traces WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName ORDER BY span_count DESC LIMIT 20" + } + }, + { + name: "Error Rate by Service (SQL)", + x: 0, y: 4, w: 24, h: 4, + config: { + configType: "sql", + displayType: "line", + connectionId: "${connectionId}", + sqlTemplate: "SELECT $__timeInterval(Timestamp) AS ts, ServiceName, countIf(StatusCode = 'STATUS_CODE_ERROR') / count() AS error_rate FROM otel_traces WHERE $__timeFilter(Timestamp) GROUP BY ServiceName, ts ORDER BY ts" + } + } + ] +} + +SQL TEMPLATE REFERENCE: + Macros (expanded before execution): + $__timeFilter(col) — col >= AND col <= (DateTime) + $__timeFilter_ms(col) — same with DateTime64 millisecond precision + $__dateFilter(col) — same with Date precision + $__timeInterval(col) — time bucket: toStartOfInterval(toDateTime(col), INTERVAL ...) + $__timeInterval_ms(col) — same with millisecond precision + $__fromTime / $__toTime — start/end as DateTime values + $__fromTime_ms / $__toTime_ms — start/end as DateTime64 values + $__interval_s — raw interval in seconds + $__filters — dashboard filter conditions (resolves to 1=1 when none) + + Query parameters: + {startDateMilliseconds:Int64} — start of date range in milliseconds + {endDateMilliseconds:Int64} — end of date range in milliseconds + {intervalSeconds:Int64} — time bucket size in seconds + {intervalMilliseconds:Int64} — time bucket size in milliseconds + + Available parameters by displayType: + line / stacked_bar — startDate, endDate, interval (all available) + table / number / pie — startDate, endDate only (no interval)`; + + if (pattern) { + const key = pattern.toLowerCase().replace(/[\s-]+/g, '_'); + const matched = Object.entries(examples).find(([k]) => k === key); + if (matched) { + return `Dashboard example for pattern: ${pattern}\n\nReplace sourceId/connectionId values with real IDs from hyperdx_list_sources.\nNOTE: Column names below (Duration, StatusCode, SpanName, etc.) are defaults for the standard schema. Call hyperdx_list_sources to get the actual column names for your sources.\n${matched[1]}`; + } + return ( + `No example found for pattern "${pattern}". Available patterns: ${Object.keys(examples).join(', ')}\n\n` + + `Showing all examples below.\n\n` + + Object.values(examples).join('\n') + ); + } + + return ( + `Complete dashboard examples for common observability patterns.\n` + + `Replace sourceId/connectionId values with real IDs from hyperdx_list_sources.\n` + + `NOTE: Column names below (Duration, StatusCode, SpanName, etc.) are defaults for the standard schema. Call hyperdx_list_sources to get the actual column names for your sources.\n\n` + + `Available patterns: ${Object.keys(examples).join(', ')}\n` + + Object.values(examples).join('\n') + ); +} + +export function buildQueryGuidePrompt(): string { + return `Reference guide for writing queries with HyperDX MCP tools (hyperdx_query and hyperdx_save_dashboard). + +== AGGREGATION FUNCTIONS (aggFn) == + + count — Count matching rows. Does NOT take a valueExpression. + sum — Sum of a numeric column. Requires valueExpression. + avg — Average of a numeric column. Requires valueExpression. + min — Minimum value. Requires valueExpression. + max — Maximum value. Requires valueExpression. + count_distinct — Count of unique values. Requires valueExpression. + quantile — Percentile value. Requires valueExpression AND level (0.5, 0.9, 0.95, or 0.99). + last_value — Most recent value of a column. Requires valueExpression. + none — Pass a raw expression unchanged. Requires valueExpression. + +Examples: + { aggFn: "count" } + { aggFn: "avg", valueExpression: "Duration" } + { aggFn: "quantile", valueExpression: "Duration", level: 0.95 } + { aggFn: "count_distinct", valueExpression: "ResourceAttributes['service.name']" } + { aggFn: "sum", valueExpression: "Duration", where: "StatusCode:STATUS_CODE_ERROR" } + +== COLUMN NAMING == + +Top-level columns (PascalCase defaults — use directly in valueExpression and groupBy): + Duration, StatusCode, SpanName, ServiceName, Body, SeverityText, + Timestamp, TraceId, SpanId, SpanKind, ParentSpanId + NOTE: These are the defaults for the standard HyperDX schema. Custom sources may + use different column names. Always verify with hyperdx_list_sources, which returns + the real column names and keyColumns expressions for each source. + +Map-type columns (bracket syntax — access keys via ['key']): + SpanAttributes['http.method'] + SpanAttributes['http.route'] + SpanAttributes['http.status_code'] + ResourceAttributes['service.name'] + ResourceAttributes['deployment.environment'] + +IMPORTANT: Always use bracket syntax for Map-type columns. Never use dot notation for Maps. + Correct: SpanAttributes['http.method'] + Incorrect: SpanAttributes.http.method + +JSON-type columns (dot notation — access nested keys via dot path): + JsonColumn.key.subkey + NOTE: Check the jsType field returned by hyperdx_list_sources to determine + whether a column is Map (use brackets) or JSON (use dots). + +== LUCENE FILTER SYNTAX == + +Used in the "where" field of select items and search tiles. + + Basic match: level:error + AND: service.name:api AND http.status_code:>=500 + OR: level:error OR level:fatal + NOT: NOT level:debug + Wildcard: service.name:front* + Phrase: Body:"connection refused" + Exists: _exists_:http.route + Range (numeric): Duration:>1000000000 + Range (inclusive): http.status_code:[400 TO 499] + Grouped: (level:error OR level:fatal) AND service.name:api + +NOTE: In Lucene filters, use dot notation for attribute keys (service.name, http.method). +This is different from valueExpression/groupBy which requires bracket syntax (SpanAttributes['http.method']). + +== SQL FILTER SYNTAX == + +Alternative to Lucene. Set whereLanguage: "sql" when using SQL syntax. + + Basic: SeverityText = 'error' + AND/OR: ServiceName = 'api' AND StatusCode = 'STATUS_CODE_ERROR' + IN: ServiceName IN ('api', 'web', 'worker') + LIKE: Body LIKE '%timeout%' + Comparison: Duration > 1000000000 + Map access: SpanAttributes['http.status_code'] = '500' + +== RAW SQL TEMPLATES == + +For configType: "sql" tiles, write ClickHouse SQL with template macros: + + MACROS (expanded before execution): + $__timeFilter(col) — col >= AND col <= + $__timeFilter_ms(col) — same with DateTime64 millisecond precision + $__dateFilter(col) — same with Date precision + $__dateTimeFilter(d, t) — filters on both Date and DateTime columns + $__timeInterval(col) — time bucket expression for GROUP BY + $__timeInterval_ms(col) — same with millisecond precision + $__fromTime / $__toTime — start/end as DateTime values + $__fromTime_ms / $__toTime_ms — start/end as DateTime64 values + $__interval_s — raw interval in seconds (for arithmetic) + $__filters — dashboard filter conditions (1=1 when none) + + QUERY PARAMETERS (ClickHouse parameterized syntax): + {startDateMilliseconds:Int64} + {endDateMilliseconds:Int64} + {intervalSeconds:Int64} + {intervalMilliseconds:Int64} + + TIME-SERIES EXAMPLE (line / stacked_bar): + SELECT + $__timeInterval(Timestamp) AS ts, + ServiceName, + count() AS requests + FROM otel_traces + WHERE $__timeFilter(Timestamp) + GROUP BY ServiceName, ts + ORDER BY ts + + TABLE EXAMPLE: + SELECT + ServiceName, + count() AS request_count, + avg(Duration) AS avg_duration, + quantile(0.95)(Duration) AS p95_duration + FROM otel_traces + WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) + AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) + GROUP BY ServiceName + ORDER BY request_count DESC + LIMIT 50 + + IMPORTANT: Always include a LIMIT clause in table/number/pie SQL queries. + +== PER-TILE TYPE CONSTRAINTS == + + number — Exactly 1 select item. No groupBy. + pie — Exactly 1 select item. groupBy defines the slices. + line — 1-20 select items. Optional groupBy splits into series. + stacked_bar — 1-20 select items. Optional groupBy splits into stacks. + table — 1-20 select items. Optional groupBy defines row groups. + search — No select items (select is a column list string). where is the filter. + markdown — No select items. Set markdown field with content. + +== asRatio == + +Set asRatio: true on line/stacked_bar/table tiles with exactly 2 select items +to plot the first as a ratio of the second. Useful for error rates: + select: [ + { aggFn: "count", where: "StatusCode:STATUS_CODE_ERROR", alias: "Errors" }, + { aggFn: "count", alias: "Total" } + ], + asRatio: true + +== COMMON MISTAKES == + +1. Using valueExpression with aggFn "count" + Wrong: { aggFn: "count", valueExpression: "Duration" } + Correct: { aggFn: "count" } + +2. Forgetting valueExpression for non-count aggFns + Wrong: { aggFn: "avg" } + Correct: { aggFn: "avg", valueExpression: "Duration" } + +3. Using dot notation for Map-type attributes in valueExpression/groupBy + Wrong: groupBy: "SpanAttributes.http.method" + Correct: groupBy: "SpanAttributes['http.method']" + NOTE: JSON-type columns DO use dot notation. Check jsType from hyperdx_list_sources. + +4. Multiple select items on number/pie tiles + Wrong: displayType: "number", select: [{ aggFn: "count" }, { aggFn: "avg", ... }] + Correct: displayType: "number", select: [{ aggFn: "count" }] + +5. Missing level for quantile + Wrong: { aggFn: "quantile", valueExpression: "Duration" } + Correct: { aggFn: "quantile", valueExpression: "Duration", level: 0.95 } + +6. Forgetting to validate tiles after saving + Always call hyperdx_query_tile after hyperdx_save_dashboard to verify each tile returns data. + +7. Using sourceId with SQL tiles or connectionId with builder tiles + Builder tiles (line, table, etc.) use sourceId. + SQL tiles (configType: "sql") use connectionId. + +8. Assuming StatusCode or SeverityText values + Values like STATUS_CODE_ERROR, Ok, error, fatal vary by deployment. + Always call hyperdx_list_sources and inspect real keyValues from the source + before writing filters that depend on these columns.`; +} diff --git a/packages/api/src/mcp/prompts/dashboards/helpers.ts b/packages/api/src/mcp/prompts/dashboards/helpers.ts new file mode 100644 index 00000000..53913fe5 --- /dev/null +++ b/packages/api/src/mcp/prompts/dashboards/helpers.ts @@ -0,0 +1,48 @@ +// ─── Source/connection summary helpers ─────────────────────────────────────── + +export function buildSourceSummary( + sources: { _id: unknown; name: string; kind: string; connection: unknown }[], + connections: { _id: unknown; name: string }[], +): string { + if (sources.length === 0 && connections.length === 0) { + return 'No sources or connections found. Call hyperdx_list_sources to discover available data.'; + } + + const lines: string[] = []; + + if (sources.length > 0) { + lines.push('AVAILABLE SOURCES (use sourceId with builder tiles):'); + for (const s of sources) { + lines.push( + ` - "${s.name}" (${s.kind}) — sourceId: "${s._id}", connectionId: "${s.connection}"`, + ); + } + } + + if (connections.length > 0) { + lines.push(''); + lines.push( + 'AVAILABLE CONNECTIONS (use connectionId with raw SQL tiles only):', + ); + for (const c of connections) { + lines.push(` - "${c.name}" — connectionId: "${c._id}"`); + } + } + + return lines.join('\n'); +} + +export function getFirstSourceId( + sources: { _id: unknown; kind: string }[], + preferredKind?: string, +): string { + const preferred = preferredKind + ? sources.find(s => s.kind === preferredKind) + : undefined; + const source = preferred ?? sources[0]; + return source ? String(source._id) : ''; +} + +export function getFirstConnectionId(connections: { _id: unknown }[]): string { + return connections[0] ? String(connections[0]._id) : ''; +} diff --git a/packages/api/src/mcp/prompts/dashboards/index.ts b/packages/api/src/mcp/prompts/dashboards/index.ts new file mode 100644 index 00000000..aec7fa31 --- /dev/null +++ b/packages/api/src/mcp/prompts/dashboards/index.ts @@ -0,0 +1,195 @@ +import { z } from 'zod'; + +import { getConnectionsByTeam } from '@/controllers/connection'; +import { getSources } from '@/controllers/sources'; +import logger from '@/utils/logger'; + +import type { PromptDefinition } from '../../tools/types'; +import { + buildCreateDashboardPrompt, + buildDashboardExamplesPrompt, + buildQueryGuidePrompt, +} from './content'; +import { + buildSourceSummary, + getFirstConnectionId, + getFirstSourceId, +} from './helpers'; + +const dashboardPrompts: PromptDefinition = (server, context) => { + const { teamId } = context; + + // ── create_dashboard ────────────────────────────────────────────────────── + + server.registerPrompt( + 'create_dashboard', + { + title: 'Create a Dashboard', + description: + 'Create a HyperDX dashboard with the MCP tools. ' + + 'Follow the recommended workflow, pick tile types, write queries, ' + + 'and validate results — using your real data sources.', + argsSchema: { + description: z + .string() + .optional() + .describe( + 'What the dashboard should monitor (e.g. "API error rates and latency")', + ), + }, + }, + async ({ description }) => { + let sourceSummary: string; + let traceSourceId: string; + let logSourceId: string; + + try { + const [sources, connections] = await Promise.all([ + getSources(teamId), + getConnectionsByTeam(teamId), + ]); + + sourceSummary = buildSourceSummary( + sources.map(s => ({ + _id: s._id, + name: s.name, + kind: s.kind, + connection: s.connection, + })), + connections.map(c => ({ _id: c._id, name: c.name })), + ); + traceSourceId = getFirstSourceId( + sources.map(s => ({ _id: s._id, kind: s.kind })), + 'trace', + ); + logSourceId = getFirstSourceId( + sources.map(s => ({ _id: s._id, kind: s.kind })), + 'log', + ); + } catch (e) { + logger.warn( + { teamId, error: e }, + 'Failed to fetch sources for create_dashboard prompt', + ); + sourceSummary = + 'Could not fetch sources. Call hyperdx_list_sources to discover available data.'; + traceSourceId = ''; + logSourceId = ''; + } + + return { + messages: [ + { + role: 'user' as const, + content: { + type: 'text' as const, + text: buildCreateDashboardPrompt( + sourceSummary, + traceSourceId, + logSourceId, + description, + ), + }, + }, + ], + }; + }, + ); + + // ── dashboard_examples ──────────────────────────────────────────────────── + + server.registerPrompt( + 'dashboard_examples', + { + title: 'Dashboard Examples', + description: + 'Get copy-paste-ready dashboard examples for common observability patterns: ' + + 'service_overview, error_tracking, latency, log_analysis, infrastructure_sql.', + argsSchema: { + pattern: z + .string() + .optional() + .describe( + 'Filter to a specific pattern: service_overview, error_tracking, latency, log_analysis, infrastructure_sql', + ), + }, + }, + async ({ pattern }) => { + let traceSourceId: string; + let logSourceId: string; + let connectionId: string; + + try { + const [sources, connections] = await Promise.all([ + getSources(teamId), + getConnectionsByTeam(teamId), + ]); + + traceSourceId = getFirstSourceId( + sources.map(s => ({ _id: s._id, kind: s.kind })), + 'trace', + ); + logSourceId = getFirstSourceId( + sources.map(s => ({ _id: s._id, kind: s.kind })), + 'log', + ); + connectionId = getFirstConnectionId( + connections.map(c => ({ _id: c._id })), + ); + } catch (e) { + logger.warn( + { teamId, error: e }, + 'Failed to fetch sources for dashboard_examples prompt', + ); + traceSourceId = ''; + logSourceId = ''; + connectionId = ''; + } + + return { + messages: [ + { + role: 'user' as const, + content: { + type: 'text' as const, + text: buildDashboardExamplesPrompt( + traceSourceId, + logSourceId, + connectionId, + pattern, + ), + }, + }, + ], + }; + }, + ); + + // ── query_guide ─────────────────────────────────────────────────────────── + + server.registerPrompt( + 'query_guide', + { + title: 'Query Writing Guide', + description: + 'Look up HyperDX query syntax: aggregation functions, ' + + 'Lucene/SQL filters, raw SQL macros, column naming, ' + + 'per-tile constraints, and common mistakes.', + }, + async () => { + return { + messages: [ + { + role: 'user' as const, + content: { + type: 'text' as const, + text: buildQueryGuidePrompt(), + }, + }, + ], + }; + }, + ); +}; + +export default dashboardPrompts; diff --git a/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts b/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts new file mode 100644 index 00000000..5d9fb48b --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/deleteDashboard.ts @@ -0,0 +1,63 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import mongoose from 'mongoose'; +import { z } from 'zod'; + +import { deleteDashboard } from '@/controllers/dashboard'; +import Dashboard from '@/models/dashboard'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; + +export function registerDeleteDashboard( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + + server.registerTool( + 'hyperdx_delete_dashboard', + { + title: 'Delete Dashboard', + description: + 'Permanently delete a dashboard by ID. Also removes any alerts attached to its tiles. ' + + 'Use hyperdx_get_dashboard (without an ID) to list available dashboard IDs.', + inputSchema: z.object({ + id: z.string().describe('Dashboard ID to delete.'), + }), + }, + withToolTracing( + 'hyperdx_delete_dashboard', + context, + async ({ id: dashboardId }) => { + if (!mongoose.Types.ObjectId.isValid(dashboardId)) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], + }; + } + + const existing = await Dashboard.findOne({ + _id: dashboardId, + team: teamId, + }).lean(); + if (!existing) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Dashboard not found' }], + }; + } + + await deleteDashboard(dashboardId, new mongoose.Types.ObjectId(teamId)); + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify({ deleted: true, id: dashboardId }, null, 2), + }, + ], + }; + }, + ), + ); +} diff --git a/packages/api/src/mcp/tools/dashboards/getDashboard.ts b/packages/api/src/mcp/tools/dashboards/getDashboard.ts new file mode 100644 index 00000000..ee9b2d73 --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/getDashboard.ts @@ -0,0 +1,87 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import mongoose from 'mongoose'; +import { z } from 'zod'; + +import * as config from '@/config'; +import { getDashboards } from '@/controllers/dashboard'; +import Dashboard from '@/models/dashboard'; +import { convertToExternalDashboard } from '@/routers/external-api/v2/utils/dashboards'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; + +export function registerGetDashboard( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + const frontendUrl = config.FRONTEND_URL; + + server.registerTool( + 'hyperdx_get_dashboard', + { + title: 'Get Dashboard(s)', + description: + 'Without an ID: list all dashboards (returns IDs, names, tags). ' + + 'With an ID: get full dashboard detail including all tiles and configuration.', + inputSchema: z.object({ + id: z + .string() + .optional() + .describe( + 'Dashboard ID. Omit to list all dashboards, provide to get full detail.', + ), + }), + }, + withToolTracing('hyperdx_get_dashboard', context, async ({ id }) => { + if (!id) { + const dashboards = await getDashboards( + new mongoose.Types.ObjectId(teamId), + ); + const output = dashboards.map(d => ({ + id: d._id.toString(), + name: d.name, + tags: d.tags, + ...(frontendUrl ? { url: `${frontendUrl}/dashboards/${d._id}` } : {}), + })); + return { + content: [ + { type: 'text' as const, text: JSON.stringify(output, null, 2) }, + ], + }; + } + + if (!mongoose.Types.ObjectId.isValid(id)) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], + }; + } + + const dashboard = await Dashboard.findOne({ _id: id, team: teamId }); + if (!dashboard) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Dashboard not found' }], + }; + } + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + ...convertToExternalDashboard(dashboard), + ...(frontendUrl + ? { url: `${frontendUrl}/dashboards/${dashboard._id}` } + : {}), + }, + null, + 2, + ), + }, + ], + }; + }), + ); +} diff --git a/packages/api/src/mcp/tools/dashboards/index.ts b/packages/api/src/mcp/tools/dashboards/index.ts new file mode 100644 index 00000000..7fc7bff4 --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/index.ts @@ -0,0 +1,23 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; + +import type { McpContext, ToolDefinition } from '../types'; +import { registerDeleteDashboard } from './deleteDashboard'; +import { registerGetDashboard } from './getDashboard'; +import { registerListSources } from './listSources'; +import { registerQueryTile } from './queryTile'; +import { registerSaveDashboard } from './saveDashboard'; + +export * from './schemas'; + +const dashboardsTools: ToolDefinition = ( + server: McpServer, + context: McpContext, +) => { + registerListSources(server, context); + registerGetDashboard(server, context); + registerSaveDashboard(server, context); + registerDeleteDashboard(server, context); + registerQueryTile(server, context); +}; + +export default dashboardsTools; diff --git a/packages/api/src/mcp/tools/dashboards/listSources.ts b/packages/api/src/mcp/tools/dashboards/listSources.ts new file mode 100644 index 00000000..18f0a0fa --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/listSources.ts @@ -0,0 +1,183 @@ +import { + convertCHDataTypeToJSType, + filterColumnMetaByType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { + getConnectionById, + getConnectionsByTeam, +} from '@/controllers/connection'; +import { getSources } from '@/controllers/sources'; +import logger from '@/utils/logger'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; + +export function registerListSources( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + + server.registerTool( + 'hyperdx_list_sources', + { + title: 'List Sources & Connections', + description: + 'List all data sources (logs, metrics, traces) and database connections available to this team. ' + + 'Returns source IDs (use as sourceId in hyperdx_query and dashboard tiles) and ' + + 'connection IDs (use as connectionId for advanced raw SQL queries). ' + + 'Each source includes its full column schema and sampled attribute keys from map columns ' + + '(e.g. SpanAttributes, ResourceAttributes). ' + + 'Column names are PascalCase (e.g. Duration, not duration). ' + + "Map attributes must be accessed via bracket syntax: SpanAttributes['key'].\n\n" + + 'NOTE: For most queries, use source IDs with the builder display types. ' + + 'Connection IDs are only needed for advanced raw SQL queries (displayType "sql").', + inputSchema: z.object({}), + }, + withToolTracing('hyperdx_list_sources', context, async () => { + const [sources, connections] = await Promise.all([ + getSources(teamId.toString()), + getConnectionsByTeam(teamId.toString()), + ]); + + const sourcesWithSchema = await Promise.all( + sources.map(async s => { + const meta: Record = { + id: s._id.toString(), + name: s.name, + kind: s.kind, + connectionId: s.connection.toString(), + timestampColumn: s.timestampValueExpression, + }; + + if ('eventAttributesExpression' in s && s.eventAttributesExpression) { + meta.eventAttributesColumn = s.eventAttributesExpression; + } + if ( + 'resourceAttributesExpression' in s && + s.resourceAttributesExpression + ) { + meta.resourceAttributesColumn = s.resourceAttributesExpression; + } + + if (s.kind === SourceKind.Trace) { + meta.keyColumns = { + spanName: s.spanNameExpression, + duration: s.durationExpression, + durationPrecision: s.durationPrecision, + statusCode: s.statusCodeExpression, + serviceName: s.serviceNameExpression, + traceId: s.traceIdExpression, + spanId: s.spanIdExpression, + }; + } else if (s.kind === SourceKind.Log) { + meta.keyColumns = { + body: s.bodyExpression, + serviceName: s.serviceNameExpression, + severityText: s.severityTextExpression, + traceId: s.traceIdExpression, + }; + } else if (s.kind === SourceKind.Metric) { + meta.metricTables = s.metricTables; + } + + // Skip column schema fetch for sources without a table (e.g. metrics + // sources store their tables in metricTables, not from.tableName). + if (s.from.tableName) { + try { + const connection = await getConnectionById( + teamId.toString(), + s.connection.toString(), + true, + ); + if (!connection) { + throw new Error(`Connection not found for source ${s._id}`); + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + const metadata = getMetadata(clickhouseClient); + + const columns = await metadata.getColumns({ + databaseName: s.from.databaseName, + tableName: s.from.tableName, + connectionId: s.connection.toString(), + }); + + meta.columns = columns.map(c => ({ + name: c.name, + type: c.type, + jsType: convertCHDataTypeToJSType(c.type), + })); + + const mapColumns = filterColumnMetaByType(columns, [ + JSDataType.Map, + ]); + const mapKeysResults: Record = {}; + await Promise.all( + (mapColumns ?? []).map(async col => { + try { + const keys = await metadata.getMapKeys({ + databaseName: s.from.databaseName, + tableName: s.from.tableName, + column: col.name, + maxKeys: 50, + connectionId: s.connection.toString(), + }); + mapKeysResults[col.name] = keys; + } catch { + // Skip columns where key sampling fails + } + }), + ); + if (Object.keys(mapKeysResults).length > 0) { + meta.mapAttributeKeys = mapKeysResults; + } + } catch (e) { + logger.warn( + { teamId, sourceId: s._id, error: e }, + 'Failed to fetch schema for source', + ); + } + } + + return meta; + }), + ); + + const output = { + sources: sourcesWithSchema, + connections: connections.map(c => ({ + id: c._id.toString(), + name: c.name, + })), + usage: { + topLevelColumns: + 'Use directly in valueExpression/groupBy with PascalCase: Duration, StatusCode, SpanName', + mapAttributes: + "Use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name']", + sourceIds: + 'Use sourceId with builder display types (line, stacked_bar, table, number, pie, search) for standard queries', + connectionIds: + 'ADVANCED: Use connectionId only with raw SQL queries (displayType "sql" or configType "sql"). ' + + 'Raw SQL is for advanced use cases like JOINs, sub-queries, or querying tables not registered as sources.', + }, + }; + return { + content: [ + { type: 'text' as const, text: JSON.stringify(output, null, 2) }, + ], + }; + }), + ); +} diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts new file mode 100644 index 00000000..5604bee8 --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -0,0 +1,97 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import mongoose from 'mongoose'; +import { z } from 'zod'; + +import Dashboard from '@/models/dashboard'; +import { convertToExternalDashboard } from '@/routers/external-api/v2/utils/dashboards'; + +import { withToolTracing } from '../../utils/tracing'; +import { parseTimeRange, runConfigTile } from '../query/helpers'; +import type { McpContext } from '../types'; + +export function registerQueryTile( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + + server.registerTool( + 'hyperdx_query_tile', + { + title: 'Query a Dashboard Tile', + description: + 'Execute the query for a specific tile on an existing dashboard. ' + + 'Useful for validating that a tile returns data or for spot-checking results ' + + 'without rebuilding the query from scratch. ' + + 'Use hyperdx_get_dashboard with an ID to find tile IDs.', + inputSchema: z.object({ + dashboardId: z.string().describe('Dashboard ID.'), + tileId: z + .string() + .describe( + 'Tile ID within the dashboard. ' + + 'Obtain from hyperdx_get_dashboard.', + ), + startTime: z + .string() + .optional() + .describe( + 'Start of the query window as ISO 8601. Default: 15 minutes ago. ' + + 'If results are empty, try a wider range (e.g. 24 hours).', + ), + endTime: z + .string() + .optional() + .describe('End of the query window as ISO 8601. Default: now.'), + }), + }, + withToolTracing( + 'hyperdx_query_tile', + context, + async ({ dashboardId, tileId, startTime, endTime }) => { + const timeRange = parseTimeRange(startTime, endTime); + if ('error' in timeRange) { + return { + isError: true, + content: [{ type: 'text' as const, text: timeRange.error }], + }; + } + const { startDate, endDate } = timeRange; + + if (!mongoose.Types.ObjectId.isValid(dashboardId)) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], + }; + } + + const dashboard = await Dashboard.findOne({ + _id: dashboardId, + team: teamId, + }); + if (!dashboard) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Dashboard not found' }], + }; + } + + const externalDashboard = convertToExternalDashboard(dashboard); + const tile = externalDashboard.tiles.find(t => t.id === tileId); + if (!tile) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Tile not found: ${tileId}. Available tile IDs: ${externalDashboard.tiles.map(t => t.id).join(', ')}`, + }, + ], + }; + } + + return runConfigTile(teamId.toString(), tile, startDate, endDate); + }, + ), + ); +} diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts new file mode 100644 index 00000000..c7ce7ef5 --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -0,0 +1,338 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { uniq } from 'lodash'; +import mongoose from 'mongoose'; +import { z } from 'zod'; + +import * as config from '@/config'; +import Dashboard from '@/models/dashboard'; +import { + cleanupDashboardAlerts, + convertExternalFiltersToInternal, + convertExternalTilesToInternal, + convertToExternalDashboard, + createDashboardBodySchema, + getMissingConnections, + getMissingSources, + resolveSavedQueryLanguage, + updateDashboardBodySchema, +} from '@/routers/external-api/v2/utils/dashboards'; +import type { ExternalDashboardTileWithId } from '@/utils/zod'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; +import { mcpTilesParam } from './schemas'; + +export function registerSaveDashboard( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + const frontendUrl = config.FRONTEND_URL; + + server.registerTool( + 'hyperdx_save_dashboard', + { + title: 'Create or Update Dashboard', + description: + 'Create a new dashboard (omit id) or update an existing one (provide id). ' + + 'Call hyperdx_list_sources first to obtain sourceId and connectionId values. ' + + 'IMPORTANT: After saving a dashboard, always run hyperdx_query_tile on each tile ' + + 'to confirm the queries work and return expected data. Tiles can silently fail ' + + 'due to incorrect filter syntax, missing attributes, or wrong column names.', + inputSchema: z.object({ + id: z + .string() + .optional() + .describe( + 'Dashboard ID. Omit to create a new dashboard, provide to update an existing one.', + ), + name: z.string().describe('Dashboard name'), + tiles: mcpTilesParam, + tags: z.array(z.string()).optional().describe('Dashboard tags'), + }), + }, + withToolTracing( + 'hyperdx_save_dashboard', + context, + async ({ id: dashboardId, name, tiles: inputTiles, tags }) => { + if (!dashboardId) { + return createDashboard({ + teamId, + frontendUrl, + name, + inputTiles, + tags, + }); + } + return updateDashboard({ + teamId, + frontendUrl, + dashboardId, + name, + inputTiles, + tags, + }); + }, + ), + ); +} + +// ─── Create helper ──────────────────────────────────────────────────────────── + +async function createDashboard({ + teamId, + frontendUrl, + name, + inputTiles, + tags, +}: { + teamId: string; + frontendUrl: string | undefined; + name: string; + inputTiles: unknown[]; + tags: string[] | undefined; +}) { + const parsed = createDashboardBodySchema.safeParse({ + name, + tiles: inputTiles, + tags, + }); + if (!parsed.success) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Validation error: ${JSON.stringify(parsed.error.errors)}`, + }, + ], + }; + } + + const { tiles, filters } = parsed.data; + const tilesWithId = tiles as ExternalDashboardTileWithId[]; + + const [missingSources, missingConnections] = await Promise.all([ + getMissingSources(teamId, tilesWithId, filters), + getMissingConnections(teamId, tilesWithId), + ]); + if (missingSources.length > 0) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Could not find source IDs: ${missingSources.join(', ')}`, + }, + ], + }; + } + if (missingConnections.length > 0) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Could not find connection IDs: ${missingConnections.join(', ')}`, + }, + ], + }; + } + + const internalTiles = convertExternalTilesToInternal(tilesWithId); + const filtersWithIds = convertExternalFiltersToInternal(filters ?? []); + + const normalizedSavedQueryLanguage = resolveSavedQueryLanguage({ + savedQuery: undefined, + savedQueryLanguage: undefined, + }); + + const newDashboard = await new Dashboard({ + name: parsed.data.name, + tiles: internalTiles, + tags: tags && uniq(tags), + filters: filtersWithIds, + savedQueryLanguage: normalizedSavedQueryLanguage, + savedFilterValues: parsed.data.savedFilterValues, + team: teamId, + }).save(); + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + ...convertToExternalDashboard(newDashboard), + ...(frontendUrl + ? { url: `${frontendUrl}/dashboards/${newDashboard._id}` } + : {}), + hint: 'Use hyperdx_query to test individual tile queries before viewing the dashboard.', + }, + null, + 2, + ), + }, + ], + }; +} + +// ─── Update helper ──────────────────────────────────────────────────────────── + +async function updateDashboard({ + teamId, + frontendUrl, + dashboardId, + name, + inputTiles, + tags, +}: { + teamId: string; + frontendUrl: string | undefined; + dashboardId: string; + name: string; + inputTiles: unknown[]; + tags: string[] | undefined; +}) { + if (!mongoose.Types.ObjectId.isValid(dashboardId)) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Invalid dashboard ID' }], + }; + } + + const parsed = updateDashboardBodySchema.safeParse({ + name, + tiles: inputTiles, + tags, + }); + if (!parsed.success) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Validation error: ${JSON.stringify(parsed.error.errors)}`, + }, + ], + }; + } + + const { tiles, filters } = parsed.data; + const tilesWithId = tiles as ExternalDashboardTileWithId[]; + + const [missingSources, missingConnections] = await Promise.all([ + getMissingSources(teamId, tilesWithId, filters), + getMissingConnections(teamId, tilesWithId), + ]); + if (missingSources.length > 0) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Could not find source IDs: ${missingSources.join(', ')}`, + }, + ], + }; + } + if (missingConnections.length > 0) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Could not find connection IDs: ${missingConnections.join(', ')}`, + }, + ], + }; + } + + const existingDashboard = await Dashboard.findOne( + { _id: dashboardId, team: teamId }, + { tiles: 1, filters: 1 }, + ).lean(); + + if (!existingDashboard) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Dashboard not found' }], + }; + } + + const existingTileIds = new Set( + (existingDashboard.tiles ?? []).map((t: { id: string }) => t.id), + ); + const existingFilterIds = new Set( + (existingDashboard.filters ?? []).map((f: { id: string }) => f.id), + ); + + const internalTiles = convertExternalTilesToInternal( + tilesWithId, + existingTileIds, + ); + + const setPayload: Record = { + name, + tiles: internalTiles, + tags: tags && uniq(tags), + }; + + if (filters !== undefined) { + setPayload.filters = convertExternalFiltersToInternal( + filters, + existingFilterIds, + ); + } + + const normalizedSavedQueryLanguage = resolveSavedQueryLanguage({ + savedQuery: undefined, + savedQueryLanguage: undefined, + }); + if (normalizedSavedQueryLanguage !== undefined) { + setPayload.savedQueryLanguage = normalizedSavedQueryLanguage; + } + + if (parsed.data.savedFilterValues !== undefined) { + setPayload.savedFilterValues = parsed.data.savedFilterValues; + } + + const updatedDashboard = await Dashboard.findOneAndUpdate( + { _id: dashboardId, team: teamId }, + { $set: setPayload }, + { new: true }, + ); + + if (!updatedDashboard) { + return { + isError: true, + content: [{ type: 'text' as const, text: 'Dashboard not found' }], + }; + } + + await cleanupDashboardAlerts({ + dashboardId, + teamId, + internalTiles, + existingTileIds, + }); + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + ...convertToExternalDashboard(updatedDashboard), + ...(frontendUrl + ? { url: `${frontendUrl}/dashboards/${updatedDashboard._id}` } + : {}), + hint: 'Use hyperdx_query to test individual tile queries before viewing the dashboard.', + }, + null, + 2, + ), + }, + ], + }; +} diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts new file mode 100644 index 00000000..b01f8c94 --- /dev/null +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -0,0 +1,320 @@ +import { + AggregateFunctionSchema, + SearchConditionLanguageSchema, +} from '@hyperdx/common-utils/dist/types'; +import { z } from 'zod'; + +import { externalQuantileLevelSchema } from '@/utils/zod'; + +// ─── Shared tile schemas for MCP dashboard tools ───────────────────────────── +const mcpTileSelectItemSchema = z + .object({ + aggFn: AggregateFunctionSchema.describe( + 'Aggregation function. "count" requires no valueExpression; all others do.', + ), + valueExpression: z + .string() + .optional() + .describe( + 'Column or expression to aggregate. Required for all aggFn except "count". ' + + 'Use PascalCase for top-level columns (e.g. "Duration", "StatusCode"). ' + + "For span attributes use: SpanAttributes['key'] (e.g. SpanAttributes['http.method']). " + + "For resource attributes use: ResourceAttributes['key'] (e.g. ResourceAttributes['service.name']).", + ), + where: z + .string() + .optional() + .default('') + .describe('Filter in Lucene syntax. Example: "level:error"'), + whereLanguage: SearchConditionLanguageSchema.optional().default('lucene'), + alias: z.string().optional().describe('Display label for this series'), + level: externalQuantileLevelSchema + .optional() + .describe('Percentile level for aggFn="quantile"'), + }) + .superRefine((data, ctx) => { + if (data.level && data.aggFn !== 'quantile') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Level can only be used with quantile aggregation function', + }); + } + if (data.valueExpression && data.aggFn === 'count') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + 'Value expression cannot be used with count aggregation function', + }); + } else if (!data.valueExpression && data.aggFn !== 'count') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + 'Value expression is required for non-count aggregation functions', + }); + } + }); + +const mcpTileLayoutSchema = z.object({ + name: z.string().describe('Tile title shown on the dashboard'), + x: z + .number() + .min(0) + .max(23) + .optional() + .default(0) + .describe('Horizontal grid position (0–23). Default 0'), + y: z + .number() + .min(0) + .optional() + .default(0) + .describe('Vertical grid position. Default 0'), + w: z + .number() + .min(1) + .max(24) + .optional() + .default(12) + .describe('Width in grid columns (1–24). Default 12'), + h: z + .number() + .min(1) + .optional() + .default(4) + .describe('Height in grid rows. Default 4'), + id: z + .string() + .max(36) + .optional() + .describe('Tile ID (auto-generated if omitted)'), +}); + +const mcpLineTileSchema = mcpTileLayoutSchema.extend({ + config: z.object({ + displayType: z.literal('line').describe('Line chart over time'), + sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + select: z + .array(mcpTileSelectItemSchema) + .min(1) + .max(20) + .describe('Metrics to plot (one series per item)'), + groupBy: z + .string() + .optional() + .describe( + 'Column to split/group by. ' + + 'Top-level columns use PascalCase (e.g. "SpanName", "StatusCode"). ' + + "Span attributes: SpanAttributes['key'] (e.g. SpanAttributes['http.method']). " + + "Resource attributes: ResourceAttributes['key'] (e.g. ResourceAttributes['service.name']).", + ), + fillNulls: z.boolean().optional().default(true), + alignDateRangeToGranularity: z.boolean().optional(), + asRatio: z + .boolean() + .optional() + .describe( + 'Plot as ratio of two metrics (requires exactly 2 select items)', + ), + }), +}); + +const mcpBarTileSchema = mcpTileLayoutSchema.extend({ + config: z.object({ + displayType: z + .literal('stacked_bar') + .describe('Stacked bar chart over time'), + sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + select: z.array(mcpTileSelectItemSchema).min(1).max(20), + groupBy: z.string().optional(), + fillNulls: z.boolean().optional().default(true), + alignDateRangeToGranularity: z.boolean().optional(), + asRatio: z.boolean().optional(), + }), +}); + +const mcpTableTileSchema = mcpTileLayoutSchema.extend({ + config: z.object({ + displayType: z.literal('table').describe('Tabular aggregated data'), + sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + select: z.array(mcpTileSelectItemSchema).min(1).max(20), + groupBy: z + .string() + .optional() + .describe( + 'Group rows by this column. Use PascalCase for top-level columns (e.g. "SpanName"). ' + + "For attributes: SpanAttributes['key'] or ResourceAttributes['key'].", + ), + orderBy: z.string().optional().describe('Sort results by this column'), + asRatio: z.boolean().optional(), + }), +}); + +const mcpNumberFormatSchema = z + .object({ + output: z + .enum(['currency', 'percent', 'byte', 'time', 'number']) + .describe( + 'Format category. "time" auto-formats durations (use factor for input unit). ' + + '"byte" formats as KB/MB/GB. "currency" prepends a symbol. "percent" appends %.', + ), + mantissa: z + .number() + .int() + .optional() + .describe('Decimal places (0–10). Not used for "time" output.'), + thousandSeparated: z + .boolean() + .optional() + .describe('Separate thousands (e.g. 1,234,567)'), + average: z + .boolean() + .optional() + .describe('Abbreviate large numbers (e.g. 1.2m)'), + decimalBytes: z + .boolean() + .optional() + .describe( + 'Use decimal base for bytes (1KB = 1000). Only for "byte" output.', + ), + factor: z + .number() + .optional() + .describe( + 'Input unit factor for "time" output. ' + + '1 = seconds, 0.001 = milliseconds, 0.000001 = microseconds, 0.000000001 = nanoseconds.', + ), + currencySymbol: z + .string() + .optional() + .describe('Currency symbol (e.g. "$"). Only for "currency" output.'), + unit: z + .string() + .optional() + .describe('Suffix appended to the value (e.g. " req/s")'), + }) + .describe( + 'Controls how the number value is formatted for display. ' + + 'Most useful: { output: "time", factor: 0.000000001 } to auto-format nanosecond durations, ' + + 'or { output: "number", mantissa: 2, thousandSeparated: true } for clean counts.', + ); + +const mcpNumberTileSchema = mcpTileLayoutSchema.extend({ + config: z.object({ + displayType: z.literal('number').describe('Single aggregate scalar value'), + sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + select: z + .array(mcpTileSelectItemSchema) + .length(1) + .describe('Exactly one metric to display'), + numberFormat: mcpNumberFormatSchema + .optional() + .describe( + 'Display formatting for the number value. Example: { output: "time", factor: 0.000000001 } ' + + 'to auto-format nanosecond durations as human-readable time.', + ), + }), +}); + +const mcpPieTileSchema = mcpTileLayoutSchema.extend({ + config: z.object({ + displayType: z.literal('pie').describe('Pie chart'), + sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + select: z.array(mcpTileSelectItemSchema).length(1), + groupBy: z + .string() + .optional() + .describe( + 'Column that defines pie slices. Use PascalCase for top-level columns. ' + + "For attributes: SpanAttributes['key'] or ResourceAttributes['key'].", + ), + }), +}); + +const mcpSearchTileSchema = mcpTileLayoutSchema.extend({ + config: z.object({ + displayType: z.literal('search').describe('Log/event search results list'), + sourceId: z.string().describe('Source ID – call hyperdx_list_sources'), + where: z + .string() + .optional() + .default('') + .describe('Filter in Lucene syntax. Example: "level:error"'), + whereLanguage: SearchConditionLanguageSchema.optional().default('lucene'), + select: z + .string() + .optional() + .default('') + .describe( + 'Columns to display (empty = defaults). Example: "body,service.name,duration"', + ), + }), +}); + +const mcpMarkdownTileSchema = mcpTileLayoutSchema.extend({ + config: z.object({ + displayType: z.literal('markdown').describe('Free-form Markdown text tile'), + markdown: z.string().optional().default(''), + }), +}); + +const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ + config: z.object({ + configType: z + .literal('sql') + .describe( + 'Must be "sql" for raw SQL tiles. ' + + 'ADVANCED: Only use raw SQL tiles when the builder tile types cannot express the query you need.', + ), + displayType: z + .enum(['line', 'stacked_bar', 'table', 'number', 'pie']) + .describe('How to render the SQL results'), + connectionId: z + .string() + .describe( + 'Connection ID (not sourceId) – call hyperdx_list_sources to find available connections', + ), + sqlTemplate: z + .string() + .describe( + 'Raw ClickHouse SQL query. Always include a LIMIT clause to avoid excessive data.\n' + + 'Use query parameters: {startDateMilliseconds:Int64}, {endDateMilliseconds:Int64}, ' + + '{intervalSeconds:Int64}, {intervalMilliseconds:Int64}.\n' + + 'Or use macros: $__timeFilter(col), $__timeFilter_ms(col), $__dateFilter(col), ' + + '$__fromTime, $__toTime, $__fromTime_ms, $__toTime_ms, ' + + '$__timeInterval(col), $__timeInterval_ms(col), $__interval_s, $__filters.\n' + + 'Example: "SELECT $__timeInterval(TimestampTime) AS ts, ServiceName, count() ' + + 'FROM otel_logs WHERE $__timeFilter(TimestampTime) AND $__filters ' + + 'GROUP BY ServiceName, ts ORDER BY ts"', + ), + fillNulls: z.boolean().optional(), + alignDateRangeToGranularity: z.boolean().optional(), + }), +}); + +const mcpTileSchema = z.union([ + mcpLineTileSchema, + mcpBarTileSchema, + mcpTableTileSchema, + mcpNumberTileSchema, + mcpPieTileSchema, + mcpSearchTileSchema, + mcpMarkdownTileSchema, + mcpSqlTileSchema, +]); + +export const mcpTilesParam = z + .array(mcpTileSchema) + .describe( + 'Array of dashboard tiles. Each tile needs a name, optional layout (x/y/w/h), and a config block. ' + + 'The config block varies by displayType – use hyperdx_list_sources for sourceId and connectionId values.\n\n' + + 'Example tiles:\n' + + '1. Line chart: { "name": "Error Rate", "config": { "displayType": "line", "sourceId": "", ' + + '"groupBy": "ResourceAttributes[\'service.name\']", "select": [{ "aggFn": "count", "where": "StatusCode:STATUS_CODE_ERROR" }] } }\n' + + '2. Table: { "name": "Top Endpoints", "config": { "displayType": "table", "sourceId": "", ' + + '"groupBy": "SpanAttributes[\'http.route\']", "select": [{ "aggFn": "count" }, { "aggFn": "avg", "valueExpression": "Duration" }] } }\n' + + '3. Number: { "name": "Total Requests", "config": { "displayType": "number", "sourceId": "", ' + + '"select": [{ "aggFn": "count" }], "numberFormat": { "output": "number", "average": true } } }\n' + + '4. Number (duration): { "name": "P95 Latency", "config": { "displayType": "number", "sourceId": "", ' + + '"select": [{ "aggFn": "quantile", "level": 0.95, "valueExpression": "Duration" }], ' + + '"numberFormat": { "output": "time", "factor": 0.000000001 } } }', + ); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts new file mode 100644 index 00000000..79d34eaa --- /dev/null +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -0,0 +1,264 @@ +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; +import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; +import type { + ChartConfigWithDateRange, + MetricTable, +} from '@hyperdx/common-utils/dist/types'; +import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; +import ms from 'ms'; + +import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; +import { + convertToInternalTileConfig, + isConfigTile, +} from '@/routers/external-api/v2/utils/dashboards'; +import { trimToolResponse } from '@/utils/trimToolResponse'; +import type { ExternalDashboardTileWithId } from '@/utils/zod'; + +// ─── Time range ────────────────────────────────────────────────────────────── + +export function parseTimeRange( + startTime?: string, + endTime?: string, +): { error: string } | { startDate: Date; endDate: Date } { + const endDate = endTime ? new Date(endTime) : new Date(); + const startDate = startTime + ? new Date(startTime) + : new Date(endDate.getTime() - ms('15m')); + if (isNaN(endDate.getTime()) || isNaN(startDate.getTime())) { + return { + error: 'Invalid startTime or endTime: must be valid ISO 8601 strings', + }; + } + return { startDate, endDate }; +} + +// ─── Result helpers ────────────────────────────────────────────────────────── + +function isEmptyResult(result: unknown): boolean { + if (result == null) return true; + if (Array.isArray(result)) return result.length === 0; + if (typeof result === 'object' && result !== null) { + const obj = result as Record; + if (Array.isArray(obj.data) && obj.data.length === 0) return true; + if (obj.rows != null && Number(obj.rows) === 0) return true; + } + return false; +} + +function formatQueryResult(result: unknown) { + const trimmedResult = trimToolResponse(result); + const isTrimmed = + JSON.stringify(trimmedResult).length < JSON.stringify(result).length; + const empty = isEmptyResult(result); + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + result: trimmedResult, + ...(isTrimmed + ? { + note: 'Result was trimmed for context size. Narrow the time range or add filters to reduce data.', + } + : {}), + ...(empty + ? { + hint: 'No data found in the queried time range. Try setting startTime to a wider window (e.g. 24 hours ago) or check that filters match existing data.', + } + : {}), + }, + null, + 2, + ), + }, + ], + }; +} + +// ─── Tile execution ────────────────────────────────────────────────────────── + +export async function runConfigTile( + teamId: string, + tile: ExternalDashboardTileWithId, + startDate: Date, + endDate: Date, + options?: { maxResults?: number }, +) { + if (!isConfigTile(tile)) { + return { + isError: true as const, + content: [ + { type: 'text' as const, text: 'Invalid tile: config field missing' }, + ], + }; + } + + const internalTile = convertToInternalTileConfig(tile); + const savedConfig = internalTile.config; + + if (!isRawSqlSavedChartConfig(savedConfig)) { + const builderConfig = savedConfig; + + if ( + !builderConfig.source || + builderConfig.displayType === DisplayType.Markdown + ) { + return { + content: [ + { + type: 'text' as const, + text: 'Markdown tile: no query to execute.', + }, + ], + }; + } + + const source = await getSource(teamId, builderConfig.source); + if (!source) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `Source not found: ${builderConfig.source}`, + }, + ], + }; + } + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + true, + ); + if (!connection) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `Connection not found for source: ${builderConfig.source}`, + }, + ], + }; + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + + const isSearch = builderConfig.displayType === DisplayType.Search; + const defaultTableSelect = + 'defaultTableSelectExpression' in source + ? source.defaultTableSelectExpression + : undefined; + const implicitColumn = + 'implicitColumnExpression' in source + ? source.implicitColumnExpression + : undefined; + const searchOverrides = isSearch + ? { + select: builderConfig.select || defaultTableSelect || '*', + groupBy: undefined, + granularity: undefined, + orderBy: [ + { + ordering: 'DESC' as const, + valueExpression: getFirstTimestampValueExpression( + source.timestampValueExpression, + ), + }, + ], + limit: { limit: options?.maxResults ?? 50, offset: 0 }, + } + : {}; + + const chartConfig = { + ...builderConfig, + ...searchOverrides, + from: { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }, + connection: source.connection.toString(), + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: implicitColumn, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + const metadata = getMetadata(clickhouseClient); + const result = await clickhouseClient.queryChartConfig({ + config: chartConfig, + metadata, + querySettings: source.querySettings, + }); + + return formatQueryResult(result); + } + + // Raw SQL tile — hydrate source fields for macro support ($__sourceTable, $__filters) + let sourceFields: { + from?: { databaseName: string; tableName: string }; + implicitColumnExpression?: string; + metricTables?: MetricTable; + } = {}; + if (savedConfig.source) { + const source = await getSource(teamId, savedConfig.source); + if (source) { + sourceFields = { + from: source.from, + implicitColumnExpression: + 'implicitColumnExpression' in source + ? source.implicitColumnExpression + : undefined, + metricTables: + source.kind === SourceKind.Metric ? source.metricTables : undefined, + }; + } + } + + const connection = await getConnectionById( + teamId, + savedConfig.connection, + true, + ); + if (!connection) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `Connection not found: ${savedConfig.connection}`, + }, + ], + }; + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + + const chartConfig = { + ...savedConfig, + ...sourceFields, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + const metadata = getMetadata(clickhouseClient); + const result = await clickhouseClient.queryChartConfig({ + config: chartConfig, + metadata, + querySettings: undefined, + }); + + return formatQueryResult(result); +} diff --git a/packages/api/src/mcp/tools/query/index.ts b/packages/api/src/mcp/tools/query/index.ts new file mode 100644 index 00000000..13a9fe47 --- /dev/null +++ b/packages/api/src/mcp/tools/query/index.ts @@ -0,0 +1,117 @@ +import { ObjectId } from 'mongodb'; + +import type { ExternalDashboardTileWithId } from '@/utils/zod'; +import { externalDashboardTileSchemaWithId } from '@/utils/zod'; + +import { withToolTracing } from '../../utils/tracing'; +import type { ToolDefinition } from '../types'; +import { parseTimeRange, runConfigTile } from './helpers'; +import { hyperdxQuerySchema } from './schemas'; + +// ─── Tool definition ───────────────────────────────────────────────────────── + +const queryTools: ToolDefinition = (server, context) => { + const { teamId } = context; + + server.registerTool( + 'hyperdx_query', + { + title: 'Query Data', + description: + 'Query observability data (logs, metrics, traces) from HyperDX. ' + + 'Use hyperdx_list_sources first to find sourceId/connectionId values. ' + + 'Set displayType to control the query shape.\n\n' + + 'PREFERRED: Use the builder display types (line, stacked_bar, table, number, pie) ' + + 'for aggregated metrics, or "search" for browsing individual log/event rows. ' + + 'These are safer, easier to construct, and cover most use cases.\n\n' + + 'ADVANCED: Use displayType "sql" only when you need capabilities the builder cannot express, ' + + 'such as JOINs, sub-queries, CTEs, or querying tables not registered as sources. ' + + 'Raw SQL requires a connectionId (not sourceId) and a hand-written ClickHouse SQL query.\n\n' + + 'Column naming: Top-level columns are PascalCase (Duration, StatusCode, SpanName). ' + + "Map attributes use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name']. " + + 'Call hyperdx_list_sources to discover available columns and attribute keys for each source.', + inputSchema: hyperdxQuerySchema, + }, + withToolTracing('hyperdx_query', context, async input => { + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return { + isError: true, + content: [{ type: 'text' as const, text: timeRange.error }], + }; + } + const { startDate, endDate } = timeRange; + + let tile: ExternalDashboardTileWithId; + + if (input.displayType === 'sql') { + tile = externalDashboardTileSchemaWithId.parse({ + id: new ObjectId().toString(), + name: 'MCP SQL', + x: 0, + y: 0, + w: 24, + h: 6, + config: { + configType: 'sql' as const, + displayType: 'table' as const, + connectionId: input.connectionId, + sqlTemplate: input.sql, + }, + }); + } else if (input.displayType === 'search') { + tile = externalDashboardTileSchemaWithId.parse({ + id: new ObjectId().toString(), + name: 'MCP Search', + x: 0, + y: 0, + w: 24, + h: 6, + config: { + displayType: 'search' as const, + sourceId: input.sourceId, + select: input.columns ?? '', + where: input.where ?? '', + whereLanguage: input.whereLanguage ?? 'lucene', + }, + }); + } else { + tile = externalDashboardTileSchemaWithId.parse({ + id: new ObjectId().toString(), + name: 'MCP Query', + x: 0, + y: 0, + w: 12, + h: 4, + config: { + displayType: input.displayType, + sourceId: input.sourceId, + select: input.select.map(s => ({ + aggFn: s.aggFn, + where: s.where ?? '', + whereLanguage: s.whereLanguage ?? 'lucene', + valueExpression: s.valueExpression, + alias: s.alias, + level: s.level, + })), + groupBy: input.groupBy ?? undefined, + orderBy: input.orderBy ?? undefined, + ...(input.granularity ? { granularity: input.granularity } : {}), + }, + }); + } + + return runConfigTile( + teamId.toString(), + tile, + startDate, + endDate, + input.displayType === 'search' + ? { maxResults: input.maxResults } + : undefined, + ); + }), + ); +}; + +export default queryTools; diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts new file mode 100644 index 00000000..ccbf9ae4 --- /dev/null +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -0,0 +1,220 @@ +import { z } from 'zod'; + +// ─── Shared schemas ────────────────────────────────────────────────────────── + +const mcpAggFnSchema = z + .enum([ + 'avg', + 'count', + 'count_distinct', + 'last_value', + 'max', + 'min', + 'quantile', + 'sum', + 'none', + ]) + .describe( + 'Aggregation function:\n' + + ' count – count matching rows (no valueExpression needed)\n' + + ' sum / avg / min / max – aggregate a numeric column (valueExpression required)\n' + + ' count_distinct – unique value count (valueExpression required)\n' + + ' quantile – percentile; also set level (valueExpression required)\n' + + ' last_value – most recent value of a column\n' + + ' none – pass a raw expression through unchanged', + ); + +const mcpSelectItemSchema = z.object({ + aggFn: mcpAggFnSchema, + valueExpression: z + .string() + .optional() + .describe( + 'Column or expression to aggregate. Required for every aggFn except "count". ' + + 'Use PascalCase for top-level columns (e.g. "Duration", "StatusCode"). ' + + "For span attributes use: SpanAttributes['key'] (e.g. SpanAttributes['http.method']). " + + "For resource attributes use: ResourceAttributes['key'] (e.g. ResourceAttributes['service.name']).", + ), + where: z + .string() + .optional() + .default('') + .describe( + 'Row filter in Lucene syntax. ' + + 'Examples: "level:error", "service.name:api AND http.status_code:>=500"', + ), + whereLanguage: z + .enum(['lucene', 'sql']) + .optional() + .default('lucene') + .describe('Query language for the where filter. Default: lucene'), + alias: z + .string() + .optional() + .describe('Display label for this series. Example: "Error rate"'), + level: z + .union([z.literal(0.5), z.literal(0.9), z.literal(0.95), z.literal(0.99)]) + .optional() + .describe( + 'Percentile level. Only applicable when aggFn is "quantile". ' + + 'Allowed values: 0.5, 0.9, 0.95, 0.99', + ), +}); + +const mcpTimeRangeSchema = z.object({ + startTime: z + .string() + .optional() + .describe( + 'Start of the query window as ISO 8601. Default: 15 minutes ago. ' + + 'If results are empty, try a wider range (e.g. 24 hours).', + ), + endTime: z + .string() + .optional() + .describe('End of the query window as ISO 8601. Default: now.'), +}); + +// ─── Discriminated union schema for hyperdx_query ─────────────────────────── + +const builderQuerySchema = mcpTimeRangeSchema.extend({ + displayType: z + .enum(['line', 'stacked_bar', 'table', 'number', 'pie']) + .describe( + 'How to visualize the query results:\n' + + ' line – time-series line chart\n' + + ' stacked_bar – time-series stacked bar chart\n' + + ' table – grouped aggregation as rows\n' + + ' number – single aggregate scalar\n' + + ' pie – pie chart (one metric, grouped)', + ), + sourceId: z + .string() + .describe( + 'Source ID. Call hyperdx_list_sources to find available sources.', + ), + select: z + .array(mcpSelectItemSchema) + .min(1) + .max(10) + .describe( + 'Metrics to compute. Each item defines an aggregation. ' + + 'For "number" display, provide exactly 1 item. ' + + 'Example: [{ aggFn: "count" }, { aggFn: "avg", valueExpression: "Duration" }]', + ), + groupBy: z + .string() + .optional() + .describe( + 'Column to group/split by. ' + + 'Top-level columns use PascalCase (e.g. "SpanName", "StatusCode"). ' + + "Span attributes: SpanAttributes['key'] (e.g. SpanAttributes['http.method']). " + + "Resource attributes: ResourceAttributes['key'] (e.g. ResourceAttributes['service.name']).", + ), + orderBy: z + .string() + .optional() + .describe('Column to sort results by (table display only).'), + granularity: z + .string() + .optional() + .describe( + 'Time bucket size for time-series charts (line, stacked_bar). ' + + 'Format: " " where unit is second, minute, hour, or day. ' + + 'Examples: "1 minute", "5 minute", "1 hour", "1 day". ' + + 'Omit to let HyperDX pick automatically based on the time range.', + ), +}); + +const searchQuerySchema = mcpTimeRangeSchema.extend({ + displayType: z + .literal('search') + .describe('Search and filter individual log/event rows'), + sourceId: z + .string() + .describe( + 'Source ID. Call hyperdx_list_sources to find available sources.', + ), + where: z + .string() + .optional() + .default('') + .describe( + 'Row filter. Examples: "level:error", "service.name:api AND duration:>500"', + ), + whereLanguage: z + .enum(['lucene', 'sql']) + .optional() + .default('lucene') + .describe('Query language for the where filter. Default: lucene'), + columns: z + .string() + .optional() + .default('') + .describe( + 'Comma-separated columns to include. Leave empty for defaults. ' + + 'Example: "body,service.name,duration"', + ), + maxResults: z + .number() + .min(1) + .max(200) + .optional() + .default(50) + .describe( + 'Maximum number of rows to return (1–200). Default: 50. ' + + 'Use smaller values to reduce response size.', + ), +}); + +const sqlQuerySchema = mcpTimeRangeSchema.extend({ + displayType: z + .literal('sql') + .describe( + 'ADVANCED: Execute raw SQL directly against ClickHouse. ' + + 'Only use this when the builder query types (line, stacked_bar, table, number, pie, search) ' + + 'cannot express the query you need — e.g. complex JOINs, sub-queries, CTEs, or ' + + 'querying tables not registered as sources. ' + + 'Prefer the builder display types for standard queries as they are safer and easier to use.', + ), + connectionId: z + .string() + .describe( + 'Connection ID (not sourceId). Call hyperdx_list_sources to find available connections.', + ), + sql: z + .string() + .describe( + 'Raw ClickHouse SQL query to execute. ' + + 'Always include a LIMIT clause to avoid returning excessive data.\n\n' + + 'QUERY PARAMETERS (ClickHouse native parameterized syntax):\n' + + ' {startDateMilliseconds:Int64} — start of date range in ms since epoch\n' + + ' {endDateMilliseconds:Int64} — end of date range in ms since epoch\n' + + ' {intervalSeconds:Int64} — time bucket size in seconds (time-series only)\n' + + ' {intervalMilliseconds:Int64} — time bucket size in milliseconds (time-series only)\n\n' + + 'MACROS (expanded before execution):\n' + + ' $__timeFilter(column) — expands to: column >= AND column <= (DateTime precision)\n' + + ' $__timeFilter_ms(column) — same but with DateTime64 millisecond precision\n' + + ' $__dateFilter(column) — same but with Date precision\n' + + ' $__dateTimeFilter(dateCol, timeCol) — filters on both a Date and DateTime column\n' + + ' $__dt(dateCol, timeCol) — alias for $__dateTimeFilter\n' + + ' $__fromTime / $__toTime — start/end as DateTime values\n' + + ' $__fromTime_ms / $__toTime_ms — start/end as DateTime64 values\n' + + ' $__timeInterval(column) — time bucket expression: toStartOfInterval(toDateTime(column), INTERVAL ...)\n' + + ' $__timeInterval_ms(column) — same with millisecond precision\n' + + ' $__interval_s — raw interval in seconds\n' + + ' $__filters — placeholder for dashboard filter conditions (resolves to 1=1 when no filters)\n\n' + + 'Example (time-series): "SELECT $__timeInterval(TimestampTime) AS ts, ServiceName, count() ' + + 'FROM otel_logs WHERE $__timeFilter(TimestampTime) GROUP BY ServiceName, ts ORDER BY ts"\n\n' + + 'Example (table): "SELECT ServiceName, count() AS n FROM otel_logs ' + + 'WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) ' + + 'AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) ' + + 'GROUP BY ServiceName ORDER BY n DESC LIMIT 20"', + ), +}); + +export const hyperdxQuerySchema = z.discriminatedUnion('displayType', [ + builderQuerySchema, + searchQuerySchema, + sqlQuerySchema, +]); diff --git a/packages/api/src/mcp/tools/types.ts b/packages/api/src/mcp/tools/types.ts new file mode 100644 index 00000000..c0d669ad --- /dev/null +++ b/packages/api/src/mcp/tools/types.ts @@ -0,0 +1,10 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; + +export type McpContext = { + teamId: string; + userId?: string; +}; + +export type ToolDefinition = (server: McpServer, context: McpContext) => void; + +export type PromptDefinition = (server: McpServer, context: McpContext) => void; diff --git a/packages/api/src/mcp/utils/tracing.ts b/packages/api/src/mcp/utils/tracing.ts new file mode 100644 index 00000000..547330d5 --- /dev/null +++ b/packages/api/src/mcp/utils/tracing.ts @@ -0,0 +1,83 @@ +import opentelemetry, { SpanStatusCode } from '@opentelemetry/api'; + +import { CODE_VERSION } from '@/config'; +import logger from '@/utils/logger'; + +import type { McpContext } from '../tools/types'; + +const mcpTracer = opentelemetry.trace.getTracer('hyperdx-mcp', CODE_VERSION); + +type ToolResult = { + content: { type: 'text'; text: string }[]; + isError?: boolean; +}; + +/** + * Wraps an MCP tool handler with tracing and structured logging. + * Creates a span for each tool invocation and logs start/end with duration. + */ +export function withToolTracing( + toolName: string, + context: McpContext, + handler: (args: TArgs) => Promise, +): (args: TArgs) => Promise { + return async (args: TArgs) => { + return mcpTracer.startActiveSpan(`mcp.tool.${toolName}`, async span => { + const startTime = Date.now(); + const logContext = { + tool: toolName, + teamId: context.teamId, + userId: context.userId, + }; + + span.setAttribute('mcp.tool.name', toolName); + span.setAttribute('mcp.team.id', context.teamId); + if (context.userId) { + span.setAttribute('mcp.user.id', context.userId); + } + + logger.info(logContext, `MCP tool invoked: ${toolName}`); + + try { + const result = await handler(args); + const durationMs = Date.now() - startTime; + + if (result.isError) { + span.setStatus({ code: SpanStatusCode.ERROR }); + span.setAttribute('mcp.tool.error', true); + logger.warn( + { ...logContext, durationMs }, + `MCP tool error: ${toolName}`, + ); + } else { + span.setStatus({ code: SpanStatusCode.OK }); + logger.info( + { ...logContext, durationMs }, + `MCP tool completed: ${toolName}`, + ); + } + + span.setAttribute('mcp.tool.duration_ms', durationMs); + span.end(); + return result; + } catch (err) { + const durationMs = Date.now() - startTime; + span.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : String(err), + }); + span.recordException( + err instanceof Error ? err : new Error(String(err)), + ); + span.setAttribute('mcp.tool.duration_ms', durationMs); + span.end(); + + logger.error( + { ...logContext, durationMs, error: err }, + `MCP tool failed: ${toolName}`, + ); + throw err; + } + }); + }; +} diff --git a/packages/api/src/middleware/error.ts b/packages/api/src/middleware/error.ts index ab913b8c..a8794b38 100644 --- a/packages/api/src/middleware/error.ts +++ b/packages/api/src/middleware/error.ts @@ -3,6 +3,7 @@ import type { NextFunction, Request, Response } from 'express'; import { IS_PROD } from '@/config'; import { BaseError, isOperationalError, StatusCode } from '@/utils/errors'; +import logger from '@/utils/logger'; // WARNING: need to keep the 4th arg for express to identify it as an error-handling middleware function export const appErrorHandler = ( @@ -11,7 +12,11 @@ export const appErrorHandler = ( res: Response, next: NextFunction, ) => { - console.error(err); + if (isOperationalError(err)) { + logger.warn({ err }, err.message); + } else { + logger.error({ err }, err.message); + } const userFacingErrorMessage = isOperationalError(err) ? err.name || err.message diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index b0b1415a..159cb07e 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -1,14 +1,13 @@ -import { ALERT_INTERVAL_TO_MINUTES } from '@hyperdx/common-utils/dist/types'; +import { + ALERT_INTERVAL_TO_MINUTES, + AlertThresholdType, +} from '@hyperdx/common-utils/dist/types'; +export { AlertThresholdType } from '@hyperdx/common-utils/dist/types'; import mongoose, { Schema } from 'mongoose'; import type { ObjectId } from '.'; import Team from './team'; -export enum AlertThresholdType { - ABOVE = 'above', - BELOW = 'below', -} - export enum AlertState { ALERT = 'ALERT', DISABLED = 'DISABLED', @@ -51,6 +50,8 @@ export interface IAlert { state: AlertState; team: ObjectId; threshold: number; + /** The upper bound for BETWEEN and NOT BETWEEN threshold types */ + thresholdMax?: number; thresholdType: AlertThresholdType; createdBy?: ObjectId; @@ -84,6 +85,10 @@ const AlertSchema = new Schema( type: Number, required: true, }, + thresholdMax: { + type: Number, + required: false, + }, thresholdType: { type: String, enum: AlertThresholdType, diff --git a/packages/api/src/models/dashboard.ts b/packages/api/src/models/dashboard.ts index 8a6cfdf8..fa24833c 100644 --- a/packages/api/src/models/dashboard.ts +++ b/packages/api/src/models/dashboard.ts @@ -7,6 +7,8 @@ import type { ObjectId } from '.'; export interface IDashboard extends z.infer { _id: ObjectId; team: ObjectId; + createdBy?: ObjectId; + updatedBy?: ObjectId; createdAt: Date; updatedAt: Date; } @@ -32,6 +34,16 @@ export default mongoose.model( savedQueryLanguage: { type: String, required: false }, savedFilterValues: { type: mongoose.Schema.Types.Array, required: false }, containers: { type: mongoose.Schema.Types.Array, required: false }, + createdBy: { + type: mongoose.Schema.Types.ObjectId, + ref: 'User', + required: false, + }, + updatedBy: { + type: mongoose.Schema.Types.ObjectId, + ref: 'User', + required: false, + }, }, { timestamps: true, diff --git a/packages/api/src/models/pinnedFilter.ts b/packages/api/src/models/pinnedFilter.ts new file mode 100644 index 00000000..5241fff4 --- /dev/null +++ b/packages/api/src/models/pinnedFilter.ts @@ -0,0 +1,49 @@ +import type { PinnedFiltersValue } from '@hyperdx/common-utils/dist/types'; +import mongoose, { Schema } from 'mongoose'; + +import type { ObjectId } from '.'; + +interface IPinnedFilter { + _id: ObjectId; + team: ObjectId; + source: ObjectId; + fields: string[]; + filters: PinnedFiltersValue; + createdAt: Date; + updatedAt: Date; +} + +const PinnedFilterSchema = new Schema( + { + team: { + type: mongoose.Schema.Types.ObjectId, + required: true, + ref: 'Team', + }, + source: { + type: mongoose.Schema.Types.ObjectId, + required: true, + ref: 'Source', + }, + fields: { + type: [String], + default: [], + }, + filters: { + type: Schema.Types.Mixed, + default: {}, + }, + }, + { + timestamps: true, + toJSON: { getters: true }, + }, +); + +// One document per team+source combination +PinnedFilterSchema.index({ team: 1, source: 1 }, { unique: true }); + +export default mongoose.model( + 'PinnedFilter', + PinnedFilterSchema, +); diff --git a/packages/api/src/models/savedSearch.ts b/packages/api/src/models/savedSearch.ts index ac9f02ca..e2893656 100644 --- a/packages/api/src/models/savedSearch.ts +++ b/packages/api/src/models/savedSearch.ts @@ -9,6 +9,8 @@ export interface ISavedSearch _id: ObjectId; team: ObjectId; source: ObjectId; + createdBy?: ObjectId; + updatedBy?: ObjectId; createdAt: Date; updatedAt: Date; } @@ -35,6 +37,16 @@ export const SavedSearch = mongoose.model( }, tags: [String], filters: [{ type: mongoose.Schema.Types.Mixed }], + createdBy: { + type: mongoose.Schema.Types.ObjectId, + ref: 'User', + required: false, + }, + updatedBy: { + type: mongoose.Schema.Types.ObjectId, + ref: 'User', + required: false, + }, }, { toJSON: { virtuals: true }, diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 5f70ec76..1b1ca941 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -82,6 +82,7 @@ type CollectorConfig = { logs_table_name: string; timeout: string; create_schema: string; + json: string; retry_on_failure: { enabled: boolean; initial_interval: string; @@ -97,6 +98,7 @@ type CollectorConfig = { ttl: string; timeout: string; create_schema: string; + json: string; retry_on_failure: { enabled: boolean; initial_interval: string; @@ -205,6 +207,7 @@ export const buildOtelCollectorConfig = ( timeout: '5s', create_schema: '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}', + json: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}', retry_on_failure: { enabled: true, initial_interval: '5s', @@ -221,6 +224,7 @@ export const buildOtelCollectorConfig = ( timeout: '5s', create_schema: '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}', + json: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE:-false}', retry_on_failure: { enabled: true, initial_interval: '5s', diff --git a/packages/api/src/routers/api/__tests__/alerts.test.ts b/packages/api/src/routers/api/__tests__/alerts.test.ts index f1b49958..7dace1c7 100644 --- a/packages/api/src/routers/api/__tests__/alerts.test.ts +++ b/packages/api/src/routers/api/__tests__/alerts.test.ts @@ -1,12 +1,21 @@ +import { + AlertThresholdType, + DisplayType, +} from '@hyperdx/common-utils/dist/types'; + import { getLoggedInAgent, getServer, makeAlertInput, + makeRawSqlAlertTile, + makeRawSqlNumberAlertTile, makeRawSqlTile, makeTile, randomMongoId, + RAW_SQL_ALERT_TEMPLATE, } from '@/fixtures'; -import Alert, { AlertSource, AlertThresholdType } from '@/models/alert'; +import Alert, { AlertSource, AlertState } from '@/models/alert'; +import AlertHistory from '@/models/alertHistory'; import Webhook, { WebhookDocument, WebhookService } from '@/models/webhook'; const MOCK_TILES = [makeTile(), makeTile(), makeTile(), makeTile(), makeTile()]; @@ -550,8 +559,61 @@ describe('alerts router', () => { await agent.delete(`/alerts/${fakeId}/silenced`).expect(404); // Should fail }); - it('rejects creating an alert on a raw SQL tile', async () => { - const rawSqlTile = makeRawSqlTile(); + it('allows creating an alert on a raw SQL line tile', async () => { + const rawSqlTile = makeRawSqlAlertTile(); + const dashboard = await agent + .post('/dashboards') + .send({ + name: 'Test Dashboard', + tiles: [rawSqlTile], + tags: [], + }) + .expect(200); + + const alert = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: rawSqlTile.id, + webhookId: webhook._id.toString(), + }), + ) + .expect(200); + expect(alert.body.data.dashboard).toBe(dashboard.body.id); + expect(alert.body.data.tileId).toBe(rawSqlTile.id); + }); + + it('allows creating an alert on a raw SQL number tile', async () => { + const rawSqlTile = makeRawSqlNumberAlertTile(); + const dashboard = await agent + .post('/dashboards') + .send({ + name: 'Test Dashboard', + tiles: [rawSqlTile], + tags: [], + }) + .expect(200); + + const alert = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: rawSqlTile.id, + webhookId: webhook._id.toString(), + }), + ) + .expect(200); + expect(alert.body.data.dashboard).toBe(dashboard.body.id); + expect(alert.body.data.tileId).toBe(rawSqlTile.id); + }); + + it('rejects creating an alert on a raw SQL table tile', async () => { + const rawSqlTile = makeRawSqlTile({ + displayType: DisplayType.Table, + sqlTemplate: RAW_SQL_ALERT_TEMPLATE, + }); const dashboard = await agent .post('/dashboards') .send({ @@ -573,9 +635,72 @@ describe('alerts router', () => { .expect(400); }); - it('rejects updating an alert to reference a raw SQL tile', async () => { + it('rejects creating an alert on a raw SQL tile without interval params', async () => { + const rawSqlTile = makeRawSqlTile({ + sqlTemplate: 'SELECT count() FROM otel_logs', + }); + const dashboard = await agent + .post('/dashboards') + .send({ + name: 'Test Dashboard', + tiles: [rawSqlTile], + tags: [], + }) + .expect(200); + + await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: rawSqlTile.id, + webhookId: webhook._id.toString(), + }), + ) + .expect(400); + }); + + it('allows updating an alert to reference a raw SQL number tile', async () => { const regularTile = makeTile(); - const rawSqlTile = makeRawSqlTile(); + const rawSqlTile = makeRawSqlNumberAlertTile(); + const dashboard = await agent + .post('/dashboards') + .send({ + name: 'Test Dashboard', + tiles: [regularTile, rawSqlTile], + tags: [], + }) + .expect(200); + + const alert = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: regularTile.id, + webhookId: webhook._id.toString(), + }), + ) + .expect(200); + + await agent + .put(`/alerts/${alert.body.data._id}`) + .send({ + ...makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: rawSqlTile.id, + webhookId: webhook._id.toString(), + }), + }) + .expect(200); + }); + + it('rejects updating an alert to reference a raw SQL table tile', async () => { + const regularTile = makeTile(); + const rawSqlTile = makeRawSqlTile({ + displayType: DisplayType.Table, + sqlTemplate: RAW_SQL_ALERT_TEMPLATE, + }); const dashboard = await agent .post('/dashboards') .send({ @@ -607,4 +732,84 @@ describe('alerts router', () => { }) .expect(400); }); + + describe('GET /alerts/:id', () => { + it('returns 404 for non-existent alert', async () => { + const fakeId = randomMongoId(); + await agent.get(`/alerts/${fakeId}`).expect(404); + }); + + it('returns alert with empty history when no history exists', async () => { + const dashboard = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + + const alert = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: dashboard.body.tiles[0].id, + webhookId: webhook._id.toString(), + }), + ) + .expect(200); + + const res = await agent.get(`/alerts/${alert.body.data._id}`).expect(200); + + expect(res.body.data._id).toBe(alert.body.data._id); + expect(res.body.data.history).toEqual([]); + expect(res.body.data.threshold).toBe(alert.body.data.threshold); + expect(res.body.data.interval).toBe(alert.body.data.interval); + expect(res.body.data.dashboard).toBeDefined(); + expect(res.body.data.tileId).toBe(dashboard.body.tiles[0].id); + }); + + it('returns alert with history entries', async () => { + const dashboard = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + + const alert = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: dashboard.body.tiles[0].id, + webhookId: webhook._id.toString(), + }), + ) + .expect(200); + + const now = new Date(Date.now() - 60000); + const earlier = new Date(Date.now() - 120000); + + await AlertHistory.create({ + alert: alert.body.data._id, + createdAt: now, + state: AlertState.ALERT, + counts: 5, + lastValues: [{ startTime: now, count: 5 }], + }); + + await AlertHistory.create({ + alert: alert.body.data._id, + createdAt: earlier, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: earlier, count: 0 }], + }); + + const res = await agent.get(`/alerts/${alert.body.data._id}`).expect(200); + + expect(res.body.data._id).toBe(alert.body.data._id); + expect(res.body.data.history).toHaveLength(2); + expect(res.body.data.history[0].state).toBe('ALERT'); + expect(res.body.data.history[0].counts).toBe(5); + expect(res.body.data.history[1].state).toBe('OK'); + expect(res.body.data.history[1].counts).toBe(0); + }); + }); }); diff --git a/packages/api/src/routers/api/__tests__/dashboard.test.ts b/packages/api/src/routers/api/__tests__/dashboard.test.ts index 04c20f2e..415c95aa 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.test.ts @@ -20,6 +20,8 @@ import { makeTile, } from '../../../fixtures'; import Alert from '../../../models/alert'; +import Dashboard from '../../../models/dashboard'; +import User from '../../../models/user'; const MOCK_DASHBOARD = { name: 'Test Dashboard', @@ -78,6 +80,45 @@ describe('dashboard router', () => { ); }); + it('sets createdBy and updatedBy on create and populates them in GET', async () => { + const created = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + + // GET all dashboards + const allDashboards = await agent.get('/dashboards').expect(200); + const dashboard = allDashboards.body.find(d => d._id === created.body.id); + expect(dashboard.createdBy).toMatchObject({ email: user.email }); + expect(dashboard.updatedBy).toMatchObject({ email: user.email }); + }); + + it('populates updatedBy with a different user after DB update', async () => { + const created = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + + // Create a second user on the same team + const secondUser = await User.create({ + email: 'second@test.com', + name: 'Second User', + team: team._id, + }); + + // Simulate a different user updating the dashboard + await Dashboard.findByIdAndUpdate(created.body.id, { + updatedBy: secondUser._id, + }); + + const allDashboards = await agent.get('/dashboards').expect(200); + const dashboard = allDashboards.body.find(d => d._id === created.body.id); + expect(dashboard.createdBy).toMatchObject({ email: user.email }); + expect(dashboard.updatedBy).toMatchObject({ + email: 'second@test.com', + }); + }); + it('can update a dashboard', async () => { const dashboard = await agent .post('/dashboards') diff --git a/packages/api/src/routers/api/__tests__/pinnedFilters.test.ts b/packages/api/src/routers/api/__tests__/pinnedFilters.test.ts new file mode 100644 index 00000000..3e86bde9 --- /dev/null +++ b/packages/api/src/routers/api/__tests__/pinnedFilters.test.ts @@ -0,0 +1,214 @@ +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { Types } from 'mongoose'; + +import { getLoggedInAgent, getServer } from '@/fixtures'; +import { Source } from '@/models/source'; + +const MOCK_SOURCE: Omit, 'id'> = { + kind: SourceKind.Log, + name: 'Test Source', + connection: new Types.ObjectId().toString(), + from: { databaseName: 'test_db', tableName: 'test_table' }, + timestampValueExpression: 'timestamp', + defaultTableSelectExpression: 'body', +}; + +describe('pinnedFilters router', () => { + const server = getServer(); + let agent: Awaited>['agent']; + let team: Awaited>['team']; + let sourceId: string; + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const result = await getLoggedInAgent(server); + agent = result.agent; + team = result.team; + + // Create a real source owned by this team + const source = await Source.create({ ...MOCK_SOURCE, team: team._id }); + sourceId = source._id.toString(); + }); + + afterEach(async () => { + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + describe('GET /pinned-filters', () => { + it('returns null when no pinned filters exist', async () => { + const res = await agent + .get(`/pinned-filters?source=${sourceId}`) + .expect(200); + + expect(res.body.team).toBeNull(); + }); + + it('rejects invalid source id', async () => { + await agent.get('/pinned-filters?source=not-an-objectid').expect(400); + }); + + it('rejects missing source param', async () => { + await agent.get('/pinned-filters').expect(400); + }); + + it('returns 404 for a source not owned by the team', async () => { + const foreignSourceId = new Types.ObjectId().toString(); + await agent.get(`/pinned-filters?source=${foreignSourceId}`).expect(404); + }); + }); + + describe('PUT /pinned-filters', () => { + it('can create pinned filters', async () => { + const res = await agent + .put('/pinned-filters') + .send({ + source: sourceId, + fields: ['ServiceName', 'SeverityText'], + filters: { ServiceName: ['web', 'api'] }, + }) + .expect(200); + + expect(res.body.fields).toEqual(['ServiceName', 'SeverityText']); + expect(res.body.filters).toEqual({ ServiceName: ['web', 'api'] }); + expect(res.body.id).toBeDefined(); + }); + + it('upserts on repeated PUT', async () => { + await agent + .put('/pinned-filters') + .send({ + source: sourceId, + fields: ['ServiceName'], + filters: { ServiceName: ['web'] }, + }) + .expect(200); + + const res = await agent + .put('/pinned-filters') + .send({ + source: sourceId, + fields: ['ServiceName', 'SeverityText'], + filters: { ServiceName: ['web', 'api'], SeverityText: ['error'] }, + }) + .expect(200); + + expect(res.body.fields).toEqual(['ServiceName', 'SeverityText']); + expect(res.body.filters).toEqual({ + ServiceName: ['web', 'api'], + SeverityText: ['error'], + }); + }); + + it('rejects invalid source id', async () => { + await agent + .put('/pinned-filters') + .send({ source: 'not-valid', fields: [], filters: {} }) + .expect(400); + }); + + it('returns 404 for a source not owned by the team', async () => { + const foreignSourceId = new Types.ObjectId().toString(); + await agent + .put('/pinned-filters') + .send({ source: foreignSourceId, fields: [], filters: {} }) + .expect(404); + }); + }); + + describe('GET + PUT round-trip', () => { + it('returns data after PUT', async () => { + await agent + .put('/pinned-filters') + .send({ + source: sourceId, + fields: ['ServiceName'], + filters: { ServiceName: ['web'] }, + }) + .expect(200); + + const res = await agent + .get(`/pinned-filters?source=${sourceId}`) + .expect(200); + + expect(res.body.team).not.toBeNull(); + expect(res.body.team.fields).toEqual(['ServiceName']); + expect(res.body.team.filters).toEqual({ ServiceName: ['web'] }); + }); + + it('can reset by sending empty fields and filters', async () => { + await agent + .put('/pinned-filters') + .send({ + source: sourceId, + fields: ['ServiceName'], + filters: { ServiceName: ['web'] }, + }) + .expect(200); + + await agent + .put('/pinned-filters') + .send({ source: sourceId, fields: [], filters: {} }) + .expect(200); + + const res = await agent + .get(`/pinned-filters?source=${sourceId}`) + .expect(200); + + expect(res.body.team).not.toBeNull(); + expect(res.body.team.fields).toEqual([]); + expect(res.body.team.filters).toEqual({}); + }); + }); + + describe('source scoping', () => { + it('pins are scoped to their source', async () => { + const source2 = await Source.create({ ...MOCK_SOURCE, team: team._id }); + + await agent + .put('/pinned-filters') + .send({ + source: sourceId, + fields: ['ServiceName'], + filters: { ServiceName: ['web'] }, + }) + .expect(200); + + const res = await agent + .get(`/pinned-filters?source=${source2._id}`) + .expect(200); + + expect(res.body.team).toBeNull(); + }); + }); + + // Note: cross-team isolation (Team B cannot read Team A's pins) is enforced + // by the MongoDB query filtering on teamId AND the source ownership check + // (getSource validates source.team === teamId). Multi-team integration tests + // are not possible in this single-team environment (register returns 409). + + describe('filter values with booleans', () => { + it('supports boolean values in filters', async () => { + await agent + .put('/pinned-filters') + .send({ + source: sourceId, + fields: ['isRootSpan'], + filters: { isRootSpan: [true, false] }, + }) + .expect(200); + + const res = await agent + .get(`/pinned-filters?source=${sourceId}`) + .expect(200); + + expect(res.body.team.filters).toEqual({ isRootSpan: [true, false] }); + }); + }); +}); diff --git a/packages/api/src/routers/api/__tests__/savedSearch.test.ts b/packages/api/src/routers/api/__tests__/savedSearch.test.ts index e9cc1a27..d0c2a78b 100644 --- a/packages/api/src/routers/api/__tests__/savedSearch.test.ts +++ b/packages/api/src/routers/api/__tests__/savedSearch.test.ts @@ -4,6 +4,8 @@ import { makeSavedSearchAlertInput, } from '@/fixtures'; import Alert from '@/models/alert'; +import { SavedSearch } from '@/models/savedSearch'; +import User from '@/models/user'; import Webhook, { WebhookDocument, WebhookService } from '@/models/webhook'; const MOCK_SAVED_SEARCH = { @@ -127,6 +129,66 @@ describe('savedSearch router', () => { expect(await Alert.findById(alert.body.data._id)).toBeNull(); }); + it('sets createdBy and updatedBy on create and populates them in GET', async () => { + const created = await agent + .post('/saved-search') + .send(MOCK_SAVED_SEARCH) + .expect(200); + + // GET all saved searches + const savedSearches = await agent.get('/saved-search').expect(200); + const savedSearch = savedSearches.body.find( + s => s._id === created.body._id, + ); + expect(savedSearch.createdBy).toMatchObject({ email: user.email }); + expect(savedSearch.updatedBy).toMatchObject({ email: user.email }); + }); + + it('populates updatedBy with a different user after DB update', async () => { + const created = await agent + .post('/saved-search') + .send(MOCK_SAVED_SEARCH) + .expect(200); + + // Create a second user on the same team + const secondUser = await User.create({ + email: 'second@test.com', + name: 'Second User', + team: team._id, + }); + + // Simulate a different user updating the saved search + await SavedSearch.findByIdAndUpdate(created.body._id, { + updatedBy: secondUser._id, + }); + + const savedSearches = await agent.get('/saved-search').expect(200); + const savedSearch = savedSearches.body.find( + s => s._id === created.body._id, + ); + expect(savedSearch.createdBy).toMatchObject({ email: user.email }); + expect(savedSearch.updatedBy).toMatchObject({ + email: 'second@test.com', + }); + }); + + it('updates updatedBy when updating a saved search via API', async () => { + const created = await agent + .post('/saved-search') + .send(MOCK_SAVED_SEARCH) + .expect(200); + + await agent + .patch(`/saved-search/${created.body._id}`) + .send({ name: 'updated name' }) + .expect(200); + + // Verify updatedBy is still set in the DB + const dbRecord = await SavedSearch.findById(created.body._id); + expect(dbRecord?.updatedBy?.toString()).toBe(user._id.toString()); + expect(dbRecord?.createdBy?.toString()).toBe(user._id.toString()); + }); + it('sets createdBy on alerts created from a saved search and populates it in list', async () => { // Create a saved search const savedSearch = await agent diff --git a/packages/api/src/routers/api/__tests__/team.test.ts b/packages/api/src/routers/api/__tests__/team.test.ts index 5ad06217..a362f844 100644 --- a/packages/api/src/routers/api/__tests__/team.test.ts +++ b/packages/api/src/routers/api/__tests__/team.test.ts @@ -31,11 +31,11 @@ describe('team router', () => { expect(_.omit(resp.body, ['_id', 'id', 'apiKey', 'createdAt'])) .toMatchInlineSnapshot(` -Object { - "allowedAuthMethods": Array [], - "name": "fake@deploysentinel.com's Team", -} -`); + { + "allowedAuthMethods": [], + "name": "fake@deploysentinel.com's Team", + } + `); }); it('GET /team/tags - no tags', async () => { @@ -43,7 +43,7 @@ Object { const resp = await agent.get('/team/tags').expect(200); - expect(resp.body.data).toMatchInlineSnapshot(`Array []`); + expect(resp.body.data).toMatchInlineSnapshot(`[]`); }); it('GET /team/tags', async () => { @@ -98,29 +98,27 @@ Object { }); const resp = await agent.get('/team/members').expect(200); - expect(resp.body.data).toMatchInlineSnapshot(` -Array [ - Object { - "_id": "${resp.body.data[0]._id}", - "email": "fake@deploysentinel.com", - "hasPasswordAuth": true, - "isCurrentUser": true, - "name": "fake@deploysentinel.com", - }, - Object { - "_id": "${user1._id}", - "email": "user1@example.com", - "hasPasswordAuth": true, - "isCurrentUser": false, - }, - Object { - "_id": "${user2._id}", - "email": "user2@example.com", - "hasPasswordAuth": true, - "isCurrentUser": false, - }, -] -`); + expect(resp.body.data.map(({ _id, ...rest }: any) => rest)) + .toMatchInlineSnapshot(` + [ + { + "email": "fake@deploysentinel.com", + "hasPasswordAuth": true, + "isCurrentUser": true, + "name": "fake@deploysentinel.com", + }, + { + "email": "user1@example.com", + "hasPasswordAuth": true, + "isCurrentUser": false, + }, + { + "email": "user2@example.com", + "hasPasswordAuth": true, + "isCurrentUser": false, + }, + ] + `); }); it('POST /team/invitation', async () => { @@ -236,17 +234,17 @@ Array [ name: i.name, })), ).toMatchInlineSnapshot(` -Array [ - Object { - "email": "user1@example.com", - "name": "User 1", - }, - Object { - "email": "user2@example.com", - "name": "User 2", - }, -] -`); + [ + { + "email": "user1@example.com", + "name": "User 1", + }, + { + "email": "user2@example.com", + "name": "User 2", + }, + ] + `); }); it('DELETE /team/member/:userId removes a user', async () => { diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index d1e2e985..25492a5d 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -1,24 +1,91 @@ -import type { AlertsApiResponse } from '@hyperdx/common-utils/dist/types'; +import type { + AlertApiResponse, + AlertsApiResponse, + AlertsPageItem, +} from '@hyperdx/common-utils/dist/types'; import express from 'express'; import { pick } from 'lodash'; import { ObjectId } from 'mongodb'; import { z } from 'zod'; import { processRequest, validateRequest } from 'zod-express-middleware'; -import { getRecentAlertHistoriesBatch } from '@/controllers/alertHistory'; +import { + getRecentAlertHistories, + getRecentAlertHistoriesBatch, +} from '@/controllers/alertHistory'; import { createAlert, deleteAlert, getAlertById, + getAlertEnhanced, getAlertsEnhanced, updateAlert, validateAlertInput, } from '@/controllers/alerts'; -import { sendJson } from '@/utils/serialization'; +import { IAlertHistory } from '@/models/alertHistory'; +import { PreSerialized, sendJson } from '@/utils/serialization'; import { alertSchema, objectIdSchema } from '@/utils/zod'; const router = express.Router(); +type EnhancedAlert = NonNullable>>; + +const formatAlertResponse = ( + alert: EnhancedAlert, + history: Omit[], +): PreSerialized => { + return { + history, + silenced: alert.silenced + ? { + by: alert.silenced.by?.email, + at: alert.silenced.at, + until: alert.silenced.until, + } + : undefined, + createdBy: alert.createdBy + ? pick(alert.createdBy, ['email', 'name']) + : undefined, + channel: pick(alert.channel, ['type']), + ...(alert.dashboard && { + dashboardId: alert.dashboard._id, + dashboard: { + tiles: alert.dashboard.tiles + .filter(tile => tile.id === alert.tileId) + .map(tile => ({ + id: tile.id, + config: { name: tile.config.name }, + })), + ...pick(alert.dashboard, ['_id', 'updatedAt', 'name', 'tags']), + }, + }), + ...(alert.savedSearch && { + savedSearchId: alert.savedSearch._id, + savedSearch: pick(alert.savedSearch, [ + '_id', + 'createdAt', + 'name', + 'updatedAt', + 'tags', + ]), + }), + ...pick(alert, [ + '_id', + 'interval', + 'scheduleOffsetMinutes', + 'scheduleStartAt', + 'threshold', + 'thresholdMax', + 'thresholdType', + 'state', + 'source', + 'tileId', + 'createdAt', + 'updatedAt', + ]), + }; +}; + type AlertsExpRes = express.Response; router.get('/', async (req, res: AlertsExpRes, next) => { try { @@ -39,63 +106,50 @@ router.get('/', async (req, res: AlertsExpRes, next) => { const data = alerts.map(alert => { const history = historyMap.get(alert._id.toString()) ?? []; - - return { - history, - silenced: alert.silenced - ? { - by: alert.silenced.by?.email, - at: alert.silenced.at, - until: alert.silenced.until, - } - : undefined, - createdBy: alert.createdBy - ? pick(alert.createdBy, ['email', 'name']) - : undefined, - channel: pick(alert.channel, ['type']), - ...(alert.dashboard && { - dashboardId: alert.dashboard._id, - dashboard: { - tiles: alert.dashboard.tiles - .filter(tile => tile.id === alert.tileId) - .map(tile => ({ - id: tile.id, - config: { name: tile.config.name }, - })), - ...pick(alert.dashboard, ['_id', 'updatedAt', 'name', 'tags']), - }, - }), - ...(alert.savedSearch && { - savedSearchId: alert.savedSearch._id, - savedSearch: pick(alert.savedSearch, [ - '_id', - 'createdAt', - 'name', - 'updatedAt', - 'tags', - ]), - }), - ...pick(alert, [ - '_id', - 'interval', - 'scheduleOffsetMinutes', - 'scheduleStartAt', - 'threshold', - 'thresholdType', - 'state', - 'source', - 'tileId', - 'createdAt', - 'updatedAt', - ]), - }; + return formatAlertResponse(alert, history); }); + sendJson(res, { data }); } catch (e) { next(e); } }); +type AlertExpRes = express.Response; +router.get( + '/:id', + validateRequest({ + params: z.object({ + id: objectIdSchema, + }), + }), + async (req, res: AlertExpRes, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + const alert = await getAlertEnhanced(req.params.id, teamId); + if (!alert) { + return res.sendStatus(404); + } + + const history = await getRecentAlertHistories({ + alertId: new ObjectId(alert._id), + interval: alert.interval, + limit: 20, + }); + + const data = formatAlertResponse(alert, history); + + sendJson(res, { data }); + } catch (e) { + next(e); + } + }, +); + router.post( '/', processRequest({ body: alertSchema }), diff --git a/packages/api/src/routers/api/pinnedFilters.ts b/packages/api/src/routers/api/pinnedFilters.ts new file mode 100644 index 00000000..d0b997ce --- /dev/null +++ b/packages/api/src/routers/api/pinnedFilters.ts @@ -0,0 +1,95 @@ +import { PinnedFiltersValueSchema } from '@hyperdx/common-utils/dist/types'; +import express from 'express'; +import { z } from 'zod'; +import { validateRequest } from 'zod-express-middleware'; + +import { + getPinnedFilters, + updatePinnedFilters, +} from '@/controllers/pinnedFilter'; +import { getSource } from '@/controllers/sources'; +import { getNonNullUserWithTeam } from '@/middleware/auth'; +import { objectIdSchema } from '@/utils/zod'; + +const router = express.Router(); + +/** + * GET /pinned-filters?source= + * Returns the team-level pinned filters for the source. + */ +router.get( + '/', + validateRequest({ + query: z.object({ + source: objectIdSchema, + }), + }), + async (req, res, next) => { + try { + const { teamId } = getNonNullUserWithTeam(req); + const { source } = req.query; + + // Verify the source belongs to this team + const sourceDoc = await getSource(teamId.toString(), source); + if (!sourceDoc) { + return res.status(404).json({ error: 'Source not found' }); + } + + const doc = await getPinnedFilters(teamId.toString(), source); + + return res.json({ + team: doc + ? { + id: doc._id.toString(), + fields: doc.fields, + filters: doc.filters, + } + : null, + }); + } catch (e) { + next(e); + } + }, +); + +const updateBodySchema = z.object({ + source: objectIdSchema, + fields: z.array(z.string().max(1024)).max(100), + filters: PinnedFiltersValueSchema, +}); + +/** + * PUT /pinned-filters + * Upserts team-level pinned filters for the given source. + */ +router.put( + '/', + validateRequest({ body: updateBodySchema }), + async (req, res, next) => { + try { + const { teamId } = getNonNullUserWithTeam(req); + const { source, fields, filters } = req.body; + + // Verify the source belongs to this team + const sourceDoc = await getSource(teamId.toString(), source); + if (!sourceDoc) { + return res.status(404).json({ error: 'Source not found' }); + } + + const doc = await updatePinnedFilters(teamId.toString(), source, { + fields, + filters, + }); + + return res.json({ + id: doc._id.toString(), + fields: doc.fields, + filters: doc.filters, + }); + } catch (e) { + next(e); + } + }, +); + +export default router; diff --git a/packages/api/src/routers/api/savedSearch.ts b/packages/api/src/routers/api/savedSearch.ts index ec75d2af..7ad06721 100644 --- a/packages/api/src/routers/api/savedSearch.ts +++ b/packages/api/src/routers/api/savedSearch.ts @@ -1,4 +1,7 @@ -import { SavedSearchSchema } from '@hyperdx/common-utils/dist/types'; +import { + SavedSearchListApiResponse, + SavedSearchSchema, +} from '@hyperdx/common-utils/dist/types'; import express from 'express'; import _ from 'lodash'; import { z } from 'zod'; @@ -16,7 +19,9 @@ import { objectIdSchema } from '@/utils/zod'; const router = express.Router(); -router.get('/', async (req, res, next) => { +type SavedSearchListExpRes = express.Response; + +router.get('/', async (req, res: SavedSearchListExpRes, next) => { try { const { teamId } = getNonNullUserWithTeam(req); @@ -37,9 +42,13 @@ router.post( }), async (req, res, next) => { try { - const { teamId } = getNonNullUserWithTeam(req); + const { teamId, userId } = getNonNullUserWithTeam(req); - const savedSearch = await createSavedSearch(teamId.toString(), req.body); + const savedSearch = await createSavedSearch( + teamId.toString(), + req.body, + userId?.toString(), + ); return res.json(savedSearch); } catch (e) { @@ -60,7 +69,7 @@ router.patch( }), async (req, res, next) => { try { - const { teamId } = getNonNullUserWithTeam(req); + const { teamId, userId } = getNonNullUserWithTeam(req); const savedSearch = await getSavedSearch( teamId.toString(), @@ -82,6 +91,7 @@ router.patch( source: savedSearch.source.toString(), ...updates, }, + userId?.toString(), ); if (!updatedSavedSearch) { diff --git a/packages/api/src/routers/external-api/__tests__/alerts.test.ts b/packages/api/src/routers/external-api/__tests__/alerts.test.ts index 68c2b082..5c5875bc 100644 --- a/packages/api/src/routers/external-api/__tests__/alerts.test.ts +++ b/packages/api/src/routers/external-api/__tests__/alerts.test.ts @@ -2,7 +2,12 @@ import _ from 'lodash'; import { ObjectId } from 'mongodb'; import request from 'supertest'; -import { getLoggedInAgent, getServer } from '../../../fixtures'; +import { + getLoggedInAgent, + getServer, + RAW_SQL_ALERT_TEMPLATE, + RAW_SQL_NUMBER_ALERT_TEMPLATE, +} from '../../../fixtures'; import { AlertSource, AlertThresholdType } from '../../../models/alert'; import Alert from '../../../models/alert'; import Dashboard from '../../../models/dashboard'; @@ -83,8 +88,13 @@ describe('External API Alerts', () => { }; // Helper to create a dashboard with a raw SQL tile for testing + // Uses Number display type by default (not alertable) for rejection tests const createTestDashboardWithRawSqlTile = async ( - options: { teamId?: any } = {}, + options: { + teamId?: any; + displayType?: string; + sqlTemplate?: string; + } = {}, ) => { const tileId = new ObjectId().toString(); const tiles = [ @@ -97,8 +107,8 @@ describe('External API Alerts', () => { h: 3, config: { configType: 'sql', - displayType: 'line', - sqlTemplate: 'SELECT 1', + displayType: options.displayType ?? 'number', + sqlTemplate: options.sqlTemplate ?? 'SELECT 1', connection: 'test-connection', }, }, @@ -716,9 +726,66 @@ describe('External API Alerts', () => { .expect(400); }); - it('should reject creating an alert on a raw SQL tile', async () => { + it('should allow creating an alert on a raw SQL line tile', async () => { const webhook = await createTestWebhook(); - const { dashboard, tileId } = await createTestDashboardWithRawSqlTile(); + const { dashboard, tileId } = await createTestDashboardWithRawSqlTile({ + displayType: 'line', + sqlTemplate: RAW_SQL_ALERT_TEMPLATE, + }); + + const alertInput = { + dashboardId: dashboard._id.toString(), + tileId, + threshold: 100, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.ABOVE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }; + + const res = await authRequest('post', ALERTS_BASE_URL) + .send(alertInput) + .expect(200); + expect(res.body.data.dashboardId).toBe(dashboard._id.toString()); + expect(res.body.data.tileId).toBe(tileId); + }); + + it('should allow creating an alert on a raw SQL number tile', async () => { + const webhook = await createTestWebhook(); + const { dashboard, tileId } = await createTestDashboardWithRawSqlTile({ + displayType: 'number', + sqlTemplate: RAW_SQL_NUMBER_ALERT_TEMPLATE, + }); + + const alertInput = { + dashboardId: dashboard._id.toString(), + tileId, + threshold: 100, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.ABOVE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }; + + const res = await authRequest('post', ALERTS_BASE_URL) + .send(alertInput) + .expect(200); + expect(res.body.data.dashboardId).toBe(dashboard._id.toString()); + expect(res.body.data.tileId).toBe(tileId); + }); + + it('should reject creating an alert on a raw SQL table tile', async () => { + const webhook = await createTestWebhook(); + const { dashboard, tileId } = await createTestDashboardWithRawSqlTile({ + displayType: 'table', + sqlTemplate: RAW_SQL_ALERT_TEMPLATE, + }); const alertInput = { dashboardId: dashboard._id.toString(), @@ -736,10 +803,36 @@ describe('External API Alerts', () => { await authRequest('post', ALERTS_BASE_URL).send(alertInput).expect(400); }); - it('should reject updating an alert to reference a raw SQL tile', async () => { + it('should reject creating an alert on a raw SQL tile without interval params', async () => { + const webhook = await createTestWebhook(); + const { dashboard, tileId } = await createTestDashboardWithRawSqlTile({ + displayType: 'line', + sqlTemplate: 'SELECT count() FROM otel_logs', + }); + + const alertInput = { + dashboardId: dashboard._id.toString(), + tileId, + threshold: 100, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.ABOVE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }; + + await authRequest('post', ALERTS_BASE_URL).send(alertInput).expect(400); + }); + + it('should reject updating an alert to reference a raw SQL table tile', async () => { const { alert, webhook } = await createTestAlert(); const { dashboard: rawSqlDashboard, tileId: rawSqlTileId } = - await createTestDashboardWithRawSqlTile(); + await createTestDashboardWithRawSqlTile({ + displayType: 'table', + sqlTemplate: RAW_SQL_ALERT_TEMPLATE, + }); const updatePayload = { threshold: 200, @@ -872,6 +965,197 @@ describe('External API Alerts', () => { }); }); + describe('BETWEEN and NOT_BETWEEN threshold types', () => { + it('should create an alert with BETWEEN threshold type', async () => { + const dashboard = await createTestDashboard(); + const webhook = await createTestWebhook(); + + const response = await authRequest('post', ALERTS_BASE_URL) + .send({ + dashboardId: dashboard._id.toString(), + tileId: dashboard.tiles[0].id, + threshold: 50, + thresholdMax: 200, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.BETWEEN, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }) + .expect(200); + + const alert = response.body.data; + expect(alert.threshold).toBe(50); + expect(alert.thresholdMax).toBe(200); + expect(alert.thresholdType).toBe(AlertThresholdType.BETWEEN); + }); + + it('should create an alert with NOT_BETWEEN threshold type', async () => { + const dashboard = await createTestDashboard(); + const webhook = await createTestWebhook(); + + const response = await authRequest('post', ALERTS_BASE_URL) + .send({ + dashboardId: dashboard._id.toString(), + tileId: dashboard.tiles[0].id, + threshold: 10, + thresholdMax: 90, + interval: '5m', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.NOT_BETWEEN, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }) + .expect(200); + + const alert = response.body.data; + expect(alert.threshold).toBe(10); + expect(alert.thresholdMax).toBe(90); + expect(alert.thresholdType).toBe(AlertThresholdType.NOT_BETWEEN); + }); + + it('should reject BETWEEN without thresholdMax', async () => { + const consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + + const dashboard = await createTestDashboard(); + const webhook = await createTestWebhook(); + + await authRequest('post', ALERTS_BASE_URL) + .send({ + dashboardId: dashboard._id.toString(), + tileId: dashboard.tiles[0].id, + threshold: 50, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.BETWEEN, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }) + .expect(400); + + consoleErrorSpy.mockRestore(); + }); + + it('should reject BETWEEN when thresholdMax < threshold', async () => { + const consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + + const dashboard = await createTestDashboard(); + const webhook = await createTestWebhook(); + + await authRequest('post', ALERTS_BASE_URL) + .send({ + dashboardId: dashboard._id.toString(), + tileId: dashboard.tiles[0].id, + threshold: 100, + thresholdMax: 50, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.BETWEEN, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }) + .expect(400); + + consoleErrorSpy.mockRestore(); + }); + + it('should allow thresholdMax equal to threshold for BETWEEN', async () => { + const dashboard = await createTestDashboard(); + const webhook = await createTestWebhook(); + + const response = await authRequest('post', ALERTS_BASE_URL) + .send({ + dashboardId: dashboard._id.toString(), + tileId: dashboard.tiles[0].id, + threshold: 100, + thresholdMax: 100, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.BETWEEN, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }) + .expect(200); + + expect(response.body.data.threshold).toBe(100); + expect(response.body.data.thresholdMax).toBe(100); + }); + + it('should update an alert to use BETWEEN threshold type', async () => { + const { alert, dashboard, webhook } = await createTestAlert(); + + const updateResponse = await authRequest( + 'put', + `${ALERTS_BASE_URL}/${alert.id}`, + ) + .send({ + dashboardId: dashboard._id.toString(), + tileId: dashboard.tiles[0].id, + threshold: 20, + thresholdMax: 80, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.BETWEEN, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }) + .expect(200); + + const updatedAlert = updateResponse.body.data; + expect(updatedAlert.threshold).toBe(20); + expect(updatedAlert.thresholdMax).toBe(80); + expect(updatedAlert.thresholdType).toBe(AlertThresholdType.BETWEEN); + }); + + it('should retrieve a BETWEEN alert with thresholdMax', async () => { + const dashboard = await createTestDashboard(); + const webhook = await createTestWebhook(); + + const createResponse = await authRequest('post', ALERTS_BASE_URL) + .send({ + dashboardId: dashboard._id.toString(), + tileId: dashboard.tiles[0].id, + threshold: 10, + thresholdMax: 50, + interval: '1h', + source: AlertSource.TILE, + thresholdType: AlertThresholdType.BETWEEN, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + }) + .expect(200); + + const getResponse = await authRequest( + 'get', + `${ALERTS_BASE_URL}/${createResponse.body.data.id}`, + ).expect(200); + + expect(getResponse.body.data.threshold).toBe(10); + expect(getResponse.body.data.thresholdMax).toBe(50); + expect(getResponse.body.data.thresholdType).toBe( + AlertThresholdType.BETWEEN, + ); + }); + }); + describe('Authentication', () => { it('should require authentication', async () => { // Create an unauthenticated agent diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts index a5faa865..d7f79173 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.test.ts @@ -3387,7 +3387,7 @@ describe('External API v2 Dashboards - new format', () => { }); }); - it('should delete alert when tile is updated from builder to raw SQL config', async () => { + it('should delete alert when tile is updated from builder to raw SQL config and the display type does not support alerts', async () => { const tileId = new ObjectId().toString(); const dashboard = await createTestDashboard({ tiles: [ @@ -3399,7 +3399,7 @@ describe('External API v2 Dashboards - new format', () => { w: 6, h: 3, config: { - displayType: 'line', + displayType: 'number', source: traceSource._id.toString(), select: [ { @@ -3455,7 +3455,7 @@ describe('External API v2 Dashboards - new format', () => { h: 3, config: { configType: 'sql', - displayType: 'line', + displayType: 'table', connectionId: connection._id.toString(), sqlTemplate: 'SELECT count() FROM otel_logs WHERE {timeFilter}', }, diff --git a/packages/api/src/routers/external-api/v2/alerts.ts b/packages/api/src/routers/external-api/v2/alerts.ts index 00684265..7633ed8f 100644 --- a/packages/api/src/routers/external-api/v2/alerts.ts +++ b/packages/api/src/routers/external-api/v2/alerts.ts @@ -34,7 +34,7 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * description: Evaluation interval. * AlertThresholdType: * type: string - * enum: [above, below] + * enum: [above, below, above_exclusive, below_or_equal, equal, not_equal, between, not_between] * description: Threshold comparison direction. * AlertSource: * type: string @@ -95,7 +95,7 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * example: "65f5e4a3b9e77c001a567890" * tileId: * type: string - * description: Tile ID for tile-based alerts. May not be a Raw-SQL-based tile. + * description: Tile ID for tile-based alerts. Must be a line, stacked bar, or number type tile. * nullable: true * example: "65f5e4a3b9e77c001a901234" * savedSearchId: @@ -110,8 +110,13 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * example: "ServiceName" * threshold: * type: number - * description: Threshold value for triggering the alert. + * description: Threshold value for triggering the alert. For between and not_between threshold types, this is the lower bound. * example: 100 + * thresholdMax: + * type: number + * nullable: true + * description: Upper bound for between and not_between threshold types. Required when thresholdType is between or not_between, must be >= threshold. + * example: 500 * interval: * $ref: '#/components/schemas/AlertInterval' * description: Evaluation interval for the alert. diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 9fa36d2d..ca8b8ebd 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -1,104 +1,30 @@ -import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; -import { SearchConditionLanguageSchema as whereLanguageSchema } from '@hyperdx/common-utils/dist/types'; import express from 'express'; import { uniq } from 'lodash'; -import { ObjectId } from 'mongodb'; import mongoose from 'mongoose'; import { z } from 'zod'; -import { deleteDashboardAlerts } from '@/controllers/alerts'; -import { getConnectionsByTeam } from '@/controllers/connection'; import { deleteDashboard } from '@/controllers/dashboard'; import { getSources } from '@/controllers/sources'; import Dashboard from '@/models/dashboard'; import { validateRequestWithEnhancedErrors as validateRequest } from '@/utils/enhancedErrors'; -import { - translateExternalChartToTileConfig, - translateExternalFilterToFilter, -} from '@/utils/externalApi'; import logger from '@/utils/logger'; -import { - ExternalDashboardFilter, - externalDashboardFilterSchema, - externalDashboardFilterSchemaWithId, - ExternalDashboardFilterWithId, - externalDashboardSavedFilterValueSchema, - externalDashboardTileListSchema, - ExternalDashboardTileWithId, - objectIdSchema, - tagsSchema, -} from '@/utils/zod'; +import { ExternalDashboardTileWithId, objectIdSchema } from '@/utils/zod'; import { + cleanupDashboardAlerts, + convertExternalFiltersToInternal, + convertExternalTilesToInternal, convertToExternalDashboard, - convertToInternalTileConfig, + createDashboardBodySchema, + getMissingConnections, + getMissingSources, isConfigTile, isRawSqlExternalTileConfig, isSeriesTile, + resolveSavedQueryLanguage, + updateDashboardBodySchema, } from './utils/dashboards'; -/** Returns an array of source IDs that are referenced in the tiles/filters but do not exist in the team's sources */ -async function getMissingSources( - team: string | mongoose.Types.ObjectId, - tiles: ExternalDashboardTileWithId[], - filters?: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[], -): Promise { - const sourceIds = new Set(); - - for (const tile of tiles) { - if (isSeriesTile(tile)) { - for (const series of tile.series) { - if ('sourceId' in series) { - sourceIds.add(series.sourceId); - } - } - } else if (isConfigTile(tile)) { - if ('sourceId' in tile.config && tile.config.sourceId) { - sourceIds.add(tile.config.sourceId); - } - } - } - - if (filters?.length) { - for (const filter of filters) { - if ('sourceId' in filter) { - sourceIds.add(filter.sourceId); - } - } - } - - const existingSources = await getSources(team.toString()); - const existingSourceIds = new Set( - existingSources.map(source => source._id.toString()), - ); - return [...sourceIds].filter(sourceId => !existingSourceIds.has(sourceId)); -} - -/** Returns an array of connection IDs that are referenced in the tiles but do not belong to the team */ -async function getMissingConnections( - team: string | mongoose.Types.ObjectId, - tiles: ExternalDashboardTileWithId[], -): Promise { - const connectionIds = new Set(); - - for (const tile of tiles) { - if (isConfigTile(tile) && isRawSqlExternalTileConfig(tile.config)) { - connectionIds.add(tile.config.connectionId); - } - } - - if (connectionIds.size === 0) return []; - - const existingConnections = await getConnectionsByTeam(team.toString()); - const existingConnectionIds = new Set( - existingConnections.map(connection => connection._id.toString()), - ); - - return [...connectionIds].filter( - connectionId => !existingConnectionIds.has(connectionId), - ); -} - async function getSourceConnectionMismatches( team: string | mongoose.Types.ObjectId, tiles: ExternalDashboardTileWithId[], @@ -123,62 +49,6 @@ async function getSourceConnectionMismatches( return sourcesWithInvalidConnections; } -type SavedQueryLanguage = z.infer; - -function resolveSavedQueryLanguage(params: { - savedQuery: string | null | undefined; - savedQueryLanguage: SavedQueryLanguage | null | undefined; -}): SavedQueryLanguage | null | undefined { - const { savedQuery, savedQueryLanguage } = params; - if (savedQueryLanguage !== undefined) return savedQueryLanguage; - if (savedQuery === null) return null; - if (savedQuery) return 'lucene'; - - return undefined; -} - -const dashboardBodyBaseShape = { - name: z.string().max(1024), - tiles: externalDashboardTileListSchema, - tags: tagsSchema, - savedQuery: z.string().nullable().optional(), - savedQueryLanguage: whereLanguageSchema.nullable().optional(), - savedFilterValues: z - .array(externalDashboardSavedFilterValueSchema) - .optional(), -}; - -function buildDashboardBodySchema(filterSchema: z.ZodTypeAny): z.ZodEffects< - z.ZodObject< - typeof dashboardBodyBaseShape & { - filters: z.ZodOptional>; - } - > -> { - return z - .object({ - ...dashboardBodyBaseShape, - filters: z.array(filterSchema).optional(), - }) - .superRefine((data, ctx) => { - if (data.savedQuery != null && data.savedQueryLanguage === null) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: - 'savedQueryLanguage cannot be null when savedQuery is provided', - path: ['savedQueryLanguage'], - }); - } - }); -} - -const createDashboardBodySchema = buildDashboardBodySchema( - externalDashboardFilterSchema, -); -const updateDashboardBodySchema = buildDashboardBodySchema( - externalDashboardFilterSchemaWithId, -); - /** * @openapi * components: @@ -1748,27 +1618,8 @@ router.post( }); } - const internalTiles = tiles.map(tile => { - const tileId = new ObjectId().toString(); - if (isConfigTile(tile)) { - return convertToInternalTileConfig({ - ...tile, - id: tileId, - }); - } - - return translateExternalChartToTileConfig({ - ...tile, - id: tileId, - }); - }); - - const filtersWithIds = (filters || []).map(filter => - translateExternalFilterToFilter({ - ...filter, - id: new ObjectId().toString(), - }), - ); + const internalTiles = convertExternalTilesToInternal(tiles); + const filtersWithIds = convertExternalFiltersToInternal(filters || []); const normalizedSavedQueryLanguage = resolveSavedQueryLanguage({ savedQuery, @@ -2001,18 +1852,10 @@ router.put( (existingDashboard?.filters ?? []).map((f: { id: string }) => f.id), ); - // Convert external tiles to internal charts format. - // Generate a new id for any tile whose id doesn't match an existing tile. - const internalTiles = tiles.map(tile => { - const tileId = existingTileIds.has(tile.id) - ? tile.id - : new ObjectId().toString(); - if (isConfigTile(tile)) { - return convertToInternalTileConfig({ ...tile, id: tileId }); - } - - return translateExternalChartToTileConfig({ ...tile, id: tileId }); - }); + const internalTiles = convertExternalTilesToInternal( + tiles, + existingTileIds, + ); const setPayload: Record = { name, @@ -2020,13 +1863,9 @@ router.put( tags: tags && uniq(tags), }; if (filters !== undefined) { - setPayload.filters = filters.map( - (filter: ExternalDashboardFilterWithId) => { - const filterId = existingFilterIds.has(filter.id) - ? filter.id - : new ObjectId().toString(); - return translateExternalFilterToFilter({ ...filter, id: filterId }); - }, + setPayload.filters = convertExternalFiltersToInternal( + filters, + existingFilterIds, ); } if (savedQuery !== undefined) { @@ -2053,21 +1892,12 @@ router.put( return res.sendStatus(404); } - // Delete alerts for tiles that are now raw SQL (unsupported) or were removed - const newTileIdSet = new Set(internalTiles.map(t => t.id)); - const tileIdsToDeleteAlerts = [ - ...internalTiles - .filter(tile => isRawSqlSavedChartConfig(tile.config)) - .map(tile => tile.id), - ...[...existingTileIds].filter(id => !newTileIdSet.has(id)), - ]; - if (tileIdsToDeleteAlerts.length > 0) { - logger.info( - { dashboardId, teamId, tileIds: tileIdsToDeleteAlerts }, - `Deleting alerts for tiles with unsupported config or removed tiles`, - ); - await deleteDashboardAlerts(dashboardId, teamId, tileIdsToDeleteAlerts); - } + await cleanupDashboardAlerts({ + dashboardId, + teamId, + internalTiles, + existingTileIds, + }); res.json({ data: convertToExternalDashboard(updatedDashboard), diff --git a/packages/api/src/routers/external-api/v2/index.ts b/packages/api/src/routers/external-api/v2/index.ts index 06fd32e1..f2f1b098 100644 --- a/packages/api/src/routers/external-api/v2/index.ts +++ b/packages/api/src/routers/external-api/v2/index.ts @@ -6,17 +6,13 @@ import chartsRouter from '@/routers/external-api/v2/charts'; import dashboardRouter from '@/routers/external-api/v2/dashboards'; import sourcesRouter from '@/routers/external-api/v2/sources'; import webhooksRouter from '@/routers/external-api/v2/webhooks'; -import rateLimiter from '@/utils/rateLimiter'; +import rateLimiter, { rateLimiterKeyGenerator } from '@/utils/rateLimiter'; const router = express.Router(); -const rateLimiterKeyGenerator = (req: express.Request): string => { - return req.headers.authorization ?? req.ip ?? 'unknown'; -}; - const defaultRateLimiter = rateLimiter({ windowMs: 60 * 1000, // 1 minute - max: 100, // Limit each IP to 100 requests per `window` + max: 100, // Limit each API key to 100 requests per `window` standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers legacyHeaders: false, // Disable the `X-RateLimit-*` headers keyGenerator: rateLimiterKeyGenerator, diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index f7de772b..f51dda6f 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -1,3 +1,4 @@ +import { displayTypeSupportsRawSqlAlerts } from '@hyperdx/common-utils/dist/core/utils'; import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { AggregateFunctionSchema, @@ -6,19 +7,35 @@ import { RawSqlSavedChartConfig, SavedChartConfig, } from '@hyperdx/common-utils/dist/types'; +import { SearchConditionLanguageSchema as whereLanguageSchema } from '@hyperdx/common-utils/dist/types'; import { pick } from 'lodash'; import _ from 'lodash'; +import mongoose from 'mongoose'; +import { z } from 'zod'; +import { deleteDashboardAlerts } from '@/controllers/alerts'; +import { getConnectionsByTeam } from '@/controllers/connection'; +import { getSources } from '@/controllers/sources'; import { DashboardDocument } from '@/models/dashboard'; -import { translateFilterToExternalFilter } from '@/utils/externalApi'; +import { + translateExternalChartToTileConfig, + translateExternalFilterToFilter, + translateFilterToExternalFilter, +} from '@/utils/externalApi'; import logger from '@/utils/logger'; import { + ExternalDashboardFilter, + externalDashboardFilterSchema, + externalDashboardFilterSchemaWithId, ExternalDashboardFilterWithId, ExternalDashboardRawSqlTileConfig, + externalDashboardSavedFilterValueSchema, ExternalDashboardSelectItem, ExternalDashboardTileConfig, + externalDashboardTileListSchema, ExternalDashboardTileWithId, externalQuantileLevelSchema, + tagsSchema, } from '@/utils/zod'; // -------------------------------------------------------------------------------- @@ -475,3 +492,220 @@ export function convertToInternalTileConfig( config: strippedConfig, }; } + +// -------------------------------------------------------------------------------- +// Shared dashboard validation helpers (used by both the REST router and MCP tools) +// -------------------------------------------------------------------------------- + +/** Returns source IDs referenced in tiles/filters that do not exist for the team */ +export async function getMissingSources( + team: string | mongoose.Types.ObjectId, + tiles: ExternalDashboardTileWithId[], + filters?: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[], +): Promise { + const sourceIds = new Set(); + + for (const tile of tiles) { + if (isSeriesTile(tile)) { + for (const series of tile.series) { + if ('sourceId' in series) { + sourceIds.add(series.sourceId); + } + } + } else if (isConfigTile(tile)) { + if ('sourceId' in tile.config && tile.config.sourceId) { + sourceIds.add(tile.config.sourceId); + } + } + } + + if (filters?.length) { + for (const filter of filters) { + if ('sourceId' in filter) { + sourceIds.add(filter.sourceId); + } + } + } + + const existingSources = await getSources(team.toString()); + const existingSourceIds = new Set( + existingSources.map(source => source._id.toString()), + ); + return [...sourceIds].filter(sourceId => !existingSourceIds.has(sourceId)); +} + +/** Returns connection IDs referenced in tiles that do not belong to the team */ +export async function getMissingConnections( + team: string | mongoose.Types.ObjectId, + tiles: ExternalDashboardTileWithId[], +): Promise { + const connectionIds = new Set(); + + for (const tile of tiles) { + if (isConfigTile(tile) && isRawSqlExternalTileConfig(tile.config)) { + connectionIds.add(tile.config.connectionId); + } + } + + if (connectionIds.size === 0) return []; + + const existingConnections = await getConnectionsByTeam(team.toString()); + const existingConnectionIds = new Set( + existingConnections.map(connection => connection._id.toString()), + ); + + return [...connectionIds].filter( + connectionId => !existingConnectionIds.has(connectionId), + ); +} + +type SavedQueryLanguage = z.infer; + +export function resolveSavedQueryLanguage(params: { + savedQuery: string | null | undefined; + savedQueryLanguage: SavedQueryLanguage | null | undefined; +}): SavedQueryLanguage | null | undefined { + const { savedQuery, savedQueryLanguage } = params; + if (savedQueryLanguage !== undefined) return savedQueryLanguage; + if (savedQuery === null) return null; + if (savedQuery) return 'lucene'; + + return undefined; +} + +const dashboardBodyBaseShape = { + name: z.string().max(1024), + tiles: externalDashboardTileListSchema, + tags: tagsSchema, + savedQuery: z.string().nullable().optional(), + savedQueryLanguage: whereLanguageSchema.nullable().optional(), + savedFilterValues: z + .array(externalDashboardSavedFilterValueSchema) + .optional(), +}; + +// -------------------------------------------------------------------------------- +// Shared tile/filter conversion helpers (used by both external API and MCP) +// -------------------------------------------------------------------------------- + +/** + * Convert external tile definitions to internal Mongoose-compatible format. + * Generates new ObjectIds for tiles that don't already have a matching ID in + * `existingTileIds` (update path) or for all tiles (create path). + */ +export function convertExternalTilesToInternal( + tiles: ExternalDashboardTileWithId[], + existingTileIds?: Set, +): DashboardDocument['tiles'] { + return tiles.map(tile => { + const tileId = + existingTileIds && tile.id && existingTileIds.has(tile.id) + ? tile.id + : new mongoose.Types.ObjectId().toString(); + const tileWithId = { ...tile, id: tileId }; + if (isConfigTile(tileWithId)) { + return convertToInternalTileConfig(tileWithId); + } + if (isSeriesTile(tileWithId)) { + return translateExternalChartToTileConfig(tileWithId); + } + // Fallback for tiles with neither config nor series — treat as empty series tile. + // This shouldn't happen with valid input, but matches the previous behavior. + return translateExternalChartToTileConfig(tileWithId as SeriesTile); + }); +} + +/** + * Convert external filter definitions to internal format, preserving IDs that + * match `existingFilterIds` (update path) or generating new ones (create path). + */ +export function convertExternalFiltersToInternal( + filters: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[], + existingFilterIds?: Set, +) { + return filters.map(filter => { + const filterId = + existingFilterIds && 'id' in filter && existingFilterIds.has(filter.id) + ? filter.id + : new mongoose.Types.ObjectId().toString(); + return translateExternalFilterToFilter({ ...filter, id: filterId }); + }); +} + +/** + * Delete alerts for tiles that were removed or converted to raw SQL + * (which doesn't support alerts). + */ +export async function cleanupDashboardAlerts({ + dashboardId, + teamId, + internalTiles, + existingTileIds, +}: { + dashboardId: string; + teamId: string | mongoose.Types.ObjectId; + internalTiles: DashboardDocument['tiles']; + existingTileIds: Set; +}) { + const newTileIdSet = new Set(internalTiles.map(t => t.id)); + const tileIdsToDeleteAlerts = [ + ...internalTiles + .filter( + tile => + isRawSqlSavedChartConfig(tile.config) && + !displayTypeSupportsRawSqlAlerts(tile.config.displayType), + ) + .map(tile => tile.id), + ...[...existingTileIds].filter(id => !newTileIdSet.has(id)), + ]; + if (tileIdsToDeleteAlerts.length > 0) { + logger.info( + { dashboardId, teamId, tileIds: tileIdsToDeleteAlerts }, + 'Deleting alerts for tiles with unsupported config or removed tiles', + ); + const teamObjectId = + teamId instanceof mongoose.Types.ObjectId + ? teamId + : new mongoose.Types.ObjectId(teamId); + await deleteDashboardAlerts( + dashboardId, + teamObjectId, + tileIdsToDeleteAlerts, + ); + } +} + +// -------------------------------------------------------------------------------- +// Body validation schemas +// -------------------------------------------------------------------------------- + +function buildDashboardBodySchema(filterSchema: z.ZodTypeAny): z.ZodEffects< + z.ZodObject< + typeof dashboardBodyBaseShape & { + filters: z.ZodOptional>; + } + > +> { + return z + .object({ + ...dashboardBodyBaseShape, + filters: z.array(filterSchema).optional(), + }) + .superRefine((data, ctx) => { + if (data.savedQuery != null && data.savedQueryLanguage === null) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + 'savedQueryLanguage cannot be null when savedQuery is provided', + path: ['savedQueryLanguage'], + }); + } + }); +} + +export const createDashboardBodySchema = buildDashboardBodySchema( + externalDashboardFilterSchema, +); +export const updateDashboardBodySchema = buildDashboardBodySchema( + externalDashboardFilterSchemaWithId, +); diff --git a/packages/api/src/tasks/__tests__/util.test.ts b/packages/api/src/tasks/__tests__/util.test.ts index fb44f089..bfb6d2a2 100644 --- a/packages/api/src/tasks/__tests__/util.test.ts +++ b/packages/api/src/tasks/__tests__/util.test.ts @@ -109,15 +109,15 @@ describe('util', () => { }); it('should handle keys with empty segments', () => { - expect(() => unflattenObject({ 'foo..bar': 'baz' })).toThrowError(); + expect(() => unflattenObject({ 'foo..bar': 'baz' })).toThrow(); }); it('should handle keys starting with separator', () => { - expect(() => unflattenObject({ '.foo.bar': 'baz' })).toThrowError(); + expect(() => unflattenObject({ '.foo.bar': 'baz' })).toThrow(); }); it('should handle keys ending with separator', () => { - expect(() => unflattenObject({ 'foo.bar.': 'baz' })).toThrowError(); + expect(() => unflattenObject({ 'foo.bar.': 'baz' })).toThrow(); }); it('should handle complex custom separator', () => { diff --git a/packages/api/src/tasks/checkAlerts/__tests__/__snapshots__/renderAlertTemplate.test.ts.snap b/packages/api/src/tasks/checkAlerts/__tests__/__snapshots__/renderAlertTemplate.test.ts.snap new file mode 100644 index 00000000..92b5b56d --- /dev/null +++ b/packages/api/src/tasks/checkAlerts/__tests__/__snapshots__/renderAlertTemplate.test.ts.snap @@ -0,0 +1,362 @@ +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state above threshold=5 alertValue=10 1`] = `"🚨 Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = `"🚨 Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state below threshold=5 alertValue=2 1`] = `"🚨 Alert for "My Search" - 2 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = `"🚨 Alert for "My Search" - 3 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state between threshold=5 alertValue=6 1`] = `"🚨 Alert for "My Search" - 6 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state equal threshold=5 alertValue=5 1`] = `"🚨 Alert for "My Search" - 5 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state not_between threshold=5 alertValue=12 1`] = `"🚨 Alert for "My Search" - 12 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state not_equal threshold=5 alertValue=10 1`] = `"🚨 Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) above threshold=5 okValue=3 1`] = `"✅ Alert for "My Search" - 3 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = `"✅ Alert for "My Search" - 3 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) below threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) between threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) equal threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) not_between threshold=5 okValue=6 1`] = `"✅ Alert for "My Search" - 6 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = `"✅ Alert for "My Search" - 5 lines found"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state above threshold=5 alertValue=10 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10 meets or exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10 exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state below threshold=5 alertValue=2 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 2 falls below 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 3 falls to or below 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state between threshold=5 alertValue=6 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 6 falls between 5 and 7"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state decimal threshold 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10.1 meets or exceeds 1.5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state equal threshold=5 alertValue=5 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 5 equals 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state integer threshold rounds value 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 11 meets or exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state not_between threshold=5 alertValue=12 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 12 falls outside 5 and 7"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state not_equal threshold=5 alertValue=10 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10 does not equal 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) above threshold=5 okValue=3 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 3 falls below 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 3 falls to or below 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) below threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 meets or exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) between threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 falls outside 5 and 7"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) equal threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 does not equal 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) not_between threshold=5 okValue=6 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 6 falls between 5 and 7"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 5 equals 5"`; + +exports[`renderAlertTemplate saved search alerts ALERT state above threshold=5 alertValue=10 1`] = ` +" +10 lines found, which meets or exceeds the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = ` +" +10 lines found, which exceeds the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state below threshold=5 alertValue=2 1`] = ` +" +2 lines found, which falls below the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = ` +" +3 lines found, which falls to or below the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state between threshold=5 alertValue=6 1`] = ` +" +6 lines found, which falls between the threshold of 5 and 7 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state equal threshold=5 alertValue=5 1`] = ` +" +5 lines found, which equals the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state not_between threshold=5 alertValue=12 1`] = ` +" +12 lines found, which falls outside the threshold of 5 and 7 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state not_equal threshold=5 alertValue=10 1`] = ` +" +10 lines found, which does not equal the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state with group 1`] = ` +"Group: "http" +10 lines found, which meets or exceeds the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) above threshold=5 okValue=3 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) below threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) between threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) equal threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) not_between threshold=5 okValue=6 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) with group 1`] = ` +"Group: "http" - The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state above threshold=5 alertValue=10 1`] = ` +" +10 meets or exceeds 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = ` +" +10 exceeds 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state below threshold=5 alertValue=2 1`] = ` +" +2 falls below 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = ` +" +3 falls to or below 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state between threshold=5 alertValue=6 1`] = ` +" +6 falls between 5 and 7 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state decimal threshold 1`] = ` +" +10.1 meets or exceeds 1.5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state equal threshold=5 alertValue=5 1`] = ` +" +5 equals 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state integer threshold rounds value 1`] = ` +" +11 meets or exceeds 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state not_between threshold=5 alertValue=12 1`] = ` +" +12 falls outside 5 and 7 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state not_equal threshold=5 alertValue=10 1`] = ` +" +10 does not equal 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state with group 1`] = ` +"Group: "us-east-1" +10 meets or exceeds 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) above threshold=5 okValue=3 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) below threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) between threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) equal threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) not_between threshold=5 okValue=6 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) with group 1`] = ` +"Group: "us-east-1" - The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts index a8c9657f..20aa33e7 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts @@ -1,6 +1,7 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { AlertState, + AlertThresholdType, SourceKind, Tile, WebhookService, @@ -20,8 +21,10 @@ import { getServer, getTestFixtureClickHouseClient, makeTile, + RAW_SQL_ALERT_TEMPLATE, + RAW_SQL_NUMBER_ALERT_TEMPLATE, } from '@/fixtures'; -import Alert, { AlertSource, AlertThresholdType } from '@/models/alert'; +import Alert, { AlertSource } from '@/models/alert'; import AlertHistory from '@/models/alertHistory'; import Connection, { IConnection } from '@/models/connection'; import Dashboard, { IDashboard } from '@/models/dashboard'; @@ -65,63 +68,901 @@ beforeAll(async () => { describe('checkAlerts', () => { describe('doesExceedThreshold', () => { it('should return true when value exceeds ABOVE threshold', () => { - expect(doesExceedThreshold(AlertThresholdType.ABOVE, 10, 11)).toBe(true); - expect(doesExceedThreshold(AlertThresholdType.ABOVE, 10, 10)).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: 10 }, + 11, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: 10 }, + 10, + ), + ).toBe(true); }); it('should return true when value is below BELOW threshold', () => { - expect(doesExceedThreshold(AlertThresholdType.BELOW, 10, 9)).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: 10 }, + 9, + ), + ).toBe(true); }); it('should return false when value equals BELOW threshold', () => { - expect(doesExceedThreshold(AlertThresholdType.BELOW, 10, 10)).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: 10 }, + 10, + ), + ).toBe(false); }); it('should return false when value is below ABOVE threshold', () => { - expect(doesExceedThreshold(AlertThresholdType.ABOVE, 10, 9)).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: 10 }, + 9, + ), + ).toBe(false); }); it('should return false when value is above BELOW threshold', () => { - expect(doesExceedThreshold(AlertThresholdType.BELOW, 10, 11)).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: 10 }, + 11, + ), + ).toBe(false); }); it('should handle zero values correctly', () => { - expect(doesExceedThreshold(AlertThresholdType.ABOVE, 0, 1)).toBe(true); - expect(doesExceedThreshold(AlertThresholdType.ABOVE, 0, 0)).toBe(true); - expect(doesExceedThreshold(AlertThresholdType.ABOVE, 0, -1)).toBe(false); - expect(doesExceedThreshold(AlertThresholdType.BELOW, 0, -1)).toBe(true); - expect(doesExceedThreshold(AlertThresholdType.BELOW, 0, 0)).toBe(false); - expect(doesExceedThreshold(AlertThresholdType.BELOW, 0, 1)).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: 0 }, + 1, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: 0 }, + 0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: 0 }, + -1, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: 0 }, + -1, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: 0 }, + 0, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: 0 }, + 1, + ), + ).toBe(false); }); it('should handle negative values correctly', () => { - expect(doesExceedThreshold(AlertThresholdType.ABOVE, -5, -3)).toBe(true); - expect(doesExceedThreshold(AlertThresholdType.ABOVE, -5, -5)).toBe(true); - expect(doesExceedThreshold(AlertThresholdType.ABOVE, -5, -7)).toBe(false); - expect(doesExceedThreshold(AlertThresholdType.BELOW, -5, -7)).toBe(true); - expect(doesExceedThreshold(AlertThresholdType.BELOW, -5, -5)).toBe(false); - expect(doesExceedThreshold(AlertThresholdType.BELOW, -5, -3)).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: -5 }, + -3, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: -5 }, + -5, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: -5 }, + -7, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: -5 }, + -7, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: -5 }, + -5, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: -5 }, + -3, + ), + ).toBe(false); }); it('should handle decimal values correctly', () => { - expect(doesExceedThreshold(AlertThresholdType.ABOVE, 10.5, 11.0)).toBe( - true, - ); - expect(doesExceedThreshold(AlertThresholdType.ABOVE, 10.5, 10.5)).toBe( - true, - ); - expect(doesExceedThreshold(AlertThresholdType.ABOVE, 10.5, 10.0)).toBe( - false, - ); - expect(doesExceedThreshold(AlertThresholdType.BELOW, 10.5, 10.0)).toBe( - true, - ); - expect(doesExceedThreshold(AlertThresholdType.BELOW, 10.5, 10.5)).toBe( - false, - ); - expect(doesExceedThreshold(AlertThresholdType.BELOW, 10.5, 11.0)).toBe( - false, - ); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: 10.5 }, + 11.0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: 10.5 }, + 10.5, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE, threshold: 10.5 }, + 10.0, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: 10.5 }, + 10.0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: 10.5 }, + 10.5, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW, threshold: 10.5 }, + 11.0, + ), + ).toBe(false); + }); + + // ABOVE_EXCLUSIVE (>) tests + it('should return true when value is strictly above ABOVE_EXCLUSIVE threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, threshold: 10 }, + 11, + ), + ).toBe(true); + }); + + it('should return false when value equals ABOVE_EXCLUSIVE threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, threshold: 10 }, + 10, + ), + ).toBe(false); + }); + + it('should return false when value is below ABOVE_EXCLUSIVE threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, threshold: 10 }, + 9, + ), + ).toBe(false); + }); + + it('should handle zero values correctly for ABOVE_EXCLUSIVE', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, threshold: 0 }, + 1, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, threshold: 0 }, + 0, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, threshold: 0 }, + -1, + ), + ).toBe(false); + }); + + it('should handle negative values correctly for ABOVE_EXCLUSIVE', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, threshold: -5 }, + -3, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, threshold: -5 }, + -5, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, threshold: -5 }, + -7, + ), + ).toBe(false); + }); + + it('should handle decimal values correctly for ABOVE_EXCLUSIVE', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, + threshold: 10.5, + }, + 11.0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, + threshold: 10.5, + }, + 10.5, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, + threshold: 10.5, + }, + 10.0, + ), + ).toBe(false); + }); + + // BELOW_OR_EQUAL (<=) tests + it('should return true when value is below BELOW_OR_EQUAL threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW_OR_EQUAL, threshold: 10 }, + 9, + ), + ).toBe(true); + }); + + it('should return true when value equals BELOW_OR_EQUAL threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW_OR_EQUAL, threshold: 10 }, + 10, + ), + ).toBe(true); + }); + + it('should return false when value is above BELOW_OR_EQUAL threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW_OR_EQUAL, threshold: 10 }, + 11, + ), + ).toBe(false); + }); + + it('should handle zero values correctly for BELOW_OR_EQUAL', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW_OR_EQUAL, threshold: 0 }, + -1, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW_OR_EQUAL, threshold: 0 }, + 0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW_OR_EQUAL, threshold: 0 }, + 1, + ), + ).toBe(false); + }); + + it('should handle negative values correctly for BELOW_OR_EQUAL', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW_OR_EQUAL, threshold: -5 }, + -7, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW_OR_EQUAL, threshold: -5 }, + -5, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.BELOW_OR_EQUAL, threshold: -5 }, + -3, + ), + ).toBe(false); + }); + + it('should handle decimal values correctly for BELOW_OR_EQUAL', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BELOW_OR_EQUAL, + threshold: 10.5, + }, + 10.0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BELOW_OR_EQUAL, + threshold: 10.5, + }, + 10.5, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BELOW_OR_EQUAL, + threshold: 10.5, + }, + 11.0, + ), + ).toBe(false); + }); + + // EQUAL (=) tests + it('should return true when value equals EQUAL threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: 10 }, + 10, + ), + ).toBe(true); + }); + + it('should return false when value is above EQUAL threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: 10 }, + 11, + ), + ).toBe(false); + }); + + it('should return false when value is below EQUAL threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: 10 }, + 9, + ), + ).toBe(false); + }); + + it('should handle zero values correctly for EQUAL', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: 0 }, + 0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: 0 }, + 1, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: 0 }, + -1, + ), + ).toBe(false); + }); + + it('should handle negative values correctly for EQUAL', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: -5 }, + -5, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: -5 }, + -3, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: -5 }, + -7, + ), + ).toBe(false); + }); + + it('should handle decimal values correctly for EQUAL', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: 10.5 }, + 10.5, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: 10.5 }, + 10.0, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.EQUAL, threshold: 10.5 }, + 11.0, + ), + ).toBe(false); + }); + + // NOT_EQUAL (≠) tests + it('should return true when value does not equal NOT_EQUAL threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: 10 }, + 11, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: 10 }, + 9, + ), + ).toBe(true); + }); + + it('should return false when value equals NOT_EQUAL threshold', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: 10 }, + 10, + ), + ).toBe(false); + }); + + it('should handle zero values correctly for NOT_EQUAL', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: 0 }, + 1, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: 0 }, + -1, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: 0 }, + 0, + ), + ).toBe(false); + }); + + it('should handle negative values correctly for NOT_EQUAL', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: -5 }, + -3, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: -5 }, + -7, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: -5 }, + -5, + ), + ).toBe(false); + }); + + it('should handle decimal values correctly for NOT_EQUAL', () => { + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: 10.5 }, + 11.0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: 10.5 }, + 10.0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_EQUAL, threshold: 10.5 }, + 10.5, + ), + ).toBe(false); + }); + + // BETWEEN tests + it('should return true when value is within BETWEEN range', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 7, + ), + ).toBe(true); + }); + + it('should return true when value equals BETWEEN lower bound (inclusive)', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 5, + ), + ).toBe(true); + }); + + it('should return true when value equals BETWEEN upper bound (inclusive)', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 10, + ), + ).toBe(true); + }); + + it('should return false when value is below BETWEEN range', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 4, + ), + ).toBe(false); + }); + + it('should return false when value is above BETWEEN range', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 11, + ), + ).toBe(false); + }); + + it('should handle zero values correctly for BETWEEN', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: -1, + thresholdMax: 1, + }, + 0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 0, + thresholdMax: 0, + }, + 0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 1, + thresholdMax: 5, + }, + 0, + ), + ).toBe(false); + }); + + it('should handle negative values correctly for BETWEEN', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: -10, + thresholdMax: -5, + }, + -7, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: -10, + thresholdMax: -5, + }, + -10, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: -10, + thresholdMax: -5, + }, + -5, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: -10, + thresholdMax: -5, + }, + -11, + ), + ).toBe(false); + }); + + it('should handle decimal values correctly for BETWEEN', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 10.0, + thresholdMax: 11.0, + }, + 10.5, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 10.0, + thresholdMax: 11.0, + }, + 9.9, + ), + ).toBe(false); + }); + + it('should return true when threshold equals thresholdMax equals value for BETWEEN', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 5, + thresholdMax: 5, + }, + 5, + ), + ).toBe(true); + }); + + it('should throw when thresholdMax is undefined for BETWEEN', () => { + expect(() => + doesExceedThreshold( + { thresholdType: AlertThresholdType.BETWEEN, threshold: 5 }, + 7, + ), + ).toThrow(/thresholdMax is required/); + }); + + // NOT_BETWEEN tests + it('should return true when value is below NOT_BETWEEN range', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 3, + ), + ).toBe(true); + }); + + it('should return true when value is above NOT_BETWEEN range', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 12, + ), + ).toBe(true); + }); + + it('should return false when value is within NOT_BETWEEN range', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 7, + ), + ).toBe(false); + }); + + it('should return false when value equals NOT_BETWEEN lower bound (inclusive)', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 5, + ), + ).toBe(false); + }); + + it('should return false when value equals NOT_BETWEEN upper bound (inclusive)', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: 5, + thresholdMax: 10, + }, + 10, + ), + ).toBe(false); + }); + + it('should handle zero values correctly for NOT_BETWEEN', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: -1, + thresholdMax: 1, + }, + 0, + ), + ).toBe(false); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: 1, + thresholdMax: 5, + }, + 0, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: -5, + thresholdMax: -1, + }, + 0, + ), + ).toBe(true); + }); + + it('should handle negative values correctly for NOT_BETWEEN', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: -10, + thresholdMax: -5, + }, + -11, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: -10, + thresholdMax: -5, + }, + -4, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: -10, + thresholdMax: -5, + }, + -7, + ), + ).toBe(false); + }); + + it('should handle decimal values correctly for NOT_BETWEEN', () => { + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: 10.0, + thresholdMax: 11.0, + }, + 9.9, + ), + ).toBe(true); + expect( + doesExceedThreshold( + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: 10.0, + thresholdMax: 11.0, + }, + 10.5, + ), + ).toBe(false); + }); + + it('should throw when thresholdMax is undefined for NOT_BETWEEN', () => { + expect(() => + doesExceedThreshold( + { thresholdType: AlertThresholdType.NOT_BETWEEN, threshold: 5 }, + 7, + ), + ).toThrow(/thresholdMax is required/); }); }); @@ -486,15 +1327,13 @@ describe('checkAlerts', () => { buildAlertMessageTemplateTitle({ view: defaultSearchView, }), - ).toMatchInlineSnapshot( - `"🚨 Alert for \\"My Search\\" - 10 lines found"`, - ); + ).toMatchInlineSnapshot(`"🚨 Alert for "My Search" - 10 lines found"`); expect( buildAlertMessageTemplateTitle({ view: defaultChartView, }), ).toMatchInlineSnapshot( - `"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 5 meets or exceeds 1"`, ); }); @@ -505,16 +1344,14 @@ describe('checkAlerts', () => { view: defaultSearchView, state: AlertState.ALERT, }), - ).toMatchInlineSnapshot( - `"🚨 Alert for \\"My Search\\" - 10 lines found"`, - ); + ).toMatchInlineSnapshot(`"🚨 Alert for "My Search" - 10 lines found"`); expect( buildAlertMessageTemplateTitle({ view: defaultChartView, state: AlertState.ALERT, }), ).toMatchInlineSnapshot( - `"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 5 meets or exceeds 1"`, ); // Test OK state (should have ✅ emoji) @@ -523,16 +1360,14 @@ describe('checkAlerts', () => { view: defaultSearchView, state: AlertState.OK, }), - ).toMatchInlineSnapshot( - `"✅ Alert for \\"My Search\\" - 10 lines found"`, - ); + ).toMatchInlineSnapshot(`"✅ Alert for "My Search" - 10 lines found"`); expect( buildAlertMessageTemplateTitle({ view: defaultChartView, state: AlertState.OK, }), ).toMatchInlineSnapshot( - `"✅ Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`, + `"✅ Alert for "Test Chart" in "My Dashboard" - 5 meets or exceeds 1"`, ); }); @@ -552,7 +1387,7 @@ describe('checkAlerts', () => { view: decimalChartView, }), ).toMatchInlineSnapshot( - `"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 1111.1 exceeds 1.5"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 1111.1 meets or exceeds 1.5"`, ); // Test with multiple decimal places @@ -570,7 +1405,7 @@ describe('checkAlerts', () => { view: multiDecimalChartView, }), ).toMatchInlineSnapshot( - `"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 1.1235 exceeds 0.1234"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 1.1235 meets or exceeds 0.1234"`, ); // Test with integer value and decimal threshold @@ -588,7 +1423,7 @@ describe('checkAlerts', () => { view: integerValueView, }), ).toMatchInlineSnapshot( - `"🚨 Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 10.00 exceeds 0.12"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 10.00 meets or exceeds 0.12"`, ); }); @@ -629,7 +1464,7 @@ describe('checkAlerts', () => { expect( translateExternalActionsToInternal('@webhook-123'), ).toMatchInlineSnapshot( - `"{{__hdx_notify_channel__ channel=\\"webhook\\" id=\\"123\\"}}"`, + `"{{__hdx_notify_channel__ channel="webhook" id="123"}}"`, ); // with multiple breaks @@ -639,44 +1474,44 @@ describe('checkAlerts', () => { @webhook-123 `), ).toMatchInlineSnapshot(` -" -{{__hdx_notify_channel__ channel=\\"webhook\\" id=\\"123\\"}} -" -`); + " + {{__hdx_notify_channel__ channel="webhook" id="123"}} + " + `); // with body string expect( translateExternalActionsToInternal('blabla @action-id'), ).toMatchInlineSnapshot( - `"blabla {{__hdx_notify_channel__ channel=\\"action\\" id=\\"id\\"}}"`, + `"blabla {{__hdx_notify_channel__ channel="action" id="id"}}"`, ); // multiple actions expect( translateExternalActionsToInternal('blabla @action-id @action2-id2'), ).toMatchInlineSnapshot( - `"blabla {{__hdx_notify_channel__ channel=\\"action\\" id=\\"id\\"}} {{__hdx_notify_channel__ channel=\\"action2\\" id=\\"id2\\"}}"`, + `"blabla {{__hdx_notify_channel__ channel="action" id="id"}} {{__hdx_notify_channel__ channel="action2" id="id2"}}"`, ); // id with special characters expect( translateExternalActionsToInternal('send @email-mike@hyperdx.io'), ).toMatchInlineSnapshot( - `"send {{__hdx_notify_channel__ channel=\\"email\\" id=\\"mike@hyperdx.io\\"}}"`, + `"send {{__hdx_notify_channel__ channel="email" id="mike@hyperdx.io"}}"`, ); // id with multiple dashes expect( translateExternalActionsToInternal('@action-id-with-multiple-dashes'), ).toMatchInlineSnapshot( - `"{{__hdx_notify_channel__ channel=\\"action\\" id=\\"id-with-multiple-dashes\\"}}"`, + `"{{__hdx_notify_channel__ channel="action" id="id-with-multiple-dashes"}}"`, ); // custom template id expect( translateExternalActionsToInternal('@action-{{action_id}}'), ).toMatchInlineSnapshot( - `"{{__hdx_notify_channel__ channel=\\"action\\" id=\\"{{action_id}}\\"}}"`, + `"{{__hdx_notify_channel__ channel="action" id="{{action_id}}"}}"`, ); }); @@ -756,7 +1591,7 @@ describe('checkAlerts', () => { text: [ '**', 'Group: "http"', - '10 lines found, expected less than 1 lines', + '10 lines found, which meets or exceeds the threshold of 1 lines', 'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)', 'Custom body ', '```', @@ -816,7 +1651,7 @@ describe('checkAlerts', () => { text: [ '**', 'Group: "http"', - '10 lines found, expected less than 1 lines', + '10 lines found, which meets or exceeds the threshold of 1 lines', 'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)', 'Custom body ', '```', @@ -923,7 +1758,7 @@ describe('checkAlerts', () => { text: [ '**', 'Group: "http"', - '10 lines found, expected less than 1 lines', + '10 lines found, which meets or exceeds the threshold of 1 lines', 'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)', '', ' Runbook URL: https://example.com', @@ -952,7 +1787,7 @@ describe('checkAlerts', () => { text: [ '**', 'Group: "http"', - '10 lines found, expected less than 1 lines', + '10 lines found, which meets or exceeds the threshold of 1 lines', 'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)', '', ' Runbook URL: https://example.com', @@ -1142,7 +1977,7 @@ describe('checkAlerts', () => { const createAlertDetails = async ( team: ITeam, - source: ISource, + source: ISource | undefined, alertConfig: Parameters[1], additionalDetails: | { @@ -1154,7 +1989,7 @@ describe('checkAlerts', () => { tile: Tile; dashboard: IDashboard; }, - ) => { + ): Promise => { const mockUserId = new mongoose.Types.ObjectId(); const alert = await createAlert(team._id, alertConfig, mockUserId); @@ -1163,14 +1998,19 @@ describe('checkAlerts', () => { 'savedSearch', ]); - const details = { - alert: enhancedAlert, - source, - previousMap: new Map(), - ...additionalDetails, - } satisfies AlertDetails; - - return details; + return additionalDetails.taskType === AlertTaskType.SAVED_SEARCH + ? { + alert: enhancedAlert, + source: source!, + previousMap: new Map(), + ...additionalDetails, + } + : { + alert: enhancedAlert, + source, + previousMap: new Map(), + ...additionalDetails, + }; }; const processAlertAtTime = async ( @@ -1623,14 +2463,14 @@ describe('checkAlerts', () => { 1, 'https://hooks.slack.com/services/123', { - text: '🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1', + text: '🚨 Alert for "Logs Count" in "My Dashboard" - 3 meets or exceeds 1', blocks: [ { text: { text: [ - `**`, + `**`, '', - '3 exceeds 1', + '3 meets or exceeds 1', 'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)', '', ].join('\n'), @@ -1643,6 +2483,70 @@ describe('checkAlerts', () => { ); }); + it.each([AlertThresholdType.BETWEEN, AlertThresholdType.NOT_BETWEEN])( + 'should not fire or record history when thresholdMax is missing for %s', + async thresholdType => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const now = new Date('2023-11-16T22:12:00.000Z'); + const eventMs = new Date('2023-11-16T22:05:00.000Z'); + + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: eventMs, + SeverityText: 'error', + Body: 'Oh no! Something went wrong!', + }, + ]); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType, + threshold: 1, + // thresholdMax intentionally omitted to simulate an invalid alert + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + await processAlertAtTime( + now, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + + // Alert should remain in its default OK state and no history/webhooks should be emitted + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + expect( + await AlertHistory.countDocuments({ alert: details.alert.id }), + ).toBe(0); + expect(slack.postMessageToWebhook).not.toHaveBeenCalled(); + }, + ); + it('TILE alert (events) - generic webhook', async () => { const fetchMock = jest.fn().mockResolvedValue({ ok: true, @@ -1815,7 +2719,7 @@ describe('checkAlerts', () => { expect(fetchMock).toHaveBeenCalledWith('https://webhook.site/123', { method: 'POST', body: JSON.stringify({ - text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1`, + text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 meets or exceeds 1`, }), headers: { 'Content-Type': 'application/json', @@ -1825,6 +2729,995 @@ describe('checkAlerts', () => { }); }); + it('TILE alert (raw SQL line chart) - should trigger and resolve', async () => { + const { team, webhook, connection, teamWebhooksById, clickhouseClient } = + await setupSavedSearchAlertTest(); + + const now = new Date('2023-11-16T22:12:00.000Z'); + // Send events in the last alert window 22:05 - 22:10 + const eventMs = now.getTime() - ms('5m'); + + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'Raw SQL alert test event 1', + }, + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'Raw SQL alert test event 2', + }, + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'Raw SQL alert test event 3', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Raw SQL Dashboard', + team: team._id, + tiles: [ + { + id: 'rawsql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'line', + sqlTemplate: RAW_SQL_ALERT_TEMPLATE, + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'rawsql1'); + if (!tile) throw new Error('tile not found for raw SQL test'); + + const details = await createAlertDetails( + team, + undefined, // No source for raw SQL tiles + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'rawsql1', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // should fetch 5m of logs and trigger alert + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Next window with no data should resolve + const nextWindow = new Date('2023-11-16T22:16:00.000Z'); + await processAlertAtTime( + nextWindow, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + + // Check alert history + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + + expect(alertHistories.length).toBe(2); + expect(alertHistories[0].state).toBe('ALERT'); + expect(alertHistories[0].counts).toBe(1); + expect(alertHistories[0].lastValues[0].count).toBeGreaterThanOrEqual(1); + expect(alertHistories[1].state).toBe('OK'); + }); + + it('TILE alert (raw SQL) - multiple rows per time bucket from GROUP BY', async () => { + const { team, webhook, connection, teamWebhooksById, clickhouseClient } = + await setupSavedSearchAlertTest(); + + const now = new Date('2023-11-16T22:12:00.000Z'); + const eventMs = now.getTime() - ms('5m'); + + // Insert logs from two different services in the same time bucket + await bulkInsertLogs([ + { + ServiceName: 'web', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'web error 1', + }, + { + ServiceName: 'web', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'web error 2', + }, + { + ServiceName: 'worker', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'worker error 1', + }, + ]); + + // SQL query that groups by ServiceName — produces multiple rows per time bucket. + // Raw SQL alerts don't have explicit groupBy, so the alert system treats + // each row independently against the threshold within a single history record. + const groupedSqlTemplate = ` + SELECT + toStartOfInterval(Timestamp, INTERVAL {intervalSeconds:Int64} second) AS ts, + ServiceName, + count() AS cnt + FROM default.otel_logs + WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) + AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) + GROUP BY ts, ServiceName + ORDER BY ts`; + + const dashboard = await new Dashboard({ + name: 'Raw SQL Grouped Dashboard', + team: team._id, + tiles: [ + { + id: 'rawsql-grouped', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'line', + sqlTemplate: groupedSqlTemplate, + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'rawsql-grouped'); + if (!tile) throw new Error('tile not found'); + + const details = await createAlertDetails( + team, + undefined, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + dashboardId: dashboard.id, + tileId: 'rawsql-grouped', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Raw SQL alerts with GROUP BY produce separate history records per group. + // web=2 (meets threshold 2), worker=1 (below threshold 2). + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }); + + expect(alertHistories.length).toBe(2); + + const webHistory = alertHistories.find(h => + h.group?.includes('ServiceName:web'), + ); + const workerHistory = alertHistories.find(h => + h.group?.includes('ServiceName:worker'), + ); + + expect(webHistory).toBeDefined(); + expect(webHistory!.state).toBe('ALERT'); + expect(webHistory!.lastValues.map(v => v.count)).toEqual([2]); + + expect(workerHistory).toBeDefined(); + expect(workerHistory!.state).toBe('OK'); + expect(workerHistory!.lastValues.map(v => v.count)).toEqual([1]); + }); + + it('TILE alert (raw SQL) - alert is evaluated using the last numeric column', async () => { + const { team, webhook, connection, teamWebhooksById, clickhouseClient } = + await setupSavedSearchAlertTest(); + + const now = new Date('2023-11-16T22:12:00.000Z'); + const eventMs = now.getTime() - ms('5m'); + + // Insert 1 error and 2 warns so the two numeric columns differ: + // error_count = 1, warn_count = 2 + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'error log', + }, + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'warn', + Body: 'warn log 1', + }, + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'warn', + Body: 'warn log 2', + }, + ]); + + // SQL query that returns multiple numeric columns (error_count, warn_count). + // The last numeric column (warn_count = 2) determines the alert. + const multiSeriesSqlTemplate = ` + SELECT + toStartOfInterval(Timestamp, INTERVAL {intervalSeconds:Int64} second) AS ts, + countIf(SeverityText = 'error') AS error_count, + countIf(SeverityText = 'warn') AS warn_count + FROM default.otel_logs + WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) + AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) + GROUP BY ts + ORDER BY ts`; + + const dashboard = await new Dashboard({ + name: 'Raw SQL Multi-Series Dashboard', + team: team._id, + tiles: [ + { + id: 'rawsql-multi', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'line', + sqlTemplate: multiSeriesSqlTemplate, + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'rawsql-multi'); + if (!tile) throw new Error('tile not found'); + + // Threshold of 2: error_count (1) does not meet it, warn_count (2) meets it. + // The alert should fire because the last numeric column (warn_count = 2) + // is the value used for threshold comparison. + const details = await createAlertDetails( + team, + undefined, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + dashboardId: dashboard.id, + tileId: 'rawsql-multi', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }); + + expect(alertHistories.length).toBe(1); + expect(alertHistories[0].state).toBe('ALERT'); + // The value is from the last numeric column (warn_count), not error_count + expect(alertHistories[0].lastValues[0].count).toBe(2); + }); + + it('TILE alert (raw SQL with macros) - $__sourceTable, $__timeFilter, and $__timeInterval', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const now = new Date('2023-11-16T22:12:00.000Z'); + const eventMs = now.getTime() - ms('5m'); + + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'macro test event 1', + }, + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'macro test event 2', + }, + ]); + + // SQL query using all three macros: + // $__sourceTable resolves to `default`.`otel_logs` from the source + // $__timeFilter(Timestamp) resolves to date range params + // $__timeInterval(Timestamp) resolves to interval bucket expression + const macroSqlTemplate = [ + 'SELECT', + ' $__timeInterval(Timestamp) AS ts,', + ' count() AS cnt', + ' FROM $__sourceTable', + ' WHERE $__timeFilter(Timestamp)', + ' GROUP BY ts', + ' ORDER BY ts', + ].join('\n'); + + const dashboard = await new Dashboard({ + name: 'Raw SQL Macro Dashboard', + team: team._id, + tiles: [ + { + id: 'rawsql-macros', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'line', + sqlTemplate: macroSqlTemplate, + connection: connection.id, + source: source.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'rawsql-macros'); + if (!tile) throw new Error('tile not found'); + + // Pass source so $__sourceTable macro can resolve + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'rawsql-macros', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }); + + expect(alertHistories.length).toBe(1); + expect(alertHistories[0].state).toBe('ALERT'); + expect(alertHistories[0].lastValues[0].count).toBe(2); + }); + + it('TILE alert (raw SQL) - catches up on multiple missed windows', async () => { + const { team, webhook, connection, teamWebhooksById, clickhouseClient } = + await setupSavedSearchAlertTest(); + + // Scenario: 5m alert interval. + // Run 1 at 22:02 — evaluates [21:55-22:00), finds 0 events → OK + // Run 2 at 22:17 — catches up missed windows, evaluates + // [22:00-22:05) — 0 events (OK) + // [22:05-22:10) — 2 events (ALERT, exceeds threshold of 1) + // [22:10-22:15) — 1 event (ALERT, meets threshold of 1) + + await bulkInsertLogs([ + // Events in the 22:05-22:10 bucket + { + ServiceName: 'api', + Timestamp: new Date('2023-11-16T22:06:00.000Z'), + SeverityText: 'error', + Body: 'missed window event 1', + }, + { + ServiceName: 'api', + Timestamp: new Date('2023-11-16T22:07:00.000Z'), + SeverityText: 'error', + Body: 'missed window event 2', + }, + // Event in the 22:10-22:15 bucket + { + ServiceName: 'api', + Timestamp: new Date('2023-11-16T22:11:00.000Z'), + SeverityText: 'error', + Body: 'missed window event 3', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Raw SQL Catchup Dashboard', + team: team._id, + tiles: [ + { + id: 'rawsql-catchup', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'line', + sqlTemplate: RAW_SQL_ALERT_TEMPLATE, + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'rawsql-catchup'); + if (!tile) throw new Error('tile not found'); + + const details = await createAlertDetails( + team, + undefined, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'rawsql-catchup', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Run 1 at 22:02 — evaluates [21:55-22:00), no events → OK history + await processAlertAtTime( + new Date('2023-11-16T22:02:00.000Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + const firstRunHistories = await AlertHistory.find({ + alert: details.alert.id, + }); + expect(firstRunHistories.length).toBe(1); + expect(firstRunHistories[0].state).toBe('OK'); + + // Run 2 at 22:17 — catches up from 22:00, evaluates + // [22:00-22:05), [22:05-22:10), [22:10-22:15) + await processAlertAtTime( + new Date('2023-11-16T22:17:00.000Z'), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + const catchupHistories = await AlertHistory.find({ + alert: details.alert.id, + createdAt: { $gt: new Date('2023-11-16T22:00:00.000Z') }, + }); + + expect(catchupHistories.length).toBe(1); + expect(catchupHistories[0].state).toBe('ALERT'); + + // lastValues should contain entries for each evaluated bucket + // Bucket 22:00-22:05 has 0 events, 22:05-22:10 has 2, 22:10-22:15 has 1 + const { lastValues } = catchupHistories[0]; + expect(lastValues.length).toBe(3); + + expect(lastValues.map(v => v.count)).toEqual([0, 2, 1]); + }); + + it('TILE alert (raw SQL Number chart) - should trigger and resolve', async () => { + const { team, webhook, connection, teamWebhooksById, clickhouseClient } = + await setupSavedSearchAlertTest(); + + const now = new Date('2023-11-16T22:12:00.000Z'); + const eventMs = now.getTime() - ms('5m'); + + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'Number chart alert test event 1', + }, + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'Number chart alert test event 2', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Number Chart Dashboard', + team: team._id, + tiles: [ + { + id: 'number1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'number', + sqlTemplate: RAW_SQL_NUMBER_ALERT_TEMPLATE, + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'number1'); + if (!tile) throw new Error('tile not found for Number chart test'); + + const details = await createAlertDetails( + team, + undefined, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'number1', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Should trigger alert (2 events > threshold of 1) + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Check alert history + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + + expect(alertHistories.length).toBe(1); + expect(alertHistories[0].state).toBe('ALERT'); + expect(alertHistories[0].counts).toBe(1); + expect(alertHistories[0].lastValues[0].count).toBe(2); + + // Next window with no new data in range should resolve + const nextWindow = new Date('2023-11-16T22:16:00.000Z'); + await processAlertAtTime( + nextWindow, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + + const allHistories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + expect(allHistories.length).toBe(2); + expect(allHistories[1].state).toBe('OK'); + }); + + it('TILE alert (raw SQL Number chart) - no data returns zero value', async () => { + const { team, webhook, connection, teamWebhooksById, clickhouseClient } = + await setupSavedSearchAlertTest(); + + const now = new Date('2023-11-16T22:12:00.000Z'); + + // No logs inserted — empty table for this time range + + const dashboard = await new Dashboard({ + name: 'Empty Number Chart Dashboard', + team: team._id, + tiles: [ + { + id: 'number-empty', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'number', + sqlTemplate: RAW_SQL_NUMBER_ALERT_TEMPLATE, + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'number-empty'); + if (!tile) throw new Error('tile not found'); + + const details = await createAlertDetails( + team, + undefined, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'number-empty', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + // count() returns 0 for no matching rows, which is below threshold of 1 + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }); + expect(alertHistories.length).toBe(1); + expect(alertHistories[0].state).toBe('OK'); + expect(alertHistories[0].lastValues[0].count).toBe(0); + }); + + it('TILE alert (raw SQL Number chart) - threshold compares with last numeric column', async () => { + const { team, webhook, connection, teamWebhooksById, clickhouseClient } = + await setupSavedSearchAlertTest(); + + const now = new Date('2023-11-16T22:12:00.000Z'); + const eventMs = now.getTime() - ms('5m'); + + // Insert 1 error and 2 warns so the two numeric columns differ: + // error_count = 1, warn_count = 2 + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'error log', + }, + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'warn', + Body: 'warn log 1', + }, + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'warn', + Body: 'warn log 2', + }, + ]); + + // SQL query that returns multiple numeric columns (error_count, warn_count). + // The last numeric column (warn_count = 2) should be used for threshold comparison. + const multiNumericSql = [ + 'SELECT', + " countIf(SeverityText = 'error') AS error_count,", + " countIf(SeverityText = 'warn') AS warn_count", + ' FROM default.otel_logs', + ' WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})', + ' AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})', + ].join(''); + + const dashboard = await new Dashboard({ + name: 'Multi-Numeric Number Chart Dashboard', + team: team._id, + tiles: [ + { + id: 'number-multi-numeric', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'number', + sqlTemplate: multiNumericSql, + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find( + (t: any) => t.id === 'number-multi-numeric', + ); + if (!tile) throw new Error('tile not found'); + + // Threshold of 2: error_count (1) does not meet it, warn_count (2) meets it. + // The alert should fire because the last numeric column (warn_count = 2) + // is the value used for threshold comparison. + const details = await createAlertDetails( + team, + undefined, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + dashboardId: dashboard.id, + tileId: 'number-multi-numeric', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }); + + expect(alertHistories.length).toBe(1); + expect(alertHistories[0].state).toBe('ALERT'); + // The value is from the last numeric column (warn_count = 2), not error_count (1) + expect(alertHistories[0].lastValues[0].count).toBe(2); + }); + + it('TILE alert (raw SQL Number chart) - only first row is compared to threshold when query returns multiple rows', async () => { + const { team, webhook, connection, teamWebhooksById, clickhouseClient } = + await setupSavedSearchAlertTest(); + + const now = new Date('2023-11-16T22:12:00.000Z'); + const eventMs = now.getTime() - ms('5m'); + + // Insert 3 events for 'web' and 1 event for 'api'. + // With ORDER BY cnt DESC, the first row will be web (cnt=3), + // the second row will be api (cnt=1). + await bulkInsertLogs([ + { + ServiceName: 'web', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'web event 1', + }, + { + ServiceName: 'web', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'web event 2', + }, + { + ServiceName: 'web', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'web event 3', + }, + { + ServiceName: 'api', + Timestamp: new Date(eventMs), + SeverityText: 'error', + Body: 'api event 1', + }, + ]); + + // SQL with GROUP BY that returns multiple rows. + // ORDER BY cnt DESC ensures the first row is web (cnt=3). + const groupBySql = [ + 'SELECT ServiceName, count() AS cnt', + ' FROM default.otel_logs', + ' WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})', + ' AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})', + ' GROUP BY ServiceName', + ' ORDER BY cnt DESC', + ].join(''); + + const dashboard = await new Dashboard({ + name: 'Number Chart First Row Dashboard', + team: team._id, + tiles: [ + { + id: 'number-first-row', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'number', + sqlTemplate: groupBySql, + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find( + (t: any) => t.id === 'number-first-row', + ); + if (!tile) throw new Error('tile not found'); + + // Threshold of 2: first row web (cnt=3) exceeds it, second row api (cnt=1) does not. + // Only the first row should be compared, so the alert should fire. + const details = await createAlertDetails( + team, + undefined, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + dashboardId: dashboard.id, + tileId: 'number-first-row', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }); + + // Number charts produce a single history (no per-group splitting) + expect(alertHistories.length).toBe(1); + expect(alertHistories[0].state).toBe('ALERT'); + // The value comes from the first row only (web cnt=3), not the second row (api cnt=1) + expect(alertHistories[0].lastValues[0].count).toBe(3); + }); + it('Group-by alerts that resolve (missing data case)', async () => { const { team, @@ -2621,14 +4514,14 @@ describe('checkAlerts', () => { 1, 'https://hooks.slack.com/services/123', { - text: '🚨 Alert for "CPU" in "My Dashboard" - 6 exceeds 1', + text: '🚨 Alert for "CPU" in "My Dashboard" - 6 meets or exceeds 1', blocks: [ { text: { text: [ - `**`, + `**`, '', - '6 exceeds 1', + '6 meets or exceeds 1', 'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)', '', ].join('\n'), @@ -2804,7 +4697,7 @@ describe('checkAlerts', () => { for (const msg of messages) { expect(msg.text).toContain('CPU by Service'); expect(msg.text).toContain('My Dashboard'); - expect(msg.text).toContain('exceeds 1'); + expect(msg.text).toContain('meets or exceeds 1'); } // Body should contain Group: "ServiceName:service-a" or "ServiceName:service-b" @@ -4616,6 +6509,896 @@ describe('checkAlerts', () => { expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); }); + + // --------------------------------------------------------------- + // Integration tests for threshold types + // Each test follows ALERT → Resolve flow with boundary-condition + // values in the resolve period. + // --------------------------------------------------------------- + + it('SAVED_SEARCH alert with ABOVE_EXCLUSIVE threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // threshold = 2, ABOVE_EXCLUSIVE means value > 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + await bulkInsertLogs([ + // Period 1: 3 error logs (should ALERT since 3 > 2) + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 3', + }, + // Period 2: exactly 2 error logs (should resolve since 2 is NOT > 2) + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + // Period 1: 3 logs, threshold is > 2, should ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, threshold is > 2, 2 is NOT > 2 so should resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('TILE alert with ABOVE_EXCLUSIVE threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + // Period 1: 3 logs (ALERT: 3 > 2) + // Period 2: exactly 2 logs (OK: 2 is NOT > 2, boundary value) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Test Dashboard', + team: team._id, + tiles: [ + { + id: 'tile-above-excl', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + name: 'Error Count', + select: [ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + valueExpression: '', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + displayType: 'line', + granularity: 'auto', + source: source.id, + groupBy: '', + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find( + (t: any) => t.id === 'tile-above-excl', + ); + if (!tile) throw new Error('tile not found'); + + // threshold = 2, ABOVE_EXCLUSIVE means > 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, + threshold: 2, + dashboardId: dashboard.id, + tileId: 'tile-above-excl', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Period 1: 3 logs, 3 > 2 → ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, 2 is NOT > 2 (boundary) → resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('SAVED_SEARCH alert with BELOW_OR_EQUAL threshold - should alert then resolve', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // threshold = 2, BELOW_OR_EQUAL means value <= 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.BELOW_OR_EQUAL, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + await bulkInsertLogs([ + // Period 1: exactly 2 error logs (should ALERT since 2 <= 2) + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + // Period 2: 3 error logs (should resolve since 3 is NOT <= 2) + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + // Period 1: 2 logs, threshold is <= 2, should ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 3 logs, threshold is <= 2, should resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('TILE alert with BELOW_OR_EQUAL threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + // Period 1: 1 log (ALERT: 1 <= 1, boundary) + // Period 2: 2 logs (OK: 2 is NOT <= 1, nearest non-matching value) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 3', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Test Dashboard', + team: team._id, + tiles: [ + { + id: 'tile-below-eq', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + name: 'Error Count', + select: [ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + valueExpression: '', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + displayType: 'line', + granularity: 'auto', + source: source.id, + groupBy: '', + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'tile-below-eq'); + if (!tile) throw new Error('tile not found'); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.BELOW_OR_EQUAL, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'tile-below-eq', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Period 1: 1 log, 1 <= 1 → ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, 2 is NOT <= 1 (near-boundary) → resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('SAVED_SEARCH alert with EQUAL threshold - should alert then resolve', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // threshold = 2, EQUAL means value == 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.EQUAL, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + await bulkInsertLogs([ + // Period 1: exactly 2 error logs (should ALERT since 2 == 2) + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + // Period 2: 3 error logs (should resolve since 3 != 2) + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + // Period 1: 2 logs, threshold is == 2, should ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 3 logs, threshold is == 2, 3 != 2 so should resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('TILE alert with EQUAL threshold - should alert then resolve at near-boundary', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + // Period 1: 3 logs (ALERT: 3 == 3) + // Period 2: 2 logs (OK: 2 != 3, nearest non-matching value) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Test Dashboard', + team: team._id, + tiles: [ + { + id: 'tile-equal', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + name: 'Error Count', + select: [ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + valueExpression: '', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + displayType: 'line', + granularity: 'auto', + source: source.id, + groupBy: '', + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'tile-equal'); + if (!tile) throw new Error('tile not found'); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.EQUAL, + threshold: 3, + dashboardId: dashboard.id, + tileId: 'tile-equal', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Period 1: 3 logs, 3 == 3 → ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, 2 != 3 (near-boundary) → resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('SAVED_SEARCH alert with NOT_EQUAL threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // threshold = 2, NOT_EQUAL means value != 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.NOT_EQUAL, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + await bulkInsertLogs([ + // Period 1: 3 error logs (should ALERT since 3 != 2) + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 3', + }, + // Period 2: exactly 2 error logs (should resolve since 2 == 2) + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + // Period 1: 3 logs, threshold is != 2, 3 != 2 so should ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, threshold is != 2, 2 == 2 so should resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('TILE alert with NOT_EQUAL threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + // Period 1: 2 logs (ALERT: 2 != 3) + // Period 2: exactly 3 logs (OK: 3 == 3, boundary value) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Test Dashboard', + team: team._id, + tiles: [ + { + id: 'tile-not-equal', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + name: 'Error Count', + select: [ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + valueExpression: '', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + displayType: 'line', + granularity: 'auto', + source: source.id, + groupBy: '', + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'tile-not-equal'); + if (!tile) throw new Error('tile not found'); + + // threshold = 3, NOT_EQUAL means != 3 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.NOT_EQUAL, + threshold: 3, + dashboardId: dashboard.id, + tileId: 'tile-not-equal', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Period 1: 2 logs, 2 != 3 → ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 3 logs, 3 == 3 (boundary) → resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); }); describe('processAlert with materialized views', () => { diff --git a/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.test.ts new file mode 100644 index 00000000..d6bd453f --- /dev/null +++ b/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.test.ts @@ -0,0 +1,451 @@ +import { + AlertState, + AlertThresholdType, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; +import mongoose from 'mongoose'; + +import { makeTile } from '@/fixtures'; +import { AlertSource } from '@/models/alert'; +import { loadProvider } from '@/tasks/checkAlerts/providers'; +import { + AlertMessageTemplateDefaultView, + buildAlertMessageTemplateTitle, + renderAlertTemplate, +} from '@/tasks/checkAlerts/template'; + +let alertProvider: any; + +beforeAll(async () => { + alertProvider = await loadProvider(); +}); + +// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion +const mockMetadata = { + getColumn: jest.fn().mockImplementation(({ column }) => { + const columnMap = { + Timestamp: { name: 'Timestamp', type: 'DateTime' }, + Body: { name: 'Body', type: 'String' }, + SeverityText: { name: 'SeverityText', type: 'String' }, + ServiceName: { name: 'ServiceName', type: 'String' }, + }; + return Promise.resolve(columnMap[column]); + }), + getColumns: jest.fn().mockResolvedValue([]), + getMapKeys: jest.fn().mockResolvedValue([]), + getMapValues: jest.fn().mockResolvedValue([]), + getAllFields: jest.fn().mockResolvedValue([]), + getTableMetadata: jest.fn().mockResolvedValue({}), + getClickHouseSettings: jest.fn().mockReturnValue({}), + setClickHouseSettings: jest.fn(), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), +} as any; + +const sampleLogsCsv = [ + '"2023-03-17 22:14:01","error","Failed to connect to database"', + '"2023-03-17 22:13:45","error","Connection timeout after 30s"', + '"2023-03-17 22:12:30","error","Retry limit exceeded"', +].join('\n'); + +// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion +const mockClickhouseClient = { + query: jest.fn().mockResolvedValue({ + json: jest.fn().mockResolvedValue({ data: [] }), + text: jest.fn().mockResolvedValue(sampleLogsCsv), + }), +} as any; + +const startTime = new Date('2023-03-17T22:10:00.000Z'); +const endTime = new Date('2023-03-17T22:15:00.000Z'); + +const makeSearchView = ( + overrides: Partial & { + thresholdType?: AlertThresholdType; + threshold?: number; + thresholdMax?: number; + value?: number; + group?: string; + } = {}, +): AlertMessageTemplateDefaultView => ({ + alert: { + thresholdType: overrides.thresholdType ?? AlertThresholdType.ABOVE, + threshold: overrides.threshold ?? 5, + thresholdMax: overrides.thresholdMax, + source: AlertSource.SAVED_SEARCH, + channel: { type: null }, + interval: '1m', + }, + source: { + id: 'fake-source-id', + kind: SourceKind.Log, + team: 'team-123', + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: 'connection-123', + name: 'Logs', + defaultTableSelectExpression: 'Timestamp, Body', + }, + savedSearch: { + _id: 'fake-saved-search-id' as any, + team: 'team-123' as any, + id: 'fake-saved-search-id', + name: 'My Search', + select: 'Body', + where: 'Body: "error"', + whereLanguage: 'lucene', + orderBy: 'timestamp', + source: 'fake-source-id' as any, + tags: ['test'], + createdAt: new Date(), + updatedAt: new Date(), + }, + attributes: {}, + granularity: '1m', + group: overrides.group, + isGroupedAlert: false, + startTime, + endTime, + value: overrides.value ?? 10, +}); + +const testTile = makeTile({ id: 'test-tile-id' }); +const makeTileView = ( + overrides: Partial & { + thresholdType?: AlertThresholdType; + threshold?: number; + thresholdMax?: number; + value?: number; + group?: string; + } = {}, +): AlertMessageTemplateDefaultView => ({ + alert: { + thresholdType: overrides.thresholdType ?? AlertThresholdType.ABOVE, + threshold: overrides.threshold ?? 5, + thresholdMax: overrides.thresholdMax, + source: AlertSource.TILE, + channel: { type: null }, + interval: '1m', + tileId: 'test-tile-id', + }, + dashboard: { + _id: new mongoose.Types.ObjectId(), + id: 'id-123', + name: 'My Dashboard', + tiles: [testTile], + team: 'team-123' as any, + tags: ['test'], + createdAt: new Date(), + updatedAt: new Date(), + }, + attributes: {}, + granularity: '5 minute', + group: overrides.group, + isGroupedAlert: false, + startTime, + endTime, + value: overrides.value ?? 10, +}); + +const render = (view: AlertMessageTemplateDefaultView, state: AlertState) => + renderAlertTemplate({ + alertProvider, + clickhouseClient: mockClickhouseClient, + metadata: mockMetadata, + state, + template: null, + title: 'Test Alert Title', + view, + teamWebhooksById: new Map(), + }); + +interface AlertCase { + thresholdType: AlertThresholdType; + threshold: number; + thresholdMax?: number; // for between-type thresholds + alertValue: number; // value that would trigger the alert + okValue: number; // value that would resolve the alert +} + +const alertCases: AlertCase[] = [ + { + thresholdType: AlertThresholdType.ABOVE, + threshold: 5, + alertValue: 10, + okValue: 3, + }, + { + thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, + threshold: 5, + alertValue: 10, + okValue: 3, + }, + { + thresholdType: AlertThresholdType.BELOW, + threshold: 5, + alertValue: 2, + okValue: 10, + }, + { + thresholdType: AlertThresholdType.BELOW_OR_EQUAL, + threshold: 5, + alertValue: 3, + okValue: 10, + }, + { + thresholdType: AlertThresholdType.EQUAL, + threshold: 5, + alertValue: 5, + okValue: 10, + }, + { + thresholdType: AlertThresholdType.NOT_EQUAL, + threshold: 5, + alertValue: 10, + okValue: 5, + }, + { + thresholdType: AlertThresholdType.BETWEEN, + threshold: 5, + thresholdMax: 7, + alertValue: 6, + okValue: 10, + }, + { + thresholdType: AlertThresholdType.NOT_BETWEEN, + threshold: 5, + thresholdMax: 7, + alertValue: 12, + okValue: 6, + }, +]; + +describe('renderAlertTemplate', () => { + describe('saved search alerts', () => { + describe('ALERT state', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold alertValue=$alertValue', + async ({ thresholdType, threshold, thresholdMax, alertValue }) => { + const result = await render( + makeSearchView({ + thresholdType, + threshold, + thresholdMax, + value: alertValue, + }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }, + ); + + it('with group', async () => { + const result = await render( + makeSearchView({ group: 'http' }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }); + }); + + describe('OK state (resolved)', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold okValue=$okValue', + async ({ thresholdType, threshold, thresholdMax, okValue }) => { + const result = await render( + makeSearchView({ + thresholdType, + threshold, + thresholdMax, + value: okValue, + }), + AlertState.OK, + ); + expect(result).toMatchSnapshot(); + }, + ); + + it('with group', async () => { + const result = await render( + makeSearchView({ group: 'http' }), + AlertState.OK, + ); + expect(result).toMatchSnapshot(); + }); + }); + }); + + describe('tile alerts', () => { + describe('ALERT state', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold alertValue=$alertValue', + async ({ thresholdType, threshold, thresholdMax, alertValue }) => { + const result = await render( + makeTileView({ + thresholdType, + threshold, + thresholdMax, + value: alertValue, + }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }, + ); + + it('with group', async () => { + const result = await render( + makeTileView({ group: 'us-east-1' }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }); + + it('decimal threshold', async () => { + const result = await render( + makeTileView({ + thresholdType: AlertThresholdType.ABOVE, + threshold: 1.5, + value: 10.123, + }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }); + + it('integer threshold rounds value', async () => { + const result = await render( + makeTileView({ + thresholdType: AlertThresholdType.ABOVE, + threshold: 5, + value: 10.789, + }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }); + }); + + describe('OK state (resolved)', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold okValue=$okValue', + async ({ thresholdType, threshold, thresholdMax, okValue }) => { + const result = await render( + makeTileView({ + thresholdType, + threshold, + thresholdMax, + value: okValue, + }), + AlertState.OK, + ); + expect(result).toMatchSnapshot(); + }, + ); + + it('with group', async () => { + const result = await render( + makeTileView({ group: 'us-east-1' }), + AlertState.OK, + ); + expect(result).toMatchSnapshot(); + }); + }); + }); +}); + +describe('buildAlertMessageTemplateTitle', () => { + describe('saved search alerts', () => { + describe('ALERT state', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold alertValue=$alertValue', + ({ thresholdType, threshold, alertValue }) => { + const result = buildAlertMessageTemplateTitle({ + view: makeSearchView({ + thresholdType, + threshold, + value: alertValue, + }), + state: AlertState.ALERT, + }); + expect(result).toMatchSnapshot(); + }, + ); + }); + + describe('OK state (resolved)', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold okValue=$okValue', + ({ thresholdType, threshold, okValue }) => { + const result = buildAlertMessageTemplateTitle({ + view: makeSearchView({ thresholdType, threshold, value: okValue }), + state: AlertState.OK, + }); + expect(result).toMatchSnapshot(); + }, + ); + }); + }); + + describe('tile alerts', () => { + describe('ALERT state', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold alertValue=$alertValue', + ({ thresholdType, threshold, thresholdMax, alertValue }) => { + const result = buildAlertMessageTemplateTitle({ + view: makeTileView({ + thresholdType, + threshold, + thresholdMax, + value: alertValue, + }), + state: AlertState.ALERT, + }); + expect(result).toMatchSnapshot(); + }, + ); + + it('decimal threshold', () => { + const result = buildAlertMessageTemplateTitle({ + view: makeTileView({ + thresholdType: AlertThresholdType.ABOVE, + threshold: 1.5, + value: 10.123, + }), + state: AlertState.ALERT, + }); + expect(result).toMatchSnapshot(); + }); + + it('integer threshold rounds value', () => { + const result = buildAlertMessageTemplateTitle({ + view: makeTileView({ + thresholdType: AlertThresholdType.ABOVE, + threshold: 5, + value: 10.789, + }), + state: AlertState.ALERT, + }); + expect(result).toMatchSnapshot(); + }); + }); + + describe('OK state (resolved)', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold okValue=$okValue', + ({ thresholdType, threshold, thresholdMax, okValue }) => { + const result = buildAlertMessageTemplateTitle({ + view: makeTileView({ + thresholdType, + threshold, + thresholdMax, + value: okValue, + }), + state: AlertState.OK, + }); + expect(result).toMatchSnapshot(); + }, + ); + }); + }); +}); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/singleInvocationAlert.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/singleInvocationAlert.test.ts index 7a6b4d89..02d5f2bc 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/singleInvocationAlert.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/singleInvocationAlert.test.ts @@ -6,7 +6,11 @@ import ms from 'ms'; import * as config from '@/config'; import { createAlert } from '@/controllers/alerts'; import { createTeam } from '@/controllers/team'; -import { bulkInsertLogs, getServer } from '@/fixtures'; +import { + bulkInsertLogs, + getServer, + RAW_SQL_NUMBER_ALERT_TEMPLATE, +} from '@/fixtures'; import Alert, { AlertSource, AlertThresholdType } from '@/models/alert'; import AlertHistory from '@/models/alertHistory'; import Connection from '@/models/connection'; @@ -244,7 +248,7 @@ describe('Single Invocation Alert Test', () => { // Verify the message body contains the search link const messageBody = webhookPayload.blocks[0].text.text; expect(messageBody).toContain('lines found'); - expect(messageBody).toContain('expected less than 1 lines'); + expect(messageBody).toContain('meets or exceeds the threshold of 1 lines'); expect(messageBody).toContain('http://app:8080/search/'); expect(messageBody).toContain('from='); expect(messageBody).toContain('to='); @@ -858,4 +862,137 @@ describe('Single Invocation Alert Test', () => { expect(dashboard.tiles[1].config.name).toBe('Second Tile Name'); expect(enhancedAlert.tileId).toBe('second-tile-id'); }); + + it('should trigger alert for raw SQL Number chart tile', async () => { + jest.spyOn(slack, 'postMessageToWebhook').mockResolvedValue(null as any); + + const team = await createTeam({ name: 'Test Team' }); + + const connection = await Connection.create({ + team: team._id, + name: 'Test Connection', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + const webhook = await new Webhook({ + team: team._id, + service: 'slack', + url: 'https://hooks.slack.com/services/test-number', + name: 'Test Webhook', + }).save(); + + const dashboard = await new Dashboard({ + name: 'Number Chart Alert Dashboard', + team: team._id, + tiles: [ + { + id: 'number-tile-1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'number', + sqlTemplate: RAW_SQL_NUMBER_ALERT_TEMPLATE, + connection: connection.id, + }, + }, + ], + }).save(); + + const mockUserId = new mongoose.Types.ObjectId(); + const alert = await createAlert( + team._id, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'number-tile-1', + name: 'Number Chart Alert', + }, + mockUserId, + ); + + const now = new Date('2023-11-16T22:12:00.000Z'); + const eventTime = new Date(now.getTime() - ms('3m')); + + // Insert logs that should be counted by the Number chart query + await bulkInsertLogs([ + { + ServiceName: 'web', + Timestamp: eventTime, + SeverityText: 'error', + Body: 'Number chart error 1', + }, + { + ServiceName: 'web', + Timestamp: eventTime, + SeverityText: 'error', + Body: 'Number chart error 2', + }, + { + ServiceName: 'web', + Timestamp: eventTime, + SeverityText: 'error', + Body: 'Number chart error 3', + }, + ]); + + const enhancedAlert: any = await Alert.findById(alert.id).populate([ + 'team', + 'savedSearch', + ]); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'number-tile-1'); + + const details: AlertDetails = { + alert: enhancedAlert, + source: undefined, + taskType: AlertTaskType.TILE, + tile: tile!, + dashboard, + previousMap: new Map(), + }; + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + + await processAlert( + now, + details, + clickhouseClient, + connection.id, + alertProvider, + new Map([[webhook.id.toString(), webhook]]), + ); + + // Verify alert state changed to ALERT + expect((await Alert.findById(enhancedAlert.id))!.state).toBe('ALERT'); + + // Verify alert history was created + const alertHistories = await AlertHistory.find({ + alert: alert.id, + }).sort({ createdAt: 1 }); + + expect(alertHistories.length).toBe(1); + expect(alertHistories[0].state).toBe('ALERT'); + expect(alertHistories[0].counts).toBe(1); + expect(alertHistories[0].lastValues.length).toBe(1); + expect(alertHistories[0].lastValues[0].count).toBe(3); + + // Verify webhook was called + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 3f0972de..90d06257 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -14,28 +14,36 @@ import { Metadata, } from '@hyperdx/common-utils/dist/core/metadata'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; -import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; +import { + aliasMapToWithClauses, + displayTypeSupportsRawSqlAlerts, + isTimeSeriesDisplayType, +} from '@hyperdx/common-utils/dist/core/utils'; import { timeBucketByGranularity } from '@hyperdx/common-utils/dist/core/utils'; import { + isBuilderChartConfig, isBuilderSavedChartConfig, + isRawSqlChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { + AlertThresholdType, BuilderChartConfigWithOptDateRange, + ChartConfigWithOptDateRange, DisplayType, getSampleWeightExpression, pickSampleWeightExpressionProps, SourceKind, } from '@hyperdx/common-utils/dist/types'; import * as fns from 'date-fns'; -import { isString } from 'lodash'; +import { isString, pick } from 'lodash'; import { ObjectId } from 'mongoose'; import mongoose from 'mongoose'; import ms from 'ms'; import { serializeError } from 'serialize-error'; import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; -import { AlertState, AlertThresholdType, IAlert } from '@/models/alert'; +import { AlertState, IAlert } from '@/models/alert'; import AlertHistory, { IAlertHistory } from '@/models/alertHistory'; import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; @@ -81,6 +89,17 @@ export const alertHasGroupBy = (details: AlertDetails): boolean => { ) { return true; } + + // Without a reliable parser, it's difficult to tell if the raw sql contains a + // group by (besides the group by on the interval), so we'll assume it might + // in the case of time series charts, and assume it will not in the case of number charts. + // Group name will just be blank if there are no group by values. + if ( + details.taskType === AlertTaskType.TILE && + isRawSqlSavedChartConfig(details.tile.config) + ) { + return details.tile.config.displayType !== DisplayType.Number; + } return false; }; @@ -119,17 +138,37 @@ export async function computeAliasWithClauses( } export const doesExceedThreshold = ( - thresholdType: AlertThresholdType, - threshold: number, + { + threshold, + thresholdType, + thresholdMax, + }: Pick, value: number, ) => { - const isThresholdTypeAbove = thresholdType === AlertThresholdType.ABOVE; - if (isThresholdTypeAbove && value >= threshold) { - return true; - } else if (!isThresholdTypeAbove && value < threshold) { - return true; + switch (thresholdType) { + case AlertThresholdType.ABOVE: + return value >= threshold; + case AlertThresholdType.BELOW: + return value < threshold; + case AlertThresholdType.ABOVE_EXCLUSIVE: + return value > threshold; + case AlertThresholdType.BELOW_OR_EQUAL: + return value <= threshold; + case AlertThresholdType.EQUAL: + return value === threshold; + case AlertThresholdType.NOT_EQUAL: + return value !== threshold; + case AlertThresholdType.BETWEEN: + case AlertThresholdType.NOT_BETWEEN: + if (thresholdMax == null) { + throw new Error( + `thresholdMax is required for threshold type "${thresholdType}"`, + ); + } + return thresholdType === AlertThresholdType.BETWEEN + ? value >= threshold && value <= thresholdMax + : value < threshold || value > thresholdMax; } - return false; }; const normalizeScheduleOffsetMinutes = ({ @@ -292,6 +331,7 @@ const fireChannelEvent = async ({ silenced: alert.silenced, source: alert.source, threshold: alert.threshold, + thresholdMax: alert.thresholdMax, thresholdType: alert.thresholdType, tileId: alert.tileId, }, @@ -370,9 +410,10 @@ const shouldSkipAlertCheck = ( // Skip if ANY previous history for this alert was created in the current window return Array.from(previousMap.entries()).some(([key, history]) => { // For grouped alerts, check any key that starts with alertId prefix - // For non-grouped alerts, check exact match with alertId + // or matches the bare alertId (empty group key case). + // For non-grouped alerts, check exact match with alertId. const isMatchingKey = hasGroupBy - ? key.startsWith(alertKeyPrefix) + ? key === alert.id || key.startsWith(alertKeyPrefix) : key === alert.id; return ( @@ -394,11 +435,11 @@ const getAlertEvaluationDateRange = ( // Find the latest createdAt among all histories for this alert let previousCreatedAt: Date | undefined; if (hasGroupBy) { - // For grouped alerts, find the latest createdAt among all groups - // Use the latest to avoid checking from old groups that might no longer exist + // For grouped alerts, find the latest createdAt among all groups. + // Also check the bare alertId key for the empty group key case. const alertKeyPrefix = getAlertKeyPrefix(alert.id); for (const [key, history] of previousMap.entries()) { - if (key.startsWith(alertKeyPrefix)) { + if (key === alert.id || key.startsWith(alertKeyPrefix)) { if (!previousCreatedAt || history.createdAt > previousCreatedAt) { previousCreatedAt = history.createdAt; } @@ -430,9 +471,10 @@ const getChartConfigFromAlert = ( connection: string, dateRange: [Date, Date], windowSizeInMins: number, -): BuilderChartConfigWithOptDateRange | undefined => { - const { alert, source } = details; +): ChartConfigWithOptDateRange | undefined => { + const { alert } = details; if (details.taskType === AlertTaskType.SAVED_SEARCH) { + const { source } = details; const savedSearch = details.savedSearch; return { connection, @@ -463,8 +505,43 @@ const getChartConfigFromAlert = ( } else if (details.taskType === AlertTaskType.TILE) { const tile = details.tile; - // Alerts are not supported for raw sql based charts - if (isRawSqlSavedChartConfig(tile.config)) return undefined; + // Raw SQL tiles: build a RawSqlChartConfig + if (isRawSqlSavedChartConfig(tile.config)) { + if (displayTypeSupportsRawSqlAlerts(tile.config.displayType)) { + return { + ...pick(tile.config, [ + 'configType', + 'sqlTemplate', + 'displayType', + 'source', + ]), + connection, + dateRange, + // Only time-series charts use interval bucketing + ...(isTimeSeriesDisplayType(tile.config.displayType) && { + granularity: `${windowSizeInMins} minute`, + }), + // Include source metadata for macro expansion ($__sourceTable) + ...(details.source && { + from: details.source.from, + metricTables: + details.source.kind === SourceKind.Metric + ? details.source.metricTables + : undefined, + }), + }; + } + return undefined; + } + + const { source } = details; + if (!source) { + logger.error( + { alertId: alert.id }, + 'Source not found for builder tile alert', + ); + return undefined; + } // Doesn't work for metric alerts yet if ( @@ -513,9 +590,21 @@ const getChartConfigFromAlert = ( return undefined; }; +type ResponseMetadata = + | { + type: 'time_series'; + timestampColumnName: string; + valueColumnNames: Set; + } + | { + type: 'single_value'; + valueColumnNames: Set; + }; + const getResponseMetadata = ( + chartConfig: ChartConfigWithOptDateRange, data: ResponseJSON>, -) => { +): ResponseMetadata | undefined => { if (!data?.meta) { return undefined; } @@ -527,39 +616,60 @@ const getResponseMetadata = ( jsType: clickhouse.convertCHDataTypeToJSType(m.type), })) ?? []; - const timestampColumnName = meta.find( - m => m.jsType === clickhouse.JSDataType.Date, - )?.name; const valueColumnNames = new Set( meta .filter(m => m.jsType === clickhouse.JSDataType.Number) .map(m => m.name), ); - if (timestampColumnName == null) { - logger.error({ meta }, 'Failed to find timestamp column'); - return undefined; - } - if (valueColumnNames.size === 0) { logger.error({ meta }, 'Failed to find value column'); return undefined; } - return { timestampColumnName, valueColumnNames }; + // Raw SQL charts with Number display type don't use interval parameters, so they cannot be treated as timeseries. + // Number-type Builder Charts are rendered as time-series, to maintain legacy behavior for existing alerts. + if ( + isRawSqlChartConfig(chartConfig) && + chartConfig.displayType === DisplayType.Number + ) { + return { type: 'single_value', valueColumnNames }; + } else { + const timestampColumnName = meta.find( + m => m.jsType === clickhouse.JSDataType.Date, + )?.name; + + if (timestampColumnName == null) { + logger.error({ meta }, 'Failed to find timestamp column'); + return undefined; + } + + return { type: 'time_series', timestampColumnName, valueColumnNames }; + } }; +/** + * Parses the following from the given alert query result: + * - `value`: the numeric value to compare against the alert threshold, taken + * from the last column in the result which is included in valueColumnNames + * - `extraFields`: an array of strings representing the names and values of + * each column in the result which is neither the timestampColumnName nor a + * valueColumnName, formatted as "columnName:value". + */ const parseAlertData = ( data: Record, - meta: { timestampColumnName: string; valueColumnNames: Set }, + meta: ResponseMetadata, ) => { let value: number | null = null; const extraFields: string[] = []; for (const [k, v] of Object.entries(data)) { if (meta.valueColumnNames.has(k)) { + // Due to output_format_json_quote_64bit_integers=1, 64-bit integers will be returned as strings. + // Parse them as integers to ensure correct threshold comparison. + // Floats are not returned as strings (unless output_format_json_quote_64bit_floats=1, which is not the default). value = isString(v) ? parseInt(v) : v; - } else if (k !== meta.timestampColumnName) { + } else if (meta.type !== 'time_series' || k !== meta.timestampColumnName) { extraFields.push(`${k}:${v}`); } } @@ -575,7 +685,8 @@ export const processAlert = async ( alertProvider: AlertProvider, teamWebhooksById: Map, ) => { - const { alert, source, previousMap } = details; + const { alert, previousMap } = details; + const source = 'source' in details ? details.source : undefined; try { const windowSizeInMins = ms(alert.interval) / 60000; const scheduleStartAt = normalizeScheduleStartAt({ @@ -680,10 +791,18 @@ export const processAlert = async ( // so we render the saved search's select separately to discover aliases // and inject them as WITH clauses into the alert query. if (details.taskType === AlertTaskType.SAVED_SEARCH) { + if (!isBuilderChartConfig(chartConfig)) { + logger.error({ + chartConfig, + message: + 'Found non-builder chart config for saved search alert, cannot compute WITH clauses', + }); + throw new Error('Expected builder chart config for saved search alert'); + } try { const withClauses = await computeAliasWithClauses( details.savedSearch, - source, + details.source, metadata, ); if (withClauses) { @@ -700,24 +819,34 @@ export const processAlert = async ( // Optimize chart config with materialized views, if available. // materializedViews exists on Log and Trace sources. const mvSource = - source.kind === SourceKind.Log || source.kind === SourceKind.Trace + source?.kind === SourceKind.Log || source?.kind === SourceKind.Trace ? source : undefined; - const optimizedChartConfig = mvSource?.materializedViews?.length - ? await tryOptimizeConfigWithMaterializedView( - chartConfig, - metadata, - clickhouseClient, - undefined, - mvSource, - ) - : chartConfig; + const optimizedChartConfig = + isBuilderChartConfig(chartConfig) && mvSource?.materializedViews?.length + ? await tryOptimizeConfigWithMaterializedView( + chartConfig, + metadata, + clickhouseClient, + undefined, + mvSource, + ) + : chartConfig; + + // Readonly = 2 means the query is readonly but can still specify query settings. + // This is done only for Raw SQL configs because it carries a minor risk of conflict with + // existing settings (which may have readonly = 1) and is not required for builder + // chart configs, which are always rendered as select statements. + const clickHouseSettings = isRawSqlChartConfig(optimizedChartConfig) + ? { readonly: '2' } + : {}; // Query for alert data const checksData = await clickhouseClient.queryChartConfig({ config: optimizedChartConfig, metadata, - querySettings: source.querySettings, + opts: { clickhouse_settings: clickHouseSettings }, + querySettings: source?.querySettings, }); logger.info( @@ -800,18 +929,74 @@ export const processAlert = async ( } }; + const sendNotificationIfResolved = async ( + previousHistory: AggregatedAlertHistory | undefined, + currentHistory: IAlertHistory, + groupKey: string, + ) => { + if ( + previousHistory?.state === AlertState.ALERT && + currentHistory.state === AlertState.OK + ) { + const lastValue = + currentHistory.lastValues[currentHistory.lastValues.length - 1]; + await trySendNotification({ + state: AlertState.OK, + group: groupKey, + totalCount: lastValue?.count || 0, + startTime: lastValue?.startTime || nowInMinsRoundDown, + }); + } + }; + + const meta = getResponseMetadata(chartConfig, checksData); + if (!meta) { + logger.error({ alertId: alert.id }, 'Failed to get response metadata'); + return; + } + + // single_value type (Raw SQL Number charts) returns a single value with no + // timestamp column, and are assumed to not have groups. + if (meta.type === 'single_value') { + // Use the date range end as the alert timestamp. + const alertTimestamp = dateRange[1]; + const history = getOrCreateHistory(''); + + // The value is taken from the last numeric column of the first row. + // The value defaults to 0. + const value = + checksData.data.length > 0 + ? (parseAlertData(checksData.data[0], meta).value ?? 0) + : 0; + + history.lastValues.push({ count: value, startTime: alertTimestamp }); + if (doesExceedThreshold(alert, value)) { + history.state = AlertState.ALERT; + history.counts += 1; + await trySendNotification({ + state: AlertState.ALERT, + group: '', + totalCount: value, + startTime: alertTimestamp, + }); + } + + // Auto-resolve + const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); + await sendNotificationIfResolved(previous, history, ''); + + const historyRecords = Array.from(histories.values()); + await alertProvider.updateAlertState(alert.id, historyRecords); + return; + } + + // ── Time-series path (Line/StackedBar charts) ── const expectedBuckets = timeBucketByGranularity( dateRange[0], dateRange[1], `${windowSizeInMins} minute`, ); - const meta = getResponseMetadata(checksData); - if (!meta) { - logger.error({ alertId: alert.id }, 'Failed to get response metadata'); - return; - } - // Group data by time bucket (grouped alerts may have multiple entries per time bucket) const checkDataByBucket = new Map< number, @@ -838,12 +1023,7 @@ export const processAlert = async ( 'No data returned from ClickHouse for time bucket', ); - // Empty periods are filled with a 0 values. - const zeroValueIsAlert = doesExceedThreshold( - alert.thresholdType, - alert.threshold, - 0, - ); + const zeroValueIsAlert = doesExceedThreshold(alert, 0); const hasAlertsInPreviousMap = previousMap .values() @@ -884,7 +1064,7 @@ export const processAlert = async ( const groupKey = hasGroupBy ? extraFields.join(', ') : ''; const history = getOrCreateHistory(groupKey); - if (doesExceedThreshold(alert.thresholdType, alert.threshold, value)) { + if (doesExceedThreshold(alert, value)) { history.state = AlertState.ALERT; await trySendNotification({ state: AlertState.ALERT, @@ -912,7 +1092,7 @@ export const processAlert = async ( if ( previousHistory.state === AlertState.ALERT && !histories.has(groupKey) && - !doesExceedThreshold(alert.thresholdType, alert.threshold, 0) + !doesExceedThreshold(alert, 0) ) { logger.info( { @@ -936,19 +1116,7 @@ export const processAlert = async ( for (const [groupKey, history] of histories.entries()) { const previousKey = computeHistoryMapKey(alert.id, groupKey); const groupPrevious = previousMap.get(previousKey); - - if ( - groupPrevious?.state === AlertState.ALERT && - history.state === AlertState.OK - ) { - const lastValue = history.lastValues[history.lastValues.length - 1]; - await trySendNotification({ - state: AlertState.OK, - group: groupKey, - totalCount: lastValue?.count || 0, - startTime: lastValue?.startTime || nowInMinsRoundDown, - }); - } + await sendNotificationIfResolved(groupPrevious, history, groupKey); } // Save all history records and update alert state diff --git a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts index d2d270b2..e72aa289 100644 --- a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts +++ b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.test.ts @@ -193,13 +193,14 @@ describe('DefaultAlertProvider', () => { // Validate source is proper ISource object const alertSource = result[0].alerts[0].source; - expect(alertSource.connection).toBe(connection.id); // Should be ObjectId, not populated IConnection - expect(alertSource.name).toBe('Test Source'); - expect(alertSource.kind).toBe('log'); - expect(alertSource.team).toBeDefined(); - expect(alertSource.from?.databaseName).toBe('default'); - expect(alertSource.from?.tableName).toBe('logs'); - expect(alertSource.timestampValueExpression).toBe('timestamp'); + expect(alertSource).toBeDefined(); + expect(alertSource!.connection).toBe(connection.id); // Should be ObjectId, not populated IConnection + expect(alertSource!.name).toBe('Test Source'); + expect(alertSource!.kind).toBe('log'); + expect(alertSource!.team).toBeDefined(); + expect(alertSource!.from?.databaseName).toBe('default'); + expect(alertSource!.from?.tableName).toBe('logs'); + expect(alertSource!.timestampValueExpression).toBe('timestamp'); // Ensure it's a plain object, not a mongoose document expect((alertSource as any).toObject).toBeUndefined(); // mongoose documents have toObject method diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index ace87f21..fa2ebe1b 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -1,4 +1,5 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { displayTypeSupportsRawSqlAlerts } from '@hyperdx/common-utils/dist/core/utils'; import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { Tile } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; @@ -108,13 +109,56 @@ async function getTileDetails( } if (isRawSqlSavedChartConfig(tile.config)) { - logger.warn({ - tileId, - dashboardId: dashboard._id, - alertId: alert.id, - message: 'skipping alert with raw sql chart config, not supported', - }); - return []; + if (!displayTypeSupportsRawSqlAlerts(tile.config.displayType)) { + logger.warn({ + tileId, + dashboardId: dashboard._id, + alertId: alert.id, + message: + 'skipping alert with raw sql chart config, only line/bar display types are supported', + }); + return []; + } + + // Raw SQL tiles store connection ID directly on the config + const connection = await Connection.findOne({ + _id: tile.config.connection, + team: alert.team, + }).select('+password'); + + if (!connection) { + logger.error({ + message: 'connection not found for raw sql tile', + connectionId: tile.config.connection, + tileId, + dashboardId: dashboard._id, + alertId: alert.id, + }); + return []; + } + + // Optionally look up source for filter/macro metadata + let source: ISource | undefined; + if (tile.config.source) { + const sourceDoc = await Source.findOne({ + _id: tile.config.source, + team: alert.team, + }); + if (sourceDoc) { + source = sourceDoc.toObject(); + } + } + + return [ + connection, + { + alert, + source, + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ]; } const source = await Source.findOne({ diff --git a/packages/api/src/tasks/checkAlerts/providers/index.ts b/packages/api/src/tasks/checkAlerts/providers/index.ts index d6338223..46cdb876 100644 --- a/packages/api/src/tasks/checkAlerts/providers/index.ts +++ b/packages/api/src/tasks/checkAlerts/providers/index.ts @@ -32,15 +32,16 @@ export type PopulatedAlertChannel = { type: 'webhook' } & { channel: IWebhook }; // the are required when the type is set accordingly. export type AlertDetails = { alert: IAlert; - source: ISource; previousMap: Map; // Map of alertId||group -> history for group-by alerts } & ( | { taskType: AlertTaskType.SAVED_SEARCH; + source: ISource; savedSearch: Omit; } | { taskType: AlertTaskType.TILE; + source?: ISource; tile: Tile; dashboard: IDashboard; } diff --git a/packages/api/src/tasks/checkAlerts/template.ts b/packages/api/src/tasks/checkAlerts/template.ts index f5494ba6..d6ecc839 100644 --- a/packages/api/src/tasks/checkAlerts/template.ts +++ b/packages/api/src/tasks/checkAlerts/template.ts @@ -8,8 +8,10 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { AlertChannelType, + AlertThresholdType, ChartConfigWithOptDateRange, DisplayType, + isRangeThresholdType, pickSampleWeightExpressionProps, SourceKind, WebhookService, @@ -24,7 +26,7 @@ import { z } from 'zod'; import * as config from '@/config'; import { AlertInput } from '@/controllers/alerts'; -import { AlertSource, AlertState, AlertThresholdType } from '@/models/alert'; +import { AlertSource, AlertState } from '@/models/alert'; import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; import { ISource } from '@/models/source'; @@ -42,6 +44,58 @@ import { truncateString } from '@/utils/common'; import logger from '@/utils/logger'; import * as slack from '@/utils/slack'; +const describeThresholdViolation = ( + thresholdType: AlertThresholdType, +): string => { + switch (thresholdType) { + case AlertThresholdType.ABOVE: + return 'meets or exceeds'; + case AlertThresholdType.ABOVE_EXCLUSIVE: + return 'exceeds'; + case AlertThresholdType.BELOW: + return 'falls below'; + case AlertThresholdType.BELOW_OR_EQUAL: + return 'falls to or below'; + case AlertThresholdType.EQUAL: + return 'equals'; + case AlertThresholdType.NOT_EQUAL: + return 'does not equal'; + case AlertThresholdType.BETWEEN: + return 'falls between'; + case AlertThresholdType.NOT_BETWEEN: + return 'falls outside'; + } +}; + +const describeThresholdResolution = ( + thresholdType: AlertThresholdType, +): string => { + switch (thresholdType) { + case AlertThresholdType.ABOVE: + return 'falls below'; + case AlertThresholdType.ABOVE_EXCLUSIVE: + return 'falls to or below'; + case AlertThresholdType.BELOW: + return 'meets or exceeds'; + case AlertThresholdType.BELOW_OR_EQUAL: + return 'exceeds'; + case AlertThresholdType.EQUAL: + return 'does not equal'; + case AlertThresholdType.NOT_EQUAL: + return 'equals'; + case AlertThresholdType.BETWEEN: + return 'falls outside'; + case AlertThresholdType.NOT_BETWEEN: + return 'falls between'; + } +}; + +const describeThreshold = (alert: AlertInput): string => { + return isRangeThresholdType(alert.thresholdType) + ? `${alert.threshold} and ${alert.thresholdMax ?? '?'}` + : `${alert.threshold}`; +}; + const MAX_MESSAGE_LENGTH = 500; const NOTIFY_FN_NAME = '__hdx_notify_channel__'; const IS_MATCH_FN_NAME = 'is_match'; @@ -376,14 +430,10 @@ export const buildAlertMessageTemplateTitle = ({ const baseTitle = template ? handlebars.compile(template)(view) : `Alert for "${tile.config.name}" in "${dashboard.name}" - ${formattedValue} ${ - doesExceedThreshold(alert.thresholdType, alert.threshold, value) - ? alert.thresholdType === AlertThresholdType.ABOVE - ? 'exceeds' - : 'falls below' - : alert.thresholdType === AlertThresholdType.ABOVE - ? 'falls below' - : 'exceeds' - } ${alert.threshold}`; + doesExceedThreshold(alert, value) + ? describeThresholdViolation(alert.thresholdType) + : describeThresholdResolution(alert.thresholdType) + } ${describeThreshold(alert)}`; return `${emoji}${baseTitle}`; } @@ -649,11 +699,7 @@ ${targetTemplate}`; } rawTemplateBody = `${group ? `Group: "${group}"` : ''} -${value} lines found, expected ${ - alert.thresholdType === AlertThresholdType.ABOVE - ? 'less than' - : 'greater than' - } ${alert.threshold} lines\n${timeRangeMessage} +${value} lines found, which ${describeThresholdViolation(alert.thresholdType)} the threshold of ${describeThreshold(alert)} lines\n${timeRangeMessage} ${targetTemplate} \`\`\` ${truncatedResults} @@ -665,14 +711,10 @@ ${truncatedResults} const formattedValue = formatValueToMatchThreshold(value, alert.threshold); rawTemplateBody = `${group ? `Group: "${group}"` : ''} ${formattedValue} ${ - doesExceedThreshold(alert.thresholdType, alert.threshold, value) - ? alert.thresholdType === AlertThresholdType.ABOVE - ? 'exceeds' - : 'falls below' - : alert.thresholdType === AlertThresholdType.ABOVE - ? 'falls below' - : 'exceeds' - } ${alert.threshold}\n${timeRangeMessage} + doesExceedThreshold(alert, value) + ? describeThresholdViolation(alert.thresholdType) + : describeThresholdResolution(alert.thresholdType) + } ${describeThreshold(alert)}\n${timeRangeMessage} ${targetTemplate}`; } diff --git a/packages/api/src/utils/__tests__/__snapshots__/logParser.test.ts.snap b/packages/api/src/utils/__tests__/__snapshots__/logParser.test.ts.snap index 1add8294..592f9242 100644 --- a/packages/api/src/utils/__tests__/__snapshots__/logParser.test.ts.snap +++ b/packages/api/src/utils/__tests__/__snapshots__/logParser.test.ts.snap @@ -1,41 +1,41 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing exports[`logParser mapObjectToKeyValuePairs 1`] = ` -Object { - "bool.names": Array [ +{ + "bool.names": [ "foo2", "good.burrito.is", ], - "bool.values": Array [ + "bool.values": [ 0, 1, ], - "number.names": Array [ + "number.names": [ "foo1", ], - "number.values": Array [ + "number.values": [ 123, ], - "string.names": Array [ + "string.names": [ "foo", "nested.foo", "array1", "array2", ], - "string.values": Array [ + "string.values": [ "123", "bar", "[456]", - "[\\"foo1\\",{\\"foo2\\":\\"bar2\\"},[{\\"foo3\\":\\"bar3\\"}]]", + "["foo1",{"foo2":"bar2"},[{"foo3":"bar3"}]]", ], } `; exports[`logParser mapObjectToKeyValuePairs 2`] = ` -Object { - "bool.names": Array [], - "bool.values": Array [], - "number.names": Array [ +{ + "bool.names": [], + "bool.values": [], + "number.names": [ "foo0", "foo1", "foo2", @@ -1061,7 +1061,7 @@ Object { "foo1022", "foo1023", ], - "number.values": Array [ + "number.values": [ 0, 1, 2, @@ -2087,7 +2087,7 @@ Object { 1022, 1023, ], - "string.names": Array [], - "string.values": Array [], + "string.names": [], + "string.values": [], } `; diff --git a/packages/api/src/utils/__tests__/trimToolResponse.test.ts b/packages/api/src/utils/__tests__/trimToolResponse.test.ts new file mode 100644 index 00000000..c6bd2eea --- /dev/null +++ b/packages/api/src/utils/__tests__/trimToolResponse.test.ts @@ -0,0 +1,121 @@ +import { trimToolResponse } from '../trimToolResponse'; + +describe('trimToolResponse', () => { + describe('small data (within maxSize)', () => { + it('should return primitive values unchanged', () => { + expect(trimToolResponse(42)).toBe(42); + expect(trimToolResponse('hello')).toBe('hello'); + expect(trimToolResponse(null)).toBeNull(); + expect(trimToolResponse(true)).toBe(true); + }); + + it('should return small arrays unchanged', () => { + const data = [1, 2, 3, 4, 5]; + expect(trimToolResponse(data)).toEqual(data); + }); + + it('should return small objects unchanged', () => { + const data = { a: 1, b: 'hello', c: [1, 2, 3] }; + expect(trimToolResponse(data)).toEqual(data); + }); + }); + + describe('large arrays', () => { + it('should trim large arrays to fit within maxSize', () => { + // Create an array that exceeds maxSize + const largeArray = Array.from({ length: 500 }, (_, i) => ({ + id: i, + data: 'x'.repeat(200), + })); + + const result = trimToolResponse(largeArray, 5000); + expect(Array.isArray(result)).toBe(true); + expect(result.length).toBeLessThan(largeArray.length); + expect(result.length).toBeGreaterThanOrEqual(10); // minimum 10 items + expect(JSON.stringify(result).length).toBeLessThanOrEqual(5000); + }); + + it('should keep at least 10 items', () => { + const largeArray = Array.from({ length: 100 }, (_, i) => ({ + id: i, + data: 'x'.repeat(500), + })); + + // maxSize so small even 10 items may exceed it, but we keep 10 minimum + const result = trimToolResponse(largeArray, 100); + expect(Array.isArray(result)).toBe(true); + expect(result.length).toBeGreaterThanOrEqual(10); + }); + + it('should not trim arrays that fit within maxSize', () => { + const smallArray = [1, 2, 3, 4, 5]; + const result = trimToolResponse(smallArray, 50000); + expect(result).toEqual(smallArray); + }); + }); + + describe('large objects', () => { + it('should trim large objects to fit within maxSize', () => { + const largeObj: Record = {}; + for (let i = 0; i < 100; i++) { + largeObj[`key_${i}`] = 'x'.repeat(200); + } + + const result = trimToolResponse(largeObj, 5000); + // The trimmed result must be smaller than the original + expect(JSON.stringify(result).length).toBeLessThan( + JSON.stringify(largeObj).length, + ); + // All keys should still be present (values are trimmed, not dropped) + expect( + Object.keys(result).filter(k => k !== '__hdx_trimmed'), + ).toHaveLength(100); + // The sentinel flag should be set to indicate trimming occurred + expect(result.__hdx_trimmed).toBe(true); + }); + + it('should not trim objects that fit within maxSize', () => { + const obj = { a: 1, b: 2 }; + const result = trimToolResponse(obj, 50000); + expect(result).toEqual(obj); + }); + }); + + describe('getAIMetadata structure', () => { + it('should handle objects with allFieldsWithKeys and keyValues', () => { + const metadataObj = { + allFieldsWithKeys: Array.from({ length: 200 }, (_, i) => ({ + field: `field_${i}`, + key: `key_${i}`, + extra: 'x'.repeat(100), + })), + keyValues: Object.fromEntries( + Array.from({ length: 200 }, (_, i) => [`kv_${i}`, 'x'.repeat(100)]), + ), + otherProp: 'preserved', + }; + + const result = trimToolResponse(metadataObj, 5000); + expect(result).toHaveProperty('allFieldsWithKeys'); + expect(result).toHaveProperty('keyValues'); + expect(result).toHaveProperty('otherProp', 'preserved'); + expect(Array.isArray(result.allFieldsWithKeys)).toBe(true); + expect(typeof result.keyValues).toBe('object'); + }); + }); + + describe('default maxSize', () => { + it('should use 50000 as default maxSize', () => { + // Create data just over default size + const data = Array.from({ length: 1000 }, (_, i) => ({ + id: i, + payload: 'x'.repeat(100), + })); + + const resultDefault = trimToolResponse(data); + const resultExplicit = trimToolResponse(data, 50000); + // Both should produce the same result + expect(resultDefault.length).toBe(resultExplicit.length); + }); + }); +}); diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 721e4ad4..80f6144c 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -1,4 +1,5 @@ import { + AlertThresholdType, BuilderSavedChartConfig, DashboardFilter, DisplayType, @@ -12,7 +13,6 @@ import { AlertDocument, AlertInterval, AlertState, - AlertThresholdType, } from '@/models/alert'; import type { DashboardDocument } from '@/models/dashboard'; import { SeriesTile } from '@/routers/external-api/v2/utils/dashboards'; @@ -228,6 +228,7 @@ export type ExternalAlert = { name?: string | null; message?: string | null; threshold: number; + thresholdMax?: number; interval: AlertInterval; scheduleOffsetMinutes?: number; scheduleStartAt?: string | null; @@ -309,6 +310,7 @@ export function translateAlertDocumentToExternalAlert( name: alertObj.name, message: alertObj.message, threshold: alertObj.threshold, + thresholdMax: alertObj.thresholdMax, interval: alertObj.interval, ...(alertObj.scheduleOffsetMinutes != null && { scheduleOffsetMinutes: alertObj.scheduleOffsetMinutes, diff --git a/packages/api/src/utils/rateLimiter.ts b/packages/api/src/utils/rateLimiter.ts index 234a653a..51320e77 100644 --- a/packages/api/src/utils/rateLimiter.ts +++ b/packages/api/src/utils/rateLimiter.ts @@ -1,5 +1,10 @@ +import express from 'express'; import rateLimit, { Options } from 'express-rate-limit'; +export const rateLimiterKeyGenerator = (req: express.Request): string => { + return req.headers.authorization ?? req.ip ?? 'unknown'; +}; + export default (config?: Partial) => { return rateLimit({ ...config, diff --git a/packages/api/src/utils/serialization.ts b/packages/api/src/utils/serialization.ts index ab79dfc0..8c536f5e 100644 --- a/packages/api/src/utils/serialization.ts +++ b/packages/api/src/utils/serialization.ts @@ -9,7 +9,7 @@ type JsonStringifiable = { toJSON(): string }; * toJSON(): string). This allows passing raw Mongoose data to sendJson() * while keeping type inference from the typed Express response. */ -type PreSerialized = T extends string +export type PreSerialized = T extends string ? string | JsonStringifiable : T extends (infer U)[] ? PreSerialized[] diff --git a/packages/api/src/utils/trimToolResponse.ts b/packages/api/src/utils/trimToolResponse.ts new file mode 100644 index 00000000..b4108399 --- /dev/null +++ b/packages/api/src/utils/trimToolResponse.ts @@ -0,0 +1,109 @@ +import logger from '@/utils/logger'; + +/** + * Trims large data structures to prevent "Request Entity Too Large" errors + * when multiple tool calls accumulate data in the conversation history. + */ +export function trimToolResponse(data: any, maxSize: number = 50000): any { + const serialized = JSON.stringify(data); + + // If data is within acceptable size, return as-is + if (serialized.length <= maxSize) { + return data; + } + + logger.warn( + `Tool response too large, trimming data. Original Size: ${serialized.length}, Max Size: ${maxSize}`, + ); + + // Handle different data structures + if (Array.isArray(data)) { + return trimArray(data, maxSize); + } + + if (typeof data === 'object' && data !== null) { + return trimObject(data, maxSize); + } + + return data; +} + +function trimArray(arr: any[], maxSize: number): any[] { + // Keep reducing array size until it fits + let result = [...arr]; + let resultSize = JSON.stringify(result).length; + + while (resultSize > maxSize && result.length > 10) { + // Keep at least 10 items + const newLength = Math.max(10, Math.floor(result.length * 0.7)); + result = result.slice(0, newLength); + resultSize = JSON.stringify(result).length; + } + + // If we're still over budget (e.g. a single item exceeds maxSize), truncate + // individual oversized items so the array itself stays within the limit. + if (resultSize > maxSize) { + result = result.map(item => { + const itemStr = JSON.stringify(item); + if (itemStr.length > maxSize) { + logger.info( + `Trimming oversized array item (${itemStr.length} bytes > ${maxSize} limit)`, + ); + if (typeof item === 'object' && item !== null) { + return trimObject(item, maxSize); + } + // Scalar that is itself too large — return a truncation marker + return { __hdx_trimmed: true, originalSize: itemStr.length }; + } + return item; + }); + } + + if (result.length < arr.length) { + logger.info(`Trimmed array from ${arr.length} to ${result.length} items`); + } + + return result; +} + +// Keys in trimObject come exclusively from Object.entries() on internal tool +// response data — never from user-supplied HTTP input — so bracket-notation +// writes are not an injection risk; see inline eslint-disable comments below. +function trimObject(obj: any, maxSize: number): any { + const entries = Object.entries(obj); + if (entries.length === 0) return obj; + + const result: any = {}; + + // Give each key an equal share of the budget so that no single large value + // crowds out the rest (e.g. a large array at key[0] eating all the budget + // before key[1] gets a chance to appear). + const perKeyBudget = Math.floor(maxSize / entries.length); + let trimmed = false; + + for (const [key, value] of entries) { + const valueStr = JSON.stringify(value); + + if (valueStr.length <= perKeyBudget) { + result[key] = value; // eslint-disable-line security/detect-object-injection + } else { + logger.info( + `Trimming oversized object value at key "${key}" (${valueStr.length} bytes > ${perKeyBudget} per-key budget)`, + ); + if (Array.isArray(value)) { + result[key] = trimArray(value, perKeyBudget); // eslint-disable-line security/detect-object-injection + } else if (typeof value === 'object' && value !== null) { + result[key] = trimObject(value, perKeyBudget); // eslint-disable-line security/detect-object-injection + } else { + result[key] = { __hdx_trimmed: true, originalSize: valueStr.length }; // eslint-disable-line security/detect-object-injection + } + trimmed = true; + } + } + + if (trimmed) { + result.__hdx_trimmed = true; + } + + return result; +} diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 2ddf02d0..bd9c1813 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -1,17 +1,19 @@ import { AggregateFunctionSchema, + AlertThresholdType, DashboardFilterSchema, MetricsDataType, NumberFormatSchema, scheduleStartAtSchema, SearchConditionLanguageSchema as whereLanguageSchema, validateAlertScheduleOffsetMinutes, + validateAlertThresholdMax, WebhookService, } from '@hyperdx/common-utils/dist/types'; import { Types } from 'mongoose'; import { z } from 'zod'; -import { AlertSource, AlertThresholdType } from '@/models/alert'; +import { AlertSource } from '@/models/alert'; export const objectIdSchema = z.string().refine(val => { return Types.ObjectId.isValid(val); @@ -510,12 +512,14 @@ export const alertSchema = z scheduleStartAt: scheduleStartAtSchema, threshold: z.number(), thresholdType: z.nativeEnum(AlertThresholdType), + thresholdMax: z.number().optional(), source: z.nativeEnum(AlertSource).default(AlertSource.SAVED_SEARCH), name: z.string().min(1).max(512).nullish(), message: z.string().min(1).max(4096).nullish(), }) .and(zSavedSearchAlert.or(zTileAlert)) - .superRefine(validateAlertScheduleOffsetMinutes); + .superRefine(validateAlertScheduleOffsetMinutes) + .superRefine(validateAlertThresholdMax); // ============================== // Webhooks diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index f2d94a5a..a1fb7569 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -5,6 +5,7 @@ "paths": { "@/*": ["./*"] }, + "types": ["jest", "node"], "outDir": "build", "isolatedModules": true, "skipLibCheck": true, diff --git a/packages/app/CHANGELOG.md b/packages/app/CHANGELOG.md index 92b4fd34..84f635db 100644 --- a/packages/app/CHANGELOG.md +++ b/packages/app/CHANGELOG.md @@ -1,5 +1,33 @@ # @hyperdx/app +## 2.23.2 + +### Patch Changes + +- 0daa5299: feat: Generate stable source IDs in local mode + +## 2.23.1 + +### Patch Changes + +- 7d1a8e54: fix: Show sidebar favorites empty state when none are starred yet +- 800689ac: feat: Add reusable EmptyState component and adopt it across pages for consistent empty/no-data states +- 2570ff84: fix: Change K8s CPU chart format from percentage to number to support both old and new OTel collector metric names +- ad71dc2e: feat: Add keyboard shortcuts modal from the Help menu + + - New **Keyboard shortcuts** item opens a modal documenting app shortcuts (command palette ⌘/Ctrl+K, search focus, time picker, tables, traces, dashboards, and more). + - Help menu items ordered by importance (documentation and setup before shortcuts and community). + - Shortcuts modal uses a readable width, row dividers, and **or** vs **+** labels so alternative keys are not confused with key chords. + +- 1bcca2cd: feat: Add alert icons to dashboard list page +- 52986a94: Fix bug when accessing session replay panel from search page +- ffc961c6: fix: Add error message and edit button when tile source is missing +- 3ffafced: feat: show error details in search event patterns +- 61db3e8b: refactor: Create TileAlertEditor component +- f8d2edde: feat: Show created/updated metadata for saved searches and dashboards +- Updated dependencies [24767c58] + - @hyperdx/common-utils@0.17.1 + ## 2.23.0 ### Minor Changes diff --git a/packages/app/eslint.config.mjs b/packages/app/eslint.config.mjs index 73977dc5..d02587a5 100644 --- a/packages/app/eslint.config.mjs +++ b/packages/app/eslint.config.mjs @@ -91,6 +91,7 @@ export default [ 'next-env.d.ts', 'playwright-report/**', '.next/**', + '.next-e2e/**', '.storybook/**', 'node_modules/**', 'out/**', @@ -121,8 +122,28 @@ export default [ ...nextPlugin.configs.recommended.rules, ...nextPlugin.configs['core-web-vitals'].rules, ...reactHooksPlugin.configs.recommended.rules, + ...eslintReactPlugin.configs['recommended-type-checked'].rules, + + // Non-default react-hooks rules + 'react-hooks/set-state-in-render': 'error', 'react-hooks/set-state-in-effect': 'warn', 'react-hooks/exhaustive-deps': 'error', + + // Disable rules from @eslint-react that have equivalent rules enabled in eslint-plugin-react-hooks + '@eslint-react/rules-of-hooks': 'off', + '@eslint-react/component-hook-factories': 'off', + '@eslint-react/exhaustive-deps': 'off', + '@eslint-react/error-boundaries': 'off', + '@eslint-react/immutability': 'off', + '@eslint-react/purity': 'off', + '@eslint-react/refs': 'off', + '@eslint-react/set-state-in-effect': 'off', + '@eslint-react/set-state-in-render': 'off', + '@eslint-react/no-nested-component-definitions': 'off', + '@eslint-react/no-nested-lazy-component-declarations': 'off', + '@eslint-react/unsupported-syntax': 'off', + '@eslint-react/use-memo': 'off', + 'react-hook-form/no-use-watch': 'error', '@eslint-react/no-unstable-default-props': 'error', '@typescript-eslint/ban-ts-comment': 'warn', @@ -205,6 +226,7 @@ export default [ rules: { // Drop date rules — new Date() / Date.now() are fine in tests 'no-restricted-syntax': ['error', ...UI_SYNTAX_RESTRICTIONS], + '@eslint-react/component-hook-factories': 'off', }, }, { diff --git a/packages/app/package.json b/packages/app/package.json index 1710c36a..b69b84be 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/app", - "version": "2.23.0", + "version": "2.23.2", "private": true, "license": "MIT", "engines": { @@ -18,7 +18,7 @@ "lint:styles": "stylelint **/*/*.{css,scss}", "ci:lint": "yarn lint && yarn tsc --noEmit && yarn lint:styles --quiet", "ci:unit": "jest --ci --coverage", - "dev:unit": "jest --watchAll --detectOpenHandles", + "dev:unit": "jest --watchAll", "test:e2e": "node scripts/run-e2e.js", "test:e2e:ci": "../../scripts/test-e2e-ci.sh", "storybook": "storybook dev -p 6006", @@ -34,15 +34,15 @@ "@dagrejs/dagre": "^1.1.5", "@hookform/resolvers": "^3.9.0", "@hyperdx/browser": "^0.22.0", - "@hyperdx/common-utils": "^0.17.0", + "@hyperdx/common-utils": "^0.17.1", "@hyperdx/node-opentelemetry": "^0.9.0", - "@mantine/core": "^7.17.8", - "@mantine/dates": "^7.17.8", - "@mantine/dropzone": "^7.17.8", - "@mantine/form": "^7.17.8", - "@mantine/hooks": "^7.17.8", - "@mantine/notifications": "^7.17.8", - "@mantine/spotlight": "^7.17.8", + "@mantine/core": "^9.0.0", + "@mantine/dates": "^9.0.0", + "@mantine/dropzone": "^9.0.0", + "@mantine/form": "^9.0.0", + "@mantine/hooks": "^9.0.0", + "@mantine/notifications": "^9.0.0", + "@mantine/spotlight": "^9.0.0", "@tabler/icons-react": "^3.39.0", "@tanstack/react-query": "^5.56.2", "@tanstack/react-query-devtools": "^5.56.2", @@ -79,7 +79,6 @@ "react-error-boundary": "^3.1.4", "react-grid-layout": "^1.3.4", "react-hook-form": "^7.43.8", - "react-hook-form-mantine": "^3.1.3", "react-hotkeys-hook": "^4.3.7", "react-json-tree": "^0.20.0", "react-markdown": "^10.1.0", @@ -140,7 +139,7 @@ "msw": "^2.3.0", "msw-storybook-addon": "^2.0.2", "postcss": "^8.4.38", - "postcss-preset-mantine": "^1.15.0", + "postcss-preset-mantine": "^1.18.0", "postcss-simple-vars": "^7.0.0", "prettier": "^3.3.2", "rimraf": "^4.4.1", diff --git a/packages/app/pages/_app.tsx b/packages/app/pages/_app.tsx index 68cdff0e..ca8fc72e 100644 --- a/packages/app/pages/_app.tsx +++ b/packages/app/pages/_app.tsx @@ -3,6 +3,7 @@ import type { NextPage } from 'next'; import type { AppProps } from 'next/app'; import Head from 'next/head'; import { NextAdapter } from 'next-query-params'; +import { env } from 'next-runtime-env'; import randomUUID from 'crypto-randomuuid'; import { enableMapSet } from 'immer'; import { QueryParamProvider } from 'use-query-params'; @@ -16,7 +17,7 @@ import { import { ReactQueryDevtools } from '@tanstack/react-query-devtools'; import { DynamicFavicon } from '@/components/DynamicFavicon'; -import { IS_LOCAL_MODE } from '@/config'; +import { IS_LOCAL_MODE, parseResourceAttributes } from '@/config'; import { DEFAULT_FONT_VAR, FONT_VAR_MAP, @@ -136,12 +137,19 @@ export default function MyApp({ Component, pageProps }: AppPropsWithLayout) { .then(res => res.json()) .then((_jsonData?: NextApiConfigResponseData) => { if (_jsonData?.apiKey) { + const frontendAttrs = parseResourceAttributes( + env('NEXT_PUBLIC_OTEL_RESOURCE_ATTRIBUTES') ?? '', + ); HyperDX.init({ apiKey: _jsonData.apiKey, consoleCapture: true, maskAllInputs: true, maskAllText: true, + // service.version is applied last so it always reflects the + // NEXT_PUBLIC_APP_VERSION and cannot be overridden by + // NEXT_PUBLIC_OTEL_RESOURCE_ATTRIBUTES. otelResourceAttributes: { + ...frontendAttrs, 'service.version': process.env.NEXT_PUBLIC_APP_VERSION, }, service: _jsonData.serviceName, diff --git a/packages/app/pages/trace/[traceId].tsx b/packages/app/pages/trace/[traceId].tsx new file mode 100644 index 00000000..61a1df71 --- /dev/null +++ b/packages/app/pages/trace/[traceId].tsx @@ -0,0 +1,41 @@ +import { useEffect } from 'react'; +import { useRouter } from 'next/router'; +import { Center, Text } from '@mantine/core'; + +import { withAppNav } from '@/layout'; +import { buildTraceRedirectUrl } from '@/utils/directTrace'; + +export function TraceRedirectPage() { + const { isReady, query, replace } = useRouter(); + const traceIdParam = Array.isArray(query.traceId) + ? query.traceId[0] + : query.traceId; + + useEffect(() => { + if (!isReady) return; + + if (!traceIdParam) { + replace('/search'); + return; + } + + replace( + buildTraceRedirectUrl({ + traceId: traceIdParam, + search: window.location.search, + }), + ); + }, [isReady, replace, traceIdParam]); + + return ( +
+ + Redirecting to search... + +
+ ); +} + +TraceRedirectPage.getLayout = withAppNav; + +export default TraceRedirectPage; diff --git a/packages/app/src/AlertsPage.tsx b/packages/app/src/AlertsPage.tsx index 6b732130..73c88c89 100644 --- a/packages/app/src/AlertsPage.tsx +++ b/packages/app/src/AlertsPage.tsx @@ -1,29 +1,15 @@ import * as React from 'react'; import Head from 'next/head'; import Link from 'next/link'; -import cx from 'classnames'; -import type { Duration } from 'date-fns'; -import { add, formatRelative } from 'date-fns'; import { - AlertHistory, AlertSource, AlertState, + isRangeThresholdType, } from '@hyperdx/common-utils/dist/types'; -import { - Alert, - Badge, - Button, - Container, - Group, - Menu, - Stack, - Tooltip, -} from '@mantine/core'; -import { notifications } from '@mantine/notifications'; +import { Alert, Anchor, Badge, Container, Group, Stack } from '@mantine/core'; import { IconAlertTriangle, IconBell, - IconBrandSlack, IconChartLine, IconCheck, IconChevronRight, @@ -31,290 +17,21 @@ import { IconInfoCircleFilled, IconTableRow, } from '@tabler/icons-react'; -import { useQueryClient } from '@tanstack/react-query'; -import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; +import { AckAlert } from '@/components/alerts/AckAlert'; +import { AlertHistoryCardList } from '@/components/alerts/AlertHistoryCards'; +import EmptyState from '@/components/EmptyState'; import { PageHeader } from '@/components/PageHeader'; import { useBrandDisplayName } from './theme/ThemeProvider'; -import { isAlertSilenceExpired } from './utils/alerts'; +import { TILE_ALERT_THRESHOLD_TYPE_OPTIONS } from './utils/alerts'; import { getWebhookChannelIcon } from './utils/webhookIcons'; import api from './api'; import { withAppNav } from './layout'; import type { AlertsPageItem } from './types'; -import { FormatTime } from './useFormatTime'; import styles from '../styles/AlertsPage.module.scss'; -function AlertHistoryCard({ - history, - alertUrl, -}: { - history: AlertHistory; - alertUrl: string; -}) { - const start = new Date(history.createdAt.toString()); - - // eslint-disable-next-line no-restricted-syntax - const today = React.useMemo(() => new Date(), []); - - const href = React.useMemo(() => { - if (!alertUrl || !history.lastValues?.[0]?.startTime) return null; - - // Create time window from alert creation to last recorded value - const to = new Date(history.createdAt).getTime(); - const from = new Date(history.lastValues[0].startTime).getTime(); - - // Construct URL with time range parameters - const url = new URL(alertUrl, window.location.origin); - url.searchParams.set('from', from.toString()); - url.searchParams.set('to', to.toString()); - url.searchParams.set('isLive', 'false'); - - return url.pathname + url.search; - }, [history, alertUrl]); - - const content = ( -
- ); - - return ( - - {href ? ( - - {content} - - ) : ( - content - )} - - ); -} - -const HISTORY_ITEMS = 18; - -function AckAlert({ alert }: { alert: AlertsPageItem }) { - const queryClient = useQueryClient(); - const silenceAlert = api.useSilenceAlert(); - const unsilenceAlert = api.useUnsilenceAlert(); - - const mutateOptions = React.useMemo( - () => ({ - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ['alerts'] }); - }, - onError: (error: any) => { - const status = error?.response?.status; - let message = 'Failed to silence alert, please try again later.'; - - if (status === 404) { - message = 'Alert not found.'; - } else if (status === 400) { - message = - 'Invalid request. Please ensure the silence duration is valid.'; - } - - notifications.show({ - color: 'red', - message, - }); - }, - }), - [queryClient], - ); - - const handleUnsilenceAlert = React.useCallback(() => { - unsilenceAlert.mutate(alert._id || '', mutateOptions); - }, [alert._id, mutateOptions, unsilenceAlert]); - - const isNoLongerMuted = React.useMemo(() => { - return isAlertSilenceExpired(alert.silenced); - }, [alert.silenced]); - - const handleSilenceAlert = React.useCallback( - (duration: Duration) => { - // eslint-disable-next-line no-restricted-syntax - const mutedUntil = add(new Date(), duration); - silenceAlert.mutate( - { - alertId: alert._id || '', - mutedUntil: mutedUntil.toISOString(), - }, - mutateOptions, - ); - }, - [alert._id, mutateOptions, silenceAlert], - ); - - if (alert.silenced?.at) { - return ( - - - - - - - - Acknowledged{' '} - {alert.silenced?.by ? ( - <> - by {alert.silenced?.by} - - ) : null}{' '} - on
- - .
-
- - - {isNoLongerMuted ? ( - 'Alert resumed.' - ) : ( - <> - Resumes . - - )} - - - {isNoLongerMuted ? 'Unacknowledge' : 'Resume alert'} - -
-
-
- ); - } - - if (alert.state === 'ALERT') { - return ( - - - - - - - - Acknowledge and silence for - - - handleSilenceAlert({ - minutes: 30, - }) - } - > - 30 minutes - - - handleSilenceAlert({ - hours: 1, - }) - } - > - 1 hour - - - handleSilenceAlert({ - hours: 6, - }) - } - > - 6 hours - - - handleSilenceAlert({ - hours: 24, - }) - } - > - 24 hours - - - - - ); - } - - return null; -} - -function AlertHistoryCardList({ - history, - alertUrl, -}: { - history: AlertHistory[]; - alertUrl: string; -}) { - const items = React.useMemo(() => { - if (history.length < HISTORY_ITEMS) { - return history; - } - return history.slice(0, HISTORY_ITEMS); - }, [history]); - - const paddingItems = React.useMemo(() => { - if (history.length > HISTORY_ITEMS) { - return []; - } - return new Array(HISTORY_ITEMS - history.length).fill(null); - }, [history]); - - return ( -
- {paddingItems.map((_, index) => ( - -
- - ))} - {items - .slice() - .reverse() - .map((history, index) => ( - - ))} -
- ); -} - function AlertDetails({ alert }: { alert: AlertsPageItem }) { const alertName = React.useMemo(() => { if (alert.source === AlertSource.TILE && alert.dashboard) { @@ -362,10 +79,19 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) { })(); const alertType = React.useMemo(() => { + const thresholdLabel = + TILE_ALERT_THRESHOLD_TYPE_OPTIONS[alert.thresholdType] ?? + alert.thresholdType; return ( <> - If value is {alert.thresholdType === 'above' ? 'over' : 'under'}{' '} + If value {thresholdLabel}{' '} {alert.threshold} + {isRangeThresholdType(alert.thresholdType) && ( + <> + {' '} + and {alert.thresholdMax ?? '-'} + + )} · ); @@ -467,7 +193,12 @@ function AlertCardList({ alerts }: { alerts: AlertsPageItem[] }) { OK {okData.length === 0 && ( -
No alerts
+ } + title="No alerts" + description="All alerts in OK state will appear here." + /> )} {okData.map((alert, index) => ( @@ -484,41 +215,60 @@ export default function AlertsPage() { const alerts = React.useMemo(() => data?.data || [], [data?.data]); return ( -
+
Alerts - {brandName} Alerts -
- - } - color="gray" - py="xs" - mt="md" - > - Alerts can be{' '} - + {isLoading ? ( +
Loading...
+ ) : isError ? ( +
Error
+ ) : alerts?.length ? ( + + } + color="gray" + py="xs" + mt="md" > - created -
{' '} - from dashboard charts and saved searches. -
- {isLoading ? ( -
Loading...
- ) : isError ? ( -
Error
- ) : alerts?.length ? ( - <> - - - ) : ( -
No alerts created yet
- )} -
+ Alerts can be{' '} + + created + {' '} + from dashboard charts and saved searches. + + + + ) : ( + } + title="No alerts created yet" + description={ + <> + Alerts can be created from{' '} + + dashboard charts + {' '} + and{' '} + + saved searches + + . + + } + /> + )}
); diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index ed30e83b..20265625 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -373,8 +373,8 @@ export const ERROR_RATE_PERCENTAGE_NUMBER_FORMAT: NumberFormat = { }; export const K8S_CPU_PERCENTAGE_NUMBER_FORMAT: NumberFormat = { - output: 'percent', - mantissa: 0, + output: 'number', + mantissa: 2, }; export const K8S_FILESYSTEM_NUMBER_FORMAT: NumberFormat = { diff --git a/packages/app/src/ClickhousePage.tsx b/packages/app/src/ClickhousePage.tsx index 3c529341..afd45ba5 100644 --- a/packages/app/src/ClickhousePage.tsx +++ b/packages/app/src/ClickhousePage.tsx @@ -368,6 +368,7 @@ function InsertsTab({ } toolbarPrefix={[ { diff --git a/packages/app/src/DBChartPage.tsx b/packages/app/src/DBChartPage.tsx index 82f2e5e2..8b9224f2 100644 --- a/packages/app/src/DBChartPage.tsx +++ b/packages/app/src/DBChartPage.tsx @@ -35,6 +35,8 @@ import { useBrandDisplayName } from '@/theme/ThemeProvider'; import { parseTimeQuery, useNewTimeQuery } from '@/timeQuery'; import { useLocalStorage } from '@/utils'; +import OnboardingModal from './components/OnboardingModal'; + // Autocomplete can focus on column/map keys // Sampled field discovery and full field discovery @@ -164,7 +166,7 @@ function AIAssistant({ Experimental - + {opened && ( // eslint-disable-next-line react-hooks/refs
@@ -235,6 +237,7 @@ function DBChartExplorerPage() { Chart Explorer - {brandName} + - + {error.details} diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index a28f8a27..fe71f241 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -11,7 +11,7 @@ import dynamic from 'next/dynamic'; import Head from 'next/head'; import Link from 'next/link'; import { useRouter } from 'next/router'; -import { formatRelative } from 'date-fns'; +import { formatDistanceToNow, formatRelative } from 'date-fns'; import produce from 'immer'; import { pick } from 'lodash'; import { parseAsArrayOf, parseAsString, useQueryState } from 'nuqs'; @@ -19,7 +19,11 @@ import { ErrorBoundary } from 'react-error-boundary'; import RGL, { WidthProvider } from 'react-grid-layout'; import { useForm, useWatch } from 'react-hook-form'; import { TableConnection } from '@hyperdx/common-utils/dist/core/metadata'; -import { convertToDashboardTemplate } from '@hyperdx/common-utils/dist/core/utils'; +import { + convertToDashboardTemplate, + displayTypeSupportsBuilderAlerts, + displayTypeSupportsRawSqlAlerts, +} from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderChartConfig, isBuilderSavedChartConfig, @@ -50,15 +54,14 @@ import { Flex, Group, Indicator, - Input, Menu, Modal, Paper, + Stack, Text, - Title, Tooltip, } from '@mantine/core'; -import { useHotkeys, useHover } from '@mantine/hooks'; +import { useHotkeys } from '@mantine/hooks'; import { notifications } from '@mantine/notifications'; import { IconArrowsMaximize, @@ -125,6 +128,7 @@ import { } from './source'; import { parseTimeQuery, useNewTimeQuery } from './timeQuery'; import { useConfirm } from './useConfirm'; +import { FormatTime } from './useFormatTime'; import { getMetricTableName } from './utils'; import { useZIndex, ZIndexContext } from './zIndex'; @@ -219,10 +223,17 @@ const Tile = forwardRef( ChartConfigWithDateRange | undefined >(undefined); - const { data: source } = useSource({ + const { data: source, isFetched: isSourceFetched } = useSource({ id: chart.config.source, }); + const isSourceMissing = + !!chart.config.source && isSourceFetched && source == null; + const isSourceUnset = + !!chart.config && + isBuilderSavedChartConfig(chart.config) && + !chart.config.source; + useEffect(() => { if (isRawSqlSavedChartConfig(chart.config)) { // Some raw SQL charts don't have a source @@ -287,9 +298,7 @@ const Tile = forwardRef( const [hovered, setHovered] = useState(false); - const alert = isBuilderSavedChartConfig(chart.config) - ? chart.config.alert - : undefined; + const alert = chart.config.alert; const alertIndicatorColor = useMemo(() => { if (!alert) { return 'transparent'; @@ -358,37 +367,39 @@ const Tile = forwardRef( }, [filters, queriedConfig, source]); const hoverToolbar = useMemo(() => { + const isRawSql = isRawSqlSavedChartConfig(chart.config); + const displayTypeSupportsAlerts = isRawSql + ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) + : displayTypeSupportsBuilderAlerts(chart.config.displayType); return ( e.stopPropagation()} key="hover-toolbar" + my={4} // Margin to ensure that the Alert Indicator doesn't clip on non-Line/Bar display types style={{ visibility: hovered ? 'visible' : 'hidden' }} > - {(chart.config.displayType === DisplayType.Line || - chart.config.displayType === DisplayType.StackedBar || - chart.config.displayType === DisplayType.Number) && - !isRawSqlSavedChartConfig(chart.config) && ( - +} - mr={4} - > - - - - - - - )} + {displayTypeSupportsAlerts && ( + +} + mr={4} + > + + + + + + + )} } > - {(queriedConfig?.displayType === DisplayType.Line || - queriedConfig?.displayType === DisplayType.StackedBar) && ( - { - onUpdateChart?.({ - ...chart, - config: { - ...chart.config, - displayType, - }, - }); - }} - /> - )} - {queriedConfig?.displayType === DisplayType.Table && ( - - - buildTableRowSearchUrl({ - row, - source, - config: queriedConfig, - dateRange: dateRange, - }) - : undefined - } - /> - - )} - {queriedConfig?.displayType === DisplayType.Number && ( - - )} - {queriedConfig?.displayType === DisplayType.Pie && ( - - )} - {effectiveMarkdownConfig?.displayType === DisplayType.Markdown && - 'markdown' in effectiveMarkdownConfig && ( - - )} - {queriedConfig?.displayType === DisplayType.Search && - isBuilderChartConfig(queriedConfig) && - isBuilderSavedChartConfig(chart.config) && ( - - + + + The data source for this tile no longer exists. Edit the + tile to select a new source. + + + + ) : isSourceUnset ? ( + + + + The data source for this tile is not set. Edit the tile to + select a data source. + + + + ) : ( + <> + {(queriedConfig?.displayType === DisplayType.Line || + queriedConfig?.displayType === DisplayType.StackedBar) && ( + { + onUpdateChart?.({ + ...chart, + config: { + ...chart.config, + displayType, }, - ], - dateRange, - select: - queriedConfig.select || - (source?.kind === SourceKind.Log || - source?.kind === SourceKind.Trace - ? source.defaultTableSelectExpression - : '') || - '', - groupBy: undefined, - granularity: undefined, + }); }} - isLive={false} - queryKeyPrefix={'search'} - variant="muted" /> - - )} + )} + {queriedConfig?.displayType === DisplayType.Table && ( + + + buildTableRowSearchUrl({ + row, + source, + config: queriedConfig, + dateRange: dateRange, + }) + : undefined + } + /> + + )} + {queriedConfig?.displayType === DisplayType.Number && ( + + )} + {queriedConfig?.displayType === DisplayType.Pie && ( + + )} + {effectiveMarkdownConfig?.displayType === + DisplayType.Markdown && + 'markdown' in effectiveMarkdownConfig && ( + + )} + {queriedConfig?.displayType === DisplayType.Search && + isBuilderChartConfig(queriedConfig) && + isBuilderSavedChartConfig(chart.config) && ( + + + + )} + + )} ); }, @@ -629,6 +663,8 @@ const Tile = forwardRef( source, dateRange, filterWarning, + isSourceMissing, + isSourceUnset, ], ); @@ -1557,16 +1593,42 @@ function DBDashboardPage({ presetConfig }: { presetConfig?: Dashboard }) { ) : ( - - - Dashboards - - - {dashboard?.name ?? 'Untitled'} - - + + + + Dashboards + + + {dashboard?.name ?? 'Untitled'} + + + {!isLocalDashboard && dashboard && ( + + {dashboard.createdBy && ( + + Created by{' '} + {dashboard.createdBy.name || dashboard.createdBy.email}.{' '} + + )} + {dashboard.updatedAt && ( + + + {dashboard.updatedBy + ? ` by ${dashboard.updatedBy.name || dashboard.updatedBy.email}` + : ''} + + } + > + {`Updated ${formatDistanceToNow(new Date(dashboard.updatedAt), { addSuffix: true })}.`} + + )} + + )} + )} - + ) => { e.preventDefault(); - handleSubmit(({ name }) => { + handleSubmit(async ({ name }) => { if (isUpdate) { if (savedSearchId == null) { throw new Error('savedSearchId is required for update'); @@ -428,11 +437,20 @@ function SaveSearchModalComponent({ onSuccess: () => { onClose(); }, + onError: error => { + console.error('Error updating saved search:', error); + notifications.show({ + color: 'red', + title: 'Error', + message: + 'An error occurred while updating your saved search. Please try again.', + }); + }, }, ); } else { - createSavedSearch.mutate( - { + try { + const savedSearch = await createSavedSearch.mutateAsync({ name, select: effectiveSelect, where: searchedConfig.where ?? '', @@ -442,18 +460,25 @@ function SaveSearchModalComponent({ orderBy: searchedConfig.orderBy ?? '', filters: searchedConfig.filters ?? [], tags: tags, - }, - { - onSuccess: savedSearch => { - router.push(`/search/${savedSearch.id}${window.location.search}`); - onClose(); - }, - }, - ); + }); + + router.push(`/search/${savedSearch.id}${window.location.search}`); + onClose(); + } catch (error) { + console.error('Error creating saved search:', error); + notifications.show({ + color: 'red', + title: 'Error', + message: + 'An error occurred while saving your search. Please try again.', + }); + } } })(); }; + const isPending = createSavedSearch.isPending || updateSavedSearch.isPending; + const { data: chartConfig } = useSearchedConfigToChartConfig(searchedConfig); return ( @@ -568,6 +593,7 @@ function SaveSearchModalComponent({ variant="primary" type="submit" disabled={!formState.isValid} + loading={isPending} > {isUpdate ? 'Update' : 'Save'} @@ -783,7 +809,7 @@ const queryStateMap = { orderBy: parseAsStringEncoded, }; -function DBSearchPage() { +export function DBSearchPage() { const brandName = useBrandDisplayName(); // Next router is laggy behind window.location, which causes race // conditions with useQueryStates, so we'll parse it directly @@ -791,6 +817,10 @@ function DBSearchPage() { const savedSearchId = paths.length === 3 ? paths[2] : null; const [searchedConfig, setSearchedConfig] = useQueryStates(queryStateMap); + const [directTraceId, setDirectTraceId] = useQueryState( + 'traceId', + parseAsStringEncoded, + ); const { data: savedSearch } = useSavedSearch( { id: `${savedSearchId}` }, @@ -808,6 +838,14 @@ function DBSearchPage() { id: searchedConfig.source, kinds: [SourceKind.Log, SourceKind.Trace], }); + const directTraceSource = + directTraceId != null && searchedSource?.kind === SourceKind.Trace + ? searchedSource + : undefined; + const chartSourceId = + directTraceId != null && !directTraceSource + ? '' + : (searchedConfig.source ?? ''); const [analysisMode, setAnalysisMode] = useQueryState( 'mode', @@ -865,7 +903,9 @@ function DBSearchPage() { where: searchedConfig.where || '', whereLanguage: searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene', - source: searchedConfig.source || defaultSourceId, + source: + searchedConfig.source || + (savedSearchId || directTraceId ? '' : defaultSourceId), filters: searchedConfig.filters ?? [], orderBy: searchedConfig.orderBy ?? '', }, @@ -964,6 +1004,10 @@ function DBSearchPage() { return; } + if (savedSearchId == null && directTraceId != null && !source) { + return; + } + // Landed on a new search - ensure we have a source selected if (savedSearchId == null && defaultSourceId && isSearchConfigEmpty) { setSearchedConfig({ @@ -982,6 +1026,7 @@ function DBSearchPage() { setSearchedConfig, savedSearchId, defaultSourceId, + directTraceId, sources, ]); @@ -1104,9 +1149,28 @@ function DBSearchPage() { const [saveSearchModalState, setSaveSearchModalState] = useState< 'create' | 'update' | undefined >(undefined); + const chartSearchConfig = useMemo( + () => ({ + select: searchedConfig.select ?? '', + source: chartSourceId, + where: searchedConfig.where ?? '', + whereLanguage: + searchedConfig.whereLanguage ?? getStoredLanguage() ?? 'lucene', + filters: searchedConfig.filters ?? [], + orderBy: searchedConfig.orderBy ?? '', + }), + [ + chartSourceId, + searchedConfig.filters, + searchedConfig.orderBy, + searchedConfig.select, + searchedConfig.where, + searchedConfig.whereLanguage, + ], + ); const { data: chartConfig, isLoading: isChartConfigLoading } = - useSearchedConfigToChartConfig(searchedConfig, defaultSearchConfig); + useSearchedConfigToChartConfig(chartSearchConfig, defaultSearchConfig); // query error handling const { hasQueryError, queryError } = useMemo(() => { @@ -1347,17 +1411,67 @@ function DBSearchPage() { const [isAlertModalOpen, { open: openAlertModal, close: closeAlertModal }] = useDisclosure(); + const directTraceRangeAppliedRef = useRef(null); + const directTraceFilterAppliedRef = useRef(null); + + useEffect(() => { + if (!isReady || !directTraceId) { + directTraceRangeAppliedRef.current = null; + return; + } + + const searchParams = new URLSearchParams(window.location.search); + if (searchParams.has('from') && searchParams.has('to')) { + return; + } + + if (directTraceRangeAppliedRef.current === directTraceId) { + return; + } + + directTraceRangeAppliedRef.current = directTraceId; + setIsLive(false); + const [start, end] = getDefaultDirectTraceDateRange(); + onTimeRangeSelect(start, end, null); + }, [directTraceId, isReady, onTimeRangeSelect, setIsLive]); + + useEffect(() => { + if (!directTraceId || !directTraceSource) { + directTraceFilterAppliedRef.current = null; + return; + } + + const nextKey = `${directTraceSource.id}:${directTraceId}`; + if (directTraceFilterAppliedRef.current === nextKey) { + return; + } + + directTraceFilterAppliedRef.current = nextKey; + setIsLive(false); + setSearchedConfig({ + source: directTraceSource.id, + where: buildDirectTraceWhereClause( + directTraceSource.traceIdExpression, + directTraceId, + ), + whereLanguage: 'sql', + filters: [], + }); + }, [directTraceId, directTraceSource, setIsLive, setSearchedConfig]); - // Add this effect to trigger initial search when component mounts useEffect(() => { if (isReady && queryReady && !isChartConfigLoading) { // Only trigger if we haven't searched yet (no time range in URL) const searchParams = new URLSearchParams(window.location.search); - if (!searchParams.has('from') && !searchParams.has('to')) { + if ( + directTraceId == null && + !searchParams.has('from') && + !searchParams.has('to') + ) { onSearch('Live Tail'); } } - }, [isReady, queryReady, isChartConfigLoading, onSearch]); + }, [directTraceId, isReady, queryReady, isChartConfigLoading, onSearch]); const { data: aliasMap } = useAliasMapFromChartConfig(dbSqlRowTableConfig); @@ -1525,6 +1639,52 @@ function DBSearchPage() { }, [setIsLive, setInterval, onTimeRangeSelect], ); + const directTraceFocusDate = useMemo( + () => + new Date( + (searchedTimeRange[0].getTime() + searchedTimeRange[1].getTime()) / 2, + ), + [searchedTimeRange], + ); + + const onDirectTraceSourceChange = useCallback( + (sourceId: string | null) => { + setIsLive(false); + if (sourceId == null) { + directTraceFilterAppliedRef.current = null; + setSearchedConfig({ + source: null, + where: '', + whereLanguage: getStoredLanguage() ?? 'lucene', + filters: [], + }); + return; + } + + const nextSource = sources?.find( + (source): source is Extract => + source.id === sourceId && isTraceSource(source), + ); + if (!nextSource || !directTraceId) { + return; + } + + setSearchedConfig({ + source: nextSource.id, + where: buildDirectTraceWhereClause( + nextSource.traceIdExpression, + directTraceId, + ), + whereLanguage: 'sql', + filters: [], + }); + }, + [directTraceId, setIsLive, setSearchedConfig, sources], + ); + + const closeDirectTraceSidePanel = useCallback(() => { + setDirectTraceId(null); + }, [setDirectTraceId]); const clearSaveSearchModalState = useCallback( () => setSaveSearchModalState(undefined), @@ -1578,9 +1738,9 @@ function DBSearchPage() { )} {savedSearch && ( - - - + + + Saved Searches @@ -1588,53 +1748,82 @@ function DBSearchPage() { {savedSearch.name} - { - updateSavedSearch.mutate({ - id: savedSearch.id, - name: editedName, - }); - }} - /> - - - - - - - - - { - deleteSavedSearch.mutate(savedSearch?.id ?? '', { - onSuccess: () => { - router.push('/search/list'); - }, - }); - }} - onClickSaveAsNew={() => { - setSaveSearchModalState('create'); - }} - /> + + {savedSearch.createdBy && ( + + Created by{' '} + {savedSearch.createdBy.name || savedSearch.createdBy.email}.{' '} + + )} + {savedSearch.updatedAt && ( + + + {savedSearch.updatedBy + ? ` by ${savedSearch.updatedBy.name || savedSearch.updatedBy.email}` + : ''} + + } + > + {`Updated ${formatDistanceToNow(new Date(savedSearch.updatedAt), { addSuffix: true })}.`} + + )} + - + +
+ { + updateSavedSearch.mutate({ + id: savedSearch.id, + name: editedName, + }); + }} + /> +
+ + + + + + + + { + deleteSavedSearch.mutate(savedSearch?.id ?? '', { + onSuccess: () => { + router.push('/search/list'); + }, + }); + }} + onClickSaveAsNew={() => { + setSaveSearchModalState('create'); + }} + /> + +
+ )} )} + {!queryReady ? ( - -
- - Please start by selecting a source and then click the play - button to query data. - -
-
+ } + title="No data to display" + description="Select a source and click the play button to query data." + /> ) : ( <>
{whereSuggestions!.map(s => ( - <> + {s.userMessage('where')} @@ -1990,7 +2186,7 @@ function DBSearchPage() { Accept - + ))} diff --git a/packages/app/src/DBSearchPageAlertModal.tsx b/packages/app/src/DBSearchPageAlertModal.tsx index 3c9ede8c..a2fbc372 100644 --- a/packages/app/src/DBSearchPageAlertModal.tsx +++ b/packages/app/src/DBSearchPageAlertModal.tsx @@ -1,7 +1,6 @@ import React from 'react'; import router from 'next/router'; -import { useForm, useWatch } from 'react-hook-form'; -import { NativeSelect, NumberInput } from 'react-hook-form-mantine'; +import { Controller, useForm, useWatch } from 'react-hook-form'; import { z } from 'zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; @@ -11,24 +10,29 @@ import { AlertSource, AlertThresholdType, Filter, + isRangeThresholdType, scheduleStartAtSchema, SearchCondition, SearchConditionLanguage, validateAlertScheduleOffsetMinutes, + validateAlertThresholdMax, zAlertChannel, } from '@hyperdx/common-utils/dist/types'; -import { Alert as MantineAlert, TextInput } from '@mantine/core'; import { Accordion, + Alert as MantineAlert, Box, Button, Group, LoadingOverlay, Modal, + NativeSelect, + NumberInput, Paper, Stack, Tabs, Text, + TextInput, } from '@mantine/core'; import { notifications } from '@mantine/notifications'; import { @@ -53,6 +57,8 @@ import { import { AlertPreviewChart } from './components/AlertPreviewChart'; import { AlertChannelForm } from './components/Alerts'; +import { AckAlert } from './components/alerts/AckAlert'; +import { AlertHistoryCardList } from './components/alerts/AlertHistoryCards'; import { AlertScheduleFields } from './components/AlertScheduleFields'; import { getStoredLanguage } from './components/SearchInput/SearchWhereInput'; import { getWebhookChannelIcon } from './utils/webhookIcons'; @@ -64,13 +70,15 @@ const SavedSearchAlertFormSchema = z .object({ interval: AlertIntervalSchema, threshold: z.number(), + thresholdMax: z.number().optional(), scheduleOffsetMinutes: z.number().int().min(0).default(0), scheduleStartAt: scheduleStartAtSchema, thresholdType: z.nativeEnum(AlertThresholdType), channel: zAlertChannel, }) .passthrough() - .superRefine(validateAlertScheduleOffsetMinutes); + .superRefine(validateAlertScheduleOffsetMinutes) + .superRefine(validateAlertThresholdMax); const AlertForm = ({ sourceId, @@ -138,12 +146,16 @@ const AlertForm = ({ }); const groupByValue = useWatch({ control, name: 'groupBy' }); const threshold = useWatch({ control, name: 'threshold' }); + const thresholdMax = useWatch({ control, name: 'thresholdMax' }); const maxScheduleOffsetMinutes = Math.max( intervalToMinutes(interval ?? '5m') - 1, 0, ); const intervalLabel = ALERT_INTERVAL_OPTIONS[interval ?? '5m']; + const { data: alertData } = api.useAlert(defaultValues?.id); + const alert = alertData?.data; + return ( @@ -157,8 +169,8 @@ const AlertForm = ({ ), )} > - - + + Trigger @@ -166,35 +178,79 @@ const AlertForm = ({ Alert when - ( + { + field.onChange(e); + if ( + isRangeThresholdType(e.currentTarget.value) && + thresholdMax == null + ) { + setValue('thresholdMax', (threshold ?? 0) + 1); + } + }} + /> + )} /> - ( + + )} /> + {isRangeThresholdType(thresholdType as AlertThresholdType) && ( + <> + + and + + ( + + )} + /> + + )} lines appear within - ( + + )} /> via - ( + + )} /> - - Send to + {groupBy && + (thresholdType === AlertThresholdType.BELOW || + thresholdType === AlertThresholdType.BELOW_OR_EQUAL || + thresholdType === AlertThresholdType.EQUAL || + thresholdType === AlertThresholdType.NOT_EQUAL) && ( + } + color="gray" + py="xs" + > + + Warning: Alerts with this threshold type and a "grouped + by" value will not alert for periods with no data for a + group. + + + )} + {(thresholdType === AlertThresholdType.EQUAL || + thresholdType === AlertThresholdType.NOT_EQUAL) && ( + } + color="gray" + py="xs" + > + + Note: Floating-point query results are not rounded during + equality comparison. + + + )} + + + + {(defaultValues?.createdBy || alert) && ( + + + {defaultValues?.createdBy && ( + + + Created by + + + {defaultValues.createdBy.name || + defaultValues.createdBy.email} + + {defaultValues.createdBy.name && ( + + {defaultValues.createdBy.email} + + )} + + )} + {alert && ( + + {alert.history.length > 0 && ( + + )} + + + )} + - {groupBy && thresholdType === AlertThresholdType.BELOW && ( - } - bg="dark" - py="xs" - > - - Warning: Alerts with a "Below (<)" threshold and a - "grouped by" value will not alert for periods with no - data for a group. - - - )} - + )} @@ -255,6 +357,7 @@ const AlertForm = ({ interval={interval} groupBy={groupByValue} threshold={threshold} + thresholdMax={thresholdMax} thresholdType={thresholdType} /> )} @@ -262,22 +365,6 @@ const AlertForm = ({ - {defaultValues?.createdBy && ( - - - Created by - - - {defaultValues.createdBy.name || defaultValues.createdBy.email} - - {defaultValues.createdBy.name && ( - - {defaultValues.createdBy.email} - - )} - - )} -
{defaultValues && ( @@ -404,6 +491,7 @@ export const DBSearchPageAlertModal = ({ }); } } catch (error) { + console.error('Error creating/updating alert:', error); notifications.show({ color: 'red', message: `Something went wrong. Please contact ${brandName} team.`, @@ -423,6 +511,7 @@ export const DBSearchPageAlertModal = ({ autoClose: 5000, }); } catch (error) { + console.error('Failed to delete alert:', error); notifications.show({ color: 'red', message: `Something went wrong. Please contact ${brandName} team.`, @@ -440,7 +529,6 @@ export const DBSearchPageAlertModal = ({ onClose={onClose} size="xl" withCloseButton={false} - zIndex={9999} > ( + <> + + Service Map - {brandName} + + + + ), + [brandName], + ); + if (!isLoading && !hasTraceSources) { return ( + {head} Service Map @@ -128,20 +139,13 @@ function DBServiceMapPage() { /> )} - } + title="No trace sources configured" + description="The Service Map visualizes relationships between your services using trace data. Configure a trace source to get started." + maw={600} > - - No trace sources configured - - - The Service Map visualizes relationships between your services using - trace data. Configure a trace source to get started. - {IS_LOCAL_MODE ? ( + +
+ ); + }, +})); + +jest.mock('@/components/DBSearchPageFilters', () => ({ + DBSearchPageFilters: () =>
, +})); + +jest.mock('@/components/DBTimeChart', () => ({ + DBTimeChart: () =>
, +})); + +jest.mock('@/components/ActiveFilterPills', () => ({ + ActiveFilterPills: () =>
, +})); +jest.mock('@/components/ContactSupportText', () => ({ + ContactSupportText: () =>
, +})); +jest.mock('@/components/FavoriteButton', () => ({ + FavoriteButton: () =>
, +})); +jest.mock('@/components/InputControlled', () => ({ + InputControlled: () =>
, +})); +jest.mock('@/components/OnboardingModal', () => () =>
); +jest.mock('@/components/SearchInput/SearchWhereInput', () => ({ + __esModule: true, + default: () =>
, + getStoredLanguage: () => 'lucene', +})); +jest.mock('@/components/SearchPageActionBar', () => () =>
); +jest.mock('@/components/SearchTotalCountChart', () => () =>
); +jest.mock('@/components/Sources/SourceForm', () => ({ + TableSourceForm: () =>
, +})); +jest.mock('@/components/SourceSelect', () => ({ + SourceSelectControlled: () =>
, +})); +jest.mock('@/components/SQLEditor/SQLInlineEditor', () => ({ + SQLInlineEditorControlled: () =>
, +})); +jest.mock('@/components/Tags', () => ({ + Tags: ({ children }: { children: React.ReactNode }) =>
{children}
, +})); +jest.mock('@/components/TimePicker', () => ({ + TimePicker: () =>
, +})); +jest.mock('../components/ChartSQLPreview', () => ({ + SQLPreview: () =>
, +})); +jest.mock('../components/DBSqlRowTableWithSidebar', () => () =>
); +jest.mock('../components/PatternTable', () => () =>
); +jest.mock('../components/Search/DBSearchHeatmapChart', () => ({ + DBSearchHeatmapChart: () =>
, +})); +jest.mock('../components/SourceSchemaPreview', () => () =>
); +jest.mock('../components/Error/ErrorBoundary', () => ({ + ErrorBoundary: ({ children }: { children: React.ReactNode }) => ( + <>{children} + ), +})); + +jest.mock('../utils/queryParsers', () => ({ + parseAsJsonEncoded: () => 'parseAsJsonEncoded', + parseAsSortingStateString: { + parse: () => null, + }, + parseAsStringEncoded: 'parseAsStringEncoded', +})); + +jest.mock('../api', () => ({ + __esModule: true, + default: { + useMe: () => ({ + data: { team: {} }, + isSuccess: true, + }), + }, +})); + +jest.mock('@/utils', () => ({ + QUERY_LOCAL_STORAGE: 'query-local-storage', + useLocalStorage: (_key: string, initialValue: unknown) => [ + initialValue, + jest.fn(), + ], + usePrevious: (value: unknown) => value, +})); + +jest.mock('@tanstack/react-query', () => ({ + useIsFetching: () => 0, +})); + +describe('DBSearchPage direct trace flow', () => { + beforeEach(() => { + jest.clearAllMocks(); + latestDirectTracePanelProps = null; + mockDirectTraceId = 'trace-123'; + mockSearchedConfig = { + source: undefined, + where: '', + select: '', + whereLanguage: undefined, + filters: [], + orderBy: '', + }; + mockSources = [ + { + id: 'trace-source', + kind: SourceKind.Trace, + name: 'Trace Source', + traceIdExpression: 'TraceId', + from: { databaseName: 'db', tableName: 'traces' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp', + implicitColumnExpression: 'Body', + connection: 'conn', + logSourceId: 'log-source', + }, + { + id: 'log-source', + kind: SourceKind.Log, + name: 'Log Source', + from: { databaseName: 'db', tableName: 'logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp', + implicitColumnExpression: 'Body', + connection: 'conn', + }, + ]; + }); + + it('opens the direct trace panel with no selected source when none is provided', async () => { + window.history.pushState({}, '', '/search?traceId=trace-123'); + + renderWithMantine(); + + await waitFor(() => { + expect(latestDirectTracePanelProps).toEqual( + expect.objectContaining({ + traceId: 'trace-123', + traceSourceId: null, + }), + ); + }); + }); + + it('applies a direct trace filter when a valid trace source is present', async () => { + mockSearchedConfig = { + ...mockSearchedConfig, + source: 'trace-source', + }; + window.history.pushState( + {}, + '', + '/search?traceId=trace-123&source=trace-source', + ); + + renderWithMantine(); + + await waitFor(() => { + expect(mockSetSearchedConfig).toHaveBeenCalledWith( + expect.objectContaining({ + source: 'trace-source', + where: "TraceId = 'trace-123'", + whereLanguage: 'sql', + filters: [], + }), + ); + }); + }); + + it('applies the default 14-day range only when from/to are absent', () => { + window.history.pushState({}, '', '/search?traceId=trace-123'); + + renderWithMantine(); + + expect(mockOnTimeRangeSelect).toHaveBeenCalled(); + + jest.clearAllMocks(); + window.history.pushState({}, '', '/search?traceId=trace-123&from=1&to=2'); + + renderWithMantine(); + + expect(mockOnTimeRangeSelect).not.toHaveBeenCalled(); + }); + + it('lets the direct trace panel update the selected source', async () => { + window.history.pushState({}, '', '/search?traceId=trace-123'); + + renderWithMantine(); + + await waitFor(() => { + expect(screen.getByTestId('direct-trace-panel')).toBeInTheDocument(); + }); + + screen.getByText('select-trace-source').click(); + + expect(mockSetSearchedConfig).toHaveBeenCalledWith( + expect.objectContaining({ + source: 'trace-source', + where: "TraceId = 'trace-123'", + whereLanguage: 'sql', + filters: [], + }), + ); + }); + + it('clears the direct trace mode when the panel closes', async () => { + window.history.pushState({}, '', '/search?traceId=trace-123'); + + renderWithMantine(); + + await waitFor(() => { + expect(screen.getByTestId('direct-trace-panel')).toBeInTheDocument(); + }); + + screen.getByText('close-trace').click(); + + expect(mockSetDirectTraceId).toHaveBeenCalledWith(null); + }); +}); diff --git a/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx b/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx index c0b6309a..6b1b47ae 100644 --- a/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx +++ b/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx @@ -39,6 +39,7 @@ jest.mock('@/hooks/useChartConfig', () => ({ jest.mock('@/source', () => ({ useSource: () => ({ data: null, isLoading: false }), + useResolvedNumberFormat: () => undefined, })); jest.mock('@/ChartUtils', () => ({ diff --git a/packages/app/src/__tests__/TraceRedirectPage.test.tsx b/packages/app/src/__tests__/TraceRedirectPage.test.tsx new file mode 100644 index 00000000..d819ae49 --- /dev/null +++ b/packages/app/src/__tests__/TraceRedirectPage.test.tsx @@ -0,0 +1,81 @@ +import React from 'react'; +import { waitFor } from '@testing-library/react'; + +import { TraceRedirectPage } from '../../pages/trace/[traceId]'; + +const mockReplace = jest.fn(); + +let mockRouter = { + isReady: true, + query: { + traceId: 'trace-123', + }, + replace: mockReplace, +}; + +jest.mock('next/router', () => ({ + useRouter: () => mockRouter, +})); + +jest.mock('@/layout', () => ({ + withAppNav: (component: unknown) => component, +})); + +describe('TraceRedirectPage', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = { + isReady: true, + query: { + traceId: 'trace-123', + }, + replace: mockReplace, + }; + }); + + it('redirects to search with the trace id query param', async () => { + window.history.pushState({}, '', '/trace/trace-123'); + + renderWithMantine(); + + await waitFor(() => { + expect(mockReplace).toHaveBeenCalledWith('/search?traceId=trace-123'); + }); + }); + + it('preserves existing query params such as source', async () => { + window.history.pushState({}, '', '/trace/trace-123?source=trace-source'); + + renderWithMantine(); + + await waitFor(() => { + expect(mockReplace).toHaveBeenCalledWith( + '/search?source=trace-source&traceId=trace-123', + ); + }); + }); + + it('redirects after router readiness changes', async () => { + mockRouter = { + ...mockRouter, + isReady: false, + }; + window.history.pushState({}, '', '/trace/trace-123'); + + const { unmount } = renderWithMantine(); + + expect(mockReplace).not.toHaveBeenCalled(); + + mockRouter = { + ...mockRouter, + isReady: true, + }; + + unmount(); + renderWithMantine(); + + await waitFor(() => { + expect(mockReplace).toHaveBeenCalledWith('/search?traceId=trace-123'); + }); + }); +}); diff --git a/packages/app/src/__tests__/config.test.ts b/packages/app/src/__tests__/config.test.ts new file mode 100644 index 00000000..e72d757b --- /dev/null +++ b/packages/app/src/__tests__/config.test.ts @@ -0,0 +1,83 @@ +import { parseResourceAttributes } from '@/config'; + +describe('parseResourceAttributes', () => { + it('parses a standard comma-separated string', () => { + const raw = + 'service.namespace=observability,deployment.environment=prod,k8s.cluster.name=us-west-2'; + expect(parseResourceAttributes(raw)).toEqual({ + 'service.namespace': 'observability', + 'deployment.environment': 'prod', + 'k8s.cluster.name': 'us-west-2', + }); + }); + + it('returns an empty object for an empty string', () => { + expect(parseResourceAttributes('')).toEqual({}); + }); + + it('handles a single key=value pair', () => { + expect(parseResourceAttributes('foo=bar')).toEqual({ foo: 'bar' }); + }); + + it('handles values containing equals signs', () => { + expect(parseResourceAttributes('url=https://example.com?a=1')).toEqual({ + url: 'https://example.com?a=1', + }); + }); + + it('skips malformed entries without an equals sign', () => { + expect(parseResourceAttributes('good=value,badentry,ok=yes')).toEqual({ + good: 'value', + ok: 'yes', + }); + }); + + it('skips entries where key is empty (leading equals)', () => { + expect(parseResourceAttributes('=nokey,valid=value')).toEqual({ + valid: 'value', + }); + }); + + it('handles trailing commas gracefully', () => { + expect(parseResourceAttributes('a=1,b=2,')).toEqual({ a: '1', b: '2' }); + }); + + it('handles leading commas gracefully', () => { + expect(parseResourceAttributes(',a=1,b=2')).toEqual({ a: '1', b: '2' }); + }); + + it('allows empty values', () => { + expect(parseResourceAttributes('key=')).toEqual({ key: '' }); + }); + + it('last value wins for duplicate keys', () => { + expect(parseResourceAttributes('k=first,k=second')).toEqual({ + k: 'second', + }); + }); + + it('decodes percent-encoded commas in values', () => { + expect(parseResourceAttributes('tags=a%2Cb%2Cc')).toEqual({ + tags: 'a,b,c', + }); + }); + + it('decodes percent-encoded equals in values', () => { + expect(parseResourceAttributes('expr=x%3D1')).toEqual({ + expr: 'x=1', + }); + }); + + it('decodes percent-encoded keys', () => { + expect(parseResourceAttributes('my%2Ekey=value')).toEqual({ + 'my.key': 'value', + }); + }); + + it('round-trips values with both encoded commas and equals', () => { + expect(parseResourceAttributes('q=a%3D1%2Cb%3D2,other=plain')).toEqual({ + q: 'a=1,b=2', + other: 'plain', + }); + }); +}); diff --git a/packages/app/src/__tests__/localStore.test.ts b/packages/app/src/__tests__/localStore.test.ts index e18838c1..34a027da 100644 --- a/packages/app/src/__tests__/localStore.test.ts +++ b/packages/app/src/__tests__/localStore.test.ts @@ -16,6 +16,7 @@ jest.mock('../utils', () => ({ import { createEntityStore, + generateDeterministicId, localSavedSearches, localSources, } from '../localStore'; @@ -104,6 +105,56 @@ describe('createEntityStore', () => { expect(store.getAll()).toHaveLength(3); }); + + describe('with generateDeterministicId', () => { + it('generates deterministic ids from item content', () => { + const storeA = createEntityStore( + 'store-det-a', + undefined, + generateDeterministicId, + ); + const storeB = createEntityStore( + 'store-det-b', + undefined, + generateDeterministicId, + ); + + const a = storeA.create({ name: 'demo-source' }); + const b = storeB.create({ name: 'demo-source' }); + + expect(a.id).toBe(b.id); + }); + + it('generates the same id regardless of property insertion order', () => { + type MultiProp = { id: string; name: string; kind: string }; + const storeA = createEntityStore( + 'store-ord-a', + undefined, + generateDeterministicId, + ); + const storeB = createEntityStore( + 'store-ord-b', + undefined, + generateDeterministicId, + ); + + const a = storeA.create({ name: 'demo', kind: 'log' }); + const b = storeB.create({ kind: 'log', name: 'demo' }); + + expect(a.id).toBe(b.id); + }); + + it('generates different ids for items with different content', () => { + const store = createEntityStore( + TEST_KEY, + undefined, + generateDeterministicId, + ); + const a = store.create({ name: 'alpha' }); + const b = store.create({ name: 'beta' }); + expect(a.id).not.toBe(b.id); + }); + }); }); describe('update', () => { @@ -273,7 +324,6 @@ describe('localSources', () => { const envDefaults = [{ id: 'env-src', name: 'Env Source' }]; mockedConfig.HDX_LOCAL_DEFAULT_SOURCES = JSON.stringify(envDefaults); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const stored = localSources.create({ name: 'My Source' } as Omit< TSource, 'id' diff --git a/packages/app/src/__tests__/pinnedFilters.test.ts b/packages/app/src/__tests__/pinnedFilters.test.ts new file mode 100644 index 00000000..564ce143 --- /dev/null +++ b/packages/app/src/__tests__/pinnedFilters.test.ts @@ -0,0 +1,169 @@ +/** + * Tests for pinned filters logic: + * - mergePinnedData (exported from searchFilters.tsx) + * - localStorage migration logic + */ + +import { mergePinnedData } from '../searchFilters'; + +describe('mergePinnedData', () => { + it('returns empty fields and filters when both are null', () => { + const result = mergePinnedData(null, null); + expect(result.fields).toEqual([]); + expect(result.filters).toEqual({}); + }); + + it('returns team data when personal is null', () => { + const team = { + fields: ['ServiceName'], + filters: { ServiceName: ['web', 'api'] }, + }; + const result = mergePinnedData(team, null); + expect(result.fields).toEqual(['ServiceName']); + expect(result.filters).toEqual({ ServiceName: ['web', 'api'] }); + }); + + it('returns personal data when team is null', () => { + const personal = { + fields: ['level'], + filters: { level: ['error'] }, + }; + const result = mergePinnedData(null, personal); + expect(result.fields).toEqual(['level']); + expect(result.filters).toEqual({ level: ['error'] }); + }); + + it('unions fields from both team and personal', () => { + const team = { fields: ['ServiceName', 'level'], filters: {} }; + const personal = { fields: ['level', 'host'], filters: {} }; + const result = mergePinnedData(team, personal); + expect(result.fields).toEqual(['ServiceName', 'level', 'host']); + }); + + it('unions filter values and deduplicates', () => { + const team = { fields: [], filters: { ServiceName: ['web', 'api'] } }; + const personal = { + fields: [], + filters: { ServiceName: ['api', 'worker'] }, + }; + const result = mergePinnedData(team, personal); + expect(result.filters.ServiceName).toEqual(['web', 'api', 'worker']); + }); + + it('merges filter keys that only exist in one side', () => { + const team = { fields: [], filters: { ServiceName: ['web'] } }; + const personal = { fields: [], filters: { level: ['error'] } }; + const result = mergePinnedData(team, personal); + expect(result.filters).toEqual({ + ServiceName: ['web'], + level: ['error'], + }); + }); + + it('handles boolean values in filters', () => { + const team = { fields: [], filters: { isRootSpan: [true] } }; + const personal = { fields: [], filters: { isRootSpan: [false] } }; + const result = mergePinnedData(team, personal); + expect(result.filters.isRootSpan).toEqual([true, false]); + }); + + it('does not duplicate boolean values', () => { + const team = { fields: [], filters: { isRootSpan: [true] } }; + const personal = { fields: [], filters: { isRootSpan: [true] } }; + const result = mergePinnedData(team, personal); + expect(result.filters.isRootSpan).toEqual([true]); + }); +}); + +describe('localStorage migration', () => { + beforeEach(() => { + window.localStorage.clear(); + }); + + afterEach(() => { + window.localStorage.clear(); + }); + + it('reads pinned filters from localStorage correctly', () => { + const sourceId = 'source123'; + const storedFilters = { + [sourceId]: { ServiceName: ['web', 'api'] }, + }; + const storedFields = { + [sourceId]: ['ServiceName', 'level'], + }; + + window.localStorage.setItem( + 'hdx-pinned-search-filters', + JSON.stringify(storedFilters), + ); + window.localStorage.setItem( + 'hdx-pinned-fields', + JSON.stringify(storedFields), + ); + + const filtersRaw = window.localStorage.getItem('hdx-pinned-search-filters'); + const fieldsRaw = window.localStorage.getItem('hdx-pinned-fields'); + + const filters = filtersRaw ? JSON.parse(filtersRaw) : {}; + const fields = fieldsRaw ? JSON.parse(fieldsRaw) : {}; + + expect(filters[sourceId]).toEqual({ ServiceName: ['web', 'api'] }); + expect(fields[sourceId]).toEqual(['ServiceName', 'level']); + }); + + it('handles missing localStorage keys gracefully', () => { + const filtersRaw = window.localStorage.getItem('hdx-pinned-search-filters'); + const fieldsRaw = window.localStorage.getItem('hdx-pinned-fields'); + + expect(filtersRaw).toBeNull(); + expect(fieldsRaw).toBeNull(); + + const filters = filtersRaw ? JSON.parse(filtersRaw) : {}; + const fields = fieldsRaw ? JSON.parse(fieldsRaw) : {}; + + expect(filters).toEqual({}); + expect(fields).toEqual({}); + }); + + it('handles corrupted localStorage data gracefully', () => { + window.localStorage.setItem('hdx-pinned-search-filters', 'not-valid-json'); + + expect(() => { + try { + const raw = window.localStorage.getItem('hdx-pinned-search-filters'); + JSON.parse(raw!); + } catch { + // Migration should catch this and continue + } + }).not.toThrow(); + }); + + it('cleans up localStorage for a specific source after migration', () => { + const sourceA = 'sourceA'; + const sourceB = 'sourceB'; + + const storedFilters = { + [sourceA]: { ServiceName: ['web'] }, + [sourceB]: { level: ['error'] }, + }; + window.localStorage.setItem( + 'hdx-pinned-search-filters', + JSON.stringify(storedFilters), + ); + + // Simulate cleanup for sourceA (as the migration would do) + const updated: Record = { ...storedFilters }; + delete updated[sourceA]; + window.localStorage.setItem( + 'hdx-pinned-search-filters', + JSON.stringify(updated), + ); + + const result = JSON.parse( + window.localStorage.getItem('hdx-pinned-search-filters')!, + ); + expect(result[sourceA]).toBeUndefined(); + expect(result[sourceB]).toEqual({ level: ['error'] }); + }); +}); diff --git a/packages/app/src/__tests__/source.test.ts b/packages/app/src/__tests__/source.test.ts index dccf3475..606122e2 100644 --- a/packages/app/src/__tests__/source.test.ts +++ b/packages/app/src/__tests__/source.test.ts @@ -1,31 +1,260 @@ -import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; +import { + SourceKind, + TLogSource, + TTraceSource, +} from '@hyperdx/common-utils/dist/types'; +import { notifications } from '@mantine/notifications'; -import { getEventBody } from '../source'; +import { + getEventBody, + getSourceValidationNotificationId, + getTraceDurationNumberFormat, + useSources, +} from '../source'; + +jest.mock('../api', () => ({ hdxServer: jest.fn() })); +jest.mock('../config', () => ({ IS_LOCAL_MODE: false })); +jest.mock('@mantine/notifications', () => ({ + notifications: { show: jest.fn() }, +})); +jest.mock('@tanstack/react-query', () => ({ + useQuery: jest.fn(), + useMutation: jest.fn(), + useQueryClient: jest.fn(), +})); + +import { useQuery } from '@tanstack/react-query'; + +import { hdxServer } from '../api'; + +const TRACE_SOURCE: TTraceSource = { + kind: SourceKind.Trace, + from: { + databaseName: 'default', + tableName: 'otel_traces', + }, + timestampValueExpression: 'Timestamp', + connection: 'test-connection', + name: 'Traces', + id: 'test-source-id', + spanNameExpression: 'SpanName', + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanKindExpression: 'SpanKind', + defaultTableSelectExpression: 'Timestamp, ServiceName', +} as TTraceSource; describe('getEventBody', () => { - // Added to prevent regression back to HDX-3361 it('returns spanNameExpression for trace kind source when both bodyExpression and spanNameExpression are present', () => { - const source = { - kind: SourceKind.Trace, - from: { - databaseName: 'default', - tableName: 'otel_traces', - }, - timestampValueExpression: 'Timestamp', - connection: 'test-connection', - name: 'Traces', - id: 'test-source-id', - spanNameExpression: 'SpanName', - durationExpression: 'Duration', - durationPrecision: 9, - traceIdExpression: 'TraceId', - spanIdExpression: 'SpanId', - parentSpanIdExpression: 'ParentSpanId', - spanKindExpression: 'SpanKind', - } as TTraceSource; - - const result = getEventBody(source); - + const result = getEventBody(TRACE_SOURCE); expect(result).toBe('SpanName'); }); }); + +describe('getTraceDurationNumberFormat', () => { + it('returns undefined for non-trace sources', () => { + const logSource = { + kind: SourceKind.Log, + id: 'log-source', + } as TLogSource; + const result = getTraceDurationNumberFormat(logSource, [ + { valueExpression: 'count()' }, + ]); + expect(result).toBeUndefined(); + }); + + it('returns undefined when source is undefined', () => { + const result = getTraceDurationNumberFormat(undefined, [ + { valueExpression: 'count()' }, + ]); + expect(result).toBeUndefined(); + }); + + it('returns undefined when select expressions do not reference duration', () => { + const result = getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'count()' }, + ]); + expect(result).toBeUndefined(); + }); + + // --- exact match --- + + it('matches when valueExpression exactly equals durationExpression', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration', aggFn: 'avg' }, + ]), + ).toEqual({ output: 'duration', factor: 1e-9 }); + }); + + it('matches without aggFn (raw expression passed through)', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration' }, + ]), + ).toEqual({ output: 'duration', factor: 1e-9 }); + }); + + // --- non-matching expressions --- + + it('does not match expressions that only contain the duration name', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'avg(Duration)' }, + ]), + ).toBeUndefined(); + }); + + it('does not match division expressions', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration/1e6' }, + ]), + ).toBeUndefined(); + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: '(Duration)/1e6' }, + ]), + ).toBeUndefined(); + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration / 1e9' }, + ]), + ).toBeUndefined(); + }); + + it('does not match modified or similar-named expressions', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration * 2' }, + ]), + ).toBeUndefined(); + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'LongerDuration' }, + ]), + ).toBeUndefined(); + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'round(Duration / 1e6, 2)' }, + ]), + ).toBeUndefined(); + }); + + // --- aggFn filtering --- + + it('returns undefined for count aggFn', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration', aggFn: 'count' }, + ]), + ).toBeUndefined(); + }); + + it('returns undefined for count_distinct aggFn', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration', aggFn: 'count_distinct' }, + ]), + ).toBeUndefined(); + }); + + it.each(['sum', 'min', 'max', 'quantile', 'avg', 'any', 'last_value'])( + 'detects duration with %s aggFn', + aggFn => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration', aggFn }, + ]), + ).toEqual({ output: 'duration', factor: 1e-9 }); + }, + ); + + it('detects duration with combinator aggFn like avgIf', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration', aggFn: 'avgIf' }, + ]), + ).toEqual({ output: 'duration', factor: 1e-9 }); + }); + + it('skips non-preserving aggFn and detects preserving one in mixed selects', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration', aggFn: 'count' }, + { valueExpression: 'Duration', aggFn: 'avg' }, + ]), + ).toEqual({ output: 'duration', factor: 1e-9 }); + }); + + it('returns undefined when only non-preserving aggFns reference duration', () => { + expect( + getTraceDurationNumberFormat(TRACE_SOURCE, [ + { valueExpression: 'Duration', aggFn: 'count' }, + { valueExpression: 'Duration', aggFn: 'count_distinct' }, + ]), + ).toBeUndefined(); + }); + + it('returns undefined when select is empty', () => { + expect(getTraceDurationNumberFormat(TRACE_SOURCE, [])).toBeUndefined(); + }); +}); + +describe('useSources validation notifications', () => { + const mockUseQuery = useQuery as jest.Mock; + const mockHdxServer = hdxServer as jest.Mock; + const mockShow = notifications.show as jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('reuses the same notification id for repeated validation errors', async () => { + let capturedQueryFn: (() => Promise) | undefined; + mockUseQuery.mockImplementation(({ queryFn }) => { + capturedQueryFn = queryFn; + return { data: [] }; + }); + + const invalidSource = { + id: 'source-1', + kind: SourceKind.Log, + name: 'Broken Source', + connection: 'conn-1', + from: { databaseName: 'default', tableName: 'logs' }, + timestampValueExpression: 'Timestamp', + // Intentionally invalid for SourceSchema to trigger validation error. + serviceNameExpression: 42, + }; + + mockHdxServer.mockReturnValue({ + json: jest.fn().mockResolvedValue([invalidSource]), + }); + + useSources(); + expect(capturedQueryFn).toBeDefined(); + + await capturedQueryFn?.(); + await capturedQueryFn?.(); + + expect(mockShow).toHaveBeenCalledTimes(2); + expect(mockShow).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + id: getSourceValidationNotificationId('source-1'), + autoClose: false, + }), + ); + expect(mockShow).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + id: getSourceValidationNotificationId('source-1'), + autoClose: false, + }), + ); + }); +}); diff --git a/packages/app/src/__tests__/timeQuery.test.tsx b/packages/app/src/__tests__/timeQuery.test.tsx index 40cf31c9..9edf4bf0 100644 --- a/packages/app/src/__tests__/timeQuery.test.tsx +++ b/packages/app/src/__tests__/timeQuery.test.tsx @@ -1,3 +1,4 @@ +/* eslint-disable @eslint-react/no-create-ref */ import * as React from 'react'; import { useImperativeHandle } from 'react'; import { useRouter } from 'next/router'; diff --git a/packages/app/src/__tests__/utils.test.ts b/packages/app/src/__tests__/utils.test.ts index 26ac4277..5f9e3344 100644 --- a/packages/app/src/__tests__/utils.test.ts +++ b/packages/app/src/__tests__/utils.test.ts @@ -6,6 +6,7 @@ import { MetricsDataType, NumberFormat } from '../types'; import * as utils from '../utils'; import { formatAttributeClause, + formatDurationMs, formatNumber, getAllMetricTables, getMetricTableName, @@ -357,6 +358,68 @@ describe('formatNumber', () => { }); }); + describe('duration format', () => { + it('formats seconds input as adaptive duration', () => { + const format: NumberFormat = { + output: 'duration', + factor: 1, + }; + expect(formatNumber(30.41, format)).toBe('30.41s'); + expect(formatNumber(0.045, format)).toBe('45ms'); + expect(formatNumber(3661, format)).toBe('1.02h'); + }); + + it('formats milliseconds input as adaptive duration', () => { + const format: NumberFormat = { + output: 'duration', + factor: 0.001, + }; + expect(formatNumber(30410, format)).toBe('30.41s'); + expect(formatNumber(45, format)).toBe('45ms'); + }); + + it('formats nanoseconds input as adaptive duration', () => { + const format: NumberFormat = { + output: 'duration', + factor: 0.000000001, + }; + expect(formatNumber(30410000000, format)).toBe('30.41s'); + expect(formatNumber(45000000, format)).toBe('45ms'); + expect(formatNumber(500, format)).toBe('0.5µs'); + }); + + it('handles zero value', () => { + const format: NumberFormat = { + output: 'duration', + factor: 1, + }; + expect(formatNumber(0, format)).toBe('0ms'); + }); + + it('defaults factor to 1 (seconds) when not specified', () => { + const format: NumberFormat = { + output: 'duration', + }; + expect(formatNumber(1.5, format)).toBe('1.5s'); + }); + + it('formats sub-millisecond values as microseconds', () => { + const format: NumberFormat = { + output: 'duration', + factor: 1, + }; + expect(formatNumber(0.0003, format)).toBe('300µs'); + }); + + it('formats large values as hours', () => { + const format: NumberFormat = { + output: 'duration', + factor: 1, + }; + expect(formatNumber(7200, format)).toBe('2h'); + }); + }); + describe('unit handling', () => { it('appends unit to formatted number', () => { const format: NumberFormat = { @@ -596,6 +659,49 @@ describe('formatNumber', () => { }); }); +describe('formatDurationMs', () => { + it('formats zero', () => { + expect(formatDurationMs(0)).toBe('0ms'); + }); + + it('formats microseconds', () => { + expect(formatDurationMs(0.5)).toBe('500µs'); + expect(formatDurationMs(0.003)).toBe('3µs'); + expect(formatDurationMs(0.01)).toBe('10µs'); + }); + + it('formats milliseconds', () => { + expect(formatDurationMs(1)).toBe('1ms'); + expect(formatDurationMs(45)).toBe('45ms'); + expect(formatDurationMs(999)).toBe('999ms'); + expect(formatDurationMs(5.5)).toBe('5.5ms'); + }); + + it('formats seconds', () => { + expect(formatDurationMs(1000)).toBe('1s'); + expect(formatDurationMs(1500)).toBe('1.5s'); + expect(formatDurationMs(30410)).toBe('30.41s'); + }); + + it('formats minutes', () => { + expect(formatDurationMs(60000)).toBe('1min'); + expect(formatDurationMs(90000)).toBe('1.5min'); + }); + + it('formats hours', () => { + expect(formatDurationMs(3600000)).toBe('1h'); + expect(formatDurationMs(7200000)).toBe('2h'); + }); + + it('handles negative values', () => { + expect(formatDurationMs(-1500)).toBe('-1.5s'); + }); + + it('handles sub-microsecond precision', () => { + expect(formatDurationMs(0.0005)).toBe('0.5µs'); + }); +}); + describe('useLocalStorage', () => { // Create a mock for localStorage let localStorageMock: jest.Mocked; diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index d80f7771..bfa195eb 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -3,6 +3,7 @@ import type { HTTPError, Options, ResponsePromise } from 'ky'; import ky from 'ky-universal'; import type { Alert, + AlertApiResponse, AlertsApiResponse, InstallationApiResponse, MeApiResponse, @@ -234,12 +235,22 @@ const api = { }).json(), }); }, + getAlertsQueryKey: () => ['alerts'] as const, + getAlertQueryKey: (alertId: string | undefined) => + ['alert', alertId] as const, useAlerts() { return useQuery({ - queryKey: [`alerts`], + queryKey: api.getAlertsQueryKey(), queryFn: () => hdxServer(`alerts`).json(), }); }, + useAlert(alertId: string | undefined) { + return useQuery({ + queryKey: api.getAlertQueryKey(alertId), + queryFn: () => hdxServer(`alerts/${alertId}`).json(), + enabled: alertId != null, + }); + }, useServices() { return useQuery({ queryKey: [`services`], diff --git a/packages/app/src/components/AlertPreviewChart.tsx b/packages/app/src/components/AlertPreviewChart.tsx index 029d0959..fd310cc4 100644 --- a/packages/app/src/components/AlertPreviewChart.tsx +++ b/packages/app/src/components/AlertPreviewChart.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import { AlertInterval, + AlertThresholdType, Filter, getSampleWeightExpression, isLogSource, @@ -25,8 +26,9 @@ type AlertPreviewChartProps = { filters?: Filter[] | null; interval: AlertInterval; groupBy?: string; - thresholdType: 'above' | 'below'; + thresholdType: AlertThresholdType; threshold: number; + thresholdMax?: number; select?: string | null; }; @@ -38,6 +40,7 @@ export const AlertPreviewChart = ({ interval, groupBy, threshold, + thresholdMax, thresholdType, select, }: AlertPreviewChartProps) => { @@ -64,7 +67,11 @@ export const AlertPreviewChart = ({ showDisplaySwitcher={false} showMVOptimizationIndicator={false} showDateRangeIndicator={false} - referenceLines={getAlertReferenceLines({ threshold, thresholdType })} + referenceLines={getAlertReferenceLines({ + threshold, + thresholdMax, + thresholdType, + })} config={{ where: where || '', whereLanguage: whereLanguage || undefined, diff --git a/packages/app/src/components/AlertScheduleFields.tsx b/packages/app/src/components/AlertScheduleFields.tsx index a5893f5a..ccdba373 100644 --- a/packages/app/src/components/AlertScheduleFields.tsx +++ b/packages/app/src/components/AlertScheduleFields.tsx @@ -8,11 +8,11 @@ import { UseFormSetValue, useWatch, } from 'react-hook-form'; -import { NumberInput } from 'react-hook-form-mantine'; import { Box, Collapse, Group, + NumberInput, Text, Tooltip, UnstyledButton, @@ -96,7 +96,7 @@ export function AlertScheduleFields({ - + Optional schedule controls for aligning alert windows. @@ -120,15 +120,20 @@ export function AlertScheduleFields({ - ( + + )} /> {offsetWindowLabel} @@ -174,7 +179,7 @@ export function AlertScheduleFields({ field.value as string | null | undefined, )} onChange={value => - field.onChange(value?.toISOString() ?? null) + field.onChange(value ? new Date(value).toISOString() : null) } error={error?.message} /> diff --git a/packages/app/src/components/AlertStatusIcon.tsx b/packages/app/src/components/AlertStatusIcon.tsx new file mode 100644 index 00000000..31031115 --- /dev/null +++ b/packages/app/src/components/AlertStatusIcon.tsx @@ -0,0 +1,31 @@ +import { AlertState } from '@hyperdx/common-utils/dist/types'; +import { Tooltip } from '@mantine/core'; +import { IconBell, IconBellFilled } from '@tabler/icons-react'; + +export function AlertStatusIcon({ + alerts, +}: { + alerts?: { state?: AlertState }[]; +}) { + if (!Array.isArray(alerts) || alerts.length === 0) return null; + const alertingCount = alerts.filter(a => a.state === AlertState.ALERT).length; + return ( + 0 + ? `${alertingCount} alert${alertingCount > 1 ? 's' : ''} triggered` + : 'Alerts configured' + } + > + {alertingCount > 0 ? ( + + ) : ( + + )} + + ); +} diff --git a/packages/app/src/components/Alerts.tsx b/packages/app/src/components/Alerts.tsx index 9f1efb50..5b2168ee 100644 --- a/packages/app/src/components/Alerts.tsx +++ b/packages/app/src/components/Alerts.tsx @@ -1,12 +1,18 @@ import { useMemo } from 'react'; -import { Control, useController } from 'react-hook-form'; -import { Select, SelectProps } from 'react-hook-form-mantine'; +import { + Control, + Controller, + FieldValues, + Path, + useController, +} from 'react-hook-form'; import { Label, ReferenceArea, ReferenceLine } from 'recharts'; import { type AlertChannelType, + AlertThresholdType, WebhookService, } from '@hyperdx/common-utils/dist/types'; -import { Button, ComboboxData, Group, Modal } from '@mantine/core'; +import { Button, ComboboxData, Group, Modal, Select } from '@mantine/core'; import { useDisclosure } from '@mantine/hooks'; import api from '@/api'; @@ -18,9 +24,13 @@ type Webhook = { name: string; }; -const WebhookChannelForm = ( - props: Partial>, -) => { +const WebhookChannelForm = ({ + control, + name, +}: { + control?: Control; + name?: string; +}) => { const { data: webhooks, refetch: refetchWebhooks } = api.useWebhooks([ WebhookService.Slack, WebhookService.Generic, @@ -48,8 +58,8 @@ const WebhookChannelForm = ( }, [webhooks]); const { field } = useController({ - control: props.control, - name: props.name!, + control, + name: name! as Path, }); const handleWebhookCreated = async (webhookId?: string) => { @@ -63,22 +73,27 @@ const WebhookChannelForm = ( return (
- - + )} /> + )} + - @@ -179,15 +244,17 @@ export default function RawSqlChartEditor({ />
- - - + {alert && ( + setValue('alert', undefined)} + error={alertErrorMessage} + warning={alertWarningMessage} + tooltip={alertTooltip} + /> + )} ); } diff --git a/packages/app/src/components/ChartEditor/RawSqlChartInstructions.tsx b/packages/app/src/components/ChartEditor/RawSqlChartInstructions.tsx index 05d7e60d..0882318c 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartInstructions.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartInstructions.tsx @@ -91,7 +91,7 @@ export function RawSqlChartInstructions({ SQL Chart Instructions - + {DISPLAY_TYPE_INSTRUCTIONS[displayType]} diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts index a90a50fe..79b18f75 100644 --- a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -6,6 +6,7 @@ import type { TSource, } from '@hyperdx/common-utils/dist/types'; import { + AlertThresholdType, DisplayType, MetricsDataType, SourceKind, @@ -610,6 +611,133 @@ describe('validateChartForm', () => { ).toHaveLength(0); }); + it('errors when raw SQL chart has alert but SQL is missing time filters and interval', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + configType: 'sql', + displayType: DisplayType.Line, + sqlTemplate: 'SELECT count() FROM logs', + connection: 'conn-1', + alert: { + interval: '1h', + threshold: 100, + thresholdType: 'above', + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + undefined, + setError, + ); + expect(errors).toContainEqual( + expect.objectContaining({ + path: 'sqlTemplate', + message: + 'SQL used for alerts must include an interval parameter or macro.', + }), + ); + }); + + it('does not error when raw SQL chart has alert and SQL includes required params', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + configType: 'sql', + displayType: DisplayType.Line, + sqlTemplate: + 'SELECT toStartOfInterval(ts, INTERVAL {intervalSeconds:Int64} SECOND) AS ts, count() FROM logs WHERE ts >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND ts <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ts', + connection: 'conn-1', + alert: { + interval: '1h', + threshold: 100, + thresholdType: 'above', + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + undefined, + setError, + ); + expect( + errors.filter( + e => e.path === 'sqlTemplate' && e.message.includes('alert'), + ), + ).toHaveLength(0); + }); + + it('does not validate SQL template for alerts when no alert is configured', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + configType: 'sql', + displayType: DisplayType.Line, + sqlTemplate: 'SELECT count() FROM logs', + connection: 'conn-1', + alert: undefined, + }), + undefined, + setError, + ); + expect( + errors.filter( + e => e.path === 'sqlTemplate' && e.message.includes('alert'), + ), + ).toHaveLength(0); + }); + + it('does not error when raw SQL Number chart has alert with date range params but no interval', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + configType: 'sql', + displayType: DisplayType.Number, + sqlTemplate: + 'SELECT count() FROM logs WHERE ts >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND ts <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64})', + connection: 'conn-1', + alert: { + interval: '1h', + threshold: 100, + thresholdType: 'above', + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + undefined, + setError, + ); + expect( + errors.filter( + e => e.path === 'sqlTemplate' && e.message.includes('alert'), + ), + ).toHaveLength(0); + }); + + it('still requires interval params for raw SQL Line chart alerts', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + configType: 'sql', + displayType: DisplayType.Line, + sqlTemplate: + 'SELECT count() FROM logs WHERE ts >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND ts <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64})', + connection: 'conn-1', + alert: { + interval: '1h', + threshold: 100, + thresholdType: 'above', + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + undefined, + setError, + ); + expect(errors).toContainEqual( + expect.objectContaining({ + path: 'sqlTemplate', + message: + 'SQL used for alerts must include an interval parameter or macro.', + }), + ); + }); + // ── Source validation ──────────────────────────────────────────────── it('errors when builder chart has no source', () => { @@ -928,6 +1056,148 @@ describe('validateChartForm', () => { expect(errors.filter(e => e.path === 'series')).toHaveLength(0); }); + // ── Alert thresholdMax validation ─────────────────────────────────── + + it('errors when between alert is missing thresholdMax', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + source: 'source-log', + alert: { + interval: '1h', + threshold: 10, + thresholdType: AlertThresholdType.BETWEEN, + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + logSource, + setError, + ); + expect(errors).toContainEqual( + expect.objectContaining({ + path: 'alert.thresholdMax', + message: + 'Upper bound is required for between/not between threshold types', + }), + ); + }); + + it('errors when not_between alert is missing thresholdMax', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + source: 'source-log', + alert: { + interval: '1h', + threshold: 10, + thresholdType: AlertThresholdType.NOT_BETWEEN, + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + logSource, + setError, + ); + expect(errors).toContainEqual( + expect.objectContaining({ + path: 'alert.thresholdMax', + }), + ); + }); + + it('errors when thresholdMax is less than threshold', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + source: 'source-log', + alert: { + interval: '1h', + threshold: 10, + thresholdMax: 5, + thresholdType: AlertThresholdType.BETWEEN, + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + logSource, + setError, + ); + expect(errors).toContainEqual( + expect.objectContaining({ + path: 'alert.thresholdMax', + message: + 'Alert threshold upper bound must be greater than or equal to the lower bound', + }), + ); + }); + + it('does not error when thresholdMax equals threshold', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + source: 'source-log', + alert: { + interval: '1h', + threshold: 10, + thresholdMax: 10, + thresholdType: AlertThresholdType.BETWEEN, + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + logSource, + setError, + ); + expect(errors.filter(e => e.path === 'alert.thresholdMax')).toHaveLength(0); + }); + + it('does not error when thresholdMax is greater than threshold for between', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + source: 'source-log', + alert: { + interval: '1h', + threshold: 10, + thresholdMax: 20, + thresholdType: AlertThresholdType.BETWEEN, + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + logSource, + setError, + ); + expect(errors.filter(e => e.path === 'alert.thresholdMax')).toHaveLength(0); + }); + + it('does not validate thresholdMax for non-range threshold types', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + source: 'source-log', + alert: { + interval: '1h', + threshold: 10, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook' }, + } as ChartEditorFormState['alert'], + }), + logSource, + setError, + ); + expect(errors.filter(e => e.path === 'alert.thresholdMax')).toHaveLength(0); + }); + + it('does not validate thresholdMax when no alert is configured', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeForm({ + source: 'source-log', + alert: undefined, + }), + logSource, + setError, + ); + expect(errors.filter(e => e.path === 'alert.thresholdMax')).toHaveLength(0); + }); + // ── Multiple validation errors at once ─────────────────────────────── it('accumulates multiple errors across different validation rules', () => { diff --git a/packages/app/src/components/ChartEditor/types.ts b/packages/app/src/components/ChartEditor/types.ts index 3e0cabc0..8bc14b78 100644 --- a/packages/app/src/components/ChartEditor/types.ts +++ b/packages/app/src/components/ChartEditor/types.ts @@ -27,6 +27,7 @@ export type SavedChartConfigWithSelectArray = Omit< export type ChartEditorFormState = Partial & Partial> & { alert?: BuilderSavedChartConfig['alert'] & { + id?: string; createdBy?: AlertWithCreatedBy['createdBy']; }; series: SavedChartConfigWithSelectArray['select']; diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index d5a28d78..56240689 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -1,5 +1,6 @@ import { omit, pick } from 'lodash'; import { Path, UseFormSetError } from 'react-hook-form'; +import { validateRawSqlForAlert } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderSavedChartConfig, isRawSqlSavedChartConfig, @@ -11,6 +12,7 @@ import { getSampleWeightExpression, isLogSource, isMetricSource, + isRangeThresholdType, isTraceSource, RawSqlChartConfig, RawSqlSavedChartConfig, @@ -75,6 +77,7 @@ export function convertFormStateToSavedChartConfig( 'compareToPreviousPeriod', 'fillNulls', 'alignDateRangeToGranularity', + 'alert', ]), sqlTemplate: form.sqlTemplate ?? '', connection: form.connection ?? '', @@ -205,7 +208,7 @@ export const validateChartForm = ( if ( !isRawSqlChart && form.displayType !== DisplayType.Markdown && - !form.source + (!form.source || !source) ) { errors.push({ path: `source`, message: 'Source is required' }); } @@ -246,6 +249,41 @@ export const validateChartForm = ( }); } + // Validate raw SQL alert has required time filters and interval parameters + if (isRawSqlChart && form.alert) { + const config = { + configType: 'sql', + sqlTemplate: form.sqlTemplate ?? '', + connection: form.connection ?? '', + from: source?.from, + displayType: form.displayType, + } satisfies RawSqlChartConfig; + const { errors: alertErrors } = validateRawSqlForAlert(config); + if (alertErrors.length > 0) { + errors.push({ + path: `sqlTemplate`, + message: alertErrors.join('. '), + }); + } + } + + // Validate thresholdMax for range threshold types (between / not between) + if (form.alert && isRangeThresholdType(form.alert.thresholdType)) { + if (form.alert.thresholdMax == null) { + errors.push({ + path: 'alert.thresholdMax', + message: + 'Upper bound is required for between/not between threshold types', + }); + } else if (form.alert.thresholdMax < form.alert.threshold) { + errors.push({ + path: 'alert.thresholdMax', + message: + 'Alert threshold upper bound must be greater than or equal to the lower bound', + }); + } + } + // Validate number and pie charts only have one series if ( !isRawSqlChart && diff --git a/packages/app/src/components/DBDeltaChart.tsx b/packages/app/src/components/DBDeltaChart.tsx index 61865a36..ed88895d 100644 --- a/packages/app/src/components/DBDeltaChart.tsx +++ b/packages/app/src/components/DBDeltaChart.tsx @@ -478,7 +478,7 @@ export default function DBDeltaChart({ {/* Legend */} {legendPrefix} - {legendPrefix && ( + {!!legendPrefix && ( { - if (!queriedConfig) return false; - if (isRawSqlChartConfig(queriedConfig)) { - return !!(queriedConfig.sqlTemplate && queriedConfig.connection); - } - return ( - ((queriedConfig.select?.length ?? 0) > 0 || - typeof queriedConfig.select === 'string') && - queriedConfig.from?.databaseName && - // tableName is empty for metric sources - (queriedConfig.from?.tableName || queriedConfig.metricTables) && - queriedConfig.timestampValueExpression - ); -}; - -type SeriesItem = NonNullable< - SavedChartConfigWithSelectArray['select'] ->[number]; - -function ChartSeriesEditorComponent({ - control, - databaseName, - dateRange, - connectionId, - index, - namePrefix, - onRemoveSeries, - onSwapSeries, - onSubmit, - setValue, - showGroupBy, - showHaving, - tableName: _tableName, - parentRef, - length, - tableSource, - errors, - clearErrors, -}: { - control: Control; - databaseName: string; - dateRange?: DateRange['dateRange']; - connectionId?: string; - index: number; - namePrefix: `series.${number}.`; - parentRef?: HTMLElement | null; - onRemoveSeries: (index: number) => void; - onSwapSeries: (from: number, to: number) => void; - onSubmit: () => void; - setValue: UseFormSetValue; - showGroupBy: boolean; - showHaving: boolean; - tableName: string; - length: number; - tableSource?: TSource; - errors?: FieldErrors; - clearErrors: UseFormClearErrors; -}) { - const aggFn = useWatch({ control, name: `${namePrefix}aggFn` }); - const aggConditionLanguage = useWatch({ - control, - name: `${namePrefix}aggConditionLanguage`, - defaultValue: 'lucene', - }); - - const metricType = useWatch({ control, name: `${namePrefix}metricType` }); - - // Initialize metricType to 'gauge' when switching to a metric source - // and reset 'custom' aggFn to 'count' since custom is not supported for metrics - useEffect(() => { - if (tableSource?.kind === SourceKind.Metric) { - if (!metricType) { - setValue(`${namePrefix}metricType`, MetricsDataType.Gauge); - } - if (aggFn === 'none') { - setValue(`${namePrefix}aggFn`, 'count'); - } - } - }, [tableSource?.kind, metricType, aggFn, namePrefix, setValue]); - - const tableName = - tableSource?.kind === SourceKind.Metric - ? getMetricTableName(tableSource, metricType) - : _tableName; - - const metricName = useWatch({ control, name: `${namePrefix}metricName` }); - const aggCondition = useWatch({ - control, - name: `${namePrefix}aggCondition`, - }); - const groupBy = useWatch({ control, name: 'groupBy' }); - - const metricTableSource = - tableSource?.kind === SourceKind.Metric ? tableSource : undefined; - - const { data: attributeSuggestions, isLoading: isLoadingAttributes } = - useFetchMetricResourceAttrs({ - databaseName, - metricType, - metricName, - tableSource: metricTableSource, - isSql: aggConditionLanguage === 'sql', - }); - - const attributeKeys = useMemo( - () => parseAttributeKeysFromSuggestions(attributeSuggestions ?? []), - [attributeSuggestions], - ); - - const { data: metricMetadata } = useFetchMetricMetadata({ - databaseName, - metricType, - metricName, - tableSource: metricTableSource, - }); - - const handleAddToWhere = useCallback( - (clause: string) => { - const currentValue = aggCondition || ''; - - const newValue = currentValue ? `${currentValue} AND ${clause}` : clause; - setValue(`${namePrefix}aggCondition`, newValue); - onSubmit(); - }, - [aggCondition, namePrefix, setValue, onSubmit], - ); - - const handleAddToGroupBy = useCallback( - (clause: string) => { - const currentValue = groupBy || ''; - const newValue = currentValue ? `${currentValue}, ${clause}` : clause; - setValue('groupBy', newValue); - onSubmit(); - }, - [groupBy, setValue, onSubmit], - ); - - const showWhere = aggFn !== 'none'; - - const tableConnection = useMemo( - () => ({ - databaseName, - tableName: tableName ?? '', - connectionId: connectionId ?? '', - metricName: - tableSource?.kind === SourceKind.Metric ? metricName : undefined, - }), - [databaseName, tableName, connectionId, metricName, tableSource], - ); - - return ( - <> - - Alias - -
- onSubmit()} - size="xs" - data-testid="series-alias-input" - /> -
- {(index ?? -1) > 0 && ( - - )} - {(index ?? -1) < length - 1 && ( - - )} - {((index ?? -1) > 0 || length > 1) && ( - - )} - - } - labelPosition="right" - mb={8} - mt="sm" - /> - -
- -
- {tableSource?.kind === SourceKind.Metric && metricType && ( -
- { - setValue(`${namePrefix}metricName`, value); - setValue(`${namePrefix}valueExpression`, 'Value'); - }} - setMetricType={value => - setValue(`${namePrefix}metricType`, value) - } - metricSource={tableSource} - data-testid="metric-name-selector" - error={errors?.metricName?.message} - onFocus={() => clearErrors(`${namePrefix}metricName`)} - /> - {metricType === 'gauge' && ( - - - - )} -
- )} - {tableSource?.kind !== SourceKind.Metric && aggFn !== 'count' && ( -
- -
- )} - {(showWhere || showGroupBy || showHaving) && ( -
- {showWhere && ( - <> - Where -
- -
- - )} - {showGroupBy && ( - <> - - Group By - -
- -
- {showHaving && ( - <> - - Having - -
- -
- - )} - - )} -
- )} -
- {tableSource?.kind === SourceKind.Metric && metricName && metricType && ( - - )} - - ); -} -const ChartSeriesEditor = ChartSeriesEditorComponent; - -const ErrorNotificationMessage = ({ - errors, -}: { - errors: { path: Path; message: string }[]; -}) => { - return ( - } - > - {errors.map(({ message }, index) => ( - {message} - ))} - - ); -}; - -const zSavedChartConfig = z - .object({ - // TODO: Chart - alert: ChartAlertBaseSchema.superRefine( - validateAlertScheduleOffsetMinutes, - ).optional(), - }) - .passthrough(); - -export default function EditTimeChartForm({ - dashboardId, - chartConfig, - displayedTimeInputValue, - dateRange, - isSaving, - onTimeRangeSearch, - setChartConfig, - setDisplayedTimeInputValue, - onSave, - onTimeRangeSelect, - onClose, - onDirtyChange, - 'data-testid': dataTestId, - submitRef, - isDashboardForm = false, - autoRun = false, -}: { - dashboardId?: string; - chartConfig: SavedChartConfig; - displayedTimeInputValue?: string; - dateRange: [Date, Date]; - isSaving?: boolean; - onTimeRangeSearch?: (value: string) => void; - setChartConfig?: (chartConfig: SavedChartConfig) => void; - setDisplayedTimeInputValue?: (value: string) => void; - onSave?: (chart: SavedChartConfig) => void; - onClose?: () => void; - onDirtyChange?: (isDirty: boolean) => void; - onTimeRangeSelect?: (start: Date, end: Date) => void; - 'data-testid'?: string; - submitRef?: React.MutableRefObject<(() => void) | undefined>; - isDashboardForm?: boolean; - autoRun?: boolean; -}) { - const formValue: ChartEditorFormState = useMemo( - () => convertSavedChartConfigToFormState(chartConfig), - [chartConfig], - ); - - const { - control, - setValue, - handleSubmit, - register, - setError, - clearErrors, - formState: { errors, isDirty, dirtyFields }, - } = useForm({ - defaultValues: formValue, - values: formValue, - resolver: zodResolver(zSavedChartConfig), - }); - - const { - fields, - append, - remove: removeSeries, - swap: swapSeries, - } = useFieldArray({ - control, - name: 'series', - }); - - useEffect(() => { - onDirtyChange?.(isDirty); - }, [isDirty, onDirtyChange]); - - const [isSampleEventsOpen, setIsSampleEventsOpen] = useState(false); - - const select = useWatch({ control, name: 'select' }); - const sourceId = useWatch({ control, name: 'source' }); - const alert = useWatch({ control, name: 'alert' }); - const seriesReturnType = useWatch({ control, name: 'seriesReturnType' }); - const groupBy = useWatch({ control, name: 'groupBy' }); - const displayType = - useWatch({ control, name: 'displayType' }) ?? DisplayType.Line; - const markdown = useWatch({ control, name: 'markdown' }); - const alertChannelType = useWatch({ control, name: 'alert.channel.type' }); - const alertScheduleOffsetMinutes = useWatch({ - control, - name: 'alert.scheduleOffsetMinutes', - }); - const granularity = useWatch({ control, name: 'granularity' }); - const maxAlertScheduleOffsetMinutes = alert?.interval - ? Math.max(intervalToMinutes(alert.interval) - 1, 0) - : 0; - const alertIntervalLabel = alert?.interval - ? TILE_ALERT_INTERVAL_OPTIONS[alert.interval] - : undefined; - const configType = useWatch({ control, name: 'configType' }); - - const chartConfigAlert = !isRawSqlSavedChartConfig(chartConfig) - ? chartConfig.alert - : undefined; - - const isRawSqlInput = - configType === 'sql' && isRawSqlDisplayType(displayType); - - const { data: tableSource } = useSource({ id: sourceId }); - const databaseName = tableSource?.from.databaseName; - const tableName = tableSource?.from.tableName; - - const activeTab = useMemo(() => { - switch (displayType) { - case DisplayType.Search: - return 'search'; - case DisplayType.Markdown: - return 'markdown'; - case DisplayType.Table: - return 'table'; - case DisplayType.Pie: - return 'pie'; - case DisplayType.Number: - return 'number'; - default: - return 'time'; - } - }, [displayType]); - - useEffect(() => { - if ( - displayType !== DisplayType.Line && - displayType !== DisplayType.Number - ) { - setValue('alert', undefined); - } - }, [displayType, setValue]); - - const showGeneratedSql = ['table', 'time', 'number', 'pie'].includes( - activeTab, - ); - - const showSampleEvents = - tableSource?.kind !== SourceKind.Metric && !isRawSqlInput; - - const [ - alignDateRangeToGranularity, - fillNulls, - compareToPreviousPeriod, - numberFormat, - ] = useWatch({ - control, - name: [ - 'alignDateRangeToGranularity', - 'fillNulls', - 'compareToPreviousPeriod', - 'numberFormat', - ], - }); - - const displaySettings: ChartConfigDisplaySettings = useMemo( - () => ({ - alignDateRangeToGranularity, - fillNulls, - compareToPreviousPeriod, - numberFormat, - }), - [ - alignDateRangeToGranularity, - fillNulls, - compareToPreviousPeriod, - numberFormat, - ], - ); - - const [ - displaySettingsOpened, - { open: openDisplaySettings, close: closeDisplaySettings }, - ] = useDisclosure(false); - - // Only update this on submit, otherwise we'll have issues - // with using the source value from the last submit - // (ex. ignoring local custom source updates) - const [queriedConfig, setQueriedConfig] = useState< - ChartConfigWithDateRange | undefined - >(undefined); - const [queriedSource, setQueriedSource] = useState( - undefined, - ); - - const setQueriedConfigAndSource = useCallback( - (config: ChartConfigWithDateRange, source: TSource | undefined) => { - setQueriedConfig(config); - setQueriedSource(source); - }, - [], - ); - - const dbTimeChartConfig = useMemo(() => { - if (!queriedConfig) { - return undefined; - } - - return { - ...queriedConfig, - granularity: alert - ? intervalToGranularity(alert.interval) - : queriedConfig.granularity, - dateRange: alert - ? extendDateRangeToInterval(queriedConfig.dateRange, alert.interval) - : queriedConfig.dateRange, - }; - }, [queriedConfig, alert]); - - const [saveToDashboardModalOpen, setSaveToDashboardModalOpen] = - useState(false); - - const onSubmit = useCallback( - (suppressErrorNotification: boolean = false) => { - handleSubmit(form => { - const isRawSqlChart = - form.configType === 'sql' && isRawSqlDisplayType(form.displayType); - - const errors = validateChartForm(form, tableSource, setError); - if (errors.length > 0) { - if (!suppressErrorNotification) { - notifications.show({ - id: 'chart-error', - title: 'Invalid Chart', - message: , - color: 'red', - }); - } - return; - } - - const savedConfig = convertFormStateToSavedChartConfig( - form, - tableSource, - ); - const queriedConfig = convertFormStateToChartConfig( - form, - dateRange, - tableSource, - ); - - if (savedConfig && queriedConfig) { - const normalizedSavedConfig = isRawSqlSavedChartConfig(savedConfig) - ? savedConfig - : { - ...savedConfig, - alert: normalizeNoOpAlertScheduleFields( - savedConfig.alert, - chartConfigAlert, - { - preserveExplicitScheduleOffsetMinutes: - dirtyFields.alert?.scheduleOffsetMinutes === true, - preserveExplicitScheduleStartAt: - dirtyFields.alert?.scheduleStartAt === true, - }, - ), - }; - setChartConfig?.(normalizedSavedConfig); - setQueriedConfigAndSource( - queriedConfig, - isRawSqlChart ? undefined : tableSource, - ); - } - })(); - }, - [ - chartConfigAlert, - dirtyFields.alert?.scheduleOffsetMinutes, - dirtyFields.alert?.scheduleStartAt, - handleSubmit, - setChartConfig, - setQueriedConfigAndSource, - tableSource, - dateRange, - setError, - ], - ); - - const onTableSortingChange = useCallback( - (sortState: SortingState | null) => { - setValue('orderBy', sortingStateToOrderByString(sortState) ?? ''); - onSubmit(); - }, - [setValue, onSubmit], - ); - - const tableSortState = useMemo( - () => - queriedConfig != null && - isBuilderChartConfig(queriedConfig) && - queriedConfig.orderBy && - typeof queriedConfig.orderBy === 'string' - ? orderByStringToSortingState(queriedConfig.orderBy) - : undefined, - [queriedConfig], - ); - - useEffect(() => { - if (submitRef) { - submitRef.current = onSubmit; - } - }, [onSubmit, submitRef]); - - const autoRunFired = useRef(false); - useEffect(() => { - if (autoRun && !autoRunFired.current && tableSource) { - autoRunFired.current = true; - onSubmit(true); - } - }, [autoRun, tableSource, onSubmit]); - - const handleSave = useCallback( - (form: ChartEditorFormState) => { - const errors = validateChartForm(form, tableSource, setError); - if (errors.length > 0) { - notifications.show({ - id: 'chart-error', - title: 'Invalid Chart', - message: , - color: 'red', - }); - return; - } - - const savedChartConfig = convertFormStateToSavedChartConfig( - form, - tableSource, - ); - - if (savedChartConfig) { - const normalizedSavedConfig = isRawSqlSavedChartConfig(savedChartConfig) - ? savedChartConfig - : { - ...savedChartConfig, - alert: normalizeNoOpAlertScheduleFields( - savedChartConfig.alert, - chartConfigAlert, - { - preserveExplicitScheduleOffsetMinutes: - dirtyFields.alert?.scheduleOffsetMinutes === true, - preserveExplicitScheduleStartAt: - dirtyFields.alert?.scheduleStartAt === true, - }, - ), - }; - - onSave?.(normalizedSavedConfig); - } - }, - [ - onSave, - tableSource, - setError, - chartConfigAlert, - dirtyFields.alert?.scheduleOffsetMinutes, - dirtyFields.alert?.scheduleStartAt, - ], - ); - - // Track previous values for detecting changes - const prevGranularityRef = useRef(granularity); - const prevDisplayTypeRef = useRef(displayType); - const prevConfigTypeRef = useRef(configType); - - useEffect(() => { - // Emulate the granularity picker auto-searching similar to dashboards - if (granularity !== prevGranularityRef.current) { - prevGranularityRef.current = granularity; - onSubmit(); - } - }, [granularity, onSubmit]); - - useEffect(() => { - const displayTypeChanged = displayType !== prevDisplayTypeRef.current; - const configTypeChanged = configType !== prevConfigTypeRef.current; - - if (displayTypeChanged || configTypeChanged) { - prevDisplayTypeRef.current = displayType; - prevConfigTypeRef.current = configType; - - if (displayType === DisplayType.Search && typeof select !== 'string') { - setValue('select', ''); - setValue('series', []); - } - - if (displayType !== DisplayType.Search && !Array.isArray(select)) { - const defaultSeries: SavedChartConfigWithSelectArray['select'] = [ - { - aggFn: 'count', - aggCondition: '', - aggConditionLanguage: getStoredLanguage() ?? 'lucene', - valueExpression: '', - }, - ]; - setValue('where', ''); - setValue('select', defaultSeries); - setValue('series', defaultSeries); - } - - // Don't auto-submit when config type changes, to avoid clearing form state (like source) - if (displayTypeChanged) { - // true = Suppress error notification (because we're auto-submitting) - onSubmit(true); - } - } - }, [displayType, select, setValue, onSubmit, configType]); - - // Emulate the date range picker auto-searching similar to dashboards - useEffect(() => { - // eslint-disable-next-line react-hooks/set-state-in-effect - setQueriedConfig((config: ChartConfigWithDateRange | undefined) => { - if (config == null) { - return config; - } - - return { - ...config, - dateRange, - }; - }); - }, [dateRange]); - - const queryReady = isQueryReady(queriedConfig); - - // The chart config to use when showing the user the generated SQL - // and explaining whether a MV can be used. - const chartConfigForExplanations: ChartConfigWithOptTimestamp | undefined = - useMemo(() => { - if (queriedConfig && isRawSqlChartConfig(queriedConfig)) - return { ...queriedConfig, dateRange }; - - if (chartConfig && isRawSqlSavedChartConfig(chartConfig)) - return { ...chartConfig, dateRange }; - - const userHasSubmittedQuery = !!queriedConfig; - const queriedSourceMatchesSelectedSource = - queriedSource?.id === tableSource?.id; - const urlParamsSourceMatchesSelectedSource = - chartConfig.source === tableSource?.id; - - const effectiveQueriedConfig = - activeTab === 'time' ? dbTimeChartConfig : queriedConfig; - - const config = - userHasSubmittedQuery && queriedSourceMatchesSelectedSource - ? effectiveQueriedConfig - : chartConfig && urlParamsSourceMatchesSelectedSource && tableSource - ? { - ...chartConfig, - dateRange, - timestampValueExpression: tableSource.timestampValueExpression, - from: tableSource.from, - connection: tableSource.connection, - } - : undefined; - - if (!config || isRawSqlChartConfig(config)) { - return undefined; - } - - // Apply the transformations that child components will apply, - // so that the MV optimization explanation and generated SQL preview - // are accurate. - if (activeTab === 'time') { - return convertToTimeChartConfig(config); - } else if (activeTab === 'number') { - return convertToNumberChartConfig(config); - } else if (activeTab === 'table') { - return convertToTableChartConfig(config); - } else if (activeTab === 'pie') { - return convertToPieChartConfig(config); - } - - return config; - }, [ - queriedConfig, - queriedSource?.id, - tableSource, - chartConfig, - dateRange, - activeTab, - dbTimeChartConfig, - ]); - - const previousDateRange = getPreviousDateRange(dateRange); - - const sampleEventsConfig = useMemo( - () => - tableSource != null && - queriedConfig != null && - isBuilderChartConfig(queriedConfig) && - queryReady - ? { - ...queriedConfig, - orderBy: [ - { - ordering: 'DESC' as const, - valueExpression: getFirstTimestampValueExpression( - tableSource.timestampValueExpression, - ), - }, - ], - dateRange, - timestampValueExpression: tableSource.timestampValueExpression, - connection: tableSource.connection, - from: tableSource.from, - limit: { limit: 200 }, - select: - ((tableSource?.kind === SourceKind.Log || - tableSource?.kind === SourceKind.Trace) && - tableSource.defaultTableSelectExpression) || - '', - filters: seriesToFilters(queriedConfig.select), - filtersLogicalOperator: 'OR' as const, - groupBy: undefined, - granularity: undefined, - having: undefined, - } - : null, - [queriedConfig, tableSource, dateRange, queryReady], - ); - - // Need to force a rerender on change as the modal will not be mounted when initially rendered - const [parentRef, setParentRef] = useState(null); - - const handleUpdateDisplaySettings = useCallback( - ({ - numberFormat, - alignDateRangeToGranularity, - fillNulls, - compareToPreviousPeriod, - }: ChartConfigDisplaySettings) => { - setValue('numberFormat', numberFormat); - setValue('alignDateRangeToGranularity', alignDateRangeToGranularity); - setValue('fillNulls', fillNulls); - setValue('compareToPreviousPeriod', compareToPreviousPeriod); - onSubmit(); - }, - [setValue, onSubmit], - ); - - const tableConnection = useMemo( - () => tcFromSource(tableSource), - [tableSource], - ); - - return ( -
- - ( - - - } - data-testid="chart-type-line" - > - Line/Bar - - } - data-testid="chart-type-table" - > - Table - - } - data-testid="chart-type-number" - > - Number - - } - data-testid="chart-type-pie" - > - Pie - - } - data-testid="chart-type-search" - > - Search - - } - data-testid="chart-type-markdown" - > - Markdown - - - - )} - /> - - - Chart Name - - - {isRawSqlDisplayType(displayType) && ( - ( - - )} - /> - )} - - - {activeTab === 'markdown' ? ( -
-