diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index e2996e88..98bcf46e 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -56,7 +56,11 @@ services: # ports: # - 9000:9000 environment: + CLICKHOUSE_HOST: http://ch-server:8123 + CLICKHOUSE_PASSWORD: api + CLICKHOUSE_USER: api EXPRESS_SESSION_SECRET: 'hyperdx is cool ๐Ÿ‘‹' + FRONTEND_URL: 'http://app:8080' MONGO_URI: 'mongodb://db:29999/hyperdx-test' NODE_ENV: ci PORT: 9000 diff --git a/packages/api/package.json b/packages/api/package.json index f0e75ade..c7a64ecf 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -8,6 +8,7 @@ }, "dependencies": { "@clickhouse/client": "^0.2.10", + "@clickhouse/client-common": "^1.9.1", "@hyperdx/lucene": "^3.1.1", "@hyperdx/node-opentelemetry": "^0.8.1", "@opentelemetry/api": "^1.8.0", @@ -33,6 +34,7 @@ "mongoose": "^6.12.0", "ms": "^2.1.3", "node-schedule": "^2.1.1", + "node-sql-parser": "^5.3.5", "object-hash": "^3.0.0", "on-headers": "^1.0.2", "passport": "^0.6.0", @@ -47,7 +49,7 @@ "sqlstring": "^2.3.3", "uuid": "^8.3.2", "winston": "^3.10.0", - "zod": "^3.22.3", + "zod": "^3.24.1", "zod-express-middleware": "^1.4.0" }, "devDependencies": { diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index ab31dec2..8ea1c8c4 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -34,7 +34,7 @@ const sess: session.SessionOptions & { cookie: session.CookieOptions } = { }; app.set('trust proxy', 1); -if (config.FRONTEND_URL && !config.IS_CI) { +if (!config.IS_CI && config.FRONTEND_URL) { const feUrl = new URL(config.FRONTEND_URL); sess.cookie.domain = feUrl.hostname; if (feUrl.protocol === 'https:') { diff --git a/packages/api/src/common/DisplayType.ts b/packages/api/src/common/DisplayType.ts new file mode 100644 index 00000000..2bd99808 --- /dev/null +++ b/packages/api/src/common/DisplayType.ts @@ -0,0 +1,9 @@ +export enum DisplayType { + Line = 'line', + StackedBar = 'stacked_bar', + Table = 'table', + Number = 'number', + Search = 'search', + Heatmap = 'heatmap', + Markdown = 'markdown', +} diff --git a/packages/api/src/common/clickhouse.ts b/packages/api/src/common/clickhouse.ts new file mode 100644 index 00000000..761b2be7 --- /dev/null +++ b/packages/api/src/common/clickhouse.ts @@ -0,0 +1,606 @@ +import { + BaseResultSet, + DataFormat, + isSuccessfulResponse, + ResponseJSON, +} from '@clickhouse/client-common'; + +import { SQLInterval } from '@/common/sqlTypes'; +import { timeBucketByGranularity } from '@/common/utils'; +import { hashCode } from '@/common/utils'; + +export const CLICKHOUSE_HOST = '/api/clickhouse-proxy'; + +export enum JSDataType { + Array = 'array', + Date = 'date', + Map = 'map', + Number = 'number', + String = 'string', + Bool = 'bool', +} + +export const convertCHDataTypeToJSType = ( + dataType: string, +): JSDataType | null => { + if (dataType.startsWith('Date')) { + return JSDataType.Date; + } else if (dataType.startsWith('Map')) { + return JSDataType.Map; + } else if (dataType.startsWith('Array')) { + return JSDataType.Array; + } else if ( + dataType.startsWith('Int') || + dataType.startsWith('UInt') || + dataType.startsWith('Float') || + // Nullable types are possible (charts) + dataType.startsWith('Nullable(Int') || + dataType.startsWith('Nullable(UInt') || + dataType.startsWith('Nullable(Float') + ) { + return JSDataType.Number; + } else if ( + dataType.startsWith('String') || + dataType.startsWith('FixedString') || + dataType.startsWith('Enum') || + dataType.startsWith('UUID') || + dataType.startsWith('IPv4') || + dataType.startsWith('IPv6') + ) { + return JSDataType.String; + } else if (dataType === 'Bool') { + return JSDataType.Bool; + } else if (dataType.startsWith('LowCardinality')) { + return convertCHDataTypeToJSType(dataType.slice(15, -1)); + } + + return null; +}; + +export const convertCHTypeToPrimitiveJSType = (dataType: string) => { + const jsType = convertCHDataTypeToJSType(dataType); + + if (jsType === JSDataType.Map || jsType === JSDataType.Array) { + throw new Error('Map type is not a primitive type'); + } else if (jsType === JSDataType.Date) { + return JSDataType.Number; + } + + return jsType; +}; + +const hash = (input: string | number) => Math.abs(hashCode(`${input}`)); +const paramHash = (str: string | number) => { + return `HYPERDX_PARAM_${hash(str)}`; +}; + +export type ChSql = { + sql: string; + params: Record; +}; + +type ParamTypes = + | ChSql + | ChSql[] + | { Identifier: string } + | { String: string } + | { Float32: number } + | { Float64: number } + | { Int32: number } + | { Int64: number } + | { UNSAFE_RAW_SQL: string } + | string; // TODO: Deprecate raw string interpolation + +export const chSql = ( + strings: TemplateStringsArray, + ...values: ParamTypes[] +): ChSql => { + const sql = strings + .map((str, i) => { + const value = values[i]; + // if (typeof value === 'string') { + // console.error('Unsafe string detected', value, 'in', strings, values); + // } + + return ( + str + + (value == null + ? '' + : typeof value === 'string' + ? value // If it's just a string sql literal + : 'UNSAFE_RAW_SQL' in value + ? value.UNSAFE_RAW_SQL + : Array.isArray(value) + ? value.map(v => v.sql).join('') + : 'sql' in value + ? value.sql + : 'Identifier' in value + ? `{${paramHash(value.Identifier)}:Identifier}` + : 'String' in value + ? `{${paramHash(value.String)}:String}` + : 'Float32' in value + ? `{${paramHash(value.Float32)}:Float32}` + : 'Float64' in value + ? `{${paramHash(value.Float64)}:Float64}` + : 'Int32' in value + ? `{${paramHash(value.Int32)}:Int32}` + : 'Int64' in value + ? `{${paramHash(value.Int64)}:Int64}` + : '') + ); + }) + .join(''); + + return { + sql, + params: values.reduce((acc, value) => { + return { + ...acc, + ...(value == null || + typeof value === 'string' || + 'UNSAFE_RAW_SQL' in value + ? {} + : Array.isArray(value) + ? value.reduce((acc, v) => { + Object.assign(acc, v.params); + return acc; + }, {}) + : 'params' in value + ? value.params + : 'Identifier' in value + ? { [paramHash(value.Identifier)]: value.Identifier } + : 'String' in value + ? { [paramHash(value.String)]: value.String } + : 'Float32' in value + ? { [paramHash(value.Float32)]: value.Float32 } + : 'Float64' in value + ? { [paramHash(value.Float64)]: value.Float64 } + : 'Int32' in value + ? { [paramHash(value.Int32)]: value.Int32 } + : 'Int64' in value + ? { [paramHash(value.Int64)]: value.Int64 } + : {}), + }; + }, {}), + }; +}; + +export const concatChSql = (sep: string, ...args: (ChSql | ChSql[])[]) => { + return args.reduce( + (acc: ChSql, arg) => { + if (Array.isArray(arg)) { + if (arg.length === 0) { + return acc; + } + + acc.sql += + (acc.sql.length > 0 ? sep : '') + arg.map(a => a.sql).join(sep); + acc.params = arg.reduce((acc, a) => { + Object.assign(acc, a.params); + return acc; + }, acc.params); + } else if (arg.sql.length > 0) { + acc.sql += `${acc.sql.length > 0 ? sep : ''}${arg.sql}`; + Object.assign(acc.params, arg.params); + } + return acc; + }, + { sql: '', params: {} }, + ); +}; + +const isChSqlEmpty = (chSql: ChSql | ChSql[]) => { + if (Array.isArray(chSql)) { + return chSql.every(c => c.sql.length === 0); + } + return chSql.sql.length === 0; +}; + +export const wrapChSqlIfNotEmpty = ( + sql: ChSql | ChSql[], + left: string, + right: string, +): ChSql | [] => { + if (isChSqlEmpty(sql)) { + return []; + } + + return chSql`${left}${sql}${right}`; +}; +export class ClickHouseQueryError extends Error { + constructor( + message: string, + public query: string, + ) { + super(message); + this.name = 'ClickHouseQueryError'; + } +} + +export function extractColumnReference( + sql: string, + maxIterations = 10, +): string | null { + let iterations = 0; + + // Loop until we remove all function calls and get just the column, with a maximum limit + while (/\w+\([^()]*\)/.test(sql) && iterations < maxIterations) { + // Replace the outermost function with its content + sql = sql.replace(/\w+\(([^()]*)\)/, '$1'); + iterations++; + } + + // If we reached the max iterations without resolving, return null to indicate an issue + return iterations < maxIterations ? sql.trim() : null; +} + +const client = { + async query({ + query, + format = 'JSON', + query_params = {}, + abort_signal, + clickhouse_settings, + host, + username, + password, + includeCredentials, + includeCorsHeader, + connectionId, + queryId, + }: { + query: string; + format?: string; + abort_signal?: AbortSignal; + query_params?: Record; + clickhouse_settings?: Record; + host?: string; + username?: string; + password?: string; + includeCredentials: boolean; + includeCorsHeader: boolean; + connectionId?: string; + queryId?: string; + }): Promise> { + const searchParams = new URLSearchParams([ + ...(includeCorsHeader ? [['add_http_cors_header', '1']] : []), + ...(connectionId ? [['hyperdx_connection_id', connectionId]] : []), + ['query', query], + ['default_format', format], + ['date_time_output_format', 'iso'], + ['wait_end_of_query', '0'], + ['cancel_http_readonly_queries_on_client_close', '1'], + ...(username ? [['user', username]] : []), + ...(password ? [['password', password]] : []), + ...(queryId ? [['query_id', queryId]] : []), + ...Object.entries(query_params).map(([key, value]) => [ + `param_${key}`, + value, + ]), + ...Object.entries(clickhouse_settings ?? {}).map(([key, value]) => [ + key, + value, + ]), + ]); + + let debugSql = ''; + try { + debugSql = parameterizedQueryToSql({ sql: query, params: query_params }); + } catch (e) { + debugSql = query; + } + + // eslint-disable-next-line no-console + console.log('--------------------------------------------------------'); + // eslint-disable-next-line no-console + console.log('Sending Query:', debugSql); + // eslint-disable-next-line no-console + console.log('--------------------------------------------------------'); + + const res = await fetch(`${host}/?${searchParams.toString()}`, { + ...(includeCredentials ? { credentials: 'include' } : {}), + signal: abort_signal, + method: 'GET', + }); + + // TODO: Send command to CH to cancel query on abort_signal + if (!res.ok) { + if (!isSuccessfulResponse(res.status)) { + const text = await res.text(); + throw new ClickHouseQueryError(`${text}`, debugSql); + } + } + + if (res.body == null) { + // TODO: Handle empty responses better? + throw new Error('Unexpected empty response from ClickHouse'); + } + + // @ts-ignore + return new BaseResultSet(res.body, format, ''); + }, +}; + +export const testLocalConnection = async ({ + host, + username, + password, +}: { + host: string; + username: string; + password: string; +}): Promise => { + try { + const result = await client.query({ + query: 'SELECT 1', + format: 'TabSeparatedRaw', + host: host, + username: username, + password: password, + includeCredentials: false, + includeCorsHeader: true, + }); + return result.text().then(text => text.trim() === '1'); + } catch (e) { + console.warn('Failed to test local connection', e); + return false; + } +}; + +export const sendQuery = async ({ + query, + format = 'JSON', + query_params = {}, + abort_signal, + clickhouse_settings, + connectionId, + queryId, +}: { + query: string; + format?: string; + query_params?: Record; + abort_signal?: AbortSignal; + clickhouse_settings?: Record; + connectionId: string; + queryId?: string; +}) => { + const IS_LOCAL_MODE = false; + + // TODO: decide what to do here + let host, username, password; + if (IS_LOCAL_MODE) { + const localConnections: any = []; + if (localConnections.length === 0) { + throw new Error('No local connection found'); + } + host = localConnections[0].host; + username = localConnections[0].username; + password = localConnections[0].password; + } + + return client.query({ + query, + format, + query_params, + abort_signal, + clickhouse_settings, + queryId, + connectionId: IS_LOCAL_MODE ? undefined : connectionId, + host: IS_LOCAL_MODE ? host : CLICKHOUSE_HOST, + username: IS_LOCAL_MODE ? username : undefined, + password: IS_LOCAL_MODE ? password : undefined, + includeCredentials: !IS_LOCAL_MODE, + includeCorsHeader: IS_LOCAL_MODE, + }); +}; + +export const tableExpr = ({ + database, + table, +}: { + database: string; + table: string; +}) => { + return chSql`${{ Identifier: database }}.${{ Identifier: table }}`; +}; + +/** + * SELECT + * aggFnIf(fieldToColumn(field), where), + * timeBucketing(Granularity, timeConversion(fieldToColumn(field))), + * FROM db.table + * WHERE where + * GROUP BY timeBucketing, fieldToColumn(groupBy) + * ORDER BY orderBy + */ + +export function parameterizedQueryToSql({ + sql, + params, +}: { + sql: string; + params: Record; +}) { + return Object.entries(params).reduce((acc, [key, value]) => { + return acc.replace(new RegExp(`{${key}:\\w+}`, 'g'), value); + }, sql); +} + +export type ColumnMetaType = { name: string; type: string }; +export function filterColumnMetaByType( + meta: Array, + types: JSDataType[], +): Array | undefined { + return meta.filter(column => + types.includes(convertCHDataTypeToJSType(column.type) as JSDataType), + ); +} + +export function inferTimestampColumn( + // from: https://github.com/ClickHouse/clickhouse-js/blob/442392c83834f313a964f9e5bd7ff44474631755/packages/client-common/src/clickhouse_types.ts#L8C3-L8C47 + meta: Array, +) { + return filterColumnMetaByType(meta, [JSDataType.Date])?.[0]; +} + +function inferValueColumns(meta: Array<{ name: string; type: string }>) { + return filterColumnMetaByType(meta, [JSDataType.Number]); +} + +function inferGroupColumns(meta: Array<{ name: string; type: string }>) { + return filterColumnMetaByType(meta, [ + JSDataType.String, + JSDataType.Map, + JSDataType.Array, + ]); +} + +// TODO: Move to ChartUtils +// Input: { ts, value1, value2, groupBy1, groupBy2 }, +// Output: { ts, [value1Name, groupBy1, groupBy2]: value1, [...]: value2 } +export function formatResponseForTimeChart({ + res, + dateRange, + granularity, + generateEmptyBuckets = true, +}: { + dateRange: [Date, Date]; + granularity?: SQLInterval; + res: ResponseJSON>; + generateEmptyBuckets?: boolean; +}) { + const meta = res.meta; + const data = res.data; + + if (meta == null) { + throw new Error('No meta data found in response'); + } + + const timestampColumn = inferTimestampColumn(meta); + const valueColumns = inferValueColumns(meta) ?? []; + const groupColumns = inferGroupColumns(meta) ?? []; + + if (timestampColumn == null) { + throw new Error( + `No timestamp column found with meta: ${JSON.stringify(meta)}`, + ); + } + + // Timestamp -> { tsCol, line1, line2, ...} + const tsBucketMap: Map> = new Map(); + const lineDataMap: { + [keyName: string]: { + dataKey: string; + displayName: string; + maxValue: number; + minValue: number; + color: string | undefined; + }; + } = {}; + + for (const row of data) { + const date = new Date(row[timestampColumn.name]); + const ts = date.getTime() / 1000; + + for (const valueColumn of valueColumns) { + const tsBucket = tsBucketMap.get(ts) ?? {}; + + const keyName = [ + valueColumn.name, + ...groupColumns.map(g => row[g.name]), + ].join(' ยท '); + + // UInt64 are returned as strings, we'll convert to number + // and accept a bit of floating point error + const rawValue = row[valueColumn.name]; + const value = + typeof rawValue === 'number' ? rawValue : Number.parseFloat(rawValue); + + tsBucketMap.set(ts, { + ...tsBucket, + [timestampColumn.name]: ts, + [keyName]: value, + }); + + // TODO: Set name and color correctly + lineDataMap[keyName] = { + dataKey: keyName, + displayName: keyName, + color: undefined, + maxValue: Math.max( + lineDataMap[keyName]?.maxValue ?? Number.NEGATIVE_INFINITY, + value, + ), + minValue: Math.min( + lineDataMap[keyName]?.minValue ?? Number.POSITIVE_INFINITY, + value, + ), + }; + } + } + + // TODO: Custom sort and truncate top N lines + const sortedLineDataMap = Object.values(lineDataMap).sort((a, b) => { + return a.maxValue - b.maxValue; + }); + + if (generateEmptyBuckets && granularity != null) { + // Zero fill TODO: Make this an option + const generatedTsBuckets = timeBucketByGranularity( + dateRange[0], + dateRange[1], + granularity, + ); + + generatedTsBuckets.forEach(date => { + const ts = date.getTime() / 1000; + const tsBucket = tsBucketMap.get(ts); + + if (tsBucket == null) { + const tsBucket: Record = { + [timestampColumn.name]: ts, + }; + + for (const line of sortedLineDataMap) { + tsBucket[line.dataKey] = 0; + } + + tsBucketMap.set(ts, tsBucket); + } else { + for (const line of sortedLineDataMap) { + if (tsBucket[line.dataKey] == null) { + tsBucket[line.dataKey] = 0; + } + } + tsBucketMap.set(ts, tsBucket); + } + }); + } + + // Sort results again by timestamp + const graphResults: { + [key: string]: number | undefined; + }[] = Array.from(tsBucketMap.values()).sort( + (a, b) => a[timestampColumn.name] - b[timestampColumn.name], + ); + + // TODO: Return line color and names + return { + // dateRange: [minDate, maxDate], + graphResults, + timestampColumn, + groupKeys: sortedLineDataMap.map(l => l.dataKey), + lineNames: sortedLineDataMap.map(l => l.displayName), + lineColors: sortedLineDataMap.map(l => l.color), + }; +} + +export type ColumnMeta = { + codec_expression: string; + comment: string; + default_expression: string; + default_type: string; + name: string; + ttl_expression: string; + type: string; +}; diff --git a/packages/api/src/common/commonTypes.ts b/packages/api/src/common/commonTypes.ts new file mode 100644 index 00000000..fd71bcaa --- /dev/null +++ b/packages/api/src/common/commonTypes.ts @@ -0,0 +1,245 @@ +import { z } from 'zod'; + +import { DisplayType } from '@/common/DisplayType'; + +// -------------------------- +// SQL TYPES +// -------------------------- +// TODO: infer types from here and replaces all types in sqlTypes.ts +export const SQLIntervalSchema = z + .string() + .regex(/^\d+ (second|minute|hour|day)$/); +export const SearchConditionSchema = z.string(); +export const SearchConditionLanguageSchema = z + .enum(['sql', 'lucene']) + .optional(); +export const AggregateFunctionSchema = z.enum([ + 'avg', + 'count', + 'count_distinct', + 'max', + 'min', + 'quantile', + 'sum', +]); +export const AggregateFunctionWithCombinatorsSchema = z + .string() + .regex(/^(\w+)If(State|Merge)$/); +export const RootValueExpressionSchema = z + .object({ + aggFn: z.union([ + AggregateFunctionSchema, + AggregateFunctionWithCombinatorsSchema, + ]), + aggCondition: SearchConditionSchema, + aggConditionLanguage: SearchConditionLanguageSchema, + valueExpression: z.string(), + }) + .or( + z.object({ + aggFn: z.literal('quantile'), + level: z.number(), + aggCondition: SearchConditionSchema, + aggConditionLanguage: SearchConditionLanguageSchema, + valueExpression: z.string(), + }), + ) + .or( + z.object({ + aggFn: z.string().optional(), + aggCondition: z.string().optional(), + aggConditionLanguage: SearchConditionLanguageSchema, + valueExpression: z.string(), + }), + ); +export const DerivedColumnSchema = z.intersection( + RootValueExpressionSchema, + z.object({ + alias: z.string().optional(), + }), +); +export const SelectListSchema = z.array(DerivedColumnSchema).or(z.string()); +export const SortSpecificationSchema = z.intersection( + RootValueExpressionSchema, + z.object({ + ordering: z.enum(['ASC', 'DESC']), + }), +); +export const SortSpecificationListSchema = z + .array(SortSpecificationSchema) + .or(z.string()); +export const LimitSchema = z.object({ + limit: z.number().optional(), + offset: z.number().optional(), +}); +export const SelectSQLStatementSchema = z.object({ + select: SelectListSchema, + from: z.object({ + databaseName: z.string(), + tableName: z.string(), + }), + where: SearchConditionSchema, + whereLanguage: SearchConditionLanguageSchema, + groupBy: SelectListSchema.optional(), + having: SearchConditionSchema.optional(), + havingLanguage: SearchConditionLanguageSchema.optional(), + orderBy: SortSpecificationListSchema.optional(), + limit: LimitSchema.optional(), +}); + +// -------------------------- +// SAVED SEARCH +// -------------------------- +export const SavedSearchSchema = z.object({ + id: z.string(), + name: z.string(), + select: z.string(), + where: z.string(), + whereLanguage: SearchConditionLanguageSchema, + source: z.string(), + tags: z.array(z.string()), + orderBy: z.string().optional(), +}); + +export type SavedSearch = z.infer; + +// -------------------------- +// DASHBOARDS +// -------------------------- +export const NumberFormatSchema = z.object({ + output: z.enum(['currency', 'percent', 'byte', 'time', 'number']), + mantissa: z.number().optional(), + thousandSeparated: z.boolean().optional(), + average: z.boolean().optional(), + decimalBytes: z.boolean().optional(), + factor: z.number().optional(), + currencySymbol: z.string().optional(), + unit: z.string().optional(), +}); + +export const SqlAstFilterSchema = z.object({ + type: z.literal('sql_ast'), + operator: z.enum(['=', '<', '>', '!=', '<=', '>=']), + left: z.string(), + right: z.string(), +}); + +export const FilterSchema = z.union([ + z.object({ + type: z.enum(['lucene', 'sql']), + condition: z.string(), + }), + SqlAstFilterSchema, +]); + +export const _ChartConfigSchema = z.object({ + displayType: z.nativeEnum(DisplayType), + numberFormat: NumberFormatSchema, + timestampValueExpression: z.string(), + implicitColumnExpression: z.string().optional(), + granularity: z.string().optional(), + markdown: z.string().optional(), + filtersLogicalOperator: z.enum(['AND', 'OR']).optional(), + filters: z.array(FilterSchema), + connection: z.string(), + fillNulls: z.number().optional(), + selectGroupBy: z.boolean().optional(), +}); + +export const ChartConfigSchema = z.intersection( + _ChartConfigSchema, + SelectSQLStatementSchema, +); + +export const SavedChartConfigSchema = z.intersection( + z.intersection( + z.object({ + name: z.string(), + source: z.string(), + }), + _ChartConfigSchema.omit({ + connection: true, + timestampValueExpression: true, + }), + ), + SelectSQLStatementSchema.omit({ + from: true, + }), +); + +export type SavedChartConfig = z.infer; + +export const TileSchema = z.object({ + id: z.string(), + x: z.number(), + y: z.number(), + w: z.number(), + h: z.number(), + config: SavedChartConfigSchema, +}); + +export type Tile = z.infer; + +export const DashboardSchema = z.object({ + id: z.string(), + name: z.string(), + tiles: z.array(TileSchema), + tags: z.array(z.string()), +}); + +export const DashboardWithoutIdSchema = DashboardSchema.omit({ id: true }); + +export const ConnectionSchema = z.object({ + id: z.string(), + name: z.string(), + host: z.string(), + username: z.string(), + password: z.string().optional(), +}); + +// -------------------------- +// TABLE SOURCES +// -------------------------- +export const SourceSchema = z.object({ + from: z.object({ + databaseName: z.string(), + tableName: z.string(), + }), + timestampValueExpression: z.string(), + connection: z.string(), + + // Common + kind: z.enum(['log', 'trace']), + id: z.string(), + name: z.string(), + displayedTimestampValueExpression: z.string().optional(), + implicitColumnExpression: z.string().optional(), + serviceNameExpression: z.string().optional(), + bodyExpression: z.string().optional(), + tableFilterExpression: z.string().optional(), + eventAttributesExpression: z.string().optional(), + resourceAttributesExpression: z.string().optional(), + defaultTableSelectExpression: z.string().optional(), + + // Logs + uniqueRowIdExpression: z.string().optional(), + severityTextExpression: z.string().optional(), + traceSourceId: z.string().optional(), + + // Traces & Logs + traceIdExpression: z.string().optional(), + spanIdExpression: z.string().optional(), + + // Traces + durationExpression: z.string().optional(), + durationPrecision: z.number().min(0).max(9).optional(), + parentSpanIdExpression: z.string().optional(), + spanNameExpression: z.string().optional(), + + spanKindExpression: z.string().optional(), + statusCodeExpression: z.string().optional(), + statusMessageExpression: z.string().optional(), + logSourceId: z.string().optional(), +}); + +export type TSource = z.infer; diff --git a/packages/api/src/common/metadata.ts b/packages/api/src/common/metadata.ts new file mode 100644 index 00000000..c8a54c1c --- /dev/null +++ b/packages/api/src/common/metadata.ts @@ -0,0 +1,436 @@ +import { + ChSql, + chSql, + ColumnMeta, + convertCHDataTypeToJSType, + filterColumnMetaByType, + JSDataType, + sendQuery, + tableExpr, +} from '@/common/clickhouse'; + +import { + ChartConfigWithDateRange, + renderChartConfig, +} from './renderChartConfig'; + +const DEFAULT_SAMPLE_SIZE = 1e6; + +class MetadataCache { + private cache = new Map(); + + // this should be getOrUpdate... or just query to follow react query + get(key: string): T | undefined { + return this.cache.get(key); + } + + async getOrFetch(key: string, query: () => Promise): Promise { + const value = this.get(key) as T | undefined; + if (value != null) { + return value; + } + + const newValue = await query(); + this.cache.set(key, newValue); + + return newValue; + } + + set(key: string, value: T) { + return this.cache.set(key, value); + } + + // TODO: This needs to be async, and use tanstack query on frontend for cache + // TODO: Implement locks for refreshing + // TODO: Shard cache by time +} + +export type TableMetadata = { + database: string; + name: string; + uuid: string; + engine: string; + is_temporary: number; + data_paths: string[]; + metadata_path: string; + metadata_modification_time: string; + metadata_version: number; + create_table_query: string; + engine_full: string; + as_select: string; + partition_key: string; + sorting_key: string; + primary_key: string; + sampling_key: string; + storage_policy: string; + total_rows: string; + total_bytes: string; + total_bytes_uncompressed: string; + parts: string; + active_parts: string; + total_marks: string; + comment: string; +}; + +export class Metadata { + private cache = new MetadataCache(); + + private static async queryTableMetadata({ + database, + table, + cache, + connectionId, + }: { + database: string; + table: string; + cache: MetadataCache; + connectionId: string; + }) { + return cache.getOrFetch(`${database}.${table}.metadata`, async () => { + const sql = chSql`SELECT * FROM system.tables where database = ${{ String: database }} AND name = ${{ String: table }}`; + const json = await sendQuery<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId, + }).then(res => res.json()); + return json.data[0]; + }); + } + + async getColumns({ + databaseName, + tableName, + connectionId, + }: { + databaseName: string; + tableName: string; + connectionId: string; + }) { + return this.cache.getOrFetch( + `${databaseName}.${tableName}.columns`, + async () => { + const sql = chSql`DESCRIBE ${tableExpr({ database: databaseName, table: tableName })}`; + const columns = await sendQuery<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId, + }) + .then(res => res.json()) + .then(d => d.data); + return columns as ColumnMeta[]; + }, + ); + } + + async getMaterializedColumnsLookupTable({ + databaseName, + tableName, + connectionId, + }: { + databaseName: string; + tableName: string; + connectionId: string; + }) { + const columns = await this.getColumns({ + databaseName, + tableName, + connectionId, + }); + + // Build up materalized fields lookup table + return new Map( + columns + .filter( + c => + c.default_type === 'MATERIALIZED' || c.default_type === 'DEFAULT', + ) + .map(c => [c.default_expression, c.name]), + ); + } + + async getColumn({ + databaseName, + tableName, + column, + matchLowercase = false, + connectionId, + }: { + databaseName: string; + tableName: string; + column: string; + matchLowercase?: boolean; + connectionId: string; + }): Promise { + const tableColumns = await this.getColumns({ + databaseName, + tableName, + connectionId, + }); + + return tableColumns.filter(c => { + if (matchLowercase) { + return c.name.toLowerCase() === column.toLowerCase(); + } + + return c.name === column; + })[0]; + } + + async getMapKeys({ + databaseName, + tableName, + column, + maxKeys = 1000, + connectionId, + }: { + databaseName: string; + tableName: string; + column: string; + maxKeys?: number; + connectionId: string; + }) { + const cachedKeys = this.cache.get( + `${databaseName}.${tableName}.${column}.keys`, + ); + + if (cachedKeys != null) { + return cachedKeys; + } + + const colMeta = await this.getColumn({ + databaseName, + tableName, + column, + connectionId, + }); + + if (colMeta == null) { + throw new Error( + `Column ${column} not found in ${databaseName}.${tableName}`, + ); + } + + let strategy: 'groupUniqArrayArray' | 'lowCardinalityKeys' = + 'groupUniqArrayArray'; + if (colMeta.type.startsWith('Map(LowCardinality(String)')) { + strategy = 'lowCardinalityKeys'; + } + + let sql: ChSql; + if (strategy === 'groupUniqArrayArray') { + sql = chSql`SELECT groupUniqArrayArray(${{ Int32: maxKeys }})(${{ + Identifier: column, + }}) as keysArr + FROM ${tableExpr({ database: databaseName, table: tableName })}`; + } else { + sql = chSql`SELECT DISTINCT lowCardinalityKeys(arrayJoin(${{ + Identifier: column, + }}.keys)) as key + FROM ${tableExpr({ database: databaseName, table: tableName })} + LIMIT ${{ + Int32: maxKeys, + }}`; + } + + return this.cache.getOrFetch( + `${databaseName}.${tableName}.${column}.keys`, + async () => { + const keys = await sendQuery<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId, + clickhouse_settings: { + max_rows_to_read: DEFAULT_SAMPLE_SIZE, + read_overflow_mode: 'break', + }, + }) + .then(res => res.json>()) + .then(d => { + let output: string[]; + if (strategy === 'groupUniqArrayArray') { + output = d.data[0].keysArr as string[]; + } else { + output = d.data.map(row => row.key) as string[]; + } + + return output.filter(r => r); + }); + return keys; + }, + ); + } + + async getMapValues({ + databaseName, + tableName, + column, + key, + maxValues = 20, + connectionId, + }: { + databaseName: string; + tableName: string; + column: string; + key?: string; + maxValues?: number; + connectionId: string; + }) { + const cachedValues = this.cache.get( + `${databaseName}.${tableName}.${column}.${key}.values`, + ); + + if (cachedValues != null) { + return cachedValues; + } + + const sql = key + ? chSql` + SELECT DISTINCT ${{ + Identifier: column, + }}[${{ String: key }}] as value + FROM ${tableExpr({ database: databaseName, table: tableName })} + WHERE value != '' + LIMIT ${{ + Int32: maxValues, + }} + ` + : chSql` + SELECT DISTINCT ${{ + Identifier: column, + }} as value + FROM ${tableExpr({ database: databaseName, table: tableName })} + WHERE value != '' + LIMIT ${{ + Int32: maxValues, + }} + `; + + return this.cache.getOrFetch( + `${databaseName}.${tableName}.${column}.${key}.values`, + async () => { + const values = await sendQuery<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId, + clickhouse_settings: { + max_rows_to_read: DEFAULT_SAMPLE_SIZE, + read_overflow_mode: 'break', + }, + }) + .then(res => res.json>()) + .then(d => d.data.map(row => row.value as string)); + return values; + }, + ); + } + + async getAllFields({ + databaseName, + tableName, + connectionId, + }: { + databaseName: string; + tableName: string; + connectionId: string; + }) { + const fields: Field[] = []; + const columns = await this.getColumns({ + databaseName, + tableName, + connectionId, + }); + + for (const c of columns) { + fields.push({ + path: [c.name], + type: c.type, + jsType: convertCHDataTypeToJSType(c.type), + }); + } + + const mapColumns = filterColumnMetaByType(columns, [JSDataType.Map]) ?? []; + + await Promise.all( + mapColumns.map(async column => { + const keys = await this.getMapKeys({ + databaseName, + tableName, + column: column.name, + connectionId, + }); + + const match = column.type.match(/Map\(.+,\s*(.+)\)/); + const chType = match?.[1] ?? 'String'; // default to string ? + + for (const key of keys) { + fields.push({ + path: [column.name, key], + type: chType, + jsType: convertCHDataTypeToJSType(chType), + }); + } + }), + ); + + return fields; + } + + async getTableMetadata({ + databaseName, + tableName, + connectionId, + }: { + databaseName: string; + tableName: string; + connectionId: string; + }) { + const tableMetadata = await Metadata.queryTableMetadata({ + cache: this.cache, + database: databaseName, + table: tableName, + connectionId, + }); + + return tableMetadata; + } + + async getKeyValues({ + chartConfig, + keys, + limit = 20, + }: { + chartConfig: ChartConfigWithDateRange; + keys: string[]; + limit?: number; + }) { + const sql = await renderChartConfig({ + ...chartConfig, + select: keys + .map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`) + .join(', '), + }); + + const json = await sendQuery<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId: chartConfig.connection, + clickhouse_settings: { + max_rows_to_read: DEFAULT_SAMPLE_SIZE, + read_overflow_mode: 'break', + }, + }).then(res => res.json()); + + return Object.entries(json.data[0]).map(([key, value]) => ({ + key: keys[parseInt(key.replace('param', ''))], + value: (value as string[])?.filter(Boolean), // remove nulls + })); + } +} + +export type Field = { + path: string[]; + type: string; + jsType: JSDataType | null; +}; + +export const metadata = new Metadata(); diff --git a/packages/api/src/common/queryParser.ts b/packages/api/src/common/queryParser.ts new file mode 100644 index 00000000..3362f0e8 --- /dev/null +++ b/packages/api/src/common/queryParser.ts @@ -0,0 +1,717 @@ +import lucene from '@hyperdx/lucene'; +import SqlString from 'sqlstring'; + +import { convertCHTypeToPrimitiveJSType } from '@/common/clickhouse'; +import { Metadata } from '@/common/metadata'; + +function encodeSpecialTokens(query: string): string { + return query + .replace(/\\\\/g, 'HDX_BACKSLASH_LITERAL') + .replace('http://', 'http_COLON_//') + .replace('https://', 'https_COLON_//') + .replace(/localhost:(\d{1,5})/, 'localhost_COLON_$1') + .replace(/\\:/g, 'HDX_COLON'); +} +function decodeSpecialTokens(query: string): string { + return query + .replace(/\\"/g, '"') + .replace(/HDX_BACKSLASH_LITERAL/g, '\\') + .replace('http_COLON_//', 'http://') + .replace('https_COLON_//', 'https://') + .replace(/localhost_COLON_(\d{1,5})/, 'localhost:$1') + .replace(/HDX_COLON/g, ':'); +} + +export function parse(query: string): lucene.AST { + return lucene.parse(encodeSpecialTokens(query)); +} + +const IMPLICIT_FIELD = ''; + +interface Serializer { + operator(op: lucene.Operator): string; + eq(field: string, term: string, isNegatedField: boolean): Promise; + isNotNull(field: string, isNegatedField: boolean): Promise; + gte(field: string, term: string): Promise; + lte(field: string, term: string): Promise; + lt(field: string, term: string): Promise; + gt(field: string, term: string): Promise; + fieldSearch( + field: string, + term: string, + isNegatedField: boolean, + prefixWildcard: boolean, + suffixWildcard: boolean, + ): Promise; + range( + field: string, + start: string, + end: string, + isNegatedField: boolean, + ): Promise; +} + +class EnglishSerializer implements Serializer { + private translateField(field: string) { + if (field === IMPLICIT_FIELD) { + return 'event'; + } + + return `'${field}'`; + } + + operator(op: lucene.Operator) { + switch (op) { + case 'NOT': + case 'AND NOT': + return 'AND NOT'; + case 'OR NOT': + return 'OR NOT'; + // @ts-ignore TODO: Types need to be fixed upstream + case '&&': + case '': + case 'AND': + return 'AND'; + // @ts-ignore TODO: Types need to be fixed upstream + case '||': + case 'OR': + return 'OR'; + default: + throw new Error(`Unexpected operator. ${op}`); + } + } + + async eq(field: string, term: string, isNegatedField: boolean) { + return `${this.translateField(field)} ${ + isNegatedField ? 'is not' : 'is' + } ${term}`; + } + + async isNotNull(field: string, isNegatedField: boolean) { + return `${this.translateField(field)} ${ + isNegatedField ? 'is null' : 'is not null' + }`; + } + + async gte(field: string, term: string) { + return `${this.translateField(field)} is greater than or equal to ${term}`; + } + + async lte(field: string, term: string) { + return `${this.translateField(field)} is less than or equal to ${term}`; + } + + async lt(field: string, term: string) { + return `${this.translateField(field)} is less than ${term}`; + } + + async gt(field: string, term: string) { + return `${this.translateField(field)} is greater than ${term}`; + } + + // async fieldSearch(field: string, term: string, isNegatedField: boolean) { + // return `${this.translateField(field)} ${ + // isNegatedField ? 'does not contain' : 'contains' + // } ${term}`; + // } + + async fieldSearch( + field: string, + term: string, + isNegatedField: boolean, + prefixWildcard: boolean, + suffixWildcard: boolean, + ) { + if (field === IMPLICIT_FIELD) { + return `${this.translateField(field)} ${ + prefixWildcard && suffixWildcard + ? isNegatedField + ? 'does not contain' + : 'contains' + : prefixWildcard + ? isNegatedField + ? 'does not end with' + : 'ends with' + : suffixWildcard + ? isNegatedField + ? 'does not start with' + : 'starts with' + : isNegatedField + ? 'does not have whole word' + : 'has whole word' + } ${term}`; + } else { + return `${this.translateField(field)} ${ + isNegatedField ? 'does not contain' : 'contains' + } ${term}`; + } + } + + async range( + field: string, + start: string, + end: string, + isNegatedField: boolean, + ) { + return `${field} ${ + isNegatedField ? 'is not' : 'is' + } between ${start} and ${end}`; + } +} + +export abstract class SQLSerializer implements Serializer { + private NOT_FOUND_QUERY = '(1 = 0)'; + + abstract getColumnForField(field: string): Promise<{ + column?: string; + propertyType?: 'string' | 'number' | 'bool'; + found: boolean; + }>; + + operator(op: lucene.Operator) { + switch (op) { + case 'NOT': + case 'AND NOT': + return 'AND NOT'; + case 'OR NOT': + return 'OR NOT'; + // @ts-ignore TODO: Types need to be fixed upstream + case '&&': + case '': + case 'AND': + return 'AND'; + // @ts-ignore TODO: Types need to be fixed upstream + case '||': + case 'OR': + return 'OR'; + default: + throw new Error(`Unexpected operator. ${op}`); + } + } + + // Only for exact string matches + async eq(field: string, term: string, isNegatedField: boolean) { + const { column, found, propertyType } = await this.getColumnForField(field); + if (!found) { + return this.NOT_FOUND_QUERY; + } + if (propertyType === 'bool') { + // numeric and boolean fields must be equality matched + const normTerm = `${term}`.trim().toLowerCase(); + return SqlString.format(`(?? ${isNegatedField ? '!' : ''}= ?)`, [ + column, + normTerm === 'true' ? 1 : normTerm === 'false' ? 0 : parseInt(normTerm), + ]); + } else if (propertyType === 'number') { + return SqlString.format( + `(${column} ${isNegatedField ? '!' : ''}= CAST(?, 'Float64'))`, + [term], + ); + } + return SqlString.format(`(${column} ${isNegatedField ? '!' : ''}= ?)`, [ + term, + ]); + } + + async isNotNull(field: string, isNegatedField: boolean) { + const { found, column } = await this.getColumnForField(field); + if (!found) { + return this.NOT_FOUND_QUERY; + } + + return `notEmpty(${column}) ${isNegatedField ? '!' : ''}= 1`; + } + + async gte(field: string, term: string) { + const { column, found } = await this.getColumnForField(field); + if (!found) { + return this.NOT_FOUND_QUERY; + } + return SqlString.format(`(${column} >= ?)`, [term]); + } + + async lte(field: string, term: string) { + const { column, found } = await this.getColumnForField(field); + if (!found) { + return this.NOT_FOUND_QUERY; + } + return SqlString.format(`(${column} <= ?)`, [term]); + } + + async lt(field: string, term: string) { + const { column, found } = await this.getColumnForField(field); + if (!found) { + return this.NOT_FOUND_QUERY; + } + return SqlString.format(`(${column} < ?)`, [term]); + } + + async gt(field: string, term: string) { + const { column, found } = await this.getColumnForField(field); + if (!found) { + return this.NOT_FOUND_QUERY; + } + return SqlString.format(`(${column} > ?)`, [term]); + } + + // TODO: Not sure if SQL really needs this or if it'll coerce itself + private attemptToParseNumber(term: string): string | number { + const number = Number.parseFloat(term); + if (Number.isNaN(number)) { + return term; + } + return number; + } + + // Ref: https://clickhouse.com/codebrowser/ClickHouse/src/Functions/HasTokenImpl.h.html#_ZN2DB12HasTokenImpl16isTokenSeparatorEDu + // Split by anything that's ascii 0-128, that's not a letter or a number + private tokenizeTerm(term: string): string[] { + return term.split(/[ -/:-@[-`{-~\t\n\r]+/).filter(t => t.length > 0); + } + + private termHasSeperators(term: string): boolean { + return term.match(/[ -/:-@[-`{-~\t\n\r]+/) != null; + } + + async fieldSearch( + field: string, + term: string, + isNegatedField: boolean, + prefixWildcard: boolean, + suffixWildcard: boolean, + ) { + const isImplicitField = field === IMPLICIT_FIELD; + const { column, propertyType, found } = await this.getColumnForField(field); + if (!found) { + return this.NOT_FOUND_QUERY; + } + // If it's a string field, we will always try to match with ilike + + if (propertyType === 'bool') { + // numeric and boolean fields must be equality matched + const normTerm = `${term}`.trim().toLowerCase(); + return SqlString.format(`(?? ${isNegatedField ? '!' : ''}= ?)`, [ + column, + normTerm === 'true' ? 1 : normTerm === 'false' ? 0 : parseInt(normTerm), + ]); + } else if (propertyType === 'number') { + return SqlString.format( + `(?? ${isNegatedField ? '!' : ''}= CAST(?, 'Float64'))`, + [column, term], + ); + } + + // // If the query is empty, or is a empty quoted string ex: "" + // // we should match all + if (term.length === 0) { + return '(1=1)'; + } + + if (isImplicitField) { + // For the _source column, we'll try to do whole word searches by default + // to utilize the token bloom filter unless a prefix/sufix wildcard is specified + if (prefixWildcard || suffixWildcard) { + return SqlString.format( + `(lower(??) ${isNegatedField ? 'NOT ' : ''}LIKE lower(?))`, + [ + column, + `${prefixWildcard ? '%' : ''}${term}${suffixWildcard ? '%' : ''}`, + ], + ); + } else { + // We can't search multiple tokens with `hasToken`, so we need to split up the term into tokens + const hasSeperators = this.termHasSeperators(term); + if (hasSeperators) { + const tokens = this.tokenizeTerm(term); + return `(${isNegatedField ? 'NOT (' : ''}${[ + ...tokens.map(token => + SqlString.format(`hasTokenCaseInsensitive(??, ?)`, [ + column, + token, + ]), + ), + // If there are symbols in the term, we'll try to match the whole term as well (ex. Scott!) + SqlString.format(`(lower(??) LIKE lower(?))`, [ + column, + `%${term}%`, + ]), + ].join(' AND ')}${isNegatedField ? ')' : ''})`; + } else { + return SqlString.format( + `(${isNegatedField ? 'NOT ' : ''}hasTokenCaseInsensitive(??, ?))`, + [column, term], + ); + } + } + } else { + const shoudUseTokenBf = isImplicitField; + return SqlString.format( + `(${column} ${isNegatedField ? 'NOT ' : ''}? ?)`, + [SqlString.raw(shoudUseTokenBf ? 'LIKE' : 'ILIKE'), `%${term}%`], + ); + } + } + + async range( + field: string, + start: string, + end: string, + isNegatedField: boolean, + ) { + const { column, found } = await this.getColumnForField(field); + if (!found) { + return this.NOT_FOUND_QUERY; + } + return SqlString.format( + `(${column} ${isNegatedField ? 'NOT ' : ''}BETWEEN ? AND ?)`, + [this.attemptToParseNumber(start), this.attemptToParseNumber(end)], + ); + } +} + +export type CustomSchemaConfig = { + databaseName: string; + implicitColumnExpression?: string; + tableName: string; + connectionId: string; +}; + +export class CustomSchemaSQLSerializerV2 extends SQLSerializer { + private metadata: Metadata; + private tableName: string; + private databaseName: string; + private implicitColumnExpression?: string; + private connectionId: string; + + constructor({ + metadata, + databaseName, + tableName, + connectionId, + implicitColumnExpression, + }: { metadata: Metadata } & CustomSchemaConfig) { + super(); + this.metadata = metadata; + this.databaseName = databaseName; + this.tableName = tableName; + this.implicitColumnExpression = implicitColumnExpression; + this.connectionId = connectionId; + } + + /** + * Translate field from user ex. column.property.subproperty to SQL expression + * Supports: + * - Materialized Columns + * - Map + * - JSON Strings (via JSONExtract) + * TODO: + * - Nested Map + * - JSONExtract for non-string types + */ + private async buildColumnExpressionFromField(field: string) { + const exactMatch = await this.metadata.getColumn({ + databaseName: this.databaseName, + tableName: this.tableName, + column: field, + connectionId: this.connectionId, + }); + + if (exactMatch) { + return { + found: true, + columnType: exactMatch.type, + columnExpression: exactMatch.name, + }; + } + + const fieldPrefix = field.split('.')[0]; + const prefixMatch = await this.metadata.getColumn({ + databaseName: this.databaseName, + tableName: this.tableName, + column: fieldPrefix, + connectionId: this.connectionId, + }); + + if (prefixMatch) { + const fieldPostfix = field.split('.').slice(1).join('.'); + + if (prefixMatch.type.startsWith('Map')) { + const valueType = prefixMatch.type.match(/,\s+(\w+)\)$/)?.[1]; + return { + found: true, + columnExpression: SqlString.format(`??[?]`, [ + prefixMatch.name, + fieldPostfix, + ]), + columnType: valueType ?? 'Unknown', + }; + } else if (prefixMatch.type === 'String') { + // TODO: Support non-strings + const nestedPaths = fieldPostfix.split('.'); + return { + found: true, + columnExpression: SqlString.format( + `JSONExtractString(??, ${Array(nestedPaths.length) + .fill('?') + .join(',')})`, + [prefixMatch.name, ...nestedPaths], + ), + columnType: 'String', + }; + } + // TODO: Support arrays and tuples + throw new Error('Unsupported column type for prefix match'); + } + + throw new Error(`Column not found: ${field}`); + } + + async getColumnForField(field: string) { + if (field === IMPLICIT_FIELD) { + if (!this.implicitColumnExpression) { + throw new Error( + 'Can not search bare text without an implicit column set.', + ); + } + + return { + column: this.implicitColumnExpression, + propertyType: 'string' as const, + found: true, + }; + } + + const expression = await this.buildColumnExpressionFromField(field); + + return { + column: expression.columnExpression, + propertyType: + convertCHTypeToPrimitiveJSType(expression.columnType) ?? undefined, + found: expression.found, + }; + } +} + +async function nodeTerm( + node: lucene.Node, + serializer: Serializer, +): Promise { + const field = node.field[0] === '-' ? node.field.slice(1) : node.field; + let isNegatedField = node.field[0] === '-'; + const isImplicitField = node.field === IMPLICIT_FIELD; + + // NodeTerm + if ((node as lucene.NodeTerm).term != null) { + const nodeTerm = node as lucene.NodeTerm; + let term = decodeSpecialTokens(nodeTerm.term); + // We should only negate the search for negated bare terms (ex. '-5') + // This meeans the field is implicit and the prefix is - + if (isImplicitField && nodeTerm.prefix === '-') { + isNegatedField = true; + } + // Otherwise, if we have a negated term for a field (ex. 'level:-5') + // we should not negate the search, and search for -5 + if (!isImplicitField && nodeTerm.prefix === '-') { + term = nodeTerm.prefix + decodeSpecialTokens(nodeTerm.term); + } + + // TODO: Decide if this is good behavior + // If the term is quoted, we should search for the exact term in a property (ex. foo:"bar") + // Implicit field searches should still use substring matching (ex. "foo bar") + if (nodeTerm.quoted && !isImplicitField) { + return serializer.eq(field, term, isNegatedField); + } + + if (!nodeTerm.quoted && term === '*') { + return serializer.isNotNull(field, isNegatedField); + } + + if (!nodeTerm.quoted && term.substring(0, 2) === '>=') { + if (isNegatedField) { + return serializer.lt(field, term.slice(2)); + } + return serializer.gte(field, term.slice(2)); + } + if (!nodeTerm.quoted && term.substring(0, 2) === '<=') { + if (isNegatedField) { + return serializer.gt(field, term.slice(2)); + } + return serializer.lte(field, term.slice(2)); + } + if (!nodeTerm.quoted && term[0] === '>') { + if (isNegatedField) { + return serializer.lte(field, term.slice(1)); + } + return serializer.gt(field, term.slice(1)); + } + if (!nodeTerm.quoted && term[0] === '<') { + if (isNegatedField) { + return serializer.gte(field, term.slice(1)); + } + return serializer.lt(field, term.slice(1)); + } + + let prefixWildcard = false; + let suffixWildcard = false; + if (!nodeTerm.quoted && term[0] === '*') { + prefixWildcard = true; + term = term.slice(1); + } + if (!nodeTerm.quoted && term[term.length - 1] === '*') { + suffixWildcard = true; + term = term.slice(0, -1); + } + + return serializer.fieldSearch( + field, + term, + isNegatedField, + prefixWildcard, + suffixWildcard, + ); + + // TODO: Handle regex, similarity, boost, prefix + } + // NodeRangedTerm + if ((node as lucene.NodeRangedTerm).inclusive != null) { + const rangedTerm = node as lucene.NodeRangedTerm; + return serializer.range( + field, + rangedTerm.term_min, + rangedTerm.term_max, + isNegatedField, + ); + } + + throw new Error(`Unexpected Node type. ${node}`); +} + +async function serialize( + ast: lucene.AST | lucene.Node, + serializer: Serializer, +): Promise { + // Node Scenarios: + // 1. NodeTerm: Single term ex. "foo:bar" + // 2. NodeRangedTerm: Two terms ex. "foo:[bar TO qux]" + if ((ast as lucene.NodeTerm).term != null) { + return await nodeTerm(ast as lucene.NodeTerm, serializer); + } + if ((ast as lucene.NodeRangedTerm).inclusive != null) { + return await nodeTerm(ast as lucene.NodeTerm, serializer); + } + + // AST Scenarios: + // 1. BinaryAST: Two terms ex. "foo:bar AND baz:qux" + // 2. LeftOnlyAST: Single term ex. "foo:bar" + if ((ast as lucene.BinaryAST).right != null) { + const binaryAST = ast as lucene.BinaryAST; + const operator = serializer.operator(binaryAST.operator); + const parenthesized = binaryAST.parenthesized; + return `${parenthesized ? '(' : ''}${await serialize( + binaryAST.left, + serializer, + )} ${operator} ${await serialize(binaryAST.right, serializer)}${ + parenthesized ? ')' : '' + }`; + } + + if ((ast as lucene.LeftOnlyAST).left != null) { + const leftOnlyAST = ast as lucene.LeftOnlyAST; + const parenthesized = leftOnlyAST.parenthesized; + // start is used when ex. "NOT foo:bar" + return `${parenthesized ? '(' : ''}${ + leftOnlyAST.start != undefined ? `${leftOnlyAST.start} ` : '' + }${await serialize(leftOnlyAST.left, serializer)}${ + parenthesized ? ')' : '' + }`; + } + + // Blank AST, means no text was parsed + return ''; +} + +// TODO: can just inline this within getSearchQuery +export async function genWhereSQL( + ast: lucene.AST, + serializer: Serializer, +): Promise { + return await serialize(ast, serializer); +} + +export class SearchQueryBuilder { + private readonly searchQ: string; + + private readonly conditions: string[]; + + private serializer: SQLSerializer; + + constructor(searchQ: string, serializer: SQLSerializer) { + this.conditions = []; + this.searchQ = searchQ; + // init default serializer + this.serializer = serializer; + } + + setSerializer(serializer: SQLSerializer) { + this.serializer = serializer; + return this; + } + + getSerializer() { + return this.serializer; + } + + private async genSearchQuery() { + if (!this.searchQ) { + return ''; + } + + // const implicitColumn = await this.serializer.getColumnForField( + // IMPLICIT_FIELD, + // ); + + // let querySql = this.searchQ + // .split(/\s+/) + // .map(queryToken => + // SqlString.format(`lower(??) LIKE lower(?)`, [ + // implicitColumn.column, + // `%${queryToken}%`, + // ]), + // ) + // .join(' AND '); + + const parsedQ = parse(this.searchQ); + + return await genWhereSQL(parsedQ, this.serializer); + } + + and(condition: string) { + if (condition && condition.trim()) { + this.conditions.push(`(${condition})`); + } + return this; + } + + async build() { + const searchQuery = await this.genSearchQuery(); + if (this.searchQ) { + this.and(searchQuery); + } + return this.conditions.join(' AND '); + } +} + +export async function genEnglishExplanation(query: string): Promise { + try { + const parsedQ = parse(query); + + if (parsedQ) { + const serializer = new EnglishSerializer(); + return await serialize(parsedQ, serializer); + } + } catch (e) { + console.warn('Parse failure', query, e); + } + + return `Message containing ${query}`; +} diff --git a/packages/api/src/common/renderChartConfig.ts b/packages/api/src/common/renderChartConfig.ts new file mode 100644 index 00000000..90c1584c --- /dev/null +++ b/packages/api/src/common/renderChartConfig.ts @@ -0,0 +1,782 @@ +import isPlainObject from 'lodash/isPlainObject'; +import * as SQLParser from 'node-sql-parser'; + +import { + ChSql, + chSql, + concatChSql, + wrapChSqlIfNotEmpty, +} from '@/common/clickhouse'; +import { DisplayType } from '@/common/DisplayType'; +import { Metadata, metadata } from '@/common/metadata'; +import { + CustomSchemaSQLSerializerV2, + SearchQueryBuilder, +} from '@/common/queryParser'; +import { + AggregateFunction, + AggregateFunctionWithCombinators, + SearchCondition, + SearchConditionLanguage, + SelectList, + SelectSQLStatement, + SortSpecificationList, + SQLInterval, +} from '@/common/sqlTypes'; +import { + convertDateRangeToGranularityString, + getFirstTimestampValueExpression, +} from '@/common/utils'; + +// FIXME: SQLParser.ColumnRef is incomplete +type ColumnRef = SQLParser.ColumnRef & { + array_index?: { + index: { type: string; value: string }; + }[]; +}; + +export type NumberFormat = { + output?: 'currency' | 'percent' | 'byte' | 'time' | 'number'; + mantissa?: number; + thousandSeparated?: boolean; + average?: boolean; + decimalBytes?: boolean; + factor?: number; + currencySymbol?: string; + unit?: string; +}; + +export type SqlAstFilter = { + type: 'sql_ast'; + operator: '=' | '<' | '>' | '!=' | '<=' | '>='; + // SQL Expressions + left: string; + right: string; +}; + +export type Filter = + | { + type: 'lucene' | 'sql'; + condition: SearchCondition; + } + | SqlAstFilter; + +// Used to actually query the data in a given chart +export type ChartConfig = { + displayType?: DisplayType; + numberFormat?: NumberFormat; + timestampValueExpression: string; + implicitColumnExpression?: string; // Where lucene will search if given bare terms + granularity?: SQLInterval | 'auto'; + markdown?: string; // Markdown Content + filtersLogicalOperator?: 'AND' | 'OR'; // Default AND + filters?: Filter[]; // Additional filters to where clause + connection: string; // Connection ID + fillNulls?: number | false; // undefined = 0, false = no fill + selectGroupBy?: boolean; // Add groupBy elements to select statement (default behavior: true) + // TODO: Color support +} & SelectSQLStatement; + +// Saved configuration, has a variable source ID that we pull at query time +export type SavedChartConfig = { + name: string; + source: string; +} & Omit; + +type DateRange = { + dateRange: [Date, Date]; + dateRangeStartInclusive?: boolean; // default true +}; + +export type ChartConfigWithDateRange = ChartConfig & DateRange; +// For non-time-based searches (ex. grab 1 row) +export type ChartConfigWithOptDateRange = Omit< + ChartConfig, + 'timestampValueExpression' +> & { + timestampValueExpression?: string; +} & Partial; + +export const FIXED_TIME_BUCKET_EXPR_ALIAS = '__hdx_time_bucket'; + +export function isUsingGroupBy( + chartConfig: ChartConfigWithOptDateRange, +): chartConfig is Omit & { + groupBy: NonNullable; +} { + return chartConfig.groupBy != null && chartConfig.groupBy.length > 0; +} + +function isUsingGranularity( + chartConfig: ChartConfigWithOptDateRange, +): chartConfig is Omit< + Omit, 'dateRange'>, + 'timestampValueExpression' +> & { + granularity: NonNullable; + dateRange: NonNullable; + timestampValueExpression: NonNullable< + ChartConfigWithDateRange['timestampValueExpression'] + >; +} { + return ( + chartConfig.timestampValueExpression != null && + chartConfig.granularity != null + ); +} + +const INVERSE_OPERATOR_MAP = { + '=': '!=', + '>': '<=', + '<': '>=', + + '!=': '=', + '<=': '>', + '>=': '<', +} as const; +export function inverseSqlAstFilter(filter: SqlAstFilter): SqlAstFilter { + return { + ...filter, + operator: + INVERSE_OPERATOR_MAP[ + filter.operator as keyof typeof INVERSE_OPERATOR_MAP + ], + }; +} + +export function isNonEmptyWhereExpr(where?: string): where is string { + return where != null && where.trim() != ''; +} + +const fastifySQL = ({ + materializedFields, + rawSQL, +}: { + materializedFields: Map; + rawSQL: string; +}) => { + // Parse the SQL AST + try { + const parser = new SQLParser.Parser(); + const ast = parser.astify(rawSQL, { + database: 'Postgresql', + }) as SQLParser.Select; + + // traveral ast and replace the left node with the materialized field + // FIXME: type node (AST type is incomplete): https://github.com/taozhi8833998/node-sql-parser/blob/42ea0b1800c5d425acb8c5ca708a1cee731aada8/types.d.ts#L474 + const traverse = ( + node: + | SQLParser.Expr + | SQLParser.ExpressionValue + | SQLParser.ExprList + | SQLParser.Function + | null, + ) => { + if (node == null) { + return; + } + + let colExpr; + switch (node.type) { + case 'column_ref': { + // FIXME: handle 'Value' type? + const _n = node as ColumnRef; + // @ts-ignore + if (typeof _n.column !== 'string') { + // @ts-ignore + colExpr = `${_n.column?.expr.value}['${_n.array_index?.[0]?.index.value}']`; + } + break; + } + case 'binary_expr': { + const _n = node as SQLParser.Expr; + if (Array.isArray(_n.left)) { + for (const left of _n.left) { + traverse(left); + } + } else { + traverse(_n.left); + } + + if (Array.isArray(_n.right)) { + for (const right of _n.right) { + traverse(right); + } + } else { + traverse(_n.right); + } + break; + } + case 'function': { + const _n = node as SQLParser.Function; + + if (_n.args?.type === 'expr_list') { + if (Array.isArray(_n.args?.value)) { + for (const arg of _n.args.value) { + traverse(arg); + } + + // ex: JSONExtractString(Body, 'message') + if ( + _n.args?.value?.[0]?.type === 'column_ref' && + _n.args?.value?.[1]?.type === 'single_quote_string' + ) { + colExpr = `${_n.name?.name?.[0]?.value}(${(_n.args?.value?.[0] as any)?.column.expr.value}, '${_n.args?.value?.[1]?.value}')`; + } + } + // when _n.args?.value is Expr + else if (isPlainObject(_n.args?.value)) { + traverse(_n.args.value); + } + } + + break; + } + default: + // ignore other types + break; + } + + if (colExpr) { + const materializedField = materializedFields.get(colExpr); + if (materializedField) { + const _n = node as ColumnRef; + // reset the node ref + for (const key in _n) { + // eslint-disable-next-line no-prototype-builtins + if (_n.hasOwnProperty(key)) { + // @ts-ignore + delete _n[key]; + } + } + _n.type = 'column_ref'; + // @ts-ignore + _n.table = null; + // @ts-ignore + _n.column = { expr: { type: 'default', value: materializedField } }; + } + } + }; + + if (Array.isArray(ast.columns)) { + for (const col of ast.columns) { + traverse(col.expr); + } + } + + traverse(ast.where); + + return parser.sqlify(ast); + } catch (e) { + console.error('[renderWhereExpression]feat: Failed to parse SQL AST', e); + return rawSQL; + } +}; + +const aggFnExpr = ({ + fn, + expr, + quantileLevel, + where, +}: { + fn: AggregateFunction | AggregateFunctionWithCombinators; + expr?: string; + quantileLevel?: number; + where?: string; +}) => { + const isCount = fn.startsWith('count'); + const isWhereUsed = isNonEmptyWhereExpr(where); + // Cast to float64 because the expr might not be a number + const unsafeExpr = { UNSAFE_RAW_SQL: `toFloat64OrNull(toString(${expr}))` }; + const whereWithExtraNullCheck = `${where} AND ${unsafeExpr.UNSAFE_RAW_SQL} IS NOT NULL`; + + if (fn.endsWith('Merge')) { + return chSql`${fn}(${{ + UNSAFE_RAW_SQL: expr ?? '', + }})`; + } + // TODO: merge this chunk with the rest of logics + else if (fn.endsWith('State')) { + if (expr == null || isCount) { + return isWhereUsed + ? chSql`${fn}(${{ UNSAFE_RAW_SQL: where }})` + : chSql`${fn}()`; + } + return chSql`${fn}(${unsafeExpr}${ + isWhereUsed ? chSql`, ${{ UNSAFE_RAW_SQL: whereWithExtraNullCheck }}` : '' + })`; + } + + if (fn === 'count') { + if (isWhereUsed) { + return chSql`${fn}If(${{ UNSAFE_RAW_SQL: where }})`; + } + return { + sql: `${fn}()`, + params: {}, + }; + } + + if (expr != null) { + if (fn === 'count_distinct') { + return chSql`count${isWhereUsed ? 'If' : ''}(DISTINCT ${{ + UNSAFE_RAW_SQL: expr, + }}${isWhereUsed ? chSql`, ${{ UNSAFE_RAW_SQL: where }}` : ''})`; + } + + if (quantileLevel != null) { + return chSql`quantile${isWhereUsed ? 'If' : ''}(${{ + // Using Float64 param leads to an added coersion, but we don't need to + // escape number values anyways + UNSAFE_RAW_SQL: Number.isFinite(quantileLevel) + ? `${quantileLevel}` + : '0', + }})(${unsafeExpr}${ + isWhereUsed + ? chSql`, ${{ UNSAFE_RAW_SQL: whereWithExtraNullCheck }}` + : '' + })`; + } + + // TODO: Verify fn is a safe/valid function + return chSql`${{ UNSAFE_RAW_SQL: fn }}${isWhereUsed ? 'If' : ''}( + ${unsafeExpr}${isWhereUsed ? chSql`, ${{ UNSAFE_RAW_SQL: whereWithExtraNullCheck }}` : ''} + )`; + } else { + throw new Error( + 'Column is required for all non-count aggregation functions', + ); + } +}; + +async function renderSelectList( + selectList: SelectList, + chartConfig: ChartConfigWithOptDateRange, + metadata: Metadata, +) { + if (typeof selectList === 'string') { + return chSql`${{ UNSAFE_RAW_SQL: selectList }}`; + } + + const materializedFields = await metadata.getMaterializedColumnsLookupTable({ + connectionId: chartConfig.connection, + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + }); + + return Promise.all( + selectList.map(async select => { + const whereClause = await renderWhereExpression({ + condition: select.aggCondition ?? '', + from: chartConfig.from, + language: select.aggConditionLanguage ?? 'lucene', + implicitColumnExpression: chartConfig.implicitColumnExpression, + metadata, + connectionId: chartConfig.connection, + }); + + let expr: ChSql; + if (select.aggFn == null) { + expr = chSql`${{ UNSAFE_RAW_SQL: select.valueExpression }}`; + } else if (select.aggFn === 'quantile') { + expr = aggFnExpr({ + fn: select.aggFn, + expr: select.valueExpression, + // @ts-ignore (TS doesn't know that we've already checked for quantile) + quantileLevel: select.level, + where: whereClause.sql, + }); + } else { + expr = aggFnExpr({ + fn: select.aggFn, + expr: select.valueExpression, + where: whereClause.sql, + }); + } + + const rawSQL = `SELECT ${expr.sql} FROM \`t\``; + // strip 'SELECT * FROM `t` WHERE ' from the sql + expr.sql = fastifySQL({ materializedFields, rawSQL }) + .replace(/^SELECT\s+/i, '') // Remove 'SELECT ' from the start + .replace(/\s+FROM `t`$/i, ''); // Remove ' FROM t' from the end + + return chSql`${expr}${ + select.alias != null + ? chSql` AS \`${{ UNSAFE_RAW_SQL: select.alias }}\`` + : [] + }`; + }), + ); +} + +function renderSortSpecificationList( + sortSpecificationList: SortSpecificationList, +) { + if (typeof sortSpecificationList === 'string') { + return chSql`${{ UNSAFE_RAW_SQL: sortSpecificationList }}`; + } + + return sortSpecificationList.map(sortSpecification => { + return chSql`${{ UNSAFE_RAW_SQL: sortSpecification.valueExpression }} ${ + sortSpecification.ordering === 'DESC' ? 'DESC' : 'ASC' + }`; + }); +} + +function timeBucketExpr({ + interval, + timestampValueExpression, + dateRange, + alias = FIXED_TIME_BUCKET_EXPR_ALIAS, +}: { + interval: SQLInterval | 'auto'; + timestampValueExpression: string; + dateRange?: [Date, Date]; + alias?: string; +}) { + const unsafeTimestampValueExpression = { + UNSAFE_RAW_SQL: getFirstTimestampValueExpression(timestampValueExpression), + }; + const unsafeInterval = { + UNSAFE_RAW_SQL: + interval === 'auto' && Array.isArray(dateRange) + ? convertDateRangeToGranularityString(dateRange, 60) + : interval, + }; + + return chSql`toStartOfInterval(toDateTime(${unsafeTimestampValueExpression}), INTERVAL ${unsafeInterval}) AS \`${{ + UNSAFE_RAW_SQL: alias, + }}\``; +} + +async function timeFilterExpr({ + timestampValueExpression, + dateRange, + dateRangeStartInclusive, + databaseName, + tableName, + metadata, + connectionId, +}: { + timestampValueExpression: string; + dateRange: [Date, Date]; + dateRangeStartInclusive: boolean; + metadata: Metadata; + connectionId: string; + databaseName: string; + tableName: string; +}) { + const valueExpressions = timestampValueExpression.split(','); + const startTime = dateRange[0].getTime(); + const endTime = dateRange[1].getTime(); + + const whereExprs = await Promise.all( + valueExpressions.map(async expr => { + const col = expr.trim(); + const columnMeta = await metadata.getColumn({ + databaseName, + tableName, + column: col, + connectionId, + }); + + const unsafeTimestampValueExpression = { + UNSAFE_RAW_SQL: col, + }; + + if (columnMeta == null) { + console.warn( + `Column ${col} not found in ${databaseName}.${tableName} while inferring type for time filter`, + ); + } + + // If it's a date type + if (columnMeta?.type === 'Date') { + return chSql`(${unsafeTimestampValueExpression} ${ + dateRangeStartInclusive ? '>=' : '>' + } toDate(fromUnixTimestamp64Milli(${{ + Int64: startTime, + }})) AND ${unsafeTimestampValueExpression} <= toDate(fromUnixTimestamp64Milli(${{ + Int64: endTime, + }})))`; + } else { + return chSql`(${unsafeTimestampValueExpression} ${ + dateRangeStartInclusive ? '>=' : '>' + } fromUnixTimestamp64Milli(${{ + Int64: startTime, + }}) AND ${unsafeTimestampValueExpression} <= fromUnixTimestamp64Milli(${{ + Int64: endTime, + }}))`; + } + }), + ); + + return concatChSql('AND', ...whereExprs); +} + +async function renderSelect( + chartConfig: ChartConfigWithOptDateRange, + metadata: Metadata, +): Promise { + /** + * SELECT + * if granularity: toStartOfInterval, + * if groupBy: groupBy, + * select + */ + const isIncludingTimeBucket = isUsingGranularity(chartConfig); + const isIncludingGroupBy = isUsingGroupBy(chartConfig); + + // TODO: clean up these await mess + return concatChSql( + ',', + await renderSelectList(chartConfig.select, chartConfig, metadata), + isIncludingGroupBy && chartConfig.selectGroupBy !== false + ? await renderSelectList(chartConfig.groupBy, chartConfig, metadata) + : [], + isIncludingTimeBucket + ? timeBucketExpr({ + interval: chartConfig.granularity, + timestampValueExpression: chartConfig.timestampValueExpression, + dateRange: chartConfig.dateRange, + }) + : [], + ); +} + +function renderFrom({ + from, +}: { + from: ChartConfigWithDateRange['from']; +}): ChSql { + return chSql`${{ Identifier: from.databaseName }}.${{ + Identifier: from.tableName, + }}`; +} + +async function renderWhereExpression({ + condition, + language, + metadata, + from, + implicitColumnExpression, + connectionId, +}: { + condition: SearchCondition; + language: SearchConditionLanguage; + metadata: Metadata; + from: ChartConfigWithDateRange['from']; + implicitColumnExpression?: string; + connectionId: string; +}): Promise { + let _condition = condition; + if (language === 'lucene') { + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName: from.databaseName, + tableName: from.tableName, + implicitColumnExpression, + connectionId: connectionId, + }); + const builder = new SearchQueryBuilder(condition, serializer); + _condition = await builder.build(); + } + + const materializedFields = await metadata.getMaterializedColumnsLookupTable({ + connectionId, + databaseName: from.databaseName, + tableName: from.tableName, + }); + + const _sqlPrefix = 'SELECT * FROM `t` WHERE '; + const rawSQL = `${_sqlPrefix}${_condition}`; + // strip 'SELECT * FROM `t` WHERE ' from the sql + _condition = fastifySQL({ materializedFields, rawSQL }).replace( + _sqlPrefix, + '', + ); + return chSql`${{ UNSAFE_RAW_SQL: _condition }}`; +} + +async function renderWhere( + chartConfig: ChartConfigWithOptDateRange, + metadata: Metadata, +): Promise { + let whereSearchCondition: ChSql | [] = []; + if (isNonEmptyWhereExpr(chartConfig.where)) { + whereSearchCondition = wrapChSqlIfNotEmpty( + await renderWhereExpression({ + condition: chartConfig.where, + from: chartConfig.from, + language: chartConfig.whereLanguage ?? 'sql', + implicitColumnExpression: chartConfig.implicitColumnExpression, + metadata, + connectionId: chartConfig.connection, + }), + '(', + ')', + ); + } + + let selectSearchConditions: ChSql[] = []; + if ( + typeof chartConfig.select != 'string' && + // Only if every select has an aggCondition, add to where clause + // otherwise we'll scan all rows anyways + chartConfig.select.every(select => isNonEmptyWhereExpr(select.aggCondition)) + ) { + selectSearchConditions = ( + await Promise.all( + chartConfig.select.map(async select => { + if (isNonEmptyWhereExpr(select.aggCondition)) { + return await renderWhereExpression({ + condition: select.aggCondition, + from: chartConfig.from, + language: select.aggConditionLanguage ?? 'sql', + implicitColumnExpression: chartConfig.implicitColumnExpression, + metadata, + connectionId: chartConfig.connection, + }); + } + return null; + }), + ) + ).filter(v => v !== null) as ChSql[]; + } + + const filterConditions = await Promise.all( + (chartConfig.filters ?? []).map(async filter => { + if (filter.type === 'sql_ast') { + return wrapChSqlIfNotEmpty( + chSql`${{ UNSAFE_RAW_SQL: filter.left }} ${filter.operator} ${{ UNSAFE_RAW_SQL: filter.right }}`, + '(', + ')', + ); + } else if (filter.type === 'lucene' || filter.type === 'sql') { + return wrapChSqlIfNotEmpty( + await renderWhereExpression({ + condition: filter.condition, + from: chartConfig.from, + language: filter.type, + implicitColumnExpression: chartConfig.implicitColumnExpression, + metadata, + connectionId: chartConfig.connection, + }), + '(', + ')', + ); + } + + throw new Error(`Unknown filter type: ${filter.type}`); + }), + ); + + return concatChSql( + ' AND ', + chartConfig.dateRange != null && + chartConfig.timestampValueExpression != null + ? await timeFilterExpr({ + timestampValueExpression: chartConfig.timestampValueExpression, + dateRange: chartConfig.dateRange, + dateRangeStartInclusive: chartConfig.dateRangeStartInclusive ?? true, + metadata, + connectionId: chartConfig.connection, + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + }) + : [], + whereSearchCondition, + // Add aggConditions to where clause to utilize index + wrapChSqlIfNotEmpty(concatChSql(' OR ', selectSearchConditions), '(', ')'), + wrapChSqlIfNotEmpty( + concatChSql( + chartConfig.filtersLogicalOperator === 'OR' ? ' OR ' : ' AND ', + ...filterConditions, + ), + '(', + ')', + ), + ); +} + +async function renderGroupBy( + chartConfig: ChartConfigWithOptDateRange, + metadata: Metadata, +): Promise { + return concatChSql( + ',', + isUsingGroupBy(chartConfig) + ? await renderSelectList(chartConfig.groupBy, chartConfig, metadata) + : [], + isUsingGranularity(chartConfig) + ? timeBucketExpr({ + interval: chartConfig.granularity, + timestampValueExpression: chartConfig.timestampValueExpression, + dateRange: chartConfig.dateRange, + }) + : [], + ); +} + +function renderOrderBy( + chartConfig: ChartConfigWithOptDateRange, +): ChSql | undefined { + const isIncludingTimeBucket = isUsingGranularity(chartConfig); + + if (chartConfig.orderBy == null && !isIncludingTimeBucket) { + return undefined; + } + + return concatChSql( + ',', + isIncludingTimeBucket + ? timeBucketExpr({ + interval: chartConfig.granularity, + timestampValueExpression: chartConfig.timestampValueExpression, + dateRange: chartConfig.dateRange, + }) + : [], + chartConfig.orderBy != null + ? renderSortSpecificationList(chartConfig.orderBy) + : [], + ); +} + +function renderLimit( + chartConfig: ChartConfigWithOptDateRange, +): ChSql | undefined { + if (chartConfig.limit == null || chartConfig.limit.limit == null) { + return undefined; + } + + const offset = + chartConfig.limit.offset != null + ? chSql` OFFSET ${{ Int32: chartConfig.limit.offset }}` + : []; + + return chSql`${{ Int32: chartConfig.limit.limit }}${offset}`; +} + +export async function renderChartConfig( + chartConfig: ChartConfigWithOptDateRange, +): Promise { + const select = await renderSelect(chartConfig, metadata); + const from = renderFrom(chartConfig); + const where = await renderWhere(chartConfig, metadata); + const groupBy = await renderGroupBy(chartConfig, metadata); + const orderBy = renderOrderBy(chartConfig); + const limit = renderLimit(chartConfig); + + return chSql`SELECT ${select} FROM ${from} ${where?.sql ? chSql`WHERE ${where}` : ''} ${ + groupBy?.sql ? chSql`GROUP BY ${groupBy}` : '' + } ${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''} ${ + limit?.sql ? chSql`LIMIT ${limit}` : '' + }`; +} + +// EditForm -> translateToQueriedChartConfig -> QueriedChartConfig +// renderFn(QueriedChartConfig) -> sql +// query(sql) -> data +// formatter(data) -> displayspecificDs +// displaySettings(QueriedChartConfig) -> displaySepcificDs +// chartComponent(displayspecificDs) -> React.Node diff --git a/packages/api/src/common/sqlTypes.ts b/packages/api/src/common/sqlTypes.ts new file mode 100644 index 00000000..2dece52d --- /dev/null +++ b/packages/api/src/common/sqlTypes.ts @@ -0,0 +1,46 @@ +// Derived from SQL grammar spec +// See: https://ronsavage.github.io/SQL/sql-2003-2.bnf.html#query%20specification + +import { z } from 'zod'; + +import { + AggregateFunctionSchema, + AggregateFunctionWithCombinatorsSchema, + DerivedColumnSchema, + SearchConditionLanguageSchema, + SearchConditionSchema, + SelectListSchema, + SortSpecificationListSchema, + SQLIntervalSchema, +} from '@/common/commonTypes'; + +export type SQLInterval = z.infer; + +export type SearchCondition = z.infer; +export type SearchConditionLanguage = z.infer< + typeof SearchConditionLanguageSchema +>; +export type AggregateFunction = z.infer; +export type AggregateFunctionWithCombinators = z.infer< + typeof AggregateFunctionWithCombinatorsSchema +>; + +export type DerivedColumn = z.infer; + +export type SelectList = z.infer; + +export type SortSpecificationList = z.infer; + +type Limit = { limit?: number; offset?: number }; + +export type SelectSQLStatement = { + select: SelectList; + from: { databaseName: string; tableName: string }; + where: SearchCondition; + whereLanguage?: SearchConditionLanguage; + groupBy?: SelectList; + having?: SearchCondition; + havingLanguage?: SearchConditionLanguage; + orderBy?: SortSpecificationList; + limit?: Limit; +}; diff --git a/packages/api/src/common/utils.ts b/packages/api/src/common/utils.ts new file mode 100644 index 00000000..a5779662 --- /dev/null +++ b/packages/api/src/common/utils.ts @@ -0,0 +1,173 @@ +// Port from ChartUtils + source.ts +import { add } from 'date-fns'; + +import type { SQLInterval } from '@/common/sqlTypes'; + +// If a user specifies a timestampValueExpression with multiple columns, +// this will return the first one. We'll want to refine this over time +export function getFirstTimestampValueExpression(valueExpression: string) { + return valueExpression.split(',')[0].trim(); +} + +export enum Granularity { + FifteenSecond = '15 second', + ThirtySecond = '30 second', + OneMinute = '1 minute', + FiveMinute = '5 minute', + TenMinute = '10 minute', + FifteenMinute = '15 minute', + ThirtyMinute = '30 minute', + OneHour = '1 hour', + TwoHour = '2 hour', + SixHour = '6 hour', + TwelveHour = '12 hour', + OneDay = '1 day', + TwoDay = '2 day', + SevenDay = '7 day', + ThirtyDay = '30 day', +} + +export function hashCode(str: string) { + let hash = 0, + i, + chr; + if (str.length === 0) return hash; + for (i = 0; i < str.length; i++) { + chr = str.charCodeAt(i); + hash = (hash << 5) - hash + chr; + hash |= 0; // Convert to 32bit integer + } + return hash; +} + +export function convertDateRangeToGranularityString( + dateRange: [Date, Date], + maxNumBuckets: number, +): Granularity { + const start = dateRange[0].getTime(); + const end = dateRange[1].getTime(); + const diffSeconds = Math.floor((end - start) / 1000); + const granularitySizeSeconds = Math.ceil(diffSeconds / maxNumBuckets); + + if (granularitySizeSeconds <= 15) { + return Granularity.FifteenSecond; + } else if (granularitySizeSeconds <= 30) { + return Granularity.ThirtySecond; + } else if (granularitySizeSeconds <= 60) { + return Granularity.OneMinute; + } else if (granularitySizeSeconds <= 5 * 60) { + return Granularity.FiveMinute; + } else if (granularitySizeSeconds <= 10 * 60) { + return Granularity.TenMinute; + } else if (granularitySizeSeconds <= 15 * 60) { + return Granularity.FifteenMinute; + } else if (granularitySizeSeconds <= 30 * 60) { + return Granularity.ThirtyMinute; + } else if (granularitySizeSeconds <= 3600) { + return Granularity.OneHour; + } else if (granularitySizeSeconds <= 2 * 3600) { + return Granularity.TwoHour; + } else if (granularitySizeSeconds <= 6 * 3600) { + return Granularity.SixHour; + } else if (granularitySizeSeconds <= 12 * 3600) { + return Granularity.TwelveHour; + } else if (granularitySizeSeconds <= 24 * 3600) { + return Granularity.OneDay; + } else if (granularitySizeSeconds <= 2 * 24 * 3600) { + return Granularity.TwoDay; + } else if (granularitySizeSeconds <= 7 * 24 * 3600) { + return Granularity.SevenDay; + } else if (granularitySizeSeconds <= 30 * 24 * 3600) { + return Granularity.ThirtyDay; + } + + return Granularity.ThirtyDay; +} + +export function convertGranularityToSeconds(granularity: SQLInterval): number { + const [num, unit] = granularity.split(' '); + const numInt = Number.parseInt(num); + switch (unit) { + case 'second': + return numInt; + case 'minute': + return numInt * 60; + case 'hour': + return numInt * 60 * 60; + case 'day': + return numInt * 60 * 60 * 24; + default: + return 0; + } +} +// Note: roundToNearestMinutes is broken in date-fns currently +// additionally it doesn't support seconds or > 30min +// so we need to write our own :( +// see: https://github.com/date-fns/date-fns/pull/3267/files +export function toStartOfInterval(date: Date, granularity: SQLInterval): Date { + const [num, unit] = granularity.split(' '); + const numInt = Number.parseInt(num); + const roundFn = Math.floor; + + switch (unit) { + case 'second': + return new Date( + Date.UTC( + date.getUTCFullYear(), + date.getUTCMonth(), + date.getUTCDate(), + date.getUTCHours(), + date.getUTCMinutes(), + roundFn(date.getUTCSeconds() / numInt) * numInt, + ), + ); + case 'minute': + return new Date( + Date.UTC( + date.getUTCFullYear(), + date.getUTCMonth(), + date.getUTCDate(), + date.getUTCHours(), + roundFn(date.getUTCMinutes() / numInt) * numInt, + ), + ); + case 'hour': + return new Date( + Date.UTC( + date.getUTCFullYear(), + date.getUTCMonth(), + date.getUTCDate(), + roundFn(date.getUTCHours() / numInt) * numInt, + ), + ); + case 'day': { + // Clickhouse uses the # of days since unix epoch to round dates + // see: https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/DateLUTImpl.h#L1059 + const daysSinceEpoch = date.getTime() / 1000 / 60 / 60 / 24; + const daysSinceEpochRounded = roundFn(daysSinceEpoch / numInt) * numInt; + + return new Date(daysSinceEpochRounded * 1000 * 60 * 60 * 24); + } + default: + return date; + } +} + +export function timeBucketByGranularity( + start: Date, + end: Date, + granularity: SQLInterval, +): Date[] { + const buckets: Date[] = []; + + let current = toStartOfInterval(start, granularity); + const granularitySeconds = convertGranularityToSeconds(granularity); + while (current < end) { + buckets.push(current); + current = add(current, { + seconds: granularitySeconds, + }); + } + + return buckets; +} diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index 7b879975..ac2209a6 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -1,16 +1,13 @@ -import { getHours, getMinutes } from 'date-fns'; import { sign, verify } from 'jsonwebtoken'; import ms from 'ms'; import { z } from 'zod'; -import * as clickhouse from '@/clickhouse'; -import { SQLSerializer } from '@/clickhouse/searchQueryParser'; import type { ObjectId } from '@/models'; import Alert, { AlertChannel, AlertInterval, AlertSource, - AlertType, + AlertThresholdType, IAlert, } from '@/models/alert'; import Dashboard, { IDashboard } from '@/models/dashboard'; @@ -20,10 +17,10 @@ import logger from '@/utils/logger'; import { alertSchema } from '@/utils/zod'; export type AlertInput = { - source: AlertSource; + source?: AlertSource; channel: AlertChannel; interval: AlertInterval; - type: AlertType; + thresholdType: AlertThresholdType; threshold: number; // Message template @@ -46,60 +43,13 @@ export type AlertInput = { }; }; -const getCron = (interval: AlertInterval) => { - const now = new Date(); - const nowMins = getMinutes(now); - const nowHours = getHours(now); - - switch (interval) { - case '1m': - return '* * * * *'; - case '5m': - return '*/5 * * * *'; - case '15m': - return '*/15 * * * *'; - case '30m': - return '*/30 * * * *'; - case '1h': - return `${nowMins} * * * *`; - case '6h': - return `${nowMins} */6 * * *`; - case '12h': - return `${nowMins} */12 * * *`; - case '1d': - return `${nowMins} ${nowHours} * * *`; - } -}; - -export const validateGroupByProperty = async ({ - groupBy, - logStreamTableVersion, - teamId, -}: { - groupBy: string; - logStreamTableVersion: number | undefined; - teamId: string; -}): Promise => { - const nowInMs = Date.now(); - const propertyTypeMappingsModel = - await clickhouse.buildLogsPropertyTypeMappingsModel( - logStreamTableVersion, - teamId, - nowInMs - ms('1d'), - nowInMs, - ); - const serializer = new SQLSerializer(propertyTypeMappingsModel); - const { found } = await serializer.getColumnForField(groupBy); - return !!found; -}; - -const makeAlert = (alert: AlertInput) => { +const makeAlert = (alert: AlertInput): Partial => { return { channel: alert.channel, interval: alert.interval, source: alert.source, threshold: alert.threshold, - type: alert.type, + thresholdType: alert.thresholdType, // Message template // If they're undefined/null, set it to null so we clear out the field @@ -109,13 +59,11 @@ const makeAlert = (alert: AlertInput) => { message: alert.message == null ? null : alert.message, // Log alerts - savedSearch: alert.savedSearchId, + savedSearch: alert.savedSearchId as unknown as ObjectId, groupBy: alert.groupBy, // Chart alerts - dashboardId: alert.dashboardId, + dashboard: alert.dashboardId as unknown as ObjectId, tileId: alert.tileId, - cron: getCron(alert.interval), - timezone: 'UTC', // TODO: support different timezone }; }; @@ -123,13 +71,13 @@ export const createAlert = async ( teamId: ObjectId, alertInput: z.infer, ) => { - if (alertInput.source === 'CHART') { + if (alertInput.source === AlertSource.TILE) { if ((await Dashboard.findById(alertInput.dashboardId)) == null) { throw new Error('Dashboard ID not found'); } } - if (alertInput.source === 'LOG') { + if (alertInput.source === AlertSource.SAVED_SEARCH) { if ((await SavedSearch.findById(alertInput.savedSearchId)) == null) { throw new Error('Saved Search ID not found'); } @@ -177,11 +125,11 @@ export const getAlertById = async ( export const getAlertsEnhanced = async (teamId: ObjectId) => { return Alert.find({ team: teamId }).populate<{ savedSearch: ISavedSearch; - dashboardId: IDashboard; + dashboard: IDashboard; silenced?: IAlert['silenced'] & { by: IUser; }; - }>(['savedSearch', 'dashboardId', 'silenced.by']); + }>(['savedSearch', 'dashboard', 'silenced.by']); }; export const deleteAlert = async (id: string, teamId: ObjectId) => { diff --git a/packages/api/src/controllers/dashboard.ts b/packages/api/src/controllers/dashboard.ts index 1e49af3b..e4a5a361 100644 --- a/packages/api/src/controllers/dashboard.ts +++ b/packages/api/src/controllers/dashboard.ts @@ -1,11 +1,11 @@ import { differenceBy, uniq } from 'lodash'; import { z } from 'zod'; +import { DashboardWithoutIdSchema, Tile } from '@/common/commonTypes'; import type { ObjectId } from '@/models'; import Alert from '@/models/alert'; import Dashboard from '@/models/dashboard'; -import { DashboardSchema, DashboardWithoutIdSchema } from '@/utils/commonTypes'; -import { chartSchema, tagsSchema } from '@/utils/zod'; +import { tagsSchema } from '@/utils/zod'; export async function getDashboards(teamId: ObjectId) { const dashboards = await Dashboard.find({ @@ -41,7 +41,7 @@ export async function deleteDashboardAndAlerts( team: teamId, }); if (dashboard) { - await Alert.deleteMany({ dashboardId: dashboard._id }); + await Alert.deleteMany({ dashboard: dashboard._id }); } } @@ -50,13 +50,11 @@ export async function updateDashboard( teamId: ObjectId, { name, - charts, - query, + tiles, tags, }: { name: string; - charts: z.infer[]; - query: string; + tiles: Tile[]; tags: z.infer; }, ) { @@ -67,8 +65,7 @@ export async function updateDashboard( }, { name, - charts, - query, + tiles, tags: tags && uniq(tags), }, { new: true }, @@ -114,7 +111,7 @@ export async function updateDashboardAndAlerts( if (deletedTileIds?.length > 0) { await Alert.deleteMany({ - dashboardId: dashboardId, + dashboard: dashboardId, tileId: { $in: deletedTileIds }, }); } diff --git a/packages/api/src/controllers/savedSearch.ts b/packages/api/src/controllers/savedSearch.ts index 2bd7d6ee..9f89919b 100644 --- a/packages/api/src/controllers/savedSearch.ts +++ b/packages/api/src/controllers/savedSearch.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; +import { SavedSearchSchema } from '@/common/commonTypes'; import { SavedSearch } from '@/models/savedSearch'; -import { SavedSearchSchema } from '@/utils/commonTypes'; type SavedSearchWithoutId = Omit, 'id'>; diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 1a941971..f66f4dde 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -1,6 +1,5 @@ import mongoose from 'mongoose'; import request from 'supertest'; -import { z } from 'zod'; import * as clickhouse from '@/clickhouse'; import { @@ -15,13 +14,15 @@ import { } from '@/utils/logParser'; import { redisClient } from '@/utils/redis'; +import { SavedChartConfig, Tile } from './common/commonTypes'; +import { DisplayType } from './common/DisplayType'; import * as config from './config'; +import { AlertInput } from './controllers/alerts'; import { getTeam } from './controllers/team'; import { findUserByEmail } from './controllers/user'; import { mongooseConnection } from './models'; +import { AlertInterval, AlertSource, AlertThresholdType } from './models/alert'; import Server from './server'; -import { Tile } from './utils/commonTypes'; -import { externalAlertSchema } from './utils/zod'; const MOCK_USER = { email: 'fake@deploysentinel.com', @@ -77,8 +78,12 @@ class MockServer extends Server { if (!config.IS_CI) { throw new Error('ONLY execute this in CI env ๐Ÿ˜ˆ !!!'); } - await super.start(); - await initCiEnvs(); + try { + await super.start(); + await initCiEnvs(); + } catch (err) { + console.error(err); + } } stop() { @@ -101,18 +106,7 @@ class MockServer extends Server { } } -class MockAPIServer extends MockServer { - protected readonly appType = 'api'; -} - -export const getServer = (appType: 'api' = 'api') => { - switch (appType) { - case 'api': - return new MockAPIServer(); - default: - throw new Error(`Invalid app type: ${appType}`); - } -}; +export const getServer = () => new MockServer(); export const getAgent = (server: MockServer) => request.agent(server.getHttpServer()); @@ -349,24 +343,32 @@ export const makeTile = (opts?: { id?: string }): Tile => ({ y: 1, w: 1, h: 1, - config: makeChart(), + config: makeChartConfig(), }); -export const makeChart = (opts?: { id?: string }) => ({ - id: opts?.id ?? randomMongoId(), +export const makeChartConfig = (opts?: { id?: string }): SavedChartConfig => ({ name: 'Test Chart', - x: 1, - y: 1, - w: 1, - h: 1, - series: [ + source: 'test-source', + displayType: DisplayType.Line, + select: [ { - type: 'time', - table: 'metrics', + aggFn: 'count', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: '', }, ], + where: '', + whereLanguage: 'lucene', + granularity: 'auto', + implicitColumnExpression: 'Body', + numberFormat: { + output: 'number', + }, + filters: [], }); +// TODO: DEPRECATED export const makeExternalChart = (opts?: { id?: string }) => ({ name: 'Test Chart', x: 1, @@ -382,50 +384,25 @@ export const makeExternalChart = (opts?: { id?: string }) => ({ ], }); -export const makeAlert = ({ +export const makeAlertInput = ({ dashboardId, + interval = '15m', + threshold = 8, tileId, }: { dashboardId: string; + interval?: AlertInterval; + threshold?: number; tileId: string; -}) => ({ +}): Partial => ({ channel: { type: 'webhook', webhookId: 'test-webhook-id', }, - interval: '15m', - threshold: 8, - type: 'presence', - source: 'CHART', + interval, + threshold, + thresholdType: AlertThresholdType.ABOVE, + source: AlertSource.TILE, dashboardId, tileId, }); - -export const makeExternalAlert = ({ - dashboardId, - chartId, - threshold = 8, - interval = '15m', - name, - message, -}: { - dashboardId: string; - chartId: string; - threshold?: number; - interval?: '15m' | '1m' | '5m' | '30m' | '1h' | '6h' | '12h' | '1d'; - name?: string; - message?: string; -}): z.infer => ({ - channel: { - type: 'slack_webhook', - webhookId: '65ad876b6b08426ab4ba7830', - }, - interval, - threshold, - threshold_type: 'above', - source: 'chart', - dashboardId, - chartId, - name, - message, -}); diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index a994985d..4b0ed660 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -2,7 +2,10 @@ import mongoose, { Schema } from 'mongoose'; import type { ObjectId } from '.'; -export type AlertType = 'presence' | 'absence'; +export enum AlertThresholdType { + ABOVE = 'above', + BELOW = 'below', +} export enum AlertState { ALERT = 'ALERT', @@ -22,35 +25,40 @@ export type AlertInterval = | '12h' | '1d'; -export type AlertChannel = { - type: 'webhook'; - webhookId: string; -}; +export type AlertChannel = + | { + type: 'webhook'; + webhookId: string; + } + | { + type: null; + }; -export type AlertSource = 'LOG' | 'CHART'; +export enum AlertSource { + SAVED_SEARCH = 'saved_search', + TILE = 'tile', +} export interface IAlert { _id: ObjectId; channel: AlertChannel; - cron: string; interval: AlertInterval; source?: AlertSource; state: AlertState; team: ObjectId; threshold: number; - timezone: string; - type: AlertType; + thresholdType: AlertThresholdType; // Message template name?: string | null; message?: string | null; - // Log alerts + // SavedSearch alerts groupBy?: string; savedSearch?: ObjectId; - // Chart alerts - dashboardId?: ObjectId; + // Tile alerts + dashboard?: ObjectId; tileId?: string; // Silenced @@ -65,26 +73,19 @@ export type AlertDocument = mongoose.HydratedDocument; const AlertSchema = new Schema( { - type: { - type: String, - required: true, - }, threshold: { type: Number, required: true, }, + thresholdType: { + type: String, + enum: AlertThresholdType, + required: false, + }, interval: { type: String, required: true, }, - timezone: { - type: String, - required: true, - }, - cron: { - type: String, - required: true, - }, channel: Schema.Types.Mixed, // slack, email, etc state: { type: String, @@ -94,7 +95,7 @@ const AlertSchema = new Schema( source: { type: String, required: false, - default: 'LOG', + default: AlertSource.SAVED_SEARCH, }, team: { type: mongoose.Schema.Types.ObjectId, @@ -123,7 +124,7 @@ const AlertSchema = new Schema( }, // Chart alerts - dashboardId: { + dashboard: { type: mongoose.Schema.Types.ObjectId, ref: 'Dashboard', required: false, diff --git a/packages/api/src/models/dashboard.ts b/packages/api/src/models/dashboard.ts index 793cd891..b1c6add4 100644 --- a/packages/api/src/models/dashboard.ts +++ b/packages/api/src/models/dashboard.ts @@ -1,7 +1,7 @@ import mongoose, { Schema } from 'mongoose'; import { z } from 'zod'; -import { DashboardSchema } from '@/utils/commonTypes'; +import { DashboardSchema } from '@/common/commonTypes'; import type { ObjectId } from '.'; diff --git a/packages/api/src/models/savedSearch.ts b/packages/api/src/models/savedSearch.ts index 297a20a3..e2245cd9 100644 --- a/packages/api/src/models/savedSearch.ts +++ b/packages/api/src/models/savedSearch.ts @@ -2,12 +2,12 @@ import mongoose, { Schema } from 'mongoose'; import { v4 as uuidv4 } from 'uuid'; import { z } from 'zod'; -import { SavedSearchSchema } from '@/utils/commonTypes'; +import { SavedSearchSchema } from '@/common/commonTypes'; type ObjectId = mongoose.Types.ObjectId; export interface ISavedSearch - extends Omit, 'source' | 'id'> { + extends Omit, 'source'> { _id: ObjectId; team: ObjectId; source: ObjectId; diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index c9ecd7ab..5c792e0f 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -1,6 +1,6 @@ import mongoose, { Schema } from 'mongoose'; -import { TSource } from '@/utils/commonTypes'; +import { TSource } from '@/common/commonTypes'; type ObjectId = mongoose.Types.ObjectId; diff --git a/packages/api/src/routers/api/__tests__/alerts.test.ts b/packages/api/src/routers/api/__tests__/alerts.test.ts index aef4e978..da6895a5 100644 --- a/packages/api/src/routers/api/__tests__/alerts.test.ts +++ b/packages/api/src/routers/api/__tests__/alerts.test.ts @@ -1,7 +1,7 @@ import { getLoggedInAgent, getServer, - makeAlert, + makeAlertInput, makeTile, randomMongoId, } from '@/fixtures'; @@ -39,13 +39,13 @@ describe('alerts router', () => { const alert = await agent .post('/alerts') .send( - makeAlert({ + makeAlertInput({ dashboardId: dashboard.body.id, tileId: dashboard.body.tiles[0].id, }), ) .expect(200); - expect(alert.body.data.dashboardId).toBe(dashboard.body.id); + expect(alert.body.data.dashboard).toBe(dashboard.body.id); expect(alert.body.data.tileId).toBe(dashboard.body.tiles[0].id); }); @@ -58,7 +58,7 @@ describe('alerts router', () => { const alert = await agent .post('/alerts') .send( - makeAlert({ + makeAlertInput({ dashboardId: resp.body.id, tileId: MOCK_TILES[0].id, }), @@ -78,7 +78,7 @@ describe('alerts router', () => { const alert = await agent .post('/alerts') .send( - makeAlert({ + makeAlertInput({ dashboardId: dashboard.body.id, tileId: MOCK_TILES[0].id, }), @@ -88,6 +88,7 @@ describe('alerts router', () => { .put(`/alerts/${alert.body.data._id}`) .send({ ...alert.body.data, + dashboardId: dashboard.body.id, // because alert.body.data stores 'dashboard' instead of 'dashboardId' threshold: 10, }) .expect(200); @@ -109,7 +110,7 @@ describe('alerts router', () => { agent .post('/alerts') .send( - makeAlert({ + makeAlertInput({ dashboardId: dashboard._id, tileId: tile.id, }), diff --git a/packages/api/src/routers/api/__tests__/dashboard.test.ts b/packages/api/src/routers/api/__tests__/dashboard.test.ts index e4e76c87..67e502df 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.test.ts @@ -1,7 +1,11 @@ -import { getLoggedInAgent, getServer, makeAlert, makeTile } from '@/fixtures'; +import { + getLoggedInAgent, + getServer, + makeAlertInput, + makeTile, +} from '@/fixtures'; const MOCK_DASHBOARD = { - id: '1', name: 'Test Dashboard', tiles: [makeTile(), makeTile(), makeTile(), makeTile(), makeTile()], tags: ['test'], @@ -22,6 +26,54 @@ describe('dashboard router', () => { await server.stop(); }); + it('can create a dashboard', async () => { + const { agent } = await getLoggedInAgent(server); + const dashboard = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + expect(dashboard.body.name).toBe(MOCK_DASHBOARD.name); + expect(dashboard.body.tiles.length).toBe(MOCK_DASHBOARD.tiles.length); + expect(dashboard.body.tiles.map(tile => tile.id)).toEqual( + MOCK_DASHBOARD.tiles.map(tile => tile.id), + ); + }); + + it('can update a dashboard', async () => { + const { agent } = await getLoggedInAgent(server); + const dashboard = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + + const updatedDashboard = await agent + .patch(`/dashboards/${dashboard.body.id}`) + .send({ + ...dashboard.body, + name: 'Updated Dashboard', + tiles: dashboard.body.tiles.slice(1), + }) + .expect(200); + expect(updatedDashboard.body.name).toBe('Updated Dashboard'); + expect(updatedDashboard.body.tiles.length).toBe( + dashboard.body.tiles.length - 1, + ); + expect(updatedDashboard.body.tiles.map(tile => tile.id)).toEqual( + dashboard.body.tiles.slice(1).map(tile => tile.id), + ); + }); + + it('can delete a dashboard', async () => { + const { agent } = await getLoggedInAgent(server); + const dashboard = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + await agent.delete(`/dashboards/${dashboard.body.id}`).expect(204); + const dashboards = await agent.get('/dashboards').expect(200); + expect(dashboards.body.length).toBe(0); + }); + it('deletes attached alerts when deleting tiles', async () => { const { agent } = await getLoggedInAgent(server); @@ -35,7 +87,7 @@ describe('dashboard router', () => { agent .post('/alerts') .send( - makeAlert({ + makeAlertInput({ dashboardId: dashboard._id, tileId: tile.id, }), diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index 3d6db90c..db51efe3 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -1,4 +1,4 @@ -import express, { NextFunction, Request, Response } from 'express'; +import express from 'express'; import _ from 'lodash'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; @@ -9,45 +9,12 @@ import { getAlertById, getAlertsEnhanced, updateAlert, - validateGroupByProperty, } from '@/controllers/alerts'; -import { getTeam } from '@/controllers/team'; import AlertHistory from '@/models/alertHistory'; import { alertSchema, objectIdSchema } from '@/utils/zod'; const router = express.Router(); -// Validate groupBy property -const validateGroupBy = async ( - req: Request, - res: Response, - next: NextFunction, -) => { - const { groupBy, source } = req.body || {}; - if (source === 'LOG' && groupBy) { - const teamId = req.user?.team; - if (teamId == null) { - return res.sendStatus(403); - } - const team = await getTeam(teamId); - if (team == null) { - return res.sendStatus(403); - } - // Validate groupBy property - const groupByValid = await validateGroupByProperty({ - groupBy, - logStreamTableVersion: team.logStreamTableVersion, - teamId: teamId.toString(), - }); - if (!groupByValid) { - return res.status(400).json({ - error: 'Invalid groupBy property', - }); - } - } - next(); -}; - router.get('/', async (req, res, next) => { try { const teamId = req.user?.team; @@ -82,17 +49,12 @@ router.get('/', async (req, res, next) => { } : undefined, channel: _.pick(alert.channel, ['type']), - ...(alert.dashboardId && { + ...(alert.dashboard && { dashboard: { - charts: alert.dashboardId.tiles - .filter(chart => chart.id === alert.tileId) - .map(chart => _.pick(chart, ['id', 'name'])), - ..._.pick(alert.dashboardId, [ - '_id', - 'name', - 'updatedAt', - 'tags', - ]), + tiles: alert.dashboard.tiles + .filter(tile => tile.id === alert.tileId) + .map(tile => _.pick(tile, ['id', 'name'])), + ..._.pick(alert.dashboard, ['_id', 'name', 'updatedAt', 'tags']), }, }), ...(alert.savedSearch && { @@ -108,8 +70,8 @@ router.get('/', async (req, res, next) => { '_id', 'interval', 'threshold', + 'thresholdType', 'state', - 'type', 'source', 'tileId', 'createdAt', @@ -129,7 +91,6 @@ router.get('/', async (req, res, next) => { router.post( '/', validateRequest({ body: alertSchema }), - validateGroupBy, async (req, res, next) => { const teamId = req.user?.team; if (teamId == null) { @@ -154,7 +115,6 @@ router.put( id: objectIdSchema, }), }), - validateGroupBy, async (req, res, next) => { try { const teamId = req.user?.team; diff --git a/packages/api/src/routers/api/connections.ts b/packages/api/src/routers/api/connections.ts index d22802a2..8a1ecf1c 100644 --- a/packages/api/src/routers/api/connections.ts +++ b/packages/api/src/routers/api/connections.ts @@ -1,6 +1,7 @@ import express from 'express'; import { validateRequest } from 'zod-express-middleware'; +import { ConnectionSchema } from '@/common/commonTypes'; import { createConnection, deleteConnection, @@ -9,7 +10,6 @@ import { updateConnection, } from '@/controllers/connection'; import { getNonNullUserWithTeam } from '@/middleware/auth'; -import { ConnectionSchema } from '@/utils/commonTypes'; const router = express.Router(); diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index ef12a2b3..54c4b654 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -4,6 +4,10 @@ import _ from 'lodash'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; +import { + DashboardSchema, + DashboardWithoutIdSchema, +} from '@/common/commonTypes'; import { createDashboard, deleteDashboardAndAlerts, @@ -13,8 +17,6 @@ import { } from '@/controllers/dashboard'; import { getNonNullUserWithTeam } from '@/middleware/auth'; import Alert from '@/models/alert'; -import Dashboard from '@/models/dashboard'; -import { DashboardSchema, DashboardWithoutIdSchema } from '@/utils/commonTypes'; import { chartSchema, objectIdSchema, tagsSchema } from '@/utils/zod'; // create routes that will get and update dashboards @@ -28,9 +30,9 @@ router.get('/', async (req, res, next) => { const alertsByDashboard = groupBy( await Alert.find({ - dashboardId: { $in: dashboards.map(d => d._id) }, + dashboard: { $in: dashboards.map(d => d._id) }, }), - 'dashboardId', + 'dashboard', ); res.json( diff --git a/packages/api/src/routers/api/savedSearch.ts b/packages/api/src/routers/api/savedSearch.ts index 23f97dea..7631eed9 100644 --- a/packages/api/src/routers/api/savedSearch.ts +++ b/packages/api/src/routers/api/savedSearch.ts @@ -3,6 +3,7 @@ import _ from 'lodash'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; +import { SavedSearchSchema } from '@/common/commonTypes'; import { createSavedSearch, deleteSavedSearch, @@ -11,7 +12,6 @@ import { updateSavedSearch, } from '@/controllers/savedSearch'; import { getNonNullUserWithTeam } from '@/middleware/auth'; -import { SavedSearchSchema } from '@/utils/commonTypes'; import { objectIdSchema } from '@/utils/zod'; const router = express.Router(); diff --git a/packages/api/src/routers/api/sources.ts b/packages/api/src/routers/api/sources.ts index 668f7f62..073b02ff 100644 --- a/packages/api/src/routers/api/sources.ts +++ b/packages/api/src/routers/api/sources.ts @@ -2,6 +2,7 @@ import express from 'express'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; +import { SourceSchema } from '@/common/commonTypes'; import { createSource, deleteSource, @@ -9,7 +10,6 @@ import { updateSource, } from '@/controllers/sources'; import { getNonNullUserWithTeam } from '@/middleware/auth'; -import { SourceSchema } from '@/utils/commonTypes'; import { objectIdSchema } from '@/utils/zod'; const router = express.Router(); diff --git a/packages/api/src/routers/external-api/__tests__/alerts.test.ts b/packages/api/src/routers/external-api/__tests__/alerts.test.ts index 5c43f23d..b6508484 100644 --- a/packages/api/src/routers/external-api/__tests__/alerts.test.ts +++ b/packages/api/src/routers/external-api/__tests__/alerts.test.ts @@ -3,18 +3,18 @@ import _ from 'lodash'; import { getLoggedInAgent, getServer, - makeChart, - makeExternalAlert, + makeAlertInput, + makeChartConfig, } from '@/fixtures'; const MOCK_DASHBOARD = { name: 'Test Dashboard', charts: [ - makeChart({ id: 'aaaaaaa' }), - makeChart({ id: 'bbbbbbb' }), - makeChart({ id: 'ccccccc' }), - makeChart({ id: 'ddddddd' }), - makeChart({ id: 'eeeeeee' }), + makeChartConfig({ id: 'aaaaaaa' }), + makeChartConfig({ id: 'bbbbbbb' }), + makeChartConfig({ id: 'ccccccc' }), + makeChartConfig({ id: 'ddddddd' }), + makeChartConfig({ id: 'eeeeeee' }), ], query: 'test query', }; @@ -48,9 +48,9 @@ describe.skip('/api/v1/alerts', () => { .post('/api/v1/alerts') .set('Authorization', `Bearer ${user?.accessKey}`) .send( - makeExternalAlert({ + makeAlertInput({ dashboardId: dashboard._id, - chartId: chart.id, + tileId: chart.id, ...(i % 2 == 0 ? { name: 'test {{hello}}', @@ -177,9 +177,9 @@ Array [ const updateAlert = await agent .put(`/api/v1/alerts/${remainingAlert.id}`) .send( - makeExternalAlert({ + makeAlertInput({ dashboardId: remainingAlert.dashboardId, - chartId: remainingAlert.chartId, + tileId: remainingAlert.chartId, threshold: 1000, interval: '1h', }), diff --git a/packages/api/src/routers/external-api/__tests__/dashboard.test.ts b/packages/api/src/routers/external-api/__tests__/dashboard.test.ts index d9930197..e7ce5486 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboard.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboard.test.ts @@ -3,7 +3,7 @@ import _ from 'lodash'; import { getLoggedInAgent, getServer, - makeExternalAlert, + makeAlertInput, makeExternalChart, } from '@/fixtures'; @@ -229,9 +229,9 @@ Object { .post('/api/v1/alerts') .set('Authorization', `Bearer ${user?.accessKey}`) .send( - makeExternalAlert({ + makeAlertInput({ dashboardId: dashboard.id, - chartId: chart.id, + tileId: chart.id, }), ) .expect(200), diff --git a/packages/api/src/routers/external-api/v1/alerts.ts b/packages/api/src/routers/external-api/v1/alerts.ts index 9ab338a3..98f3ecd6 100644 --- a/packages/api/src/routers/external-api/v1/alerts.ts +++ b/packages/api/src/routers/external-api/v1/alerts.ts @@ -1,4 +1,4 @@ -import express, { NextFunction, Request, Response } from 'express'; +import express from 'express'; import _ from 'lodash'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; @@ -9,50 +9,11 @@ import { getAlertById, getAlerts, updateAlert, - validateGroupByProperty, } from '@/controllers/alerts'; -import { getTeam } from '@/controllers/team'; -import { - externalAlertSchema, - objectIdSchema, - translateAlertDocumentToExternalAlert, - translateExternalAlertToInternalAlert, -} from '@/utils/zod'; +import { alertSchema, objectIdSchema } from '@/utils/zod'; const router = express.Router(); -// TODO: Dedup with private API router -// Validate groupBy property -const validateGroupBy = async ( - req: Request, - res: Response, - next: NextFunction, -) => { - const { groupBy, source } = req.body || {}; - if (source === 'LOG' && groupBy) { - const teamId = req.user?.team; - if (teamId == null) { - return res.sendStatus(403); - } - const team = await getTeam(teamId); - if (team == null) { - return res.sendStatus(403); - } - // Validate groupBy property - const groupByValid = await validateGroupByProperty({ - groupBy, - logStreamTableVersion: team.logStreamTableVersion, - teamId: teamId.toString(), - }); - if (!groupByValid) { - return res.status(400).json({ - error: 'Invalid groupBy property', - }); - } - } - next(); -}; - router.get( '/:id', validateRequest({ @@ -74,7 +35,7 @@ router.get( } return res.json({ - data: translateAlertDocumentToExternalAlert(alert), + data: alert, }); } catch (e) { next(e); @@ -92,49 +53,37 @@ router.get('/', async (req, res, next) => { const alerts = await getAlerts(teamId); return res.json({ - data: alerts.map(alert => { - return translateAlertDocumentToExternalAlert(alert); - }), + data: alerts, }); } catch (e) { next(e); } }); -router.post( - '/', - validateRequest({ body: externalAlertSchema }), - validateGroupBy, - async (req, res, next) => { - const teamId = req.user?.team; - if (teamId == null) { - return res.sendStatus(403); - } - try { - const alertInput = req.body; +router.post('/', async (req, res, next) => { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + try { + const alertInput = req.body; - const internalAlert = translateExternalAlertToInternalAlert(alertInput); - - return res.json({ - data: translateAlertDocumentToExternalAlert( - await createAlert(teamId, internalAlert), - ), - }); - } catch (e) { - next(e); - } - }, -); + return res.json({ + data: await createAlert(teamId, alertInput), + }); + } catch (e) { + next(e); + } +}); router.put( '/:id', validateRequest({ - body: externalAlertSchema, + body: alertSchema, params: z.object({ id: objectIdSchema, }), }), - validateGroupBy, async (req, res, next) => { try { const teamId = req.user?.team; @@ -145,15 +94,14 @@ router.put( const { id } = req.params; const alertInput = req.body; - const internalAlert = translateExternalAlertToInternalAlert(alertInput); - const alert = await updateAlert(id, teamId, internalAlert); + const alert = await updateAlert(id, teamId, alertInput); if (alert == null) { return res.sendStatus(404); } res.json({ - data: translateAlertDocumentToExternalAlert(alert), + data: alert, }); } catch (e) { next(e); diff --git a/packages/api/src/routers/external-api/v1/dashboards.ts b/packages/api/src/routers/external-api/v1/dashboards.ts index bbf54f84..32c070af 100644 --- a/packages/api/src/routers/external-api/v1/dashboards.ts +++ b/packages/api/src/routers/external-api/v1/dashboards.ts @@ -4,6 +4,7 @@ import { ObjectId } from 'mongodb'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; +import { TileSchema } from '@/common/commonTypes'; import { deleteDashboardAndAlerts, updateDashboard, @@ -130,8 +131,7 @@ router.put( }), body: z.object({ name: z.string().max(1024), - charts: z.array(externalChartSchemaWithId), - query: z.string().max(2048), + tiles: z.array(TileSchema), tags: tagsSchema, }), }), @@ -146,16 +146,11 @@ router.put( return res.sendStatus(400); } - const { name, charts, query, tags } = req.body ?? {}; - - const internalCharts = charts.map(chart => { - return translateExternalChartToInternalChart(chart); - }); + const { name, tiles, tags } = req.body ?? {}; const updatedDashboard = await updateDashboard(dashboardId, teamId, { name, - charts: internalCharts, - query, + tiles, tags, }); diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 00a52b90..28b4b335 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -2,19 +2,19 @@ import http from 'http'; import gracefulShutdown from 'http-graceful-shutdown'; import { serializeError } from 'serialize-error'; -import apiServer from './api-app'; -import * as config from './config'; -import { connectDB, mongooseConnection } from './models'; -import logger from './utils/logger'; -import redisClient from './utils/redis'; +import app from '@/api-app'; +import * as config from '@/config'; +import { connectDB, mongooseConnection } from '@/models'; +import logger from '@/utils/logger'; +import redisClient from '@/utils/redis'; export default class Server { protected shouldHandleGracefulShutdown = true; protected httpServer!: http.Server; - private async createServer() { - return http.createServer(apiServer); + private createServer() { + return http.createServer(app); } protected async shutdown(signal?: string) { @@ -45,7 +45,7 @@ export default class Server { } async start() { - this.httpServer = await this.createServer(); + this.httpServer = this.createServer(); this.httpServer.keepAliveTimeout = 61000; // Ensure all inactive connections are terminated by the ALB, by setting this a few seconds higher than the ALB idle timeout this.httpServer.headersTimeout = 62000; // Ensure the headersTimeout is set higher than the keepAliveTimeout due to this nodejs regression bug: https://github.com/nodejs/node/issues/27363 diff --git a/packages/api/src/tasks/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/__tests__/checkAlerts.test.ts index bee01194..c66e0bf2 100644 --- a/packages/api/src/tasks/__tests__/checkAlerts.test.ts +++ b/packages/api/src/tasks/__tests__/checkAlerts.test.ts @@ -2,25 +2,23 @@ import ms from 'ms'; +import { createAlert } from '@/controllers/alerts'; +import { createTeam } from '@/controllers/team'; import { buildMetricSeries, generateBuildTeamEventFn, getServer, + makeTile, mockLogsPropertyTypeMappingsModel, mockSpyMetricPropertyTypeMappingsModel, } from '@/fixtures'; -import { LogType } from '@/utils/logParser'; - -import * as clickhouse from '../../clickhouse'; -import { createAlert } from '../../controllers/alerts'; -import { createTeam } from '../../controllers/team'; -import AlertHistory from '../../models/alertHistory'; -import Dashboard from '../../models/dashboard'; -import LogView from '../../models/logView'; -import Webhook from '../../models/webhook'; -import * as slack from '../../utils/slack'; -import * as checkAlert from '../checkAlerts'; +import { AlertSource, AlertThresholdType } from '@/models/alert'; +import AlertHistory from '@/models/alertHistory'; +import Dashboard from '@/models/dashboard'; +import LogView from '@/models/logView'; +import Webhook from '@/models/webhook'; import { + AlertMessageTemplateDefaultView, buildAlertMessageTemplateHdxLink, buildAlertMessageTemplateTitle, buildLogSearchLink, @@ -32,8 +30,23 @@ import { renderAlertTemplate, roundDownToXMinutes, translateExternalActionsToInternal, -} from '../checkAlerts'; +} from '@/tasks/checkAlerts'; +import { LogType } from '@/utils/logParser'; +import * as slack from '@/utils/slack'; +const MOCK_DASHBOARD = { + name: 'Test Dashboard', + tiles: [makeTile(), makeTile()], + tags: ['test'], +}; + +const MOCK_SOURCE = {}; + +const MOCK_SAVED_SEARCH: any = { + id: 'fake-saved-search-id', +}; + +// TODO: fix tests describe.skip('checkAlerts', () => { afterAll(async () => { await clickhouse.client.close(); @@ -67,21 +80,16 @@ describe.skip('checkAlerts', () => { buildLogSearchLink({ startTime: new Date('2023-03-17T22:13:03.103Z'), endTime: new Date('2023-03-17T22:13:59.103Z'), - logViewId: '123', + savedSearch: MOCK_SAVED_SEARCH, }), - ).toBe( - 'http://localhost:9090/search/123?from=1679091183103&to=1679091239103', - ); + ).toMatchInlineSnapshot(''); expect( buildLogSearchLink({ startTime: new Date('2023-03-17T22:13:03.103Z'), endTime: new Date('2023-03-17T22:13:59.103Z'), - logViewId: '123', - q: '๐Ÿฑ foo:"bar"', + savedSearch: MOCK_SAVED_SEARCH, }), - ).toBe( - 'http://localhost:9090/search/123?from=1679091183103&to=1679091239103&q=%F0%9F%90%B1+foo%3A%22bar%22', - ); + ).toMatchInlineSnapshot(''); }); it('doesExceedThreshold', () => { @@ -145,50 +153,58 @@ describe.skip('checkAlerts', () => { }); describe('Alert Templates', () => { - const defaultSearchView: any = { + const defaultSearchView: AlertMessageTemplateDefaultView = { alert: { - threshold_type: 'above', + thresholdType: AlertThresholdType.ABOVE, threshold: 1, - source: 'search', - groupBy: 'span_name', + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: 'fake-webhook-id', + }, + interval: '1m', }, savedSearch: { - id: 'id-123', - query: 'level:error', + _id: 'fake-saved-search-id' as any, + team: 'team-123' as any, + id: 'fake-saved-search-id', name: 'My Search', + select: 'Body', + where: 'Body: "error"', + whereLanguage: 'lucene', + orderBy: 'timestamp', + source: 'fake-source-id' as any, + tags: ['test'], }, - team: { - id: 'team-123', - logStreamTableVersion: 1, - }, + attributes: {}, + granularity: '1m', group: 'http', startTime: new Date('2023-03-17T22:13:03.103Z'), endTime: new Date('2023-03-17T22:13:59.103Z'), value: 10, }; - const defaultChartView: any = { + const defaultChartView: AlertMessageTemplateDefaultView = { alert: { - threshold_type: 'below', - threshold: 10, - source: 'chart', - groupBy: 'span_name', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: 'fake-webhook-id', + }, + interval: '1m', }, dashboard: { id: 'id-123', name: 'My Dashboard', - charts: [ - { - name: 'My Chart', - }, - ], - }, - team: { - id: 'team-123', - logStreamTableVersion: 1, + tiles: [makeTile()], + team: 'team-123' as any, + tags: ['test'], }, startTime: new Date('2023-03-17T22:13:03.103Z'), endTime: new Date('2023-03-17T22:13:59.103Z'), + attributes: {}, granularity: '5 minute', value: 5, }; @@ -209,11 +225,15 @@ describe.skip('checkAlerts', () => { }); it('buildAlertMessageTemplateHdxLink', () => { - expect(buildAlertMessageTemplateHdxLink(defaultSearchView)).toBe( - 'http://localhost:9090/search/id-123?from=1679091183103&to=1679091239103&q=level%3Aerror+span_name%3A%22http%22', + expect( + buildAlertMessageTemplateHdxLink(defaultSearchView), + ).toMatchInlineSnapshot( + `"http://app:8080/search/fake-saved-search-id?from=1679091183103&to=1679091239103"`, ); - expect(buildAlertMessageTemplateHdxLink(defaultChartView)).toBe( - 'http://localhost:9090/dashboards/id-123?from=1679089083103&granularity=5+minute&to=1679093339103', + expect( + buildAlertMessageTemplateHdxLink(defaultChartView), + ).toMatchInlineSnapshot( + `"http://app:8080/dashboards/id-123?from=1679089083103&granularity=5+minute&to=1679093339103"`, ); }); @@ -222,23 +242,25 @@ describe.skip('checkAlerts', () => { buildAlertMessageTemplateTitle({ view: defaultSearchView, }), - ).toBe('Alert for "My Search" - 10 lines found'); + ).toMatchInlineSnapshot(`"Alert for \\"My Search\\" - 10 lines found"`); expect( buildAlertMessageTemplateTitle({ view: defaultChartView, }), - ).toBe('Alert for "My Chart" in "My Dashboard" - 5 falls below 10'); + ).toMatchInlineSnapshot( + `"Alert for \\"Test Chart\\" in \\"My Dashboard\\" - 5 exceeds 1"`, + ); }); it('getDefaultExternalAction', () => { expect( getDefaultExternalAction({ channel: { - type: 'slack_webhook', + type: 'webhook', webhookId: '123', }, } as any), - ).toBe('@slack_webhook-123'); + ).toBe('@webhook-123'); expect( getDefaultExternalAction({ channel: { @@ -251,20 +273,20 @@ describe.skip('checkAlerts', () => { it('translateExternalActionsToInternal', () => { // normal expect( - translateExternalActionsToInternal('@slack_webhook-123'), + translateExternalActionsToInternal('@webhook-123'), ).toMatchInlineSnapshot( - `"{{__hdx_notify_channel__ channel=\\"slack_webhook\\" id=\\"123\\"}}"`, + `"{{__hdx_notify_channel__ channel=\\"webhook\\" id=\\"123\\"}}"`, ); // with multiple breaks expect( translateExternalActionsToInternal(` -@slack_webhook-123 +@webhook-123 `), ).toMatchInlineSnapshot(` " -{{__hdx_notify_channel__ channel=\\"slack_webhook\\" id=\\"123\\"}} +{{__hdx_notify_channel__ channel=\\"webhook\\" id=\\"123\\"}} " `); @@ -306,20 +328,6 @@ describe.skip('checkAlerts', () => { it('renderAlertTemplate - with existing channel', async () => { jest.spyOn(slack, 'postMessageToWebhook').mockResolvedValue(null as any); - jest.spyOn(clickhouse, 'getLogBatch').mockResolvedValueOnce({ - data: [ - { - timestamp: '2023-11-16T22:10:00.000Z', - severity_text: 'error', - body: 'Oh no! Something went wrong!', - }, - { - timestamp: '2023-11-16T22:15:00.000Z', - severity_text: 'info', - body: 'All good!', - }, - ], - } as any); const team = await createTeam({ name: 'My Team' }); const webhook = await new Webhook({ @@ -330,13 +338,13 @@ describe.skip('checkAlerts', () => { }).save(); await renderAlertTemplate({ - template: 'Custom body @slack_webhook-My_Web', // partial name should work + template: 'Custom body @webhook-My_Web', // partial name should work view: { ...defaultSearchView, alert: { ...defaultSearchView.alert, channel: { - type: 'slack_webhook', + type: 'webhook', webhookId: webhook._id.toString(), }, }, @@ -344,7 +352,6 @@ describe.skip('checkAlerts', () => { title: 'Alert for "My Search" - 10 lines found', team: { id: team._id.toString(), - logStreamTableVersion: team.logStreamTableVersion, }, }); @@ -356,20 +363,6 @@ describe.skip('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') .mockResolvedValueOnce(null as any); - jest.spyOn(clickhouse, 'getLogBatch').mockResolvedValueOnce({ - data: [ - { - timestamp: '2023-11-16T22:10:00.000Z', - severity_text: 'error', - body: 'Oh no! Something went wrong!', - }, - { - timestamp: '2023-11-16T22:15:00.000Z', - severity_text: 'info', - body: 'All good!', - }, - ], - } as any); const team = await createTeam({ name: 'My Team' }); await new Webhook({ @@ -380,7 +373,7 @@ describe.skip('checkAlerts', () => { }).save(); await renderAlertTemplate({ - template: 'Custom body @slack_webhook-My_Web', // partial name should work + template: 'Custom body @webhook-My_Web', // partial name should work view: { ...defaultSearchView, alert: { @@ -393,7 +386,6 @@ describe.skip('checkAlerts', () => { title: 'Alert for "My Search" - 10 lines found', team: { id: team._id.toString(), - logStreamTableVersion: team.logStreamTableVersion, }, }); @@ -406,13 +398,12 @@ describe.skip('checkAlerts', () => { { text: { text: [ - '**', + '**', 'Group: "http"', '10 lines found, expected less than 1 lines', 'Custom body ', '```', - 'Nov 16 22:10:00Z [error] Oh no! Something went wrong!', - 'Nov 16 22:15:00Z [info] All good!', + '', '```', ].join('\n'), type: 'mrkdwn', @@ -428,20 +419,6 @@ describe.skip('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') .mockResolvedValueOnce(null as any); - jest.spyOn(clickhouse, 'getLogBatch').mockResolvedValueOnce({ - data: [ - { - timestamp: '2023-11-16T22:10:00.000Z', - severity_text: 'error', - body: 'Oh no! Something went wrong!', - }, - { - timestamp: '2023-11-16T22:15:00.000Z', - severity_text: 'info', - body: 'All good!', - }, - ], - } as any); const team = await createTeam({ name: 'My Team' }); await new Webhook({ @@ -452,7 +429,7 @@ describe.skip('checkAlerts', () => { }).save(); await renderAlertTemplate({ - template: 'Custom body @slack_webhook-{{attributes.webhookName}}', // partial name should work + template: 'Custom body @webhook-{{attributes.webhookName}}', // partial name should work view: { ...defaultSearchView, alert: { @@ -468,7 +445,6 @@ describe.skip('checkAlerts', () => { title: 'Alert for "My Search" - 10 lines found', team: { id: team._id.toString(), - logStreamTableVersion: team.logStreamTableVersion, }, }); @@ -481,13 +457,12 @@ describe.skip('checkAlerts', () => { { text: { text: [ - '**', + '**', 'Group: "http"', '10 lines found, expected less than 1 lines', 'Custom body ', '```', - 'Nov 16 22:10:00Z [error] Oh no! Something went wrong!', - 'Nov 16 22:15:00Z [info] All good!', + '', '```', ].join('\n'), type: 'mrkdwn', @@ -501,20 +476,6 @@ describe.skip('checkAlerts', () => { it('renderAlertTemplate - #is_match with single action', async () => { jest.spyOn(slack, 'postMessageToWebhook').mockResolvedValue(null as any); - jest.spyOn(clickhouse, 'getLogBatch').mockResolvedValueOnce({ - data: [ - { - timestamp: '2023-11-16T22:10:00.000Z', - severity_text: 'error', - body: 'Oh no! Something went wrong!', - }, - { - timestamp: '2023-11-16T22:15:00.000Z', - severity_text: 'info', - body: 'All good!', - }, - ], - } as any); const team = await createTeam({ name: 'My Team' }); await new Webhook({ @@ -535,10 +496,10 @@ describe.skip('checkAlerts', () => { {{#is_match "attributes.k8s.pod.name" "otel-collector-123"}} Runbook URL: {{attributes.runbook.url}} hi i matched - @slack_webhook-My_Web + @webhook-My_Web {{/is_match}} -@slack_webhook-Another_Webhook +@webhook-Another_Webhook `, // partial name should work view: { ...defaultSearchView, @@ -562,14 +523,13 @@ describe.skip('checkAlerts', () => { title: 'Alert for "My Search" - 10 lines found', team: { id: team._id.toString(), - logStreamTableVersion: team.logStreamTableVersion, }, }); - // @slack_webhook should not be called + // @webhook should not be called await renderAlertTemplate({ template: - '{{#is_match "attributes.host" "web"}} @slack_webhook-My_Web {{/is_match}}', // partial name should work + '{{#is_match "attributes.host" "web"}} @webhook-My_Web {{/is_match}}', // partial name should work view: { ...defaultSearchView, alert: { @@ -585,7 +545,6 @@ describe.skip('checkAlerts', () => { title: 'Alert for "My Search" - 10 lines found', team: { id: team._id.toString(), - logStreamTableVersion: team.logStreamTableVersion, }, }); @@ -598,7 +557,7 @@ describe.skip('checkAlerts', () => { { text: { text: [ - '**', + '**', 'Group: "http"', '10 lines found, expected less than 1 lines', '', @@ -608,8 +567,7 @@ describe.skip('checkAlerts', () => { '', '', '```', - 'Nov 16 22:10:00Z [error] Oh no! Something went wrong!', - 'Nov 16 22:15:00Z [info] All good!', + '', '```', ].join('\n'), type: 'mrkdwn', @@ -627,7 +585,7 @@ describe.skip('checkAlerts', () => { { text: { text: [ - '**', + '**', 'Group: "http"', '10 lines found, expected less than 1 lines', '', @@ -637,8 +595,7 @@ describe.skip('checkAlerts', () => { '', '', '```', - 'Nov 16 22:10:00Z [error] Oh no! Something went wrong!', - 'Nov 16 22:15:00Z [info] All good!', + '', '```', ].join('\n'), type: 'mrkdwn', @@ -667,7 +624,7 @@ describe.skip('checkAlerts', () => { await server.stop(); }); - it('LOG alert - slack webhook', async () => { + it('SAVED_SEARCH alert - slack webhook', async () => { jest .spyOn(slack, 'postMessageToWebhook') .mockResolvedValueOnce(null as any); @@ -688,16 +645,6 @@ describe.skip('checkAlerts', () => { rows: 0, data: [], } as any); - jest.spyOn(clickhouse, 'getLogBatch').mockResolvedValueOnce({ - rows: 1, - data: [ - { - timestamp: '2023-11-16T22:10:00.000Z', - severity_text: 'error', - body: 'Oh no! Something went wrong!', - }, - ], - } as any); const team = await createTeam({ name: 'My Team' }); const logView = await new LogView({ @@ -765,7 +712,6 @@ describe.skip('checkAlerts', () => { groupBy: alert.groupBy, q: logView.query, startTime: new Date('2023-11-16T22:05:00.000Z'), - tableVersion: team.logStreamTableVersion, teamId: logView.team._id.toString(), windowSizeInMins: 5, }); @@ -1302,7 +1248,6 @@ describe.skip('checkAlerts', () => { groupBy: alert.groupBy, q: logView.query, startTime: new Date('2023-11-16T22:05:00.000Z'), - tableVersion: team.logStreamTableVersion, teamId: logView.team._id.toString(), windowSizeInMins: 5, }); diff --git a/packages/api/src/tasks/checkAlerts.ts b/packages/api/src/tasks/checkAlerts.ts index 195572ac..0200f9e6 100644 --- a/packages/api/src/tasks/checkAlerts.ts +++ b/packages/api/src/tasks/checkAlerts.ts @@ -1,4 +1,3 @@ -// @ts-nocheck TODO: Fix When Restoring Alerts // -------------------------------------------------------- // -------------- EXECUTE EVERY MINUTE -------------------- // -------------------------------------------------------- @@ -12,61 +11,65 @@ import ms from 'ms'; import PromisedHandlebars from 'promised-handlebars'; import { serializeError } from 'serialize-error'; import { URLSearchParams } from 'url'; -import { z } from 'zod'; -import * as clickhouse from '@/clickhouse'; +import { Tile } from '@/common/commonTypes'; +import { DisplayType } from '@/common/DisplayType'; +import { + ChartConfigWithOptDateRange, + FIXED_TIME_BUCKET_EXPR_ALIAS, +} from '@/common/renderChartConfig'; +import { renderChartConfig } from '@/common/renderChartConfig'; import * as config from '@/config'; +import { AlertInput } from '@/controllers/alerts'; import { ObjectId } from '@/models'; -import Alert, { AlertDocument, AlertState } from '@/models/alert'; +import Alert, { + AlertDocument, + AlertSource, + AlertState, + AlertThresholdType, + IAlert, +} from '@/models/alert'; import AlertHistory, { IAlertHistory } from '@/models/alertHistory'; import Dashboard, { IDashboard } from '@/models/dashboard'; -import LogView from '@/models/logView'; +import { ISavedSearch } from '@/models/savedSearch'; +import { ISource, Source } from '@/models/source'; import { ITeam } from '@/models/team'; import Webhook, { IWebhook } from '@/models/webhook'; import { convertMsToGranularityString, truncateString } from '@/utils/common'; -import { translateDashboardDocumentToExternalDashboard } from '@/utils/externalApi'; import logger from '@/utils/logger'; import * as slack from '@/utils/slack'; -import { - externalAlertSchema, - translateAlertDocumentToExternalAlert, -} from '@/utils/zod'; - -type EnhancedDashboard = Omit & { team: ITeam }; const MAX_MESSAGE_LENGTH = 500; const NOTIFY_FN_NAME = '__hdx_notify_channel__'; const IS_MATCH_FN_NAME = 'is_match'; -const getLogViewEnhanced = async (logViewId: ObjectId) => { - const logView = await LogView.findById(logViewId).populate<{ - team: ITeam; - }>('team'); - if (!logView) { - throw new Error(`LogView ${logViewId} not found `); - } - return logView; +type EnhancedSavedSearch = Omit & { + source: ISource; }; +const getAlerts = () => + Alert.find({}).populate<{ + team: ITeam; + savedSearch?: EnhancedSavedSearch; + dashboard?: IDashboard; + }>(['team', 'savedSearch', 'savedSearch.source', 'dashboard']); + +type EnhancedAlert = Awaited>[0]; + export const buildLogSearchLink = ({ endTime, - logViewId, - q, + savedSearch, startTime, }: { endTime: Date; - logViewId: string; - q?: string; + savedSearch: EnhancedSavedSearch; startTime: Date; }) => { - const url = new URL(`${config.FRONTEND_URL}/search/${logViewId}`); + const url = new URL(`${config.FRONTEND_URL}/search/${savedSearch.id}`); const queryParams = new URLSearchParams({ from: startTime.getTime().toString(), to: endTime.getTime().toString(), }); - if (q) { - queryParams.append('q', q); - } url.search = queryParams.toString(); return url.toString(); }; @@ -143,22 +146,14 @@ export const expandToNestedObject = ( // ----------------- Alert Message Template ------------------- // ------------------------------------------------------------ // should match the external alert schema -type AlertMessageTemplateDefaultView = { - // FIXME: do we want to include groupBy in the external alert schema? - alert: z.infer & { groupBy?: string }; +export type AlertMessageTemplateDefaultView = { + alert: AlertInput; attributes: ReturnType; - dashboard: ReturnType< - typeof translateDashboardDocumentToExternalDashboard - > | null; + dashboard?: IDashboard | null; endTime: Date; granularity: string; group?: string; - // TODO: use a translation function ? - savedSearch: { - id: string; - name: string; - query: string; - } | null; + savedSearch?: EnhancedSavedSearch | null; startTime: Date; value: number; }; @@ -180,7 +175,7 @@ export const notifyChannel = async ({ }; }) => { switch (channel) { - case 'slack_webhook': { + case 'webhook': { const webhook = await Webhook.findOne({ team: team.id, ...(mongoose.isValidObjectId(id) @@ -312,26 +307,21 @@ export const buildAlertMessageTemplateHdxLink = ({ dashboard, endTime, granularity, - group, savedSearch, startTime, }: AlertMessageTemplateDefaultView) => { - if (alert.source === 'search') { + if (alert.source === AlertSource.SAVED_SEARCH) { if (savedSearch == null) { - throw new Error('Source is LOG but logView is null'); + throw new Error(`Source is ${alert.source} but savedSearch is null`); } - const searchQuery = alert.groupBy - ? `${savedSearch.query} ${alert.groupBy}:"${group}"` - : savedSearch.query; return buildLogSearchLink({ endTime, - logViewId: savedSearch.id, - q: searchQuery, + savedSearch, startTime, }); - } else if (alert.source === 'chart') { + } else if (alert.source === AlertSource.TILE) { if (dashboard == null) { - throw new Error('Source is CHART but dashboard is null'); + throw new Error(`Source is ${alert.source} but dashboard is null`); } return buildChartLink({ dashboardId: dashboard.id, @@ -352,31 +342,31 @@ export const buildAlertMessageTemplateTitle = ({ }) => { const { alert, dashboard, savedSearch, value } = view; const handlebars = Handlebars.create(); - if (alert.source === 'search') { + if (alert.source === AlertSource.SAVED_SEARCH) { if (savedSearch == null) { - throw new Error('Source is LOG but logView is null'); + throw new Error(`Source is ${alert.source} but savedSearch is null`); } // TODO: using template engine to render the title return template ? handlebars.compile(template)(view) : `Alert for "${savedSearch.name}" - ${value} lines found`; - } else if (alert.source === 'chart') { + } else if (alert.source === AlertSource.TILE) { if (dashboard == null) { - throw new Error('Source is CHART but dashboard is null'); + throw new Error(`Source is ${alert.source} but dashboard is null`); } - const chart = dashboard.charts[0]; + const tile = dashboard.tiles[0]; return template ? handlebars.compile(template)(view) - : `Alert for "${chart.name}" in "${dashboard.name}" - ${value} ${ + : `Alert for "${tile.config.name}" in "${dashboard.name}" - ${value} ${ doesExceedThreshold( - alert.threshold_type === 'above', + alert.thresholdType === AlertThresholdType.ABOVE, alert.threshold, value, ) - ? alert.threshold_type === 'above' + ? alert.thresholdType === AlertThresholdType.ABOVE ? 'exceeds' : 'falls below' - : alert.threshold_type === 'above' + : alert.thresholdType === AlertThresholdType.ABOVE ? 'falls below' : 'exceeds' } ${alert.threshold}`; @@ -388,10 +378,7 @@ export const buildAlertMessageTemplateTitle = ({ export const getDefaultExternalAction = ( alert: AlertMessageTemplateDefaultView['alert'], ) => { - if ( - alert.channel.type === 'slack_webhook' && - alert.channel.webhookId != null - ) { + if (alert.channel.type === 'webhook' && alert.channel.webhookId != null) { return `@${alert.channel.type}-${alert.channel.webhookId}`; } return null; @@ -421,7 +408,6 @@ export const renderAlertTemplate = async ({ view: AlertMessageTemplateDefaultView; team: { id: string; - logStreamTableVersion?: ITeam['logStreamTableVersion']; }; }) => { const { alert, dashboard, endTime, group, savedSearch, startTime, value } = @@ -461,7 +447,7 @@ export const renderAlertTemplate = async ({ NOTIFY_FN_NAME, async (options: { hash: Record }) => { const { channel, id } = options.hash; - if (channel !== 'slack_webhook') { + if (channel !== 'webhook') { throw new Error(`Unsupported channel type: ${channel}`); } // render id template @@ -487,24 +473,23 @@ export const renderAlertTemplate = async ({ // TODO: support advanced routing with template engine // users should be able to use '@' syntax to trigger alerts - if (alert.source === 'search') { + if (alert.source === AlertSource.SAVED_SEARCH) { if (savedSearch == null) { - throw new Error('Source is LOG but logView is null'); + throw new Error(`Source is ${alert.source} but savedSearch is null`); } - const searchQuery = alert.groupBy - ? `${savedSearch.query} ${alert.groupBy}:"${group}"` - : savedSearch.query; // TODO: show group + total count for group-by alerts - const results = await clickhouse.getLogBatch({ - endTime: endTime.getTime(), - limit: 5, - offset: 0, - order: clickhouse.SortOrder.Desc, - q: searchQuery, - startTime: startTime.getTime(), - tableVersion: team.logStreamTableVersion, - teamId: team.id, - }); + const results: any = { data: [] }; + // IMPLEMENT ME: fetching sample logs using renderChartConfig + // await clickhouse.getLogBatch({ + // endTime: endTime.getTime(), + // limit: 5, + // offset: 0, + // order: clickhouse.SortOrder.Desc, + // q: searchQuery, + // startTime: startTime.getTime(), + // tableVersion: team.logStreamTableVersion, + // teamId: team.id, + // }); const truncatedResults = truncateString( results.data .map(row => { @@ -522,27 +507,29 @@ export const renderAlertTemplate = async ({ ); rawTemplateBody = `${group ? `Group: "${group}"` : ''} ${value} lines found, expected ${ - alert.threshold_type === 'above' ? 'less than' : 'greater than' + alert.thresholdType === AlertThresholdType.ABOVE + ? 'less than' + : 'greater than' } ${alert.threshold} lines ${targetTemplate} \`\`\` ${truncatedResults} \`\`\``; - } else if (alert.source === 'chart') { + } else if (alert.source === AlertSource.TILE) { if (dashboard == null) { - throw new Error('Source is CHART but dashboard is null'); + throw new Error(`Source is ${alert.source} but dashboard is null`); } rawTemplateBody = `${group ? `Group: "${group}"` : ''} ${value} ${ doesExceedThreshold( - alert.threshold_type === 'above', + alert.thresholdType === AlertThresholdType.ABOVE, alert.threshold, value, ) - ? alert.threshold_type === 'above' + ? alert.thresholdType === AlertThresholdType.ABOVE ? 'exceeds' : 'falls below' - : alert.threshold_type === 'above' + : alert.thresholdType === AlertThresholdType.ABOVE ? 'falls below' : 'exceeds' } ${alert.threshold} @@ -563,25 +550,21 @@ ${targetTemplate}`; const fireChannelEvent = async ({ alert, attributes, - dashboard, endTime, group, - logView, startTime, totalCount, windowSizeInMins, }: { - alert: AlertDocument; + alert: EnhancedAlert; attributes: Record; // TODO: support other types than string - dashboard: EnhancedDashboard | null; endTime: Date; group?: string; - logView: Awaited> | null; startTime: Date; totalCount: number; windowSizeInMins: number; }) => { - const team = logView?.team ?? dashboard?.team; + const team = alert.team; if (team == null) { throw new Error('Team not found'); } @@ -598,30 +581,25 @@ const fireChannelEvent = async ({ const attributesNested = expandToNestedObject(attributes); const templateView: AlertMessageTemplateDefaultView = { alert: { - ...translateAlertDocumentToExternalAlert(alert), + channel: alert.channel, + dashboardId: alert.dashboard?.id, groupBy: alert.groupBy, + interval: alert.interval, + message: alert.message, + name: alert.name, + savedSearchId: alert.savedSearch?.id, + silenced: alert.silenced, + source: alert.source, + threshold: alert.threshold, + thresholdType: alert.thresholdType, + tileId: alert.tileId, }, attributes: attributesNested, - dashboard: dashboard - ? translateDashboardDocumentToExternalDashboard({ - _id: dashboard._id, - name: dashboard.name, - query: dashboard.query, - team: team._id, - charts: dashboard.charts, - tags: dashboard.tags, - }) - : null, + dashboard: alert.dashboard, endTime, granularity: `${windowSizeInMins} minute`, group, - savedSearch: logView - ? { - id: logView._id.toString(), - name: logView.name, - query: logView.query, - } - : null, + savedSearch: alert.savedSearch, startTime, value: totalCount, }; @@ -635,7 +613,6 @@ const fireChannelEvent = async ({ view: templateView, team: { id: team._id.toString(), - logStreamTableVersion: team.logStreamTableVersion, }, }); }; @@ -644,7 +621,7 @@ export const roundDownTo = (roundTo: number) => (x: Date) => new Date(Math.floor(x.getTime() / roundTo) * roundTo); export const roundDownToXMinutes = (x: number) => roundDownTo(1000 * 60 * x); -export const processAlert = async (now: Date, alert: AlertDocument) => { +export const processAlert = async (now: Date, alert: EnhancedAlert) => { try { const previous: IAlertHistory | undefined = ( await AlertHistory.find({ alert: alert._id }) @@ -674,114 +651,115 @@ export const processAlert = async (now: Date, alert: AlertDocument) => { const checkEndTime = nowInMinsRoundDown; // Logs Source - let checksData: - | Awaited> - | Awaited> - | null = null; - let logView: Awaited> | null = null; - let targetDashboard: EnhancedDashboard | null = null; - if (alert.source === 'LOG' && alert.logView) { - logView = await getLogViewEnhanced(alert.logView); - // TODO: use getLogsChart instead so we can deprecate checkAlert - checksData = await clickhouse.checkAlert({ - endTime: checkEndTime, - groupBy: alert.groupBy, - q: logView.query, - startTime: checkStartTime, - tableVersion: logView.team.logStreamTableVersion, - teamId: logView.team._id.toString(), - windowSizeInMins, - }); + const checksData: { + data: { + __hdx_time_bucket: string; + [key: string]: any; + }[]; + rows: number; + } | null = { + data: [], + rows: 0, + }; + let chartConfig: ChartConfigWithOptDateRange; + if (alert.source === AlertSource.SAVED_SEARCH && alert.savedSearch) { + chartConfig = { + select: alert.savedSearch.select, + connection: alert.savedSearch.source.connection.toString(), + where: alert.savedSearch.where, + from: alert.savedSearch.source.from, + orderBy: alert.savedSearch.orderBy, + dateRange: [checkStartTime, checkEndTime], + whereLanguage: alert.savedSearch.whereLanguage, + granularity: `${windowSizeInMins} minute`, + }; logger.info({ - message: 'Received alert metric [LOG source]', + message: `Received alert metric [${alert.source} source]`, alert, - logView, + savedSearch: alert.savedSearch, checksData, checkStartTime, checkEndTime, }); } // Chart Source - else if (alert.source === 'CHART' && alert.dashboardId && alert.chartId) { - const dashboard = await Dashboard.findOne( - { - _id: alert.dashboardId, - 'charts.id': alert.chartId, - }, - { - name: 1, - charts: { - $elemMatch: { - id: alert.chartId, - }, - }, - }, - ).populate<{ - team: ITeam; - }>('team'); + else if ( + alert.source === AlertSource.TILE && + alert.dashboard && + alert.tileId + ) { + // filter tiles + alert.dashboard.tiles = alert.dashboard.tiles.filter( + tile => tile.id === alert.tileId, + ); if ( - dashboard && - Array.isArray(dashboard.charts) && - dashboard.charts.length === 1 + alert.dashboard && + Array.isArray(alert.dashboard.tiles) && + alert.dashboard.tiles.length === 1 ) { - const chart = dashboard.charts[0]; // Doesn't work for metric alerts yet const MAX_NUM_GROUPS = 20; // TODO: assuming that the chart has only 1 series for now - const firstSeries = chart.series[0]; - if (firstSeries.type === 'time' && firstSeries.table === 'logs') { - targetDashboard = dashboard; - const startTimeMs = fns.getTime(checkStartTime); - const endTimeMs = fns.getTime(checkEndTime); - - checksData = await clickhouse.getMultiSeriesChartLegacyFormat({ - series: chart.series, - endTime: endTimeMs, - granularity: `${windowSizeInMins} minute`, - maxNumGroups: MAX_NUM_GROUPS, - startTime: startTimeMs, - tableVersion: dashboard.team.logStreamTableVersion, - teamId: dashboard.team._id.toString(), - seriesReturnType: chart.seriesReturnType, + const firstTile = alert.dashboard.tiles[0]; + if (firstTile.config.displayType === DisplayType.Line) { + // fetch source data + const _source = await Source.findOne({ + _id: firstTile.config.source, }); - } else if ( - firstSeries.type === 'time' && - firstSeries.table === 'metrics' && - firstSeries.field - ) { - targetDashboard = dashboard; - const startTimeMs = fns.getTime(checkStartTime); - const endTimeMs = fns.getTime(checkEndTime); - checksData = await clickhouse.getMultiSeriesChartLegacyFormat({ - series: chart.series.map(series => { - if ('field' in series && series.field != null) { - const [metricName, rawMetricDataType] = - series.field.split(' - '); - const metricDataType = z - .nativeEnum(clickhouse.MetricsDataType) - .parse(rawMetricDataType); - return { - ...series, - metricDataType, - field: metricName, - }; - } - return series; - }), - endTime: endTimeMs, + if (!_source) { + throw new Error('Source not found'); + } + // TODO: FIXED TYPE + // @ts-ignore + chartConfig = { + ...firstTile.config, + connection: _source.connection.toString(), + from: _source.from, + dateRange: [checkStartTime, checkEndTime], granularity: `${windowSizeInMins} minute`, - maxNumGroups: MAX_NUM_GROUPS, - startTime: startTimeMs, - tableVersion: dashboard.team.logStreamTableVersion, - teamId: dashboard.team._id.toString(), - seriesReturnType: chart.seriesReturnType, - }); + }; } + // else if ( + // firstTile.type === 'time' && + // firstTile.table === 'metrics' && + // firstTile.field + // ) { + // targetDashboard = dashboard; + // const startTimeMs = fns.getTime(checkStartTime); + // const endTimeMs = fns.getTime(checkEndTime); + // ***************************************************** + // IMPLEMENT ME: implement query using renderChartConfig + // ***************************************************** + // checksData = await clickhouse.getMultiSeriesChartLegacyFormat({ + // series: chart.series.map(series => { + // if ('field' in series && series.field != null) { + // const [metricName, rawMetricDataType] = + // series.field.split(' - '); + // const metricDataType = z + // .nativeEnum(clickhouse.MetricsDataType) + // .parse(rawMetricDataType); + // return { + // ...series, + // metricDataType, + // field: metricName, + // }; + // } + // return series; + // }), + // endTime: endTimeMs, + // granularity: `${windowSizeInMins} minute`, + // maxNumGroups: MAX_NUM_GROUPS, + // startTime: startTimeMs, + // tableVersion: dashboard.team.logStreamTableVersion, + // teamId: dashboard.team._id.toString(), + // seriesReturnType: chart.seriesReturnType, + // }); + // } } logger.info({ - message: 'Received alert metric [CHART source]', + message: `Received alert metric [${alert.source} source]`, alert, checksData, checkStartTime, @@ -814,10 +792,10 @@ export const processAlert = async (now: Date, alert: AlertDocument) => { const totalCount = isString(checkData.data) ? parseInt(checkData.data) : checkData.data; - const bucketStart = new Date(checkData.ts_bucket * 1000); + const bucketStart = new Date(checkData[FIXED_TIME_BUCKET_EXPR_ALIAS]); if ( doesExceedThreshold( - alert.type === 'presence', + alert.thresholdType === AlertThresholdType.ABOVE, alert.threshold, totalCount, ) @@ -834,12 +812,10 @@ export const processAlert = async (now: Date, alert: AlertDocument) => { await fireChannelEvent({ alert, attributes: checkData.attributes, - dashboard: targetDashboard, endTime: fns.addMinutes(bucketStart, windowSizeInMins), group: Array.isArray(checkData.group) ? checkData.group.join(', ') : checkData.group, - logView, startTime: bucketStart, totalCount, windowSizeInMins, @@ -876,7 +852,7 @@ export const processAlert = async (now: Date, alert: AlertDocument) => { export default async () => { const now = new Date(); - const alerts = await Alert.find({}); + const alerts = await getAlerts(); logger.info(`Going to process ${alerts.length} alerts`); await Promise.all(alerts.map(alert => processAlert(now, alert))); }; diff --git a/packages/api/src/utils/commonTypes.ts b/packages/api/src/utils/commonTypes.ts deleted file mode 100644 index 9320de49..00000000 --- a/packages/api/src/utils/commonTypes.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { z } from 'zod'; - -// -------------------------- -// SAVED SEARCH -// -------------------------- - -export const SavedSearchSchema = z.object({ - id: z.string(), - name: z.string(), - select: z.string(), - where: z.string(), - whereLanguage: z.string().optional(), - source: z.string(), - tags: z.array(z.string()), - orderBy: z.string().optional(), -}); - -export type SavedSearch = z.infer; - -// -------------------------- -// DASHBOARDS -// -------------------------- - -// TODO: Define this -export const SavedChartConfigSchema = z.any(); - -export const TileSchema = z.object({ - id: z.string(), - x: z.number(), - y: z.number(), - w: z.number(), - h: z.number(), - config: SavedChartConfigSchema, -}); - -export type Tile = z.infer; -export const DashboardSchema = z.object({ - id: z.string(), - name: z.string(), - tiles: z.array(TileSchema), - tags: z.array(z.string()), -}); - -export const DashboardWithoutIdSchema = DashboardSchema.omit({ id: true }); - -export const ConnectionSchema = z.object({ - id: z.string(), - name: z.string(), - host: z.string(), - username: z.string(), - password: z.string().optional(), -}); - -// -------------------------- -// TABLE SOURCES -// -------------------------- -export const SourceSchema = z.object({ - from: z.object({ - databaseName: z.string(), - tableName: z.string(), - }), - timestampValueExpression: z.string(), - connection: z.string(), - - // Common - kind: z.enum(['log', 'trace']), - id: z.string(), - name: z.string(), - displayedTimestampValueExpression: z.string().optional(), - implicitColumnExpression: z.string().optional(), - serviceNameExpression: z.string().optional(), - bodyExpression: z.string().optional(), - tableFilterExpression: z.string().optional(), - eventAttributesExpression: z.string().optional(), - resourceAttributesExpression: z.string().optional(), - defaultTableSelectExpression: z.string().optional(), - - // Logs - uniqueRowIdExpression: z.string().optional(), - severityTextExpression: z.string().optional(), - traceSourceId: z.string().optional(), - - // Traces & Logs - traceIdExpression: z.string().optional(), - spanIdExpression: z.string().optional(), - - // Traces - durationExpression: z.string().optional(), - durationPrecision: z.number().min(0).max(9).optional(), - parentSpanIdExpression: z.string().optional(), - spanNameExpression: z.string().optional(), - - spanKindExpression: z.string().optional(), - statusCodeExpression: z.string().optional(), - statusMessageExpression: z.string().optional(), - logSourceId: z.string().optional(), -}); - -export type TSource = z.infer; diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 31148f0b..1b89fa05 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -2,7 +2,7 @@ import { Types } from 'mongoose'; import { z } from 'zod'; import { AggFn, MetricsDataType } from '@/clickhouse'; -import { AlertDocument } from '@/models/alert'; +import { AlertSource, AlertThresholdType } from '@/models/alert'; export const objectIdSchema = z.string().refine(val => { return Types.ObjectId.isValid(val); @@ -201,14 +201,14 @@ export const zChannel = z.object({ webhookId: z.string().min(1), }); -export const zLogAlert = z.object({ - source: z.literal('LOG'), +export const zSavedSearchAlert = z.object({ + source: z.literal(AlertSource.SAVED_SEARCH), groupBy: z.string().optional(), savedSearchId: z.string().min(1), }); -export const zChartAlert = z.object({ - source: z.literal('CHART'), +export const zTileAlert = z.object({ + source: z.literal(AlertSource.TILE), tileId: z.string().min(1), dashboardId: z.string().min(1), }); @@ -218,101 +218,9 @@ export const alertSchema = z channel: zChannel, interval: z.enum(['1m', '5m', '15m', '30m', '1h', '6h', '12h', '1d']), threshold: z.number().min(0), - type: z.enum(['presence', 'absence']), - source: z.enum(['LOG', 'CHART']).default('LOG'), + thresholdType: z.nativeEnum(AlertThresholdType), + source: z.nativeEnum(AlertSource).default(AlertSource.SAVED_SEARCH), name: z.string().min(1).max(512).nullish(), message: z.string().min(1).max(4096).nullish(), }) - .and(zLogAlert.or(zChartAlert)); - -// ============================== -// External API Alerts -// ============================== - -export const externalSlackWebhookAlertChannel = z.object({ - type: z.literal('slack_webhook'), - webhookId: objectIdSchema, -}); - -export const externalSearchAlertSchema = z.object({ - source: z.literal('search'), - groupBy: z.string().optional(), - savedSearchId: objectIdSchema, -}); - -export const externalChartAlertSchema = z.object({ - source: z.literal('chart'), - chartId: z.string().min(1), - dashboardId: objectIdSchema, -}); - -export const externalAlertSchema = z - .object({ - channel: externalSlackWebhookAlertChannel, - interval: z.enum(['1m', '5m', '15m', '30m', '1h', '6h', '12h', '1d']), - threshold: z.number().min(0), - threshold_type: z.enum(['above', 'below']), - source: z.enum(['search', 'chart']).default('search'), - name: z.string().min(1).max(512).nullish(), - message: z.string().min(1).max(4096).nullish(), - }) - .and(externalSearchAlertSchema.or(externalChartAlertSchema)); - -export const externalAlertSchemaWithId = externalAlertSchema.and( - z.object({ - id: objectIdSchema, - }), -); - -// TODO: move this to utils file since its not zod instance -export const translateExternalAlertToInternalAlert = ( - alertInput: z.infer, -): z.infer => { - return { - interval: alertInput.interval, - threshold: alertInput.threshold, - type: alertInput.threshold_type === 'above' ? 'presence' : 'absence', - channel: { - ...alertInput.channel, - type: 'webhook', - }, - name: alertInput.name, - message: alertInput.message, - ...(alertInput.source === 'search' && alertInput.savedSearchId - ? { source: 'LOG', savedSearchId: alertInput.savedSearchId } - : alertInput.source === 'chart' && alertInput.dashboardId - ? { - source: 'CHART', - dashboardId: alertInput.dashboardId, - tileId: alertInput.chartId, - } - : ({} as never)), - }; -}; - -// TODO: move this to utils file since its not zod instance -export const translateAlertDocumentToExternalAlert = ( - alertDoc: AlertDocument, -): z.infer => { - return { - id: alertDoc._id.toString(), - interval: alertDoc.interval, - threshold: alertDoc.threshold, - threshold_type: alertDoc.type === 'absence' ? 'below' : 'above', - channel: { - ...alertDoc.channel, - type: 'slack_webhook', - }, - name: alertDoc.name, - message: alertDoc.message, - ...(alertDoc.source === 'LOG' && alertDoc.savedSearch - ? { source: 'search', savedSearchId: alertDoc.savedSearch.toString() } - : alertDoc.source === 'CHART' && alertDoc.dashboardId - ? { - source: 'chart', - dashboardId: alertDoc.dashboardId.toString(), - chartId: alertDoc.tileId as string, - } - : ({} as never)), - }; -}; + .and(zSavedSearchAlert.or(zTileAlert)); diff --git a/packages/app/package.json b/packages/app/package.json index df14d4b6..1bba3475 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -97,7 +97,7 @@ "uplot": "^1.6.30", "uplot-react": "^1.2.2", "use-query-params": "^2.1.2", - "zod": "^3.22.3" + "zod": "^3.24.1" }, "devDependencies": { "@chromatic-com/storybook": "^1.5.0", diff --git a/yarn.lock b/yarn.lock index 510fa8e2..bddc1dcd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3133,6 +3133,13 @@ __metadata: languageName: node linkType: hard +"@clickhouse/client-common@npm:^1.9.1": + version: 1.9.1 + resolution: "@clickhouse/client-common@npm:1.9.1" + checksum: 10c0/d4df33c16e8ead60c8d656418c81f4cacd16f63c26d462bb265fa4892c0e0f40c5ddf447724befea36ffed569dae685049a9fb6f6210e3362bcbe632e15dfc70 + languageName: node + linkType: hard + "@clickhouse/client-web@npm:^1.7.0": version: 1.7.0 resolution: "@clickhouse/client-web@npm:1.7.0" @@ -4016,6 +4023,7 @@ __metadata: resolution: "@hyperdx/api@workspace:packages/api" dependencies: "@clickhouse/client": "npm:^0.2.10" + "@clickhouse/client-common": "npm:^1.9.1" "@hyperdx/lucene": "npm:^3.1.1" "@hyperdx/node-opentelemetry": "npm:^0.8.1" "@opentelemetry/api": "npm:^1.8.0" @@ -4063,6 +4071,7 @@ __metadata: mongoose: "npm:^6.12.0" ms: "npm:^2.1.3" node-schedule: "npm:^2.1.1" + node-sql-parser: "npm:^5.3.5" nodemon: "npm:^2.0.20" object-hash: "npm:^3.0.0" on-headers: "npm:^1.0.2" @@ -4085,7 +4094,7 @@ __metadata: typescript: "npm:^4.9.5" uuid: "npm:^8.3.2" winston: "npm:^3.10.0" - zod: "npm:^3.22.3" + zod: "npm:^3.24.1" zod-express-middleware: "npm:^1.4.0" languageName: unknown linkType: soft @@ -4214,7 +4223,7 @@ __metadata: uplot: "npm:^1.6.30" uplot-react: "npm:^1.2.2" use-query-params: "npm:^2.1.2" - zod: "npm:^3.22.3" + zod: "npm:^3.24.1" languageName: unknown linkType: soft @@ -21000,6 +21009,16 @@ __metadata: languageName: node linkType: hard +"node-sql-parser@npm:^5.3.5": + version: 5.3.5 + resolution: "node-sql-parser@npm:5.3.5" + dependencies: + "@types/pegjs": "npm:^0.10.0" + big-integer: "npm:^1.6.48" + checksum: 10c0/221c0e5d582adf9e87a4357cc437f6f66e925eaefa889f05b8e93375274b0246edbca11a673e7d44a29888ca0ec6005b5614ac08361b0e9171d11c66ce17e91a + languageName: node + linkType: hard + "nodemon@npm:^2.0.20": version: 2.0.20 resolution: "nodemon@npm:2.0.20" @@ -27912,13 +27931,6 @@ __metadata: languageName: node linkType: hard -"zod@npm:^3.22.3": - version: 3.22.3 - resolution: "zod@npm:3.22.3" - checksum: 10c0/cb4b24aed7dec98552eb9042e88cbd645455bf2830e5704174d2da96f554dabad4630e3b4f6623e1b6562b9eaa43535a37b7f2011f29b8d8e9eabe1ddf3b656b - languageName: node - linkType: hard - "zod@npm:^3.22.4": version: 3.23.8 resolution: "zod@npm:3.23.8" @@ -27926,6 +27938,13 @@ __metadata: languageName: node linkType: hard +"zod@npm:^3.24.1": + version: 3.24.1 + resolution: "zod@npm:3.24.1" + checksum: 10c0/0223d21dbaa15d8928fe0da3b54696391d8e3e1e2d0283a1a070b5980a1dbba945ce631c2d1eccc088fdbad0f2dfa40155590bf83732d3ac4fcca2cc9237591b + languageName: node + linkType: hard + "zwitch@npm:^2.0.0": version: 2.0.4 resolution: "zwitch@npm:2.0.4"