mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
style: use common utils package (api and app) (#555)
This commit is contained in:
Warren
GitHub
parent
189d1d05f1
commit
a70080e533
88 changed files with 544 additions and 6136 deletions
24
.github/workflows/main.yml
vendored
24
.github/workflows/main.yml
vendored
|
|
@ -13,13 +13,17 @@ jobs:
|
|||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v3
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: '.nvmrc'
|
||||
cache-dependency-path: 'yarn.lock'
|
||||
cache: 'yarn'
|
||||
- name: Install root dependencies
|
||||
uses: bahmutov/npm-install@v1
|
||||
run: yarn install
|
||||
- name: Build dependencies
|
||||
run: make ci-build
|
||||
- name: Install core libs
|
||||
run: sudo apt-get install --yes curl bc
|
||||
- name: Run lint + type check
|
||||
|
|
@ -29,13 +33,17 @@ jobs:
|
|||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v3
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: '.nvmrc'
|
||||
cache-dependency-path: 'yarn.lock'
|
||||
cache: 'yarn'
|
||||
- name: Install root dependencies
|
||||
uses: bahmutov/npm-install@v1
|
||||
run: yarn install
|
||||
- name: Build dependencies
|
||||
run: make ci-build
|
||||
- name: Run unit tests
|
||||
run: make ci-unit
|
||||
integration:
|
||||
|
|
@ -43,9 +51,9 @@ jobs:
|
|||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v3
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: '.nvmrc'
|
||||
- name: Expose GitHub Runtime
|
||||
|
|
|
|||
8
.github/workflows/release.yml
vendored
8
.github/workflows/release.yml
vendored
|
|
@ -9,13 +9,15 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v3
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: '.nvmrc'
|
||||
cache-dependency-path: 'yarn.lock'
|
||||
cache: 'yarn'
|
||||
- name: Install root dependencies
|
||||
uses: bahmutov/npm-install@v1
|
||||
run: yarn install
|
||||
- name: Create Release Pull Request or Publish to npm
|
||||
id: changesets
|
||||
uses: changesets/action@v1
|
||||
|
|
|
|||
10
Makefile
10
Makefile
|
|
@ -27,6 +27,10 @@ dev-down:
|
|||
dev-lint:
|
||||
npx nx run-many -t lint:fix
|
||||
|
||||
.PHONY: ci-build
|
||||
ci-build:
|
||||
npx nx run-many -t ci:build
|
||||
|
||||
.PHONY: ci-lint
|
||||
ci-lint:
|
||||
npx nx run-many -t ci:lint
|
||||
|
|
@ -43,12 +47,6 @@ dev-int:
|
|||
.PHONY: ci-int
|
||||
ci-int:
|
||||
docker compose -p int -f ./docker-compose.ci.yml run --rm api ci:int
|
||||
# @echo "\n\n"
|
||||
# @echo "Checking otel-collector...\n"
|
||||
# curl -v http://localhost:23133
|
||||
# @echo "\n\n"
|
||||
# @echo "Checking ingestor...\n"
|
||||
# curl -v http://localhost:28686/health
|
||||
|
||||
.PHONY: dev-unit
|
||||
dev-unit:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@clickhouse/client": "^0.2.10",
|
||||
"@clickhouse/client-common": "^1.9.1",
|
||||
"@hyperdx/common-utils": "^0.0.9",
|
||||
"@hyperdx/lucene": "^3.1.1",
|
||||
"@hyperdx/node-opentelemetry": "^0.8.1",
|
||||
"@opentelemetry/api": "^1.8.0",
|
||||
|
|
@ -34,7 +34,6 @@
|
|||
"mongoose": "^6.12.0",
|
||||
"ms": "^2.1.3",
|
||||
"node-schedule": "^2.1.1",
|
||||
"node-sql-parser": "^5.3.5",
|
||||
"object-hash": "^3.0.0",
|
||||
"on-headers": "^1.0.2",
|
||||
"passport": "^0.6.0",
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
export enum DisplayType {
|
||||
Line = 'line',
|
||||
StackedBar = 'stacked_bar',
|
||||
Table = 'table',
|
||||
Number = 'number',
|
||||
Search = 'search',
|
||||
Heatmap = 'heatmap',
|
||||
Markdown = 'markdown',
|
||||
}
|
||||
|
|
@ -1,606 +0,0 @@
|
|||
import {
|
||||
BaseResultSet,
|
||||
DataFormat,
|
||||
isSuccessfulResponse,
|
||||
ResponseJSON,
|
||||
} from '@clickhouse/client-common';
|
||||
|
||||
import { SQLInterval } from '@/common/sqlTypes';
|
||||
import { hashCode } from '@/common/utils';
|
||||
import { timeBucketByGranularity } from '@/common/utils';
|
||||
|
||||
export const PROXY_CLICKHOUSE_HOST = '/api/clickhouse-proxy';
|
||||
|
||||
export enum JSDataType {
|
||||
Array = 'array',
|
||||
Date = 'date',
|
||||
Map = 'map',
|
||||
Number = 'number',
|
||||
String = 'string',
|
||||
Bool = 'bool',
|
||||
}
|
||||
|
||||
export const convertCHDataTypeToJSType = (
|
||||
dataType: string,
|
||||
): JSDataType | null => {
|
||||
if (dataType.startsWith('Date')) {
|
||||
return JSDataType.Date;
|
||||
} else if (dataType.startsWith('Map')) {
|
||||
return JSDataType.Map;
|
||||
} else if (dataType.startsWith('Array')) {
|
||||
return JSDataType.Array;
|
||||
} else if (
|
||||
dataType.startsWith('Int') ||
|
||||
dataType.startsWith('UInt') ||
|
||||
dataType.startsWith('Float') ||
|
||||
// Nullable types are possible (charts)
|
||||
dataType.startsWith('Nullable(Int') ||
|
||||
dataType.startsWith('Nullable(UInt') ||
|
||||
dataType.startsWith('Nullable(Float')
|
||||
) {
|
||||
return JSDataType.Number;
|
||||
} else if (
|
||||
dataType.startsWith('String') ||
|
||||
dataType.startsWith('FixedString') ||
|
||||
dataType.startsWith('Enum') ||
|
||||
dataType.startsWith('UUID') ||
|
||||
dataType.startsWith('IPv4') ||
|
||||
dataType.startsWith('IPv6')
|
||||
) {
|
||||
return JSDataType.String;
|
||||
} else if (dataType === 'Bool') {
|
||||
return JSDataType.Bool;
|
||||
} else if (dataType.startsWith('LowCardinality')) {
|
||||
return convertCHDataTypeToJSType(dataType.slice(15, -1));
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
export const convertCHTypeToPrimitiveJSType = (dataType: string) => {
|
||||
const jsType = convertCHDataTypeToJSType(dataType);
|
||||
|
||||
if (jsType === JSDataType.Map || jsType === JSDataType.Array) {
|
||||
throw new Error('Map type is not a primitive type');
|
||||
} else if (jsType === JSDataType.Date) {
|
||||
return JSDataType.Number;
|
||||
}
|
||||
|
||||
return jsType;
|
||||
};
|
||||
|
||||
const hash = (input: string | number) => Math.abs(hashCode(`${input}`));
|
||||
const paramHash = (str: string | number) => {
|
||||
return `HYPERDX_PARAM_${hash(str)}`;
|
||||
};
|
||||
|
||||
export type ChSql = {
|
||||
sql: string;
|
||||
params: Record<string, any>;
|
||||
};
|
||||
|
||||
type ParamTypes =
|
||||
| ChSql
|
||||
| ChSql[]
|
||||
| { Identifier: string }
|
||||
| { String: string }
|
||||
| { Float32: number }
|
||||
| { Float64: number }
|
||||
| { Int32: number }
|
||||
| { Int64: number }
|
||||
| { UNSAFE_RAW_SQL: string }
|
||||
| string; // TODO: Deprecate raw string interpolation
|
||||
|
||||
export const chSql = (
|
||||
strings: TemplateStringsArray,
|
||||
...values: ParamTypes[]
|
||||
): ChSql => {
|
||||
const sql = strings
|
||||
.map((str, i) => {
|
||||
const value = values[i];
|
||||
// if (typeof value === 'string') {
|
||||
// console.error('Unsafe string detected', value, 'in', strings, values);
|
||||
// }
|
||||
|
||||
return (
|
||||
str +
|
||||
(value == null
|
||||
? ''
|
||||
: typeof value === 'string'
|
||||
? value // If it's just a string sql literal
|
||||
: 'UNSAFE_RAW_SQL' in value
|
||||
? value.UNSAFE_RAW_SQL
|
||||
: Array.isArray(value)
|
||||
? value.map(v => v.sql).join('')
|
||||
: 'sql' in value
|
||||
? value.sql
|
||||
: 'Identifier' in value
|
||||
? `{${paramHash(value.Identifier)}:Identifier}`
|
||||
: 'String' in value
|
||||
? `{${paramHash(value.String)}:String}`
|
||||
: 'Float32' in value
|
||||
? `{${paramHash(value.Float32)}:Float32}`
|
||||
: 'Float64' in value
|
||||
? `{${paramHash(value.Float64)}:Float64}`
|
||||
: 'Int32' in value
|
||||
? `{${paramHash(value.Int32)}:Int32}`
|
||||
: 'Int64' in value
|
||||
? `{${paramHash(value.Int64)}:Int64}`
|
||||
: '')
|
||||
);
|
||||
})
|
||||
.join('');
|
||||
|
||||
return {
|
||||
sql,
|
||||
params: values.reduce((acc, value) => {
|
||||
return {
|
||||
...acc,
|
||||
...(value == null ||
|
||||
typeof value === 'string' ||
|
||||
'UNSAFE_RAW_SQL' in value
|
||||
? {}
|
||||
: Array.isArray(value)
|
||||
? value.reduce((acc, v) => {
|
||||
Object.assign(acc, v.params);
|
||||
return acc;
|
||||
}, {})
|
||||
: 'params' in value
|
||||
? value.params
|
||||
: 'Identifier' in value
|
||||
? { [paramHash(value.Identifier)]: value.Identifier }
|
||||
: 'String' in value
|
||||
? { [paramHash(value.String)]: value.String }
|
||||
: 'Float32' in value
|
||||
? { [paramHash(value.Float32)]: value.Float32 }
|
||||
: 'Float64' in value
|
||||
? { [paramHash(value.Float64)]: value.Float64 }
|
||||
: 'Int32' in value
|
||||
? { [paramHash(value.Int32)]: value.Int32 }
|
||||
: 'Int64' in value
|
||||
? { [paramHash(value.Int64)]: value.Int64 }
|
||||
: {}),
|
||||
};
|
||||
}, {}),
|
||||
};
|
||||
};
|
||||
|
||||
export const concatChSql = (sep: string, ...args: (ChSql | ChSql[])[]) => {
|
||||
return args.reduce(
|
||||
(acc: ChSql, arg) => {
|
||||
if (Array.isArray(arg)) {
|
||||
if (arg.length === 0) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
acc.sql +=
|
||||
(acc.sql.length > 0 ? sep : '') + arg.map(a => a.sql).join(sep);
|
||||
acc.params = arg.reduce((acc, a) => {
|
||||
Object.assign(acc, a.params);
|
||||
return acc;
|
||||
}, acc.params);
|
||||
} else if (arg.sql.length > 0) {
|
||||
acc.sql += `${acc.sql.length > 0 ? sep : ''}${arg.sql}`;
|
||||
Object.assign(acc.params, arg.params);
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{ sql: '', params: {} },
|
||||
);
|
||||
};
|
||||
|
||||
const isChSqlEmpty = (chSql: ChSql | ChSql[]) => {
|
||||
if (Array.isArray(chSql)) {
|
||||
return chSql.every(c => c.sql.length === 0);
|
||||
}
|
||||
return chSql.sql.length === 0;
|
||||
};
|
||||
|
||||
export const wrapChSqlIfNotEmpty = (
|
||||
sql: ChSql | ChSql[],
|
||||
left: string,
|
||||
right: string,
|
||||
): ChSql | [] => {
|
||||
if (isChSqlEmpty(sql)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return chSql`${left}${sql}${right}`;
|
||||
};
|
||||
export class ClickHouseQueryError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public query: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ClickHouseQueryError';
|
||||
}
|
||||
}
|
||||
|
||||
export function extractColumnReference(
|
||||
sql: string,
|
||||
maxIterations = 10,
|
||||
): string | null {
|
||||
let iterations = 0;
|
||||
|
||||
// Loop until we remove all function calls and get just the column, with a maximum limit
|
||||
while (/\w+\([^()]*\)/.test(sql) && iterations < maxIterations) {
|
||||
// Replace the outermost function with its content
|
||||
sql = sql.replace(/\w+\(([^()]*)\)/, '$1');
|
||||
iterations++;
|
||||
}
|
||||
|
||||
// If we reached the max iterations without resolving, return null to indicate an issue
|
||||
return iterations < maxIterations ? sql.trim() : null;
|
||||
}
|
||||
|
||||
export const client = {
|
||||
async query<T extends DataFormat>({
|
||||
query,
|
||||
format = 'JSON',
|
||||
query_params = {},
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
host,
|
||||
username,
|
||||
password,
|
||||
includeCredentials,
|
||||
includeCorsHeader,
|
||||
connectionId,
|
||||
queryId,
|
||||
}: {
|
||||
query: string;
|
||||
format?: string;
|
||||
abort_signal?: AbortSignal;
|
||||
query_params?: Record<string, any>;
|
||||
clickhouse_settings?: Record<string, any>;
|
||||
host?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
includeCredentials: boolean;
|
||||
includeCorsHeader: boolean;
|
||||
connectionId?: string;
|
||||
queryId?: string;
|
||||
}): Promise<BaseResultSet<any, T>> {
|
||||
const searchParams = new URLSearchParams([
|
||||
...(includeCorsHeader ? [['add_http_cors_header', '1']] : []),
|
||||
...(connectionId ? [['hyperdx_connection_id', connectionId]] : []),
|
||||
['query', query],
|
||||
['default_format', format],
|
||||
['date_time_output_format', 'iso'],
|
||||
['wait_end_of_query', '0'],
|
||||
['cancel_http_readonly_queries_on_client_close', '1'],
|
||||
...(username ? [['user', username]] : []),
|
||||
...(password ? [['password', password]] : []),
|
||||
...(queryId ? [['query_id', queryId]] : []),
|
||||
...Object.entries(query_params).map(([key, value]) => [
|
||||
`param_${key}`,
|
||||
value,
|
||||
]),
|
||||
...Object.entries(clickhouse_settings ?? {}).map(([key, value]) => [
|
||||
key,
|
||||
value,
|
||||
]),
|
||||
]);
|
||||
|
||||
let debugSql = '';
|
||||
try {
|
||||
debugSql = parameterizedQueryToSql({ sql: query, params: query_params });
|
||||
} catch (e) {
|
||||
debugSql = query;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('--------------------------------------------------------');
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Sending Query:', debugSql);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('--------------------------------------------------------');
|
||||
|
||||
const res = await fetch(`${host}/?${searchParams.toString()}`, {
|
||||
...(includeCredentials ? { credentials: 'include' } : {}),
|
||||
signal: abort_signal,
|
||||
method: 'GET',
|
||||
});
|
||||
|
||||
// TODO: Send command to CH to cancel query on abort_signal
|
||||
if (!res.ok) {
|
||||
if (!isSuccessfulResponse(res.status)) {
|
||||
const text = await res.text();
|
||||
throw new ClickHouseQueryError(`${text}`, debugSql);
|
||||
}
|
||||
}
|
||||
|
||||
if (res.body == null) {
|
||||
// TODO: Handle empty responses better?
|
||||
throw new Error('Unexpected empty response from ClickHouse');
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
return new BaseResultSet(res.body, format, '');
|
||||
},
|
||||
};
|
||||
|
||||
export const testLocalConnection = async ({
|
||||
host,
|
||||
username,
|
||||
password,
|
||||
}: {
|
||||
host: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}): Promise<boolean> => {
|
||||
try {
|
||||
const result = await client.query({
|
||||
query: 'SELECT 1',
|
||||
format: 'TabSeparatedRaw',
|
||||
host: host,
|
||||
username: username,
|
||||
password: password,
|
||||
includeCredentials: false,
|
||||
includeCorsHeader: true,
|
||||
});
|
||||
return result.text().then(text => text.trim() === '1');
|
||||
} catch (e) {
|
||||
console.warn('Failed to test local connection', e);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export const sendQuery = async <T extends DataFormat>({
|
||||
query,
|
||||
format = 'JSON',
|
||||
query_params = {},
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
connectionId,
|
||||
queryId,
|
||||
}: {
|
||||
query: string;
|
||||
format?: string;
|
||||
query_params?: Record<string, any>;
|
||||
abort_signal?: AbortSignal;
|
||||
clickhouse_settings?: Record<string, any>;
|
||||
connectionId: string;
|
||||
queryId?: string;
|
||||
}) => {
|
||||
const IS_LOCAL_MODE = false;
|
||||
|
||||
// TODO: decide what to do here
|
||||
let host, username, password;
|
||||
if (IS_LOCAL_MODE) {
|
||||
const localConnections: any = [];
|
||||
if (localConnections.length === 0) {
|
||||
throw new Error('No local connection found');
|
||||
}
|
||||
host = localConnections[0].host;
|
||||
username = localConnections[0].username;
|
||||
password = localConnections[0].password;
|
||||
}
|
||||
|
||||
return client.query<T>({
|
||||
query,
|
||||
format,
|
||||
query_params,
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
queryId,
|
||||
connectionId: IS_LOCAL_MODE ? undefined : connectionId,
|
||||
host: IS_LOCAL_MODE ? host : PROXY_CLICKHOUSE_HOST,
|
||||
username: IS_LOCAL_MODE ? username : undefined,
|
||||
password: IS_LOCAL_MODE ? password : undefined,
|
||||
includeCredentials: !IS_LOCAL_MODE,
|
||||
includeCorsHeader: IS_LOCAL_MODE,
|
||||
});
|
||||
};
|
||||
|
||||
export const tableExpr = ({
|
||||
database,
|
||||
table,
|
||||
}: {
|
||||
database: string;
|
||||
table: string;
|
||||
}) => {
|
||||
return chSql`${{ Identifier: database }}.${{ Identifier: table }}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* SELECT
|
||||
* aggFnIf(fieldToColumn(field), where),
|
||||
* timeBucketing(Granularity, timeConversion(fieldToColumn(field))),
|
||||
* FROM db.table
|
||||
* WHERE where
|
||||
* GROUP BY timeBucketing, fieldToColumn(groupBy)
|
||||
* ORDER BY orderBy
|
||||
*/
|
||||
|
||||
export function parameterizedQueryToSql({
|
||||
sql,
|
||||
params,
|
||||
}: {
|
||||
sql: string;
|
||||
params: Record<string, any>;
|
||||
}) {
|
||||
return Object.entries(params).reduce((acc, [key, value]) => {
|
||||
return acc.replace(new RegExp(`{${key}:\\w+}`, 'g'), value);
|
||||
}, sql);
|
||||
}
|
||||
|
||||
export type ColumnMetaType = { name: string; type: string };
|
||||
export function filterColumnMetaByType(
|
||||
meta: Array<ColumnMetaType>,
|
||||
types: JSDataType[],
|
||||
): Array<ColumnMetaType> | undefined {
|
||||
return meta.filter(column =>
|
||||
types.includes(convertCHDataTypeToJSType(column.type) as JSDataType),
|
||||
);
|
||||
}
|
||||
|
||||
export function inferTimestampColumn(
|
||||
// from: https://github.com/ClickHouse/clickhouse-js/blob/442392c83834f313a964f9e5bd7ff44474631755/packages/client-common/src/clickhouse_types.ts#L8C3-L8C47
|
||||
meta: Array<ColumnMetaType>,
|
||||
) {
|
||||
return filterColumnMetaByType(meta, [JSDataType.Date])?.[0];
|
||||
}
|
||||
|
||||
function inferValueColumns(meta: Array<{ name: string; type: string }>) {
|
||||
return filterColumnMetaByType(meta, [JSDataType.Number]);
|
||||
}
|
||||
|
||||
function inferGroupColumns(meta: Array<{ name: string; type: string }>) {
|
||||
return filterColumnMetaByType(meta, [
|
||||
JSDataType.String,
|
||||
JSDataType.Map,
|
||||
JSDataType.Array,
|
||||
]);
|
||||
}
|
||||
|
||||
// TODO: Move to ChartUtils
|
||||
// Input: { ts, value1, value2, groupBy1, groupBy2 },
|
||||
// Output: { ts, [value1Name, groupBy1, groupBy2]: value1, [...]: value2 }
|
||||
export function formatResponseForTimeChart({
|
||||
res,
|
||||
dateRange,
|
||||
granularity,
|
||||
generateEmptyBuckets = true,
|
||||
}: {
|
||||
dateRange: [Date, Date];
|
||||
granularity?: SQLInterval;
|
||||
res: ResponseJSON<Record<string, any>>;
|
||||
generateEmptyBuckets?: boolean;
|
||||
}) {
|
||||
const meta = res.meta;
|
||||
const data = res.data;
|
||||
|
||||
if (meta == null) {
|
||||
throw new Error('No meta data found in response');
|
||||
}
|
||||
|
||||
const timestampColumn = inferTimestampColumn(meta);
|
||||
const valueColumns = inferValueColumns(meta) ?? [];
|
||||
const groupColumns = inferGroupColumns(meta) ?? [];
|
||||
|
||||
if (timestampColumn == null) {
|
||||
throw new Error(
|
||||
`No timestamp column found with meta: ${JSON.stringify(meta)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Timestamp -> { tsCol, line1, line2, ...}
|
||||
const tsBucketMap: Map<number, Record<string, any>> = new Map();
|
||||
const lineDataMap: {
|
||||
[keyName: string]: {
|
||||
dataKey: string;
|
||||
displayName: string;
|
||||
maxValue: number;
|
||||
minValue: number;
|
||||
color: string | undefined;
|
||||
};
|
||||
} = {};
|
||||
|
||||
for (const row of data) {
|
||||
const date = new Date(row[timestampColumn.name]);
|
||||
const ts = date.getTime() / 1000;
|
||||
|
||||
for (const valueColumn of valueColumns) {
|
||||
const tsBucket = tsBucketMap.get(ts) ?? {};
|
||||
|
||||
const keyName = [
|
||||
valueColumn.name,
|
||||
...groupColumns.map(g => row[g.name]),
|
||||
].join(' · ');
|
||||
|
||||
// UInt64 are returned as strings, we'll convert to number
|
||||
// and accept a bit of floating point error
|
||||
const rawValue = row[valueColumn.name];
|
||||
const value =
|
||||
typeof rawValue === 'number' ? rawValue : Number.parseFloat(rawValue);
|
||||
|
||||
tsBucketMap.set(ts, {
|
||||
...tsBucket,
|
||||
[timestampColumn.name]: ts,
|
||||
[keyName]: value,
|
||||
});
|
||||
|
||||
// TODO: Set name and color correctly
|
||||
lineDataMap[keyName] = {
|
||||
dataKey: keyName,
|
||||
displayName: keyName,
|
||||
color: undefined,
|
||||
maxValue: Math.max(
|
||||
lineDataMap[keyName]?.maxValue ?? Number.NEGATIVE_INFINITY,
|
||||
value,
|
||||
),
|
||||
minValue: Math.min(
|
||||
lineDataMap[keyName]?.minValue ?? Number.POSITIVE_INFINITY,
|
||||
value,
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Custom sort and truncate top N lines
|
||||
const sortedLineDataMap = Object.values(lineDataMap).sort((a, b) => {
|
||||
return a.maxValue - b.maxValue;
|
||||
});
|
||||
|
||||
if (generateEmptyBuckets && granularity != null) {
|
||||
// Zero fill TODO: Make this an option
|
||||
const generatedTsBuckets = timeBucketByGranularity(
|
||||
dateRange[0],
|
||||
dateRange[1],
|
||||
granularity,
|
||||
);
|
||||
|
||||
generatedTsBuckets.forEach(date => {
|
||||
const ts = date.getTime() / 1000;
|
||||
const tsBucket = tsBucketMap.get(ts);
|
||||
|
||||
if (tsBucket == null) {
|
||||
const tsBucket: Record<string, any> = {
|
||||
[timestampColumn.name]: ts,
|
||||
};
|
||||
|
||||
for (const line of sortedLineDataMap) {
|
||||
tsBucket[line.dataKey] = 0;
|
||||
}
|
||||
|
||||
tsBucketMap.set(ts, tsBucket);
|
||||
} else {
|
||||
for (const line of sortedLineDataMap) {
|
||||
if (tsBucket[line.dataKey] == null) {
|
||||
tsBucket[line.dataKey] = 0;
|
||||
}
|
||||
}
|
||||
tsBucketMap.set(ts, tsBucket);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Sort results again by timestamp
|
||||
const graphResults: {
|
||||
[key: string]: number | undefined;
|
||||
}[] = Array.from(tsBucketMap.values()).sort(
|
||||
(a, b) => a[timestampColumn.name] - b[timestampColumn.name],
|
||||
);
|
||||
|
||||
// TODO: Return line color and names
|
||||
return {
|
||||
// dateRange: [minDate, maxDate],
|
||||
graphResults,
|
||||
timestampColumn,
|
||||
groupKeys: sortedLineDataMap.map(l => l.dataKey),
|
||||
lineNames: sortedLineDataMap.map(l => l.displayName),
|
||||
lineColors: sortedLineDataMap.map(l => l.color),
|
||||
};
|
||||
}
|
||||
|
||||
export type ColumnMeta = {
|
||||
codec_expression: string;
|
||||
comment: string;
|
||||
default_expression: string;
|
||||
default_type: string;
|
||||
name: string;
|
||||
ttl_expression: string;
|
||||
type: string;
|
||||
};
|
||||
|
|
@ -1,245 +0,0 @@
|
|||
import { z } from 'zod';
|
||||
|
||||
import { DisplayType } from '@/common/DisplayType';
|
||||
|
||||
// --------------------------
|
||||
// SQL TYPES
|
||||
// --------------------------
|
||||
// TODO: infer types from here and replaces all types in sqlTypes.ts
|
||||
export const SQLIntervalSchema = z
|
||||
.string()
|
||||
.regex(/^\d+ (second|minute|hour|day)$/);
|
||||
export const SearchConditionSchema = z.string();
|
||||
export const SearchConditionLanguageSchema = z
|
||||
.enum(['sql', 'lucene'])
|
||||
.optional();
|
||||
export const AggregateFunctionSchema = z.enum([
|
||||
'avg',
|
||||
'count',
|
||||
'count_distinct',
|
||||
'max',
|
||||
'min',
|
||||
'quantile',
|
||||
'sum',
|
||||
]);
|
||||
export const AggregateFunctionWithCombinatorsSchema = z
|
||||
.string()
|
||||
.regex(/^(\w+)If(State|Merge)$/);
|
||||
export const RootValueExpressionSchema = z
|
||||
.object({
|
||||
aggFn: z.union([
|
||||
AggregateFunctionSchema,
|
||||
AggregateFunctionWithCombinatorsSchema,
|
||||
]),
|
||||
aggCondition: SearchConditionSchema,
|
||||
aggConditionLanguage: SearchConditionLanguageSchema,
|
||||
valueExpression: z.string(),
|
||||
})
|
||||
.or(
|
||||
z.object({
|
||||
aggFn: z.literal('quantile'),
|
||||
level: z.number(),
|
||||
aggCondition: SearchConditionSchema,
|
||||
aggConditionLanguage: SearchConditionLanguageSchema,
|
||||
valueExpression: z.string(),
|
||||
}),
|
||||
)
|
||||
.or(
|
||||
z.object({
|
||||
aggFn: z.string().optional(),
|
||||
aggCondition: z.string().optional(),
|
||||
aggConditionLanguage: SearchConditionLanguageSchema,
|
||||
valueExpression: z.string(),
|
||||
}),
|
||||
);
|
||||
export const DerivedColumnSchema = z.intersection(
|
||||
RootValueExpressionSchema,
|
||||
z.object({
|
||||
alias: z.string().optional(),
|
||||
}),
|
||||
);
|
||||
export const SelectListSchema = z.array(DerivedColumnSchema).or(z.string());
|
||||
export const SortSpecificationSchema = z.intersection(
|
||||
RootValueExpressionSchema,
|
||||
z.object({
|
||||
ordering: z.enum(['ASC', 'DESC']),
|
||||
}),
|
||||
);
|
||||
export const SortSpecificationListSchema = z
|
||||
.array(SortSpecificationSchema)
|
||||
.or(z.string());
|
||||
export const LimitSchema = z.object({
|
||||
limit: z.number().optional(),
|
||||
offset: z.number().optional(),
|
||||
});
|
||||
export const SelectSQLStatementSchema = z.object({
|
||||
select: SelectListSchema,
|
||||
from: z.object({
|
||||
databaseName: z.string(),
|
||||
tableName: z.string(),
|
||||
}),
|
||||
where: SearchConditionSchema,
|
||||
whereLanguage: SearchConditionLanguageSchema,
|
||||
groupBy: SelectListSchema.optional(),
|
||||
having: SearchConditionSchema.optional(),
|
||||
havingLanguage: SearchConditionLanguageSchema.optional(),
|
||||
orderBy: SortSpecificationListSchema.optional(),
|
||||
limit: LimitSchema.optional(),
|
||||
});
|
||||
|
||||
// --------------------------
|
||||
// SAVED SEARCH
|
||||
// --------------------------
|
||||
export const SavedSearchSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
select: z.string(),
|
||||
where: z.string(),
|
||||
whereLanguage: SearchConditionLanguageSchema,
|
||||
source: z.string(),
|
||||
tags: z.array(z.string()),
|
||||
orderBy: z.string().optional(),
|
||||
});
|
||||
|
||||
export type SavedSearch = z.infer<typeof SavedSearchSchema>;
|
||||
|
||||
// --------------------------
|
||||
// DASHBOARDS
|
||||
// --------------------------
|
||||
export const NumberFormatSchema = z.object({
|
||||
output: z.enum(['currency', 'percent', 'byte', 'time', 'number']),
|
||||
mantissa: z.number().optional(),
|
||||
thousandSeparated: z.boolean().optional(),
|
||||
average: z.boolean().optional(),
|
||||
decimalBytes: z.boolean().optional(),
|
||||
factor: z.number().optional(),
|
||||
currencySymbol: z.string().optional(),
|
||||
unit: z.string().optional(),
|
||||
});
|
||||
|
||||
export const SqlAstFilterSchema = z.object({
|
||||
type: z.literal('sql_ast'),
|
||||
operator: z.enum(['=', '<', '>', '!=', '<=', '>=']),
|
||||
left: z.string(),
|
||||
right: z.string(),
|
||||
});
|
||||
|
||||
export const FilterSchema = z.union([
|
||||
z.object({
|
||||
type: z.enum(['lucene', 'sql']),
|
||||
condition: z.string(),
|
||||
}),
|
||||
SqlAstFilterSchema,
|
||||
]);
|
||||
|
||||
export const _ChartConfigSchema = z.object({
|
||||
displayType: z.nativeEnum(DisplayType),
|
||||
numberFormat: NumberFormatSchema.optional(),
|
||||
timestampValueExpression: z.string(),
|
||||
implicitColumnExpression: z.string().optional(),
|
||||
granularity: z.string().optional(),
|
||||
markdown: z.string().optional(),
|
||||
filtersLogicalOperator: z.enum(['AND', 'OR']).optional(),
|
||||
filters: z.array(FilterSchema).optional(),
|
||||
connection: z.string(),
|
||||
fillNulls: z.number().optional(),
|
||||
selectGroupBy: z.boolean().optional(),
|
||||
});
|
||||
|
||||
export const ChartConfigSchema = z.intersection(
|
||||
_ChartConfigSchema,
|
||||
SelectSQLStatementSchema,
|
||||
);
|
||||
|
||||
export const SavedChartConfigSchema = z.intersection(
|
||||
z.intersection(
|
||||
z.object({
|
||||
name: z.string(),
|
||||
source: z.string(),
|
||||
}),
|
||||
_ChartConfigSchema.omit({
|
||||
connection: true,
|
||||
timestampValueExpression: true,
|
||||
}),
|
||||
),
|
||||
SelectSQLStatementSchema.omit({
|
||||
from: true,
|
||||
}),
|
||||
);
|
||||
|
||||
export type SavedChartConfig = z.infer<typeof SavedChartConfigSchema>;
|
||||
|
||||
export const TileSchema = z.object({
|
||||
id: z.string(),
|
||||
x: z.number(),
|
||||
y: z.number(),
|
||||
w: z.number(),
|
||||
h: z.number(),
|
||||
config: SavedChartConfigSchema,
|
||||
});
|
||||
|
||||
export type Tile = z.infer<typeof TileSchema>;
|
||||
|
||||
export const DashboardSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
tiles: z.array(TileSchema),
|
||||
tags: z.array(z.string()),
|
||||
});
|
||||
|
||||
export const DashboardWithoutIdSchema = DashboardSchema.omit({ id: true });
|
||||
|
||||
export const ConnectionSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
host: z.string(),
|
||||
username: z.string(),
|
||||
password: z.string().optional(),
|
||||
});
|
||||
|
||||
// --------------------------
|
||||
// TABLE SOURCES
|
||||
// --------------------------
|
||||
export const SourceSchema = z.object({
|
||||
from: z.object({
|
||||
databaseName: z.string(),
|
||||
tableName: z.string(),
|
||||
}),
|
||||
timestampValueExpression: z.string(),
|
||||
connection: z.string(),
|
||||
|
||||
// Common
|
||||
kind: z.enum(['log', 'trace']),
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
displayedTimestampValueExpression: z.string().optional(),
|
||||
implicitColumnExpression: z.string().optional(),
|
||||
serviceNameExpression: z.string().optional(),
|
||||
bodyExpression: z.string().optional(),
|
||||
tableFilterExpression: z.string().optional(),
|
||||
eventAttributesExpression: z.string().optional(),
|
||||
resourceAttributesExpression: z.string().optional(),
|
||||
defaultTableSelectExpression: z.string().optional(),
|
||||
|
||||
// Logs
|
||||
uniqueRowIdExpression: z.string().optional(),
|
||||
severityTextExpression: z.string().optional(),
|
||||
traceSourceId: z.string().optional(),
|
||||
|
||||
// Traces & Logs
|
||||
traceIdExpression: z.string().optional(),
|
||||
spanIdExpression: z.string().optional(),
|
||||
|
||||
// Traces
|
||||
durationExpression: z.string().optional(),
|
||||
durationPrecision: z.number().min(0).max(9).optional(),
|
||||
parentSpanIdExpression: z.string().optional(),
|
||||
spanNameExpression: z.string().optional(),
|
||||
|
||||
spanKindExpression: z.string().optional(),
|
||||
statusCodeExpression: z.string().optional(),
|
||||
statusMessageExpression: z.string().optional(),
|
||||
logSourceId: z.string().optional(),
|
||||
});
|
||||
|
||||
export type TSource = z.infer<typeof SourceSchema>;
|
||||
|
|
@ -1,436 +0,0 @@
|
|||
import {
|
||||
ChSql,
|
||||
chSql,
|
||||
ColumnMeta,
|
||||
convertCHDataTypeToJSType,
|
||||
filterColumnMetaByType,
|
||||
JSDataType,
|
||||
sendQuery,
|
||||
tableExpr,
|
||||
} from '@/common/clickhouse';
|
||||
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
renderChartConfig,
|
||||
} from './renderChartConfig';
|
||||
|
||||
const DEFAULT_SAMPLE_SIZE = 1e6;
|
||||
|
||||
class MetadataCache {
|
||||
private cache = new Map<string, any>();
|
||||
|
||||
// this should be getOrUpdate... or just query to follow react query
|
||||
get<T>(key: string): T | undefined {
|
||||
return this.cache.get(key);
|
||||
}
|
||||
|
||||
async getOrFetch<T>(key: string, query: () => Promise<T>): Promise<T> {
|
||||
const value = this.get(key) as T | undefined;
|
||||
if (value != null) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const newValue = await query();
|
||||
this.cache.set(key, newValue);
|
||||
|
||||
return newValue;
|
||||
}
|
||||
|
||||
set<T>(key: string, value: T) {
|
||||
return this.cache.set(key, value);
|
||||
}
|
||||
|
||||
// TODO: This needs to be async, and use tanstack query on frontend for cache
|
||||
// TODO: Implement locks for refreshing
|
||||
// TODO: Shard cache by time
|
||||
}
|
||||
|
||||
export type TableMetadata = {
|
||||
database: string;
|
||||
name: string;
|
||||
uuid: string;
|
||||
engine: string;
|
||||
is_temporary: number;
|
||||
data_paths: string[];
|
||||
metadata_path: string;
|
||||
metadata_modification_time: string;
|
||||
metadata_version: number;
|
||||
create_table_query: string;
|
||||
engine_full: string;
|
||||
as_select: string;
|
||||
partition_key: string;
|
||||
sorting_key: string;
|
||||
primary_key: string;
|
||||
sampling_key: string;
|
||||
storage_policy: string;
|
||||
total_rows: string;
|
||||
total_bytes: string;
|
||||
total_bytes_uncompressed: string;
|
||||
parts: string;
|
||||
active_parts: string;
|
||||
total_marks: string;
|
||||
comment: string;
|
||||
};
|
||||
|
||||
export class Metadata {
|
||||
private cache = new MetadataCache();
|
||||
|
||||
private static async queryTableMetadata({
|
||||
database,
|
||||
table,
|
||||
cache,
|
||||
connectionId,
|
||||
}: {
|
||||
database: string;
|
||||
table: string;
|
||||
cache: MetadataCache;
|
||||
connectionId: string;
|
||||
}) {
|
||||
return cache.getOrFetch(`${database}.${table}.metadata`, async () => {
|
||||
const sql = chSql`SELECT * FROM system.tables where database = ${{ String: database }} AND name = ${{ String: table }}`;
|
||||
const json = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
}).then(res => res.json<TableMetadata>());
|
||||
return json.data[0];
|
||||
});
|
||||
}
|
||||
|
||||
async getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
}) {
|
||||
return this.cache.getOrFetch<ColumnMeta[]>(
|
||||
`${databaseName}.${tableName}.columns`,
|
||||
async () => {
|
||||
const sql = chSql`DESCRIBE ${tableExpr({ database: databaseName, table: tableName })}`;
|
||||
const columns = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
})
|
||||
.then(res => res.json())
|
||||
.then(d => d.data);
|
||||
return columns as ColumnMeta[];
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async getMaterializedColumnsLookupTable({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const columns = await this.getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
// Build up materalized fields lookup table
|
||||
return new Map(
|
||||
columns
|
||||
.filter(
|
||||
c =>
|
||||
c.default_type === 'MATERIALIZED' || c.default_type === 'DEFAULT',
|
||||
)
|
||||
.map(c => [c.default_expression, c.name]),
|
||||
);
|
||||
}
|
||||
|
||||
async getColumn({
|
||||
databaseName,
|
||||
tableName,
|
||||
column,
|
||||
matchLowercase = false,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
column: string;
|
||||
matchLowercase?: boolean;
|
||||
connectionId: string;
|
||||
}): Promise<ColumnMeta | undefined> {
|
||||
const tableColumns = await this.getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
return tableColumns.filter(c => {
|
||||
if (matchLowercase) {
|
||||
return c.name.toLowerCase() === column.toLowerCase();
|
||||
}
|
||||
|
||||
return c.name === column;
|
||||
})[0];
|
||||
}
|
||||
|
||||
async getMapKeys({
|
||||
databaseName,
|
||||
tableName,
|
||||
column,
|
||||
maxKeys = 1000,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
column: string;
|
||||
maxKeys?: number;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const cachedKeys = this.cache.get<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.keys`,
|
||||
);
|
||||
|
||||
if (cachedKeys != null) {
|
||||
return cachedKeys;
|
||||
}
|
||||
|
||||
const colMeta = await this.getColumn({
|
||||
databaseName,
|
||||
tableName,
|
||||
column,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
if (colMeta == null) {
|
||||
throw new Error(
|
||||
`Column ${column} not found in ${databaseName}.${tableName}`,
|
||||
);
|
||||
}
|
||||
|
||||
let strategy: 'groupUniqArrayArray' | 'lowCardinalityKeys' =
|
||||
'groupUniqArrayArray';
|
||||
if (colMeta.type.startsWith('Map(LowCardinality(String)')) {
|
||||
strategy = 'lowCardinalityKeys';
|
||||
}
|
||||
|
||||
let sql: ChSql;
|
||||
if (strategy === 'groupUniqArrayArray') {
|
||||
sql = chSql`SELECT groupUniqArrayArray(${{ Int32: maxKeys }})(${{
|
||||
Identifier: column,
|
||||
}}) as keysArr
|
||||
FROM ${tableExpr({ database: databaseName, table: tableName })}`;
|
||||
} else {
|
||||
sql = chSql`SELECT DISTINCT lowCardinalityKeys(arrayJoin(${{
|
||||
Identifier: column,
|
||||
}}.keys)) as key
|
||||
FROM ${tableExpr({ database: databaseName, table: tableName })}
|
||||
LIMIT ${{
|
||||
Int32: maxKeys,
|
||||
}}`;
|
||||
}
|
||||
|
||||
return this.cache.getOrFetch<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.keys`,
|
||||
async () => {
|
||||
const keys = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
})
|
||||
.then(res => res.json<Record<string, unknown>>())
|
||||
.then(d => {
|
||||
let output: string[];
|
||||
if (strategy === 'groupUniqArrayArray') {
|
||||
output = d.data[0].keysArr as string[];
|
||||
} else {
|
||||
output = d.data.map(row => row.key) as string[];
|
||||
}
|
||||
|
||||
return output.filter(r => r);
|
||||
});
|
||||
return keys;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async getMapValues({
|
||||
databaseName,
|
||||
tableName,
|
||||
column,
|
||||
key,
|
||||
maxValues = 20,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
column: string;
|
||||
key?: string;
|
||||
maxValues?: number;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const cachedValues = this.cache.get<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.${key}.values`,
|
||||
);
|
||||
|
||||
if (cachedValues != null) {
|
||||
return cachedValues;
|
||||
}
|
||||
|
||||
const sql = key
|
||||
? chSql`
|
||||
SELECT DISTINCT ${{
|
||||
Identifier: column,
|
||||
}}[${{ String: key }}] as value
|
||||
FROM ${tableExpr({ database: databaseName, table: tableName })}
|
||||
WHERE value != ''
|
||||
LIMIT ${{
|
||||
Int32: maxValues,
|
||||
}}
|
||||
`
|
||||
: chSql`
|
||||
SELECT DISTINCT ${{
|
||||
Identifier: column,
|
||||
}} as value
|
||||
FROM ${tableExpr({ database: databaseName, table: tableName })}
|
||||
WHERE value != ''
|
||||
LIMIT ${{
|
||||
Int32: maxValues,
|
||||
}}
|
||||
`;
|
||||
|
||||
return this.cache.getOrFetch<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.${key}.values`,
|
||||
async () => {
|
||||
const values = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
})
|
||||
.then(res => res.json<Record<string, unknown>>())
|
||||
.then(d => d.data.map(row => row.value as string));
|
||||
return values;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async getAllFields({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const fields: Field[] = [];
|
||||
const columns = await this.getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
for (const c of columns) {
|
||||
fields.push({
|
||||
path: [c.name],
|
||||
type: c.type,
|
||||
jsType: convertCHDataTypeToJSType(c.type),
|
||||
});
|
||||
}
|
||||
|
||||
const mapColumns = filterColumnMetaByType(columns, [JSDataType.Map]) ?? [];
|
||||
|
||||
await Promise.all(
|
||||
mapColumns.map(async column => {
|
||||
const keys = await this.getMapKeys({
|
||||
databaseName,
|
||||
tableName,
|
||||
column: column.name,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
const match = column.type.match(/Map\(.+,\s*(.+)\)/);
|
||||
const chType = match?.[1] ?? 'String'; // default to string ?
|
||||
|
||||
for (const key of keys) {
|
||||
fields.push({
|
||||
path: [column.name, key],
|
||||
type: chType,
|
||||
jsType: convertCHDataTypeToJSType(chType),
|
||||
});
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
return fields;
|
||||
}
|
||||
|
||||
async getTableMetadata({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const tableMetadata = await Metadata.queryTableMetadata({
|
||||
cache: this.cache,
|
||||
database: databaseName,
|
||||
table: tableName,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
return tableMetadata;
|
||||
}
|
||||
|
||||
async getKeyValues({
|
||||
chartConfig,
|
||||
keys,
|
||||
limit = 20,
|
||||
}: {
|
||||
chartConfig: ChartConfigWithDateRange;
|
||||
keys: string[];
|
||||
limit?: number;
|
||||
}) {
|
||||
const sql = await renderChartConfig({
|
||||
...chartConfig,
|
||||
select: keys
|
||||
.map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`)
|
||||
.join(', '),
|
||||
});
|
||||
|
||||
const json = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId: chartConfig.connection,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
}).then(res => res.json<any>());
|
||||
|
||||
return Object.entries(json.data[0]).map(([key, value]) => ({
|
||||
key: keys[parseInt(key.replace('param', ''))],
|
||||
value: (value as string[])?.filter(Boolean), // remove nulls
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
export type Field = {
|
||||
path: string[];
|
||||
type: string;
|
||||
jsType: JSDataType | null;
|
||||
};
|
||||
|
||||
export const metadata = new Metadata();
|
||||
|
|
@ -1,717 +0,0 @@
|
|||
import lucene from '@hyperdx/lucene';
|
||||
import SqlString from 'sqlstring';
|
||||
|
||||
import { convertCHTypeToPrimitiveJSType } from '@/common/clickhouse';
|
||||
import { Metadata } from '@/common/metadata';
|
||||
|
||||
function encodeSpecialTokens(query: string): string {
|
||||
return query
|
||||
.replace(/\\\\/g, 'HDX_BACKSLASH_LITERAL')
|
||||
.replace('http://', 'http_COLON_//')
|
||||
.replace('https://', 'https_COLON_//')
|
||||
.replace(/localhost:(\d{1,5})/, 'localhost_COLON_$1')
|
||||
.replace(/\\:/g, 'HDX_COLON');
|
||||
}
|
||||
function decodeSpecialTokens(query: string): string {
|
||||
return query
|
||||
.replace(/\\"/g, '"')
|
||||
.replace(/HDX_BACKSLASH_LITERAL/g, '\\')
|
||||
.replace('http_COLON_//', 'http://')
|
||||
.replace('https_COLON_//', 'https://')
|
||||
.replace(/localhost_COLON_(\d{1,5})/, 'localhost:$1')
|
||||
.replace(/HDX_COLON/g, ':');
|
||||
}
|
||||
|
||||
export function parse(query: string): lucene.AST {
|
||||
return lucene.parse(encodeSpecialTokens(query));
|
||||
}
|
||||
|
||||
const IMPLICIT_FIELD = '<implicit>';
|
||||
|
||||
interface Serializer {
|
||||
operator(op: lucene.Operator): string;
|
||||
eq(field: string, term: string, isNegatedField: boolean): Promise<string>;
|
||||
isNotNull(field: string, isNegatedField: boolean): Promise<string>;
|
||||
gte(field: string, term: string): Promise<string>;
|
||||
lte(field: string, term: string): Promise<string>;
|
||||
lt(field: string, term: string): Promise<string>;
|
||||
gt(field: string, term: string): Promise<string>;
|
||||
fieldSearch(
|
||||
field: string,
|
||||
term: string,
|
||||
isNegatedField: boolean,
|
||||
prefixWildcard: boolean,
|
||||
suffixWildcard: boolean,
|
||||
): Promise<string>;
|
||||
range(
|
||||
field: string,
|
||||
start: string,
|
||||
end: string,
|
||||
isNegatedField: boolean,
|
||||
): Promise<string>;
|
||||
}
|
||||
|
||||
class EnglishSerializer implements Serializer {
|
||||
private translateField(field: string) {
|
||||
if (field === IMPLICIT_FIELD) {
|
||||
return 'event';
|
||||
}
|
||||
|
||||
return `'${field}'`;
|
||||
}
|
||||
|
||||
operator(op: lucene.Operator) {
|
||||
switch (op) {
|
||||
case 'NOT':
|
||||
case 'AND NOT':
|
||||
return 'AND NOT';
|
||||
case 'OR NOT':
|
||||
return 'OR NOT';
|
||||
// @ts-ignore TODO: Types need to be fixed upstream
|
||||
case '&&':
|
||||
case '<implicit>':
|
||||
case 'AND':
|
||||
return 'AND';
|
||||
// @ts-ignore TODO: Types need to be fixed upstream
|
||||
case '||':
|
||||
case 'OR':
|
||||
return 'OR';
|
||||
default:
|
||||
throw new Error(`Unexpected operator. ${op}`);
|
||||
}
|
||||
}
|
||||
|
||||
async eq(field: string, term: string, isNegatedField: boolean) {
|
||||
return `${this.translateField(field)} ${
|
||||
isNegatedField ? 'is not' : 'is'
|
||||
} ${term}`;
|
||||
}
|
||||
|
||||
async isNotNull(field: string, isNegatedField: boolean) {
|
||||
return `${this.translateField(field)} ${
|
||||
isNegatedField ? 'is null' : 'is not null'
|
||||
}`;
|
||||
}
|
||||
|
||||
async gte(field: string, term: string) {
|
||||
return `${this.translateField(field)} is greater than or equal to ${term}`;
|
||||
}
|
||||
|
||||
async lte(field: string, term: string) {
|
||||
return `${this.translateField(field)} is less than or equal to ${term}`;
|
||||
}
|
||||
|
||||
async lt(field: string, term: string) {
|
||||
return `${this.translateField(field)} is less than ${term}`;
|
||||
}
|
||||
|
||||
async gt(field: string, term: string) {
|
||||
return `${this.translateField(field)} is greater than ${term}`;
|
||||
}
|
||||
|
||||
// async fieldSearch(field: string, term: string, isNegatedField: boolean) {
|
||||
// return `${this.translateField(field)} ${
|
||||
// isNegatedField ? 'does not contain' : 'contains'
|
||||
// } ${term}`;
|
||||
// }
|
||||
|
||||
async fieldSearch(
|
||||
field: string,
|
||||
term: string,
|
||||
isNegatedField: boolean,
|
||||
prefixWildcard: boolean,
|
||||
suffixWildcard: boolean,
|
||||
) {
|
||||
if (field === IMPLICIT_FIELD) {
|
||||
return `${this.translateField(field)} ${
|
||||
prefixWildcard && suffixWildcard
|
||||
? isNegatedField
|
||||
? 'does not contain'
|
||||
: 'contains'
|
||||
: prefixWildcard
|
||||
? isNegatedField
|
||||
? 'does not end with'
|
||||
: 'ends with'
|
||||
: suffixWildcard
|
||||
? isNegatedField
|
||||
? 'does not start with'
|
||||
: 'starts with'
|
||||
: isNegatedField
|
||||
? 'does not have whole word'
|
||||
: 'has whole word'
|
||||
} ${term}`;
|
||||
} else {
|
||||
return `${this.translateField(field)} ${
|
||||
isNegatedField ? 'does not contain' : 'contains'
|
||||
} ${term}`;
|
||||
}
|
||||
}
|
||||
|
||||
async range(
|
||||
field: string,
|
||||
start: string,
|
||||
end: string,
|
||||
isNegatedField: boolean,
|
||||
) {
|
||||
return `${field} ${
|
||||
isNegatedField ? 'is not' : 'is'
|
||||
} between ${start} and ${end}`;
|
||||
}
|
||||
}
|
||||
|
||||
export abstract class SQLSerializer implements Serializer {
|
||||
private NOT_FOUND_QUERY = '(1 = 0)';
|
||||
|
||||
abstract getColumnForField(field: string): Promise<{
|
||||
column?: string;
|
||||
propertyType?: 'string' | 'number' | 'bool';
|
||||
found: boolean;
|
||||
}>;
|
||||
|
||||
operator(op: lucene.Operator) {
|
||||
switch (op) {
|
||||
case 'NOT':
|
||||
case 'AND NOT':
|
||||
return 'AND NOT';
|
||||
case 'OR NOT':
|
||||
return 'OR NOT';
|
||||
// @ts-ignore TODO: Types need to be fixed upstream
|
||||
case '&&':
|
||||
case '<implicit>':
|
||||
case 'AND':
|
||||
return 'AND';
|
||||
// @ts-ignore TODO: Types need to be fixed upstream
|
||||
case '||':
|
||||
case 'OR':
|
||||
return 'OR';
|
||||
default:
|
||||
throw new Error(`Unexpected operator. ${op}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Only for exact string matches
|
||||
async eq(field: string, term: string, isNegatedField: boolean) {
|
||||
const { column, found, propertyType } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
if (propertyType === 'bool') {
|
||||
// numeric and boolean fields must be equality matched
|
||||
const normTerm = `${term}`.trim().toLowerCase();
|
||||
return SqlString.format(`(?? ${isNegatedField ? '!' : ''}= ?)`, [
|
||||
column,
|
||||
normTerm === 'true' ? 1 : normTerm === 'false' ? 0 : parseInt(normTerm),
|
||||
]);
|
||||
} else if (propertyType === 'number') {
|
||||
return SqlString.format(
|
||||
`(${column} ${isNegatedField ? '!' : ''}= CAST(?, 'Float64'))`,
|
||||
[term],
|
||||
);
|
||||
}
|
||||
return SqlString.format(`(${column} ${isNegatedField ? '!' : ''}= ?)`, [
|
||||
term,
|
||||
]);
|
||||
}
|
||||
|
||||
async isNotNull(field: string, isNegatedField: boolean) {
|
||||
const { found, column } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
|
||||
return `notEmpty(${column}) ${isNegatedField ? '!' : ''}= 1`;
|
||||
}
|
||||
|
||||
async gte(field: string, term: string) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(`(${column} >= ?)`, [term]);
|
||||
}
|
||||
|
||||
async lte(field: string, term: string) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(`(${column} <= ?)`, [term]);
|
||||
}
|
||||
|
||||
async lt(field: string, term: string) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(`(${column} < ?)`, [term]);
|
||||
}
|
||||
|
||||
async gt(field: string, term: string) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(`(${column} > ?)`, [term]);
|
||||
}
|
||||
|
||||
// TODO: Not sure if SQL really needs this or if it'll coerce itself
|
||||
private attemptToParseNumber(term: string): string | number {
|
||||
const number = Number.parseFloat(term);
|
||||
if (Number.isNaN(number)) {
|
||||
return term;
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
// Ref: https://clickhouse.com/codebrowser/ClickHouse/src/Functions/HasTokenImpl.h.html#_ZN2DB12HasTokenImpl16isTokenSeparatorEDu
|
||||
// Split by anything that's ascii 0-128, that's not a letter or a number
|
||||
private tokenizeTerm(term: string): string[] {
|
||||
return term.split(/[ -/:-@[-`{-~\t\n\r]+/).filter(t => t.length > 0);
|
||||
}
|
||||
|
||||
private termHasSeperators(term: string): boolean {
|
||||
return term.match(/[ -/:-@[-`{-~\t\n\r]+/) != null;
|
||||
}
|
||||
|
||||
async fieldSearch(
|
||||
field: string,
|
||||
term: string,
|
||||
isNegatedField: boolean,
|
||||
prefixWildcard: boolean,
|
||||
suffixWildcard: boolean,
|
||||
) {
|
||||
const isImplicitField = field === IMPLICIT_FIELD;
|
||||
const { column, propertyType, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
// If it's a string field, we will always try to match with ilike
|
||||
|
||||
if (propertyType === 'bool') {
|
||||
// numeric and boolean fields must be equality matched
|
||||
const normTerm = `${term}`.trim().toLowerCase();
|
||||
return SqlString.format(`(?? ${isNegatedField ? '!' : ''}= ?)`, [
|
||||
column,
|
||||
normTerm === 'true' ? 1 : normTerm === 'false' ? 0 : parseInt(normTerm),
|
||||
]);
|
||||
} else if (propertyType === 'number') {
|
||||
return SqlString.format(
|
||||
`(?? ${isNegatedField ? '!' : ''}= CAST(?, 'Float64'))`,
|
||||
[column, term],
|
||||
);
|
||||
}
|
||||
|
||||
// // If the query is empty, or is a empty quoted string ex: ""
|
||||
// // we should match all
|
||||
if (term.length === 0) {
|
||||
return '(1=1)';
|
||||
}
|
||||
|
||||
if (isImplicitField) {
|
||||
// For the _source column, we'll try to do whole word searches by default
|
||||
// to utilize the token bloom filter unless a prefix/sufix wildcard is specified
|
||||
if (prefixWildcard || suffixWildcard) {
|
||||
return SqlString.format(
|
||||
`(lower(??) ${isNegatedField ? 'NOT ' : ''}LIKE lower(?))`,
|
||||
[
|
||||
column,
|
||||
`${prefixWildcard ? '%' : ''}${term}${suffixWildcard ? '%' : ''}`,
|
||||
],
|
||||
);
|
||||
} else {
|
||||
// We can't search multiple tokens with `hasToken`, so we need to split up the term into tokens
|
||||
const hasSeperators = this.termHasSeperators(term);
|
||||
if (hasSeperators) {
|
||||
const tokens = this.tokenizeTerm(term);
|
||||
return `(${isNegatedField ? 'NOT (' : ''}${[
|
||||
...tokens.map(token =>
|
||||
SqlString.format(`hasTokenCaseInsensitive(??, ?)`, [
|
||||
column,
|
||||
token,
|
||||
]),
|
||||
),
|
||||
// If there are symbols in the term, we'll try to match the whole term as well (ex. Scott!)
|
||||
SqlString.format(`(lower(??) LIKE lower(?))`, [
|
||||
column,
|
||||
`%${term}%`,
|
||||
]),
|
||||
].join(' AND ')}${isNegatedField ? ')' : ''})`;
|
||||
} else {
|
||||
return SqlString.format(
|
||||
`(${isNegatedField ? 'NOT ' : ''}hasTokenCaseInsensitive(??, ?))`,
|
||||
[column, term],
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const shoudUseTokenBf = isImplicitField;
|
||||
return SqlString.format(
|
||||
`(${column} ${isNegatedField ? 'NOT ' : ''}? ?)`,
|
||||
[SqlString.raw(shoudUseTokenBf ? 'LIKE' : 'ILIKE'), `%${term}%`],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async range(
|
||||
field: string,
|
||||
start: string,
|
||||
end: string,
|
||||
isNegatedField: boolean,
|
||||
) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(
|
||||
`(${column} ${isNegatedField ? 'NOT ' : ''}BETWEEN ? AND ?)`,
|
||||
[this.attemptToParseNumber(start), this.attemptToParseNumber(end)],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export type CustomSchemaConfig = {
|
||||
databaseName: string;
|
||||
implicitColumnExpression?: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
};
|
||||
|
||||
export class CustomSchemaSQLSerializerV2 extends SQLSerializer {
|
||||
private metadata: Metadata;
|
||||
private tableName: string;
|
||||
private databaseName: string;
|
||||
private implicitColumnExpression?: string;
|
||||
private connectionId: string;
|
||||
|
||||
constructor({
|
||||
metadata,
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
implicitColumnExpression,
|
||||
}: { metadata: Metadata } & CustomSchemaConfig) {
|
||||
super();
|
||||
this.metadata = metadata;
|
||||
this.databaseName = databaseName;
|
||||
this.tableName = tableName;
|
||||
this.implicitColumnExpression = implicitColumnExpression;
|
||||
this.connectionId = connectionId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate field from user ex. column.property.subproperty to SQL expression
|
||||
* Supports:
|
||||
* - Materialized Columns
|
||||
* - Map
|
||||
* - JSON Strings (via JSONExtract)
|
||||
* TODO:
|
||||
* - Nested Map
|
||||
* - JSONExtract for non-string types
|
||||
*/
|
||||
private async buildColumnExpressionFromField(field: string) {
|
||||
const exactMatch = await this.metadata.getColumn({
|
||||
databaseName: this.databaseName,
|
||||
tableName: this.tableName,
|
||||
column: field,
|
||||
connectionId: this.connectionId,
|
||||
});
|
||||
|
||||
if (exactMatch) {
|
||||
return {
|
||||
found: true,
|
||||
columnType: exactMatch.type,
|
||||
columnExpression: exactMatch.name,
|
||||
};
|
||||
}
|
||||
|
||||
const fieldPrefix = field.split('.')[0];
|
||||
const prefixMatch = await this.metadata.getColumn({
|
||||
databaseName: this.databaseName,
|
||||
tableName: this.tableName,
|
||||
column: fieldPrefix,
|
||||
connectionId: this.connectionId,
|
||||
});
|
||||
|
||||
if (prefixMatch) {
|
||||
const fieldPostfix = field.split('.').slice(1).join('.');
|
||||
|
||||
if (prefixMatch.type.startsWith('Map')) {
|
||||
const valueType = prefixMatch.type.match(/,\s+(\w+)\)$/)?.[1];
|
||||
return {
|
||||
found: true,
|
||||
columnExpression: SqlString.format(`??[?]`, [
|
||||
prefixMatch.name,
|
||||
fieldPostfix,
|
||||
]),
|
||||
columnType: valueType ?? 'Unknown',
|
||||
};
|
||||
} else if (prefixMatch.type === 'String') {
|
||||
// TODO: Support non-strings
|
||||
const nestedPaths = fieldPostfix.split('.');
|
||||
return {
|
||||
found: true,
|
||||
columnExpression: SqlString.format(
|
||||
`JSONExtractString(??, ${Array(nestedPaths.length)
|
||||
.fill('?')
|
||||
.join(',')})`,
|
||||
[prefixMatch.name, ...nestedPaths],
|
||||
),
|
||||
columnType: 'String',
|
||||
};
|
||||
}
|
||||
// TODO: Support arrays and tuples
|
||||
throw new Error('Unsupported column type for prefix match');
|
||||
}
|
||||
|
||||
throw new Error(`Column not found: ${field}`);
|
||||
}
|
||||
|
||||
async getColumnForField(field: string) {
|
||||
if (field === IMPLICIT_FIELD) {
|
||||
if (!this.implicitColumnExpression) {
|
||||
throw new Error(
|
||||
'Can not search bare text without an implicit column set.',
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
column: this.implicitColumnExpression,
|
||||
propertyType: 'string' as const,
|
||||
found: true,
|
||||
};
|
||||
}
|
||||
|
||||
const expression = await this.buildColumnExpressionFromField(field);
|
||||
|
||||
return {
|
||||
column: expression.columnExpression,
|
||||
propertyType:
|
||||
convertCHTypeToPrimitiveJSType(expression.columnType) ?? undefined,
|
||||
found: expression.found,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function nodeTerm(
|
||||
node: lucene.Node,
|
||||
serializer: Serializer,
|
||||
): Promise<string> {
|
||||
const field = node.field[0] === '-' ? node.field.slice(1) : node.field;
|
||||
let isNegatedField = node.field[0] === '-';
|
||||
const isImplicitField = node.field === IMPLICIT_FIELD;
|
||||
|
||||
// NodeTerm
|
||||
if ((node as lucene.NodeTerm).term != null) {
|
||||
const nodeTerm = node as lucene.NodeTerm;
|
||||
let term = decodeSpecialTokens(nodeTerm.term);
|
||||
// We should only negate the search for negated bare terms (ex. '-5')
|
||||
// This meeans the field is implicit and the prefix is -
|
||||
if (isImplicitField && nodeTerm.prefix === '-') {
|
||||
isNegatedField = true;
|
||||
}
|
||||
// Otherwise, if we have a negated term for a field (ex. 'level:-5')
|
||||
// we should not negate the search, and search for -5
|
||||
if (!isImplicitField && nodeTerm.prefix === '-') {
|
||||
term = nodeTerm.prefix + decodeSpecialTokens(nodeTerm.term);
|
||||
}
|
||||
|
||||
// TODO: Decide if this is good behavior
|
||||
// If the term is quoted, we should search for the exact term in a property (ex. foo:"bar")
|
||||
// Implicit field searches should still use substring matching (ex. "foo bar")
|
||||
if (nodeTerm.quoted && !isImplicitField) {
|
||||
return serializer.eq(field, term, isNegatedField);
|
||||
}
|
||||
|
||||
if (!nodeTerm.quoted && term === '*') {
|
||||
return serializer.isNotNull(field, isNegatedField);
|
||||
}
|
||||
|
||||
if (!nodeTerm.quoted && term.substring(0, 2) === '>=') {
|
||||
if (isNegatedField) {
|
||||
return serializer.lt(field, term.slice(2));
|
||||
}
|
||||
return serializer.gte(field, term.slice(2));
|
||||
}
|
||||
if (!nodeTerm.quoted && term.substring(0, 2) === '<=') {
|
||||
if (isNegatedField) {
|
||||
return serializer.gt(field, term.slice(2));
|
||||
}
|
||||
return serializer.lte(field, term.slice(2));
|
||||
}
|
||||
if (!nodeTerm.quoted && term[0] === '>') {
|
||||
if (isNegatedField) {
|
||||
return serializer.lte(field, term.slice(1));
|
||||
}
|
||||
return serializer.gt(field, term.slice(1));
|
||||
}
|
||||
if (!nodeTerm.quoted && term[0] === '<') {
|
||||
if (isNegatedField) {
|
||||
return serializer.gte(field, term.slice(1));
|
||||
}
|
||||
return serializer.lt(field, term.slice(1));
|
||||
}
|
||||
|
||||
let prefixWildcard = false;
|
||||
let suffixWildcard = false;
|
||||
if (!nodeTerm.quoted && term[0] === '*') {
|
||||
prefixWildcard = true;
|
||||
term = term.slice(1);
|
||||
}
|
||||
if (!nodeTerm.quoted && term[term.length - 1] === '*') {
|
||||
suffixWildcard = true;
|
||||
term = term.slice(0, -1);
|
||||
}
|
||||
|
||||
return serializer.fieldSearch(
|
||||
field,
|
||||
term,
|
||||
isNegatedField,
|
||||
prefixWildcard,
|
||||
suffixWildcard,
|
||||
);
|
||||
|
||||
// TODO: Handle regex, similarity, boost, prefix
|
||||
}
|
||||
// NodeRangedTerm
|
||||
if ((node as lucene.NodeRangedTerm).inclusive != null) {
|
||||
const rangedTerm = node as lucene.NodeRangedTerm;
|
||||
return serializer.range(
|
||||
field,
|
||||
rangedTerm.term_min,
|
||||
rangedTerm.term_max,
|
||||
isNegatedField,
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected Node type. ${node}`);
|
||||
}
|
||||
|
||||
async function serialize(
|
||||
ast: lucene.AST | lucene.Node,
|
||||
serializer: Serializer,
|
||||
): Promise<string> {
|
||||
// Node Scenarios:
|
||||
// 1. NodeTerm: Single term ex. "foo:bar"
|
||||
// 2. NodeRangedTerm: Two terms ex. "foo:[bar TO qux]"
|
||||
if ((ast as lucene.NodeTerm).term != null) {
|
||||
return await nodeTerm(ast as lucene.NodeTerm, serializer);
|
||||
}
|
||||
if ((ast as lucene.NodeRangedTerm).inclusive != null) {
|
||||
return await nodeTerm(ast as lucene.NodeTerm, serializer);
|
||||
}
|
||||
|
||||
// AST Scenarios:
|
||||
// 1. BinaryAST: Two terms ex. "foo:bar AND baz:qux"
|
||||
// 2. LeftOnlyAST: Single term ex. "foo:bar"
|
||||
if ((ast as lucene.BinaryAST).right != null) {
|
||||
const binaryAST = ast as lucene.BinaryAST;
|
||||
const operator = serializer.operator(binaryAST.operator);
|
||||
const parenthesized = binaryAST.parenthesized;
|
||||
return `${parenthesized ? '(' : ''}${await serialize(
|
||||
binaryAST.left,
|
||||
serializer,
|
||||
)} ${operator} ${await serialize(binaryAST.right, serializer)}${
|
||||
parenthesized ? ')' : ''
|
||||
}`;
|
||||
}
|
||||
|
||||
if ((ast as lucene.LeftOnlyAST).left != null) {
|
||||
const leftOnlyAST = ast as lucene.LeftOnlyAST;
|
||||
const parenthesized = leftOnlyAST.parenthesized;
|
||||
// start is used when ex. "NOT foo:bar"
|
||||
return `${parenthesized ? '(' : ''}${
|
||||
leftOnlyAST.start != undefined ? `${leftOnlyAST.start} ` : ''
|
||||
}${await serialize(leftOnlyAST.left, serializer)}${
|
||||
parenthesized ? ')' : ''
|
||||
}`;
|
||||
}
|
||||
|
||||
// Blank AST, means no text was parsed
|
||||
return '';
|
||||
}
|
||||
|
||||
// TODO: can just inline this within getSearchQuery
|
||||
export async function genWhereSQL(
|
||||
ast: lucene.AST,
|
||||
serializer: Serializer,
|
||||
): Promise<string> {
|
||||
return await serialize(ast, serializer);
|
||||
}
|
||||
|
||||
export class SearchQueryBuilder {
|
||||
private readonly searchQ: string;
|
||||
|
||||
private readonly conditions: string[];
|
||||
|
||||
private serializer: SQLSerializer;
|
||||
|
||||
constructor(searchQ: string, serializer: SQLSerializer) {
|
||||
this.conditions = [];
|
||||
this.searchQ = searchQ;
|
||||
// init default serializer
|
||||
this.serializer = serializer;
|
||||
}
|
||||
|
||||
setSerializer(serializer: SQLSerializer) {
|
||||
this.serializer = serializer;
|
||||
return this;
|
||||
}
|
||||
|
||||
getSerializer() {
|
||||
return this.serializer;
|
||||
}
|
||||
|
||||
private async genSearchQuery() {
|
||||
if (!this.searchQ) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// const implicitColumn = await this.serializer.getColumnForField(
|
||||
// IMPLICIT_FIELD,
|
||||
// );
|
||||
|
||||
// let querySql = this.searchQ
|
||||
// .split(/\s+/)
|
||||
// .map(queryToken =>
|
||||
// SqlString.format(`lower(??) LIKE lower(?)`, [
|
||||
// implicitColumn.column,
|
||||
// `%${queryToken}%`,
|
||||
// ]),
|
||||
// )
|
||||
// .join(' AND ');
|
||||
|
||||
const parsedQ = parse(this.searchQ);
|
||||
|
||||
return await genWhereSQL(parsedQ, this.serializer);
|
||||
}
|
||||
|
||||
and(condition: string) {
|
||||
if (condition && condition.trim()) {
|
||||
this.conditions.push(`(${condition})`);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
async build() {
|
||||
const searchQuery = await this.genSearchQuery();
|
||||
if (this.searchQ) {
|
||||
this.and(searchQuery);
|
||||
}
|
||||
return this.conditions.join(' AND ');
|
||||
}
|
||||
}
|
||||
|
||||
export async function genEnglishExplanation(query: string): Promise<string> {
|
||||
try {
|
||||
const parsedQ = parse(query);
|
||||
|
||||
if (parsedQ) {
|
||||
const serializer = new EnglishSerializer();
|
||||
return await serialize(parsedQ, serializer);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Parse failure', query, e);
|
||||
}
|
||||
|
||||
return `Message containing ${query}`;
|
||||
}
|
||||
|
|
@ -1,782 +0,0 @@
|
|||
import isPlainObject from 'lodash/isPlainObject';
|
||||
import * as SQLParser from 'node-sql-parser';
|
||||
|
||||
import {
|
||||
ChSql,
|
||||
chSql,
|
||||
concatChSql,
|
||||
wrapChSqlIfNotEmpty,
|
||||
} from '@/common/clickhouse';
|
||||
import { DisplayType } from '@/common/DisplayType';
|
||||
import { Metadata, metadata } from '@/common/metadata';
|
||||
import {
|
||||
CustomSchemaSQLSerializerV2,
|
||||
SearchQueryBuilder,
|
||||
} from '@/common/queryParser';
|
||||
import {
|
||||
AggregateFunction,
|
||||
AggregateFunctionWithCombinators,
|
||||
SearchCondition,
|
||||
SearchConditionLanguage,
|
||||
SelectList,
|
||||
SelectSQLStatement,
|
||||
SortSpecificationList,
|
||||
SQLInterval,
|
||||
} from '@/common/sqlTypes';
|
||||
import {
|
||||
convertDateRangeToGranularityString,
|
||||
getFirstTimestampValueExpression,
|
||||
} from '@/common/utils';
|
||||
|
||||
// FIXME: SQLParser.ColumnRef is incomplete
|
||||
type ColumnRef = SQLParser.ColumnRef & {
|
||||
array_index?: {
|
||||
index: { type: string; value: string };
|
||||
}[];
|
||||
};
|
||||
|
||||
export type NumberFormat = {
|
||||
output?: 'currency' | 'percent' | 'byte' | 'time' | 'number';
|
||||
mantissa?: number;
|
||||
thousandSeparated?: boolean;
|
||||
average?: boolean;
|
||||
decimalBytes?: boolean;
|
||||
factor?: number;
|
||||
currencySymbol?: string;
|
||||
unit?: string;
|
||||
};
|
||||
|
||||
export type SqlAstFilter = {
|
||||
type: 'sql_ast';
|
||||
operator: '=' | '<' | '>' | '!=' | '<=' | '>=';
|
||||
// SQL Expressions
|
||||
left: string;
|
||||
right: string;
|
||||
};
|
||||
|
||||
export type Filter =
|
||||
| {
|
||||
type: 'lucene' | 'sql';
|
||||
condition: SearchCondition;
|
||||
}
|
||||
| SqlAstFilter;
|
||||
|
||||
// Used to actually query the data in a given chart
|
||||
export type ChartConfig = {
|
||||
displayType?: DisplayType;
|
||||
numberFormat?: NumberFormat;
|
||||
timestampValueExpression: string;
|
||||
implicitColumnExpression?: string; // Where lucene will search if given bare terms
|
||||
granularity?: SQLInterval | 'auto';
|
||||
markdown?: string; // Markdown Content
|
||||
filtersLogicalOperator?: 'AND' | 'OR'; // Default AND
|
||||
filters?: Filter[]; // Additional filters to where clause
|
||||
connection: string; // Connection ID
|
||||
fillNulls?: number | false; // undefined = 0, false = no fill
|
||||
selectGroupBy?: boolean; // Add groupBy elements to select statement (default behavior: true)
|
||||
// TODO: Color support
|
||||
} & SelectSQLStatement;
|
||||
|
||||
// Saved configuration, has a variable source ID that we pull at query time
|
||||
export type SavedChartConfig = {
|
||||
name: string;
|
||||
source: string;
|
||||
} & Omit<ChartConfig, 'timestampValueExpression' | 'from' | 'connection'>;
|
||||
|
||||
type DateRange = {
|
||||
dateRange: [Date, Date];
|
||||
dateRangeStartInclusive?: boolean; // default true
|
||||
};
|
||||
|
||||
export type ChartConfigWithDateRange = ChartConfig & DateRange;
|
||||
// For non-time-based searches (ex. grab 1 row)
|
||||
export type ChartConfigWithOptDateRange = Omit<
|
||||
ChartConfig,
|
||||
'timestampValueExpression'
|
||||
> & {
|
||||
timestampValueExpression?: string;
|
||||
} & Partial<DateRange>;
|
||||
|
||||
export const FIXED_TIME_BUCKET_EXPR_ALIAS = '__hdx_time_bucket';
|
||||
|
||||
export function isUsingGroupBy(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): chartConfig is Omit<ChartConfigWithDateRange, 'groupBy'> & {
|
||||
groupBy: NonNullable<ChartConfigWithDateRange['groupBy']>;
|
||||
} {
|
||||
return chartConfig.groupBy != null && chartConfig.groupBy.length > 0;
|
||||
}
|
||||
|
||||
function isUsingGranularity(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): chartConfig is Omit<
|
||||
Omit<Omit<ChartConfigWithDateRange, 'granularity'>, 'dateRange'>,
|
||||
'timestampValueExpression'
|
||||
> & {
|
||||
granularity: NonNullable<ChartConfigWithDateRange['granularity']>;
|
||||
dateRange: NonNullable<ChartConfigWithDateRange['dateRange']>;
|
||||
timestampValueExpression: NonNullable<
|
||||
ChartConfigWithDateRange['timestampValueExpression']
|
||||
>;
|
||||
} {
|
||||
return (
|
||||
chartConfig.timestampValueExpression != null &&
|
||||
chartConfig.granularity != null
|
||||
);
|
||||
}
|
||||
|
||||
const INVERSE_OPERATOR_MAP = {
|
||||
'=': '!=',
|
||||
'>': '<=',
|
||||
'<': '>=',
|
||||
|
||||
'!=': '=',
|
||||
'<=': '>',
|
||||
'>=': '<',
|
||||
} as const;
|
||||
export function inverseSqlAstFilter(filter: SqlAstFilter): SqlAstFilter {
|
||||
return {
|
||||
...filter,
|
||||
operator:
|
||||
INVERSE_OPERATOR_MAP[
|
||||
filter.operator as keyof typeof INVERSE_OPERATOR_MAP
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function isNonEmptyWhereExpr(where?: string): where is string {
|
||||
return where != null && where.trim() != '';
|
||||
}
|
||||
|
||||
const fastifySQL = ({
|
||||
materializedFields,
|
||||
rawSQL,
|
||||
}: {
|
||||
materializedFields: Map<string, string>;
|
||||
rawSQL: string;
|
||||
}) => {
|
||||
// Parse the SQL AST
|
||||
try {
|
||||
const parser = new SQLParser.Parser();
|
||||
const ast = parser.astify(rawSQL, {
|
||||
database: 'Postgresql',
|
||||
}) as SQLParser.Select;
|
||||
|
||||
// traveral ast and replace the left node with the materialized field
|
||||
// FIXME: type node (AST type is incomplete): https://github.com/taozhi8833998/node-sql-parser/blob/42ea0b1800c5d425acb8c5ca708a1cee731aada8/types.d.ts#L474
|
||||
const traverse = (
|
||||
node:
|
||||
| SQLParser.Expr
|
||||
| SQLParser.ExpressionValue
|
||||
| SQLParser.ExprList
|
||||
| SQLParser.Function
|
||||
| null,
|
||||
) => {
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
let colExpr;
|
||||
switch (node.type) {
|
||||
case 'column_ref': {
|
||||
// FIXME: handle 'Value' type?
|
||||
const _n = node as ColumnRef;
|
||||
// @ts-ignore
|
||||
if (typeof _n.column !== 'string') {
|
||||
// @ts-ignore
|
||||
colExpr = `${_n.column?.expr.value}['${_n.array_index?.[0]?.index.value}']`;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'binary_expr': {
|
||||
const _n = node as SQLParser.Expr;
|
||||
if (Array.isArray(_n.left)) {
|
||||
for (const left of _n.left) {
|
||||
traverse(left);
|
||||
}
|
||||
} else {
|
||||
traverse(_n.left);
|
||||
}
|
||||
|
||||
if (Array.isArray(_n.right)) {
|
||||
for (const right of _n.right) {
|
||||
traverse(right);
|
||||
}
|
||||
} else {
|
||||
traverse(_n.right);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'function': {
|
||||
const _n = node as SQLParser.Function;
|
||||
|
||||
if (_n.args?.type === 'expr_list') {
|
||||
if (Array.isArray(_n.args?.value)) {
|
||||
for (const arg of _n.args.value) {
|
||||
traverse(arg);
|
||||
}
|
||||
|
||||
// ex: JSONExtractString(Body, 'message')
|
||||
if (
|
||||
_n.args?.value?.[0]?.type === 'column_ref' &&
|
||||
_n.args?.value?.[1]?.type === 'single_quote_string'
|
||||
) {
|
||||
colExpr = `${_n.name?.name?.[0]?.value}(${(_n.args?.value?.[0] as any)?.column.expr.value}, '${_n.args?.value?.[1]?.value}')`;
|
||||
}
|
||||
}
|
||||
// when _n.args?.value is Expr
|
||||
else if (isPlainObject(_n.args?.value)) {
|
||||
traverse(_n.args.value);
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
default:
|
||||
// ignore other types
|
||||
break;
|
||||
}
|
||||
|
||||
if (colExpr) {
|
||||
const materializedField = materializedFields.get(colExpr);
|
||||
if (materializedField) {
|
||||
const _n = node as ColumnRef;
|
||||
// reset the node ref
|
||||
for (const key in _n) {
|
||||
// eslint-disable-next-line no-prototype-builtins
|
||||
if (_n.hasOwnProperty(key)) {
|
||||
// @ts-ignore
|
||||
delete _n[key];
|
||||
}
|
||||
}
|
||||
_n.type = 'column_ref';
|
||||
// @ts-ignore
|
||||
_n.table = null;
|
||||
// @ts-ignore
|
||||
_n.column = { expr: { type: 'default', value: materializedField } };
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (Array.isArray(ast.columns)) {
|
||||
for (const col of ast.columns) {
|
||||
traverse(col.expr);
|
||||
}
|
||||
}
|
||||
|
||||
traverse(ast.where);
|
||||
|
||||
return parser.sqlify(ast);
|
||||
} catch (e) {
|
||||
console.error('[renderWhereExpression]feat: Failed to parse SQL AST', e);
|
||||
return rawSQL;
|
||||
}
|
||||
};
|
||||
|
||||
const aggFnExpr = ({
|
||||
fn,
|
||||
expr,
|
||||
quantileLevel,
|
||||
where,
|
||||
}: {
|
||||
fn: AggregateFunction | AggregateFunctionWithCombinators;
|
||||
expr?: string;
|
||||
quantileLevel?: number;
|
||||
where?: string;
|
||||
}) => {
|
||||
const isCount = fn.startsWith('count');
|
||||
const isWhereUsed = isNonEmptyWhereExpr(where);
|
||||
// Cast to float64 because the expr might not be a number
|
||||
const unsafeExpr = { UNSAFE_RAW_SQL: `toFloat64OrNull(toString(${expr}))` };
|
||||
const whereWithExtraNullCheck = `${where} AND ${unsafeExpr.UNSAFE_RAW_SQL} IS NOT NULL`;
|
||||
|
||||
if (fn.endsWith('Merge')) {
|
||||
return chSql`${fn}(${{
|
||||
UNSAFE_RAW_SQL: expr ?? '',
|
||||
}})`;
|
||||
}
|
||||
// TODO: merge this chunk with the rest of logics
|
||||
else if (fn.endsWith('State')) {
|
||||
if (expr == null || isCount) {
|
||||
return isWhereUsed
|
||||
? chSql`${fn}(${{ UNSAFE_RAW_SQL: where }})`
|
||||
: chSql`${fn}()`;
|
||||
}
|
||||
return chSql`${fn}(${unsafeExpr}${
|
||||
isWhereUsed ? chSql`, ${{ UNSAFE_RAW_SQL: whereWithExtraNullCheck }}` : ''
|
||||
})`;
|
||||
}
|
||||
|
||||
if (fn === 'count') {
|
||||
if (isWhereUsed) {
|
||||
return chSql`${fn}If(${{ UNSAFE_RAW_SQL: where }})`;
|
||||
}
|
||||
return {
|
||||
sql: `${fn}()`,
|
||||
params: {},
|
||||
};
|
||||
}
|
||||
|
||||
if (expr != null) {
|
||||
if (fn === 'count_distinct') {
|
||||
return chSql`count${isWhereUsed ? 'If' : ''}(DISTINCT ${{
|
||||
UNSAFE_RAW_SQL: expr,
|
||||
}}${isWhereUsed ? chSql`, ${{ UNSAFE_RAW_SQL: where }}` : ''})`;
|
||||
}
|
||||
|
||||
if (quantileLevel != null) {
|
||||
return chSql`quantile${isWhereUsed ? 'If' : ''}(${{
|
||||
// Using Float64 param leads to an added coersion, but we don't need to
|
||||
// escape number values anyways
|
||||
UNSAFE_RAW_SQL: Number.isFinite(quantileLevel)
|
||||
? `${quantileLevel}`
|
||||
: '0',
|
||||
}})(${unsafeExpr}${
|
||||
isWhereUsed
|
||||
? chSql`, ${{ UNSAFE_RAW_SQL: whereWithExtraNullCheck }}`
|
||||
: ''
|
||||
})`;
|
||||
}
|
||||
|
||||
// TODO: Verify fn is a safe/valid function
|
||||
return chSql`${{ UNSAFE_RAW_SQL: fn }}${isWhereUsed ? 'If' : ''}(
|
||||
${unsafeExpr}${isWhereUsed ? chSql`, ${{ UNSAFE_RAW_SQL: whereWithExtraNullCheck }}` : ''}
|
||||
)`;
|
||||
} else {
|
||||
throw new Error(
|
||||
'Column is required for all non-count aggregation functions',
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
async function renderSelectList(
|
||||
selectList: SelectList,
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
metadata: Metadata,
|
||||
) {
|
||||
if (typeof selectList === 'string') {
|
||||
return chSql`${{ UNSAFE_RAW_SQL: selectList }}`;
|
||||
}
|
||||
|
||||
const materializedFields = await metadata.getMaterializedColumnsLookupTable({
|
||||
connectionId: chartConfig.connection,
|
||||
databaseName: chartConfig.from.databaseName,
|
||||
tableName: chartConfig.from.tableName,
|
||||
});
|
||||
|
||||
return Promise.all(
|
||||
selectList.map(async select => {
|
||||
const whereClause = await renderWhereExpression({
|
||||
condition: select.aggCondition ?? '',
|
||||
from: chartConfig.from,
|
||||
language: select.aggConditionLanguage ?? 'lucene',
|
||||
implicitColumnExpression: chartConfig.implicitColumnExpression,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
});
|
||||
|
||||
let expr: ChSql;
|
||||
if (select.aggFn == null) {
|
||||
expr = chSql`${{ UNSAFE_RAW_SQL: select.valueExpression }}`;
|
||||
} else if (select.aggFn === 'quantile') {
|
||||
expr = aggFnExpr({
|
||||
fn: select.aggFn,
|
||||
expr: select.valueExpression,
|
||||
// @ts-ignore (TS doesn't know that we've already checked for quantile)
|
||||
quantileLevel: select.level,
|
||||
where: whereClause.sql,
|
||||
});
|
||||
} else {
|
||||
expr = aggFnExpr({
|
||||
fn: select.aggFn,
|
||||
expr: select.valueExpression,
|
||||
where: whereClause.sql,
|
||||
});
|
||||
}
|
||||
|
||||
const rawSQL = `SELECT ${expr.sql} FROM \`t\``;
|
||||
// strip 'SELECT * FROM `t` WHERE ' from the sql
|
||||
expr.sql = fastifySQL({ materializedFields, rawSQL })
|
||||
.replace(/^SELECT\s+/i, '') // Remove 'SELECT ' from the start
|
||||
.replace(/\s+FROM `t`$/i, ''); // Remove ' FROM t' from the end
|
||||
|
||||
return chSql`${expr}${
|
||||
select.alias != null
|
||||
? chSql` AS \`${{ UNSAFE_RAW_SQL: select.alias }}\``
|
||||
: []
|
||||
}`;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function renderSortSpecificationList(
|
||||
sortSpecificationList: SortSpecificationList,
|
||||
) {
|
||||
if (typeof sortSpecificationList === 'string') {
|
||||
return chSql`${{ UNSAFE_RAW_SQL: sortSpecificationList }}`;
|
||||
}
|
||||
|
||||
return sortSpecificationList.map(sortSpecification => {
|
||||
return chSql`${{ UNSAFE_RAW_SQL: sortSpecification.valueExpression }} ${
|
||||
sortSpecification.ordering === 'DESC' ? 'DESC' : 'ASC'
|
||||
}`;
|
||||
});
|
||||
}
|
||||
|
||||
function timeBucketExpr({
|
||||
interval,
|
||||
timestampValueExpression,
|
||||
dateRange,
|
||||
alias = FIXED_TIME_BUCKET_EXPR_ALIAS,
|
||||
}: {
|
||||
interval: SQLInterval | 'auto';
|
||||
timestampValueExpression: string;
|
||||
dateRange?: [Date, Date];
|
||||
alias?: string;
|
||||
}) {
|
||||
const unsafeTimestampValueExpression = {
|
||||
UNSAFE_RAW_SQL: getFirstTimestampValueExpression(timestampValueExpression),
|
||||
};
|
||||
const unsafeInterval = {
|
||||
UNSAFE_RAW_SQL:
|
||||
interval === 'auto' && Array.isArray(dateRange)
|
||||
? convertDateRangeToGranularityString(dateRange, 60)
|
||||
: interval,
|
||||
};
|
||||
|
||||
return chSql`toStartOfInterval(toDateTime(${unsafeTimestampValueExpression}), INTERVAL ${unsafeInterval}) AS \`${{
|
||||
UNSAFE_RAW_SQL: alias,
|
||||
}}\``;
|
||||
}
|
||||
|
||||
async function timeFilterExpr({
|
||||
timestampValueExpression,
|
||||
dateRange,
|
||||
dateRangeStartInclusive,
|
||||
databaseName,
|
||||
tableName,
|
||||
metadata,
|
||||
connectionId,
|
||||
}: {
|
||||
timestampValueExpression: string;
|
||||
dateRange: [Date, Date];
|
||||
dateRangeStartInclusive: boolean;
|
||||
metadata: Metadata;
|
||||
connectionId: string;
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
}) {
|
||||
const valueExpressions = timestampValueExpression.split(',');
|
||||
const startTime = dateRange[0].getTime();
|
||||
const endTime = dateRange[1].getTime();
|
||||
|
||||
const whereExprs = await Promise.all(
|
||||
valueExpressions.map(async expr => {
|
||||
const col = expr.trim();
|
||||
const columnMeta = await metadata.getColumn({
|
||||
databaseName,
|
||||
tableName,
|
||||
column: col,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
const unsafeTimestampValueExpression = {
|
||||
UNSAFE_RAW_SQL: col,
|
||||
};
|
||||
|
||||
if (columnMeta == null) {
|
||||
console.warn(
|
||||
`Column ${col} not found in ${databaseName}.${tableName} while inferring type for time filter`,
|
||||
);
|
||||
}
|
||||
|
||||
// If it's a date type
|
||||
if (columnMeta?.type === 'Date') {
|
||||
return chSql`(${unsafeTimestampValueExpression} ${
|
||||
dateRangeStartInclusive ? '>=' : '>'
|
||||
} toDate(fromUnixTimestamp64Milli(${{
|
||||
Int64: startTime,
|
||||
}})) AND ${unsafeTimestampValueExpression} <= toDate(fromUnixTimestamp64Milli(${{
|
||||
Int64: endTime,
|
||||
}})))`;
|
||||
} else {
|
||||
return chSql`(${unsafeTimestampValueExpression} ${
|
||||
dateRangeStartInclusive ? '>=' : '>'
|
||||
} fromUnixTimestamp64Milli(${{
|
||||
Int64: startTime,
|
||||
}}) AND ${unsafeTimestampValueExpression} <= fromUnixTimestamp64Milli(${{
|
||||
Int64: endTime,
|
||||
}}))`;
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
return concatChSql('AND', ...whereExprs);
|
||||
}
|
||||
|
||||
async function renderSelect(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
metadata: Metadata,
|
||||
): Promise<ChSql> {
|
||||
/**
|
||||
* SELECT
|
||||
* if granularity: toStartOfInterval,
|
||||
* if groupBy: groupBy,
|
||||
* select
|
||||
*/
|
||||
const isIncludingTimeBucket = isUsingGranularity(chartConfig);
|
||||
const isIncludingGroupBy = isUsingGroupBy(chartConfig);
|
||||
|
||||
// TODO: clean up these await mess
|
||||
return concatChSql(
|
||||
',',
|
||||
await renderSelectList(chartConfig.select, chartConfig, metadata),
|
||||
isIncludingGroupBy && chartConfig.selectGroupBy !== false
|
||||
? await renderSelectList(chartConfig.groupBy, chartConfig, metadata)
|
||||
: [],
|
||||
isIncludingTimeBucket
|
||||
? timeBucketExpr({
|
||||
interval: chartConfig.granularity,
|
||||
timestampValueExpression: chartConfig.timestampValueExpression,
|
||||
dateRange: chartConfig.dateRange,
|
||||
})
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
function renderFrom({
|
||||
from,
|
||||
}: {
|
||||
from: ChartConfigWithDateRange['from'];
|
||||
}): ChSql {
|
||||
return chSql`${{ Identifier: from.databaseName }}.${{
|
||||
Identifier: from.tableName,
|
||||
}}`;
|
||||
}
|
||||
|
||||
async function renderWhereExpression({
|
||||
condition,
|
||||
language,
|
||||
metadata,
|
||||
from,
|
||||
implicitColumnExpression,
|
||||
connectionId,
|
||||
}: {
|
||||
condition: SearchCondition;
|
||||
language: SearchConditionLanguage;
|
||||
metadata: Metadata;
|
||||
from: ChartConfigWithDateRange['from'];
|
||||
implicitColumnExpression?: string;
|
||||
connectionId: string;
|
||||
}): Promise<ChSql> {
|
||||
let _condition = condition;
|
||||
if (language === 'lucene') {
|
||||
const serializer = new CustomSchemaSQLSerializerV2({
|
||||
metadata,
|
||||
databaseName: from.databaseName,
|
||||
tableName: from.tableName,
|
||||
implicitColumnExpression,
|
||||
connectionId: connectionId,
|
||||
});
|
||||
const builder = new SearchQueryBuilder(condition, serializer);
|
||||
_condition = await builder.build();
|
||||
}
|
||||
|
||||
const materializedFields = await metadata.getMaterializedColumnsLookupTable({
|
||||
connectionId,
|
||||
databaseName: from.databaseName,
|
||||
tableName: from.tableName,
|
||||
});
|
||||
|
||||
const _sqlPrefix = 'SELECT * FROM `t` WHERE ';
|
||||
const rawSQL = `${_sqlPrefix}${_condition}`;
|
||||
// strip 'SELECT * FROM `t` WHERE ' from the sql
|
||||
_condition = fastifySQL({ materializedFields, rawSQL }).replace(
|
||||
_sqlPrefix,
|
||||
'',
|
||||
);
|
||||
return chSql`${{ UNSAFE_RAW_SQL: _condition }}`;
|
||||
}
|
||||
|
||||
async function renderWhere(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
metadata: Metadata,
|
||||
): Promise<ChSql> {
|
||||
let whereSearchCondition: ChSql | [] = [];
|
||||
if (isNonEmptyWhereExpr(chartConfig.where)) {
|
||||
whereSearchCondition = wrapChSqlIfNotEmpty(
|
||||
await renderWhereExpression({
|
||||
condition: chartConfig.where,
|
||||
from: chartConfig.from,
|
||||
language: chartConfig.whereLanguage ?? 'sql',
|
||||
implicitColumnExpression: chartConfig.implicitColumnExpression,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
}),
|
||||
'(',
|
||||
')',
|
||||
);
|
||||
}
|
||||
|
||||
let selectSearchConditions: ChSql[] = [];
|
||||
if (
|
||||
typeof chartConfig.select != 'string' &&
|
||||
// Only if every select has an aggCondition, add to where clause
|
||||
// otherwise we'll scan all rows anyways
|
||||
chartConfig.select.every(select => isNonEmptyWhereExpr(select.aggCondition))
|
||||
) {
|
||||
selectSearchConditions = (
|
||||
await Promise.all(
|
||||
chartConfig.select.map(async select => {
|
||||
if (isNonEmptyWhereExpr(select.aggCondition)) {
|
||||
return await renderWhereExpression({
|
||||
condition: select.aggCondition,
|
||||
from: chartConfig.from,
|
||||
language: select.aggConditionLanguage ?? 'sql',
|
||||
implicitColumnExpression: chartConfig.implicitColumnExpression,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
)
|
||||
).filter(v => v !== null) as ChSql[];
|
||||
}
|
||||
|
||||
const filterConditions = await Promise.all(
|
||||
(chartConfig.filters ?? []).map(async filter => {
|
||||
if (filter.type === 'sql_ast') {
|
||||
return wrapChSqlIfNotEmpty(
|
||||
chSql`${{ UNSAFE_RAW_SQL: filter.left }} ${filter.operator} ${{ UNSAFE_RAW_SQL: filter.right }}`,
|
||||
'(',
|
||||
')',
|
||||
);
|
||||
} else if (filter.type === 'lucene' || filter.type === 'sql') {
|
||||
return wrapChSqlIfNotEmpty(
|
||||
await renderWhereExpression({
|
||||
condition: filter.condition,
|
||||
from: chartConfig.from,
|
||||
language: filter.type,
|
||||
implicitColumnExpression: chartConfig.implicitColumnExpression,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
}),
|
||||
'(',
|
||||
')',
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(`Unknown filter type: ${filter.type}`);
|
||||
}),
|
||||
);
|
||||
|
||||
return concatChSql(
|
||||
' AND ',
|
||||
chartConfig.dateRange != null &&
|
||||
chartConfig.timestampValueExpression != null
|
||||
? await timeFilterExpr({
|
||||
timestampValueExpression: chartConfig.timestampValueExpression,
|
||||
dateRange: chartConfig.dateRange,
|
||||
dateRangeStartInclusive: chartConfig.dateRangeStartInclusive ?? true,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
databaseName: chartConfig.from.databaseName,
|
||||
tableName: chartConfig.from.tableName,
|
||||
})
|
||||
: [],
|
||||
whereSearchCondition,
|
||||
// Add aggConditions to where clause to utilize index
|
||||
wrapChSqlIfNotEmpty(concatChSql(' OR ', selectSearchConditions), '(', ')'),
|
||||
wrapChSqlIfNotEmpty(
|
||||
concatChSql(
|
||||
chartConfig.filtersLogicalOperator === 'OR' ? ' OR ' : ' AND ',
|
||||
...filterConditions,
|
||||
),
|
||||
'(',
|
||||
')',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async function renderGroupBy(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
metadata: Metadata,
|
||||
): Promise<ChSql | undefined> {
|
||||
return concatChSql(
|
||||
',',
|
||||
isUsingGroupBy(chartConfig)
|
||||
? await renderSelectList(chartConfig.groupBy, chartConfig, metadata)
|
||||
: [],
|
||||
isUsingGranularity(chartConfig)
|
||||
? timeBucketExpr({
|
||||
interval: chartConfig.granularity,
|
||||
timestampValueExpression: chartConfig.timestampValueExpression,
|
||||
dateRange: chartConfig.dateRange,
|
||||
})
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
function renderOrderBy(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): ChSql | undefined {
|
||||
const isIncludingTimeBucket = isUsingGranularity(chartConfig);
|
||||
|
||||
if (chartConfig.orderBy == null && !isIncludingTimeBucket) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return concatChSql(
|
||||
',',
|
||||
isIncludingTimeBucket
|
||||
? timeBucketExpr({
|
||||
interval: chartConfig.granularity,
|
||||
timestampValueExpression: chartConfig.timestampValueExpression,
|
||||
dateRange: chartConfig.dateRange,
|
||||
})
|
||||
: [],
|
||||
chartConfig.orderBy != null
|
||||
? renderSortSpecificationList(chartConfig.orderBy)
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
function renderLimit(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): ChSql | undefined {
|
||||
if (chartConfig.limit == null || chartConfig.limit.limit == null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const offset =
|
||||
chartConfig.limit.offset != null
|
||||
? chSql` OFFSET ${{ Int32: chartConfig.limit.offset }}`
|
||||
: [];
|
||||
|
||||
return chSql`${{ Int32: chartConfig.limit.limit }}${offset}`;
|
||||
}
|
||||
|
||||
export async function renderChartConfig(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): Promise<ChSql> {
|
||||
const select = await renderSelect(chartConfig, metadata);
|
||||
const from = renderFrom(chartConfig);
|
||||
const where = await renderWhere(chartConfig, metadata);
|
||||
const groupBy = await renderGroupBy(chartConfig, metadata);
|
||||
const orderBy = renderOrderBy(chartConfig);
|
||||
const limit = renderLimit(chartConfig);
|
||||
|
||||
return chSql`SELECT ${select} FROM ${from} ${where?.sql ? chSql`WHERE ${where}` : ''} ${
|
||||
groupBy?.sql ? chSql`GROUP BY ${groupBy}` : ''
|
||||
} ${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''} ${
|
||||
limit?.sql ? chSql`LIMIT ${limit}` : ''
|
||||
}`;
|
||||
}
|
||||
|
||||
// EditForm -> translateToQueriedChartConfig -> QueriedChartConfig
|
||||
// renderFn(QueriedChartConfig) -> sql
|
||||
// query(sql) -> data
|
||||
// formatter(data) -> displayspecificDs
|
||||
// displaySettings(QueriedChartConfig) -> displaySepcificDs
|
||||
// chartComponent(displayspecificDs) -> React.Node
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
// Derived from SQL grammar spec
|
||||
// See: https://ronsavage.github.io/SQL/sql-2003-2.bnf.html#query%20specification
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
import {
|
||||
AggregateFunctionSchema,
|
||||
AggregateFunctionWithCombinatorsSchema,
|
||||
DerivedColumnSchema,
|
||||
SearchConditionLanguageSchema,
|
||||
SearchConditionSchema,
|
||||
SelectListSchema,
|
||||
SortSpecificationListSchema,
|
||||
SQLIntervalSchema,
|
||||
} from '@/common/commonTypes';
|
||||
|
||||
export type SQLInterval = z.infer<typeof SQLIntervalSchema>;
|
||||
|
||||
export type SearchCondition = z.infer<typeof SearchConditionSchema>;
|
||||
export type SearchConditionLanguage = z.infer<
|
||||
typeof SearchConditionLanguageSchema
|
||||
>;
|
||||
export type AggregateFunction = z.infer<typeof AggregateFunctionSchema>;
|
||||
export type AggregateFunctionWithCombinators = z.infer<
|
||||
typeof AggregateFunctionWithCombinatorsSchema
|
||||
>;
|
||||
|
||||
export type DerivedColumn = z.infer<typeof DerivedColumnSchema>;
|
||||
|
||||
export type SelectList = z.infer<typeof SelectListSchema>;
|
||||
|
||||
export type SortSpecificationList = z.infer<typeof SortSpecificationListSchema>;
|
||||
|
||||
type Limit = { limit?: number; offset?: number };
|
||||
|
||||
export type SelectSQLStatement = {
|
||||
select: SelectList;
|
||||
from: { databaseName: string; tableName: string };
|
||||
where: SearchCondition;
|
||||
whereLanguage?: SearchConditionLanguage;
|
||||
groupBy?: SelectList;
|
||||
having?: SearchCondition;
|
||||
havingLanguage?: SearchConditionLanguage;
|
||||
orderBy?: SortSpecificationList;
|
||||
limit?: Limit;
|
||||
};
|
||||
|
|
@ -1,181 +0,0 @@
|
|||
// Port from ChartUtils + source.ts
|
||||
import { add } from 'date-fns';
|
||||
|
||||
import type { SQLInterval } from '@/common/sqlTypes';
|
||||
|
||||
export const isBrowser: boolean =
|
||||
typeof window !== 'undefined' && typeof window.document !== 'undefined';
|
||||
|
||||
export const isNode: boolean =
|
||||
typeof process !== 'undefined' &&
|
||||
process.versions != null &&
|
||||
process.versions.node != null;
|
||||
|
||||
// If a user specifies a timestampValueExpression with multiple columns,
|
||||
// this will return the first one. We'll want to refine this over time
|
||||
export function getFirstTimestampValueExpression(valueExpression: string) {
|
||||
return valueExpression.split(',')[0].trim();
|
||||
}
|
||||
|
||||
export enum Granularity {
|
||||
FifteenSecond = '15 second',
|
||||
ThirtySecond = '30 second',
|
||||
OneMinute = '1 minute',
|
||||
FiveMinute = '5 minute',
|
||||
TenMinute = '10 minute',
|
||||
FifteenMinute = '15 minute',
|
||||
ThirtyMinute = '30 minute',
|
||||
OneHour = '1 hour',
|
||||
TwoHour = '2 hour',
|
||||
SixHour = '6 hour',
|
||||
TwelveHour = '12 hour',
|
||||
OneDay = '1 day',
|
||||
TwoDay = '2 day',
|
||||
SevenDay = '7 day',
|
||||
ThirtyDay = '30 day',
|
||||
}
|
||||
|
||||
export function hashCode(str: string) {
|
||||
let hash = 0,
|
||||
i,
|
||||
chr;
|
||||
if (str.length === 0) return hash;
|
||||
for (i = 0; i < str.length; i++) {
|
||||
chr = str.charCodeAt(i);
|
||||
hash = (hash << 5) - hash + chr;
|
||||
hash |= 0; // Convert to 32bit integer
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
export function convertDateRangeToGranularityString(
|
||||
dateRange: [Date, Date],
|
||||
maxNumBuckets: number,
|
||||
): Granularity {
|
||||
const start = dateRange[0].getTime();
|
||||
const end = dateRange[1].getTime();
|
||||
const diffSeconds = Math.floor((end - start) / 1000);
|
||||
const granularitySizeSeconds = Math.ceil(diffSeconds / maxNumBuckets);
|
||||
|
||||
if (granularitySizeSeconds <= 15) {
|
||||
return Granularity.FifteenSecond;
|
||||
} else if (granularitySizeSeconds <= 30) {
|
||||
return Granularity.ThirtySecond;
|
||||
} else if (granularitySizeSeconds <= 60) {
|
||||
return Granularity.OneMinute;
|
||||
} else if (granularitySizeSeconds <= 5 * 60) {
|
||||
return Granularity.FiveMinute;
|
||||
} else if (granularitySizeSeconds <= 10 * 60) {
|
||||
return Granularity.TenMinute;
|
||||
} else if (granularitySizeSeconds <= 15 * 60) {
|
||||
return Granularity.FifteenMinute;
|
||||
} else if (granularitySizeSeconds <= 30 * 60) {
|
||||
return Granularity.ThirtyMinute;
|
||||
} else if (granularitySizeSeconds <= 3600) {
|
||||
return Granularity.OneHour;
|
||||
} else if (granularitySizeSeconds <= 2 * 3600) {
|
||||
return Granularity.TwoHour;
|
||||
} else if (granularitySizeSeconds <= 6 * 3600) {
|
||||
return Granularity.SixHour;
|
||||
} else if (granularitySizeSeconds <= 12 * 3600) {
|
||||
return Granularity.TwelveHour;
|
||||
} else if (granularitySizeSeconds <= 24 * 3600) {
|
||||
return Granularity.OneDay;
|
||||
} else if (granularitySizeSeconds <= 2 * 24 * 3600) {
|
||||
return Granularity.TwoDay;
|
||||
} else if (granularitySizeSeconds <= 7 * 24 * 3600) {
|
||||
return Granularity.SevenDay;
|
||||
} else if (granularitySizeSeconds <= 30 * 24 * 3600) {
|
||||
return Granularity.ThirtyDay;
|
||||
}
|
||||
|
||||
return Granularity.ThirtyDay;
|
||||
}
|
||||
|
||||
export function convertGranularityToSeconds(granularity: SQLInterval): number {
|
||||
const [num, unit] = granularity.split(' ');
|
||||
const numInt = Number.parseInt(num);
|
||||
switch (unit) {
|
||||
case 'second':
|
||||
return numInt;
|
||||
case 'minute':
|
||||
return numInt * 60;
|
||||
case 'hour':
|
||||
return numInt * 60 * 60;
|
||||
case 'day':
|
||||
return numInt * 60 * 60 * 24;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
// Note: roundToNearestMinutes is broken in date-fns currently
|
||||
// additionally it doesn't support seconds or > 30min
|
||||
// so we need to write our own :(
|
||||
// see: https://github.com/date-fns/date-fns/pull/3267/files
|
||||
export function toStartOfInterval(date: Date, granularity: SQLInterval): Date {
|
||||
const [num, unit] = granularity.split(' ');
|
||||
const numInt = Number.parseInt(num);
|
||||
const roundFn = Math.floor;
|
||||
|
||||
switch (unit) {
|
||||
case 'second':
|
||||
return new Date(
|
||||
Date.UTC(
|
||||
date.getUTCFullYear(),
|
||||
date.getUTCMonth(),
|
||||
date.getUTCDate(),
|
||||
date.getUTCHours(),
|
||||
date.getUTCMinutes(),
|
||||
roundFn(date.getUTCSeconds() / numInt) * numInt,
|
||||
),
|
||||
);
|
||||
case 'minute':
|
||||
return new Date(
|
||||
Date.UTC(
|
||||
date.getUTCFullYear(),
|
||||
date.getUTCMonth(),
|
||||
date.getUTCDate(),
|
||||
date.getUTCHours(),
|
||||
roundFn(date.getUTCMinutes() / numInt) * numInt,
|
||||
),
|
||||
);
|
||||
case 'hour':
|
||||
return new Date(
|
||||
Date.UTC(
|
||||
date.getUTCFullYear(),
|
||||
date.getUTCMonth(),
|
||||
date.getUTCDate(),
|
||||
roundFn(date.getUTCHours() / numInt) * numInt,
|
||||
),
|
||||
);
|
||||
case 'day': {
|
||||
// Clickhouse uses the # of days since unix epoch to round dates
|
||||
// see: https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/DateLUTImpl.h#L1059
|
||||
const daysSinceEpoch = date.getTime() / 1000 / 60 / 60 / 24;
|
||||
const daysSinceEpochRounded = roundFn(daysSinceEpoch / numInt) * numInt;
|
||||
|
||||
return new Date(daysSinceEpochRounded * 1000 * 60 * 60 * 24);
|
||||
}
|
||||
default:
|
||||
return date;
|
||||
}
|
||||
}
|
||||
|
||||
export function timeBucketByGranularity(
|
||||
start: Date,
|
||||
end: Date,
|
||||
granularity: SQLInterval,
|
||||
): Date[] {
|
||||
const buckets: Date[] = [];
|
||||
|
||||
let current = toStartOfInterval(start, granularity);
|
||||
const granularitySeconds = convertGranularityToSeconds(granularity);
|
||||
while (current < end) {
|
||||
buckets.push(current);
|
||||
current = add(current, {
|
||||
seconds: granularitySeconds,
|
||||
});
|
||||
}
|
||||
|
||||
return buckets;
|
||||
}
|
||||
|
|
@ -1,7 +1,10 @@
|
|||
import {
|
||||
DashboardWithoutIdSchema,
|
||||
Tile,
|
||||
} from '@hyperdx/common-utils/dist/types';
|
||||
import { differenceBy, uniq } from 'lodash';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { DashboardWithoutIdSchema, Tile } from '@/common/commonTypes';
|
||||
import type { ObjectId } from '@/models';
|
||||
import Alert from '@/models/alert';
|
||||
import Dashboard from '@/models/dashboard';
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { SavedSearchSchema } from '@hyperdx/common-utils/dist/types';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { SavedSearchSchema } from '@/common/commonTypes';
|
||||
import { SavedSearch } from '@/models/savedSearch';
|
||||
|
||||
type SavedSearchWithoutId = Omit<z.infer<typeof SavedSearchSchema>, 'id'>;
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
import { createClient } from '@clickhouse/client';
|
||||
import * as commonClickhouse from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import {
|
||||
DisplayType,
|
||||
SavedChartConfig,
|
||||
Tile,
|
||||
} from '@hyperdx/common-utils/dist/types';
|
||||
import mongoose from 'mongoose';
|
||||
import request from 'supertest';
|
||||
|
||||
import * as clickhouse from '@/clickhouse';
|
||||
import * as commonClickhouse from '@/common/clickhouse';
|
||||
import { SavedChartConfig, Tile } from '@/common/commonTypes';
|
||||
import { DisplayType } from '@/common/DisplayType';
|
||||
import * as config from '@/config';
|
||||
import { AlertInput } from '@/controllers/alerts';
|
||||
import { getTeam } from '@/controllers/team';
|
||||
|
|
@ -216,24 +219,6 @@ export const clearRedis = async () => {
|
|||
// ------------------------------------------------
|
||||
// ------------------ Clickhouse ------------------
|
||||
// ------------------------------------------------
|
||||
export const mockClientQuery = () => {
|
||||
if (!config.IS_CI) {
|
||||
throw new Error('ONLY execute this in CI env 😈 !!!');
|
||||
}
|
||||
jest
|
||||
.spyOn(commonClickhouse.client, 'query')
|
||||
.mockImplementation(async (args: any) => {
|
||||
const nodeClient = createClient({
|
||||
host: config.CLICKHOUSE_HOST,
|
||||
username: config.CLICKHOUSE_USER,
|
||||
password: config.CLICKHOUSE_PASSWORD,
|
||||
});
|
||||
return nodeClient.query(args) as unknown as ReturnType<
|
||||
typeof commonClickhouse.client.query
|
||||
>;
|
||||
});
|
||||
};
|
||||
|
||||
export const clearClickhouseTables = async () => {
|
||||
if (!config.IS_CI) {
|
||||
throw new Error('ONLY execute this in CI env 😈 !!!');
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
import { DashboardSchema } from '@hyperdx/common-utils/dist/types';
|
||||
import mongoose, { Schema } from 'mongoose';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { DashboardSchema } from '@/common/commonTypes';
|
||||
|
||||
import type { ObjectId } from '.';
|
||||
|
||||
export interface IDashboard extends z.infer<typeof DashboardSchema> {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
import { SavedSearchSchema } from '@hyperdx/common-utils/dist/types';
|
||||
import mongoose, { Schema } from 'mongoose';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { SavedSearchSchema } from '@/common/commonTypes';
|
||||
|
||||
type ObjectId = mongoose.Types.ObjectId;
|
||||
|
||||
export interface ISavedSearch
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import { TSource } from '@hyperdx/common-utils/dist/types';
|
||||
import mongoose, { Schema } from 'mongoose';
|
||||
|
||||
import { TSource } from '@/common/commonTypes';
|
||||
|
||||
type ObjectId = mongoose.Types.ObjectId;
|
||||
|
||||
export interface ISource extends Omit<TSource, 'connection'> {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { ConnectionSchema } from '@hyperdx/common-utils/dist/types';
|
||||
import express from 'express';
|
||||
import { validateRequest } from 'zod-express-middleware';
|
||||
|
||||
import { ConnectionSchema } from '@/common/commonTypes';
|
||||
import {
|
||||
createConnection,
|
||||
deleteConnection,
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import {
|
||||
DashboardSchema,
|
||||
DashboardWithoutIdSchema,
|
||||
} from '@hyperdx/common-utils/dist/types';
|
||||
import express from 'express';
|
||||
import { differenceBy, groupBy, uniq } from 'lodash';
|
||||
import _ from 'lodash';
|
||||
import { z } from 'zod';
|
||||
import { validateRequest } from 'zod-express-middleware';
|
||||
|
||||
import {
|
||||
DashboardSchema,
|
||||
DashboardWithoutIdSchema,
|
||||
} from '@/common/commonTypes';
|
||||
import {
|
||||
createDashboard,
|
||||
deleteDashboardAndAlerts,
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import { SavedSearchSchema } from '@hyperdx/common-utils/dist/types';
|
||||
import express from 'express';
|
||||
import _ from 'lodash';
|
||||
import { z } from 'zod';
|
||||
import { validateRequest } from 'zod-express-middleware';
|
||||
|
||||
import { SavedSearchSchema } from '@/common/commonTypes';
|
||||
import {
|
||||
createSavedSearch,
|
||||
deleteSavedSearch,
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { SourceSchema } from '@hyperdx/common-utils/dist/types';
|
||||
import express from 'express';
|
||||
import { z } from 'zod';
|
||||
import { validateRequest } from 'zod-express-middleware';
|
||||
|
||||
import { SourceSchema } from '@/common/commonTypes';
|
||||
import {
|
||||
createSource,
|
||||
deleteSource,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import { TileSchema } from '@hyperdx/common-utils/dist/types';
|
||||
import express from 'express';
|
||||
import { uniq } from 'lodash';
|
||||
import { ObjectId } from 'mongodb';
|
||||
import { z } from 'zod';
|
||||
import { validateRequest } from 'zod-express-middleware';
|
||||
|
||||
import { TileSchema } from '@/common/commonTypes';
|
||||
import {
|
||||
deleteDashboardAndAlerts,
|
||||
updateDashboard,
|
||||
|
|
|
|||
|
|
@ -1,15 +1,9 @@
|
|||
import ms from 'ms';
|
||||
|
||||
import * as clickhouse from '@/common/clickhouse';
|
||||
import * as config from '@/config';
|
||||
import { createAlert } from '@/controllers/alerts';
|
||||
import { createTeam } from '@/controllers/team';
|
||||
import {
|
||||
bulkInsertLogs,
|
||||
getServer,
|
||||
makeTile,
|
||||
mockClientQuery,
|
||||
} from '@/fixtures';
|
||||
import { bulkInsertLogs, getServer, makeTile } from '@/fixtures';
|
||||
import Alert, { AlertSource, AlertThresholdType } from '@/models/alert';
|
||||
import AlertHistory from '@/models/alertHistory';
|
||||
import Connection from '@/models/connection';
|
||||
|
|
@ -625,7 +619,6 @@ describe('checkAlerts', () => {
|
|||
});
|
||||
|
||||
it('SAVED_SEARCH alert - slack webhook', async () => {
|
||||
mockClientQuery();
|
||||
jest
|
||||
.spyOn(slack, 'postMessageToWebhook')
|
||||
.mockResolvedValueOnce(null as any);
|
||||
|
|
@ -758,7 +751,6 @@ describe('checkAlerts', () => {
|
|||
});
|
||||
|
||||
it('TILE alert - slack webhook', async () => {
|
||||
mockClientQuery();
|
||||
jest
|
||||
.spyOn(slack, 'postMessageToWebhook')
|
||||
.mockResolvedValueOnce(null as any);
|
||||
|
|
@ -920,8 +912,6 @@ describe('checkAlerts', () => {
|
|||
});
|
||||
|
||||
it('TILE alert - generic webhook', async () => {
|
||||
mockClientQuery();
|
||||
|
||||
jest.spyOn(checkAlert, 'handleSendGenericWebhook');
|
||||
|
||||
const fetchMock = jest.fn().mockResolvedValue({});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
// --------------------------------------------------------
|
||||
// -------------- EXECUTE EVERY MINUTE --------------------
|
||||
// --------------------------------------------------------
|
||||
import * as clickhouse from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { getMetadata } from '@hyperdx/common-utils/dist/metadata';
|
||||
import {
|
||||
ChartConfigWithOptDateRange,
|
||||
renderChartConfig,
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { DisplayType } from '@hyperdx/common-utils/dist/types';
|
||||
import * as fns from 'date-fns';
|
||||
import * as fnsTz from 'date-fns-tz';
|
||||
import Handlebars, { HelperOptions } from 'handlebars';
|
||||
|
|
@ -12,16 +19,9 @@ import PromisedHandlebars from 'promised-handlebars';
|
|||
import { serializeError } from 'serialize-error';
|
||||
import { URLSearchParams } from 'url';
|
||||
|
||||
import * as clickhouse from '@/common/clickhouse';
|
||||
import { convertCHDataTypeToJSType } from '@/common/clickhouse';
|
||||
import { DisplayType } from '@/common/DisplayType';
|
||||
import {
|
||||
ChartConfigWithOptDateRange,
|
||||
FIXED_TIME_BUCKET_EXPR_ALIAS,
|
||||
} from '@/common/renderChartConfig';
|
||||
import { renderChartConfig } from '@/common/renderChartConfig';
|
||||
import * as config from '@/config';
|
||||
import { AlertInput } from '@/controllers/alerts';
|
||||
import { getConnectionById } from '@/controllers/connection';
|
||||
import Alert, {
|
||||
AlertSource,
|
||||
AlertState,
|
||||
|
|
@ -737,9 +737,28 @@ export const processAlert = async (now: Date, alert: EnhancedAlert) => {
|
|||
return;
|
||||
}
|
||||
|
||||
const query = await renderChartConfig(chartConfig);
|
||||
const checksData = await clickhouse
|
||||
.sendQuery<'JSON'>({
|
||||
const connection = await getConnectionById(
|
||||
alert.team._id.toString(),
|
||||
connectionId,
|
||||
true,
|
||||
);
|
||||
|
||||
if (connection == null) {
|
||||
logger.error({
|
||||
message: 'Connection not found',
|
||||
alertId: alert.id,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const clickhouseClient = new clickhouse.ClickhouseClient({
|
||||
host: connection.host,
|
||||
username: connection.username,
|
||||
password: connection.password,
|
||||
});
|
||||
const metadata = getMetadata(clickhouseClient);
|
||||
const query = await renderChartConfig(chartConfig, metadata);
|
||||
const checksData = await clickhouseClient
|
||||
.query<'JSON'>({
|
||||
query: query.sql,
|
||||
query_params: query.params,
|
||||
format: 'JSON',
|
||||
|
|
@ -768,7 +787,7 @@ export const processAlert = async (now: Date, alert: EnhancedAlert) => {
|
|||
const meta =
|
||||
checksData.meta?.map(m => ({
|
||||
...m,
|
||||
jsType: convertCHDataTypeToJSType(m.type),
|
||||
jsType: clickhouse.convertCHDataTypeToJSType(m.type),
|
||||
})) ?? [];
|
||||
|
||||
const timestampColumnName = meta.find(
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@
|
|||
"@codemirror/lang-sql": "^6.7.0",
|
||||
"@hookform/resolvers": "^3.9.0",
|
||||
"@hyperdx/browser": "^0.21.1",
|
||||
"@hyperdx/lucene": "^3.1.1",
|
||||
"@hyperdx/common-utils": "^0.0.9",
|
||||
"@hyperdx/node-opentelemetry": "^0.8.1",
|
||||
"@lezer/highlight": "^1.2.0",
|
||||
"@mantine/core": "7.9.2",
|
||||
|
|
@ -64,7 +64,6 @@
|
|||
"next-seo": "^4.28.1",
|
||||
"nextra": "2.0.1",
|
||||
"nextra-theme-docs": "^2.0.2",
|
||||
"node-sql-parser": "^5.3.2",
|
||||
"numbro": "^2.4.0",
|
||||
"nuqs": "^1.17.0",
|
||||
"object-hash": "^3.0.0",
|
||||
|
|
@ -115,7 +114,6 @@
|
|||
"@testing-library/react": "^14.2.1",
|
||||
"@testing-library/user-event": "^14.5.2",
|
||||
"@types/crypto-js": "^4",
|
||||
"@types/hyperdx__lucene": "npm:@types/lucene@*",
|
||||
"@types/intercom-web": "^2.8.18",
|
||||
"@types/jest": "^28.1.6",
|
||||
"@types/lodash": "^4.14.186",
|
||||
|
|
|
|||
|
|
@ -989,6 +989,7 @@ export default function AppNav({ fixed = false }: { fixed?: boolean }) {
|
|||
|
||||
<AppNavLinkGroups
|
||||
name="dashboards"
|
||||
/* @ts-ignore */
|
||||
groups={groupedFilteredDashboardsList}
|
||||
renderLink={renderDashboardLink}
|
||||
forceExpandGroups={!!dashboardsListQ}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
useQueryState,
|
||||
} from 'nuqs';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { DisplayType } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
Button,
|
||||
Code,
|
||||
|
|
@ -19,12 +20,12 @@ import {
|
|||
} from '@mantine/core';
|
||||
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
|
||||
|
||||
import { getClickhouseClient } from '@/clickhouse';
|
||||
|
||||
import { ConnectionSelectControlled } from './components/ConnectionSelect';
|
||||
import DBTableChart from './components/DBTableChart';
|
||||
import { DBTimeChart } from './components/DBTimeChart';
|
||||
import { SQLEditorControlled } from './components/SQLEditor';
|
||||
import { sendQuery } from './clickhouse';
|
||||
import { DisplayType } from './DisplayType';
|
||||
|
||||
function useBenchmarkQueryIds({
|
||||
queries,
|
||||
|
|
@ -36,6 +37,7 @@ function useBenchmarkQueryIds({
|
|||
iterations?: number;
|
||||
}) {
|
||||
const enabled = queries.length > 0 && connections.length > 0;
|
||||
const clickhouseClient = getClickhouseClient();
|
||||
|
||||
return useQuery({
|
||||
enabled,
|
||||
|
|
@ -50,17 +52,19 @@ function useBenchmarkQueryIds({
|
|||
// ask CH to use ours instead
|
||||
const queryId = crypto.randomUUID();
|
||||
|
||||
await sendQuery({
|
||||
query: shuffledQueries[j],
|
||||
connectionId: connections[j],
|
||||
format: 'NULL',
|
||||
clickhouse_settings: {
|
||||
min_bytes_to_use_direct_io: '1',
|
||||
use_query_cache: 0,
|
||||
wait_end_of_query: 1,
|
||||
},
|
||||
queryId,
|
||||
}).then(res => res.text());
|
||||
await clickhouseClient
|
||||
.query({
|
||||
query: shuffledQueries[j],
|
||||
connectionId: connections[j],
|
||||
format: 'NULL',
|
||||
clickhouse_settings: {
|
||||
min_bytes_to_use_direct_io: '1',
|
||||
use_query_cache: 0,
|
||||
wait_end_of_query: 1,
|
||||
},
|
||||
queryId,
|
||||
})
|
||||
.then(res => res.text());
|
||||
|
||||
queryIds[j] ??= [];
|
||||
queryIds[j].push(queryId);
|
||||
|
|
@ -90,16 +94,19 @@ function useEstimates(
|
|||
},
|
||||
options: Omit<UseQueryOptions<any>, 'queryKey' | 'queryFn'> = {},
|
||||
) {
|
||||
const clickhouseClient = getClickhouseClient();
|
||||
return useQuery({
|
||||
queryKey: ['estimate', queries, connections],
|
||||
queryFn: async () => {
|
||||
return Promise.all(
|
||||
queries.map((query, i) =>
|
||||
sendQuery({
|
||||
query: `EXPLAIN ESTIMATE ${query}`,
|
||||
format: 'JSON',
|
||||
connectionId: connections[i],
|
||||
}).then(res => res.json()),
|
||||
clickhouseClient
|
||||
.query({
|
||||
query: `EXPLAIN ESTIMATE ${query}`,
|
||||
format: 'JSON',
|
||||
connectionId: connections[i],
|
||||
})
|
||||
.then(res => res.json()),
|
||||
),
|
||||
);
|
||||
},
|
||||
|
|
@ -117,16 +124,18 @@ function useIndexes(
|
|||
},
|
||||
options: Omit<UseQueryOptions<any>, 'queryKey' | 'queryFn'> = {},
|
||||
) {
|
||||
const clickhouseClient = getClickhouseClient();
|
||||
return useQuery({
|
||||
queryKey: ['indexes', queries, connections],
|
||||
queryFn: async () => {
|
||||
return Promise.all(
|
||||
queries.map((query, i) =>
|
||||
sendQuery({
|
||||
query: `EXPLAIN indexes=1, json=1, description = 0 ${query}`,
|
||||
format: 'TSVRaw',
|
||||
connectionId: connections[i],
|
||||
})
|
||||
clickhouseClient
|
||||
.query({
|
||||
query: `EXPLAIN indexes=1, json=1, description = 0 ${query}`,
|
||||
format: 'TSVRaw',
|
||||
connectionId: connections[i],
|
||||
})
|
||||
.then(res => res.text())
|
||||
.then(res => JSON.parse(res)),
|
||||
),
|
||||
|
|
|
|||
|
|
@ -2,6 +2,11 @@ import { useMemo, useRef } from 'react';
|
|||
import { add } from 'date-fns';
|
||||
import Select from 'react-select';
|
||||
import AsyncSelect from 'react-select/async';
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
SavedChartConfig,
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { DisplayType, SQLInterval } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
Divider,
|
||||
Group,
|
||||
|
|
@ -9,16 +14,11 @@ import {
|
|||
Select as MSelect,
|
||||
} from '@mantine/core';
|
||||
|
||||
import type { SQLInterval } from '@/sqlTypes';
|
||||
|
||||
import { NumberFormatInput } from './components/NumberFormat';
|
||||
import api from './api';
|
||||
import Checkbox from './Checkbox';
|
||||
import { DisplayType } from './DisplayType';
|
||||
import FieldMultiSelect from './FieldMultiSelect';
|
||||
import MetricTagFilterInput from './MetricTagFilterInput';
|
||||
import { SavedChartConfig } from './renderChartConfig';
|
||||
import { ChartConfigWithDateRange } from './renderChartConfig';
|
||||
import SearchInput from './SearchInput';
|
||||
import { AggFn, ChartSeries, MetricsDataType, SourceTable } from './types';
|
||||
import { NumberFormat } from './types';
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
useQueryStates,
|
||||
} from 'nuqs';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { DisplayType } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
Box,
|
||||
BoxComponentProps,
|
||||
|
|
@ -22,7 +23,6 @@ import {
|
|||
import { ConnectionSelectControlled } from '@/components/ConnectionSelect';
|
||||
import { DBTimeChart } from '@/components/DBTimeChart';
|
||||
import { TimePicker } from '@/components/TimePicker';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import { withAppNav } from '@/layout';
|
||||
|
||||
import { ChartBox } from './components/ChartBox';
|
||||
|
|
|
|||
|
|
@ -1,15 +1,14 @@
|
|||
import { useCallback } from 'react';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { parseAsJson, parseAsStringEnum, useQueryState } from 'nuqs';
|
||||
import { SavedChartConfig } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { Box } from '@mantine/core';
|
||||
|
||||
import { DEFAULT_CHART_CONFIG, Granularity } from '@/ChartUtils';
|
||||
import EditTimeChartForm from '@/components/DBEditTimeChartForm';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import { withAppNav } from '@/layout';
|
||||
import { parseTimeQuery, useNewTimeQuery } from '@/timeQuery';
|
||||
|
||||
import { SavedChartConfig } from './renderChartConfig';
|
||||
import { useSources } from './source';
|
||||
|
||||
// Autocomplete can focus on column/map keys
|
||||
|
|
|
|||
|
|
@ -17,6 +17,16 @@ import { ErrorBoundary } from 'react-error-boundary';
|
|||
import RGL, { WidthProvider } from 'react-grid-layout';
|
||||
import { Controller, useForm } from 'react-hook-form';
|
||||
import { useHotkeys } from 'react-hotkeys-hook';
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
Filter,
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import {
|
||||
DisplayType,
|
||||
SearchCondition,
|
||||
SearchConditionLanguage,
|
||||
SQLInterval,
|
||||
} from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
ActionIcon,
|
||||
Badge,
|
||||
|
|
@ -54,8 +64,6 @@ import {
|
|||
useCreateDashboard,
|
||||
useDeleteDashboard,
|
||||
} from '@/dashboard';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import { ChartConfigWithDateRange, Filter } from '@/renderChartConfig';
|
||||
|
||||
import DBRowSidePanel from './components/DBRowSidePanel';
|
||||
import OnboardingModal from './components/OnboardingModal';
|
||||
|
|
@ -74,11 +82,6 @@ import {
|
|||
useSource,
|
||||
useSources,
|
||||
} from './source';
|
||||
import {
|
||||
SearchCondition,
|
||||
SearchConditionLanguage,
|
||||
SQLInterval,
|
||||
} from './sqlTypes';
|
||||
import { Tags } from './Tags';
|
||||
import { parseTimeQuery, useNewTimeQuery } from './timeQuery';
|
||||
import { useConfirm } from './useConfirm';
|
||||
|
|
|
|||
|
|
@ -19,6 +19,12 @@ import {
|
|||
import { useForm } from 'react-hook-form';
|
||||
import { z } from 'zod';
|
||||
import { zodResolver } from '@hookform/resolvers/zod';
|
||||
import {
|
||||
ChartConfig,
|
||||
ChartConfigWithDateRange,
|
||||
Filter,
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { DisplayType } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
ActionIcon,
|
||||
Box,
|
||||
|
|
@ -53,15 +59,9 @@ import { SQLInlineEditorControlled } from '@/components/SQLInlineEditor';
|
|||
import { TimePicker } from '@/components/TimePicker';
|
||||
import WhereLanguageControlled from '@/components/WhereLanguageControlled';
|
||||
import { IS_LOCAL_MODE } from '@/config';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { useExplainQuery } from '@/hooks/useExplainQuery';
|
||||
import { withAppNav } from '@/layout';
|
||||
import {
|
||||
ChartConfig,
|
||||
ChartConfigWithDateRange,
|
||||
Filter,
|
||||
} from '@/renderChartConfig';
|
||||
import {
|
||||
useCreateSavedSearch,
|
||||
useDeleteSavedSearch,
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
export enum DisplayType {
|
||||
Line = 'line',
|
||||
StackedBar = 'stacked_bar',
|
||||
Table = 'table',
|
||||
Number = 'number',
|
||||
Search = 'search',
|
||||
Heatmap = 'heatmap',
|
||||
Markdown = 'markdown',
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@ import {
|
|||
XAxis,
|
||||
YAxis,
|
||||
} from 'recharts';
|
||||
import { DisplayType } from '@hyperdx/common-utils/dist/types';
|
||||
import { Popover } from '@mantine/core';
|
||||
import { notifications } from '@mantine/notifications';
|
||||
|
||||
|
|
@ -38,7 +39,6 @@ import {
|
|||
seriesColumns,
|
||||
seriesToUrlSearchQueryParam,
|
||||
} from '@/ChartUtils';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import type { ChartSeries, NumberFormat } from '@/types';
|
||||
import { COLORS, formatNumber, getColorProps, truncateMiddle } from '@/utils';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { genEnglishExplanation } from '@hyperdx/common-utils/dist/queryParser';
|
||||
|
||||
import api from '@/api';
|
||||
import AutocompleteInput from '@/AutocompleteInput';
|
||||
import { genEnglishExplanation } from '@/queryParser';
|
||||
|
||||
export default function SearchInput({
|
||||
inputRef,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { useController, UseControllerProps } from 'react-hook-form';
|
||||
import { useHotkeys } from 'react-hotkeys-hook';
|
||||
import { genEnglishExplanation } from '@hyperdx/common-utils/dist/queryParser';
|
||||
|
||||
import AutocompleteInput from '@/AutocompleteInput';
|
||||
import { useAllFields } from '@/hooks/useMetadata';
|
||||
import { genEnglishExplanation } from '@/queryParser';
|
||||
|
||||
export default function SearchInputV2({
|
||||
database,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ import {
|
|||
useQueryStates,
|
||||
} from 'nuqs';
|
||||
import { UseControllerProps, useForm } from 'react-hook-form';
|
||||
import type { Filter } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { DisplayType, TSource } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
Box,
|
||||
Button,
|
||||
|
|
@ -22,7 +24,6 @@ import {
|
|||
INTEGER_NUMBER_FORMAT,
|
||||
MS_NUMBER_FORMAT,
|
||||
} from '@/ChartUtils';
|
||||
import { TSource } from '@/commonTypes';
|
||||
import { ChartBox } from '@/components/ChartBox';
|
||||
import DBHistogramChart from '@/components/DBHistogramChart';
|
||||
import DBListBarChart from '@/components/DBListBarChart';
|
||||
|
|
@ -36,10 +37,8 @@ import { SourceSelectControlled } from '@/components/SourceSelect';
|
|||
import { SQLInlineEditorControlled } from '@/components/SQLInlineEditor';
|
||||
import { TimePicker } from '@/components/TimePicker';
|
||||
import WhereLanguageControlled from '@/components/WhereLanguageControlled';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { withAppNav } from '@/layout';
|
||||
import { Filter } from '@/renderChartConfig';
|
||||
import SearchInputV2 from '@/SearchInputV2';
|
||||
import { getExpressions } from '@/serviceDashboard';
|
||||
import { useSource, useSources } from '@/source';
|
||||
|
|
|
|||
|
|
@ -1,619 +1,50 @@
|
|||
import type { ResponseJSON } from '@clickhouse/client';
|
||||
import { isSuccessfulResponse } from '@clickhouse/client-common';
|
||||
import { DataFormat, ResultSet } from '@clickhouse/client-web';
|
||||
import {
|
||||
chSql,
|
||||
ClickhouseClient,
|
||||
ColumnMeta,
|
||||
} from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
|
||||
|
||||
import { loginHook } from '@/api';
|
||||
import { timeBucketByGranularity } from '@/ChartUtils';
|
||||
import { CLICKHOUSE_HOST, IS_LOCAL_MODE } from '@/config';
|
||||
import { IS_LOCAL_MODE } from '@/config';
|
||||
import { getLocalConnections } from '@/connection';
|
||||
import { SQLInterval } from '@/sqlTypes';
|
||||
import { hashCode } from '@/utils';
|
||||
|
||||
export enum JSDataType {
|
||||
Array = 'array',
|
||||
Date = 'date',
|
||||
Map = 'map',
|
||||
Number = 'number',
|
||||
String = 'string',
|
||||
Bool = 'bool',
|
||||
}
|
||||
const PROXY_CLICKHOUSE_HOST = '/api/clickhouse-proxy';
|
||||
|
||||
export const convertCHDataTypeToJSType = (
|
||||
dataType: string,
|
||||
): JSDataType | null => {
|
||||
if (dataType.startsWith('Date')) {
|
||||
return JSDataType.Date;
|
||||
} else if (dataType.startsWith('Map')) {
|
||||
return JSDataType.Map;
|
||||
} else if (dataType.startsWith('Array')) {
|
||||
return JSDataType.Array;
|
||||
} else if (
|
||||
dataType.startsWith('Int') ||
|
||||
dataType.startsWith('UInt') ||
|
||||
dataType.startsWith('Float') ||
|
||||
// Nullable types are possible (charts)
|
||||
dataType.startsWith('Nullable(Int') ||
|
||||
dataType.startsWith('Nullable(UInt') ||
|
||||
dataType.startsWith('Nullable(Float')
|
||||
) {
|
||||
return JSDataType.Number;
|
||||
} else if (
|
||||
dataType.startsWith('String') ||
|
||||
dataType.startsWith('FixedString') ||
|
||||
dataType.startsWith('Enum') ||
|
||||
dataType.startsWith('UUID') ||
|
||||
dataType.startsWith('IPv4') ||
|
||||
dataType.startsWith('IPv6')
|
||||
) {
|
||||
return JSDataType.String;
|
||||
} else if (dataType === 'Bool') {
|
||||
return JSDataType.Bool;
|
||||
} else if (dataType.startsWith('LowCardinality')) {
|
||||
return convertCHDataTypeToJSType(dataType.slice(15, -1));
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
export const convertCHTypeToPrimitiveJSType = (dataType: string) => {
|
||||
const jsType = convertCHDataTypeToJSType(dataType);
|
||||
|
||||
if (jsType === JSDataType.Map || jsType === JSDataType.Array) {
|
||||
throw new Error('Map type is not a primitive type');
|
||||
} else if (jsType === JSDataType.Date) {
|
||||
return JSDataType.Number;
|
||||
}
|
||||
|
||||
return jsType;
|
||||
};
|
||||
|
||||
const hash = (input: string | number) => Math.abs(hashCode(`${input}`));
|
||||
const paramHash = (str: string | number) => {
|
||||
return `HYPERDX_PARAM_${hash(str)}`;
|
||||
};
|
||||
|
||||
export type ChSql = {
|
||||
sql: string;
|
||||
params: Record<string, any>;
|
||||
};
|
||||
|
||||
type ParamTypes =
|
||||
| ChSql
|
||||
| ChSql[]
|
||||
| { Identifier: string }
|
||||
| { String: string }
|
||||
| { Float32: number }
|
||||
| { Float64: number }
|
||||
| { Int32: number }
|
||||
| { Int64: number }
|
||||
| { UNSAFE_RAW_SQL: string }
|
||||
| string; // TODO: Deprecate raw string interpolation
|
||||
|
||||
export const chSql = (
|
||||
strings: TemplateStringsArray,
|
||||
...values: ParamTypes[]
|
||||
): ChSql => {
|
||||
const sql = strings
|
||||
.map((str, i) => {
|
||||
const value = values[i];
|
||||
// if (typeof value === 'string') {
|
||||
// console.error('Unsafe string detected', value, 'in', strings, values);
|
||||
// }
|
||||
|
||||
return (
|
||||
str +
|
||||
(value == null
|
||||
? ''
|
||||
: typeof value === 'string'
|
||||
? value // If it's just a string sql literal
|
||||
: 'UNSAFE_RAW_SQL' in value
|
||||
? value.UNSAFE_RAW_SQL
|
||||
: Array.isArray(value)
|
||||
? value.map(v => v.sql).join('')
|
||||
: 'sql' in value
|
||||
? value.sql
|
||||
: 'Identifier' in value
|
||||
? `{${paramHash(value.Identifier)}:Identifier}`
|
||||
: 'String' in value
|
||||
? `{${paramHash(value.String)}:String}`
|
||||
: 'Float32' in value
|
||||
? `{${paramHash(value.Float32)}:Float32}`
|
||||
: 'Float64' in value
|
||||
? `{${paramHash(value.Float64)}:Float64}`
|
||||
: 'Int32' in value
|
||||
? `{${paramHash(value.Int32)}:Int32}`
|
||||
: 'Int64' in value
|
||||
? `{${paramHash(value.Int64)}:Int64}`
|
||||
: '')
|
||||
);
|
||||
})
|
||||
.join('');
|
||||
|
||||
return {
|
||||
sql,
|
||||
params: values.reduce((acc, value) => {
|
||||
return {
|
||||
...acc,
|
||||
...(value == null ||
|
||||
typeof value === 'string' ||
|
||||
'UNSAFE_RAW_SQL' in value
|
||||
? {}
|
||||
: Array.isArray(value)
|
||||
? value.reduce((acc, v) => {
|
||||
Object.assign(acc, v.params);
|
||||
return acc;
|
||||
}, {})
|
||||
: 'params' in value
|
||||
? value.params
|
||||
: 'Identifier' in value
|
||||
? { [paramHash(value.Identifier)]: value.Identifier }
|
||||
: 'String' in value
|
||||
? { [paramHash(value.String)]: value.String }
|
||||
: 'Float32' in value
|
||||
? { [paramHash(value.Float32)]: value.Float32 }
|
||||
: 'Float64' in value
|
||||
? { [paramHash(value.Float64)]: value.Float64 }
|
||||
: 'Int32' in value
|
||||
? { [paramHash(value.Int32)]: value.Int32 }
|
||||
: 'Int64' in value
|
||||
? { [paramHash(value.Int64)]: value.Int64 }
|
||||
: {}),
|
||||
};
|
||||
}, {}),
|
||||
};
|
||||
};
|
||||
|
||||
export const concatChSql = (sep: string, ...args: (ChSql | ChSql[])[]) => {
|
||||
return args.reduce(
|
||||
(acc: ChSql, arg) => {
|
||||
if (Array.isArray(arg)) {
|
||||
if (arg.length === 0) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
acc.sql +=
|
||||
(acc.sql.length > 0 ? sep : '') + arg.map(a => a.sql).join(sep);
|
||||
acc.params = arg.reduce((acc, a) => {
|
||||
Object.assign(acc, a.params);
|
||||
return acc;
|
||||
}, acc.params);
|
||||
} else if (arg.sql.length > 0) {
|
||||
acc.sql += `${acc.sql.length > 0 ? sep : ''}${arg.sql}`;
|
||||
Object.assign(acc.params, arg.params);
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{ sql: '', params: {} },
|
||||
);
|
||||
};
|
||||
|
||||
const isChSqlEmpty = (chSql: ChSql | ChSql[]) => {
|
||||
if (Array.isArray(chSql)) {
|
||||
return chSql.every(c => c.sql.length === 0);
|
||||
}
|
||||
return chSql.sql.length === 0;
|
||||
};
|
||||
|
||||
export const wrapChSqlIfNotEmpty = (
|
||||
sql: ChSql | ChSql[],
|
||||
left: string,
|
||||
right: string,
|
||||
): ChSql | [] => {
|
||||
if (isChSqlEmpty(sql)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return chSql`${left}${sql}${right}`;
|
||||
};
|
||||
export class ClickHouseQueryError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public query: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ClickHouseQueryError';
|
||||
}
|
||||
}
|
||||
|
||||
export function extractColumnReference(
|
||||
sql: string,
|
||||
maxIterations = 10,
|
||||
): string | null {
|
||||
let iterations = 0;
|
||||
|
||||
// Loop until we remove all function calls and get just the column, with a maximum limit
|
||||
while (/\w+\([^()]*\)/.test(sql) && iterations < maxIterations) {
|
||||
// Replace the outermost function with its content
|
||||
sql = sql.replace(/\w+\(([^()]*)\)/, '$1');
|
||||
iterations++;
|
||||
}
|
||||
|
||||
// If we reached the max iterations without resolving, return null to indicate an issue
|
||||
return iterations < maxIterations ? sql.trim() : null;
|
||||
}
|
||||
|
||||
const client = {
|
||||
async query<T extends DataFormat>({
|
||||
query,
|
||||
format = 'JSON',
|
||||
query_params = {},
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
host,
|
||||
username,
|
||||
password,
|
||||
includeCredentials,
|
||||
includeCorsHeader,
|
||||
connectionId,
|
||||
queryId,
|
||||
}: {
|
||||
query: string;
|
||||
format?: string;
|
||||
abort_signal?: AbortSignal;
|
||||
query_params?: Record<string, any>;
|
||||
clickhouse_settings?: Record<string, any>;
|
||||
host?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
includeCredentials: boolean;
|
||||
includeCorsHeader: boolean;
|
||||
connectionId?: string;
|
||||
queryId?: string;
|
||||
}): Promise<ResultSet<T>> {
|
||||
const searchParams = new URLSearchParams([
|
||||
...(includeCorsHeader ? [['add_http_cors_header', '1']] : []),
|
||||
...(connectionId ? [['hyperdx_connection_id', connectionId]] : []),
|
||||
['query', query],
|
||||
['default_format', format],
|
||||
['date_time_output_format', 'iso'],
|
||||
['wait_end_of_query', '0'],
|
||||
['cancel_http_readonly_queries_on_client_close', '1'],
|
||||
...(username ? [['user', username]] : []),
|
||||
...(password ? [['password', password]] : []),
|
||||
...(queryId ? [['query_id', queryId]] : []),
|
||||
...Object.entries(query_params).map(([key, value]) => [
|
||||
`param_${key}`,
|
||||
value,
|
||||
]),
|
||||
...Object.entries(clickhouse_settings ?? {}).map(([key, value]) => [
|
||||
key,
|
||||
value,
|
||||
]),
|
||||
]);
|
||||
|
||||
let debugSql = '';
|
||||
try {
|
||||
debugSql = parameterizedQueryToSql({ sql: query, params: query_params });
|
||||
} catch (e) {
|
||||
debugSql = query;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('--------------------------------------------------------');
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Sending Query:', debugSql);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('--------------------------------------------------------');
|
||||
|
||||
const res = await fetch(`${host}/?${searchParams.toString()}`, {
|
||||
...(includeCredentials ? { credentials: 'include' } : {}),
|
||||
signal: abort_signal,
|
||||
method: 'GET',
|
||||
});
|
||||
|
||||
// TODO: Send command to CH to cancel query on abort_signal
|
||||
if (!res.ok) {
|
||||
loginHook({} as any, {}, res as any);
|
||||
|
||||
if (!isSuccessfulResponse(res.status)) {
|
||||
const text = await res.text();
|
||||
throw new ClickHouseQueryError(`${text}`, debugSql);
|
||||
}
|
||||
}
|
||||
|
||||
if (res.body == null) {
|
||||
// TODO: Handle empty responses better?
|
||||
throw new Error('Unexpected empty response from ClickHouse');
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
return new ResultSet(res.body, format, '');
|
||||
},
|
||||
};
|
||||
|
||||
export const testLocalConnection = async ({
|
||||
host,
|
||||
username,
|
||||
password,
|
||||
}: {
|
||||
host: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}): Promise<boolean> => {
|
||||
try {
|
||||
const result = await client.query({
|
||||
query: 'SELECT 1',
|
||||
format: 'TabSeparatedRaw',
|
||||
host: host,
|
||||
username: username,
|
||||
password: password,
|
||||
includeCredentials: false,
|
||||
includeCorsHeader: true,
|
||||
});
|
||||
return result.text().then(text => text.trim() === '1');
|
||||
} catch (e) {
|
||||
console.warn('Failed to test local connection', e);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export const sendQuery = async <T extends DataFormat>({
|
||||
query,
|
||||
format = 'JSON',
|
||||
query_params = {},
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
connectionId,
|
||||
queryId,
|
||||
}: {
|
||||
query: string;
|
||||
format?: string;
|
||||
query_params?: Record<string, any>;
|
||||
abort_signal?: AbortSignal;
|
||||
clickhouse_settings?: Record<string, any>;
|
||||
connectionId: string;
|
||||
queryId?: string;
|
||||
}) => {
|
||||
let host, username, password;
|
||||
export const getClickhouseClient = () => {
|
||||
if (IS_LOCAL_MODE) {
|
||||
const localConnections = getLocalConnections();
|
||||
if (localConnections.length === 0) {
|
||||
throw new Error('No local connection found');
|
||||
}
|
||||
host = localConnections[0].host;
|
||||
username = localConnections[0].username;
|
||||
password = localConnections[0].password;
|
||||
}
|
||||
|
||||
return client.query<T>({
|
||||
query,
|
||||
format,
|
||||
query_params,
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
queryId,
|
||||
connectionId: IS_LOCAL_MODE ? undefined : connectionId,
|
||||
host: IS_LOCAL_MODE ? host : CLICKHOUSE_HOST,
|
||||
username: IS_LOCAL_MODE ? username : undefined,
|
||||
password: IS_LOCAL_MODE ? password : undefined,
|
||||
includeCredentials: !IS_LOCAL_MODE,
|
||||
includeCorsHeader: IS_LOCAL_MODE,
|
||||
});
|
||||
};
|
||||
|
||||
export const tableExpr = ({
|
||||
database,
|
||||
table,
|
||||
}: {
|
||||
database: string;
|
||||
table: string;
|
||||
}) => {
|
||||
return chSql`${{ Identifier: database }}.${{ Identifier: table }}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* SELECT
|
||||
* aggFnIf(fieldToColumn(field), where),
|
||||
* timeBucketing(Granularity, timeConversion(fieldToColumn(field))),
|
||||
* FROM db.table
|
||||
* WHERE where
|
||||
* GROUP BY timeBucketing, fieldToColumn(groupBy)
|
||||
* ORDER BY orderBy
|
||||
*/
|
||||
|
||||
export function parameterizedQueryToSql({
|
||||
sql,
|
||||
params,
|
||||
}: {
|
||||
sql: string;
|
||||
params: Record<string, any>;
|
||||
}) {
|
||||
return Object.entries(params).reduce((acc, [key, value]) => {
|
||||
return acc.replace(new RegExp(`{${key}:\\w+}`, 'g'), value);
|
||||
}, sql);
|
||||
}
|
||||
|
||||
export type ColumnMetaType = { name: string; type: string };
|
||||
export function filterColumnMetaByType(
|
||||
meta: Array<ColumnMetaType>,
|
||||
types: JSDataType[],
|
||||
): Array<ColumnMetaType> | undefined {
|
||||
return meta.filter(column =>
|
||||
types.includes(convertCHDataTypeToJSType(column.type) as JSDataType),
|
||||
);
|
||||
}
|
||||
|
||||
export function inferTimestampColumn(
|
||||
// from: https://github.com/ClickHouse/clickhouse-js/blob/442392c83834f313a964f9e5bd7ff44474631755/packages/client-common/src/clickhouse_types.ts#L8C3-L8C47
|
||||
meta: Array<ColumnMetaType>,
|
||||
) {
|
||||
return filterColumnMetaByType(meta, [JSDataType.Date])?.[0];
|
||||
}
|
||||
|
||||
function inferValueColumns(meta: Array<{ name: string; type: string }>) {
|
||||
return filterColumnMetaByType(meta, [JSDataType.Number]);
|
||||
}
|
||||
|
||||
function inferGroupColumns(meta: Array<{ name: string; type: string }>) {
|
||||
return filterColumnMetaByType(meta, [
|
||||
JSDataType.String,
|
||||
JSDataType.Map,
|
||||
JSDataType.Array,
|
||||
]);
|
||||
}
|
||||
|
||||
// TODO: Move to ChartUtils
|
||||
// Input: { ts, value1, value2, groupBy1, groupBy2 },
|
||||
// Output: { ts, [value1Name, groupBy1, groupBy2]: value1, [...]: value2 }
|
||||
export function formatResponseForTimeChart({
|
||||
res,
|
||||
dateRange,
|
||||
granularity,
|
||||
generateEmptyBuckets = true,
|
||||
}: {
|
||||
dateRange: [Date, Date];
|
||||
granularity?: SQLInterval;
|
||||
res: ResponseJSON<Record<string, any>>;
|
||||
generateEmptyBuckets?: boolean;
|
||||
}) {
|
||||
const meta = res.meta;
|
||||
const data = res.data;
|
||||
|
||||
if (meta == null) {
|
||||
throw new Error('No meta data found in response');
|
||||
}
|
||||
|
||||
const timestampColumn = inferTimestampColumn(meta);
|
||||
const valueColumns = inferValueColumns(meta) ?? [];
|
||||
const groupColumns = inferGroupColumns(meta) ?? [];
|
||||
|
||||
if (timestampColumn == null) {
|
||||
throw new Error(
|
||||
`No timestamp column found with meta: ${JSON.stringify(meta)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Timestamp -> { tsCol, line1, line2, ...}
|
||||
const tsBucketMap: Map<number, Record<string, any>> = new Map();
|
||||
const lineDataMap: {
|
||||
[keyName: string]: {
|
||||
dataKey: string;
|
||||
displayName: string;
|
||||
maxValue: number;
|
||||
minValue: number;
|
||||
color: string | undefined;
|
||||
};
|
||||
} = {};
|
||||
|
||||
for (const row of data) {
|
||||
const date = new Date(row[timestampColumn.name]);
|
||||
const ts = date.getTime() / 1000;
|
||||
|
||||
for (const valueColumn of valueColumns) {
|
||||
const tsBucket = tsBucketMap.get(ts) ?? {};
|
||||
|
||||
const keyName = [
|
||||
valueColumn.name,
|
||||
...groupColumns.map(g => row[g.name]),
|
||||
].join(' · ');
|
||||
|
||||
// UInt64 are returned as strings, we'll convert to number
|
||||
// and accept a bit of floating point error
|
||||
const rawValue = row[valueColumn.name];
|
||||
const value =
|
||||
typeof rawValue === 'number' ? rawValue : Number.parseFloat(rawValue);
|
||||
|
||||
tsBucketMap.set(ts, {
|
||||
...tsBucket,
|
||||
[timestampColumn.name]: ts,
|
||||
[keyName]: value,
|
||||
console.warn('No local connection found');
|
||||
return new ClickhouseClient({
|
||||
host: '',
|
||||
});
|
||||
|
||||
// TODO: Set name and color correctly
|
||||
lineDataMap[keyName] = {
|
||||
dataKey: keyName,
|
||||
displayName: keyName,
|
||||
color: undefined,
|
||||
maxValue: Math.max(
|
||||
lineDataMap[keyName]?.maxValue ?? Number.NEGATIVE_INFINITY,
|
||||
value,
|
||||
),
|
||||
minValue: Math.min(
|
||||
lineDataMap[keyName]?.minValue ?? Number.POSITIVE_INFINITY,
|
||||
value,
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Custom sort and truncate top N lines
|
||||
const sortedLineDataMap = Object.values(lineDataMap).sort((a, b) => {
|
||||
return a.maxValue - b.maxValue;
|
||||
});
|
||||
|
||||
if (generateEmptyBuckets && granularity != null) {
|
||||
// Zero fill TODO: Make this an option
|
||||
const generatedTsBuckets = timeBucketByGranularity(
|
||||
dateRange[0],
|
||||
dateRange[1],
|
||||
granularity,
|
||||
);
|
||||
|
||||
generatedTsBuckets.forEach(date => {
|
||||
const ts = date.getTime() / 1000;
|
||||
const tsBucket = tsBucketMap.get(ts);
|
||||
|
||||
if (tsBucket == null) {
|
||||
const tsBucket: Record<string, any> = {
|
||||
[timestampColumn.name]: ts,
|
||||
};
|
||||
|
||||
for (const line of sortedLineDataMap) {
|
||||
tsBucket[line.dataKey] = 0;
|
||||
}
|
||||
|
||||
tsBucketMap.set(ts, tsBucket);
|
||||
} else {
|
||||
for (const line of sortedLineDataMap) {
|
||||
if (tsBucket[line.dataKey] == null) {
|
||||
tsBucket[line.dataKey] = 0;
|
||||
}
|
||||
}
|
||||
tsBucketMap.set(ts, tsBucket);
|
||||
}
|
||||
return new ClickhouseClient({
|
||||
host: localConnections[0].host,
|
||||
username: localConnections[0].username,
|
||||
password: localConnections[0].password,
|
||||
});
|
||||
}
|
||||
|
||||
// Sort results again by timestamp
|
||||
const graphResults: {
|
||||
[key: string]: number | undefined;
|
||||
}[] = Array.from(tsBucketMap.values()).sort(
|
||||
(a, b) => a[timestampColumn.name] - b[timestampColumn.name],
|
||||
);
|
||||
|
||||
// TODO: Return line color and names
|
||||
return {
|
||||
// dateRange: [minDate, maxDate],
|
||||
graphResults,
|
||||
timestampColumn,
|
||||
groupKeys: sortedLineDataMap.map(l => l.dataKey),
|
||||
lineNames: sortedLineDataMap.map(l => l.displayName),
|
||||
lineColors: sortedLineDataMap.map(l => l.color),
|
||||
};
|
||||
}
|
||||
|
||||
export type ColumnMeta = {
|
||||
codec_expression: string;
|
||||
comment: string;
|
||||
default_expression: string;
|
||||
default_type: string;
|
||||
name: string;
|
||||
ttl_expression: string;
|
||||
type: string;
|
||||
return new ClickhouseClient({
|
||||
host: PROXY_CLICKHOUSE_HOST,
|
||||
});
|
||||
};
|
||||
|
||||
export function useDatabasesDirect(
|
||||
{ connectionId }: { connectionId: string },
|
||||
options?: Omit<UseQueryOptions<any, Error>, 'queryKey'>,
|
||||
) {
|
||||
const clickhouseClient = getClickhouseClient();
|
||||
return useQuery<ResponseJSON<ColumnMeta>, Error>({
|
||||
queryKey: [`direct_datasources/databases`, connectionId],
|
||||
queryFn: async () => {
|
||||
const json = await sendQuery({
|
||||
query: 'SHOW DATABASES',
|
||||
connectionId,
|
||||
}).then(res => res.json());
|
||||
const json = await clickhouseClient
|
||||
.query({
|
||||
query: 'SHOW DATABASES',
|
||||
connectionId,
|
||||
})
|
||||
.then(res => res.json());
|
||||
|
||||
return json;
|
||||
},
|
||||
|
|
@ -626,15 +57,18 @@ export function useTablesDirect(
|
|||
{ database, connectionId }: { database: string; connectionId: string },
|
||||
options?: Omit<UseQueryOptions<any, Error>, 'queryKey'>,
|
||||
) {
|
||||
const clickhouseClient = getClickhouseClient();
|
||||
return useQuery<ResponseJSON<ColumnMeta>, Error>({
|
||||
queryKey: [`direct_datasources/databases/${database}/tables`],
|
||||
queryFn: async () => {
|
||||
const paramSql = chSql`SHOW TABLES FROM ${{ Identifier: database }}`;
|
||||
const json = await sendQuery({
|
||||
query: paramSql.sql,
|
||||
query_params: paramSql.params,
|
||||
connectionId,
|
||||
}).then(res => res.json());
|
||||
const json = await clickhouseClient
|
||||
.query({
|
||||
query: paramSql.sql,
|
||||
query_params: paramSql.params,
|
||||
connectionId,
|
||||
})
|
||||
.then(res => res.json());
|
||||
|
||||
return json;
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,98 +0,0 @@
|
|||
import { z } from 'zod';
|
||||
|
||||
// --------------------------
|
||||
// SAVED SEARCH
|
||||
// --------------------------
|
||||
|
||||
export const SavedSearchSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
select: z.string(),
|
||||
where: z.string(),
|
||||
whereLanguage: z.string().optional(),
|
||||
source: z.string(),
|
||||
tags: z.array(z.string()),
|
||||
orderBy: z.string().optional(),
|
||||
});
|
||||
|
||||
export type SavedSearch = z.infer<typeof SavedSearchSchema>;
|
||||
|
||||
// --------------------------
|
||||
// DASHBOARDS
|
||||
// --------------------------
|
||||
|
||||
// TODO: Define this
|
||||
export const SavedChartConfigSchema = z.any();
|
||||
|
||||
export const TileSchema = z.object({
|
||||
id: z.string(),
|
||||
x: z.number(),
|
||||
y: z.number(),
|
||||
w: z.number(),
|
||||
h: z.number(),
|
||||
config: SavedChartConfigSchema,
|
||||
});
|
||||
|
||||
export const DashboardSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
tiles: z.array(TileSchema),
|
||||
tags: z.array(z.string()),
|
||||
});
|
||||
|
||||
export const DashboardWithoutIdSchema = DashboardSchema.omit({ id: true });
|
||||
|
||||
export const ConnectionSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
host: z.string(),
|
||||
username: z.string(),
|
||||
password: z.string().optional(),
|
||||
});
|
||||
|
||||
// --------------------------
|
||||
// TABLE SOURCES
|
||||
// --------------------------
|
||||
export const SourceSchema = z.object({
|
||||
from: z.object({
|
||||
databaseName: z.string(),
|
||||
tableName: z.string(),
|
||||
}),
|
||||
timestampValueExpression: z.string(),
|
||||
connection: z.string(),
|
||||
|
||||
// Common
|
||||
kind: z.enum(['log', 'trace']),
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
displayedTimestampValueExpression: z.string().optional(),
|
||||
implicitColumnExpression: z.string().optional(),
|
||||
serviceNameExpression: z.string().optional(),
|
||||
bodyExpression: z.string().optional(),
|
||||
tableFilterExpression: z.string().optional(), // Future use for v1 compatibility
|
||||
eventAttributesExpression: z.string().optional(),
|
||||
resourceAttributesExpression: z.string().optional(),
|
||||
defaultTableSelectExpression: z.string().optional(), // Default SELECT for search tables
|
||||
// uniqueRowIdExpression: z.string().optional(), // TODO: Allow users to configure how to identify rows uniquely
|
||||
|
||||
// Logs
|
||||
severityTextExpression: z.string().optional(),
|
||||
traceSourceId: z.string().optional(),
|
||||
|
||||
// Traces & Logs
|
||||
traceIdExpression: z.string().optional(),
|
||||
spanIdExpression: z.string().optional(),
|
||||
|
||||
// Traces
|
||||
durationExpression: z.string().optional(),
|
||||
durationPrecision: z.number().min(0).max(9).optional(),
|
||||
parentSpanIdExpression: z.string().optional(),
|
||||
spanKindExpression: z.string().optional(),
|
||||
spanNameExpression: z.string().optional(),
|
||||
statusCodeExpression: z.string().optional(),
|
||||
statusMessageExpression: z.string().optional(),
|
||||
|
||||
logSourceId: z.string().optional(),
|
||||
});
|
||||
|
||||
export type TSource = z.infer<typeof SourceSchema>;
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
import { format } from 'sql-formatter';
|
||||
import { sql } from '@codemirror/lang-sql';
|
||||
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { Paper } from '@mantine/core';
|
||||
import CodeMirror from '@uiw/react-codemirror';
|
||||
|
||||
import { useRenderedSqlChartConfig } from '@/hooks/useChartConfig';
|
||||
import { ChartConfigWithDateRange } from '@/renderChartConfig';
|
||||
|
||||
function tryFormat(data?: string) {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import { useCallback, useState } from 'react';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { testLocalConnection } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { Box, Button, Flex, Group, Stack, Text } from '@mantine/core';
|
||||
|
||||
import api from '@/api';
|
||||
import { testLocalConnection } from '@/clickhouse';
|
||||
import { InputControlled } from '@/components/InputControlled';
|
||||
import { IS_LOCAL_MODE } from '@/config';
|
||||
import {
|
||||
|
|
|
|||
|
|
@ -10,15 +10,15 @@ import {
|
|||
XAxis,
|
||||
YAxis,
|
||||
} from 'recharts';
|
||||
import { Box, Flex, Group, Pagination, Text } from '@mantine/core';
|
||||
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import {
|
||||
ChartConfigWithOptDateRange,
|
||||
Filter,
|
||||
inverseSqlAstFilter,
|
||||
SqlAstFilter,
|
||||
} from '@/renderChartConfig';
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { Box, Flex, Group, Pagination, Text } from '@mantine/core';
|
||||
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { truncateMiddle } from '@/utils';
|
||||
|
||||
import styles from '../../styles/HDXLineChart.module.scss';
|
||||
|
|
|
|||
|
|
@ -7,6 +7,12 @@ import {
|
|||
UseFormSetValue,
|
||||
UseFormWatch,
|
||||
} from 'react-hook-form';
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
Filter,
|
||||
SavedChartConfig,
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { DisplayType, SelectList } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
Accordion,
|
||||
Box,
|
||||
|
|
@ -28,19 +34,12 @@ import DBTableChart from '@/components/DBTableChart';
|
|||
import { DBTimeChart } from '@/components/DBTimeChart';
|
||||
import { SQLInlineEditorControlled } from '@/components/SQLInlineEditor';
|
||||
import { TimePicker } from '@/components/TimePicker';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import { GranularityPickerControlled } from '@/GranularityPicker';
|
||||
import type { Filter } from '@/renderChartConfig';
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
SavedChartConfig,
|
||||
} from '@/renderChartConfig';
|
||||
import SearchInputV2 from '@/SearchInputV2';
|
||||
import { getFirstTimestampValueExpression, useSource } from '@/source';
|
||||
import { parseTimeQuery } from '@/timeQuery';
|
||||
|
||||
import HDXMarkdownChart from '../HDXMarkdownChart';
|
||||
import { SelectList } from '../sqlTypes';
|
||||
|
||||
import { AggFnSelectControlled } from './AggFnSelect';
|
||||
import DBNumberChart from './DBNumberChart';
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@ import dynamic from 'next/dynamic';
|
|||
import type { Plugin } from 'uplot';
|
||||
import uPlot from 'uplot';
|
||||
import UplotReact from 'uplot-react';
|
||||
import { inferTimestampColumn } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { DisplayType } from '@hyperdx/common-utils/dist/types';
|
||||
import { Divider, Paper, Text } from '@mantine/core';
|
||||
import { useElementSize } from '@mantine/hooks';
|
||||
|
||||
|
|
@ -10,10 +13,7 @@ import {
|
|||
convertDateRangeToGranularityString,
|
||||
timeBucketByGranularity,
|
||||
} from '@/ChartUtils';
|
||||
import { inferTimestampColumn } from '@/clickhouse';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { ChartConfigWithDateRange } from '@/renderChartConfig';
|
||||
import { Chart, NumberFormat } from '@/types';
|
||||
import { FormatTime } from '@/useFormatTime';
|
||||
import { formatNumber } from '@/utils';
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import {
|
|||
YAxis,
|
||||
} from 'recharts';
|
||||
import { CategoricalChartState } from 'recharts/types/chart/types';
|
||||
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { Box, Code, Text } from '@mantine/core';
|
||||
|
||||
import { ClickHouseQueryError } from '@/clickhouse';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { ChartConfigWithDateRange } from '@/renderChartConfig';
|
||||
import { omit } from '@/utils';
|
||||
import { generateSearchUrl } from '@/utils';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import { useMemo } from 'react';
|
||||
import Link from 'next/link';
|
||||
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { Box, Code, Flex, HoverCard, Text } from '@mantine/core';
|
||||
import { FloatingPosition } from '@mantine/core/lib/components/Floating';
|
||||
|
||||
import { ClickHouseQueryError } from '@/clickhouse';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { ChartConfigWithDateRange } from '@/renderChartConfig';
|
||||
import type { NumberFormat } from '@/types';
|
||||
import { omit } from '@/utils';
|
||||
import { formatNumber, semanticKeyedColor } from '@/utils';
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { Box, Code, Flex, Text } from '@mantine/core';
|
||||
|
||||
import { ClickHouseQueryError } from '@/clickhouse';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { ChartConfigWithDateRange } from '@/renderChartConfig';
|
||||
import { formatNumber, omit } from '@/utils';
|
||||
|
||||
import { SQLPreview } from './ChartSQLPreview';
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import router from 'next/router';
|
|||
import { useAtom, useAtomValue } from 'jotai';
|
||||
import { atomWithStorage } from 'jotai/utils';
|
||||
import get from 'lodash/get';
|
||||
import { TSource } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
ActionIcon,
|
||||
Button,
|
||||
|
|
@ -15,7 +16,6 @@ import {
|
|||
import { useDebouncedValue } from '@mantine/hooks';
|
||||
import { notifications } from '@mantine/notifications';
|
||||
|
||||
import { TSource } from '@/commonTypes';
|
||||
import HyperJson, { GetLineActions, LineAction } from '@/components/HyperJson';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { getEventBody, getFirstTimestampValueExpression } from '@/source';
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ import { parseAsStringEnum, useQueryState } from 'nuqs';
|
|||
import { ErrorBoundary } from 'react-error-boundary';
|
||||
import { useHotkeys } from 'react-hotkeys-hook';
|
||||
import Drawer from 'react-modern-drawer';
|
||||
import { TSource } from '@hyperdx/common-utils/dist/types';
|
||||
import { Box } from '@mantine/core';
|
||||
import { useClickOutside } from '@mantine/hooks';
|
||||
|
||||
import { TSource } from '@/commonTypes';
|
||||
import DBRowSidePanelHeader from '@/components/DBRowSidePanelHeader';
|
||||
import { LogSidePanelKbdShortcuts } from '@/LogSidePanelElements';
|
||||
import { getEventBody } from '@/source';
|
||||
|
|
|
|||
|
|
@ -4,6 +4,14 @@ import curry from 'lodash/curry';
|
|||
import { Button, Modal } from 'react-bootstrap';
|
||||
import { CSVLink } from 'react-csv';
|
||||
import { useHotkeys } from 'react-hotkeys-hook';
|
||||
import {
|
||||
ClickHouseQueryError,
|
||||
convertCHDataTypeToJSType,
|
||||
extractColumnReference,
|
||||
JSDataType,
|
||||
} from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { SelectList } from '@hyperdx/common-utils/dist/types';
|
||||
import { Box, Code, Flex, Text } from '@mantine/core';
|
||||
import { FetchNextPageOptions } from '@tanstack/react-query';
|
||||
import {
|
||||
|
|
@ -17,17 +25,9 @@ import {
|
|||
} from '@tanstack/react-table';
|
||||
import { useVirtualizer } from '@tanstack/react-virtual';
|
||||
|
||||
import {
|
||||
ClickHouseQueryError,
|
||||
convertCHDataTypeToJSType,
|
||||
extractColumnReference,
|
||||
JSDataType,
|
||||
} from '@/clickhouse';
|
||||
import { useTableMetadata } from '@/hooks/useMetadata';
|
||||
import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery';
|
||||
import useRowWhere from '@/hooks/useRowWhere';
|
||||
import { ChartConfigWithDateRange } from '@/renderChartConfig';
|
||||
import { SelectList } from '@/sqlTypes';
|
||||
import { UNDEFINED_WIDTH } from '@/tableUtils';
|
||||
import { FormatTime } from '@/useFormatTime';
|
||||
import { useUserPreferences } from '@/useUserPreferences';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import {
|
||||
Button,
|
||||
Checkbox,
|
||||
|
|
@ -14,7 +15,6 @@ import {
|
|||
} from '@mantine/core';
|
||||
|
||||
import { useAllFields, useGetKeyValues } from '@/hooks/useMetadata';
|
||||
import { ChartConfigWithDateRange } from '@/renderChartConfig';
|
||||
import { useSearchPageFilterState } from '@/searchFilters';
|
||||
import { mergePath } from '@/utils';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import { useMemo } from 'react';
|
||||
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { Box, Code, Text } from '@mantine/core';
|
||||
|
||||
import { ClickHouseQueryError } from '@/clickhouse';
|
||||
import { Table } from '@/HDXMultiSeriesTableChart';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { ChartConfigWithOptDateRange } from '@/renderChartConfig';
|
||||
import { omit } from '@/utils';
|
||||
|
||||
import { SQLPreview } from './ChartSQLPreview';
|
||||
|
|
|
|||
|
|
@ -2,6 +2,12 @@ import { useMemo, useState } from 'react';
|
|||
import Link from 'next/link';
|
||||
import cx from 'classnames';
|
||||
import { add } from 'date-fns';
|
||||
import {
|
||||
ClickHouseQueryError,
|
||||
formatResponseForTimeChart,
|
||||
} from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { DisplayType } from '@hyperdx/common-utils/dist/types';
|
||||
import { Box, Button, Code, Collapse, Text } from '@mantine/core';
|
||||
import { useDisclosure } from '@mantine/hooks';
|
||||
|
||||
|
|
@ -11,11 +17,8 @@ import {
|
|||
useTimeChartSettings,
|
||||
} from '@/ChartUtils';
|
||||
import { convertGranularityToSeconds } from '@/ChartUtils';
|
||||
import { ClickHouseQueryError, formatResponseForTimeChart } from '@/clickhouse';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import { MemoChart } from '@/HDXMultiSeriesTimeChart';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { ChartConfigWithDateRange } from '@/renderChartConfig';
|
||||
|
||||
import { SQLPreview } from './ChartSQLPreview';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { useCallback, useMemo, useState } from 'react';
|
||||
import { TSource } from '@hyperdx/common-utils/dist/types';
|
||||
import { Text } from '@mantine/core';
|
||||
|
||||
import { TSource } from '@/commonTypes';
|
||||
import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery';
|
||||
import useRowWhere from '@/hooks/useRowWhere';
|
||||
import {
|
||||
|
|
|
|||
|
|
@ -11,11 +11,6 @@ import CodeMirror, {
|
|||
ReactCodeMirrorRef,
|
||||
} from '@uiw/react-codemirror';
|
||||
|
||||
import { useAllFields } from '@/hooks/useMetadata';
|
||||
import { Field } from '@/metadata';
|
||||
|
||||
import InputLanguageSwitch from './InputLanguageSwitch';
|
||||
|
||||
type SQLInlineEditorProps = {
|
||||
value: string;
|
||||
onChange: (value: string) => void;
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { useController, UseControllerProps } from 'react-hook-form';
|
|||
import { useHotkeys } from 'react-hotkeys-hook';
|
||||
import { acceptCompletion, startCompletion } from '@codemirror/autocomplete';
|
||||
import { sql, SQLDialect } from '@codemirror/lang-sql';
|
||||
import { Field } from '@hyperdx/common-utils/dist/metadata';
|
||||
import { Paper, Text } from '@mantine/core';
|
||||
import CodeMirror, {
|
||||
Compartment,
|
||||
|
|
@ -13,7 +14,6 @@ import CodeMirror, {
|
|||
} from '@uiw/react-codemirror';
|
||||
|
||||
import { useAllFields } from '@/hooks/useMetadata';
|
||||
import { Field } from '@/metadata';
|
||||
|
||||
import InputLanguageSwitch from './InputLanguageSwitch';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { useCallback, useMemo } from 'react';
|
||||
import { parseAsString, useQueryState } from 'nuqs';
|
||||
import Drawer from 'react-modern-drawer';
|
||||
import type { Filter } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { Grid, Group, Text } from '@mantine/core';
|
||||
|
||||
import { INTEGER_NUMBER_FORMAT, MS_NUMBER_FORMAT } from '@/ChartUtils';
|
||||
|
|
@ -8,7 +9,6 @@ import { ChartBox } from '@/components/ChartBox';
|
|||
import { DBTimeChart } from '@/components/DBTimeChart';
|
||||
import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils';
|
||||
import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile';
|
||||
import { Filter } from '@/renderChartConfig';
|
||||
import { getExpressions } from '@/serviceDashboard';
|
||||
import { useSource } from '@/source';
|
||||
import { useZIndex, ZIndexContext } from '@/zIndex';
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { TSource } from '@hyperdx/common-utils/dist/types';
|
||||
import { Group, Text } from '@mantine/core';
|
||||
|
||||
import { MS_NUMBER_FORMAT } from '@/ChartUtils';
|
||||
import { TSource } from '@/commonTypes';
|
||||
import { ChartBox } from '@/components/ChartBox';
|
||||
import DBListBarChart from '@/components/DBListBarChart';
|
||||
import { getExpressions } from '@/serviceDashboard';
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { useCallback, useMemo } from 'react';
|
||||
import { parseAsString, useQueryState } from 'nuqs';
|
||||
import Drawer from 'react-modern-drawer';
|
||||
import type { Filter } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { Grid, Group, Text } from '@mantine/core';
|
||||
|
||||
import {
|
||||
|
|
@ -12,7 +13,6 @@ import { DBTimeChart } from '@/components/DBTimeChart';
|
|||
import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils';
|
||||
import ServiceDashboardEndpointPerformanceChart from '@/components/ServiceDashboardEndpointPerformanceChart';
|
||||
import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile';
|
||||
import { Filter } from '@/renderChartConfig';
|
||||
import { getExpressions } from '@/serviceDashboard';
|
||||
import { EndpointLatencyChart } from '@/ServicesDashboardPage';
|
||||
import { useSource } from '@/source';
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import type { Filter } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { TSource } from '@hyperdx/common-utils/dist/types';
|
||||
import { Box, Code, Group, Text } from '@mantine/core';
|
||||
|
||||
import { ClickHouseQueryError } from '@/clickhouse';
|
||||
import { TSource } from '@/commonTypes';
|
||||
import { ChartBox } from '@/components/ChartBox';
|
||||
import { DBSqlRowTable } from '@/components/DBRowTable';
|
||||
import { useQueriedChartConfig } from '@/hooks/useChartConfig';
|
||||
import { Filter } from '@/renderChartConfig';
|
||||
import { getExpressions } from '@/serviceDashboard';
|
||||
|
||||
import { SQLPreview } from './ChartSQLPreview';
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
UseFormSetValue,
|
||||
UseFormWatch,
|
||||
} from 'react-hook-form';
|
||||
import { TSource } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
Anchor,
|
||||
Box,
|
||||
|
|
@ -25,11 +26,8 @@ import {
|
|||
import { useDebouncedCallback } from '@mantine/hooks';
|
||||
import { notifications } from '@mantine/notifications';
|
||||
|
||||
import { JSDataType } from '@/clickhouse';
|
||||
import { TSource } from '@/commonTypes';
|
||||
import { SourceSelectControlled } from '@/components/SourceSelect';
|
||||
import { useConnections } from '@/connection';
|
||||
import { Field } from '@/metadata';
|
||||
import {
|
||||
inferTableSourceConfig,
|
||||
useCreateSource,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import { env } from 'next-runtime-env';
|
||||
|
||||
export const CLICKHOUSE_HOST = '/api/clickhouse-proxy';
|
||||
|
||||
// ONLY USED IN LOCAL MODE
|
||||
// ex: NEXT_PUBLIC_HDX_LOCAL_DEFAULT_CONNECTIONS='[{"id":"local","name":"Demo","host":"https://demo-ch.hyperdx.io","username":"demo","password":"demo"}]' NEXT_PUBLIC_HDX_LOCAL_DEFAULT_SOURCES='[{"id":"l701179602","kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"default","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"l-758211293","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage"},{"id":"l-758211293","kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"default","tableName":"otel_logs"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","traceSourceId":"l701179602"}]' yarn dev:local
|
||||
export const HDX_LOCAL_DEFAULT_CONNECTIONS = env(
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import store from 'store2';
|
||||
import { testLocalConnection } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
|
||||
import { hdxServer } from '@/api';
|
||||
import { testLocalConnection } from '@/clickhouse';
|
||||
import { HDX_LOCAL_DEFAULT_CONNECTIONS, IS_LOCAL_MODE } from '@/config';
|
||||
import { parseJSON } from '@/utils';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { useCallback, useMemo } from 'react';
|
||||
import { parseAsJson, useQueryState } from 'nuqs';
|
||||
import { SavedChartConfig } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
|
||||
import { SavedChartConfig } from '@/renderChartConfig';
|
||||
import { hashCode } from '@/utils';
|
||||
|
||||
import { hdxServer } from './api';
|
||||
|
|
|
|||
|
|
@ -1,14 +1,23 @@
|
|||
import objectHash from 'object-hash';
|
||||
|
||||
import { ChSql, chSql, parameterizedQueryToSql } from '@/clickhouse';
|
||||
import {
|
||||
ChSql,
|
||||
chSql,
|
||||
parameterizedQueryToSql,
|
||||
} from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import {
|
||||
ChartConfigWithOptDateRange,
|
||||
FIXED_TIME_BUCKET_EXPR_ALIAS,
|
||||
isNonEmptyWhereExpr,
|
||||
isUsingGroupBy,
|
||||
renderChartConfig,
|
||||
} from '@/renderChartConfig';
|
||||
import { AggregateFunction, DerivedColumn, SQLInterval } from '@/sqlTypes';
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import {
|
||||
AggregateFunction,
|
||||
DerivedColumn,
|
||||
SQLInterval,
|
||||
} from '@hyperdx/common-utils/dist/types';
|
||||
|
||||
import { getMetadata } from '@/metadata';
|
||||
|
||||
const HDX_DATABASE = 'hyperdx'; // all materialized views should sit in this database
|
||||
|
||||
|
|
@ -107,7 +116,7 @@ export const buildMTViewSelectQuery = async (
|
|||
orderBy: undefined,
|
||||
limit: undefined,
|
||||
};
|
||||
const mtViewSQL = await renderChartConfig(_config);
|
||||
const mtViewSQL = await renderChartConfig(_config, getMetadata());
|
||||
const mtViewSQLHash = objectHash.sha1(mtViewSQL);
|
||||
const mtViewName = `${chartConfig.from.tableName}_mv_${mtViewSQLHash}`;
|
||||
const renderMTViewConfig = {
|
||||
|
|
@ -139,7 +148,7 @@ export const buildMTViewSelectQuery = async (
|
|||
),
|
||||
renderMTViewConfig: async () => {
|
||||
try {
|
||||
return await renderChartConfig(renderMTViewConfig);
|
||||
return await renderChartConfig(renderMTViewConfig, getMetadata());
|
||||
} catch (e) {
|
||||
console.error('Failed to render MTView config', e);
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -1,23 +1,25 @@
|
|||
import { format } from 'sql-formatter';
|
||||
import { ResponseJSON } from '@clickhouse/client-web';
|
||||
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
|
||||
|
||||
import {
|
||||
ClickHouseQueryError,
|
||||
parameterizedQueryToSql,
|
||||
sendQuery,
|
||||
} from '@/clickhouse';
|
||||
import { IS_MTVIEWS_ENABLED } from '@/config';
|
||||
import { buildMTViewSelectQuery } from '@/hdxMTViews';
|
||||
} from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import {
|
||||
ChartConfigWithOptDateRange,
|
||||
renderChartConfig,
|
||||
} from '@/renderChartConfig';
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
|
||||
|
||||
import { getClickhouseClient } from '@/clickhouse';
|
||||
import { IS_MTVIEWS_ENABLED } from '@/config';
|
||||
import { buildMTViewSelectQuery } from '@/hdxMTViews';
|
||||
import { getMetadata } from '@/metadata';
|
||||
|
||||
export function useQueriedChartConfig(
|
||||
config: ChartConfigWithOptDateRange,
|
||||
options?: Partial<UseQueryOptions<ResponseJSON<any>>>,
|
||||
) {
|
||||
const clickhouseClient = getClickhouseClient();
|
||||
return useQuery<ResponseJSON<any>, ClickHouseQueryError | Error>({
|
||||
queryKey: [config],
|
||||
queryFn: async ({ signal }) => {
|
||||
|
|
@ -33,10 +35,10 @@ export function useQueriedChartConfig(
|
|||
query = await renderMTViewConfig();
|
||||
}
|
||||
if (query == null) {
|
||||
query = await renderChartConfig(config);
|
||||
query = await renderChartConfig(config, getMetadata());
|
||||
}
|
||||
|
||||
const resultSet = await sendQuery<'JSON'>({
|
||||
const resultSet = await clickhouseClient.query<'JSON'>({
|
||||
query: query.sql,
|
||||
query_params: query.params,
|
||||
format: 'JSON',
|
||||
|
|
@ -59,7 +61,7 @@ export function useRenderedSqlChartConfig(
|
|||
return useQuery<string>({
|
||||
queryKey: ['renderedSql', config],
|
||||
queryFn: async () => {
|
||||
const query = await renderChartConfig(config);
|
||||
const query = await renderChartConfig(config, getMetadata());
|
||||
|
||||
return format(parameterizedQueryToSql(query));
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,20 +1,22 @@
|
|||
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
|
||||
|
||||
import { sendQuery } from '@/clickhouse';
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
renderChartConfig,
|
||||
} from '@/renderChartConfig';
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import { useQuery, UseQueryOptions } from '@tanstack/react-query';
|
||||
|
||||
import { getClickhouseClient } from '@/clickhouse';
|
||||
import { getMetadata } from '@/metadata';
|
||||
|
||||
export function useExplainQuery(
|
||||
config: ChartConfigWithDateRange,
|
||||
options?: Omit<UseQueryOptions<any>, 'queryKey' | 'queryFn'>,
|
||||
) {
|
||||
const clickhouseClient = getClickhouseClient();
|
||||
const { data, isLoading, error } = useQuery({
|
||||
queryKey: ['explain', config],
|
||||
queryFn: async ({ signal }) => {
|
||||
const query = await renderChartConfig(config);
|
||||
const response = await sendQuery<'JSONEachRow'>({
|
||||
const query = await renderChartConfig(config, getMetadata());
|
||||
const response = await clickhouseClient.query<'JSONEachRow'>({
|
||||
query: `EXPLAIN ESTIMATE ${query.sql}`,
|
||||
query_params: query.params,
|
||||
format: 'JSONEachRow',
|
||||
|
|
|
|||
|
|
@ -1,17 +1,13 @@
|
|||
import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { Field, TableMetadata } from '@hyperdx/common-utils/dist/metadata';
|
||||
import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import {
|
||||
keepPreviousData,
|
||||
useQuery,
|
||||
UseQueryOptions,
|
||||
} from '@tanstack/react-query';
|
||||
|
||||
import {
|
||||
ColumnMeta,
|
||||
ColumnMetaType,
|
||||
filterColumnMetaByType,
|
||||
JSDataType,
|
||||
} from '@/clickhouse';
|
||||
import { Field, metadata, TableMetadata } from '@/metadata';
|
||||
import { ChartConfigWithDateRange } from '@/renderChartConfig';
|
||||
import { getMetadata } from '@/metadata';
|
||||
|
||||
export function useColumns(
|
||||
{
|
||||
|
|
@ -28,6 +24,7 @@ export function useColumns(
|
|||
return useQuery<ColumnMeta[]>({
|
||||
queryKey: ['useMetadata.useColumns', { databaseName, tableName }],
|
||||
queryFn: async () => {
|
||||
const metadata = getMetadata();
|
||||
return metadata.getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
|
|
@ -50,6 +47,7 @@ export function useAllFields(
|
|||
},
|
||||
options?: Partial<UseQueryOptions<Field[]>>,
|
||||
) {
|
||||
const metadata = getMetadata();
|
||||
return useQuery<Field[]>({
|
||||
queryKey: ['useMetadata.useAllFields', { databaseName, tableName }],
|
||||
queryFn: async () => {
|
||||
|
|
@ -75,6 +73,7 @@ export function useTableMetadata(
|
|||
},
|
||||
options?: Omit<UseQueryOptions<any, Error>, 'queryKey'>,
|
||||
) {
|
||||
const metadata = getMetadata();
|
||||
return useQuery<TableMetadata>({
|
||||
queryKey: ['useMetadata.useTableMetadata', { databaseName, tableName }],
|
||||
queryFn: async () => {
|
||||
|
|
@ -96,6 +95,7 @@ export function useGetKeyValues({
|
|||
chartConfig: ChartConfigWithDateRange;
|
||||
keys: string[];
|
||||
}) {
|
||||
const metadata = getMetadata();
|
||||
return useQuery({
|
||||
queryKey: ['useMetadata.useGetKeyValues', { chartConfig, keys }],
|
||||
queryFn: async () => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,14 @@
|
|||
import { useMemo } from 'react';
|
||||
import ms from 'ms';
|
||||
import { ResponseJSON, Row } from '@clickhouse/client-web';
|
||||
import {
|
||||
ClickHouseQueryError,
|
||||
ColumnMetaType,
|
||||
} from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
renderChartConfig,
|
||||
} from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
import {
|
||||
QueryClient,
|
||||
QueryFunction,
|
||||
|
|
@ -8,11 +16,8 @@ import {
|
|||
useQueryClient,
|
||||
} from '@tanstack/react-query';
|
||||
|
||||
import { ClickHouseQueryError, ColumnMetaType, sendQuery } from '@/clickhouse';
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
renderChartConfig,
|
||||
} from '@/renderChartConfig';
|
||||
import { getClickhouseClient } from '@/clickhouse';
|
||||
import { getMetadata } from '@/metadata';
|
||||
import { omit } from '@/utils';
|
||||
|
||||
function queryKeyFn(prefix: string, config: ChartConfigWithDateRange) {
|
||||
|
|
@ -44,23 +49,28 @@ const queryFn: QueryFunction<
|
|||
const isStreamingIncrementally = !meta.hasPreviousQueries || pageParam > 0;
|
||||
|
||||
const config = queryKey[1];
|
||||
const query = await renderChartConfig({
|
||||
...config,
|
||||
limit: {
|
||||
limit: config.limit?.limit,
|
||||
offset: pageParam,
|
||||
const query = await renderChartConfig(
|
||||
{
|
||||
...config,
|
||||
limit: {
|
||||
limit: config.limit?.limit,
|
||||
offset: pageParam,
|
||||
},
|
||||
},
|
||||
});
|
||||
getMetadata(),
|
||||
);
|
||||
|
||||
const resultSet = await sendQuery<'JSONCompactEachRowWithNamesAndTypes'>({
|
||||
query: query.sql,
|
||||
query_params: query.params,
|
||||
format: 'JSONCompactEachRowWithNamesAndTypes',
|
||||
abort_signal: signal,
|
||||
connectionId: config.connection,
|
||||
});
|
||||
const clickhouseClient = getClickhouseClient();
|
||||
const resultSet =
|
||||
await clickhouseClient.query<'JSONCompactEachRowWithNamesAndTypes'>({
|
||||
query: query.sql,
|
||||
query_params: query.params,
|
||||
format: 'JSONCompactEachRowWithNamesAndTypes',
|
||||
abort_signal: signal,
|
||||
connectionId: config.connection,
|
||||
});
|
||||
|
||||
const stream = resultSet.stream<unknown[]>();
|
||||
const stream = resultSet.stream();
|
||||
|
||||
const reader = stream.getReader();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
import { useCallback, useMemo } from 'react';
|
||||
import MD5 from 'crypto-js/md5';
|
||||
import SqlString from 'sqlstring';
|
||||
|
||||
import {
|
||||
ColumnMetaType,
|
||||
convertCHDataTypeToJSType,
|
||||
JSDataType,
|
||||
} from '@/clickhouse';
|
||||
} from '@hyperdx/common-utils/dist/clickhouse';
|
||||
|
||||
const MAX_STRING_LENGTH = 512;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,440 +1,5 @@
|
|||
import {
|
||||
ChSql,
|
||||
chSql,
|
||||
ColumnMeta,
|
||||
convertCHDataTypeToJSType,
|
||||
filterColumnMetaByType,
|
||||
JSDataType,
|
||||
sendQuery,
|
||||
tableExpr,
|
||||
} from '@/clickhouse';
|
||||
import { getMetadata as _getMetadata } from '@hyperdx/common-utils/dist/metadata';
|
||||
|
||||
import {
|
||||
ChartConfigWithDateRange,
|
||||
renderChartConfig,
|
||||
} from './renderChartConfig';
|
||||
import { getClickhouseClient } from '@/clickhouse';
|
||||
|
||||
const DEFAULT_SAMPLE_SIZE = 1e6;
|
||||
|
||||
class MetadataCache {
|
||||
private cache = new Map<string, any>();
|
||||
|
||||
// this should be getOrUpdate... or just query to follow react query
|
||||
get<T>(key: string): T | undefined {
|
||||
return this.cache.get(key);
|
||||
}
|
||||
|
||||
async getOrFetch<T>(key: string, query: () => Promise<T>): Promise<T> {
|
||||
const value = this.get(key) as T | undefined;
|
||||
if (value != null) {
|
||||
return value;
|
||||
}
|
||||
|
||||
// Request a lock
|
||||
let newValue!: T;
|
||||
await navigator.locks.request(`MedataCache.${key}`, async lock => {
|
||||
newValue = await query();
|
||||
this.cache.set(key, newValue);
|
||||
});
|
||||
|
||||
return newValue;
|
||||
}
|
||||
|
||||
set<T>(key: string, value: T) {
|
||||
return this.cache.set(key, value);
|
||||
}
|
||||
|
||||
// TODO: This needs to be async, and use tanstack query on frontend for cache
|
||||
// TODO: Implement locks for refreshing
|
||||
// TODO: Shard cache by time
|
||||
}
|
||||
|
||||
export type TableMetadata = {
|
||||
database: string;
|
||||
name: string;
|
||||
uuid: string;
|
||||
engine: string;
|
||||
is_temporary: number;
|
||||
data_paths: string[];
|
||||
metadata_path: string;
|
||||
metadata_modification_time: string;
|
||||
metadata_version: number;
|
||||
create_table_query: string;
|
||||
engine_full: string;
|
||||
as_select: string;
|
||||
partition_key: string;
|
||||
sorting_key: string;
|
||||
primary_key: string;
|
||||
sampling_key: string;
|
||||
storage_policy: string;
|
||||
total_rows: string;
|
||||
total_bytes: string;
|
||||
total_bytes_uncompressed: string;
|
||||
parts: string;
|
||||
active_parts: string;
|
||||
total_marks: string;
|
||||
comment: string;
|
||||
};
|
||||
|
||||
export class Metadata {
|
||||
private cache = new MetadataCache();
|
||||
|
||||
private static async queryTableMetadata({
|
||||
database,
|
||||
table,
|
||||
cache,
|
||||
connectionId,
|
||||
}: {
|
||||
database: string;
|
||||
table: string;
|
||||
cache: MetadataCache;
|
||||
connectionId: string;
|
||||
}) {
|
||||
return cache.getOrFetch(`${database}.${table}.metadata`, async () => {
|
||||
const sql = chSql`SELECT * FROM system.tables where database = ${{ String: database }} AND name = ${{ String: table }}`;
|
||||
const json = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
}).then(res => res.json<TableMetadata>());
|
||||
return json.data[0];
|
||||
});
|
||||
}
|
||||
|
||||
async getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
}) {
|
||||
return this.cache.getOrFetch<ColumnMeta[]>(
|
||||
`${databaseName}.${tableName}.columns`,
|
||||
async () => {
|
||||
const sql = chSql`DESCRIBE ${tableExpr({ database: databaseName, table: tableName })}`;
|
||||
const columns = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
})
|
||||
.then(res => res.json())
|
||||
.then(d => d.data);
|
||||
return columns as ColumnMeta[];
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async getMaterializedColumnsLookupTable({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const columns = await this.getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
// Build up materalized fields lookup table
|
||||
return new Map(
|
||||
columns
|
||||
.filter(
|
||||
c =>
|
||||
c.default_type === 'MATERIALIZED' || c.default_type === 'DEFAULT',
|
||||
)
|
||||
.map(c => [c.default_expression, c.name]),
|
||||
);
|
||||
}
|
||||
|
||||
async getColumn({
|
||||
databaseName,
|
||||
tableName,
|
||||
column,
|
||||
matchLowercase = false,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
column: string;
|
||||
matchLowercase?: boolean;
|
||||
connectionId: string;
|
||||
}): Promise<ColumnMeta | undefined> {
|
||||
const tableColumns = await this.getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
return tableColumns.filter(c => {
|
||||
if (matchLowercase) {
|
||||
return c.name.toLowerCase() === column.toLowerCase();
|
||||
}
|
||||
|
||||
return c.name === column;
|
||||
})[0];
|
||||
}
|
||||
|
||||
async getMapKeys({
|
||||
databaseName,
|
||||
tableName,
|
||||
column,
|
||||
maxKeys = 1000,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
column: string;
|
||||
maxKeys?: number;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const cachedKeys = this.cache.get<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.keys`,
|
||||
);
|
||||
|
||||
if (cachedKeys != null) {
|
||||
return cachedKeys;
|
||||
}
|
||||
|
||||
const colMeta = await this.getColumn({
|
||||
databaseName,
|
||||
tableName,
|
||||
column,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
if (colMeta == null) {
|
||||
throw new Error(
|
||||
`Column ${column} not found in ${databaseName}.${tableName}`,
|
||||
);
|
||||
}
|
||||
|
||||
let strategy: 'groupUniqArrayArray' | 'lowCardinalityKeys' =
|
||||
'groupUniqArrayArray';
|
||||
if (colMeta.type.startsWith('Map(LowCardinality(String)')) {
|
||||
strategy = 'lowCardinalityKeys';
|
||||
}
|
||||
|
||||
let sql: ChSql;
|
||||
if (strategy === 'groupUniqArrayArray') {
|
||||
sql = chSql`SELECT groupUniqArrayArray(${{ Int32: maxKeys }})(${{
|
||||
Identifier: column,
|
||||
}}) as keysArr
|
||||
FROM ${tableExpr({ database: databaseName, table: tableName })}`;
|
||||
} else {
|
||||
sql = chSql`SELECT DISTINCT lowCardinalityKeys(arrayJoin(${{
|
||||
Identifier: column,
|
||||
}}.keys)) as key
|
||||
FROM ${tableExpr({ database: databaseName, table: tableName })}
|
||||
LIMIT ${{
|
||||
Int32: maxKeys,
|
||||
}}`;
|
||||
}
|
||||
|
||||
return this.cache.getOrFetch<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.keys`,
|
||||
async () => {
|
||||
const keys = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
})
|
||||
.then(res => res.json<Record<string, unknown>>())
|
||||
.then(d => {
|
||||
let output: string[];
|
||||
if (strategy === 'groupUniqArrayArray') {
|
||||
output = d.data[0].keysArr as string[];
|
||||
} else {
|
||||
output = d.data.map(row => row.key) as string[];
|
||||
}
|
||||
|
||||
return output.filter(r => r);
|
||||
});
|
||||
return keys;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async getMapValues({
|
||||
databaseName,
|
||||
tableName,
|
||||
column,
|
||||
key,
|
||||
maxValues = 20,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
column: string;
|
||||
key?: string;
|
||||
maxValues?: number;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const cachedValues = this.cache.get<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.${key}.values`,
|
||||
);
|
||||
|
||||
if (cachedValues != null) {
|
||||
return cachedValues;
|
||||
}
|
||||
|
||||
const sql = key
|
||||
? chSql`
|
||||
SELECT DISTINCT ${{
|
||||
Identifier: column,
|
||||
}}[${{ String: key }}] as value
|
||||
FROM ${tableExpr({ database: databaseName, table: tableName })}
|
||||
WHERE value != ''
|
||||
LIMIT ${{
|
||||
Int32: maxValues,
|
||||
}}
|
||||
`
|
||||
: chSql`
|
||||
SELECT DISTINCT ${{
|
||||
Identifier: column,
|
||||
}} as value
|
||||
FROM ${tableExpr({ database: databaseName, table: tableName })}
|
||||
WHERE value != ''
|
||||
LIMIT ${{
|
||||
Int32: maxValues,
|
||||
}}
|
||||
`;
|
||||
|
||||
return this.cache.getOrFetch<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.${key}.values`,
|
||||
async () => {
|
||||
const values = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
})
|
||||
.then(res => res.json<Record<string, unknown>>())
|
||||
.then(d => d.data.map(row => row.value as string));
|
||||
return values;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async getAllFields({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const fields: Field[] = [];
|
||||
const columns = await this.getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
for (const c of columns) {
|
||||
fields.push({
|
||||
path: [c.name],
|
||||
type: c.type,
|
||||
jsType: convertCHDataTypeToJSType(c.type),
|
||||
});
|
||||
}
|
||||
|
||||
const mapColumns = filterColumnMetaByType(columns, [JSDataType.Map]) ?? [];
|
||||
|
||||
await Promise.all(
|
||||
mapColumns.map(async column => {
|
||||
const keys = await this.getMapKeys({
|
||||
databaseName,
|
||||
tableName,
|
||||
column: column.name,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
const match = column.type.match(/Map\(.+,\s*(.+)\)/);
|
||||
const chType = match?.[1] ?? 'String'; // default to string ?
|
||||
|
||||
for (const key of keys) {
|
||||
fields.push({
|
||||
path: [column.name, key],
|
||||
type: chType,
|
||||
jsType: convertCHDataTypeToJSType(chType),
|
||||
});
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
return fields;
|
||||
}
|
||||
|
||||
async getTableMetadata({
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
}: {
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const tableMetadata = await Metadata.queryTableMetadata({
|
||||
cache: this.cache,
|
||||
database: databaseName,
|
||||
table: tableName,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
return tableMetadata;
|
||||
}
|
||||
|
||||
async getKeyValues({
|
||||
chartConfig,
|
||||
keys,
|
||||
limit = 20,
|
||||
}: {
|
||||
chartConfig: ChartConfigWithDateRange;
|
||||
keys: string[];
|
||||
limit?: number;
|
||||
}) {
|
||||
const sql = await renderChartConfig({
|
||||
...chartConfig,
|
||||
select: keys
|
||||
.map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`)
|
||||
.join(', '),
|
||||
});
|
||||
|
||||
const json = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId: chartConfig.connection,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
}).then(res => res.json<any>());
|
||||
|
||||
return Object.entries(json.data[0]).map(([key, value]) => ({
|
||||
key: keys[parseInt(key.replace('param', ''))],
|
||||
value: (value as string[])?.filter(Boolean), // remove nulls
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
export type Field = {
|
||||
path: string[];
|
||||
type: string;
|
||||
jsType: JSDataType | null;
|
||||
};
|
||||
|
||||
export const metadata = new Metadata();
|
||||
export const getMetadata = () => _getMetadata(getClickhouseClient());
|
||||
|
|
|
|||
|
|
@ -1,717 +0,0 @@
|
|||
import SqlString from 'sqlstring';
|
||||
import lucene from '@hyperdx/lucene';
|
||||
|
||||
import { convertCHTypeToPrimitiveJSType } from './clickhouse';
|
||||
import { Metadata } from './metadata';
|
||||
|
||||
function encodeSpecialTokens(query: string): string {
|
||||
return query
|
||||
.replace(/\\\\/g, 'HDX_BACKSLASH_LITERAL')
|
||||
.replace('http://', 'http_COLON_//')
|
||||
.replace('https://', 'https_COLON_//')
|
||||
.replace(/localhost:(\d{1,5})/, 'localhost_COLON_$1')
|
||||
.replace(/\\:/g, 'HDX_COLON');
|
||||
}
|
||||
function decodeSpecialTokens(query: string): string {
|
||||
return query
|
||||
.replace(/\\"/g, '"')
|
||||
.replace(/HDX_BACKSLASH_LITERAL/g, '\\')
|
||||
.replace('http_COLON_//', 'http://')
|
||||
.replace('https_COLON_//', 'https://')
|
||||
.replace(/localhost_COLON_(\d{1,5})/, 'localhost:$1')
|
||||
.replace(/HDX_COLON/g, ':');
|
||||
}
|
||||
|
||||
export function parse(query: string): lucene.AST {
|
||||
return lucene.parse(encodeSpecialTokens(query));
|
||||
}
|
||||
|
||||
const IMPLICIT_FIELD = '<implicit>';
|
||||
|
||||
interface Serializer {
|
||||
operator(op: lucene.Operator): string;
|
||||
eq(field: string, term: string, isNegatedField: boolean): Promise<string>;
|
||||
isNotNull(field: string, isNegatedField: boolean): Promise<string>;
|
||||
gte(field: string, term: string): Promise<string>;
|
||||
lte(field: string, term: string): Promise<string>;
|
||||
lt(field: string, term: string): Promise<string>;
|
||||
gt(field: string, term: string): Promise<string>;
|
||||
fieldSearch(
|
||||
field: string,
|
||||
term: string,
|
||||
isNegatedField: boolean,
|
||||
prefixWildcard: boolean,
|
||||
suffixWildcard: boolean,
|
||||
): Promise<string>;
|
||||
range(
|
||||
field: string,
|
||||
start: string,
|
||||
end: string,
|
||||
isNegatedField: boolean,
|
||||
): Promise<string>;
|
||||
}
|
||||
|
||||
class EnglishSerializer implements Serializer {
|
||||
private translateField(field: string) {
|
||||
if (field === IMPLICIT_FIELD) {
|
||||
return 'event';
|
||||
}
|
||||
|
||||
return `'${field}'`;
|
||||
}
|
||||
|
||||
operator(op: lucene.Operator) {
|
||||
switch (op) {
|
||||
case 'NOT':
|
||||
case 'AND NOT':
|
||||
return 'AND NOT';
|
||||
case 'OR NOT':
|
||||
return 'OR NOT';
|
||||
// @ts-ignore TODO: Types need to be fixed upstream
|
||||
case '&&':
|
||||
case '<implicit>':
|
||||
case 'AND':
|
||||
return 'AND';
|
||||
// @ts-ignore TODO: Types need to be fixed upstream
|
||||
case '||':
|
||||
case 'OR':
|
||||
return 'OR';
|
||||
default:
|
||||
throw new Error(`Unexpected operator. ${op}`);
|
||||
}
|
||||
}
|
||||
|
||||
async eq(field: string, term: string, isNegatedField: boolean) {
|
||||
return `${this.translateField(field)} ${
|
||||
isNegatedField ? 'is not' : 'is'
|
||||
} ${term}`;
|
||||
}
|
||||
|
||||
async isNotNull(field: string, isNegatedField: boolean) {
|
||||
return `${this.translateField(field)} ${
|
||||
isNegatedField ? 'is null' : 'is not null'
|
||||
}`;
|
||||
}
|
||||
|
||||
async gte(field: string, term: string) {
|
||||
return `${this.translateField(field)} is greater than or equal to ${term}`;
|
||||
}
|
||||
|
||||
async lte(field: string, term: string) {
|
||||
return `${this.translateField(field)} is less than or equal to ${term}`;
|
||||
}
|
||||
|
||||
async lt(field: string, term: string) {
|
||||
return `${this.translateField(field)} is less than ${term}`;
|
||||
}
|
||||
|
||||
async gt(field: string, term: string) {
|
||||
return `${this.translateField(field)} is greater than ${term}`;
|
||||
}
|
||||
|
||||
// async fieldSearch(field: string, term: string, isNegatedField: boolean) {
|
||||
// return `${this.translateField(field)} ${
|
||||
// isNegatedField ? 'does not contain' : 'contains'
|
||||
// } ${term}`;
|
||||
// }
|
||||
|
||||
async fieldSearch(
|
||||
field: string,
|
||||
term: string,
|
||||
isNegatedField: boolean,
|
||||
prefixWildcard: boolean,
|
||||
suffixWildcard: boolean,
|
||||
) {
|
||||
if (field === IMPLICIT_FIELD) {
|
||||
return `${this.translateField(field)} ${
|
||||
prefixWildcard && suffixWildcard
|
||||
? isNegatedField
|
||||
? 'does not contain'
|
||||
: 'contains'
|
||||
: prefixWildcard
|
||||
? isNegatedField
|
||||
? 'does not end with'
|
||||
: 'ends with'
|
||||
: suffixWildcard
|
||||
? isNegatedField
|
||||
? 'does not start with'
|
||||
: 'starts with'
|
||||
: isNegatedField
|
||||
? 'does not have whole word'
|
||||
: 'has whole word'
|
||||
} ${term}`;
|
||||
} else {
|
||||
return `${this.translateField(field)} ${
|
||||
isNegatedField ? 'does not contain' : 'contains'
|
||||
} ${term}`;
|
||||
}
|
||||
}
|
||||
|
||||
async range(
|
||||
field: string,
|
||||
start: string,
|
||||
end: string,
|
||||
isNegatedField: boolean,
|
||||
) {
|
||||
return `${field} ${
|
||||
isNegatedField ? 'is not' : 'is'
|
||||
} between ${start} and ${end}`;
|
||||
}
|
||||
}
|
||||
|
||||
export abstract class SQLSerializer implements Serializer {
|
||||
private NOT_FOUND_QUERY = '(1 = 0)';
|
||||
|
||||
abstract getColumnForField(field: string): Promise<{
|
||||
column?: string;
|
||||
propertyType?: 'string' | 'number' | 'bool';
|
||||
found: boolean;
|
||||
}>;
|
||||
|
||||
operator(op: lucene.Operator) {
|
||||
switch (op) {
|
||||
case 'NOT':
|
||||
case 'AND NOT':
|
||||
return 'AND NOT';
|
||||
case 'OR NOT':
|
||||
return 'OR NOT';
|
||||
// @ts-ignore TODO: Types need to be fixed upstream
|
||||
case '&&':
|
||||
case '<implicit>':
|
||||
case 'AND':
|
||||
return 'AND';
|
||||
// @ts-ignore TODO: Types need to be fixed upstream
|
||||
case '||':
|
||||
case 'OR':
|
||||
return 'OR';
|
||||
default:
|
||||
throw new Error(`Unexpected operator. ${op}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Only for exact string matches
|
||||
async eq(field: string, term: string, isNegatedField: boolean) {
|
||||
const { column, found, propertyType } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
if (propertyType === 'bool') {
|
||||
// numeric and boolean fields must be equality matched
|
||||
const normTerm = `${term}`.trim().toLowerCase();
|
||||
return SqlString.format(`(?? ${isNegatedField ? '!' : ''}= ?)`, [
|
||||
column,
|
||||
normTerm === 'true' ? 1 : normTerm === 'false' ? 0 : parseInt(normTerm),
|
||||
]);
|
||||
} else if (propertyType === 'number') {
|
||||
return SqlString.format(
|
||||
`(${column} ${isNegatedField ? '!' : ''}= CAST(?, 'Float64'))`,
|
||||
[term],
|
||||
);
|
||||
}
|
||||
return SqlString.format(`(${column} ${isNegatedField ? '!' : ''}= ?)`, [
|
||||
term,
|
||||
]);
|
||||
}
|
||||
|
||||
async isNotNull(field: string, isNegatedField: boolean) {
|
||||
const { found, column } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
|
||||
return `notEmpty(${column}) ${isNegatedField ? '!' : ''}= 1`;
|
||||
}
|
||||
|
||||
async gte(field: string, term: string) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(`(${column} >= ?)`, [term]);
|
||||
}
|
||||
|
||||
async lte(field: string, term: string) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(`(${column} <= ?)`, [term]);
|
||||
}
|
||||
|
||||
async lt(field: string, term: string) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(`(${column} < ?)`, [term]);
|
||||
}
|
||||
|
||||
async gt(field: string, term: string) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(`(${column} > ?)`, [term]);
|
||||
}
|
||||
|
||||
// TODO: Not sure if SQL really needs this or if it'll coerce itself
|
||||
private attemptToParseNumber(term: string): string | number {
|
||||
const number = Number.parseFloat(term);
|
||||
if (Number.isNaN(number)) {
|
||||
return term;
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
// Ref: https://clickhouse.com/codebrowser/ClickHouse/src/Functions/HasTokenImpl.h.html#_ZN2DB12HasTokenImpl16isTokenSeparatorEDu
|
||||
// Split by anything that's ascii 0-128, that's not a letter or a number
|
||||
private tokenizeTerm(term: string): string[] {
|
||||
return term.split(/[ -/:-@[-`{-~\t\n\r]+/).filter(t => t.length > 0);
|
||||
}
|
||||
|
||||
private termHasSeperators(term: string): boolean {
|
||||
return term.match(/[ -/:-@[-`{-~\t\n\r]+/) != null;
|
||||
}
|
||||
|
||||
async fieldSearch(
|
||||
field: string,
|
||||
term: string,
|
||||
isNegatedField: boolean,
|
||||
prefixWildcard: boolean,
|
||||
suffixWildcard: boolean,
|
||||
) {
|
||||
const isImplicitField = field === IMPLICIT_FIELD;
|
||||
const { column, propertyType, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
// If it's a string field, we will always try to match with ilike
|
||||
|
||||
if (propertyType === 'bool') {
|
||||
// numeric and boolean fields must be equality matched
|
||||
const normTerm = `${term}`.trim().toLowerCase();
|
||||
return SqlString.format(`(?? ${isNegatedField ? '!' : ''}= ?)`, [
|
||||
column,
|
||||
normTerm === 'true' ? 1 : normTerm === 'false' ? 0 : parseInt(normTerm),
|
||||
]);
|
||||
} else if (propertyType === 'number') {
|
||||
return SqlString.format(
|
||||
`(?? ${isNegatedField ? '!' : ''}= CAST(?, 'Float64'))`,
|
||||
[column, term],
|
||||
);
|
||||
}
|
||||
|
||||
// // If the query is empty, or is a empty quoted string ex: ""
|
||||
// // we should match all
|
||||
if (term.length === 0) {
|
||||
return '(1=1)';
|
||||
}
|
||||
|
||||
if (isImplicitField) {
|
||||
// For the _source column, we'll try to do whole word searches by default
|
||||
// to utilize the token bloom filter unless a prefix/sufix wildcard is specified
|
||||
if (prefixWildcard || suffixWildcard) {
|
||||
return SqlString.format(
|
||||
`(lower(??) ${isNegatedField ? 'NOT ' : ''}LIKE lower(?))`,
|
||||
[
|
||||
column,
|
||||
`${prefixWildcard ? '%' : ''}${term}${suffixWildcard ? '%' : ''}`,
|
||||
],
|
||||
);
|
||||
} else {
|
||||
// We can't search multiple tokens with `hasToken`, so we need to split up the term into tokens
|
||||
const hasSeperators = this.termHasSeperators(term);
|
||||
if (hasSeperators) {
|
||||
const tokens = this.tokenizeTerm(term);
|
||||
return `(${isNegatedField ? 'NOT (' : ''}${[
|
||||
...tokens.map(token =>
|
||||
SqlString.format(`hasTokenCaseInsensitive(??, ?)`, [
|
||||
column,
|
||||
token,
|
||||
]),
|
||||
),
|
||||
// If there are symbols in the term, we'll try to match the whole term as well (ex. Scott!)
|
||||
SqlString.format(`(lower(??) LIKE lower(?))`, [
|
||||
column,
|
||||
`%${term}%`,
|
||||
]),
|
||||
].join(' AND ')}${isNegatedField ? ')' : ''})`;
|
||||
} else {
|
||||
return SqlString.format(
|
||||
`(${isNegatedField ? 'NOT ' : ''}hasTokenCaseInsensitive(??, ?))`,
|
||||
[column, term],
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const shoudUseTokenBf = isImplicitField;
|
||||
return SqlString.format(
|
||||
`(${column} ${isNegatedField ? 'NOT ' : ''}? ?)`,
|
||||
[SqlString.raw(shoudUseTokenBf ? 'LIKE' : 'ILIKE'), `%${term}%`],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async range(
|
||||
field: string,
|
||||
start: string,
|
||||
end: string,
|
||||
isNegatedField: boolean,
|
||||
) {
|
||||
const { column, found } = await this.getColumnForField(field);
|
||||
if (!found) {
|
||||
return this.NOT_FOUND_QUERY;
|
||||
}
|
||||
return SqlString.format(
|
||||
`(${column} ${isNegatedField ? 'NOT ' : ''}BETWEEN ? AND ?)`,
|
||||
[this.attemptToParseNumber(start), this.attemptToParseNumber(end)],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export type CustomSchemaConfig = {
|
||||
databaseName: string;
|
||||
implicitColumnExpression?: string;
|
||||
tableName: string;
|
||||
connectionId: string;
|
||||
};
|
||||
|
||||
export class CustomSchemaSQLSerializerV2 extends SQLSerializer {
|
||||
private metadata: Metadata;
|
||||
private tableName: string;
|
||||
private databaseName: string;
|
||||
private implicitColumnExpression?: string;
|
||||
private connectionId: string;
|
||||
|
||||
constructor({
|
||||
metadata,
|
||||
databaseName,
|
||||
tableName,
|
||||
connectionId,
|
||||
implicitColumnExpression,
|
||||
}: { metadata: Metadata } & CustomSchemaConfig) {
|
||||
super();
|
||||
this.metadata = metadata;
|
||||
this.databaseName = databaseName;
|
||||
this.tableName = tableName;
|
||||
this.implicitColumnExpression = implicitColumnExpression;
|
||||
this.connectionId = connectionId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate field from user ex. column.property.subproperty to SQL expression
|
||||
* Supports:
|
||||
* - Materialized Columns
|
||||
* - Map
|
||||
* - JSON Strings (via JSONExtract)
|
||||
* TODO:
|
||||
* - Nested Map
|
||||
* - JSONExtract for non-string types
|
||||
*/
|
||||
private async buildColumnExpressionFromField(field: string) {
|
||||
const exactMatch = await this.metadata.getColumn({
|
||||
databaseName: this.databaseName,
|
||||
tableName: this.tableName,
|
||||
column: field,
|
||||
connectionId: this.connectionId,
|
||||
});
|
||||
|
||||
if (exactMatch) {
|
||||
return {
|
||||
found: true,
|
||||
columnType: exactMatch.type,
|
||||
columnExpression: exactMatch.name,
|
||||
};
|
||||
}
|
||||
|
||||
const fieldPrefix = field.split('.')[0];
|
||||
const prefixMatch = await this.metadata.getColumn({
|
||||
databaseName: this.databaseName,
|
||||
tableName: this.tableName,
|
||||
column: fieldPrefix,
|
||||
connectionId: this.connectionId,
|
||||
});
|
||||
|
||||
if (prefixMatch) {
|
||||
const fieldPostfix = field.split('.').slice(1).join('.');
|
||||
|
||||
if (prefixMatch.type.startsWith('Map')) {
|
||||
const valueType = prefixMatch.type.match(/,\s+(\w+)\)$/)?.[1];
|
||||
return {
|
||||
found: true,
|
||||
columnExpression: SqlString.format(`??[?]`, [
|
||||
prefixMatch.name,
|
||||
fieldPostfix,
|
||||
]),
|
||||
columnType: valueType ?? 'Unknown',
|
||||
};
|
||||
} else if (prefixMatch.type === 'String') {
|
||||
// TODO: Support non-strings
|
||||
const nestedPaths = fieldPostfix.split('.');
|
||||
return {
|
||||
found: true,
|
||||
columnExpression: SqlString.format(
|
||||
`JSONExtractString(??, ${Array(nestedPaths.length)
|
||||
.fill('?')
|
||||
.join(',')})`,
|
||||
[prefixMatch.name, ...nestedPaths],
|
||||
),
|
||||
columnType: 'String',
|
||||
};
|
||||
}
|
||||
// TODO: Support arrays and tuples
|
||||
throw new Error('Unsupported column type for prefix match');
|
||||
}
|
||||
|
||||
throw new Error(`Column not found: ${field}`);
|
||||
}
|
||||
|
||||
async getColumnForField(field: string) {
|
||||
if (field === IMPLICIT_FIELD) {
|
||||
if (!this.implicitColumnExpression) {
|
||||
throw new Error(
|
||||
'Can not search bare text without an implicit column set.',
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
column: this.implicitColumnExpression,
|
||||
propertyType: 'string' as const,
|
||||
found: true,
|
||||
};
|
||||
}
|
||||
|
||||
const expression = await this.buildColumnExpressionFromField(field);
|
||||
|
||||
return {
|
||||
column: expression.columnExpression,
|
||||
propertyType:
|
||||
convertCHTypeToPrimitiveJSType(expression.columnType) ?? undefined,
|
||||
found: expression.found,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function nodeTerm(
|
||||
node: lucene.Node,
|
||||
serializer: Serializer,
|
||||
): Promise<string> {
|
||||
const field = node.field[0] === '-' ? node.field.slice(1) : node.field;
|
||||
let isNegatedField = node.field[0] === '-';
|
||||
const isImplicitField = node.field === IMPLICIT_FIELD;
|
||||
|
||||
// NodeTerm
|
||||
if ((node as lucene.NodeTerm).term != null) {
|
||||
const nodeTerm = node as lucene.NodeTerm;
|
||||
let term = decodeSpecialTokens(nodeTerm.term);
|
||||
// We should only negate the search for negated bare terms (ex. '-5')
|
||||
// This meeans the field is implicit and the prefix is -
|
||||
if (isImplicitField && nodeTerm.prefix === '-') {
|
||||
isNegatedField = true;
|
||||
}
|
||||
// Otherwise, if we have a negated term for a field (ex. 'level:-5')
|
||||
// we should not negate the search, and search for -5
|
||||
if (!isImplicitField && nodeTerm.prefix === '-') {
|
||||
term = nodeTerm.prefix + decodeSpecialTokens(nodeTerm.term);
|
||||
}
|
||||
|
||||
// TODO: Decide if this is good behavior
|
||||
// If the term is quoted, we should search for the exact term in a property (ex. foo:"bar")
|
||||
// Implicit field searches should still use substring matching (ex. "foo bar")
|
||||
if (nodeTerm.quoted && !isImplicitField) {
|
||||
return serializer.eq(field, term, isNegatedField);
|
||||
}
|
||||
|
||||
if (!nodeTerm.quoted && term === '*') {
|
||||
return serializer.isNotNull(field, isNegatedField);
|
||||
}
|
||||
|
||||
if (!nodeTerm.quoted && term.substring(0, 2) === '>=') {
|
||||
if (isNegatedField) {
|
||||
return serializer.lt(field, term.slice(2));
|
||||
}
|
||||
return serializer.gte(field, term.slice(2));
|
||||
}
|
||||
if (!nodeTerm.quoted && term.substring(0, 2) === '<=') {
|
||||
if (isNegatedField) {
|
||||
return serializer.gt(field, term.slice(2));
|
||||
}
|
||||
return serializer.lte(field, term.slice(2));
|
||||
}
|
||||
if (!nodeTerm.quoted && term[0] === '>') {
|
||||
if (isNegatedField) {
|
||||
return serializer.lte(field, term.slice(1));
|
||||
}
|
||||
return serializer.gt(field, term.slice(1));
|
||||
}
|
||||
if (!nodeTerm.quoted && term[0] === '<') {
|
||||
if (isNegatedField) {
|
||||
return serializer.gte(field, term.slice(1));
|
||||
}
|
||||
return serializer.lt(field, term.slice(1));
|
||||
}
|
||||
|
||||
let prefixWildcard = false;
|
||||
let suffixWildcard = false;
|
||||
if (!nodeTerm.quoted && term[0] === '*') {
|
||||
prefixWildcard = true;
|
||||
term = term.slice(1);
|
||||
}
|
||||
if (!nodeTerm.quoted && term[term.length - 1] === '*') {
|
||||
suffixWildcard = true;
|
||||
term = term.slice(0, -1);
|
||||
}
|
||||
|
||||
return serializer.fieldSearch(
|
||||
field,
|
||||
term,
|
||||
isNegatedField,
|
||||
prefixWildcard,
|
||||
suffixWildcard,
|
||||
);
|
||||
|
||||
// TODO: Handle regex, similarity, boost, prefix
|
||||
}
|
||||
// NodeRangedTerm
|
||||
if ((node as lucene.NodeRangedTerm).inclusive != null) {
|
||||
const rangedTerm = node as lucene.NodeRangedTerm;
|
||||
return serializer.range(
|
||||
field,
|
||||
rangedTerm.term_min,
|
||||
rangedTerm.term_max,
|
||||
isNegatedField,
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected Node type. ${node}`);
|
||||
}
|
||||
|
||||
async function serialize(
|
||||
ast: lucene.AST | lucene.Node,
|
||||
serializer: Serializer,
|
||||
): Promise<string> {
|
||||
// Node Scenarios:
|
||||
// 1. NodeTerm: Single term ex. "foo:bar"
|
||||
// 2. NodeRangedTerm: Two terms ex. "foo:[bar TO qux]"
|
||||
if ((ast as lucene.NodeTerm).term != null) {
|
||||
return await nodeTerm(ast as lucene.NodeTerm, serializer);
|
||||
}
|
||||
if ((ast as lucene.NodeRangedTerm).inclusive != null) {
|
||||
return await nodeTerm(ast as lucene.NodeTerm, serializer);
|
||||
}
|
||||
|
||||
// AST Scenarios:
|
||||
// 1. BinaryAST: Two terms ex. "foo:bar AND baz:qux"
|
||||
// 2. LeftOnlyAST: Single term ex. "foo:bar"
|
||||
if ((ast as lucene.BinaryAST).right != null) {
|
||||
const binaryAST = ast as lucene.BinaryAST;
|
||||
const operator = serializer.operator(binaryAST.operator);
|
||||
const parenthesized = binaryAST.parenthesized;
|
||||
return `${parenthesized ? '(' : ''}${await serialize(
|
||||
binaryAST.left,
|
||||
serializer,
|
||||
)} ${operator} ${await serialize(binaryAST.right, serializer)}${
|
||||
parenthesized ? ')' : ''
|
||||
}`;
|
||||
}
|
||||
|
||||
if ((ast as lucene.LeftOnlyAST).left != null) {
|
||||
const leftOnlyAST = ast as lucene.LeftOnlyAST;
|
||||
const parenthesized = leftOnlyAST.parenthesized;
|
||||
// start is used when ex. "NOT foo:bar"
|
||||
return `${parenthesized ? '(' : ''}${
|
||||
leftOnlyAST.start != undefined ? `${leftOnlyAST.start} ` : ''
|
||||
}${await serialize(leftOnlyAST.left, serializer)}${
|
||||
parenthesized ? ')' : ''
|
||||
}`;
|
||||
}
|
||||
|
||||
// Blank AST, means no text was parsed
|
||||
return '';
|
||||
}
|
||||
|
||||
// TODO: can just inline this within getSearchQuery
|
||||
export async function genWhereSQL(
|
||||
ast: lucene.AST,
|
||||
serializer: Serializer,
|
||||
): Promise<string> {
|
||||
return await serialize(ast, serializer);
|
||||
}
|
||||
|
||||
export class SearchQueryBuilder {
|
||||
private readonly searchQ: string;
|
||||
|
||||
private readonly conditions: string[];
|
||||
|
||||
private serializer: SQLSerializer;
|
||||
|
||||
constructor(searchQ: string, serializer: SQLSerializer) {
|
||||
this.conditions = [];
|
||||
this.searchQ = searchQ;
|
||||
// init default serializer
|
||||
this.serializer = serializer;
|
||||
}
|
||||
|
||||
setSerializer(serializer: SQLSerializer) {
|
||||
this.serializer = serializer;
|
||||
return this;
|
||||
}
|
||||
|
||||
getSerializer() {
|
||||
return this.serializer;
|
||||
}
|
||||
|
||||
private async genSearchQuery() {
|
||||
if (!this.searchQ) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// const implicitColumn = await this.serializer.getColumnForField(
|
||||
// IMPLICIT_FIELD,
|
||||
// );
|
||||
|
||||
// let querySql = this.searchQ
|
||||
// .split(/\s+/)
|
||||
// .map(queryToken =>
|
||||
// SqlString.format(`lower(??) LIKE lower(?)`, [
|
||||
// implicitColumn.column,
|
||||
// `%${queryToken}%`,
|
||||
// ]),
|
||||
// )
|
||||
// .join(' AND ');
|
||||
|
||||
const parsedQ = parse(this.searchQ);
|
||||
|
||||
return await genWhereSQL(parsedQ, this.serializer);
|
||||
}
|
||||
|
||||
and(condition: string) {
|
||||
if (condition && condition.trim()) {
|
||||
this.conditions.push(`(${condition})`);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
async build() {
|
||||
const searchQuery = await this.genSearchQuery();
|
||||
if (this.searchQ) {
|
||||
this.and(searchQuery);
|
||||
}
|
||||
return this.conditions.join(' AND ');
|
||||
}
|
||||
}
|
||||
|
||||
export async function genEnglishExplanation(query: string): Promise<string> {
|
||||
try {
|
||||
const parsedQ = parse(query);
|
||||
|
||||
if (parsedQ) {
|
||||
const serializer = new EnglishSerializer();
|
||||
return await serialize(parsedQ, serializer);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Parse failure', query, e);
|
||||
}
|
||||
|
||||
return `Message containing ${query}`;
|
||||
}
|
||||
|
|
@ -1,768 +0,0 @@
|
|||
import isPlainObject from 'lodash/isPlainObject';
|
||||
import * as SQLParser from 'node-sql-parser';
|
||||
|
||||
import { convertDateRangeToGranularityString } from '@/ChartUtils';
|
||||
import { ChSql, chSql, concatChSql, wrapChSqlIfNotEmpty } from '@/clickhouse';
|
||||
import { DisplayType } from '@/DisplayType';
|
||||
import { Metadata, metadata } from '@/metadata';
|
||||
import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser';
|
||||
import { getFirstTimestampValueExpression } from '@/source';
|
||||
import {
|
||||
AggregateFunction,
|
||||
AggregateFunctionWithCombinators,
|
||||
SearchCondition,
|
||||
SearchConditionLanguage,
|
||||
SelectList,
|
||||
SelectSQLStatement,
|
||||
SortSpecificationList,
|
||||
SQLInterval,
|
||||
} from '@/sqlTypes';
|
||||
|
||||
// FIXME: SQLParser.ColumnRef is incomplete
|
||||
type ColumnRef = SQLParser.ColumnRef & {
|
||||
array_index?: {
|
||||
index: { type: string; value: string };
|
||||
}[];
|
||||
};
|
||||
|
||||
export type NumberFormat = {
|
||||
output?: 'currency' | 'percent' | 'byte' | 'time' | 'number';
|
||||
mantissa?: number;
|
||||
thousandSeparated?: boolean;
|
||||
average?: boolean;
|
||||
decimalBytes?: boolean;
|
||||
factor?: number;
|
||||
currencySymbol?: string;
|
||||
unit?: string;
|
||||
};
|
||||
|
||||
export type SqlAstFilter = {
|
||||
type: 'sql_ast';
|
||||
operator: '=' | '<' | '>' | '!=' | '<=' | '>=';
|
||||
// SQL Expressions
|
||||
left: string;
|
||||
right: string;
|
||||
};
|
||||
|
||||
export type Filter =
|
||||
| {
|
||||
type: 'lucene' | 'sql';
|
||||
condition: SearchCondition;
|
||||
}
|
||||
| SqlAstFilter;
|
||||
|
||||
// Used to actually query the data in a given chart
|
||||
export type ChartConfig = {
|
||||
displayType?: DisplayType;
|
||||
numberFormat?: NumberFormat;
|
||||
timestampValueExpression: string;
|
||||
implicitColumnExpression?: string; // Where lucene will search if given bare terms
|
||||
granularity?: SQLInterval | 'auto';
|
||||
markdown?: string; // Markdown Content
|
||||
filtersLogicalOperator?: 'AND' | 'OR'; // Default AND
|
||||
filters?: Filter[]; // Additional filters to where clause
|
||||
connection: string; // Connection ID
|
||||
fillNulls?: number | false; // undefined = 0, false = no fill
|
||||
selectGroupBy?: boolean; // Add groupBy elements to select statement (default behavior: true)
|
||||
// TODO: Color support
|
||||
} & SelectSQLStatement;
|
||||
|
||||
// Saved configuration, has a variable source ID that we pull at query time
|
||||
export type SavedChartConfig = {
|
||||
name: string;
|
||||
source: string;
|
||||
} & Omit<ChartConfig, 'timestampValueExpression' | 'from' | 'connection'>;
|
||||
|
||||
type DateRange = {
|
||||
dateRange: [Date, Date];
|
||||
dateRangeStartInclusive?: boolean; // default true
|
||||
};
|
||||
|
||||
export type ChartConfigWithDateRange = ChartConfig & DateRange;
|
||||
// For non-time-based searches (ex. grab 1 row)
|
||||
export type ChartConfigWithOptDateRange = Omit<
|
||||
ChartConfig,
|
||||
'timestampValueExpression'
|
||||
> & {
|
||||
timestampValueExpression?: string;
|
||||
} & Partial<DateRange>;
|
||||
|
||||
export const FIXED_TIME_BUCKET_EXPR_ALIAS = '__hdx_time_bucket';
|
||||
|
||||
export function isUsingGroupBy(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): chartConfig is Omit<ChartConfigWithDateRange, 'groupBy'> & {
|
||||
groupBy: NonNullable<ChartConfigWithDateRange['groupBy']>;
|
||||
} {
|
||||
return chartConfig.groupBy != null && chartConfig.groupBy.length > 0;
|
||||
}
|
||||
|
||||
function isUsingGranularity(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): chartConfig is Omit<
|
||||
Omit<Omit<ChartConfigWithDateRange, 'granularity'>, 'dateRange'>,
|
||||
'timestampValueExpression'
|
||||
> & {
|
||||
granularity: NonNullable<ChartConfigWithDateRange['granularity']>;
|
||||
dateRange: NonNullable<ChartConfigWithDateRange['dateRange']>;
|
||||
timestampValueExpression: NonNullable<
|
||||
ChartConfigWithDateRange['timestampValueExpression']
|
||||
>;
|
||||
} {
|
||||
return (
|
||||
chartConfig.timestampValueExpression != null &&
|
||||
chartConfig.granularity != null
|
||||
);
|
||||
}
|
||||
|
||||
const INVERSE_OPERATOR_MAP = {
|
||||
'=': '!=',
|
||||
'>': '<=',
|
||||
'<': '>=',
|
||||
|
||||
'!=': '=',
|
||||
'<=': '>',
|
||||
'>=': '<',
|
||||
} as const;
|
||||
export function inverseSqlAstFilter(filter: SqlAstFilter): SqlAstFilter {
|
||||
return {
|
||||
...filter,
|
||||
operator:
|
||||
INVERSE_OPERATOR_MAP[
|
||||
filter.operator as keyof typeof INVERSE_OPERATOR_MAP
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function isNonEmptyWhereExpr(where?: string): where is string {
|
||||
return where != null && where.trim() != '';
|
||||
}
|
||||
|
||||
const fastifySQL = ({
|
||||
materializedFields,
|
||||
rawSQL,
|
||||
}: {
|
||||
materializedFields: Map<string, string>;
|
||||
rawSQL: string;
|
||||
}) => {
|
||||
// Parse the SQL AST
|
||||
try {
|
||||
const parser = new SQLParser.Parser();
|
||||
const ast = parser.astify(rawSQL, {
|
||||
database: 'Postgresql',
|
||||
}) as SQLParser.Select;
|
||||
|
||||
// traveral ast and replace the left node with the materialized field
|
||||
// FIXME: type node (AST type is incomplete): https://github.com/taozhi8833998/node-sql-parser/blob/42ea0b1800c5d425acb8c5ca708a1cee731aada8/types.d.ts#L474
|
||||
const traverse = (
|
||||
node:
|
||||
| SQLParser.Expr
|
||||
| SQLParser.ExpressionValue
|
||||
| SQLParser.ExprList
|
||||
| SQLParser.Function
|
||||
| null,
|
||||
) => {
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
let colExpr;
|
||||
switch (node.type) {
|
||||
case 'column_ref': {
|
||||
// FIXME: handle 'Value' type?
|
||||
const _n = node as ColumnRef;
|
||||
if (typeof _n.column !== 'string') {
|
||||
colExpr = `${_n.column?.expr.value}['${_n.array_index?.[0]?.index.value}']`;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'binary_expr': {
|
||||
const _n = node as SQLParser.Expr;
|
||||
if (Array.isArray(_n.left)) {
|
||||
for (const left of _n.left) {
|
||||
traverse(left);
|
||||
}
|
||||
} else {
|
||||
traverse(_n.left);
|
||||
}
|
||||
|
||||
if (Array.isArray(_n.right)) {
|
||||
for (const right of _n.right) {
|
||||
traverse(right);
|
||||
}
|
||||
} else {
|
||||
traverse(_n.right);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'function': {
|
||||
const _n = node as SQLParser.Function;
|
||||
|
||||
if (_n.args?.type === 'expr_list') {
|
||||
if (Array.isArray(_n.args?.value)) {
|
||||
for (const arg of _n.args.value) {
|
||||
traverse(arg);
|
||||
}
|
||||
|
||||
// ex: JSONExtractString(Body, 'message')
|
||||
if (
|
||||
_n.args?.value?.[0]?.type === 'column_ref' &&
|
||||
_n.args?.value?.[1]?.type === 'single_quote_string'
|
||||
) {
|
||||
colExpr = `${_n.name?.name?.[0]?.value}(${(_n.args?.value?.[0] as any)?.column.expr.value}, '${_n.args?.value?.[1]?.value}')`;
|
||||
}
|
||||
}
|
||||
// when _n.args?.value is Expr
|
||||
else if (isPlainObject(_n.args?.value)) {
|
||||
traverse(_n.args.value);
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
default:
|
||||
// ignore other types
|
||||
break;
|
||||
}
|
||||
|
||||
if (colExpr) {
|
||||
const materializedField = materializedFields.get(colExpr);
|
||||
if (materializedField) {
|
||||
const _n = node as ColumnRef;
|
||||
// reset the node ref
|
||||
for (const key in _n) {
|
||||
// eslint-disable-next-line no-prototype-builtins
|
||||
if (_n.hasOwnProperty(key)) {
|
||||
// @ts-ignore
|
||||
delete _n[key];
|
||||
}
|
||||
}
|
||||
_n.type = 'column_ref';
|
||||
_n.table = null;
|
||||
_n.column = { expr: { type: 'default', value: materializedField } };
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (Array.isArray(ast.columns)) {
|
||||
for (const col of ast.columns) {
|
||||
traverse(col.expr);
|
||||
}
|
||||
}
|
||||
|
||||
traverse(ast.where);
|
||||
|
||||
return parser.sqlify(ast);
|
||||
} catch (e) {
|
||||
console.error('[renderWhereExpression]feat: Failed to parse SQL AST', e);
|
||||
return rawSQL;
|
||||
}
|
||||
};
|
||||
|
||||
const aggFnExpr = ({
|
||||
fn,
|
||||
expr,
|
||||
quantileLevel,
|
||||
where,
|
||||
}: {
|
||||
fn: AggregateFunction | AggregateFunctionWithCombinators;
|
||||
expr?: string;
|
||||
quantileLevel?: number;
|
||||
where?: string;
|
||||
}) => {
|
||||
const isCount = fn.startsWith('count');
|
||||
const isWhereUsed = isNonEmptyWhereExpr(where);
|
||||
// Cast to float64 because the expr might not be a number
|
||||
const unsafeExpr = { UNSAFE_RAW_SQL: `toFloat64OrNull(toString(${expr}))` };
|
||||
const whereWithExtraNullCheck = `${where} AND ${unsafeExpr.UNSAFE_RAW_SQL} IS NOT NULL`;
|
||||
|
||||
if (fn.endsWith('Merge')) {
|
||||
return chSql`${fn}(${{
|
||||
UNSAFE_RAW_SQL: expr ?? '',
|
||||
}})`;
|
||||
}
|
||||
// TODO: merge this chunk with the rest of logics
|
||||
else if (fn.endsWith('State')) {
|
||||
if (expr == null || isCount) {
|
||||
return isWhereUsed
|
||||
? chSql`${fn}(${{ UNSAFE_RAW_SQL: where }})`
|
||||
: chSql`${fn}()`;
|
||||
}
|
||||
return chSql`${fn}(${unsafeExpr}${
|
||||
isWhereUsed ? chSql`, ${{ UNSAFE_RAW_SQL: whereWithExtraNullCheck }}` : ''
|
||||
})`;
|
||||
}
|
||||
|
||||
if (fn === 'count') {
|
||||
if (isWhereUsed) {
|
||||
return chSql`${fn}If(${{ UNSAFE_RAW_SQL: where }})`;
|
||||
}
|
||||
return {
|
||||
sql: `${fn}()`,
|
||||
params: {},
|
||||
};
|
||||
}
|
||||
|
||||
if (expr != null) {
|
||||
if (fn === 'count_distinct') {
|
||||
return chSql`count${isWhereUsed ? 'If' : ''}(DISTINCT ${{
|
||||
UNSAFE_RAW_SQL: expr,
|
||||
}}${isWhereUsed ? chSql`, ${{ UNSAFE_RAW_SQL: where }}` : ''})`;
|
||||
}
|
||||
|
||||
if (quantileLevel != null) {
|
||||
return chSql`quantile${isWhereUsed ? 'If' : ''}(${{
|
||||
// Using Float64 param leads to an added coersion, but we don't need to
|
||||
// escape number values anyways
|
||||
UNSAFE_RAW_SQL: Number.isFinite(quantileLevel)
|
||||
? `${quantileLevel}`
|
||||
: '0',
|
||||
}})(${unsafeExpr}${
|
||||
isWhereUsed
|
||||
? chSql`, ${{ UNSAFE_RAW_SQL: whereWithExtraNullCheck }}`
|
||||
: ''
|
||||
})`;
|
||||
}
|
||||
|
||||
// TODO: Verify fn is a safe/valid function
|
||||
return chSql`${{ UNSAFE_RAW_SQL: fn }}${isWhereUsed ? 'If' : ''}(
|
||||
${unsafeExpr}${isWhereUsed ? chSql`, ${{ UNSAFE_RAW_SQL: whereWithExtraNullCheck }}` : ''}
|
||||
)`;
|
||||
} else {
|
||||
throw new Error(
|
||||
'Column is required for all non-count aggregation functions',
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
async function renderSelectList(
|
||||
selectList: SelectList,
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
metadata: Metadata,
|
||||
) {
|
||||
if (typeof selectList === 'string') {
|
||||
return chSql`${{ UNSAFE_RAW_SQL: selectList }}`;
|
||||
}
|
||||
|
||||
const materializedFields = await metadata.getMaterializedColumnsLookupTable({
|
||||
connectionId: chartConfig.connection,
|
||||
databaseName: chartConfig.from.databaseName,
|
||||
tableName: chartConfig.from.tableName,
|
||||
});
|
||||
|
||||
return Promise.all(
|
||||
selectList.map(async select => {
|
||||
const whereClause = await renderWhereExpression({
|
||||
condition: select.aggCondition ?? '',
|
||||
from: chartConfig.from,
|
||||
language: select.aggConditionLanguage ?? 'lucene',
|
||||
implicitColumnExpression: chartConfig.implicitColumnExpression,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
});
|
||||
|
||||
let expr: ChSql;
|
||||
if (select.aggFn == null) {
|
||||
expr = chSql`${{ UNSAFE_RAW_SQL: select.valueExpression }}`;
|
||||
} else if (select.aggFn === 'quantile') {
|
||||
expr = aggFnExpr({
|
||||
fn: select.aggFn,
|
||||
expr: select.valueExpression,
|
||||
// @ts-ignore (TS doesn't know that we've already checked for quantile)
|
||||
quantileLevel: select.level,
|
||||
where: whereClause.sql,
|
||||
});
|
||||
} else {
|
||||
expr = aggFnExpr({
|
||||
fn: select.aggFn,
|
||||
expr: select.valueExpression,
|
||||
where: whereClause.sql,
|
||||
});
|
||||
}
|
||||
|
||||
const rawSQL = `SELECT ${expr.sql} FROM \`t\``;
|
||||
// strip 'SELECT * FROM `t` WHERE ' from the sql
|
||||
expr.sql = fastifySQL({ materializedFields, rawSQL })
|
||||
.replace(/^SELECT\s+/i, '') // Remove 'SELECT ' from the start
|
||||
.replace(/\s+FROM `t`$/i, ''); // Remove ' FROM t' from the end
|
||||
|
||||
return chSql`${expr}${
|
||||
select.alias != null
|
||||
? chSql` AS \`${{ UNSAFE_RAW_SQL: select.alias }}\``
|
||||
: []
|
||||
}`;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function renderSortSpecificationList(
|
||||
sortSpecificationList: SortSpecificationList,
|
||||
) {
|
||||
if (typeof sortSpecificationList === 'string') {
|
||||
return chSql`${{ UNSAFE_RAW_SQL: sortSpecificationList }}`;
|
||||
}
|
||||
|
||||
return sortSpecificationList.map(sortSpecification => {
|
||||
return chSql`${{ UNSAFE_RAW_SQL: sortSpecification.valueExpression }} ${
|
||||
sortSpecification.ordering === 'DESC' ? 'DESC' : 'ASC'
|
||||
}`;
|
||||
});
|
||||
}
|
||||
|
||||
function timeBucketExpr({
|
||||
interval,
|
||||
timestampValueExpression,
|
||||
dateRange,
|
||||
alias = FIXED_TIME_BUCKET_EXPR_ALIAS,
|
||||
}: {
|
||||
interval: SQLInterval | 'auto';
|
||||
timestampValueExpression: string;
|
||||
dateRange?: [Date, Date];
|
||||
alias?: string;
|
||||
}) {
|
||||
const unsafeTimestampValueExpression = {
|
||||
UNSAFE_RAW_SQL: getFirstTimestampValueExpression(timestampValueExpression),
|
||||
};
|
||||
const unsafeInterval = {
|
||||
UNSAFE_RAW_SQL:
|
||||
interval === 'auto' && Array.isArray(dateRange)
|
||||
? convertDateRangeToGranularityString(dateRange, 60)
|
||||
: interval,
|
||||
};
|
||||
|
||||
return chSql`toStartOfInterval(toDateTime(${unsafeTimestampValueExpression}), INTERVAL ${unsafeInterval}) AS \`${{
|
||||
UNSAFE_RAW_SQL: alias,
|
||||
}}\``;
|
||||
}
|
||||
|
||||
async function timeFilterExpr({
|
||||
timestampValueExpression,
|
||||
dateRange,
|
||||
dateRangeStartInclusive,
|
||||
databaseName,
|
||||
tableName,
|
||||
metadata,
|
||||
connectionId,
|
||||
}: {
|
||||
timestampValueExpression: string;
|
||||
dateRange: [Date, Date];
|
||||
dateRangeStartInclusive: boolean;
|
||||
metadata: Metadata;
|
||||
connectionId: string;
|
||||
databaseName: string;
|
||||
tableName: string;
|
||||
}) {
|
||||
const valueExpressions = timestampValueExpression.split(',');
|
||||
const startTime = dateRange[0].getTime();
|
||||
const endTime = dateRange[1].getTime();
|
||||
|
||||
const whereExprs = await Promise.all(
|
||||
valueExpressions.map(async expr => {
|
||||
const col = expr.trim();
|
||||
const columnMeta = await metadata.getColumn({
|
||||
databaseName,
|
||||
tableName,
|
||||
column: col,
|
||||
connectionId,
|
||||
});
|
||||
|
||||
const unsafeTimestampValueExpression = {
|
||||
UNSAFE_RAW_SQL: col,
|
||||
};
|
||||
|
||||
if (columnMeta == null) {
|
||||
console.warn(
|
||||
`Column ${col} not found in ${databaseName}.${tableName} while inferring type for time filter`,
|
||||
);
|
||||
}
|
||||
|
||||
// If it's a date type
|
||||
if (columnMeta?.type === 'Date') {
|
||||
return chSql`(${unsafeTimestampValueExpression} ${
|
||||
dateRangeStartInclusive ? '>=' : '>'
|
||||
} toDate(fromUnixTimestamp64Milli(${{
|
||||
Int64: startTime,
|
||||
}})) AND ${unsafeTimestampValueExpression} <= toDate(fromUnixTimestamp64Milli(${{
|
||||
Int64: endTime,
|
||||
}})))`;
|
||||
} else {
|
||||
return chSql`(${unsafeTimestampValueExpression} ${
|
||||
dateRangeStartInclusive ? '>=' : '>'
|
||||
} fromUnixTimestamp64Milli(${{
|
||||
Int64: startTime,
|
||||
}}) AND ${unsafeTimestampValueExpression} <= fromUnixTimestamp64Milli(${{
|
||||
Int64: endTime,
|
||||
}}))`;
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
return concatChSql('AND', ...whereExprs);
|
||||
}
|
||||
|
||||
async function renderSelect(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
metadata: Metadata,
|
||||
): Promise<ChSql> {
|
||||
/**
|
||||
* SELECT
|
||||
* if granularity: toStartOfInterval,
|
||||
* if groupBy: groupBy,
|
||||
* select
|
||||
*/
|
||||
const isIncludingTimeBucket = isUsingGranularity(chartConfig);
|
||||
const isIncludingGroupBy = isUsingGroupBy(chartConfig);
|
||||
|
||||
// TODO: clean up these await mess
|
||||
return concatChSql(
|
||||
',',
|
||||
await renderSelectList(chartConfig.select, chartConfig, metadata),
|
||||
isIncludingGroupBy && chartConfig.selectGroupBy !== false
|
||||
? await renderSelectList(chartConfig.groupBy, chartConfig, metadata)
|
||||
: [],
|
||||
isIncludingTimeBucket
|
||||
? timeBucketExpr({
|
||||
interval: chartConfig.granularity,
|
||||
timestampValueExpression: chartConfig.timestampValueExpression,
|
||||
dateRange: chartConfig.dateRange,
|
||||
})
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
function renderFrom({
|
||||
from,
|
||||
}: {
|
||||
from: ChartConfigWithDateRange['from'];
|
||||
}): ChSql {
|
||||
return chSql`${{ Identifier: from.databaseName }}.${{
|
||||
Identifier: from.tableName,
|
||||
}}`;
|
||||
}
|
||||
|
||||
async function renderWhereExpression({
|
||||
condition,
|
||||
language,
|
||||
metadata,
|
||||
from,
|
||||
implicitColumnExpression,
|
||||
connectionId,
|
||||
}: {
|
||||
condition: SearchCondition;
|
||||
language: SearchConditionLanguage;
|
||||
metadata: Metadata;
|
||||
from: ChartConfigWithDateRange['from'];
|
||||
implicitColumnExpression?: string;
|
||||
connectionId: string;
|
||||
}): Promise<ChSql> {
|
||||
let _condition = condition;
|
||||
if (language === 'lucene') {
|
||||
const serializer = new CustomSchemaSQLSerializerV2({
|
||||
metadata,
|
||||
databaseName: from.databaseName,
|
||||
tableName: from.tableName,
|
||||
implicitColumnExpression,
|
||||
connectionId: connectionId,
|
||||
});
|
||||
const builder = new SearchQueryBuilder(condition, serializer);
|
||||
_condition = await builder.build();
|
||||
}
|
||||
|
||||
const materializedFields = await metadata.getMaterializedColumnsLookupTable({
|
||||
connectionId,
|
||||
databaseName: from.databaseName,
|
||||
tableName: from.tableName,
|
||||
});
|
||||
|
||||
const _sqlPrefix = 'SELECT * FROM `t` WHERE ';
|
||||
const rawSQL = `${_sqlPrefix}${_condition}`;
|
||||
// strip 'SELECT * FROM `t` WHERE ' from the sql
|
||||
_condition = fastifySQL({ materializedFields, rawSQL }).replace(
|
||||
_sqlPrefix,
|
||||
'',
|
||||
);
|
||||
return chSql`${{ UNSAFE_RAW_SQL: _condition }}`;
|
||||
}
|
||||
|
||||
async function renderWhere(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
metadata: Metadata,
|
||||
): Promise<ChSql> {
|
||||
let whereSearchCondition: ChSql | [] = [];
|
||||
if (isNonEmptyWhereExpr(chartConfig.where)) {
|
||||
whereSearchCondition = wrapChSqlIfNotEmpty(
|
||||
await renderWhereExpression({
|
||||
condition: chartConfig.where,
|
||||
from: chartConfig.from,
|
||||
language: chartConfig.whereLanguage ?? 'sql',
|
||||
implicitColumnExpression: chartConfig.implicitColumnExpression,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
}),
|
||||
'(',
|
||||
')',
|
||||
);
|
||||
}
|
||||
|
||||
let selectSearchConditions: ChSql[] = [];
|
||||
if (
|
||||
typeof chartConfig.select != 'string' &&
|
||||
// Only if every select has an aggCondition, add to where clause
|
||||
// otherwise we'll scan all rows anyways
|
||||
chartConfig.select.every(select => isNonEmptyWhereExpr(select.aggCondition))
|
||||
) {
|
||||
selectSearchConditions = (
|
||||
await Promise.all(
|
||||
chartConfig.select.map(async select => {
|
||||
if (isNonEmptyWhereExpr(select.aggCondition)) {
|
||||
return await renderWhereExpression({
|
||||
condition: select.aggCondition,
|
||||
from: chartConfig.from,
|
||||
language: select.aggConditionLanguage ?? 'sql',
|
||||
implicitColumnExpression: chartConfig.implicitColumnExpression,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
)
|
||||
).filter(v => v !== null) as ChSql[];
|
||||
}
|
||||
|
||||
const filterConditions = await Promise.all(
|
||||
(chartConfig.filters ?? []).map(async filter => {
|
||||
if (filter.type === 'sql_ast') {
|
||||
return wrapChSqlIfNotEmpty(
|
||||
chSql`${{ UNSAFE_RAW_SQL: filter.left }} ${filter.operator} ${{ UNSAFE_RAW_SQL: filter.right }}`,
|
||||
'(',
|
||||
')',
|
||||
);
|
||||
} else if (filter.type === 'lucene' || filter.type === 'sql') {
|
||||
return wrapChSqlIfNotEmpty(
|
||||
await renderWhereExpression({
|
||||
condition: filter.condition,
|
||||
from: chartConfig.from,
|
||||
language: filter.type,
|
||||
implicitColumnExpression: chartConfig.implicitColumnExpression,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
}),
|
||||
'(',
|
||||
')',
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(`Unknown filter type: ${filter.type}`);
|
||||
}),
|
||||
);
|
||||
|
||||
return concatChSql(
|
||||
' AND ',
|
||||
chartConfig.dateRange != null &&
|
||||
chartConfig.timestampValueExpression != null
|
||||
? await timeFilterExpr({
|
||||
timestampValueExpression: chartConfig.timestampValueExpression,
|
||||
dateRange: chartConfig.dateRange,
|
||||
dateRangeStartInclusive: chartConfig.dateRangeStartInclusive ?? true,
|
||||
metadata,
|
||||
connectionId: chartConfig.connection,
|
||||
databaseName: chartConfig.from.databaseName,
|
||||
tableName: chartConfig.from.tableName,
|
||||
})
|
||||
: [],
|
||||
whereSearchCondition,
|
||||
// Add aggConditions to where clause to utilize index
|
||||
wrapChSqlIfNotEmpty(concatChSql(' OR ', selectSearchConditions), '(', ')'),
|
||||
wrapChSqlIfNotEmpty(
|
||||
concatChSql(
|
||||
chartConfig.filtersLogicalOperator === 'OR' ? ' OR ' : ' AND ',
|
||||
...filterConditions,
|
||||
),
|
||||
'(',
|
||||
')',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async function renderGroupBy(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
metadata: Metadata,
|
||||
): Promise<ChSql | undefined> {
|
||||
return concatChSql(
|
||||
',',
|
||||
isUsingGroupBy(chartConfig)
|
||||
? await renderSelectList(chartConfig.groupBy, chartConfig, metadata)
|
||||
: [],
|
||||
isUsingGranularity(chartConfig)
|
||||
? timeBucketExpr({
|
||||
interval: chartConfig.granularity,
|
||||
timestampValueExpression: chartConfig.timestampValueExpression,
|
||||
dateRange: chartConfig.dateRange,
|
||||
})
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
function renderOrderBy(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): ChSql | undefined {
|
||||
const isIncludingTimeBucket = isUsingGranularity(chartConfig);
|
||||
|
||||
if (chartConfig.orderBy == null && !isIncludingTimeBucket) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return concatChSql(
|
||||
',',
|
||||
isIncludingTimeBucket
|
||||
? timeBucketExpr({
|
||||
interval: chartConfig.granularity,
|
||||
timestampValueExpression: chartConfig.timestampValueExpression,
|
||||
dateRange: chartConfig.dateRange,
|
||||
})
|
||||
: [],
|
||||
chartConfig.orderBy != null
|
||||
? renderSortSpecificationList(chartConfig.orderBy)
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
function renderLimit(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): ChSql | undefined {
|
||||
if (chartConfig.limit == null || chartConfig.limit.limit == null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const offset =
|
||||
chartConfig.limit.offset != null
|
||||
? chSql` OFFSET ${{ Int32: chartConfig.limit.offset }}`
|
||||
: [];
|
||||
|
||||
return chSql`${{ Int32: chartConfig.limit.limit }}${offset}`;
|
||||
}
|
||||
|
||||
export async function renderChartConfig(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
): Promise<ChSql> {
|
||||
const select = await renderSelect(chartConfig, metadata);
|
||||
const from = renderFrom(chartConfig);
|
||||
const where = await renderWhere(chartConfig, metadata);
|
||||
const groupBy = await renderGroupBy(chartConfig, metadata);
|
||||
const orderBy = renderOrderBy(chartConfig);
|
||||
const limit = renderLimit(chartConfig);
|
||||
|
||||
return chSql`SELECT ${select} FROM ${from} ${where?.sql ? chSql`WHERE ${where}` : ''} ${
|
||||
groupBy?.sql ? chSql`GROUP BY ${groupBy}` : ''
|
||||
} ${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''} ${
|
||||
limit?.sql ? chSql`LIMIT ${limit}` : ''
|
||||
}`;
|
||||
}
|
||||
|
||||
// EditForm -> translateToQueriedChartConfig -> QueriedChartConfig
|
||||
// renderFn(QueriedChartConfig) -> sql
|
||||
// query(sql) -> data
|
||||
// formatter(data) -> displayspecificDs
|
||||
// displaySettings(QueriedChartConfig) -> displaySepcificDs
|
||||
// chartComponent(displayspecificDs) -> React.Node
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import { z } from 'zod';
|
||||
import { SavedSearch } from '@hyperdx/common-utils/dist/types';
|
||||
import {
|
||||
useMutation,
|
||||
useQuery,
|
||||
|
|
@ -7,7 +8,6 @@ import {
|
|||
} from '@tanstack/react-query';
|
||||
|
||||
import { hdxServer } from './api';
|
||||
import { SavedSearch } from './commonTypes';
|
||||
import { IS_LOCAL_MODE } from './config';
|
||||
|
||||
export function useSavedSearches() {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import React from 'react';
|
||||
import produce from 'immer';
|
||||
|
||||
import { Filter } from './renderChartConfig';
|
||||
import type { Filter } from '@hyperdx/common-utils/dist/renderChartConfig';
|
||||
|
||||
export type FilterState = {
|
||||
[key: string]: Set<string>;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { TSource } from '@/commonTypes';
|
||||
import { TSource } from '@hyperdx/common-utils/dist/types';
|
||||
|
||||
function getDefaults() {
|
||||
const spanAttributeField = 'SpanAttributes';
|
||||
|
|
|
|||
|
|
@ -1,20 +1,21 @@
|
|||
import omit from 'lodash/omit';
|
||||
import objectHash from 'object-hash';
|
||||
import store from 'store2';
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
|
||||
import { hdxServer } from '@/api';
|
||||
import {
|
||||
ColumnMeta,
|
||||
extractColumnReference,
|
||||
filterColumnMetaByType,
|
||||
JSDataType,
|
||||
} from '@/clickhouse';
|
||||
import { TSource } from '@/commonTypes';
|
||||
} from '@hyperdx/common-utils/dist/clickhouse';
|
||||
import { TSource } from '@hyperdx/common-utils/dist/types';
|
||||
import { hashCode } from '@hyperdx/common-utils/dist/utils';
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
|
||||
import { hdxServer } from '@/api';
|
||||
import { HDX_LOCAL_DEFAULT_SOURCES } from '@/config';
|
||||
import { IS_LOCAL_MODE } from '@/config';
|
||||
import { metadata } from '@/metadata';
|
||||
import { hashCode, parseJSON } from '@/utils';
|
||||
import { getMetadata } from '@/metadata';
|
||||
import { parseJSON } from '@/utils';
|
||||
|
||||
const LOCAL_STORE_SOUCES_KEY = 'hdx-local-source';
|
||||
|
||||
|
|
@ -199,6 +200,7 @@ export async function inferTableSourceConfig({
|
|||
}): Promise<
|
||||
Partial<Omit<TSource, 'id' | 'name' | 'from' | 'connection' | 'kind'>>
|
||||
> {
|
||||
const metadata = getMetadata();
|
||||
const columns = await metadata.getColumns({
|
||||
databaseName,
|
||||
tableName,
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
// Derived from SQL grammar spec
|
||||
// See: https://ronsavage.github.io/SQL/sql-2003-2.bnf.html#query%20specification
|
||||
|
||||
export type SQLInterval =
|
||||
| `${number} second`
|
||||
| `${number} minute`
|
||||
| `${number} hour`
|
||||
| `${number} day`;
|
||||
|
||||
export type SearchCondition = string;
|
||||
export type SearchConditionLanguage = 'sql' | 'lucene' | undefined;
|
||||
export type AggregateFunction =
|
||||
| 'avg'
|
||||
| 'count'
|
||||
| 'count_distinct'
|
||||
| 'max'
|
||||
| 'min'
|
||||
| 'quantile'
|
||||
| 'sum';
|
||||
export type AggregateFunctionWithCombinators =
|
||||
| `${AggregateFunction}If`
|
||||
| `${AggregateFunction}IfState`
|
||||
| `${AggregateFunction}IfMerge`
|
||||
| `${AggregateFunction}State`
|
||||
| `${AggregateFunction}Merge`;
|
||||
type RootValueExpression =
|
||||
| {
|
||||
aggFn: AggregateFunction | AggregateFunctionWithCombinators;
|
||||
aggCondition: SearchCondition;
|
||||
aggConditionLanguage?: SearchConditionLanguage;
|
||||
valueExpression: string;
|
||||
}
|
||||
| {
|
||||
aggFn: 'quantile';
|
||||
level: number;
|
||||
aggCondition: SearchCondition;
|
||||
aggConditionLanguage?: SearchConditionLanguage;
|
||||
valueExpression: string;
|
||||
}
|
||||
| {
|
||||
aggFn?: undefined;
|
||||
aggCondition?: undefined;
|
||||
aggConditionLanguage?: undefined;
|
||||
valueExpression: string; // always wrapped by aggFn, ex: col + 5, can contain aggregation functions ex. sum(col) + 5 with undefined aggregation
|
||||
};
|
||||
|
||||
export type DerivedColumn = RootValueExpression & { alias?: string }; // AS myColName
|
||||
|
||||
export type SelectList = DerivedColumn[] | string; // Serialized Select List
|
||||
|
||||
type SortSpecification = RootValueExpression & { ordering: 'ASC' | 'DESC' };
|
||||
export type SortSpecificationList = SortSpecification[] | string;
|
||||
|
||||
type Limit = { limit?: number; offset?: number };
|
||||
|
||||
export type SelectSQLStatement = {
|
||||
select: SelectList;
|
||||
from: { databaseName: string; tableName: string };
|
||||
where: SearchCondition;
|
||||
whereLanguage?: SearchConditionLanguage;
|
||||
groupBy?: SelectList;
|
||||
having?: SearchCondition;
|
||||
havingLanguage?: SearchConditionLanguage;
|
||||
orderBy?: SortSpecificationList;
|
||||
limit?: Limit;
|
||||
};
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
import { z } from 'zod';
|
||||
|
||||
import { DashboardSchema, SavedSearchSchema } from './commonTypes';
|
||||
import {
|
||||
DashboardSchema,
|
||||
SavedSearchSchema,
|
||||
} from '@hyperdx/common-utils/dist/types';
|
||||
|
||||
export type Team = {
|
||||
allowedAuthMethods: any[];
|
||||
|
|
|
|||
5
packages/common-utils/README.md
Normal file
5
packages/common-utils/README.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
## How to test the package with HyperDX app (browser) locally
|
||||
|
||||
1. Run `yarn dev` in the root directory of this project (packages/common-utils)
|
||||
2. Run `yarn dev:local` in the root directory of app project (packages/app)
|
||||
3. You should be able to test utils with the app
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "@hyperdx/common-utils",
|
||||
"description": "Common utilities for HyperDX application",
|
||||
"version": "0.0.1",
|
||||
"version": "0.0.9",
|
||||
"license": "MIT",
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
|
|
@ -13,7 +13,9 @@
|
|||
"node": ">=18.12.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@clickhouse/client-common": "^1.9.1",
|
||||
"@clickhouse/client": "^1.10.1",
|
||||
"@clickhouse/client-common": "^1.10.1",
|
||||
"@clickhouse/client-web": "^1.10.1",
|
||||
"@hyperdx/lucene": "^3.1.1",
|
||||
"date-fns": "^2.28.0",
|
||||
"date-fns-tz": "^2.0.0",
|
||||
|
|
@ -22,6 +24,7 @@
|
|||
"object-hash": "^3.0.0",
|
||||
"semver": "^7.5.2",
|
||||
"sqlstring": "^2.3.3",
|
||||
"store2": "^2.14.4",
|
||||
"uuid": "^8.3.2",
|
||||
"zod": "^3.24.1"
|
||||
},
|
||||
|
|
@ -46,11 +49,13 @@
|
|||
"typescript": "^4.9.5"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "tsup --watch",
|
||||
"build": "tsup",
|
||||
"ci:build": "tsup",
|
||||
"lint": "eslint --quiet . --ext .ts",
|
||||
"lint:fix": "eslint . --ext .ts --fix",
|
||||
"ci:lint": "yarn lint && yarn tsc --noEmit",
|
||||
"ci:int": "jest --runInBand --ci --forceExit --coverage",
|
||||
"dev:int": "jest --watchAll --runInBand --detectOpenHandles"
|
||||
"ci:unit": "jest --runInBand --ci --forceExit --coverage",
|
||||
"dev:unit": "jest --watchAll --runInBand --detectOpenHandles"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,14 +1,13 @@
|
|||
import {
|
||||
import type {
|
||||
BaseResultSet,
|
||||
DataFormat,
|
||||
isSuccessfulResponse,
|
||||
ResponseHeaders,
|
||||
ResponseJSON,
|
||||
} from '@clickhouse/client-common';
|
||||
import { isSuccessfulResponse } from '@clickhouse/client-common';
|
||||
|
||||
import { SQLInterval } from '@/types';
|
||||
import { hashCode, timeBucketByGranularity } from '@/utils';
|
||||
|
||||
export const PROXY_CLICKHOUSE_HOST = '/api/clickhouse-proxy';
|
||||
import { hashCode, isBrowser, isNode, timeBucketByGranularity } from '@/utils';
|
||||
|
||||
export enum JSDataType {
|
||||
Array = 'array',
|
||||
|
|
@ -19,6 +18,14 @@ export enum JSDataType {
|
|||
Bool = 'bool',
|
||||
}
|
||||
|
||||
export const getResponseHeaders = (response: Response): ResponseHeaders => {
|
||||
const headers: ResponseHeaders = {};
|
||||
response.headers.forEach((value, key) => {
|
||||
headers[key] = value;
|
||||
});
|
||||
return headers;
|
||||
};
|
||||
|
||||
export const convertCHDataTypeToJSType = (
|
||||
dataType: string,
|
||||
): JSDataType | null => {
|
||||
|
|
@ -233,18 +240,30 @@ export function extractColumnReference(
|
|||
return iterations < maxIterations ? sql.trim() : null;
|
||||
}
|
||||
|
||||
export const client = {
|
||||
export type ClickhouseClientOptions = {
|
||||
host: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
};
|
||||
|
||||
export class ClickhouseClient {
|
||||
private readonly host: string;
|
||||
private readonly username?: string;
|
||||
private readonly password?: string;
|
||||
|
||||
constructor({ host, username, password }: ClickhouseClientOptions) {
|
||||
this.host = host;
|
||||
this.username = username;
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
// https://github.com/ClickHouse/clickhouse-js/blob/1ebdd39203730bb99fad4c88eac35d9a5e96b34a/packages/client-web/src/connection/web_connection.ts#L151
|
||||
async query<T extends DataFormat>({
|
||||
query,
|
||||
format = 'JSON',
|
||||
query_params = {},
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
host,
|
||||
username,
|
||||
password,
|
||||
includeCredentials,
|
||||
includeCorsHeader,
|
||||
connectionId,
|
||||
queryId,
|
||||
}: {
|
||||
|
|
@ -253,24 +272,24 @@ export const client = {
|
|||
abort_signal?: AbortSignal;
|
||||
query_params?: Record<string, any>;
|
||||
clickhouse_settings?: Record<string, any>;
|
||||
host?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
includeCredentials: boolean;
|
||||
includeCorsHeader: boolean;
|
||||
connectionId?: string;
|
||||
queryId?: string;
|
||||
}): Promise<BaseResultSet<any, T>> {
|
||||
const isLocalMode = this.username != null && this.password != null;
|
||||
const includeCredentials = !isLocalMode;
|
||||
const includeCorsHeader = isLocalMode;
|
||||
const _connectionId = isLocalMode ? undefined : connectionId;
|
||||
|
||||
const searchParams = new URLSearchParams([
|
||||
...(includeCorsHeader ? [['add_http_cors_header', '1']] : []),
|
||||
...(connectionId ? [['hyperdx_connection_id', connectionId]] : []),
|
||||
...(_connectionId ? [['hyperdx_connection_id', _connectionId]] : []),
|
||||
['query', query],
|
||||
['default_format', format],
|
||||
['date_time_output_format', 'iso'],
|
||||
['wait_end_of_query', '0'],
|
||||
['cancel_http_readonly_queries_on_client_close', '1'],
|
||||
...(username ? [['user', username]] : []),
|
||||
...(password ? [['password', password]] : []),
|
||||
...(this.username ? [['user', this.username]] : []),
|
||||
...(this.password ? [['password', this.password]] : []),
|
||||
...(queryId ? [['query_id', queryId]] : []),
|
||||
...Object.entries(query_params).map(([key, value]) => [
|
||||
`param_${key}`,
|
||||
|
|
@ -296,29 +315,63 @@ export const client = {
|
|||
// eslint-disable-next-line no-console
|
||||
console.log('--------------------------------------------------------');
|
||||
|
||||
const res = await fetch(`${host}/?${searchParams.toString()}`, {
|
||||
...(includeCredentials ? { credentials: 'include' } : {}),
|
||||
signal: abort_signal,
|
||||
method: 'GET',
|
||||
});
|
||||
if (isBrowser) {
|
||||
// TODO: check if we can use the client-web directly
|
||||
const { ResultSet } = await import('@clickhouse/client-web');
|
||||
// https://github.com/ClickHouse/clickhouse-js/blob/1ebdd39203730bb99fad4c88eac35d9a5e96b34a/packages/client-web/src/connection/web_connection.ts#L200C7-L200C23
|
||||
const response = await fetch(`${this.host}/?${searchParams.toString()}`, {
|
||||
...(includeCredentials ? { credentials: 'include' } : {}),
|
||||
signal: abort_signal,
|
||||
method: 'GET',
|
||||
});
|
||||
|
||||
// TODO: Send command to CH to cancel query on abort_signal
|
||||
if (!res.ok) {
|
||||
if (!isSuccessfulResponse(res.status)) {
|
||||
const text = await res.text();
|
||||
throw new ClickHouseQueryError(`${text}`, debugSql);
|
||||
// TODO: Send command to CH to cancel query on abort_signal
|
||||
if (!response.ok) {
|
||||
if (!isSuccessfulResponse(response.status)) {
|
||||
const text = await response.text();
|
||||
throw new ClickHouseQueryError(`${text}`, debugSql);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (res.body == null) {
|
||||
// TODO: Handle empty responses better?
|
||||
throw new Error('Unexpected empty response from ClickHouse');
|
||||
}
|
||||
if (response.body == null) {
|
||||
// TODO: Handle empty responses better?
|
||||
throw new Error('Unexpected empty response from ClickHouse');
|
||||
}
|
||||
return new ResultSet<T>(
|
||||
response.body,
|
||||
format as T,
|
||||
queryId ?? '',
|
||||
getResponseHeaders(response),
|
||||
);
|
||||
} else if (isNode) {
|
||||
const { createClient } = await import('@clickhouse/client');
|
||||
const _client = createClient({
|
||||
host: this.host,
|
||||
username: this.username,
|
||||
password: this.password,
|
||||
clickhouse_settings: {
|
||||
date_time_output_format: 'iso',
|
||||
wait_end_of_query: 0,
|
||||
cancel_http_readonly_queries_on_client_close: 1,
|
||||
},
|
||||
});
|
||||
|
||||
// @ts-ignore
|
||||
return new BaseResultSet(res.body, format, '');
|
||||
},
|
||||
};
|
||||
// TODO: Custom error handling
|
||||
return _client.query({
|
||||
query,
|
||||
query_params,
|
||||
format: format as T,
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
query_id: queryId,
|
||||
}) as unknown as BaseResultSet<any, T>;
|
||||
} else {
|
||||
throw new Error(
|
||||
'ClickhouseClient is only supported in the browser or node environment',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const testLocalConnection = async ({
|
||||
host,
|
||||
|
|
@ -330,14 +383,10 @@ export const testLocalConnection = async ({
|
|||
password: string;
|
||||
}): Promise<boolean> => {
|
||||
try {
|
||||
const client = new ClickhouseClient({ host, username, password });
|
||||
const result = await client.query({
|
||||
query: 'SELECT 1',
|
||||
format: 'TabSeparatedRaw',
|
||||
host: host,
|
||||
username: username,
|
||||
password: password,
|
||||
includeCredentials: false,
|
||||
includeCorsHeader: true,
|
||||
});
|
||||
return result.text().then(text => text.trim() === '1');
|
||||
} catch (e) {
|
||||
|
|
@ -346,53 +395,6 @@ export const testLocalConnection = async ({
|
|||
}
|
||||
};
|
||||
|
||||
export const sendQuery = async <T extends DataFormat>({
|
||||
query,
|
||||
format = 'JSON',
|
||||
query_params = {},
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
connectionId,
|
||||
queryId,
|
||||
}: {
|
||||
query: string;
|
||||
format?: string;
|
||||
query_params?: Record<string, any>;
|
||||
abort_signal?: AbortSignal;
|
||||
clickhouse_settings?: Record<string, any>;
|
||||
connectionId: string;
|
||||
queryId?: string;
|
||||
}) => {
|
||||
const IS_LOCAL_MODE = false;
|
||||
|
||||
// TODO: decide what to do here
|
||||
let host, username, password;
|
||||
if (IS_LOCAL_MODE) {
|
||||
const localConnections: any = [];
|
||||
if (localConnections.length === 0) {
|
||||
throw new Error('No local connection found');
|
||||
}
|
||||
host = localConnections[0].host;
|
||||
username = localConnections[0].username;
|
||||
password = localConnections[0].password;
|
||||
}
|
||||
|
||||
return client.query<T>({
|
||||
query,
|
||||
format,
|
||||
query_params,
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
queryId,
|
||||
connectionId: IS_LOCAL_MODE ? undefined : connectionId,
|
||||
host: IS_LOCAL_MODE ? host : PROXY_CLICKHOUSE_HOST,
|
||||
username: IS_LOCAL_MODE ? username : undefined,
|
||||
password: IS_LOCAL_MODE ? password : undefined,
|
||||
includeCredentials: !IS_LOCAL_MODE,
|
||||
includeCorsHeader: IS_LOCAL_MODE,
|
||||
});
|
||||
};
|
||||
|
||||
export const tableExpr = ({
|
||||
database,
|
||||
table,
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import {
|
||||
ChSql,
|
||||
chSql,
|
||||
ClickhouseClient,
|
||||
ColumnMeta,
|
||||
convertCHDataTypeToJSType,
|
||||
filterColumnMetaByType,
|
||||
JSDataType,
|
||||
sendQuery,
|
||||
tableExpr,
|
||||
} from '@/clickhouse';
|
||||
import {
|
||||
|
|
@ -72,9 +72,14 @@ export type TableMetadata = {
|
|||
};
|
||||
|
||||
export class Metadata {
|
||||
private readonly clickhouseClient: ClickhouseClient;
|
||||
private cache = new MetadataCache();
|
||||
|
||||
private static async queryTableMetadata({
|
||||
constructor(clickhouseClient: ClickhouseClient) {
|
||||
this.clickhouseClient = clickhouseClient;
|
||||
}
|
||||
|
||||
private async queryTableMetadata({
|
||||
database,
|
||||
table,
|
||||
cache,
|
||||
|
|
@ -87,11 +92,13 @@ export class Metadata {
|
|||
}) {
|
||||
return cache.getOrFetch(`${database}.${table}.metadata`, async () => {
|
||||
const sql = chSql`SELECT * FROM system.tables where database = ${{ String: database }} AND name = ${{ String: table }}`;
|
||||
const json = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
}).then(res => res.json<TableMetadata>());
|
||||
const json = await this.clickhouseClient
|
||||
.query<'JSON'>({
|
||||
connectionId,
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
})
|
||||
.then(res => res.json<TableMetadata>());
|
||||
return json.data[0];
|
||||
});
|
||||
}
|
||||
|
|
@ -109,11 +116,12 @@ export class Metadata {
|
|||
`${databaseName}.${tableName}.columns`,
|
||||
async () => {
|
||||
const sql = chSql`DESCRIBE ${tableExpr({ database: databaseName, table: tableName })}`;
|
||||
const columns = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
})
|
||||
const columns = await this.clickhouseClient
|
||||
.query<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
})
|
||||
.then(res => res.json())
|
||||
.then(d => d.data);
|
||||
return columns as ColumnMeta[];
|
||||
|
|
@ -234,15 +242,16 @@ export class Metadata {
|
|||
return this.cache.getOrFetch<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.keys`,
|
||||
async () => {
|
||||
const keys = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
})
|
||||
const keys = await this.clickhouseClient
|
||||
.query<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
})
|
||||
.then(res => res.json<Record<string, unknown>>())
|
||||
.then(d => {
|
||||
let output: string[];
|
||||
|
|
@ -307,15 +316,16 @@ export class Metadata {
|
|||
return this.cache.getOrFetch<string[]>(
|
||||
`${databaseName}.${tableName}.${column}.${key}.values`,
|
||||
async () => {
|
||||
const values = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
})
|
||||
const values = await this.clickhouseClient
|
||||
.query<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
})
|
||||
.then(res => res.json<Record<string, unknown>>())
|
||||
.then(d => d.data.map(row => row.value as string));
|
||||
return values;
|
||||
|
|
@ -383,7 +393,7 @@ export class Metadata {
|
|||
tableName: string;
|
||||
connectionId: string;
|
||||
}) {
|
||||
const tableMetadata = await Metadata.queryTableMetadata({
|
||||
const tableMetadata = await this.queryTableMetadata({
|
||||
cache: this.cache,
|
||||
database: databaseName,
|
||||
table: tableName,
|
||||
|
|
@ -402,22 +412,27 @@ export class Metadata {
|
|||
keys: string[];
|
||||
limit?: number;
|
||||
}) {
|
||||
const sql = await renderChartConfig({
|
||||
...chartConfig,
|
||||
select: keys
|
||||
.map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`)
|
||||
.join(', '),
|
||||
});
|
||||
|
||||
const json = await sendQuery<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId: chartConfig.connection,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
const sql = await renderChartConfig(
|
||||
{
|
||||
...chartConfig,
|
||||
select: keys
|
||||
.map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`)
|
||||
.join(', '),
|
||||
},
|
||||
}).then(res => res.json<any>());
|
||||
this,
|
||||
);
|
||||
|
||||
const json = await this.clickhouseClient
|
||||
.query<'JSON'>({
|
||||
query: sql.sql,
|
||||
query_params: sql.params,
|
||||
connectionId: chartConfig.connection,
|
||||
clickhouse_settings: {
|
||||
max_rows_to_read: DEFAULT_SAMPLE_SIZE,
|
||||
read_overflow_mode: 'break',
|
||||
},
|
||||
})
|
||||
.then(res => res.json<any>());
|
||||
|
||||
return Object.entries(json.data[0]).map(([key, value]) => ({
|
||||
key: keys[parseInt(key.replace('param', ''))],
|
||||
|
|
@ -432,4 +447,5 @@ export type Field = {
|
|||
jsType: JSDataType | null;
|
||||
};
|
||||
|
||||
export const metadata = new Metadata();
|
||||
export const getMetadata = (clickhouseClient: ClickhouseClient) =>
|
||||
new Metadata(clickhouseClient);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import isPlainObject from 'lodash/isPlainObject';
|
|||
import * as SQLParser from 'node-sql-parser';
|
||||
|
||||
import { ChSql, chSql, concatChSql, wrapChSqlIfNotEmpty } from '@/clickhouse';
|
||||
import { Metadata, metadata } from '@/metadata';
|
||||
import { Metadata } from '@/metadata';
|
||||
import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser';
|
||||
import {
|
||||
AggregateFunction,
|
||||
|
|
@ -751,6 +751,7 @@ function renderLimit(
|
|||
|
||||
export async function renderChartConfig(
|
||||
chartConfig: ChartConfigWithOptDateRange,
|
||||
metadata: Metadata,
|
||||
): Promise<ChSql> {
|
||||
const select = await renderSelect(chartConfig, metadata);
|
||||
const from = renderFrom(chartConfig);
|
||||
|
|
|
|||
|
|
@ -176,13 +176,13 @@ export const FilterSchema = z.union([
|
|||
|
||||
export const _ChartConfigSchema = z.object({
|
||||
displayType: z.nativeEnum(DisplayType),
|
||||
numberFormat: NumberFormatSchema,
|
||||
numberFormat: NumberFormatSchema.optional(),
|
||||
timestampValueExpression: z.string(),
|
||||
implicitColumnExpression: z.string().optional(),
|
||||
granularity: z.string().optional(),
|
||||
markdown: z.string().optional(),
|
||||
filtersLogicalOperator: z.enum(['AND', 'OR']).optional(),
|
||||
filters: z.array(FilterSchema),
|
||||
filters: z.array(FilterSchema).optional(),
|
||||
connection: z.string(),
|
||||
fillNulls: z.number().optional(),
|
||||
selectGroupBy: z.boolean().optional(),
|
||||
|
|
|
|||
|
|
@ -179,3 +179,20 @@ export function timeBucketByGranularity(
|
|||
|
||||
return buckets;
|
||||
}
|
||||
|
||||
export const _useTry = <T>(fn: () => T): [null | Error | unknown, null | T] => {
|
||||
let output: T | null = null;
|
||||
let error: any = null;
|
||||
try {
|
||||
output = fn();
|
||||
return [error, output];
|
||||
} catch (e) {
|
||||
error = e;
|
||||
return [error, output];
|
||||
}
|
||||
};
|
||||
|
||||
export const parseJSON = <T = any>(json: string) => {
|
||||
const [error, result] = _useTry<T>(() => JSON.parse(json));
|
||||
return result;
|
||||
};
|
||||
|
|
|
|||
57
yarn.lock
57
yarn.lock
|
|
@ -3126,6 +3126,13 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@clickhouse/client-common@npm:1.10.1, @clickhouse/client-common@npm:^1.10.1":
|
||||
version: 1.10.1
|
||||
resolution: "@clickhouse/client-common@npm:1.10.1"
|
||||
checksum: 10c0/11638657ab9f5b7ffe8d1625215ec6be5377fe2ff9d7010228dfbdcc4edb9613b59c77d169c7ede4c16b38090942815d262b9e6d8425bce045d7014d32db56cb
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@clickhouse/client-common@npm:1.7.0":
|
||||
version: 1.7.0
|
||||
resolution: "@clickhouse/client-common@npm:1.7.0"
|
||||
|
|
@ -3133,10 +3140,12 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@clickhouse/client-common@npm:^1.9.1":
|
||||
version: 1.9.1
|
||||
resolution: "@clickhouse/client-common@npm:1.9.1"
|
||||
checksum: 10c0/d4df33c16e8ead60c8d656418c81f4cacd16f63c26d462bb265fa4892c0e0f40c5ddf447724befea36ffed569dae685049a9fb6f6210e3362bcbe632e15dfc70
|
||||
"@clickhouse/client-web@npm:^1.10.1":
|
||||
version: 1.10.1
|
||||
resolution: "@clickhouse/client-web@npm:1.10.1"
|
||||
dependencies:
|
||||
"@clickhouse/client-common": "npm:1.10.1"
|
||||
checksum: 10c0/4b4041171331670c8ca89727d425152a8d2057460ef2b3ac9c765479a0dc99080b4ff10c8100ebbcc1e3083de3719d7aefd5b74a28866a5e1a9c9702779c9b92
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
|
|
@ -3158,6 +3167,15 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@clickhouse/client@npm:^1.10.1":
|
||||
version: 1.10.1
|
||||
resolution: "@clickhouse/client@npm:1.10.1"
|
||||
dependencies:
|
||||
"@clickhouse/client-common": "npm:1.10.1"
|
||||
checksum: 10c0/a0c1c7541be7ba2691b584cd905282bab622411f165b504e61a2cf22e075fc6afcd3105b29ddc939a46b47a6e641f95161d2323b7cf831ae2a4cf920945dbd60
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@clickhouse/client@npm:^1.7.0":
|
||||
version: 1.7.0
|
||||
resolution: "@clickhouse/client@npm:1.7.0"
|
||||
|
|
@ -4198,7 +4216,7 @@ __metadata:
|
|||
resolution: "@hyperdx/api@workspace:packages/api"
|
||||
dependencies:
|
||||
"@clickhouse/client": "npm:^0.2.10"
|
||||
"@clickhouse/client-common": "npm:^1.9.1"
|
||||
"@hyperdx/common-utils": "npm:^0.0.9"
|
||||
"@hyperdx/lucene": "npm:^3.1.1"
|
||||
"@hyperdx/node-opentelemetry": "npm:^0.8.1"
|
||||
"@opentelemetry/api": "npm:^1.8.0"
|
||||
|
|
@ -4246,7 +4264,6 @@ __metadata:
|
|||
mongoose: "npm:^6.12.0"
|
||||
ms: "npm:^2.1.3"
|
||||
node-schedule: "npm:^2.1.1"
|
||||
node-sql-parser: "npm:^5.3.5"
|
||||
nodemon: "npm:^2.0.20"
|
||||
object-hash: "npm:^3.0.0"
|
||||
on-headers: "npm:^1.0.2"
|
||||
|
|
@ -4286,7 +4303,7 @@ __metadata:
|
|||
"@hookform/devtools": "npm:^4.3.1"
|
||||
"@hookform/resolvers": "npm:^3.9.0"
|
||||
"@hyperdx/browser": "npm:^0.21.1"
|
||||
"@hyperdx/lucene": "npm:^3.1.1"
|
||||
"@hyperdx/common-utils": "npm:^0.0.9"
|
||||
"@hyperdx/node-opentelemetry": "npm:^0.8.1"
|
||||
"@jedmao/location": "npm:^3.0.0"
|
||||
"@lezer/highlight": "npm:^1.2.0"
|
||||
|
|
@ -4314,7 +4331,6 @@ __metadata:
|
|||
"@testing-library/react": "npm:^14.2.1"
|
||||
"@testing-library/user-event": "npm:^14.5.2"
|
||||
"@types/crypto-js": "npm:^4"
|
||||
"@types/hyperdx__lucene": "npm:@types/lucene@*"
|
||||
"@types/intercom-web": "npm:^2.8.18"
|
||||
"@types/jest": "npm:^28.1.6"
|
||||
"@types/lodash": "npm:^4.14.186"
|
||||
|
|
@ -4355,7 +4371,6 @@ __metadata:
|
|||
next-seo: "npm:^4.28.1"
|
||||
nextra: "npm:2.0.1"
|
||||
nextra-theme-docs: "npm:^2.0.2"
|
||||
node-sql-parser: "npm:^5.3.2"
|
||||
numbro: "npm:^2.4.0"
|
||||
nuqs: "npm:^1.17.0"
|
||||
object-hash: "npm:^3.0.0"
|
||||
|
|
@ -4412,11 +4427,13 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@hyperdx/common-utils@workspace:packages/common-utils":
|
||||
"@hyperdx/common-utils@npm:^0.0.9, @hyperdx/common-utils@workspace:packages/common-utils":
|
||||
version: 0.0.0-use.local
|
||||
resolution: "@hyperdx/common-utils@workspace:packages/common-utils"
|
||||
dependencies:
|
||||
"@clickhouse/client-common": "npm:^1.9.1"
|
||||
"@clickhouse/client": "npm:^1.10.1"
|
||||
"@clickhouse/client-common": "npm:^1.10.1"
|
||||
"@clickhouse/client-web": "npm:^1.10.1"
|
||||
"@hyperdx/lucene": "npm:^3.1.1"
|
||||
"@types/hyperdx__lucene": "npm:@types/lucene@*"
|
||||
"@types/jest": "npm:^28.1.1"
|
||||
|
|
@ -4436,6 +4453,7 @@ __metadata:
|
|||
rimraf: "npm:^4.4.1"
|
||||
semver: "npm:^7.5.2"
|
||||
sqlstring: "npm:^2.3.3"
|
||||
store2: "npm:^2.14.4"
|
||||
supertest: "npm:^6.3.1"
|
||||
ts-jest: "npm:^28.0.5"
|
||||
ts-node: "npm:^10.8.1"
|
||||
|
|
@ -21533,16 +21551,6 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"node-sql-parser@npm:^5.3.2":
|
||||
version: 5.3.2
|
||||
resolution: "node-sql-parser@npm:5.3.2"
|
||||
dependencies:
|
||||
"@types/pegjs": "npm:^0.10.0"
|
||||
big-integer: "npm:^1.6.48"
|
||||
checksum: 10c0/4c28bbb156aab433387fbda8e2f5ab5dc76d231b77b9d58cd55e9f5b63d16dc2cd96002d242c6fd602c10f89baea1a33ef1cd2047b8e4a1b15008476f2c2c969
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"node-sql-parser@npm:^5.3.5":
|
||||
version: 5.3.5
|
||||
resolution: "node-sql-parser@npm:5.3.5"
|
||||
|
|
@ -25766,6 +25774,13 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"store2@npm:^2.14.4":
|
||||
version: 2.14.4
|
||||
resolution: "store2@npm:2.14.4"
|
||||
checksum: 10c0/3453c9c8c153c760e6290395a7bc23669df5dc8a6e8a49f9b3187dbb9f86d14b58705aa4f17fad6b536d4b04fe3e66ea5bde12c1352abd52c6b303bbf5757ab6
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"storybook@npm:^8.1.5":
|
||||
version: 8.1.5
|
||||
resolution: "storybook@npm:8.1.5"
|
||||
|
|
|
|||
Loading…
Reference in a new issue