mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
fix: Fix sidebar when selecting JSON property (#1231)
Closes HDX-2042 Closes HDX-2524 Closes HDX-2307 Closes #1010 # Summary This PR fixes errors that occurred when attempting to open the sidebar by clicking a log table row using a JSON logs table schema. The error was caused by `node-sql-parser` throwing exceptions when parsing SQL with JSON Expressions, resulting in HyperDX being unable to extract aliases from the SQL. In the long term, we'll want to have a true ClickHouse SQL parser. In the short term, this is fixed by: 1. Finding and replacing all JSON expressions in the sql with placeholder tokens, prior to parsing with node-sql-parser 2. Parsing with node-sql-parser to find aliases correctly 3. Replacing the placeholder tokens with the original JSON expressions ## Testing (All of the following use a JSON schema) ### Before <details> <summary>When selecting a JSON column with an alias</summary> <img width="1126" height="96" alt="Screenshot 2025-10-01 at 2 28 19 PM" src="https://github.com/user-attachments/assets/c35ed870-9986-4b30-9890-e1ca8ff6c92c" /> <img width="372" height="142" alt="Screenshot 2025-10-01 at 2 28 06 PM" src="https://github.com/user-attachments/assets/d65fdce4-6625-4308-b5d0-6f845a0f2f05" /> </details> <details> <summary>When filtering by a JSON column and using an alias on a non-JSON property</summary> <img width="800" height="103" alt="Screenshot 2025-10-01 at 2 29 44 PM" src="https://github.com/user-attachments/assets/aa7faabb-316b-4103-8840-74ac08519efb" /> <img width="372" height="142" alt="Screenshot 2025-10-01 at 2 28 06 PM" src="https://github.com/user-attachments/assets/eb86cce5-eee4-40f9-af93-2451bff32444" /> </details> ### After <details> <summary>When selecting a JSON column with an alias</summary> <img width="1126" height="96" alt="Screenshot 2025-10-01 at 2 28 19 PM" src="https://github.com/user-attachments/assets/678ba290-5215-4cc5-8fee-1bf67955aaa2" /> <img width="725" height="696" alt="Screenshot 2025-10-01 at 2 30 42 PM" src="https://github.com/user-attachments/assets/5da48109-a0cd-4b5f-a5e3-bd700116d81b" /> </details> <details> <summary>When filtering by a JSON column and using an alias on a non-JSON property</summary> <img width="800" height="103" alt="Screenshot 2025-10-01 at 2 29 44 PM" src="https://github.com/user-attachments/assets/715de816-639e-4ffd-9e09-341bd0b2ee4a" /> <img width="1271" height="888" alt="Screenshot 2025-10-01 at 2 30 24 PM" src="https://github.com/user-attachments/assets/b3b766de-be70-4161-b9ca-8aae9330b5f2" /> </details>
This commit is contained in:
Drew Davis
GitHub
parent
f4c352393d
commit
b46ae2f204
6 changed files with 627 additions and 8 deletions
6
.changeset/sharp-phones-travel.md
Normal file
6
.changeset/sharp-phones-travel.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
"@hyperdx/common-utils": patch
|
||||
"@hyperdx/app": patch
|
||||
---
|
||||
|
||||
fix: Fix sidebar when selecting JSON property
|
||||
|
|
@ -66,7 +66,7 @@ export function processRowToWhereClause(
|
|||
|
||||
// Currently we can't distinguish null or 'null'
|
||||
if (value === 'null') {
|
||||
return SqlString.format(`isNull(??)`, [column]);
|
||||
return SqlString.format(`isNull(??)`, [valueExpr]);
|
||||
}
|
||||
if (value.length > 1000 || column.length > 1000) {
|
||||
console.warn('Search value/object key too large.');
|
||||
|
|
|
|||
|
|
@ -200,6 +200,27 @@ describe('chSqlToAliasMap - alias unit test', () => {
|
|||
};
|
||||
expect(res).toEqual(aliasMap);
|
||||
});
|
||||
|
||||
it('Alias, with JSON expressions', () => {
|
||||
const chSqlInput: ChSql = {
|
||||
sql: "SELECT Timestamp as ts,ResourceAttributes.service.name as service,toStartOfDay(LogAttributes.start.`time`) as start_time,Body,TimestampTime,ServiceName,TimestampTime FROM {HYPERDX_PARAM_1544803905:Identifier}.{HYPERDX_PARAM_129845054:Identifier} WHERE (TimestampTime >= fromUnixTimestamp64Milli({HYPERDX_PARAM_1456399765:Int64}) AND TimestampTime <= fromUnixTimestamp64Milli({HYPERDX_PARAM_1719057412:Int64})) AND (`ResourceAttributes`.`service`.`name` = 'serviceName') ORDER BY TimestampTime DESC LIMIT {HYPERDX_PARAM_49586:Int32} OFFSET {HYPERDX_PARAM_48:Int32}",
|
||||
params: {
|
||||
HYPERDX_PARAM_1544803905: 'default',
|
||||
HYPERDX_PARAM_129845054: 'otel_logs',
|
||||
HYPERDX_PARAM_1456399765: 1743038742000,
|
||||
HYPERDX_PARAM_1719057412: 1743040542000,
|
||||
HYPERDX_PARAM_49586: 200,
|
||||
HYPERDX_PARAM_48: 0,
|
||||
},
|
||||
};
|
||||
const res = chSqlToAliasMap(chSqlInput);
|
||||
const aliasMap = {
|
||||
ts: 'Timestamp',
|
||||
service: 'ResourceAttributes.service.name',
|
||||
start_time: 'toStartOfDay(LogAttributes.start.`time`)',
|
||||
};
|
||||
expect(res).toEqual(aliasMap);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeRatio', () => {
|
||||
|
|
|
|||
|
|
@ -10,11 +10,13 @@ import {
|
|||
|
||||
import {
|
||||
convertToDashboardTemplate,
|
||||
findJsonExpressions,
|
||||
formatDate,
|
||||
getFirstOrderingItem,
|
||||
isFirstOrderByAscending,
|
||||
isJsonExpression,
|
||||
isTimestampExpressionInFirstOrderBy,
|
||||
removeTrailingDirection,
|
||||
replaceJsonExpressions,
|
||||
splitAndTrimCSV,
|
||||
splitAndTrimWithBracket,
|
||||
} from '../utils';
|
||||
|
|
@ -674,4 +676,443 @@ describe('utils', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('isJsonExpression', () => {
|
||||
it('should return false for expressions without dots', () => {
|
||||
expect(isJsonExpression('col')).toBe(false);
|
||||
expect(isJsonExpression('columnName')).toBe(false);
|
||||
expect(isJsonExpression('column_name')).toBe(false);
|
||||
});
|
||||
|
||||
it('should return true for simple JSON expressions', () => {
|
||||
expect(isJsonExpression('col.key')).toBe(true);
|
||||
expect(isJsonExpression('column.property')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return true for nested JSON expressions', () => {
|
||||
expect(isJsonExpression('col.key.nestedKey')).toBe(true);
|
||||
expect(isJsonExpression('a.b.c')).toBe(true);
|
||||
expect(isJsonExpression('json_col.col3.c')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return true for JSON expressions with double quotes', () => {
|
||||
expect(isJsonExpression('"json_col"."key"')).toBe(true);
|
||||
expect(isJsonExpression('"a"."b"."cde"')).toBe(true);
|
||||
expect(isJsonExpression('"a_b.2c".b."c."')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return true for JSON expressions with backticks', () => {
|
||||
expect(isJsonExpression('`a`.`b`.`cde`')).toBe(true);
|
||||
expect(isJsonExpression('`col`.`key`')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return true for mixed quoting styles', () => {
|
||||
expect(isJsonExpression('"a".b.`c`')).toBe(true);
|
||||
expect(isJsonExpression('a."b".c')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false for expressions with only one non-numeric part', () => {
|
||||
expect(isJsonExpression('col.')).toBe(false);
|
||||
expect(isJsonExpression('.col')).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false for decimal numbers', () => {
|
||||
expect(isJsonExpression('10.50')).toBe(false);
|
||||
expect(isJsonExpression('2.3')).toBe(false);
|
||||
expect(isJsonExpression('1.5')).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false for table.column references with numeric column', () => {
|
||||
expect(isJsonExpression('table.1')).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false for expressions with empty parts', () => {
|
||||
expect(isJsonExpression('.')).toBe(false);
|
||||
expect(isJsonExpression('..')).toBe(false);
|
||||
expect(isJsonExpression('a..')).toBe(false);
|
||||
});
|
||||
|
||||
it('should handle dots inside double quotes correctly', () => {
|
||||
expect(isJsonExpression('"a.b.c"')).toBe(false);
|
||||
expect(isJsonExpression('"a.b"."c.d"')).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle dots inside backticks correctly', () => {
|
||||
expect(isJsonExpression('`a.b.c`')).toBe(false);
|
||||
expect(isJsonExpression('`a.b`.`c.d`')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return true for mixed quoted and unquoted parts', () => {
|
||||
expect(isJsonExpression('"col.with.dots".key')).toBe(true);
|
||||
expect(isJsonExpression('col."key.with.dots"')).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle complex quoted identifiers', () => {
|
||||
expect(isJsonExpression('"table.name"."column.name"."nested"')).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle expressions with underscores and numbers', () => {
|
||||
expect(isJsonExpression('col_1.key_2.nested_3')).toBe(true);
|
||||
expect(isJsonExpression('table123.column456')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false for single quoted identifier', () => {
|
||||
expect(isJsonExpression('"singleColumn"')).toBe(false);
|
||||
expect(isJsonExpression('`singleColumn`')).toBe(false);
|
||||
});
|
||||
|
||||
it('should handle type specifiers', () => {
|
||||
expect(isJsonExpression('a.b.:UInt64')).toBe(true);
|
||||
expect(isJsonExpression('col.key.:String')).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle whitespace in parts', () => {
|
||||
expect(isJsonExpression('a . b')).toBe(true);
|
||||
expect(isJsonExpression('a.b. c')).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle leading whitespace', () => {
|
||||
expect(isJsonExpression(' a.b.c')).toBe(true);
|
||||
expect(isJsonExpression(' col.key')).toBe(true);
|
||||
expect(isJsonExpression('\ta.b')).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle trailing whitespace', () => {
|
||||
expect(isJsonExpression('a.b.c ')).toBe(true);
|
||||
expect(isJsonExpression('col.key ')).toBe(true);
|
||||
expect(isJsonExpression('a.b\t')).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle leading and trailing whitespace', () => {
|
||||
expect(isJsonExpression(' a.b.c ')).toBe(true);
|
||||
expect(isJsonExpression(' col.key ')).toBe(true);
|
||||
expect(isJsonExpression('\ta.b\t')).toBe(true);
|
||||
});
|
||||
|
||||
it('should correctly handle single quoted strings', () => {
|
||||
expect(isJsonExpression("'a'.b.c")).toBe(false);
|
||||
expect(isJsonExpression("'a'.'b'")).toBe(false);
|
||||
expect(isJsonExpression("'a' . 'b'")).toBe(false);
|
||||
expect(isJsonExpression("'")).toBe(false);
|
||||
expect(isJsonExpression("''")).toBe(false);
|
||||
expect(isJsonExpression("`'a'`.b")).toBe(true);
|
||||
expect(isJsonExpression("`'a`.b")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findJsonExpressions', () => {
|
||||
it('should handle empty expression', () => {
|
||||
const sql = '';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find a single JSON expression', () => {
|
||||
const sql = 'SELECT a.b.c as alias1, col2 as alias2 FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 7, expr: 'a.b.c' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find multiple JSON expression', () => {
|
||||
const sql = 'SELECT a.b.c, d.e, col2 FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [
|
||||
{ index: 7, expr: 'a.b.c' },
|
||||
{ index: 14, expr: 'd.e' },
|
||||
];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expression with type specifier', () => {
|
||||
const sql = 'SELECT a.b.:UInt64, col2 FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 7, expr: 'a.b.:UInt64' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expression with complex type specifier', () => {
|
||||
const sql = 'SELECT a.b.:Array(String) , col2 FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 7, expr: 'a.b.:Array(String)' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expressions in WHERE clause ', () => {
|
||||
const sql =
|
||||
'SELECT col2 FROM table WHERE a.b.:UInt64 = 1 AND toStartOfDay(a.date) = today()';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [
|
||||
{ index: 29, expr: 'a.b.:UInt64' },
|
||||
{ index: 62, expr: 'a.date' },
|
||||
];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expressions in function calls', () => {
|
||||
const sql = "SELECT JSONExtractString(a.b.c, 'key') FROM table";
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 25, expr: 'a.b.c' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not find JSON expressions in quoted strings', () => {
|
||||
const sql =
|
||||
"SELECT a.b.c, ResourceAttributes['key.key2'], 'a.b.c' FROM table";
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [
|
||||
{
|
||||
index: 7,
|
||||
expr: 'a.b.c',
|
||||
},
|
||||
];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expressions in math expression', () => {
|
||||
const sql =
|
||||
'SELECT toStartOfDay(a.date + INTERVAL 1 DAY), toStartOfDay(a.date+INTERVAL 1 DAY)';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [
|
||||
{ index: 20, expr: 'a.date' },
|
||||
{ index: 59, expr: 'a.date' },
|
||||
];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not infinite loop due to unterminated strings', () => {
|
||||
const sql = 'SELECT "';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not infinite loop due to trailing whitespace', () => {
|
||||
const sql = 'SELECT ';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not infinite loop due to mismatched parenthesis', () => {
|
||||
const sql = 'SELECT (';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not infinite loop due to trailing json type specifier', () => {
|
||||
const sql = 'SELECT a.b.:UInt64';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 7, expr: 'a.b.:UInt64' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not find JSON expressions in string that has escaped single quote', () => {
|
||||
const sql = "SELECT 'a.b''''a.b.:UInt64', col2, c.d FROM table";
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 35, expr: 'c.d' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not find JSON expressions in string that has escaped single quote 2', () => {
|
||||
const sql = "SELECT '\\'a.b', col2, c.d FROM table";
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 22, expr: 'c.d' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expressions with underscores and numbers', () => {
|
||||
const sql = 'SELECT json_col.col3.c FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 7, expr: 'json_col.col3.c' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expressions with backticks', () => {
|
||||
const sql = 'SELECT `a`.`b`.`cde` FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 7, expr: '`a`.`b`.`cde`' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expressions with double quotes', () => {
|
||||
const sql = 'SELECT "a"."b"."cde" FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 7, expr: '"a"."b"."cde"' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expressions in tuple', () => {
|
||||
const sql = 'SELECT (a.b, c.d.e) FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [
|
||||
{ index: 8, expr: 'a.b' },
|
||||
{ index: 13, expr: 'c.d.e' },
|
||||
];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not find JSON expressions inside identifiers', () => {
|
||||
const sql = 'SELECT "a.b.c" FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expressions with weird identifier quoting', () => {
|
||||
const sql = 'SELECT "a_b.2c".b."c." FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 7, expr: '"a_b.2c".b."c."' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find JSON expressions after *', () => {
|
||||
const sql = 'SELECT *, a.b.c FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 10, expr: 'a.b.c' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not find a decimal number expression', () => {
|
||||
const sql = 'SELECT 10.50, 2.3, 2, 1.5 - a.b FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [{ index: 28, expr: 'a.b' }];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not find a . as a JSON expression', () => {
|
||||
const sql = 'SELECT . FROM table';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find a JSON expression with an identifier containing a single-quote', () => {
|
||||
const sql = `SELECT Timestamp,ServiceName,SeverityText,Body,ResourceAttributes.hyperdx.distro."version'" FROM default.otel_logs WHERE (Timestamp >= fromUnixTimestamp64Milli(1759756098000) AND Timestamp <= fromUnixTimestamp64Milli(1759756998000)) ORDER BY Timestamp DESC`;
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [
|
||||
{ index: 47, expr: `ResourceAttributes.hyperdx.distro."version'"` },
|
||||
{ index: 97, expr: `default.otel_logs` },
|
||||
];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find a JSON expression with an identifier containing a double-quote', () => {
|
||||
const sql =
|
||||
'SELECT Timestamp,ServiceName,SeverityText,Body,ResourceAttributes.hyperdx.distro.`"version"`';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [
|
||||
{ index: 47, expr: 'ResourceAttributes.hyperdx.distro.`"version"`' },
|
||||
];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should find a JSON expression with an identifier containing a backtick', () => {
|
||||
const sql =
|
||||
'SELECT Timestamp,ServiceName,SeverityText,Body,ResourceAttributes.hyperdx.distro."`version`"';
|
||||
const actual = findJsonExpressions(sql);
|
||||
const expected = [
|
||||
{ index: 47, expr: 'ResourceAttributes.hyperdx.distro."`version`"' },
|
||||
];
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('replaceJsonAccesses', () => {
|
||||
it('should handle empty expression', () => {
|
||||
const sql = '';
|
||||
const actual = replaceJsonExpressions(sql);
|
||||
const expected = { replacements: new Map(), sqlWithReplacements: '' };
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should replace a single JSON access', () => {
|
||||
const sql = 'SELECT a.b.c as alias1, col2 as alias2 FROM table';
|
||||
const actual = replaceJsonExpressions(sql);
|
||||
const expected = {
|
||||
replacements: new Map([['__hdx_json_replacement_0', 'a.b.c']]),
|
||||
sqlWithReplacements:
|
||||
'SELECT __hdx_json_replacement_0 as alias1, col2 as alias2 FROM table',
|
||||
};
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should replace multiple JSON access', () => {
|
||||
const sql = 'SELECT a.b.c, d.e, col2 FROM table';
|
||||
const actual = replaceJsonExpressions(sql);
|
||||
const expected = {
|
||||
replacements: new Map([
|
||||
['__hdx_json_replacement_0', 'a.b.c'],
|
||||
['__hdx_json_replacement_1', 'd.e'],
|
||||
]),
|
||||
sqlWithReplacements:
|
||||
'SELECT __hdx_json_replacement_0, __hdx_json_replacement_1, col2 FROM table',
|
||||
};
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should replace JSON access with type specifier', () => {
|
||||
const sql = 'SELECT a.b.:UInt64, col2 FROM table';
|
||||
const actual = replaceJsonExpressions(sql);
|
||||
const expected = {
|
||||
replacements: new Map([['__hdx_json_replacement_0', 'a.b.:UInt64']]),
|
||||
sqlWithReplacements: 'SELECT __hdx_json_replacement_0, col2 FROM table',
|
||||
};
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should replace JSON access with complex type specifier', () => {
|
||||
const sql = 'SELECT a.b.:Array(String), col2 FROM table';
|
||||
const actual = replaceJsonExpressions(sql);
|
||||
const expected = {
|
||||
replacements: new Map([
|
||||
['__hdx_json_replacement_0', 'a.b.:Array(String)'],
|
||||
]),
|
||||
sqlWithReplacements: 'SELECT __hdx_json_replacement_0, col2 FROM table',
|
||||
};
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should replace JSON expressions in WHERE clause ', () => {
|
||||
const sql =
|
||||
'SELECT col2 FROM table WHERE a.b.:UInt64 = 1 AND toStartOfDay(a.date) = today()';
|
||||
const actual = replaceJsonExpressions(sql);
|
||||
const expected = {
|
||||
replacements: new Map([
|
||||
['__hdx_json_replacement_0', 'a.b.:UInt64'],
|
||||
['__hdx_json_replacement_1', 'a.date'],
|
||||
]),
|
||||
sqlWithReplacements:
|
||||
'SELECT col2 FROM table WHERE __hdx_json_replacement_0 = 1 AND toStartOfDay(__hdx_json_replacement_1) = today()',
|
||||
};
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should replace JSON expressions in function calls', () => {
|
||||
const sql = "SELECT JSONExtractString(a.b.c, 'key') FROM table";
|
||||
const actual = replaceJsonExpressions(sql);
|
||||
const expected = {
|
||||
replacements: new Map([['__hdx_json_replacement_0', 'a.b.c']]),
|
||||
sqlWithReplacements:
|
||||
"SELECT JSONExtractString(__hdx_json_replacement_0, 'key') FROM table",
|
||||
};
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should not replace JSON expressions in quoted strings', () => {
|
||||
const sql =
|
||||
"SELECT a.b.c, ResourceAttributes['key.key2'], 'a.b.c' FROM table";
|
||||
const actual = replaceJsonExpressions(sql);
|
||||
const expected = {
|
||||
replacements: new Map([['__hdx_json_replacement_0', 'a.b.c']]),
|
||||
sqlWithReplacements:
|
||||
"SELECT __hdx_json_replacement_0, ResourceAttributes['key.key2'], 'a.b.c' FROM table",
|
||||
};
|
||||
expect(actual).toEqual(expected);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import type {
|
|||
ResponseJSON,
|
||||
Row,
|
||||
} from '@clickhouse/client-common';
|
||||
import { isSuccessfulResponse } from '@clickhouse/client-common';
|
||||
import type { ClickHouseClient as WebClickHouseClient } from '@clickhouse/client-web';
|
||||
import * as SQLParser from 'node-sql-parser';
|
||||
import objectHash from 'object-hash';
|
||||
|
|
@ -18,8 +17,12 @@ import {
|
|||
setChartSelectsAlias,
|
||||
splitChartConfigs,
|
||||
} from '@/renderChartConfig';
|
||||
import { ChartConfigWithOptDateRange, SQLInterval } from '@/types';
|
||||
import { hashCode, splitAndTrimWithBracket } from '@/utils';
|
||||
import { ChartConfigWithOptDateRange } from '@/types';
|
||||
import {
|
||||
hashCode,
|
||||
replaceJsonExpressions,
|
||||
splitAndTrimWithBracket,
|
||||
} from '@/utils';
|
||||
|
||||
// export @clickhouse/client-common types
|
||||
export type {
|
||||
|
|
@ -686,8 +689,13 @@ export function chSqlToAliasMap(
|
|||
|
||||
try {
|
||||
const sql = parameterizedQueryToSql(chSql);
|
||||
|
||||
// Replace JSON expressions with replacement tokens so that node-sql-parser can parse the SQL
|
||||
const { sqlWithReplacements, replacements: jsonReplacementsToExpressions } =
|
||||
replaceJsonExpressions(sql);
|
||||
|
||||
const parser = new SQLParser.Parser();
|
||||
const ast = parser.astify(sql, {
|
||||
const ast = parser.astify(sqlWithReplacements, {
|
||||
database: 'Postgresql',
|
||||
parseOptions: { includeLocations: true },
|
||||
}) as SQLParser.Select;
|
||||
|
|
@ -703,7 +711,7 @@ export function chSqlToAliasMap(
|
|||
: // normal alias
|
||||
column.expr.column.expr.value;
|
||||
} else if (column.expr.loc != null) {
|
||||
aliasMap[column.as] = sql.slice(
|
||||
aliasMap[column.as] = sqlWithReplacements.slice(
|
||||
column.expr.loc.start.offset,
|
||||
column.expr.loc.end.offset,
|
||||
);
|
||||
|
|
@ -713,8 +721,23 @@ export function chSqlToAliasMap(
|
|||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Replace the JSON replacement tokens with the original JSON expressions
|
||||
for (const [alias, aliasExpression] of Object.entries(aliasMap)) {
|
||||
for (const [replacement, original] of jsonReplacementsToExpressions) {
|
||||
if (aliasExpression.includes(replacement)) {
|
||||
aliasMap[alias] = aliasExpression.replaceAll(replacement, original);
|
||||
}
|
||||
}
|
||||
}
|
||||
return aliasMap;
|
||||
} catch (e) {
|
||||
console.error('Error parsing alias map', e, 'for query', chSql);
|
||||
console.error(
|
||||
'Error parsing alias map with JSON removed',
|
||||
e,
|
||||
'for query',
|
||||
chSql,
|
||||
);
|
||||
}
|
||||
|
||||
return aliasMap;
|
||||
|
|
|
|||
|
|
@ -87,6 +87,134 @@ export function getFirstTimestampValueExpression(valueExpression: string) {
|
|||
return splitAndTrimWithBracket(valueExpression)[0];
|
||||
}
|
||||
|
||||
/** Returns true if the given expression is a JSON expression, eg. `col.key.nestedKey` or "json_col"."key" */
|
||||
export const isJsonExpression = (expr: string) => {
|
||||
if (!expr.includes('.')) return false;
|
||||
|
||||
let isInDoubleQuote = false;
|
||||
let isInBacktick = false;
|
||||
let isInSingleQuote = false;
|
||||
|
||||
const parts: string[] = [];
|
||||
let current = '';
|
||||
for (const c of expr) {
|
||||
if (c === "'" && !isInDoubleQuote && !isInBacktick) {
|
||||
isInSingleQuote = !isInSingleQuote;
|
||||
} else if (isInSingleQuote) {
|
||||
continue;
|
||||
} else if (c === '"' && !isInBacktick) {
|
||||
isInDoubleQuote = !isInDoubleQuote;
|
||||
current += c;
|
||||
} else if (c === '`' && !isInDoubleQuote) {
|
||||
isInBacktick = !isInBacktick;
|
||||
current += c;
|
||||
} else if (c === '.' && !isInDoubleQuote && !isInBacktick) {
|
||||
parts.push(current);
|
||||
current = '';
|
||||
} else {
|
||||
current += c;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isInDoubleQuote && !isInBacktick) {
|
||||
parts.push(current);
|
||||
}
|
||||
|
||||
if (parts.some(p => p.trim().length === 0)) return false;
|
||||
|
||||
return (
|
||||
parts.filter(
|
||||
p =>
|
||||
p.trim().length > 0 &&
|
||||
isNaN(Number(p)) &&
|
||||
!(p.startsWith("'") && p.endsWith("'")),
|
||||
).length > 1
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Finds and returns expressions within the given SQL string that represent JSON references (eg. `col.key.nestedKey`)
|
||||
*
|
||||
* Note - This function does not distinguish between json references and `table.column` references - both are returned.
|
||||
*/
|
||||
export function findJsonExpressions(sql: string) {
|
||||
const expressions: { index: number; expr: string }[] = [];
|
||||
|
||||
let isInDoubleQuote = false;
|
||||
let isInBacktick = false;
|
||||
|
||||
let currentExpr = '';
|
||||
const finishExpression = (expr: string, endIndex: number) => {
|
||||
if (isJsonExpression(expr)) {
|
||||
expressions.push({ index: endIndex - expr.length, expr });
|
||||
}
|
||||
currentExpr = '';
|
||||
};
|
||||
|
||||
let i = 0;
|
||||
let isInJsonTypeSpecifier = false;
|
||||
while (i < sql.length) {
|
||||
const c = sql.charAt(i);
|
||||
if (c === "'" && !isInDoubleQuote && !isInBacktick) {
|
||||
// Skip string literals
|
||||
while (i < sql.length && sql.charAt(i) !== c) {
|
||||
i++;
|
||||
}
|
||||
currentExpr = '';
|
||||
} else if (c === '"' && !isInBacktick) {
|
||||
isInDoubleQuote = !isInDoubleQuote;
|
||||
currentExpr += c;
|
||||
} else if (c === '`' && !isInDoubleQuote) {
|
||||
isInBacktick = !isInBacktick;
|
||||
currentExpr += c;
|
||||
} else if (/[\s{},+*/[\]]/.test(c)) {
|
||||
isInJsonTypeSpecifier = false;
|
||||
finishExpression(currentExpr, i);
|
||||
} else if ('()'.includes(c) && !isInJsonTypeSpecifier) {
|
||||
finishExpression(currentExpr, i);
|
||||
} else if (c === ':') {
|
||||
isInJsonTypeSpecifier = true;
|
||||
currentExpr += c;
|
||||
} else {
|
||||
currentExpr += c;
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
|
||||
finishExpression(currentExpr, i);
|
||||
return expressions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces expressions within the given SQL string that represent JSON expressions (eg. `col.key.nestedKey`).
|
||||
* Such expression are replaced with placeholders like `__hdx_json_replacement_0`. The resulting string and a
|
||||
* map of replacements --> original expressions is returned.
|
||||
*
|
||||
* Note - This function does not distinguish between json references and `table.column` references - both are replaced.
|
||||
*/
|
||||
export function replaceJsonExpressions(sql: string) {
|
||||
const jsonExpressions = findJsonExpressions(sql);
|
||||
|
||||
const replacements = new Map<string, string>();
|
||||
let sqlWithReplacements = sql;
|
||||
let indexOffsetFromInserts = 0;
|
||||
let replacementCounter = 0;
|
||||
for (const { expr, index } of jsonExpressions) {
|
||||
const replacement = `__hdx_json_replacement_${replacementCounter++}`;
|
||||
replacements.set(replacement, expr);
|
||||
|
||||
const effectiveIndex = index + indexOffsetFromInserts;
|
||||
sqlWithReplacements =
|
||||
sqlWithReplacements.slice(0, effectiveIndex) +
|
||||
replacement +
|
||||
sqlWithReplacements.slice(effectiveIndex + expr.length);
|
||||
indexOffsetFromInserts += replacement.length - expr.length;
|
||||
}
|
||||
|
||||
return { sqlWithReplacements, replacements };
|
||||
}
|
||||
|
||||
export enum Granularity {
|
||||
FifteenSecond = '15 second',
|
||||
ThirtySecond = '30 second',
|
||||
|
|
|
|||
Loading…
Reference in a new issue