Improve Support for Dynamic and JSON(<parameters>) Types (#1251)

1. Improves getJSONColumnNames to support JSON type with arguments
2. Improves useRowWhere to support arrays and JSON types

How I tested this:
1. Created a new table and setup various types of formats (`JSON, JSON(max_dynamic_paths = 3, test_key String), Dynamic, Array(Tuple(test_key String, test_value Dynamic)), Map(String, Dynamic)`)
2. Reproduced exact scenario from tickt and fixed (`getJSONColumnNames`)
4. Clicked around the app with various other types, observed that `Dynamic` fields could not be clicked if they had JSON or Array formats. Fixed that (`useRowWhere`)
5. Added tests for changes, ensured existing tests worked for `Dynamuc` useRowWhere by reproducing them in app 

Fixes HDX-2523

.changeset/nervous-pots-join.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/app": patch
+---
+
+Improve Support for Dynamic and JSON(<parameters>) Types

packages/app/src/components/DBRowDataPanel.tsx

@@ -115,7 +115,13 @@ export function useRowData({
 }
 
 export function getJSONColumnNames(meta: ResponseJSON['meta'] | undefined) {
-  return meta?.filter(m => m.type === 'JSON').map(m => m.name) ?? [];
+  return (
+    meta
+      // The type could either be just 'JSON' or it could be 'JSON(<parameters>)'
+      // this is a basic way to match both cases
+      ?.filter(m => m.type === 'JSON' || m.type.startsWith('JSON('))
+      .map(m => m.name) ?? []
+  );
 }
 
 export function RowDataPanel({

packages/app/src/components/DBRowOverviewPanel.tsx

@@ -295,7 +295,14 @@ export function RowOverviewPanel({
                   {...(onPropertyAddClick
                     ? {
                         onPropertyAddClick,
-                        sqlExpression: `${source.resourceAttributesExpression}['${key}']`,
+                        sqlExpression:
+                          source.resourceAttributesExpression &&
+                          jsonColumns?.includes(
+                            source.resourceAttributesExpression,
+                          )
+                            ? // If resource attributes is a JSON column, we need to cast the key to a string so we can run where X in Y queries
+                              `toString(${source.resourceAttributesExpression}.${key})`
+                            : `${source.resourceAttributesExpression}['${key}']`,
                       }
                     : {
                         onPropertyAddClick: undefined,

packages/app/src/components/__tests__/DBRowDataPanel.test.ts

@@ -0,0 +1,15 @@
+import { getJSONColumnNames } from '../DBRowDataPanel';
+
+describe('DBRowDataPanel', () => {
+  describe('getJSONColumnNames', () => {
+    it('should return JSON column names', () => {
+      const meta = [
+        { name: 'col1', type: 'String' },
+        { name: 'col2', type: 'JSON' },
+        { name: 'col3', type: 'JSON(1)' },
+      ];
+      const result = getJSONColumnNames(meta);
+      expect(result).toEqual(['col2', 'col3']);
+    });
+  });
+});

packages/app/src/hooks/__tests__/useRowWhere.test.tsx

@@ -173,7 +173,9 @@ describe('processRowToWhereClause', () => {
     const row = { dynamic_field: '"quoted_value"' };
     const result = processRowToWhereClause(row, columnMap);
 
-    expect(result).toBe("toString(dynamic_field)='quoted_value'");
+    expect(result).toBe(
+      "toJSONString(dynamic_field) = toJSONString(JSONExtract('\\\"quoted_value\\\"', 'Dynamic'))",
+    );
   });
 
   it('should handle Dynamic columns with escaped values', () => {
@@ -192,7 +194,47 @@ describe('processRowToWhereClause', () => {
     const row = { dynamic_field: '{\\"took\\":7, not a valid json' };
     const result = processRowToWhereClause(row, columnMap);
     expect(result).toBe(
-      'toString(dynamic_field)=\'{\\"took\\":7, not a valid json\'',
+      "toJSONString(dynamic_field) = toJSONString(JSONExtract('{\\\\\\\"took\\\\\\\":7, not a valid json', 'Dynamic'))",
+    );
+  });
+
+  it('should handle Dynamic columns with nested values', () => {
+    const columnMap = new Map([
+      [
+        'dynamic_field',
+        {
+          name: 'dynamic_field',
+          type: 'Dynamic',
+          valueExpr: 'dynamic_field',
+          jsType: JSDataType.Dynamic,
+        },
+      ],
+    ]);
+
+    const row = { dynamic_field: "{'foo': {'bar': 'baz'}}" };
+    const result = processRowToWhereClause(row, columnMap);
+    expect(result).toBe(
+      "toJSONString(dynamic_field) = toJSONString(JSONExtract('{\\'foo\\': {\\'bar\\': \\'baz\\'}}', 'Dynamic'))",
+    );
+  });
+
+  it('should handle Dynamic columns with array values', () => {
+    const columnMap = new Map([
+      [
+        'dynamic_field',
+        {
+          name: 'dynamic_field',
+          type: 'Dynamic',
+          valueExpr: 'dynamic_field',
+          jsType: JSDataType.Dynamic,
+        },
+      ],
+    ]);
+
+    const row = { dynamic_field: "['foo', 'bar']" };
+    const result = processRowToWhereClause(row, columnMap);
+    expect(result).toBe(
+      "toJSONString(dynamic_field) = toJSONString(JSONExtract('[\\'foo\\', \\'bar\\']', 'Dynamic'))",
     );
   });
 

packages/app/src/hooks/useRowWhere.tsx

@@ -72,19 +72,15 @@ export function processRowToWhereClause(
             console.warn('Search value/object key too large.');
           }
           // TODO: update when JSON type have new version
-          // will not work for array/object dyanmic data
 
-          // escaped strings needs raw, becuase sqlString will add another layer of escaping
-          // data other than array/object will alwayas return with dobule quote(because of CH)
-          // remove dobule qoute to search correctly
-          return SqlString.format(`toString(?)='?'`, [
-            SqlString.raw(valueExpr),
-            SqlString.raw(
-              value[0] === '"' && value[value.length - 1] === '"'
-                ? value.slice(1, -1)
-                : value,
-            ),
-          ]);
+          // escaped strings needs raw, because sqlString will add another layer of escaping
+          // data other than array/object will always return with double quote(because of CH)
+          // remove double quote to search correctly
+          return SqlString.format(
+            "toJSONString(?) = toJSONString(JSONExtract(?, 'Dynamic'))",
+            [SqlString.raw(valueExpr), value],
+          );
+
         default:
           // Handle nullish values
           if (value == null) {