feat: Include displayed timestamp in default order by (#1279)

Closes HDX-2593 # Summary This PR adds a source's `displayedTimestampValueExpression` (if one exists) to the default order by on the search page. ## Motivation In schemas like our default otel_logs table, there are two timestamp columns: `TimestampTime DateTime` (1-second precision) and a `Timestamp DateTime64(9)` (nanosecond precision). `TimestampTime` is preferred for filtering because it is more granular and in the primary key. However, ordering by `TimestampTime` alone results in an arbitrary order of events within each second: <img width="646" height="158" alt="Screenshot 2025-10-17 at 2 28 50 PM" src="https://github.com/user-attachments/assets/298a340f-387d-4fdf-9298-622388bb6962" /> ## Details The HyperDX source configuration form already supports configuring a 'Displayed Timestamp Column" for a log source. This PR adds the same option for Trace sources. This field is inferred from the otel logs and traces schemas as `Timestamp`. <img width="999" height="383" alt="Screenshot 2025-10-17 at 2 30 13 PM" src="https://github.com/user-attachments/assets/db1ed1eb-7ab1-4d6a-a702-b45b4d2274af" /> If the source has a displayed timestamp column configured, and if this column is different than the source's timestamp value expression, then this field will be added to the default order by which is generated for the search page. This results in a more precise ordering of the events in the logs table within each second: <img width="950" height="233" alt="Screenshot 2025-10-17 at 2 33 16 PM" src="https://github.com/user-attachments/assets/1d8447c5-ce4c-40e5-bce6-f681fe881436" />
2026-04-21 13:37:15 +00:00 · 2025-10-27 14:44:48 +01:00 · 2025-10-27 14:44:48 +01:00 · 21614b94aa
commit 21614b94aa
parent ab7af41f7e
4 changed files with 153 additions and 30 deletions
--- a/.changeset/twelve-beers-buy.md
+++ b/.changeset/twelve-beers-buy.md
@ -0,0 +1,5 @@
+---
+"@hyperdx/app": patch
+---
+
+feat: Include displayed timestamp in default order by
--- a/packages/app/src/DBSearchPage.tsx
+++ b/packages/app/src/DBSearchPage.tsx
@ -532,14 +532,26 @@ function useSearchedConfigToChartConfig({

 function optimizeDefaultOrderBy(
  timestampExpr: string,
+  displayedTimestampExpr: string | undefined,
  sortingKey: string | undefined,
 ) {
  const defaultModifier = 'DESC';
-  const fallbackOrderByItems = [
-    getFirstTimestampValueExpression(timestampExpr ?? ''),
-    defaultModifier,
-  ];
-  const fallbackOrderBy = fallbackOrderByItems.join(' ');
+  const firstTimestampValueExpression =
+    getFirstTimestampValueExpression(timestampExpr ?? '') ?? '';
+  const defaultOrderByItems = [firstTimestampValueExpression];
+  const trimmedDisplayedTimestampExpr = displayedTimestampExpr?.trim();
+
+  if (
+    trimmedDisplayedTimestampExpr &&
+    trimmedDisplayedTimestampExpr !== firstTimestampValueExpression
+  ) {
+    defaultOrderByItems.push(trimmedDisplayedTimestampExpr);
+  }
+
+  const fallbackOrderBy =
+    defaultOrderByItems.length > 1
+      ? `(${defaultOrderByItems.join(', ')}) ${defaultModifier}`
+      : `${defaultOrderByItems[0]} ${defaultModifier}`;

  if (!sortingKey) return fallbackOrderBy;

@ -547,13 +559,17 @@ function optimizeDefaultOrderBy(
  const sortKeys = splitAndTrimWithBracket(sortingKey);
  for (let i = 0; i < sortKeys.length; i++) {
    const sortKey = sortKeys[i];
-    if (sortKey.includes('toStartOf') && sortKey.includes(timestampExpr)) {
+    if (
+      sortKey.includes('toStartOf') &&
+      sortKey.includes(firstTimestampValueExpression)
+    ) {
      orderByArr.push(sortKey);
    } else if (
-      sortKey === timestampExpr ||
+      sortKey === firstTimestampValueExpression ||
      (sortKey.startsWith('toUnixTimestamp') &&
-        sortKey.includes(timestampExpr)) ||
-      (sortKey.startsWith('toDateTime') && sortKey.includes(timestampExpr))
+        sortKey.includes(firstTimestampValueExpression)) ||
+      (sortKey.startsWith('toDateTime') &&
+        sortKey.includes(firstTimestampValueExpression))
    ) {
      if (orderByArr.length === 0) {
        // fallback if the first sort key is the timestamp sort key
@ -562,6 +578,8 @@ function optimizeDefaultOrderBy(
        orderByArr.push(sortKey);
        break;
      }
+    } else if (sortKey === trimmedDisplayedTimestampExpr) {
+      orderByArr.push(sortKey);
    }
  }

@ -570,7 +588,16 @@ function optimizeDefaultOrderBy(
    return fallbackOrderBy;
  }

-  return `(${orderByArr.join(', ')}) ${defaultModifier}`;
+  if (
+    trimmedDisplayedTimestampExpr &&
+    !orderByArr.includes(trimmedDisplayedTimestampExpr)
+  ) {
+    orderByArr.push(trimmedDisplayedTimestampExpr);
+  }
+
+  return orderByArr.length > 1
+    ? `(${orderByArr.join(', ')}) ${defaultModifier}`
+    : `${orderByArr[0]} ${defaultModifier}`;
 }

 export function useDefaultOrderBy(sourceID: string | undefined | null) {
@ -582,6 +609,7 @@ export function useDefaultOrderBy(sourceID: string | undefined | null) {
    () =>
      optimizeDefaultOrderBy(
        source?.timestampValueExpression ?? '',
+        source?.displayedTimestampValueExpression,
        tableMetadata?.sorting_key,
      ),
    [source, tableMetadata],
--- a/packages/app/src/tests/DBSearchPage.test.tsx
+++ b/packages/app/src/tests/DBSearchPage.test.tsx
@ -17,82 +17,157 @@ describe('useDefaultOrderBy', () => {

  describe('optimizeOrderBy function', () => {
    describe('should handle', () => {
-      const mockSource = {
-        timestampValueExpression: 'Timestamp',
-      };
-
      const testCases = [
        {
-          input: undefined,
+          sortingKey: undefined,
          expected: 'Timestamp DESC',
        },
        {
-          input: '',
+          sortingKey: '',
          expected: 'Timestamp DESC',
        },
        {
          // Traces Table
-          input: 'ServiceName, SpanName, toDateTime(Timestamp)',
+          sortingKey: 'ServiceName, SpanName, toDateTime(Timestamp)',
          expected: 'Timestamp DESC',
        },
        {
          // Optimized Traces Table
-          input:
+          sortingKey:
            'toStartOfHour(Timestamp), ServiceName, SpanName, toDateTime(Timestamp)',
          expected: '(toStartOfHour(Timestamp), toDateTime(Timestamp)) DESC',
        },
        {
          // Unsupported for now as it's not a great sort key, want to just
          // use default behavior for this
-          input: 'toDateTime(Timestamp), ServiceName, SpanName, Timestamp',
+          sortingKey: 'toDateTime(Timestamp), ServiceName, SpanName, Timestamp',
          expected: 'Timestamp DESC',
        },
        {
          // Unsupported prefix sort key
-          input: 'toDateTime(Timestamp), ServiceName, SpanName',
+          sortingKey: 'toDateTime(Timestamp), ServiceName, SpanName',
          expected: 'Timestamp DESC',
        },
        {
          // Inverted sort key order, we should not try to optimize this
-          input:
+          sortingKey:
            'ServiceName, toDateTime(Timestamp), SeverityText, toStartOfHour(Timestamp)',
          expected: 'Timestamp DESC',
        },
        {
-          input: 'toStartOfHour(Timestamp), other_column, Timestamp',
+          sortingKey: 'toStartOfHour(Timestamp), other_column, Timestamp',
          expected: '(toStartOfHour(Timestamp), Timestamp) DESC',
        },
        {
-          input: 'Timestamp, other_column',
+          sortingKey: 'Timestamp, other_column',
          expected: 'Timestamp DESC',
        },
        {
-          input: 'user_id, toStartOfHour(Timestamp), status, Timestamp',
+          sortingKey: 'user_id, toStartOfHour(Timestamp), status, Timestamp',
          expected: '(toStartOfHour(Timestamp), Timestamp) DESC',
        },
        {
-          input:
+          sortingKey:
            'toStartOfMinute(Timestamp), user_id, status, toUnixTimestamp(Timestamp)',
          expected:
            '(toStartOfMinute(Timestamp), toUnixTimestamp(Timestamp)) DESC',
        },
        {
          // test variation of toUnixTimestamp
-          input:
+          sortingKey:
            'toStartOfMinute(Timestamp), user_id, status, toUnixTimestamp64Nano(Timestamp)',
          expected:
            '(toStartOfMinute(Timestamp), toUnixTimestamp64Nano(Timestamp)) DESC',
        },
        {
-          input:
+          sortingKey:
            'toUnixTimestamp(toStartOfMinute(Timestamp)), user_id, status, Timestamp',
          expected:
            '(toUnixTimestamp(toStartOfMinute(Timestamp)), Timestamp) DESC',
        },
+        {
+          sortingKey: 'toStartOfMinute(Timestamp), user_id, status, Timestamp',
+          timestampValueExpression: 'Timestamp, toStartOfMinute(Timestamp)',
+          expected: '(toStartOfMinute(Timestamp), Timestamp) DESC',
+        },
+        {
+          sortingKey: 'Timestamp',
+          displayedTimestampValueExpression: 'Timestamp64',
+          expected: '(Timestamp, Timestamp64) DESC',
+        },
+        {
+          sortingKey: 'Timestamp',
+          displayedTimestampValueExpression: 'Timestamp64 ',
+          expected: '(Timestamp, Timestamp64) DESC',
+        },
+        {
+          sortingKey: 'Timestamp',
+          displayedTimestampValueExpression: 'Timestamp',
+          expected: 'Timestamp DESC',
+        },
+        {
+          sortingKey: 'Timestamp',
+          displayedTimestampValueExpression: '',
+          expected: 'Timestamp DESC',
+        },
+        {
+          sortingKey: 'Timestamp, ServiceName, Timestamp64',
+          displayedTimestampValueExpression: 'Timestamp64',
+          expected: '(Timestamp, Timestamp64) DESC',
+        },
+        {
+          sortingKey:
+            'toStartOfMinute(Timestamp), Timestamp, ServiceName, Timestamp64',
+          displayedTimestampValueExpression: 'Timestamp64',
+          expected: '(toStartOfMinute(Timestamp), Timestamp, Timestamp64) DESC',
+        },
+        {
+          sortingKey:
+            'toStartOfMinute(Timestamp), Timestamp64, ServiceName, Timestamp',
+          displayedTimestampValueExpression: 'Timestamp64',
+          expected: '(toStartOfMinute(Timestamp), Timestamp64, Timestamp) DESC',
+        },
+        {
+          sortingKey: 'SomeOtherTimeColumn',
+          displayedTimestampValueExpression: 'Timestamp64',
+          expected: '(Timestamp, Timestamp64) DESC',
+        },
+        {
+          sortingKey: '',
+          displayedTimestampValueExpression: 'Timestamp64',
+          expected: '(Timestamp, Timestamp64) DESC',
+        },
+        {
+          sortingKey: 'ServiceName, TimestampTime, Timestamp',
+          timestampValueExpression: 'TimestampTime, Timestamp',
+          displayedTimestampValueExpression: 'Timestamp',
+          expected: '(TimestampTime, Timestamp) DESC',
+        },
+        {
+          sortingKey: 'ServiceName, TimestampTime, Timestamp',
+          timestampValueExpression: 'Timestamp, TimestampTime',
+          displayedTimestampValueExpression: 'Timestamp',
+          expected: 'Timestamp DESC',
+        },
+        {
+          sortingKey: '',
+          timestampValueExpression: 'Timestamp, TimestampTime',
+          displayedTimestampValueExpression: '',
+          expected: 'Timestamp DESC',
+        },
      ];
      for (const testCase of testCases) {
-        it(`${testCase.input}`, () => {
-          const mockTableMetadata = { sorting_key: testCase.input };
+        it(`${testCase.sortingKey}`, () => {
+          const mockSource = {
+            timestampValueExpression:
+              testCase.timestampValueExpression || 'Timestamp',
+            displayedTimestampValueExpression:
+              testCase.displayedTimestampValueExpression,
+          };
+
+          const mockTableMetadata = {
+            sorting_key: testCase.sortingKey,
+          };

          jest.spyOn(sourceModule, 'useSource').mockReturnValue({
            data: mockSource,
--- a/packages/app/src/components/SourceForm.tsx
+++ b/packages/app/src/components/SourceForm.tsx
@ -285,7 +285,7 @@ export function LogTableModelForm({ control, watch }: TableModelProps) {
        </FormRow>
        <FormRow
          label={'Displayed Timestamp Column'}
-          helpText="This DateTime column is used to display search results."
+          helpText="This DateTime column is used to display and order search results."
        >
          <SQLInlineEditorControlled
            tableConnection={{
@ -631,6 +631,21 @@ export function TraceTableModelForm({ control, watch }: TableModelProps) {
          placeholder="SpanName"
        />
      </FormRow>
+      <FormRow
+        label={'Displayed Timestamp Column'}
+        helpText="This DateTime column is used to display and order search results."
+      >
+        <SQLInlineEditorControlled
+          tableConnection={{
+            databaseName,
+            tableName,
+            connectionId,
+          }}
+          control={control}
+          name="displayedTimestampValueExpression"
+          disableKeywordAutocomplete
+        />
+      </FormRow>
    </Stack>
  );
 }