diff --git a/.changeset/modern-coats-exercise.md b/.changeset/modern-coats-exercise.md new file mode 100644 index 00000000..4a296e15 --- /dev/null +++ b/.changeset/modern-coats-exercise.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +refactor: decouple clickhouse client into browser.ts and node.ts diff --git a/.changeset/popular-geese-sin.md b/.changeset/popular-geese-sin.md new file mode 100644 index 00000000..d42dcb2e --- /dev/null +++ b/.changeset/popular-geese-sin.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +bump: default request_timeout to 1hr diff --git a/packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts b/packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts index 50ed071a..82032162 100644 --- a/packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts +++ b/packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts @@ -1,6 +1,7 @@ // TODO: we might want to move this test file to common-utils package -import { ChSql, ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse'; +import { ChSql } from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { getMetadata } from '@hyperdx/common-utils/dist/metadata'; import { renderChartConfig } from '@hyperdx/common-utils/dist/renderChartConfig'; import { diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 1e9e949d..7f3365b5 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -1,4 +1,4 @@ -import { getJSNativeCreateClient } from '@hyperdx/common-utils/dist/clickhouse'; +import { createNativeClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { DisplayType, SavedChartConfig, @@ -38,8 +38,7 @@ let clickhouseClient: any; const getClickhouseClient = async () => { if (!clickhouseClient) { - const createClient = await getJSNativeCreateClient(); - clickhouseClient = createClient({ + clickhouseClient = createNativeClient({ url: config.CLICKHOUSE_HOST, username: config.CLICKHOUSE_USER, password: config.CLICKHOUSE_PASSWORD, diff --git a/packages/api/src/routers/external-api/__tests__/charts.test.ts b/packages/api/src/routers/external-api/__tests__/charts.test.ts index 9a257f43..f996a1e8 100644 --- a/packages/api/src/routers/external-api/__tests__/charts.test.ts +++ b/packages/api/src/routers/external-api/__tests__/charts.test.ts @@ -1,4 +1,4 @@ -import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { MetricsDataType } from '@hyperdx/common-utils/dist/types'; import { ObjectId } from 'mongodb'; diff --git a/packages/api/src/routers/external-api/v2/charts.ts b/packages/api/src/routers/external-api/v2/charts.ts index f04c4486..96de969e 100644 --- a/packages/api/src/routers/external-api/v2/charts.ts +++ b/packages/api/src/routers/external-api/v2/charts.ts @@ -1,4 +1,4 @@ -import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { getMetadata } from '@hyperdx/common-utils/dist/metadata'; import { ChartConfigWithOptDateRange, diff --git a/packages/api/src/tasks/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/__tests__/checkAlerts.test.ts index 0b07e340..91107492 100644 --- a/packages/api/src/tasks/__tests__/checkAlerts.test.ts +++ b/packages/api/src/tasks/__tests__/checkAlerts.test.ts @@ -1,4 +1,4 @@ -import * as clickhouse from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import mongoose from 'mongoose'; import ms from 'ms'; @@ -725,7 +725,7 @@ describe('checkAlerts', () => { savedSearch, }; - const clickhouseClient = new clickhouse.ClickhouseClient({ + const clickhouseClient = new ClickhouseClient({ host: connection.host, username: connection.username, password: connection.password, @@ -965,7 +965,7 @@ describe('checkAlerts', () => { dashboard, }; - const clickhouseClient = new clickhouse.ClickhouseClient({ + const clickhouseClient = new ClickhouseClient({ host: connection.host, username: connection.username, password: connection.password, @@ -1195,7 +1195,7 @@ describe('checkAlerts', () => { dashboard, }; - const clickhouseClient = new clickhouse.ClickhouseClient({ + const clickhouseClient = new ClickhouseClient({ host: connection.host, username: connection.username, password: connection.password, @@ -1404,7 +1404,7 @@ describe('checkAlerts', () => { dashboard, }; - const clickhouseClient = new clickhouse.ClickhouseClient({ + const clickhouseClient = new ClickhouseClient({ host: connection.host, username: connection.username, password: connection.password, diff --git a/packages/api/src/tasks/__tests__/singleInvocationAlert.test.ts b/packages/api/src/tasks/__tests__/singleInvocationAlert.test.ts index 15f65fda..99090daf 100644 --- a/packages/api/src/tasks/__tests__/singleInvocationAlert.test.ts +++ b/packages/api/src/tasks/__tests__/singleInvocationAlert.test.ts @@ -1,4 +1,4 @@ -import * as clickhouse from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { createServer } from 'http'; import mongoose from 'mongoose'; import ms from 'ms'; @@ -188,7 +188,7 @@ describe('Single Invocation Alert Test', () => { taskType: AlertTaskType.SAVED_SEARCH, savedSearch, }; - const clickhouseClient = new clickhouse.ClickhouseClient({ + const clickhouseClient = new ClickhouseClient({ host: connection.host, username: connection.username, password: connection.password, @@ -404,7 +404,7 @@ describe('Single Invocation Alert Test', () => { dashboard, }; - const clickhouseClient = new clickhouse.ClickhouseClient({ + const clickhouseClient = new ClickhouseClient({ host: connection.host, username: connection.username, password: connection.password, diff --git a/packages/api/src/tasks/checkAlerts.ts b/packages/api/src/tasks/checkAlerts.ts index b43415ae..43524f47 100644 --- a/packages/api/src/tasks/checkAlerts.ts +++ b/packages/api/src/tasks/checkAlerts.ts @@ -3,6 +3,7 @@ // -------------------------------------------------------- import PQueue from '@esm2cjs/p-queue'; import * as clickhouse from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { getMetadata, Metadata } from '@hyperdx/common-utils/dist/metadata'; import { ChartConfigWithOptDateRange, @@ -67,7 +68,7 @@ const fireChannelEvent = async ({ alert: IAlert; alertProvider: AlertProvider; attributes: Record; // TODO: support other types than string - clickhouseClient: clickhouse.ClickhouseClient; + clickhouseClient: ClickhouseClient; dashboard?: IDashboard | null; endTime: Date; group?: string; @@ -138,7 +139,7 @@ const fireChannelEvent = async ({ export const processAlert = async ( now: Date, details: AlertDetails, - clickhouseClient: clickhouse.ClickhouseClient, + clickhouseClient: ClickhouseClient, connectionId: string, alertProvider: AlertProvider, ) => { @@ -397,7 +398,7 @@ export default class CheckAlertTask implements HdxTask { }); } - const clickhouseClient = new clickhouse.ClickhouseClient({ + const clickhouseClient = new ClickhouseClient({ host: conn.host, username: conn.username, password: conn.password, diff --git a/packages/api/src/tasks/template.ts b/packages/api/src/tasks/template.ts index ca4041a0..d49bedc9 100644 --- a/packages/api/src/tasks/template.ts +++ b/packages/api/src/tasks/template.ts @@ -1,4 +1,5 @@ import * as clickhouse from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { Metadata } from '@hyperdx/common-utils/dist/metadata'; import { renderChartConfig } from '@hyperdx/common-utils/dist/renderChartConfig'; import { @@ -357,7 +358,7 @@ export const renderAlertTemplate = async ({ team, }: { alertProvider: AlertProvider; - clickhouseClient: clickhouse.ClickhouseClient; + clickhouseClient: ClickhouseClient; metadata: Metadata; template?: string | null; title: string; diff --git a/packages/api/src/tasks/usageStats.ts b/packages/api/src/tasks/usageStats.ts index 25bd8269..e9969916 100644 --- a/packages/api/src/tasks/usageStats.ts +++ b/packages/api/src/tasks/usageStats.ts @@ -1,7 +1,5 @@ -import { - ClickhouseClient, - ResponseJSON, -} from '@hyperdx/common-utils/dist/clickhouse'; +import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types'; import * as HyperDX from '@hyperdx/node-opentelemetry'; import ms from 'ms'; diff --git a/packages/app/src/clickhouse.ts b/packages/app/src/clickhouse.ts index 7e0bb1b3..46af9a8f 100644 --- a/packages/app/src/clickhouse.ts +++ b/packages/app/src/clickhouse.ts @@ -7,10 +7,10 @@ import { chSql, - ClickhouseClient, ColumnMeta, ResponseJSON, } from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; import { useQuery, UseQueryOptions } from '@tanstack/react-query'; import { IS_LOCAL_MODE } from '@/config'; diff --git a/packages/app/src/components/ConnectionForm.tsx b/packages/app/src/components/ConnectionForm.tsx index e1804242..d41f3f7c 100644 --- a/packages/app/src/components/ConnectionForm.tsx +++ b/packages/app/src/components/ConnectionForm.tsx @@ -1,6 +1,6 @@ import { useCallback, useEffect, useState } from 'react'; import { useForm } from 'react-hook-form'; -import { testLocalConnection } from '@hyperdx/common-utils/dist/clickhouse'; +import { testLocalConnection } from '@hyperdx/common-utils/dist/clickhouse/browser'; import { Connection } from '@hyperdx/common-utils/dist/types'; import { Box, Button, Flex, Group, Stack, Text, Tooltip } from '@mantine/core'; import { notifications } from '@mantine/notifications'; diff --git a/packages/app/src/connection.ts b/packages/app/src/connection.ts index 5bca92d7..1aa7a684 100644 --- a/packages/app/src/connection.ts +++ b/packages/app/src/connection.ts @@ -1,5 +1,5 @@ import store from 'store2'; -import { testLocalConnection } from '@hyperdx/common-utils/dist/clickhouse'; +import { testLocalConnection } from '@hyperdx/common-utils/dist/clickhouse/browser'; import { Connection } from '@hyperdx/common-utils/dist/types'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index ffd7f41b..4c1d53a3 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -1,6 +1,6 @@ import React from 'react'; import * as metadataModule from '@hyperdx/app/src/metadata'; -import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; import { Metadata, MetadataCache } from '@hyperdx/common-utils/dist/metadata'; import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 8484a89d..11668dd7 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1,4 +1,4 @@ -import { ClickhouseClient } from '../clickhouse'; +import { ClickhouseClient } from '../clickhouse/node'; import { Metadata, MetadataCache } from '../metadata'; import * as renderChartConfigModule from '../renderChartConfig'; import { ChartConfigWithDateRange } from '../types'; diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index 5727b4d3..d20690df 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -1,4 +1,4 @@ -import { ClickhouseClient } from '@/clickhouse'; +import { ClickhouseClient } from '@/clickhouse/node'; import { getMetadata } from '@/metadata'; import { CustomSchemaSQLSerializerV2 } from '@/queryParser'; diff --git a/packages/common-utils/src/clickhouse/browser.ts b/packages/common-utils/src/clickhouse/browser.ts new file mode 100644 index 00000000..f76a9a93 --- /dev/null +++ b/packages/common-utils/src/clickhouse/browser.ts @@ -0,0 +1,145 @@ +import type { + BaseResultSet, + ClickHouseSettings, + DataFormat, +} from '@clickhouse/client-common'; +import { createClient } from '@clickhouse/client-web'; + +import { + BaseClickhouseClient, + ClickhouseClientOptions, + parameterizedQueryToSql, + QueryInputs, +} from './index'; + +const localModeFetch: typeof fetch = (input, init) => { + if (!init) init = {}; + const url = new URL( + input instanceof URL ? input : input instanceof Request ? input.url : input, + ); + + // CORS is unhappy with the authorization header, so we will supply as query params instead + const auth: string = init.headers?.['Authorization']; + const [username, password] = window + .atob(auth.substring('Bearer'.length)) + .split(':'); + delete init.headers?.['Authorization']; + delete init.headers?.['authorization']; + if (username) url.searchParams.set('user', username); + if (password) url.searchParams.set('password', password); + + return fetch(`${url.toString()}`, init); +}; + +const standardModeFetch: typeof fetch = (input, init) => { + if (!init) init = {}; + // authorization is handled on the backend, don't send this header + delete init.headers?.['Authorization']; + delete init.headers?.['authorization']; + return fetch(input, init); +}; + +export const testLocalConnection = async ({ + host, + username, + password, +}: { + host: string; + username: string; + password: string; +}): Promise => { + try { + const client = new ClickhouseClient({ host, username, password }); + const result = await client.query({ + query: 'SELECT 1', + format: 'TabSeparatedRaw', + }); + return result.text().then(text => text.trim() === '1'); + } catch (e) { + console.warn('Failed to test local connection', e); + return false; + } +}; + +export class ClickhouseClient extends BaseClickhouseClient { + constructor(options: ClickhouseClientOptions) { + super(options); + } + + protected async __query({ + query, + format = 'JSON' as Format, + query_params = {}, + abort_signal, + clickhouse_settings: external_clickhouse_settings, + connectionId, + queryId, + }: QueryInputs): Promise> { + let debugSql = ''; + try { + debugSql = parameterizedQueryToSql({ sql: query, params: query_params }); + } catch (e) { + debugSql = query; + } + let _url = this.host; + + // eslint-disable-next-line no-console + console.log('--------------------------------------------------------'); + // eslint-disable-next-line no-console + console.log('Sending Query:', debugSql); + // eslint-disable-next-line no-console + console.log('--------------------------------------------------------'); + + let clickhouse_settings = structuredClone( + external_clickhouse_settings || {}, + ); + if (clickhouse_settings?.max_rows_to_read && this.maxRowReadOnly) { + delete clickhouse_settings['max_rows_to_read']; + } + + clickhouse_settings = { + date_time_output_format: 'iso', + wait_end_of_query: 0, + cancel_http_readonly_queries_on_client_close: 1, + ...clickhouse_settings, + }; + const http_headers: { [header: string]: string } = { + ...(connectionId && connectionId !== 'local' + ? { 'x-hyperdx-connection-id': connectionId } + : {}), + }; + let myFetch: typeof fetch; + const isLocalMode = this.username != null && this.password != null; + if (isLocalMode) { + myFetch = localModeFetch; + clickhouse_settings.add_http_cors_header = 1; + } else { + _url = `${window.origin}${this.host}`; // this.host is just a pathname in this scenario + myFetch = standardModeFetch; + } + + const url = new URL(_url); + const clickhouseClient = createClient({ + url: url.origin, + pathname: url.pathname, + http_headers, + clickhouse_settings, + username: this.username ?? '', + password: this.password ?? '', + // Disable keep-alive to prevent multiple concurrent dashboard requests from exceeding the 64KB payload size limit. + keep_alive: { + enabled: false, + }, + fetch: myFetch, + request_timeout: this.requestTimeout, + }); + return clickhouseClient.query({ + query, + query_params, + format, + abort_signal, + clickhouse_settings, + query_id: queryId, + }) as Promise>; + } +} diff --git a/packages/common-utils/src/clickhouse.ts b/packages/common-utils/src/clickhouse/index.ts similarity index 75% rename from packages/common-utils/src/clickhouse.ts rename to packages/common-utils/src/clickhouse/index.ts index b8c59d31..b1905a55 100644 --- a/packages/common-utils/src/clickhouse.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -10,15 +10,14 @@ import { isSuccessfulResponse } from '@clickhouse/client-common'; import * as SQLParser from 'node-sql-parser'; import objectHash from 'object-hash'; +import { Metadata } from '@/metadata'; import { renderChartConfig, setChartSelectsAlias, splitChartConfigs, } from '@/renderChartConfig'; import { ChartConfigWithOptDateRange, SQLInterval } from '@/types'; -import { hashCode, isBrowser, isNode, timeBucketByGranularity } from '@/utils'; - -import { Metadata } from './metadata'; +import { hashCode } from '@/utils'; // export @clickhouse/client-common types export type { @@ -357,34 +356,7 @@ export const computeResultSetRatio = (resultSet: ResponseJSON) => { return result; }; -const localModeFetch: typeof fetch = (input, init) => { - if (!init) init = {}; - const url = new URL( - input instanceof URL ? input : input instanceof Request ? input.url : input, - ); - - // CORS is unhappy with the authorization header, so we will supply as query params instead - const auth: string = init.headers?.['Authorization']; - const [username, password] = window - .atob(auth.substring('Bearer'.length)) - .split(':'); - delete init.headers?.['Authorization']; - delete init.headers?.['authorization']; - if (username) url.searchParams.set('user', username); - if (password) url.searchParams.set('password', password); - - return fetch(`${url.toString()}`, init); -}; - -const standardModeFetch: typeof fetch = (input, init) => { - if (!init) init = {}; - // authorization is handled on the backend, don't send this header - delete init.headers?.['Authorization']; - delete init.headers?.['authorization']; - return fetch(input, init); -}; - -interface QueryInputs { +export interface QueryInputs { query: string; format?: Format; abort_signal?: AbortSignal; @@ -400,32 +372,17 @@ export type ClickhouseClientOptions = { password?: string; }; -export const getJSNativeCreateClient = async () => { - if (isBrowser) { - // Only import client-web in browser environment - const { createClient } = await import('@clickhouse/client-web'); - return createClient; - } else if (isNode) { - // Use require with eval to prevent webpack from analyzing this import - // This ensures @clickhouse/client is never bundled in browser builds - const { createClient } = eval('require')( - '@clickhouse/client', - ) as typeof import('@clickhouse/client'); - return createClient; - } - throw new Error('Unsupported environment'); -}; - -export class ClickhouseClient { - private readonly host: string; - private readonly username?: string; - private readonly password?: string; +export abstract class BaseClickhouseClient { + protected readonly host: string; + protected readonly username?: string; + protected readonly password?: string; /* * Some clickhouse db's (the demo instance for example) make the * max_rows_to_read setting readonly and the query will fail if you try to * query with max_rows_to_read specified */ - private maxRowReadOnly: boolean; + protected maxRowReadOnly: boolean; + protected requestTimeout: number = 3600000; // TODO: make configurable constructor({ host, username, password }: ClickhouseClientOptions) { this.host = host; @@ -479,111 +436,9 @@ export class ClickhouseClient { throw new Error('ClickHouseClient query impossible codepath'); } - // https://github.com/ClickHouse/clickhouse-js/blob/1ebdd39203730bb99fad4c88eac35d9a5e96b34a/packages/client-web/src/connection/web_connection.ts#L151 - private async __query({ - query, - format = 'JSON' as Format, - query_params = {}, - abort_signal, - clickhouse_settings: external_clickhouse_settings, - connectionId, - queryId, - }: QueryInputs): Promise> { - let debugSql = ''; - try { - debugSql = parameterizedQueryToSql({ sql: query, params: query_params }); - } catch (e) { - debugSql = query; - } - let _url = this.host; - - // eslint-disable-next-line no-console - console.log('--------------------------------------------------------'); - // eslint-disable-next-line no-console - console.log('Sending Query:', debugSql); - // eslint-disable-next-line no-console - console.log('--------------------------------------------------------'); - - let clickhouse_settings = structuredClone( - external_clickhouse_settings || {}, - ); - if (clickhouse_settings?.max_rows_to_read && this.maxRowReadOnly) { - delete clickhouse_settings['max_rows_to_read']; - } - - const createClient = await getJSNativeCreateClient(); - - if (isBrowser) { - clickhouse_settings = { - date_time_output_format: 'iso', - wait_end_of_query: 0, - cancel_http_readonly_queries_on_client_close: 1, - ...clickhouse_settings, - }; - const http_headers = { - ...(connectionId && connectionId !== 'local' - ? { 'x-hyperdx-connection-id': connectionId } - : {}), - }; - let myFetch: typeof fetch; - const isLocalMode = this.username != null && this.password != null; - if (isLocalMode) { - myFetch = localModeFetch; - clickhouse_settings.add_http_cors_header = 1; - } else { - _url = `${window.origin}${this.host}`; // this.host is just a pathname in this scenario - myFetch = standardModeFetch; - } - - const url = new URL(_url); - const clickhouseClient = createClient({ - url: url.origin, - pathname: url.pathname, - http_headers, - clickhouse_settings, - username: this.username ?? '', - password: this.password ?? '', - // Disable keep-alive to prevent multiple concurrent dashboard requests from exceeding the 64KB payload size limit. - keep_alive: { - enabled: false, - }, - fetch: myFetch, - }); - return clickhouseClient.query({ - query, - query_params, - format, - abort_signal, - clickhouse_settings, - query_id: queryId, - }) as Promise>; - } else if (isNode) { - const _client = createClient({ - url: this.host, - username: this.username, - password: this.password, - }); - - // TODO: Custom error handling - return _client.query({ - query, - query_params, - format, - abort_signal, - clickhouse_settings: { - date_time_output_format: 'iso', - wait_end_of_query: 0, - cancel_http_readonly_queries_on_client_close: 1, - ...clickhouse_settings, - }, - query_id: queryId, - }) as unknown as Promise>; - } else { - throw new Error( - 'ClickhouseClient is only supported in the browser or node environment', - ); - } - } + protected abstract __query( + inputs: QueryInputs, + ): Promise>; // TODO: only used when multi-series 'metrics' is selected (no effects on the events chart) // eventually we want to generate union CTEs on the db side instead of computing it on the client side @@ -681,28 +536,6 @@ export class ClickhouseClient { } } -export const testLocalConnection = async ({ - host, - username, - password, -}: { - host: string; - username: string; - password: string; -}): Promise => { - try { - const client = new ClickhouseClient({ host, username, password }); - const result = await client.query({ - query: 'SELECT 1', - format: 'TabSeparatedRaw', - }); - return result.text().then(text => text.trim() === '1'); - } catch (e) { - console.warn('Failed to test local connection', e); - return false; - } -}; - export const tableExpr = ({ database, table, diff --git a/packages/common-utils/src/clickhouse/node.ts b/packages/common-utils/src/clickhouse/node.ts new file mode 100644 index 00000000..70d155ac --- /dev/null +++ b/packages/common-utils/src/clickhouse/node.ts @@ -0,0 +1,74 @@ +import { createClient } from '@clickhouse/client'; +import type { + BaseResultSet, + ClickHouseSettings, + DataFormat, +} from '@clickhouse/client-common'; + +import { + BaseClickhouseClient, + ClickhouseClientOptions, + parameterizedQueryToSql, + QueryInputs, +} from './index'; + +// for api fixtures +export { createClient as createNativeClient }; + +export class ClickhouseClient extends BaseClickhouseClient { + constructor(options: ClickhouseClientOptions) { + super(options); + } + + protected async __query({ + query, + format = 'JSON' as Format, + query_params = {}, + abort_signal, + clickhouse_settings: external_clickhouse_settings, + queryId, + }: QueryInputs): Promise> { + let debugSql = ''; + try { + debugSql = parameterizedQueryToSql({ sql: query, params: query_params }); + } catch (e) { + debugSql = query; + } + + // eslint-disable-next-line no-console + console.log('--------------------------------------------------------'); + // eslint-disable-next-line no-console + console.log('Sending Query:', debugSql); + // eslint-disable-next-line no-console + console.log('--------------------------------------------------------'); + + const clickhouse_settings = structuredClone( + external_clickhouse_settings || {}, + ); + if (clickhouse_settings?.max_rows_to_read && this.maxRowReadOnly) { + delete clickhouse_settings['max_rows_to_read']; + } + + const _client = createClient({ + url: this.host, + username: this.username, + password: this.password, + request_timeout: this.requestTimeout, + }); + + // TODO: Custom error handling + return _client.query({ + query, + query_params, + format, + abort_signal, + clickhouse_settings: { + date_time_output_format: 'iso', + wait_end_of_query: 0, + cancel_http_readonly_queries_on_client_close: 1, + ...clickhouse_settings, + }, + query_id: queryId, + }) as unknown as Promise>; + } +} diff --git a/packages/common-utils/src/metadata.ts b/packages/common-utils/src/metadata.ts index 8551ea65..1dd8b6bb 100644 --- a/packages/common-utils/src/metadata.ts +++ b/packages/common-utils/src/metadata.ts @@ -1,9 +1,9 @@ import type { ClickHouseSettings } from '@clickhouse/client-common'; import { + BaseClickhouseClient, ChSql, chSql, - ClickhouseClient, ColumnMeta, convertCHDataTypeToJSType, filterColumnMetaByType, @@ -91,12 +91,12 @@ export type TableMetadata = { }; export class Metadata { - private readonly clickhouseClient: ClickhouseClient; + private readonly clickhouseClient: BaseClickhouseClient; private readonly cache: MetadataCache; private readonly clickhouseSettings: ClickHouseSettings; constructor( - clickhouseClient: ClickhouseClient, + clickhouseClient: BaseClickhouseClient, cache: MetadataCache, settings?: ClickHouseSettings, ) { @@ -528,5 +528,5 @@ const __LOCAL_CACHE__ = new MetadataCache(); // TODO: better to init the Metadata object on the client side // also the client should be able to choose the cache strategy -export const getMetadata = (clickhouseClient: ClickhouseClient) => +export const getMetadata = (clickhouseClient: BaseClickhouseClient) => new Metadata(clickhouseClient, __LOCAL_CACHE__);