mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
Adopt clickhouse-js and POST method for all queries (#829)
Ref: HDX-1688 All modes for querying a clickhouse instance now go through clickhouse-js. This change is agnostic of the clickhouse server instance. This was tested in localmode tested against image tags - 23-alpine - 24-alpine - 25.4-alpine The docker command run for these was `docker run --ulimit nofile=262144:262144 -p 8123:8123 -p 9000:9000 --env CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 clickhouse/clickhouse-server:IMAGE_TAG_HERE`. Then I just ran `yarn dev` with a modified docker-compose.dev.yml where the `ports` section under `ch-server` is removed, and the otel-collector sends to `host.docker.internal:9000` instead of `ch-server:9000`.
This commit is contained in:
Aaron Knudtson
GitHub
parent
cb26b31914
commit
43a9ca1901
3 changed files with 73 additions and 93 deletions
7
.changeset/orange-carrots-cheer.md
Normal file
7
.changeset/orange-carrots-cheer.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
"@hyperdx/common-utils": patch
|
||||
"@hyperdx/api": patch
|
||||
"@hyperdx/app": patch
|
||||
---
|
||||
|
||||
adopt clickhouse-js for all client side queries
|
||||
|
|
@ -69,11 +69,6 @@ const getConnection: RequestHandler =
|
|||
// prettier-ignore-next-line
|
||||
async (req, res, next) => {
|
||||
try {
|
||||
if (req.headers['authorization'] === 'Basic Og==') {
|
||||
// this means username & password === 0, which indicates we must be
|
||||
// doing some other authorization mechanism (probably connection_id)
|
||||
delete req.headers['authorization'];
|
||||
}
|
||||
const { teamId } = getNonNullUserWithTeam(req);
|
||||
const connection_id = req.headers['x-hyperdx-connection-id']!; // ! because zod already validated
|
||||
delete req.headers['x-hyperdx-connection-id'];
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type {
|
||||
BaseResultSet,
|
||||
ClickHouseSettings,
|
||||
DataFormat,
|
||||
ResponseHeaders,
|
||||
ResponseJSON,
|
||||
|
|
@ -327,6 +328,31 @@ export const computeResultSetRatio = (resultSet: ResponseJSON<any>) => {
|
|||
return result;
|
||||
};
|
||||
|
||||
const localModeFetch: typeof fetch = (input, init) => {
|
||||
if (!init) init = {};
|
||||
const url = new URL(
|
||||
input instanceof URL ? input : input instanceof Request ? input.url : input,
|
||||
);
|
||||
|
||||
// CORS is unhappy with the authorization header, so we will supply as query params instead
|
||||
const auth: string = init.headers?.['Authorization'];
|
||||
const [username, password] = window
|
||||
.atob(auth.substring('Bearer'.length))
|
||||
.split(':');
|
||||
delete init.headers?.['Authorization'];
|
||||
if (username) url.searchParams.set('user', username);
|
||||
if (password) url.searchParams.set('password', password);
|
||||
|
||||
return fetch(`${url.toString()}`, init);
|
||||
};
|
||||
|
||||
const standardModeFetch: typeof fetch = (input, init) => {
|
||||
if (!init) init = {};
|
||||
// authorization is handled on the backend, don't send this header
|
||||
delete init.headers?.['Authorization'];
|
||||
return fetch(input, init);
|
||||
};
|
||||
|
||||
export type ClickhouseClientOptions = {
|
||||
host: string;
|
||||
username?: string;
|
||||
|
|
@ -368,6 +394,7 @@ export class ClickhouseClient {
|
|||
} catch (e) {
|
||||
debugSql = query;
|
||||
}
|
||||
let _url = this.host;
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('--------------------------------------------------------');
|
||||
|
|
@ -378,99 +405,50 @@ export class ClickhouseClient {
|
|||
|
||||
if (isBrowser) {
|
||||
// TODO: check if we can use the client-web directly
|
||||
const { createClient, ResultSet } = await import(
|
||||
'@clickhouse/client-web'
|
||||
);
|
||||
const { createClient } = await import('@clickhouse/client-web');
|
||||
|
||||
const clickhouse_settings: ClickHouseSettings = {
|
||||
date_time_output_format: 'iso',
|
||||
wait_end_of_query: 0,
|
||||
cancel_http_readonly_queries_on_client_close: 1,
|
||||
};
|
||||
const http_headers = {
|
||||
...(connectionId && connectionId !== 'local'
|
||||
? { 'x-hyperdx-connection-id': connectionId }
|
||||
: {}),
|
||||
};
|
||||
let myFetch: typeof fetch;
|
||||
const isLocalMode = this.username != null && this.password != null;
|
||||
if (isLocalMode) {
|
||||
// LocalMode may potentially interact directly with a db, so it needs to
|
||||
// send a get request. @clickhouse/client-web does not currently support
|
||||
// querying via GET
|
||||
const includeCredentials = !isLocalMode;
|
||||
const includeCorsHeader = isLocalMode;
|
||||
|
||||
const searchParams = new URLSearchParams([
|
||||
...(includeCorsHeader ? [['add_http_cors_header', '1']] : []),
|
||||
['query', query],
|
||||
['default_format', format],
|
||||
['date_time_output_format', 'iso'],
|
||||
['wait_end_of_query', '0'],
|
||||
['cancel_http_readonly_queries_on_client_close', '1'],
|
||||
...(this.username ? [['user', this.username]] : []),
|
||||
...(this.password ? [['password', this.password]] : []),
|
||||
...(queryId ? [['query_id', queryId]] : []),
|
||||
...Object.entries(query_params).map(([key, value]) => [
|
||||
`param_${key}`,
|
||||
value,
|
||||
]),
|
||||
...Object.entries(clickhouse_settings ?? {}).map(([key, value]) => [
|
||||
key,
|
||||
value,
|
||||
]),
|
||||
]);
|
||||
const headers = {};
|
||||
if (!isLocalMode && connectionId) {
|
||||
headers['x-hyperdx-connection-id'] = connectionId;
|
||||
}
|
||||
// https://github.com/ClickHouse/clickhouse-js/blob/1ebdd39203730bb99fad4c88eac35d9a5e96b34a/packages/client-web/src/connection/web_connection.ts#L200C7-L200C23
|
||||
const response = await fetch(
|
||||
`${this.host}/?${searchParams.toString()}`,
|
||||
{
|
||||
...(includeCredentials ? { credentials: 'include' } : {}),
|
||||
signal: abort_signal,
|
||||
method: 'GET',
|
||||
headers,
|
||||
},
|
||||
);
|
||||
|
||||
// TODO: Send command to CH to cancel query on abort_signal
|
||||
if (!response.ok) {
|
||||
if (!isSuccessfulResponse(response.status)) {
|
||||
const text = await response.text();
|
||||
throw new ClickHouseQueryError(`${text}`, debugSql);
|
||||
}
|
||||
}
|
||||
|
||||
if (response.body == null) {
|
||||
// TODO: Handle empty responses better?
|
||||
throw new Error('Unexpected empty response from ClickHouse');
|
||||
}
|
||||
return new ResultSet<Format>(
|
||||
response.body,
|
||||
format,
|
||||
queryId ?? '',
|
||||
getResponseHeaders(response),
|
||||
);
|
||||
myFetch = localModeFetch;
|
||||
clickhouse_settings.add_http_cors_header = 1;
|
||||
} else {
|
||||
if (connectionId === undefined) {
|
||||
throw new Error('ConnectionId must be defined');
|
||||
}
|
||||
const clickhouseClient = createClient({
|
||||
url: window.origin,
|
||||
pathname: this.host,
|
||||
http_headers: { 'x-hyperdx-connection-id': connectionId },
|
||||
clickhouse_settings: {
|
||||
date_time_output_format: 'iso',
|
||||
wait_end_of_query: 0,
|
||||
cancel_http_readonly_queries_on_client_close: 1,
|
||||
},
|
||||
username: '',
|
||||
password: '',
|
||||
// Disable keep-alive to prevent multiple concurrent dashboard requests from exceeding the 64KB payload size limit.
|
||||
keep_alive: {
|
||||
enabled: false,
|
||||
},
|
||||
});
|
||||
return clickhouseClient.query<Format>({
|
||||
query,
|
||||
query_params,
|
||||
format,
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
query_id: queryId,
|
||||
}) as Promise<BaseResultSet<ReadableStream, Format>>;
|
||||
_url = `${window.origin}${this.host}`; // this.host is just a pathname in this scenario
|
||||
myFetch = standardModeFetch;
|
||||
}
|
||||
|
||||
const url = new URL(_url);
|
||||
const clickhouseClient = createClient({
|
||||
url: url.origin,
|
||||
pathname: url.pathname,
|
||||
http_headers,
|
||||
clickhouse_settings,
|
||||
username: this.username ?? '',
|
||||
password: this.password ?? '',
|
||||
// Disable keep-alive to prevent multiple concurrent dashboard requests from exceeding the 64KB payload size limit.
|
||||
keep_alive: {
|
||||
enabled: false,
|
||||
},
|
||||
fetch: myFetch,
|
||||
});
|
||||
return clickhouseClient.query<Format>({
|
||||
query,
|
||||
query_params,
|
||||
format,
|
||||
abort_signal,
|
||||
clickhouse_settings,
|
||||
query_id: queryId,
|
||||
}) as Promise<BaseResultSet<ReadableStream, Format>>;
|
||||
} else if (isNode) {
|
||||
const { createClient } = await import('@clickhouse/client');
|
||||
const _client = createClient({
|
||||
|
|
|
|||
Loading…
Reference in a new issue