diff --git a/deployment/services/usage-ingestor.ts b/deployment/services/usage-ingestor.ts index 603a57b01..e0632c21b 100644 --- a/deployment/services/usage-ingestor.ts +++ b/deployment/services/usage-ingestor.ts @@ -41,6 +41,8 @@ export function deployUsageIngestor({ CLICKHOUSE_PORT: clickhouse.config.port, CLICKHOUSE_USERNAME: clickhouse.config.username, CLICKHOUSE_PASSWORD: clickhouse.config.password, + CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: '30000', // flush data after max 30 seconds + CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: '200000000', // flush data when the buffer reaches 200MB ...(clickhouse.config.cloud ? { CLICKHOUSE_MIRROR_PROTOCOL: clickhouse.config.cloud.protocol, diff --git a/docker-compose.community.yml b/docker-compose.community.yml index 76690d892..1670ab3f3 100644 --- a/docker-compose.community.yml +++ b/docker-compose.community.yml @@ -22,7 +22,7 @@ services: - ./.hive/postgres:/var/lib/postgresql/data clickhouse: - image: clickhouse/clickhouse-server:22.9-alpine + image: clickhouse/clickhouse-server:22.11-alpine healthcheck: test: ['CMD', 'wget', '--spider', '-q', 'localhost:8123/ping'] interval: 5s diff --git a/integration-tests/docker-compose.yml b/integration-tests/docker-compose.yml index 88e9e8fcb..30cb2da2b 100644 --- a/integration-tests/docker-compose.yml +++ b/integration-tests/docker-compose.yml @@ -18,8 +18,10 @@ services: - 'stack' clickhouse: - image: clickhouse/clickhouse-server:22.9-alpine + image: clickhouse/clickhouse-server:22.11-alpine volumes: + - ./volumes/clickhouse/logs:/var/log/clickhouse-server + - ./volumes/clickhouse/db:/var/lib/clickhouse - ../packages/services/storage/configs/clickhouse:/etc/clickhouse-server/conf.d healthcheck: test: ['CMD', 'wget', '--spider', '-q', 'localhost:8123/ping'] @@ -428,6 +430,8 @@ services: CLICKHOUSE_PORT: 8123 CLICKHOUSE_USERNAME: test CLICKHOUSE_PASSWORD: test + CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: 500 + CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: 1000 PORT: 3007 usage-estimator: diff --git a/integration-tests/scripts/filter-log.mjs b/integration-tests/scripts/filter-log.mjs new file mode 100644 index 000000000..c15e26c80 --- /dev/null +++ b/integration-tests/scripts/filter-log.mjs @@ -0,0 +1,19 @@ +import path from 'path'; +import fs from 'fs'; +import { fileURLToPath } from 'url'; + +/** + * node ./scripts/filter-log.mjs usage usage-ingestor emails + * + * Creates the `dockest-filtered.log` file with logs from the `dockest.log` file but only for the provided services. + */ +const [, , ...services] = process.argv; + +const serviceNames = services.map(name => name.replace(/[^a-z]/i, '-')); +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const logPath = path.resolve(__dirname, '../dockest.log'); +const filteredLogPath = path.resolve(__dirname, '../dockest-filtered.log'); +const logLines = fs.readFileSync(logPath, 'utf-8').split('\n'); +const filteredLogLines = logLines.filter(line => serviceNames.some(name => line.includes(name))); + +fs.writeFileSync(filteredLogPath, filteredLogLines.join('\n')); diff --git a/packages/libraries/client/src/version.ts b/packages/libraries/client/src/version.ts index dbd94aed6..7f19ed108 100644 --- a/packages/libraries/client/src/version.ts +++ b/packages/libraries/client/src/version.ts @@ -1 +1 @@ -export const version = '0.21.2'; +export const version = '0.21.3'; diff --git a/packages/services/storage/docker-compose.yml b/packages/services/storage/docker-compose.yml index a1ceea278..26cdb2acb 100644 --- a/packages/services/storage/docker-compose.yml +++ b/packages/services/storage/docker-compose.yml @@ -38,7 +38,7 @@ services: - './volumes/redis/db:/bitnami/redis/data' clickhouse: - image: clickhouse/clickhouse-server:22.9-alpine + image: clickhouse/clickhouse-server:22.11-alpine environment: CLICKHOUSE_USER: test CLICKHOUSE_PASSWORD: test diff --git a/packages/services/usage-ingestor/.env.template b/packages/services/usage-ingestor/.env.template index 8c61ab429..03eba1245 100644 --- a/packages/services/usage-ingestor/.env.template +++ b/packages/services/usage-ingestor/.env.template @@ -8,4 +8,6 @@ CLICKHOUSE_HOST="localhost" CLICKHOUSE_PORT="8123" CLICKHOUSE_USERNAME="test" CLICKHOUSE_PASSWORD="test" +CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS="30000" +CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE="200000000" PORT=4002 \ No newline at end of file diff --git a/packages/services/usage-ingestor/src/environment.ts b/packages/services/usage-ingestor/src/environment.ts index b7b4949d7..798036b1a 100644 --- a/packages/services/usage-ingestor/src/environment.ts +++ b/packages/services/usage-ingestor/src/environment.ts @@ -63,6 +63,8 @@ const ClickHouseModel = zod.object({ CLICKHOUSE_PORT: NumberFromString, CLICKHOUSE_USERNAME: zod.string(), CLICKHOUSE_PASSWORD: zod.string(), + CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: emptyString(NumberFromString.optional()), + CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: emptyString(NumberFromString.optional()), }); const ClickHouseMirrorModel = zod.union([ @@ -72,6 +74,8 @@ const ClickHouseMirrorModel = zod.union([ CLICKHOUSE_MIRROR_PORT: NumberFromString, CLICKHOUSE_MIRROR_USERNAME: zod.string(), CLICKHOUSE_MIRROR_PASSWORD: zod.string(), + CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: emptyString(NumberFromString.optional()), + CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: emptyString(NumberFromString.optional()), }), zod.object({}), ]); @@ -172,6 +176,8 @@ export const env = { port: clickhouse.CLICKHOUSE_PORT, username: clickhouse.CLICKHOUSE_USERNAME, password: clickhouse.CLICKHOUSE_PASSWORD, + async_insert_busy_timeout_ms: clickhouse.CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS ?? 30_000, + async_insert_max_data_size: clickhouse.CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE ?? 200_000_000, }, clickhouseMirror: 'CLICKHOUSE_MIRROR_PROTOCOL' in clickhouseMirror @@ -181,6 +187,10 @@ export const env = { port: clickhouseMirror.CLICKHOUSE_MIRROR_PORT, username: clickhouseMirror.CLICKHOUSE_MIRROR_USERNAME, password: clickhouseMirror.CLICKHOUSE_MIRROR_PASSWORD, + async_insert_busy_timeout_ms: + clickhouse.CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS ?? 30_000, + async_insert_max_data_size: + clickhouse.CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE ?? 200_000_000, } : null, heartbeat: base.HEARTBEAT_ENDPOINT ? { endpoint: base.HEARTBEAT_ENDPOINT } : null, diff --git a/packages/services/usage-ingestor/src/index.ts b/packages/services/usage-ingestor/src/index.ts index 92bcc822a..163632d9b 100644 --- a/packages/services/usage-ingestor/src/index.ts +++ b/packages/services/usage-ingestor/src/index.ts @@ -32,13 +32,7 @@ async function main() { try { const { readiness, start, stop } = createIngestor({ logger: server.log, - clickhouse: { - protocol: env.clickhouse.protocol, - host: env.clickhouse.host, - port: env.clickhouse.port, - username: env.clickhouse.username, - password: env.clickhouse.password, - }, + clickhouse: env.clickhouse, clickhouseMirror: env.clickhouseMirror, kafka: { topic: env.kafka.topic, diff --git a/packages/services/usage-ingestor/src/writer.ts b/packages/services/usage-ingestor/src/writer.ts index 7d24b7df2..739879a3b 100644 --- a/packages/services/usage-ingestor/src/writer.ts +++ b/packages/services/usage-ingestor/src/writer.ts @@ -24,6 +24,8 @@ export interface ClickHouseConfig { port: number; username: string; password: string; + async_insert_busy_timeout_ms: number; + async_insert_max_data_size: number; } const operationsFields = operationsOrder.join(', '); @@ -165,8 +167,8 @@ async function writeCsv( query, async_insert: 1, wait_for_async_insert: 0, - async_insert_busy_timeout_ms: 30_000, - async_insert_max_data_size: 200_000_000, + async_insert_busy_timeout_ms: config.async_insert_busy_timeout_ms, + async_insert_max_data_size: config.async_insert_max_data_size, }, username: config.username, password: config.password,