Update clickhouse/clickhouse-server Docker tag to v22.11 (#695)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Kamil Kisiela <kamil.kisiela@gmail.com>

deployment/services/usage-ingestor.ts

@@ -41,6 +41,8 @@ export function deployUsageIngestor({
     CLICKHOUSE_PORT: clickhouse.config.port,
     CLICKHOUSE_USERNAME: clickhouse.config.username,
     CLICKHOUSE_PASSWORD: clickhouse.config.password,
+    CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: '30000', // flush data after max 30 seconds
+    CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: '200000000', // flush data when the buffer reaches 200MB
     ...(clickhouse.config.cloud
       ? {
           CLICKHOUSE_MIRROR_PROTOCOL: clickhouse.config.cloud.protocol,

docker-compose.community.yml

@@ -22,7 +22,7 @@ services:
       - ./.hive/postgres:/var/lib/postgresql/data
 
   clickhouse:
-    image: clickhouse/clickhouse-server:22.9-alpine
+    image: clickhouse/clickhouse-server:22.11-alpine
     healthcheck:
       test: ['CMD', 'wget', '--spider', '-q', 'localhost:8123/ping']
       interval: 5s

integration-tests/docker-compose.yml

@@ -18,8 +18,10 @@ services:
       - 'stack'
 
   clickhouse:
-    image: clickhouse/clickhouse-server:22.9-alpine
+    image: clickhouse/clickhouse-server:22.11-alpine
     volumes:
+      - ./volumes/clickhouse/logs:/var/log/clickhouse-server
+      - ./volumes/clickhouse/db:/var/lib/clickhouse
       - ../packages/services/storage/configs/clickhouse:/etc/clickhouse-server/conf.d
     healthcheck:
       test: ['CMD', 'wget', '--spider', '-q', 'localhost:8123/ping']
@@ -428,6 +430,8 @@ services:
       CLICKHOUSE_PORT: 8123
       CLICKHOUSE_USERNAME: test
       CLICKHOUSE_PASSWORD: test
+      CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: 500
+      CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: 1000
       PORT: 3007
 
   usage-estimator:

integration-tests/scripts/filter-log.mjs

@@ -0,0 +1,19 @@
+import path from 'path';
+import fs from 'fs';
+import { fileURLToPath } from 'url';
+
+/**
+ * node ./scripts/filter-log.mjs usage usage-ingestor emails
+ *
+ * Creates the `dockest-filtered.log` file with logs from the `dockest.log` file but only for the provided services.
+ */
+const [, , ...services] = process.argv;
+
+const serviceNames = services.map(name => name.replace(/[^a-z]/i, '-'));
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const logPath = path.resolve(__dirname, '../dockest.log');
+const filteredLogPath = path.resolve(__dirname, '../dockest-filtered.log');
+const logLines = fs.readFileSync(logPath, 'utf-8').split('\n');
+const filteredLogLines = logLines.filter(line => serviceNames.some(name => line.includes(name)));
+
+fs.writeFileSync(filteredLogPath, filteredLogLines.join('\n'));

packages/libraries/client/src/version.ts

@@ -1 +1 @@
-export const version = '0.21.2';
+export const version = '0.21.3';

packages/services/storage/docker-compose.yml

@@ -38,7 +38,7 @@ services:
       - './volumes/redis/db:/bitnami/redis/data'
 
   clickhouse:
-    image: clickhouse/clickhouse-server:22.9-alpine
+    image: clickhouse/clickhouse-server:22.11-alpine
     environment:
       CLICKHOUSE_USER: test
       CLICKHOUSE_PASSWORD: test

packages/services/usage-ingestor/.env.template

@@ -8,4 +8,6 @@ CLICKHOUSE_HOST="localhost"
 CLICKHOUSE_PORT="8123"
 CLICKHOUSE_USERNAME="test"
 CLICKHOUSE_PASSWORD="test"
+CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS="30000"
+CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE="200000000"
 PORT=4002

packages/services/usage-ingestor/src/environment.ts

@@ -63,6 +63,8 @@ const ClickHouseModel = zod.object({
   CLICKHOUSE_PORT: NumberFromString,
   CLICKHOUSE_USERNAME: zod.string(),
   CLICKHOUSE_PASSWORD: zod.string(),
+  CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: emptyString(NumberFromString.optional()),
+  CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: emptyString(NumberFromString.optional()),
 });
 
 const ClickHouseMirrorModel = zod.union([
@@ -72,6 +74,8 @@ const ClickHouseMirrorModel = zod.union([
     CLICKHOUSE_MIRROR_PORT: NumberFromString,
     CLICKHOUSE_MIRROR_USERNAME: zod.string(),
     CLICKHOUSE_MIRROR_PASSWORD: zod.string(),
+    CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: emptyString(NumberFromString.optional()),
+    CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: emptyString(NumberFromString.optional()),
   }),
   zod.object({}),
 ]);
@@ -172,6 +176,8 @@ export const env = {
     port: clickhouse.CLICKHOUSE_PORT,
     username: clickhouse.CLICKHOUSE_USERNAME,
     password: clickhouse.CLICKHOUSE_PASSWORD,
+    async_insert_busy_timeout_ms: clickhouse.CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS ?? 30_000,
+    async_insert_max_data_size: clickhouse.CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE ?? 200_000_000,
   },
   clickhouseMirror:
     'CLICKHOUSE_MIRROR_PROTOCOL' in clickhouseMirror
@@ -181,6 +187,10 @@ export const env = {
           port: clickhouseMirror.CLICKHOUSE_MIRROR_PORT,
           username: clickhouseMirror.CLICKHOUSE_MIRROR_USERNAME,
           password: clickhouseMirror.CLICKHOUSE_MIRROR_PASSWORD,
+          async_insert_busy_timeout_ms:
+            clickhouse.CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS ?? 30_000,
+          async_insert_max_data_size:
+            clickhouse.CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE ?? 200_000_000,
         }
       : null,
   heartbeat: base.HEARTBEAT_ENDPOINT ? { endpoint: base.HEARTBEAT_ENDPOINT } : null,

packages/services/usage-ingestor/src/index.ts

@@ -32,13 +32,7 @@ async function main() {
   try {
     const { readiness, start, stop } = createIngestor({
       logger: server.log,
-      clickhouse: {
-        protocol: env.clickhouse.protocol,
-        host: env.clickhouse.host,
-        port: env.clickhouse.port,
-        username: env.clickhouse.username,
-        password: env.clickhouse.password,
-      },
+      clickhouse: env.clickhouse,
       clickhouseMirror: env.clickhouseMirror,
       kafka: {
         topic: env.kafka.topic,

packages/services/usage-ingestor/src/writer.ts

@@ -24,6 +24,8 @@ export interface ClickHouseConfig {
   port: number;
   username: string;
   password: string;
+  async_insert_busy_timeout_ms: number;
+  async_insert_max_data_size: number;
 }
 
 const operationsFields = operationsOrder.join(', ');
@@ -165,8 +167,8 @@ async function writeCsv(
         query,
         async_insert: 1,
         wait_for_async_insert: 0,
-        async_insert_busy_timeout_ms: 30_000,
-        async_insert_max_data_size: 200_000_000,
+        async_insert_busy_timeout_ms: config.async_insert_busy_timeout_ms,
+        async_insert_max_data_size: config.async_insert_max_data_size,
       },
       username: config.username,
       password: config.password,