feat: setup clickhouse migration tool + update metrics schema (add is_delta + is_monotonic fields) (#138)

.changeset/little-colts-teach.md

@@ -0,0 +1,7 @@
+---
+'@hyperdx/api': minor
+'@hyperdx/app': minor
+---
+
+feat: add is_delta + is_monotonic fields to metric_stream table (REQUIRES DB
+MIGRATION)

.changeset/purple-trains-cheer.md

@@ -0,0 +1,6 @@
+---
+'@hyperdx/api': patch
+'@hyperdx/app': patch
+---
+
+feat: setup clickhouse migration tool

Makefile

@@ -48,9 +48,13 @@ dev-unit:
 ci-unit:
 	npx nx run-many -t ci:unit
 
+# TODO: check db connections before running the migration CLIs
 .PHONY: dev-migrate-db
 dev-migrate-db:
+	@echo "Migrating Mongo db...\n"
 	npx nx run @hyperdx/api:dev:migrate-db
+	@echo "Migrating ClickHouse db...\n"
+	npx nx run @hyperdx/api:dev:migrate-ch
 
 .PHONY: build-local
 build-local:

README.md

@@ -98,6 +98,13 @@ the `.env` file. After making your changes, rebuild images with
 
 **DB Migration**
 
+Before running the migration, you'll need to install the migration tools:
+
+1. Install local dependencies with `make`.
+2. Install
+   [golang-migrate](https://github.com/golang-migrate/migrate/tree/master/cmd/migrate)
+   CLI.
+
 You can initiate the DB migration process by executing `make dev-migrate-db`.
 This will run the migration scripts in `/packages/api/migrations` against the
 local DB.

packages/api/migrate-mongo-config.ts

@@ -15,7 +15,7 @@ export = {
   },
 
   // The migrations dir, can be an relative or absolute path. Only edit this when really necessary.
-  migrationsDir: 'migrations',
+  migrationsDir: 'migrations/mongo',
 
   // The mongodb collection where the applied changes are stored. Only edit this when really necessary.
   changelogCollectionName: 'changelog',

packages/api/migrations/ch/000001_add_is_delta_n_is_monotonic_fields_to_metric_stream_table.down.sql

@@ -0,0 +1,3 @@
+ALTER TABLE default.metric_stream DROP COLUMN is_delta;
+
+ALTER TABLE default.metric_stream DROP COLUMN is_monotonic;

packages/api/migrations/ch/000001_add_is_delta_n_is_monotonic_fields_to_metric_stream_table.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE default.metric_stream ADD COLUMN is_delta Boolean CODEC(Delta, ZSTD(1));
+
+ALTER TABLE default.metric_stream ADD COLUMN is_monotonic Boolean CODEC(Delta, ZSTD(1));

packages/api/package.json

@@ -93,6 +93,8 @@
     "ci:int": "jest --runInBand --ci --forceExit --coverage",
     "dev:int": "jest --watchAll --runInBand --detectOpenHandles",
     "dev:migrate-db-create": "ts-node node_modules/.bin/migrate-mongo create -f migrate-mongo-config.ts",
-    "dev:migrate-db": "ts-node node_modules/.bin/migrate-mongo up -f migrate-mongo-config.ts"
+    "dev:migrate-db": "ts-node node_modules/.bin/migrate-mongo up -f migrate-mongo-config.ts",
+    "dev:migrate-ch-create": "migrate create -ext sql -dir ./migrations/ch -seq",
+    "dev:migrate-ch": "migrate -database 'clickhouse://localhost:9000?database=default&x-multi-statement=true' -path ./migrations/ch up"
   }
 }

packages/api/src/clickhouse/index.ts

@@ -317,6 +317,8 @@ export const connect = async () => {
         value Float64 CODEC(ZSTD(1)),
         flags UInt32  CODEC(ZSTD(1)),
         unit String CODEC(ZSTD(1)),
+        is_delta Boolean CODEC(Delta, ZSTD(1)),
+        is_monotonic Boolean CODEC(Delta, ZSTD(1)),
         _string_attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
         _created_at DateTime64(9, 'UTC') DEFAULT toDateTime64(now(), 9) CODEC(Delta(8), ZSTD(1)),
         _timestamp_sort_key Int64 MATERIALIZED toUnixTimestamp64Nano(coalesce(timestamp, _created_at)),