diff --git a/.changeset/many-dodos-lay.md b/.changeset/many-dodos-lay.md new file mode 100644 index 000000000..11ab4a694 --- /dev/null +++ b/.changeset/many-dodos-lay.md @@ -0,0 +1,74 @@ +--- +'hive': minor +--- + +Create new ClickHouse materialized views for faster affected app deployment lookups in schema checks and schema +version. + +**Caution**: If you are relying on the app deployments feature for schema checks it is recommended to manually perform the following migration against your ClickHouse database after deploying this version to ensure data consistency. + +Substitute `$CLICKHOUSE_DB_USER` and `$CLICKHOUSE_DB_PASSWORD`, with the same credentials that execute these migration. + +```sql +CREATE TABLE "tmp_app_deployments_backfill_target_id" ( + "app_deployment_id" String + , "target_id" LowCardinality(String) +) +ENGINE = Memory +; + +INSERT INTO "tmp_app_deployments_backfill_target_id" +SELECT + "app_deployment_id" + , "target_id" +FROM + "app_deployments" +; + +CREATE DICTIONARY "tmp_app_deployments_target_dict" ( + "app_deployment_id" String + , "target_id" String +) +PRIMARY KEY "app_deployment_id" +SOURCE(CLICKHOUSE( + TABLE "tmp_app_deployments_backfill_target_id" + USER '$CLICKHOUSE_DB_USER' + PASSWORD '$CLICKHOUSE_DB_PASSWORD' +)) +LAYOUT(HASHED()) +LIFETIME(3600) +; + +ALTER TABLE + "app_deployment_documents" +UPDATE + "target_id" = dictGetString('tmp_app_deployments_target_dict', 'target_id', "app_deployment_id") +WHERE + "target_id" = '' +; + +INSERT INTO "app_deployment_document_coordinates" ( + "target_id" + , "coordinate" + , "app_deployment_id" + , "document_hash" + , "operation_name" +) +SELECT + "target_id" + , arrayJoin("schema_coordinates") AS "schema_coordinate" + , "app_deployment_id" + , "document_hash" + , "operation_name" +FROM + "app_deployment_documents" +WHERE + "target_id" != "" +; + +DROP DICTIONARY "tmp_app_deployments_target_dict" +; + +DROP TABLE "tmp_app_deployments_backfill_target_id" +; +``` diff --git a/packages/migrations/src/clickhouse-actions/017-affected-app-deployments-performance.ts b/packages/migrations/src/clickhouse-actions/017-affected-app-deployments-performance.ts new file mode 100644 index 000000000..0963d3fac --- /dev/null +++ b/packages/migrations/src/clickhouse-actions/017-affected-app-deployments-performance.ts @@ -0,0 +1,36 @@ +import type { Action } from '../clickhouse'; + +export const action: Action = async exec => { + await exec(` + ALTER TABLE app_deployment_documents + ADD COLUMN IF NOT EXISTS target_id LowCardinality(String) + ; + `); + + await exec(` + CREATE TABLE IF NOT EXISTS "app_deployment_document_coordinates" ( + "target_id" LowCardinality(String) + , "coordinate" LowCardinality(String) + , "app_deployment_id" LowCardinality(String) + , "document_hash" String + , "operation_name" String + ) + ENGINE = ReplacingMergeTree + ORDER BY ("target_id", "coordinate", "app_deployment_id", "document_hash") + ; + `); + + await exec(` + CREATE MATERIALIZED VIEW IF NOT EXISTS "mv_documents_by_coordinate" + TO "app_deployment_document_coordinates" + AS + SELECT + "target_id" + , arrayJoin("schema_coordinates") AS "coordinate" + , "app_deployment_id" + , "document_hash" + , "operation_name" + FROM "app_deployment_documents" + ; + `); +}; diff --git a/packages/migrations/src/clickhouse.ts b/packages/migrations/src/clickhouse.ts index 719ec032f..3751639e2 100644 --- a/packages/migrations/src/clickhouse.ts +++ b/packages/migrations/src/clickhouse.ts @@ -177,6 +177,7 @@ export async function migrateClickHouse( import('./clickhouse-actions/014-audit-logs-access-token'), import('./clickhouse-actions/015-otel-trace'), import('./clickhouse-actions/016-subgraph-otel-traces-cleanup'), + import('./clickhouse-actions/017-affected-app-deployments-performance'), ]); async function actionRunner(action: Action, index: number) { diff --git a/packages/services/api/src/modules/app-deployments/providers/persisted-document-ingester.ts b/packages/services/api/src/modules/app-deployments/providers/persisted-document-ingester.ts index a47aa7777..1119f3e21 100644 --- a/packages/services/api/src/modules/app-deployments/providers/persisted-document-ingester.ts +++ b/packages/services/api/src/modules/app-deployments/providers/persisted-document-ingester.ts @@ -270,7 +270,8 @@ export class PersistedDocumentIngester { await this.clickhouse.insert({ query: c_sql` INSERT INTO "app_deployment_documents" ( - "app_deployment_id" + "target_id" + , "app_deployment_id" , "document_hash" , "document_body" , "operation_name" @@ -279,6 +280,7 @@ export class PersistedDocumentIngester { ) FORMAT CSV`, data: args.documents.map(document => [ + args.targetId, document.appDeploymentId, document.hash, document.body,