feat: add app_deployment_documents.target_id column (#7832)

.changeset/many-dodos-lay.md

@@ -0,0 +1,74 @@
+---
+'hive': minor
+---
+
+Create new ClickHouse materialized views for faster affected app deployment lookups in schema checks and schema
+version.
+
+**Caution**: If you are relying on the app deployments feature for schema checks it is recommended to manually perform the following migration against your ClickHouse database after deploying this version to ensure data consistency.
+
+Substitute `$CLICKHOUSE_DB_USER` and `$CLICKHOUSE_DB_PASSWORD`, with the same credentials that execute these migration.
+
+```sql
+CREATE TABLE "tmp_app_deployments_backfill_target_id" (
+  "app_deployment_id" String
+  , "target_id" LowCardinality(String)
+)
+ENGINE = Memory
+;
+
+INSERT INTO "tmp_app_deployments_backfill_target_id"
+SELECT
+  "app_deployment_id"
+  , "target_id"
+FROM
+  "app_deployments"
+;
+
+CREATE DICTIONARY "tmp_app_deployments_target_dict" (
+  "app_deployment_id" String
+  , "target_id" String
+)
+PRIMARY KEY "app_deployment_id"
+SOURCE(CLICKHOUSE(
+    TABLE "tmp_app_deployments_backfill_target_id"
+    USER '$CLICKHOUSE_DB_USER'
+    PASSWORD '$CLICKHOUSE_DB_PASSWORD'
+))
+LAYOUT(HASHED())
+LIFETIME(3600)
+;
+
+ALTER TABLE
+  "app_deployment_documents"
+UPDATE 
+  "target_id" = dictGetString('tmp_app_deployments_target_dict', 'target_id', "app_deployment_id")
+WHERE
+  "target_id" = ''
+;
+
+INSERT INTO "app_deployment_document_coordinates" (
+  "target_id"
+  , "coordinate"
+  , "app_deployment_id"
+  , "document_hash"
+  , "operation_name"
+)
+SELECT
+  "target_id"
+  , arrayJoin("schema_coordinates") AS "schema_coordinate"
+  , "app_deployment_id"
+  , "document_hash"
+  , "operation_name"
+FROM
+  "app_deployment_documents"
+WHERE
+  "target_id" != ""
+;
+
+DROP DICTIONARY "tmp_app_deployments_target_dict"
+;
+
+DROP TABLE "tmp_app_deployments_backfill_target_id"
+;
+```

packages/migrations/src/clickhouse-actions/017-affected-app-deployments-performance.ts

@@ -0,0 +1,36 @@
+import type { Action } from '../clickhouse';
+
+export const action: Action = async exec => {
+  await exec(`
+    ALTER TABLE app_deployment_documents
+    ADD COLUMN IF NOT EXISTS target_id LowCardinality(String)
+    ;
+  `);
+
+  await exec(`
+    CREATE TABLE IF NOT EXISTS "app_deployment_document_coordinates" (
+      "target_id" LowCardinality(String)
+      , "coordinate" LowCardinality(String)
+      , "app_deployment_id" LowCardinality(String)
+      , "document_hash" String
+      , "operation_name" String
+    )
+    ENGINE = ReplacingMergeTree
+    ORDER BY ("target_id", "coordinate", "app_deployment_id", "document_hash")
+    ;
+  `);
+
+  await exec(`
+    CREATE MATERIALIZED VIEW IF NOT EXISTS "mv_documents_by_coordinate"
+    TO "app_deployment_document_coordinates"
+    AS
+    SELECT
+      "target_id"
+      , arrayJoin("schema_coordinates") AS "coordinate"
+      , "app_deployment_id"
+      , "document_hash"
+      , "operation_name"
+    FROM "app_deployment_documents"
+    ;
+  `);
+};

packages/migrations/src/clickhouse.ts

@@ -177,6 +177,7 @@ export async function migrateClickHouse(
     import('./clickhouse-actions/014-audit-logs-access-token'),
     import('./clickhouse-actions/015-otel-trace'),
     import('./clickhouse-actions/016-subgraph-otel-traces-cleanup'),
+    import('./clickhouse-actions/017-affected-app-deployments-performance'),
   ]);
 
   async function actionRunner(action: Action, index: number) {

packages/services/api/src/modules/app-deployments/providers/persisted-document-ingester.ts

@@ -270,7 +270,8 @@ export class PersistedDocumentIngester {
     await this.clickhouse.insert({
       query: c_sql`
         INSERT INTO "app_deployment_documents" (
-          "app_deployment_id"
+          "target_id"
+          , "app_deployment_id"
           , "document_hash"
           , "document_body"
           , "operation_name"
@@ -279,6 +280,7 @@ export class PersistedDocumentIngester {
         )
         FORMAT CSV`,
       data: args.documents.map(document => [
+        args.targetId,
         document.appDeploymentId,
         document.hash,
         document.body,