♻️ refactor: improve db sql performance (#9283)

* improve sql

* improve db index

* improve db index

* improve db index

docs/development/database-schema.dbml

@@ -419,6 +419,9 @@ table messages {
     topic_id [name: 'messages_topic_id_idx']
     parent_id [name: 'messages_parent_id_idx']
     quota_id [name: 'messages_quota_id_idx']
+    user_id [name: 'messages_user_id_idx']
+    session_id [name: 'messages_session_id_idx']
+    thread_id [name: 'messages_thread_id_idx']
   }
 }
 
@@ -624,6 +627,7 @@ table chunks {
 
   indexes {
     (client_id, user_id) [name: 'chunks_client_id_user_id_unique', unique]
+    user_id [name: 'chunks_user_id_idx']
   }
 }
 
@@ -637,6 +641,7 @@ table embeddings {
 
   indexes {
     (client_id, user_id) [name: 'embeddings_client_id_user_id_unique', unique]
+    chunk_id [name: 'embeddings_chunk_id_idx']
   }
 }
 
@@ -836,6 +841,8 @@ table sessions {
   indexes {
     (slug, user_id) [name: 'slug_user_id_unique', unique]
     (client_id, user_id) [name: 'sessions_client_id_user_id_unique', unique]
+    user_id [name: 'sessions_user_id_idx']
+    (id, user_id) [name: 'sessions_id_user_id_idx']
   }
 }
 
@@ -886,6 +893,8 @@ table topics {
 
   indexes {
     (client_id, user_id) [name: 'topics_client_id_user_id_unique', unique]
+    user_id [name: 'topics_user_id_idx']
+    (id, user_id) [name: 'topics_id_user_id_idx']
   }
 }
 

packages/database/migrations/0031_add_agent_index.sql

@@ -1,6 +1,12 @@
--- 将超过 1000 字符的 description 截断为 1000 字符
+-- Truncate title to 255 characters if it exceeds the limit
+UPDATE agents
+SET title = LEFT(title, 255)
+WHERE LENGTH(title) > 255;--> statement-breakpoint
+
+-- Truncate description to 1000 characters if it exceeds the limit
 UPDATE agents
 SET description = LEFT(description, 1000)
 WHERE LENGTH(description) > 1000;--> statement-breakpoint
+
 CREATE INDEX IF NOT EXISTS "agents_title_idx" ON "agents" USING btree ("title");--> statement-breakpoint
 CREATE INDEX IF NOT EXISTS "agents_description_idx" ON "agents" USING btree ("description");

packages/database/migrations/0032_improve_agents_field.sql

@@ -1,6 +1,2 @@
--- 将超过 255 字符的 title 截断为 255 字符
-UPDATE agents
-SET title = LEFT(title, 255)
-WHERE LENGTH(title) > 255;--> statement-breakpoint
 ALTER TABLE "agents" ALTER COLUMN "title" SET DATA TYPE varchar(255);--> statement-breakpoint
 ALTER TABLE "agents" ALTER COLUMN "description" SET DATA TYPE varchar(1000);

packages/database/migrations/0033_modern_mercury.sql

@@ -0,0 +1,18 @@
+-- 解决 chunks 表慢查询
+CREATE INDEX IF NOT EXISTS "chunks_user_id_idx" ON "chunks" USING btree ("user_id");--> statement-breakpoint
+
+-- 解决 topics 表批量删除慢查询
+CREATE INDEX IF NOT EXISTS "topics_user_id_idx" ON "topics" USING btree ("user_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "topics_id_user_id_idx" ON "topics" USING btree ("id","user_id");--> statement-breakpoint
+
+-- 解决 sessions 表删除慢查询
+CREATE INDEX IF NOT EXISTS "sessions_user_id_idx" ON "sessions" USING btree ("user_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "sessions_id_user_id_idx" ON "sessions" USING btree ("id","user_id");--> statement-breakpoint
+
+-- 解决 messages 表统计查询慢查询
+CREATE INDEX IF NOT EXISTS "messages_user_id_idx" ON "messages" USING btree ("user_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "messages_session_id_idx" ON "messages" USING btree ("session_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "messages_thread_id_idx" ON "messages" USING btree ("thread_id");--> statement-breakpoint
+
+-- 解决 embeddings 删除慢查询
+CREATE INDEX IF NOT EXISTS "embeddings_chunk_id_idx" ON "embeddings" USING btree ("chunk_id");--> statement-breakpoint

packages/database/migrations/meta/_journal.json

@@ -231,6 +231,13 @@
       "when": 1757993755131,
       "tag": "0032_improve_agents_field",
       "breakpoints": true
+    },
+    {
+      "idx": 33,
+      "version": "7",
+      "when": 1758012348218,
+      "tag": "0033_modern_mercury",
+      "breakpoints": true
     }
   ],
   "version": "6"

packages/database/src/core/migrations.json

@@ -581,22 +581,39 @@
   },
   {
     "sql": [
-      "-- 将超过 2000 字符的 description 截断为 2000 字符\nUPDATE agents\nSET description = LEFT(description, 2000)\nWHERE LENGTH(description) > 2000;",
-      "\nCREATE INDEX IF NOT EXISTS \"agents_title_idx\" ON \"agents\" USING btree (\"title\");",
+      "-- Truncate title to 255 characters if it exceeds the limit\nUPDATE agents\nSET title = LEFT(title, 255)\nWHERE LENGTH(title) > 255;",
+      "\n\n-- Truncate description to 1000 characters if it exceeds the limit\nUPDATE agents\nSET description = LEFT(description, 1000)\nWHERE LENGTH(description) > 1000;",
+      "\n\nCREATE INDEX IF NOT EXISTS \"agents_title_idx\" ON \"agents\" USING btree (\"title\");",
       "\nCREATE INDEX IF NOT EXISTS \"agents_description_idx\" ON \"agents\" USING btree (\"description\");\n"
     ],
     "bps": true,
     "folderMillis": 1757902833213,
-    "hash": "39b690dc849ae2bb09357a9ab9d1f2e8f3e37f609f636ab192670a2651d4f605"
+    "hash": "4b2e0d87485cd7b3aa4dd06edfd73b78d195137263776c7d1c3df6eb11c3bc62"
   },
   {
     "sql": [
-      "-- 将超过 255 字符的 title 截断为 255 字符\nUPDATE agents\nSET title = LEFT(title, 255)\nWHERE LENGTH(title) > 255;",
-      "\nALTER TABLE \"agents\" ALTER COLUMN \"title\" SET DATA TYPE varchar(255);",
+      "ALTER TABLE \"agents\" ALTER COLUMN \"title\" SET DATA TYPE varchar(255);",
       "\nALTER TABLE \"agents\" ALTER COLUMN \"description\" SET DATA TYPE varchar(1000);\n"
     ],
     "bps": true,
     "folderMillis": 1757993755131,
-    "hash": "0e23099a1ae4d5a40fd9ea5667c3d1939548d98488906742b0f029cde272ef27"
+    "hash": "bafc1c74796a4c69342e61d28997b0e3a38f1e7d022ba92d9cbf31e00b5748c8"
+  },
+  {
+    "sql": [
+      "-- 解决 chunks 表慢查询\nCREATE INDEX IF NOT EXISTS \"chunks_user_id_idx\" ON \"chunks\" USING btree (\"user_id\");",
+      "\n\n-- 解决 topics 表批量删除慢查询\nCREATE INDEX IF NOT EXISTS \"topics_user_id_idx\" ON \"topics\" USING btree (\"user_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"topics_id_user_id_idx\" ON \"topics\" USING btree (\"id\",\"user_id\");",
+      "\n\n-- 解决 sessions 表删除慢查询\nCREATE INDEX IF NOT EXISTS \"sessions_user_id_idx\" ON \"sessions\" USING btree (\"user_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"sessions_id_user_id_idx\" ON \"sessions\" USING btree (\"id\",\"user_id\");",
+      "\n\n-- 解决 messages 表统计查询慢查询\nCREATE INDEX IF NOT EXISTS \"messages_user_id_idx\" ON \"messages\" USING btree (\"user_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"messages_session_id_idx\" ON \"messages\" USING btree (\"session_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"messages_thread_id_idx\" ON \"messages\" USING btree (\"thread_id\");",
+      "\n\n-- 解决 embeddings 删除慢查询\nCREATE INDEX IF NOT EXISTS \"embeddings_chunk_id_idx\" ON \"embeddings\" USING btree (\"chunk_id\");",
+      "\n"
+    ],
+    "bps": true,
+    "folderMillis": 1758012348218,
+    "hash": "ce04ef4cde2db479d28ff08dced8383052c5052c904bab8343b5493fa10b0679"
   }
 ]

packages/database/src/schemas/message.ts

@@ -18,12 +18,12 @@ import { GroundingSearch } from '@/types/search';
 import { idGenerator } from '../utils/idGenerator';
 import { timestamps } from './_helpers';
 import { agents } from './agent';
+import { chatGroups } from './chatGroup';
 import { files } from './file';
 import { chunks, embeddings } from './rag';
 import { sessions } from './session';
 import { threads, topics } from './topic';
 import { users } from './user';
-import { chatGroups } from './chatGroup';
 
 // @ts-ignore
 export const messages = pgTable(
@@ -70,16 +70,17 @@ export const messages = pgTable(
     targetId: text('target_id'),
     ...timestamps,
   },
-  (table) => ({
-    createdAtIdx: index('messages_created_at_idx').on(table.createdAt),
-    messageClientIdUnique: uniqueIndex('message_client_id_user_unique').on(
-      table.clientId,
-      table.userId,
-    ),
-    topicIdIdx: index('messages_topic_id_idx').on(table.topicId),
-    parentIdIdx: index('messages_parent_id_idx').on(table.parentId),
-    quotaIdIdx: index('messages_quota_id_idx').on(table.quotaId),
-  }),
+  (table) => [
+    index('messages_created_at_idx').on(table.createdAt),
+    uniqueIndex('message_client_id_user_unique').on(table.clientId, table.userId),
+    index('messages_topic_id_idx').on(table.topicId),
+    index('messages_parent_id_idx').on(table.parentId),
+    index('messages_quota_id_idx').on(table.quotaId),
+
+    index('messages_user_id_idx').on(table.userId),
+    index('messages_session_id_idx').on(table.sessionId),
+    index('messages_thread_id_idx').on(table.threadId),
+  ],
 );
 
 // if the message container a plugin

packages/database/src/schemas/rag.ts

@@ -1,5 +1,6 @@
 /* eslint-disable sort-keys-fix/sort-keys-fix  */
 import {
+  index,
   integer,
   jsonb,
   pgTable,
@@ -29,9 +30,10 @@ export const chunks = pgTable(
 
     ...timestamps,
   },
-  (t) => ({
-    clientIdUnique: uniqueIndex('chunks_client_id_user_id_unique').on(t.clientId, t.userId),
-  }),
+  (t) => [
+    uniqueIndex('chunks_client_id_user_id_unique').on(t.clientId, t.userId),
+    index('chunks_user_id_idx').on(t.userId),
+  ],
 );
 
 export type NewChunkItem = typeof chunks.$inferInsert & { fileId?: string };
@@ -75,9 +77,11 @@ export const embeddings = pgTable(
     clientId: text('client_id'),
     userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
   },
-  (t) => ({
-    clientIdUnique: uniqueIndex('embeddings_client_id_user_id_unique').on(t.clientId, t.userId),
-  }),
+  (t) => [
+    uniqueIndex('embeddings_client_id_user_id_unique').on(t.clientId, t.userId),
+    // improve delete embeddings query
+    index('embeddings_chunk_id_idx').on(t.chunkId),
+  ],
 );
 
 export type NewEmbeddingsItem = typeof embeddings.$inferInsert;

packages/database/src/schemas/session.ts

@@ -1,5 +1,5 @@
 /* eslint-disable sort-keys-fix/sort-keys-fix  */
-import { boolean, integer, pgTable, text, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
+import { boolean, index, integer, pgTable, text, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
 import { createInsertSchema } from 'drizzle-zod';
 
 import { idGenerator, randomSlug } from '../utils/idGenerator';
@@ -64,11 +64,13 @@ export const sessions = pgTable(
 
     ...timestamps,
   },
-  (t) => ({
-    slugUserIdUnique: uniqueIndex('slug_user_id_unique').on(t.slug, t.userId),
+  (t) => [
+    uniqueIndex('slug_user_id_unique').on(t.slug, t.userId),
+    uniqueIndex('sessions_client_id_user_id_unique').on(t.clientId, t.userId),
 
-    clientIdUnique: uniqueIndex('sessions_client_id_user_id_unique').on(t.clientId, t.userId),
-  }),
+    index('sessions_user_id_idx').on(t.userId),
+    index('sessions_id_user_id_idx').on(t.id, t.userId),
+  ],
 );
 
 export const insertSessionSchema = createInsertSchema(sessions);

packages/database/src/schemas/topic.ts

@@ -1,15 +1,15 @@
 /* eslint-disable sort-keys-fix/sort-keys-fix  */
-import { boolean, jsonb, pgTable, primaryKey, text, uniqueIndex } from 'drizzle-orm/pg-core';
+import { boolean, index, jsonb, pgTable, primaryKey, text, uniqueIndex } from 'drizzle-orm/pg-core';
 import { createInsertSchema } from 'drizzle-zod';
 
 import { ChatTopicMetadata } from '@/types/topic';
 
 import { idGenerator } from '../utils/idGenerator';
 import { createdAt, timestamps, timestamptz } from './_helpers';
+import { chatGroups } from './chatGroup';
 import { documents } from './document';
 import { sessions } from './session';
 import { users } from './user';
-import { chatGroups } from './chatGroup';
 
 export const topics = pgTable(
   'topics',
@@ -29,7 +29,11 @@ export const topics = pgTable(
     metadata: jsonb('metadata').$type<ChatTopicMetadata | undefined>(),
     ...timestamps,
   },
-  (t) => [uniqueIndex('topics_client_id_user_id_unique').on(t.clientId, t.userId)],
+  (t) => [
+    uniqueIndex('topics_client_id_user_id_unique').on(t.clientId, t.userId),
+    index('topics_user_id_idx').on(t.userId),
+    index('topics_id_user_id_idx').on(t.id, t.userId),
+  ],
 );
 
 export type NewTopic = typeof topics.$inferInsert;