From 3aa1960eec5aa7cfdb29ff7682c9ee9e62601689 Mon Sep 17 00:00:00 2001 From: Rohan Lahori <64496391+rohanlahori@users.noreply.github.com> Date: Fri, 20 Jun 2025 11:02:01 +0530 Subject: [PATCH 1/3] migration fix (#13053) --- ...s.ts => 1737530238311-CreateTablesForToojetAiConversations.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/migrations/{1740399879253-CreateAiConversations.ts => 1737530238311-CreateTablesForToojetAiConversations.ts} (100%) diff --git a/server/migrations/1740399879253-CreateAiConversations.ts b/server/migrations/1737530238311-CreateTablesForToojetAiConversations.ts similarity index 100% rename from server/migrations/1740399879253-CreateAiConversations.ts rename to server/migrations/1737530238311-CreateTablesForToojetAiConversations.ts From 9aa9675b602552d062b5a086c347da58c5e54242 Mon Sep 17 00:00:00 2001 From: Adish M <44204658+adishM98@users.noreply.github.com> Date: Fri, 20 Jun 2025 12:59:17 +0530 Subject: [PATCH 2/3] Adding Cloud plugin scripts (#12995) --- package.json | 8 +++ server/package.json | 6 ++ server/scripts/plugins-install.ts | 75 +++++++++++++++++++++++++ server/scripts/plugins-reload.ts | 84 ++++++++++++++++++++++++++++ server/scripts/plugins-uninstall.ts | 85 +++++++++++++++++++++++++++++ 5 files changed, 258 insertions(+) create mode 100644 server/scripts/plugins-install.ts create mode 100644 server/scripts/plugins-reload.ts create mode 100644 server/scripts/plugins-uninstall.ts diff --git a/package.json b/package.json index ba210b0a8a..33850bd03f 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,8 @@ "postbuild:server": "npm --prefix server prune --production", "build": "npm run build:plugins:prod && npm run build:frontend && npm run build:server", "start:prod": "npm --prefix server run start:prod", + "cloud:setup": "npm run db:setup && npm run plugins:install && npm run plugins:uninstall && npm run plugins:reload", + "cloud:setup:prod": "npm run db:setup:prod && npm run plugins:install:prod && npm run plugins:uninstall:prod && npm run plugins:reload:prod", "db:create": "npm --prefix server run db:create", "db:create:prod": "npm --prefix server run db:create:prod", "db:migrate": "npm --prefix server run db:migrate", @@ -51,6 +53,12 @@ "db:reset": "npm --prefix server run db:reset", "db:drop": "npm --prefix server run db:drop", "deploy": "cp -a frontend/build/. public/", + "plugins:install": "npm --prefix server run plugins:install", + "plugins:install:prod": "npm --prefix server run plugins:install:prod", + "plugins:uninstall": "npm --prefix server run plugins:uninstall", + "plugins:uninstall:prod": "npm --prefix server run plugins:uninstall:prod", + "plugins:reload": "npm --prefix server run plugins:reload", + "plugins:reload:prod": "npm --prefix server run plugins:reload:prod", "heroku-postbuild": "./heroku-postbuild.sh", "prepare": "husky install", "update-version": "node update-version.js" diff --git a/server/package.json b/server/package.json index e90223cccd..9de2850bba 100644 --- a/server/package.json +++ b/server/package.json @@ -34,6 +34,12 @@ "db:setup": "npm run db:create && npm run db:migrate", "db:setup:prod": "npm run db:create:prod && npm run db:migrate:prod", "db:reset": "npm run db:drop && npm run db:setup", + "plugins:install": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/plugins-install.ts", + "plugins:install:prod": "node dist/scripts/plugins-install.js", + "plugins:uninstall": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/plugins-uninstall.ts", + "plugins:uninstall:prod": "node dist/scripts/plugins-uninstall.js", + "plugins:reload": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/plugins-reload.ts", + "plugins:reload:prod": "node dist/scripts/plugins-reload.js", "typeorm": "typeorm-ts-node-commonjs", "copy-schemas": "copyfiles -u 4 src/dto/validators/schemas/**/*.json dist/src/dto/validators/schemas", "worker:dev": "WORKER=true NODE_ENV=development nest start --watch", diff --git a/server/scripts/plugins-install.ts b/server/scripts/plugins-install.ts new file mode 100644 index 0000000000..1d647890e6 --- /dev/null +++ b/server/scripts/plugins-install.ts @@ -0,0 +1,75 @@ +import * as availablePlugins from '../src/assets/marketplace/plugins.json'; +import { AppModule } from '../src/app.module'; +import { CreatePluginDto } from '@dto/create-plugin.dto'; +import { EntityManager } from 'typeorm'; +import { INestApplication } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; +import { Plugin } from 'src/entities/plugin.entity'; +import { PluginsService } from '@services/plugins.service'; +import { getEnvVars } from './database-config-utils'; +import { validateSync } from 'class-validator'; + +const ENV_VARS = getEnvVars(); + +async function bootstrap() { + const nestApp = await NestFactory.create(AppModule, { + logger: ['error', 'warn'], + }); + + await validateAndInstallPlugins(nestApp); + + await nestApp.close(); + process.exit(0); +} + +async function validateAndInstallPlugins(nestApp: INestApplication) { + const pluginsService = nestApp.get(PluginsService); + const pluginsToInstall = fetchPluginsToInstall(); + const validPluginDtos: CreatePluginDto[] = []; + const invalidPluginDtos: CreatePluginDto[] = []; + const entityManager = nestApp.get(EntityManager); + + console.log('Plugins to install:', pluginsToInstall); + + for (const pluginId of pluginsToInstall) { + const pluginDto = Object.assign(new CreatePluginDto(), findPluginDetails(pluginId)); + const validationErrors = validateSync(pluginDto); + + if (validationErrors.length === 0) { + const plugin = await entityManager.findOne(Plugin, { where: { pluginId: pluginDto.id } }); + plugin ? invalidPluginDtos.push(pluginDto) : validPluginDtos.push(pluginDto); + } else { + console.log(`Plugin with ID '${pluginId}' has validation errors:`, validationErrors); + invalidPluginDtos.push(pluginDto); + } + } + + invalidPluginDtos.length > 0 && + console.log( + 'Skipping invalid plugins:', + invalidPluginDtos.map((dto) => dto.id), + '\n' + ); + + for (const dto of validPluginDtos) { + await pluginsService.install(dto); + console.log('Installed:', dto.id); + } +} + +function findPluginDetails(pluginId: string) { + return availablePlugins.find((p: { id: string }) => p.id === pluginId); +} + +function fetchPluginsToInstall(): string[] { + if (!ENV_VARS.PLUGINS_TO_INSTALL) return []; + + return sanitizedArray(ENV_VARS.PLUGINS_TO_INSTALL); +} + +function sanitizedArray(string: string): string[] { + return [...new Set(string.split(',').map((p: string) => p.trim()))]; +} + +// eslint-disable-next-line @typescript-eslint/no-floating-promises +bootstrap(); diff --git a/server/scripts/plugins-reload.ts b/server/scripts/plugins-reload.ts new file mode 100644 index 0000000000..338976ae01 --- /dev/null +++ b/server/scripts/plugins-reload.ts @@ -0,0 +1,84 @@ +import { getEnvVars } from './database-config-utils'; +import { NestFactory } from '@nestjs/core'; +import { AppModule } from '../src/app.module'; +import { INestApplication } from '@nestjs/common'; +import { PluginsService } from '@services/plugins.service'; +import { CreatePluginDto } from '@dto/create-plugin.dto'; +import * as availablePlugins from 'src/assets/marketplace/plugins.json'; +import { validateSync } from 'class-validator'; +import { EntityManager } from 'typeorm'; +import { Plugin } from 'src/entities/plugin.entity'; + +const ENV_VARS = getEnvVars(); + +async function bootstrap() { + const nestApp = await NestFactory.create(AppModule, { + logger: ['error', 'warn'], + }); + + await validateAndReloadPlugins(nestApp); + + await nestApp.close(); + process.exit(0); +} + +async function validateAndReloadPlugins(nestApp: INestApplication) { + const pluginsService = nestApp.get(PluginsService); + const pluginsToReload = fetchPluginsToReload(); + const validPluginDtos: CreatePluginDto[] = []; + const invalidPluginDtos: CreatePluginDto[] = []; + + console.log('Plugins to reload:', pluginsToReload); + + for (const pluginId of pluginsToReload) { + const pluginDto = Object.assign(new CreatePluginDto(), findPluginDetails(pluginId)); + const validationErrors = validateSync(pluginDto); + + if (validationErrors.length === 0) { + validPluginDtos.push(pluginDto); + } else { + console.log(`Plugin with ID '${pluginId}' has validation errors:`, validationErrors); + invalidPluginDtos.push(pluginDto); + } + } + + invalidPluginDtos.length > 0 && + console.log( + 'Skipping invalid plugins:', + invalidPluginDtos.map((dto) => dto.id), + '\n' + ); + + for (const dto of validPluginDtos) { + const entityManager = nestApp.get(EntityManager); + const plugins = await entityManager.find(Plugin, { where: { pluginId: dto.id } }); + const pluginDbIds = []; + + // Note: Plugins are installed at instance level. But there is no uniqueness check for the plugin. + // This means that same plugin can be installed multiple times but this is restricted at UI. + // Hence when reloading, we are reloading all the plugins installed of the same name. + // If in future we support installing different versions of the same plugin, we should remove it selectively. + for (const plugin of plugins) { + await pluginsService.reload(plugin.id); + pluginDbIds.push(plugin.id); + } + + console.log('Reloaded:', dto.id, pluginDbIds); + } +} + +function findPluginDetails(pluginId: string) { + return availablePlugins.find((p: { id: string }) => p.id === pluginId); +} + +function fetchPluginsToReload(): string[] { + if (!ENV_VARS.PLUGINS_TO_RELOAD) return []; + return sanitizeArray(ENV_VARS.PLUGINS_TO_RELOAD); +} + +function sanitizeArray(pluginsToReload: string): string[] { + return [...new Set(pluginsToReload.split(',').map((pluginId: string) => pluginId.trim()))]; +} + +// eslint-disable-next-line @typescript-eslint/no-floating-promises +bootstrap(); diff --git a/server/scripts/plugins-uninstall.ts b/server/scripts/plugins-uninstall.ts new file mode 100644 index 0000000000..5b3b2b8b2d --- /dev/null +++ b/server/scripts/plugins-uninstall.ts @@ -0,0 +1,85 @@ +import * as availablePlugins from 'src/assets/marketplace/plugins.json'; +import { AppModule } from 'src/app.module'; +import { CreatePluginDto } from '@dto/create-plugin.dto'; +import { EntityManager } from 'typeorm'; +import { INestApplication } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; +import { Plugin } from 'src/entities/plugin.entity'; +import { PluginsService } from '@services/plugins.service'; +import { getEnvVars } from './database-config-utils'; +import { validateSync } from 'class-validator'; + +const ENV_VARS = getEnvVars(); + +async function bootstrap() { + const nestApp = await NestFactory.create(AppModule, { + logger: ['error', 'warn'], + }); + + await validateAndUninstallPlugins(nestApp); + + await nestApp.close(); + process.exit(0); +} + +async function validateAndUninstallPlugins(nestApp: INestApplication) { + const pluginsService = nestApp.get(PluginsService); + const pluginsToUninstall = fetchPluginsToUninstall(); + const validPluginDtos: CreatePluginDto[] = []; + const invalidPluginDtos: CreatePluginDto[] = []; + + console.log('Plugins to uninstall:', pluginsToUninstall); + + for (const pluginId of pluginsToUninstall) { + const pluginDto = Object.assign(new CreatePluginDto(), findPluginDetails(pluginId)); + const validationErrors = validateSync(pluginDto); + + if (validationErrors.length === 0) { + validPluginDtos.push(pluginDto); + } else { + console.log(`Plugin with ID '${pluginId}' has validation errors:`, validationErrors); + invalidPluginDtos.push(pluginDto); + } + } + + invalidPluginDtos.length > 0 && + console.log( + 'Skipping invalid plugins:', + invalidPluginDtos.map((dto) => dto.id), + '\n' + ); + + for (const dto of validPluginDtos) { + const entityManager = nestApp.get(EntityManager); + const plugins = await entityManager.find(Plugin, { where: { pluginId: dto.id } }); + const pluginDbIds = []; + + // Note: Plugins are installed at instance level. But there is no uniqueness check for the plugin. + // This means that same plugin can be installed multiple times but this is restricted at UI. + // Hence when removing, we are removing all the plugins installed of the same name. + // If in future we support installing different versions of the same plugin, we should remove it selectively. + for (const plugin of plugins) { + await pluginsService.remove(plugin.id); + pluginDbIds.push(plugin.id); + } + + console.log('Uninstalled:', dto.id, pluginDbIds); + } +} + +function findPluginDetails(pluginId: string) { + return availablePlugins.find((p: { id: string }) => p.id === pluginId); +} + +function fetchPluginsToUninstall(): string[] { + if (!ENV_VARS.PLUGINS_TO_UNINSTALL) return []; + + return sanitizedArray(ENV_VARS.PLUGINS_TO_UNINSTALL); +} + +function sanitizedArray(string: string): string[] { + return [...new Set(string.split(',').map((p: string) => p.trim()))]; +} + +// eslint-disable-next-line @typescript-eslint/no-floating-promises +bootstrap(); From 2c146ad3e879aac8358471d38969b85678d070cb Mon Sep 17 00:00:00 2001 From: Midhun G S Date: Fri, 20 Jun 2025 14:30:52 +0530 Subject: [PATCH 3/3] fix for plugin scripts (#13055) --- server/scripts/plugins-install.ts | 6 +++--- server/scripts/plugins-reload.ts | 6 +++--- server/scripts/plugins-uninstall.ts | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/scripts/plugins-install.ts b/server/scripts/plugins-install.ts index 1d647890e6..f80a9d5622 100644 --- a/server/scripts/plugins-install.ts +++ b/server/scripts/plugins-install.ts @@ -1,11 +1,11 @@ import * as availablePlugins from '../src/assets/marketplace/plugins.json'; -import { AppModule } from '../src/app.module'; -import { CreatePluginDto } from '@dto/create-plugin.dto'; +import { AppModule } from '@modules/app/module'; +import { CreatePluginDto } from '@modules/plugins/dto'; import { EntityManager } from 'typeorm'; import { INestApplication } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import { Plugin } from 'src/entities/plugin.entity'; -import { PluginsService } from '@services/plugins.service'; +import { PluginsService } from '@modules/plugins/service'; import { getEnvVars } from './database-config-utils'; import { validateSync } from 'class-validator'; diff --git a/server/scripts/plugins-reload.ts b/server/scripts/plugins-reload.ts index 338976ae01..288bf2c53f 100644 --- a/server/scripts/plugins-reload.ts +++ b/server/scripts/plugins-reload.ts @@ -1,9 +1,9 @@ import { getEnvVars } from './database-config-utils'; import { NestFactory } from '@nestjs/core'; -import { AppModule } from '../src/app.module'; +import { AppModule } from '@modules/app/module'; import { INestApplication } from '@nestjs/common'; -import { PluginsService } from '@services/plugins.service'; -import { CreatePluginDto } from '@dto/create-plugin.dto'; +import { PluginsService } from '@modules/plugins/service'; +import { CreatePluginDto } from '@modules/plugins/dto'; import * as availablePlugins from 'src/assets/marketplace/plugins.json'; import { validateSync } from 'class-validator'; import { EntityManager } from 'typeorm'; diff --git a/server/scripts/plugins-uninstall.ts b/server/scripts/plugins-uninstall.ts index 5b3b2b8b2d..095598465e 100644 --- a/server/scripts/plugins-uninstall.ts +++ b/server/scripts/plugins-uninstall.ts @@ -1,11 +1,11 @@ import * as availablePlugins from 'src/assets/marketplace/plugins.json'; -import { AppModule } from 'src/app.module'; -import { CreatePluginDto } from '@dto/create-plugin.dto'; +import { AppModule } from '@modules/app/module'; +import { CreatePluginDto } from '@modules/plugins/dto'; import { EntityManager } from 'typeorm'; import { INestApplication } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import { Plugin } from 'src/entities/plugin.entity'; -import { PluginsService } from '@services/plugins.service'; +import { PluginsService } from '@modules/plugins/service'; import { getEnvVars } from './database-config-utils'; import { validateSync } from 'class-validator';