diff --git a/package.json b/package.json index ba210b0a8a..33850bd03f 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,8 @@ "postbuild:server": "npm --prefix server prune --production", "build": "npm run build:plugins:prod && npm run build:frontend && npm run build:server", "start:prod": "npm --prefix server run start:prod", + "cloud:setup": "npm run db:setup && npm run plugins:install && npm run plugins:uninstall && npm run plugins:reload", + "cloud:setup:prod": "npm run db:setup:prod && npm run plugins:install:prod && npm run plugins:uninstall:prod && npm run plugins:reload:prod", "db:create": "npm --prefix server run db:create", "db:create:prod": "npm --prefix server run db:create:prod", "db:migrate": "npm --prefix server run db:migrate", @@ -51,6 +53,12 @@ "db:reset": "npm --prefix server run db:reset", "db:drop": "npm --prefix server run db:drop", "deploy": "cp -a frontend/build/. public/", + "plugins:install": "npm --prefix server run plugins:install", + "plugins:install:prod": "npm --prefix server run plugins:install:prod", + "plugins:uninstall": "npm --prefix server run plugins:uninstall", + "plugins:uninstall:prod": "npm --prefix server run plugins:uninstall:prod", + "plugins:reload": "npm --prefix server run plugins:reload", + "plugins:reload:prod": "npm --prefix server run plugins:reload:prod", "heroku-postbuild": "./heroku-postbuild.sh", "prepare": "husky install", "update-version": "node update-version.js" diff --git a/server/migrations/1740399879253-CreateAiConversations.ts b/server/migrations/1737530238311-CreateTablesForToojetAiConversations.ts similarity index 100% rename from server/migrations/1740399879253-CreateAiConversations.ts rename to server/migrations/1737530238311-CreateTablesForToojetAiConversations.ts diff --git a/server/package.json b/server/package.json index e90223cccd..9de2850bba 100644 --- a/server/package.json +++ b/server/package.json @@ -34,6 +34,12 @@ "db:setup": "npm run db:create && npm run db:migrate", "db:setup:prod": "npm run db:create:prod && npm run db:migrate:prod", "db:reset": "npm run db:drop && npm run db:setup", + "plugins:install": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/plugins-install.ts", + "plugins:install:prod": "node dist/scripts/plugins-install.js", + "plugins:uninstall": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/plugins-uninstall.ts", + "plugins:uninstall:prod": "node dist/scripts/plugins-uninstall.js", + "plugins:reload": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/plugins-reload.ts", + "plugins:reload:prod": "node dist/scripts/plugins-reload.js", "typeorm": "typeorm-ts-node-commonjs", "copy-schemas": "copyfiles -u 4 src/dto/validators/schemas/**/*.json dist/src/dto/validators/schemas", "worker:dev": "WORKER=true NODE_ENV=development nest start --watch", diff --git a/server/scripts/plugins-install.ts b/server/scripts/plugins-install.ts new file mode 100644 index 0000000000..f80a9d5622 --- /dev/null +++ b/server/scripts/plugins-install.ts @@ -0,0 +1,75 @@ +import * as availablePlugins from '../src/assets/marketplace/plugins.json'; +import { AppModule } from '@modules/app/module'; +import { CreatePluginDto } from '@modules/plugins/dto'; +import { EntityManager } from 'typeorm'; +import { INestApplication } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; +import { Plugin } from 'src/entities/plugin.entity'; +import { PluginsService } from '@modules/plugins/service'; +import { getEnvVars } from './database-config-utils'; +import { validateSync } from 'class-validator'; + +const ENV_VARS = getEnvVars(); + +async function bootstrap() { + const nestApp = await NestFactory.create(AppModule, { + logger: ['error', 'warn'], + }); + + await validateAndInstallPlugins(nestApp); + + await nestApp.close(); + process.exit(0); +} + +async function validateAndInstallPlugins(nestApp: INestApplication) { + const pluginsService = nestApp.get(PluginsService); + const pluginsToInstall = fetchPluginsToInstall(); + const validPluginDtos: CreatePluginDto[] = []; + const invalidPluginDtos: CreatePluginDto[] = []; + const entityManager = nestApp.get(EntityManager); + + console.log('Plugins to install:', pluginsToInstall); + + for (const pluginId of pluginsToInstall) { + const pluginDto = Object.assign(new CreatePluginDto(), findPluginDetails(pluginId)); + const validationErrors = validateSync(pluginDto); + + if (validationErrors.length === 0) { + const plugin = await entityManager.findOne(Plugin, { where: { pluginId: pluginDto.id } }); + plugin ? invalidPluginDtos.push(pluginDto) : validPluginDtos.push(pluginDto); + } else { + console.log(`Plugin with ID '${pluginId}' has validation errors:`, validationErrors); + invalidPluginDtos.push(pluginDto); + } + } + + invalidPluginDtos.length > 0 && + console.log( + 'Skipping invalid plugins:', + invalidPluginDtos.map((dto) => dto.id), + '\n' + ); + + for (const dto of validPluginDtos) { + await pluginsService.install(dto); + console.log('Installed:', dto.id); + } +} + +function findPluginDetails(pluginId: string) { + return availablePlugins.find((p: { id: string }) => p.id === pluginId); +} + +function fetchPluginsToInstall(): string[] { + if (!ENV_VARS.PLUGINS_TO_INSTALL) return []; + + return sanitizedArray(ENV_VARS.PLUGINS_TO_INSTALL); +} + +function sanitizedArray(string: string): string[] { + return [...new Set(string.split(',').map((p: string) => p.trim()))]; +} + +// eslint-disable-next-line @typescript-eslint/no-floating-promises +bootstrap(); diff --git a/server/scripts/plugins-reload.ts b/server/scripts/plugins-reload.ts new file mode 100644 index 0000000000..288bf2c53f --- /dev/null +++ b/server/scripts/plugins-reload.ts @@ -0,0 +1,84 @@ +import { getEnvVars } from './database-config-utils'; +import { NestFactory } from '@nestjs/core'; +import { AppModule } from '@modules/app/module'; +import { INestApplication } from '@nestjs/common'; +import { PluginsService } from '@modules/plugins/service'; +import { CreatePluginDto } from '@modules/plugins/dto'; +import * as availablePlugins from 'src/assets/marketplace/plugins.json'; +import { validateSync } from 'class-validator'; +import { EntityManager } from 'typeorm'; +import { Plugin } from 'src/entities/plugin.entity'; + +const ENV_VARS = getEnvVars(); + +async function bootstrap() { + const nestApp = await NestFactory.create(AppModule, { + logger: ['error', 'warn'], + }); + + await validateAndReloadPlugins(nestApp); + + await nestApp.close(); + process.exit(0); +} + +async function validateAndReloadPlugins(nestApp: INestApplication) { + const pluginsService = nestApp.get(PluginsService); + const pluginsToReload = fetchPluginsToReload(); + const validPluginDtos: CreatePluginDto[] = []; + const invalidPluginDtos: CreatePluginDto[] = []; + + console.log('Plugins to reload:', pluginsToReload); + + for (const pluginId of pluginsToReload) { + const pluginDto = Object.assign(new CreatePluginDto(), findPluginDetails(pluginId)); + const validationErrors = validateSync(pluginDto); + + if (validationErrors.length === 0) { + validPluginDtos.push(pluginDto); + } else { + console.log(`Plugin with ID '${pluginId}' has validation errors:`, validationErrors); + invalidPluginDtos.push(pluginDto); + } + } + + invalidPluginDtos.length > 0 && + console.log( + 'Skipping invalid plugins:', + invalidPluginDtos.map((dto) => dto.id), + '\n' + ); + + for (const dto of validPluginDtos) { + const entityManager = nestApp.get(EntityManager); + const plugins = await entityManager.find(Plugin, { where: { pluginId: dto.id } }); + const pluginDbIds = []; + + // Note: Plugins are installed at instance level. But there is no uniqueness check for the plugin. + // This means that same plugin can be installed multiple times but this is restricted at UI. + // Hence when reloading, we are reloading all the plugins installed of the same name. + // If in future we support installing different versions of the same plugin, we should remove it selectively. + for (const plugin of plugins) { + await pluginsService.reload(plugin.id); + pluginDbIds.push(plugin.id); + } + + console.log('Reloaded:', dto.id, pluginDbIds); + } +} + +function findPluginDetails(pluginId: string) { + return availablePlugins.find((p: { id: string }) => p.id === pluginId); +} + +function fetchPluginsToReload(): string[] { + if (!ENV_VARS.PLUGINS_TO_RELOAD) return []; + return sanitizeArray(ENV_VARS.PLUGINS_TO_RELOAD); +} + +function sanitizeArray(pluginsToReload: string): string[] { + return [...new Set(pluginsToReload.split(',').map((pluginId: string) => pluginId.trim()))]; +} + +// eslint-disable-next-line @typescript-eslint/no-floating-promises +bootstrap(); diff --git a/server/scripts/plugins-uninstall.ts b/server/scripts/plugins-uninstall.ts new file mode 100644 index 0000000000..095598465e --- /dev/null +++ b/server/scripts/plugins-uninstall.ts @@ -0,0 +1,85 @@ +import * as availablePlugins from 'src/assets/marketplace/plugins.json'; +import { AppModule } from '@modules/app/module'; +import { CreatePluginDto } from '@modules/plugins/dto'; +import { EntityManager } from 'typeorm'; +import { INestApplication } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; +import { Plugin } from 'src/entities/plugin.entity'; +import { PluginsService } from '@modules/plugins/service'; +import { getEnvVars } from './database-config-utils'; +import { validateSync } from 'class-validator'; + +const ENV_VARS = getEnvVars(); + +async function bootstrap() { + const nestApp = await NestFactory.create(AppModule, { + logger: ['error', 'warn'], + }); + + await validateAndUninstallPlugins(nestApp); + + await nestApp.close(); + process.exit(0); +} + +async function validateAndUninstallPlugins(nestApp: INestApplication) { + const pluginsService = nestApp.get(PluginsService); + const pluginsToUninstall = fetchPluginsToUninstall(); + const validPluginDtos: CreatePluginDto[] = []; + const invalidPluginDtos: CreatePluginDto[] = []; + + console.log('Plugins to uninstall:', pluginsToUninstall); + + for (const pluginId of pluginsToUninstall) { + const pluginDto = Object.assign(new CreatePluginDto(), findPluginDetails(pluginId)); + const validationErrors = validateSync(pluginDto); + + if (validationErrors.length === 0) { + validPluginDtos.push(pluginDto); + } else { + console.log(`Plugin with ID '${pluginId}' has validation errors:`, validationErrors); + invalidPluginDtos.push(pluginDto); + } + } + + invalidPluginDtos.length > 0 && + console.log( + 'Skipping invalid plugins:', + invalidPluginDtos.map((dto) => dto.id), + '\n' + ); + + for (const dto of validPluginDtos) { + const entityManager = nestApp.get(EntityManager); + const plugins = await entityManager.find(Plugin, { where: { pluginId: dto.id } }); + const pluginDbIds = []; + + // Note: Plugins are installed at instance level. But there is no uniqueness check for the plugin. + // This means that same plugin can be installed multiple times but this is restricted at UI. + // Hence when removing, we are removing all the plugins installed of the same name. + // If in future we support installing different versions of the same plugin, we should remove it selectively. + for (const plugin of plugins) { + await pluginsService.remove(plugin.id); + pluginDbIds.push(plugin.id); + } + + console.log('Uninstalled:', dto.id, pluginDbIds); + } +} + +function findPluginDetails(pluginId: string) { + return availablePlugins.find((p: { id: string }) => p.id === pluginId); +} + +function fetchPluginsToUninstall(): string[] { + if (!ENV_VARS.PLUGINS_TO_UNINSTALL) return []; + + return sanitizedArray(ENV_VARS.PLUGINS_TO_UNINSTALL); +} + +function sanitizedArray(string: string): string[] { + return [...new Set(string.split(',').map((p: string) => p.trim()))]; +} + +// eslint-disable-next-line @typescript-eslint/no-floating-promises +bootstrap();