Merge branch 'main' into fix/hide-modules-tab

package.json

@@ -40,6 +40,8 @@
     "postbuild:server": "npm --prefix server prune --production",
     "build": "npm run build:plugins:prod && npm run build:frontend && npm run build:server",
     "start:prod": "npm --prefix server run start:prod",
+    "cloud:setup": "npm run db:setup && npm run plugins:install && npm run plugins:uninstall && npm run plugins:reload",
+    "cloud:setup:prod": "npm run db:setup:prod && npm run plugins:install:prod && npm run plugins:uninstall:prod && npm run plugins:reload:prod",
     "db:create": "npm --prefix server run db:create",
     "db:create:prod": "npm --prefix server run db:create:prod",
     "db:migrate": "npm --prefix server run db:migrate",
@@ -51,6 +53,12 @@
     "db:reset": "npm --prefix server run db:reset",
     "db:drop": "npm --prefix server run db:drop",
     "deploy": "cp -a frontend/build/. public/",
+    "plugins:install": "npm --prefix server run plugins:install",
+    "plugins:install:prod": "npm --prefix server run plugins:install:prod",
+    "plugins:uninstall": "npm --prefix server run plugins:uninstall",
+    "plugins:uninstall:prod": "npm --prefix server run plugins:uninstall:prod",
+    "plugins:reload": "npm --prefix server run plugins:reload",
+    "plugins:reload:prod": "npm --prefix server run plugins:reload:prod",
     "heroku-postbuild": "./heroku-postbuild.sh",
     "prepare": "husky install",
     "update-version": "node update-version.js"

server/package.json

@@ -34,6 +34,12 @@
     "db:setup": "npm run db:create && npm run db:migrate",
     "db:setup:prod": "npm run db:create:prod && npm run db:migrate:prod",
     "db:reset": "npm run db:drop && npm run db:setup",
+    "plugins:install": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/plugins-install.ts",
+    "plugins:install:prod": "node dist/scripts/plugins-install.js",
+    "plugins:uninstall": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/plugins-uninstall.ts",
+    "plugins:uninstall:prod": "node dist/scripts/plugins-uninstall.js",
+    "plugins:reload": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/plugins-reload.ts",
+    "plugins:reload:prod": "node dist/scripts/plugins-reload.js",
     "typeorm": "typeorm-ts-node-commonjs",
     "copy-schemas": "copyfiles -u 4 src/dto/validators/schemas/**/*.json dist/src/dto/validators/schemas",
     "worker:dev": "WORKER=true NODE_ENV=development nest start --watch",

server/scripts/plugins-install.ts

@@ -0,0 +1,75 @@
+import * as availablePlugins from '../src/assets/marketplace/plugins.json';
+import { AppModule } from '@modules/app/module';
+import { CreatePluginDto } from '@modules/plugins/dto';
+import { EntityManager } from 'typeorm';
+import { INestApplication } from '@nestjs/common';
+import { NestFactory } from '@nestjs/core';
+import { Plugin } from 'src/entities/plugin.entity';
+import { PluginsService } from '@modules/plugins/service';
+import { getEnvVars } from './database-config-utils';
+import { validateSync } from 'class-validator';
+
+const ENV_VARS = getEnvVars();
+
+async function bootstrap() {
+  const nestApp = await NestFactory.create(AppModule, {
+    logger: ['error', 'warn'],
+  });
+
+  await validateAndInstallPlugins(nestApp);
+
+  await nestApp.close();
+  process.exit(0);
+}
+
+async function validateAndInstallPlugins(nestApp: INestApplication) {
+  const pluginsService = nestApp.get(PluginsService);
+  const pluginsToInstall = fetchPluginsToInstall();
+  const validPluginDtos: CreatePluginDto[] = [];
+  const invalidPluginDtos: CreatePluginDto[] = [];
+  const entityManager = nestApp.get(EntityManager);
+
+  console.log('Plugins to install:', pluginsToInstall);
+
+  for (const pluginId of pluginsToInstall) {
+    const pluginDto = Object.assign(new CreatePluginDto(), findPluginDetails(pluginId));
+    const validationErrors = validateSync(pluginDto);
+
+    if (validationErrors.length === 0) {
+      const plugin = await entityManager.findOne(Plugin, { where: { pluginId: pluginDto.id } });
+      plugin ? invalidPluginDtos.push(pluginDto) : validPluginDtos.push(pluginDto);
+    } else {
+      console.log(`Plugin with ID '${pluginId}' has validation errors:`, validationErrors);
+      invalidPluginDtos.push(pluginDto);
+    }
+  }
+
+  invalidPluginDtos.length > 0 &&
+    console.log(
+      'Skipping invalid plugins:',
+      invalidPluginDtos.map((dto) => dto.id),
+      '\n'
+    );
+
+  for (const dto of validPluginDtos) {
+    await pluginsService.install(dto);
+    console.log('Installed:', dto.id);
+  }
+}
+
+function findPluginDetails(pluginId: string) {
+  return availablePlugins.find((p: { id: string }) => p.id === pluginId);
+}
+
+function fetchPluginsToInstall(): string[] {
+  if (!ENV_VARS.PLUGINS_TO_INSTALL) return [];
+
+  return sanitizedArray(ENV_VARS.PLUGINS_TO_INSTALL);
+}
+
+function sanitizedArray(string: string): string[] {
+  return [...new Set(string.split(',').map((p: string) => p.trim()))];
+}
+
+// eslint-disable-next-line @typescript-eslint/no-floating-promises
+bootstrap();

server/scripts/plugins-reload.ts

@@ -0,0 +1,84 @@
+import { getEnvVars } from './database-config-utils';
+import { NestFactory } from '@nestjs/core';
+import { AppModule } from '@modules/app/module';
+import { INestApplication } from '@nestjs/common';
+import { PluginsService } from '@modules/plugins/service';
+import { CreatePluginDto } from '@modules/plugins/dto';
+import * as availablePlugins from 'src/assets/marketplace/plugins.json';
+import { validateSync } from 'class-validator';
+import { EntityManager } from 'typeorm';
+import { Plugin } from 'src/entities/plugin.entity';
+
+const ENV_VARS = getEnvVars();
+
+async function bootstrap() {
+  const nestApp = await NestFactory.create(AppModule, {
+    logger: ['error', 'warn'],
+  });
+
+  await validateAndReloadPlugins(nestApp);
+
+  await nestApp.close();
+  process.exit(0);
+}
+
+async function validateAndReloadPlugins(nestApp: INestApplication) {
+  const pluginsService = nestApp.get(PluginsService);
+  const pluginsToReload = fetchPluginsToReload();
+  const validPluginDtos: CreatePluginDto[] = [];
+  const invalidPluginDtos: CreatePluginDto[] = [];
+
+  console.log('Plugins to reload:', pluginsToReload);
+
+  for (const pluginId of pluginsToReload) {
+    const pluginDto = Object.assign(new CreatePluginDto(), findPluginDetails(pluginId));
+    const validationErrors = validateSync(pluginDto);
+
+    if (validationErrors.length === 0) {
+      validPluginDtos.push(pluginDto);
+    } else {
+      console.log(`Plugin with ID '${pluginId}' has validation errors:`, validationErrors);
+      invalidPluginDtos.push(pluginDto);
+    }
+  }
+
+  invalidPluginDtos.length > 0 &&
+    console.log(
+      'Skipping invalid plugins:',
+      invalidPluginDtos.map((dto) => dto.id),
+      '\n'
+    );
+
+  for (const dto of validPluginDtos) {
+    const entityManager = nestApp.get(EntityManager);
+    const plugins = await entityManager.find(Plugin, { where: { pluginId: dto.id } });
+    const pluginDbIds = [];
+
+    // Note: Plugins are installed at instance level. But there is no uniqueness check for the plugin.
+    // This means that same plugin can be installed multiple times but this is restricted at UI.
+    // Hence when reloading, we are reloading all the plugins installed of the same name.
+    // If in future we support installing different versions of the same plugin, we should remove it selectively.
+    for (const plugin of plugins) {
+      await pluginsService.reload(plugin.id);
+      pluginDbIds.push(plugin.id);
+    }
+
+    console.log('Reloaded:', dto.id, pluginDbIds);
+  }
+}
+
+function findPluginDetails(pluginId: string) {
+  return availablePlugins.find((p: { id: string }) => p.id === pluginId);
+}
+
+function fetchPluginsToReload(): string[] {
+  if (!ENV_VARS.PLUGINS_TO_RELOAD) return [];
+  return sanitizeArray(ENV_VARS.PLUGINS_TO_RELOAD);
+}
+
+function sanitizeArray(pluginsToReload: string): string[] {
+  return [...new Set(pluginsToReload.split(',').map((pluginId: string) => pluginId.trim()))];
+}
+
+// eslint-disable-next-line @typescript-eslint/no-floating-promises
+bootstrap();

server/scripts/plugins-uninstall.ts

@@ -0,0 +1,85 @@
+import * as availablePlugins from 'src/assets/marketplace/plugins.json';
+import { AppModule } from '@modules/app/module';
+import { CreatePluginDto } from '@modules/plugins/dto';
+import { EntityManager } from 'typeorm';
+import { INestApplication } from '@nestjs/common';
+import { NestFactory } from '@nestjs/core';
+import { Plugin } from 'src/entities/plugin.entity';
+import { PluginsService } from '@modules/plugins/service';
+import { getEnvVars } from './database-config-utils';
+import { validateSync } from 'class-validator';
+
+const ENV_VARS = getEnvVars();
+
+async function bootstrap() {
+  const nestApp = await NestFactory.create(AppModule, {
+    logger: ['error', 'warn'],
+  });
+
+  await validateAndUninstallPlugins(nestApp);
+
+  await nestApp.close();
+  process.exit(0);
+}
+
+async function validateAndUninstallPlugins(nestApp: INestApplication) {
+  const pluginsService = nestApp.get(PluginsService);
+  const pluginsToUninstall = fetchPluginsToUninstall();
+  const validPluginDtos: CreatePluginDto[] = [];
+  const invalidPluginDtos: CreatePluginDto[] = [];
+
+  console.log('Plugins to uninstall:', pluginsToUninstall);
+
+  for (const pluginId of pluginsToUninstall) {
+    const pluginDto = Object.assign(new CreatePluginDto(), findPluginDetails(pluginId));
+    const validationErrors = validateSync(pluginDto);
+
+    if (validationErrors.length === 0) {
+      validPluginDtos.push(pluginDto);
+    } else {
+      console.log(`Plugin with ID '${pluginId}' has validation errors:`, validationErrors);
+      invalidPluginDtos.push(pluginDto);
+    }
+  }
+
+  invalidPluginDtos.length > 0 &&
+    console.log(
+      'Skipping invalid plugins:',
+      invalidPluginDtos.map((dto) => dto.id),
+      '\n'
+    );
+
+  for (const dto of validPluginDtos) {
+    const entityManager = nestApp.get(EntityManager);
+    const plugins = await entityManager.find(Plugin, { where: { pluginId: dto.id } });
+    const pluginDbIds = [];
+
+    // Note: Plugins are installed at instance level. But there is no uniqueness check for the plugin.
+    // This means that same plugin can be installed multiple times but this is restricted at UI.
+    // Hence when removing, we are removing all the plugins installed of the same name.
+    // If in future we support installing different versions of the same plugin, we should remove it selectively.
+    for (const plugin of plugins) {
+      await pluginsService.remove(plugin.id);
+      pluginDbIds.push(plugin.id);
+    }
+
+    console.log('Uninstalled:', dto.id, pluginDbIds);
+  }
+}
+
+function findPluginDetails(pluginId: string) {
+  return availablePlugins.find((p: { id: string }) => p.id === pluginId);
+}
+
+function fetchPluginsToUninstall(): string[] {
+  if (!ENV_VARS.PLUGINS_TO_UNINSTALL) return [];
+
+  return sanitizedArray(ENV_VARS.PLUGINS_TO_UNINSTALL);
+}
+
+function sanitizedArray(string: string): string[] {
+  return [...new Set(string.split(',').map((p: string) => p.trim()))];
+}
+
+// eslint-disable-next-line @typescript-eslint/no-floating-promises
+bootstrap();