[v2-beta] migration improvement (#5047)

* migration improvement

* fix

* added constants and comments

server/data-migrations/1667076251897-BackfillDataSources.ts

@@ -51,8 +51,8 @@ export class BackfillDataSources1667076251897 implements MigrationInterface {
     let runjsDS, restapiDS;
     for (const kind of ['runjs', 'restapi']) {
       const dataSourceResult = await entityManager.query(
-        'insert into data_sources (name, kind, app_version_id, app_id) values ($1, $2, $3, $4) returning "id"',
-        [`${kind}default`, `${kind}default`, version.id, version.app_id]
+        'insert into data_sources (name, kind, app_version_id, app_id, type) values ($1, $2, $3, $4, $5) returning "id"',
+        [`${kind}default`, kind, version.id, version.app_id, 'static']
       );
 
       if (kind === 'runjs') {
@@ -79,10 +79,7 @@ export class BackfillDataSources1667076251897 implements MigrationInterface {
 
   public async down(queryRunner: QueryRunner): Promise<void> {
     const entityManager = queryRunner.manager;
-    const defaultDataSources = await entityManager.query('select id from data_sources where kind = $1 or kind = $2', [
-      'runjsdefault',
-      'restapidefault',
-    ]);
+    const defaultDataSources = await entityManager.query('select id from data_sources where type = $1', ['static']);
 
     if (defaultDataSources?.length) {
       await entityManager.query(

server/migrations/1671815159504-addDataSourceType.ts

@@ -0,0 +1,22 @@
+import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm';
+
+export class addDataSourceType1671815159504 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    // Adding column type: static : restapi, runjs and tooljetdb default: normal data sources
+    await queryRunner.addColumn(
+      'data_sources',
+      new TableColumn({
+        name: 'type',
+        type: 'enum',
+        enumName: 'type',
+        enum: ['static', 'default'],
+        default: `'default'`,
+        isNullable: false,
+      })
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropColumn('data_sources', 'type');
+  }
+}

server/src/controllers/data_queries.controller.ts

@@ -24,6 +24,7 @@ import { decode } from 'js-base64';
 import { dbTransactionWrap } from 'src/helpers/utils.helper';
 import { EntityManager } from 'typeorm';
 import { DataSource } from 'src/entities/data_source.entity';
+import { DataSourceTypes } from 'src/helpers/data_source.constants';
 
 @Controller('data_queries')
 export class DataQueriesController {
@@ -49,11 +50,7 @@ export class DataQueriesController {
 
     // serialize
     for (const query of queries) {
-      if (
-        query.dataSource.kind === 'runjsdefault' ||
-        query.dataSource.kind === 'restapidefault' ||
-        query.dataSource.kind === 'tooljetdbdefault'
-      ) {
+      if (query.dataSource.type === DataSourceTypes.STATIC) {
         delete query['dataSourceId'];
       }
       delete query['dataSource'];
@@ -207,7 +204,7 @@ export class DataQueriesController {
       ...query,
       dataSource: query['data_source_id']
         ? await this.dataSourcesService.findOne(query['data_source_id'])
-        : await this.dataSourcesService.findDataSourceByKind(`${kind}default`, appVersionId, environmentId),
+        : await this.dataSourcesService.findDefaultDataSourceByKind(kind, appVersionId, environmentId),
     };
 
     const ability = await this.appsAbilityFactory.appsActions(user, dataQueryEntity.dataSource.app.id);

server/src/entities/data_query.entity.ts

@@ -63,16 +63,6 @@ export class DataQuery extends BaseEntity {
 
   @AfterLoad()
   updateKind() {
-    if (this.dataSource) {
-      if (this.dataSource.kind === 'restapidefault') {
-        this.kind = 'restapi';
-      } else if (this.dataSource.kind === 'runjsdefault') {
-        this.kind = 'runjs';
-      } else if (this.dataSource.kind === 'tooljetdbdefault') {
-        this.kind = 'tooljetdb';
-      } else {
-        this.kind = this.dataSource.kind;
-      }
-    }
+    this.kind = this.dataSource?.kind;
   }
 }

server/src/entities/data_source.entity.ts

@@ -1,3 +1,4 @@
+import { DataSourceTypes } from 'src/helpers/data_source.constants';
 import {
   Entity,
   Column,
@@ -29,6 +30,15 @@ export class DataSource extends BaseEntity {
   @Column({ name: 'kind' })
   kind: string;
 
+  @Column({
+    type: 'enum',
+    enumName: 'type',
+    name: 'type',
+    enum: [DataSourceTypes.STATIC, DataSourceTypes.DEFAULT],
+    default: DataSourceTypes.DEFAULT,
+  })
+  type: string;
+
   @Column({ name: 'plugin_id' })
   pluginId: string;
 

server/src/helpers/data_source.constants.ts

@@ -0,0 +1,4 @@
+export enum DataSourceTypes {
+  STATIC = 'static',
+  DEFAULT = 'default',
+}

server/src/services/app_import_export.service.ts

@@ -14,6 +14,7 @@ import { AppEnvironment } from 'src/entities/app_environments.entity';
 import { DataSourceOptions } from 'src/entities/data_source_options.entity';
 import { AppEnvironmentService } from './app_environments.service';
 import { convertAppDefinitionFromSinglePageToMultiPage } from '../../lib/single-page-to-and-from-multipage-definition-conversion';
+import { DataSourceTypes } from 'src/helpers/data_source.constants';
 
 @Injectable()
 export class AppImportExportService {
@@ -298,15 +299,15 @@ export class AppImportExportService {
     for (const appVersion of appVersions) {
       const dsKindsToCreate = [];
 
-      if (!dataSources?.some((ds) => ds.kind === 'restapidefault')) {
+      if (!dataSources?.some((ds) => ds.kind === 'restapi' && ds.type === DataSourceTypes.STATIC)) {
         dsKindsToCreate.push('restapi');
       }
 
-      if (!dataSources?.some((ds) => ds.kind === 'runjsdefault')) {
+      if (!dataSources?.some((ds) => ds.kind === 'runjs' && ds.type === DataSourceTypes.STATIC)) {
         dsKindsToCreate.push('runjs');
       }
 
-      if (!dataSources?.some((ds) => ds.kind === 'tooljetdbdefault')) {
+      if (!dataSources?.some((ds) => ds.kind === 'tooljetdb' && ds.type === DataSourceTypes.STATIC)) {
         dsKindsToCreate.push('tooljetdb');
       }
 

server/src/services/data_queries.service.ts

@@ -82,10 +82,7 @@ export class DataQueriesService {
   async fetchServiceAndParsedParams(dataSource, dataQuery, queryOptions, organization_id) {
     const sourceOptions = await this.parseSourceOptions(dataSource.options);
     const parsedQueryOptions = await this.parseQueryOptions(dataQuery.options, queryOptions, organization_id);
-    const dsKind = ['restapidefault', 'runjsdefault', 'tooljetdbdefault'].includes(dataSource.kind)
-      ? dataSource.kind.split('default')[0]
-      : dataSource.kind;
-    const service = await this.pluginsHelper.getService(dataSource.pluginId, dsKind);
+    const service = await this.pluginsHelper.getService(dataSource.pluginId, dataSource.kind);
 
     return { service, sourceOptions, parsedQueryOptions };
   }

server/src/services/data_sources.service.ts

@@ -9,6 +9,7 @@ import { PluginsHelper } from '../helpers/plugins.helper';
 import { AppEnvironmentService } from './app_environments.service';
 import { App } from 'src/entities/app.entity';
 import { AppEnvironment } from 'src/entities/app_environments.entity';
+import { DataSourceTypes } from 'src/helpers/data_source.constants';
 
 @Injectable()
 export class DataSourcesService {
@@ -37,26 +38,25 @@ export class DataSourcesService {
         .leftJoinAndSelect('plugin.operationsFile', 'operationsFile')
         .where('data_source_options.environmentId = :selectedEnvironmentId', { selectedEnvironmentId })
         .andWhere('data_source.appVersionId = :appVersionId', { appVersionId })
+        .andWhere('data_source.type != :staticType', { staticType: DataSourceTypes.STATIC })
         .getMany();
 
       //remove tokenData from restapi datasources
-      const dataSources = result
-        ?.map((ds) => {
-          if (ds.kind === 'restapi') {
-            const options = {};
-            Object.keys(ds.dataSourceOptions?.[0]?.options || {}).filter((key) => {
-              if (key !== 'tokenData') {
-                return (options[key] = ds.dataSourceOptions[0].options[key]);
-              }
-            });
-            ds.options = options;
-          } else {
-            ds.options = { ...(ds.dataSourceOptions?.[0]?.options || {}) };
-          }
-          delete ds['dataSourceOptions'];
-          return ds;
-        })
-        ?.filter((ds) => ds.kind !== 'restapidefault' && ds.kind !== 'runjsdefault' && ds.kind !== 'tooljetdbdefault');
+      const dataSources = result?.map((ds) => {
+        if (ds.kind === 'restapi') {
+          const options = {};
+          Object.keys(ds.dataSourceOptions?.[0]?.options || {}).filter((key) => {
+            if (key !== 'tokenData') {
+              return (options[key] = ds.dataSourceOptions[0].options[key]);
+            }
+          });
+          ds.options = options;
+        } else {
+          ds.options = { ...(ds.dataSourceOptions?.[0]?.options || {}) };
+        }
+        delete ds['dataSourceOptions'];
+        return ds;
+      });
 
       return dataSources;
     });
@@ -92,13 +92,13 @@ export class DataSourcesService {
     ).app;
   }
 
-  async findDataSourceByKind(kind: string, appVersionId?: string, environmentId?: string) {
+  async findDefaultDataSourceByKind(kind: string, appVersionId?: string, environmentId?: string) {
     return await dbTransactionWrap(async (manager: EntityManager) => {
       const currentEnv = environmentId
         ? await manager.findOneOrFail(AppEnvironment, { where: { id: environmentId } })
         : await manager.findOneOrFail(AppEnvironment, { where: { isDefault: true, appVersionId } });
-      return await this.dataSourcesRepository.findOneOrFail({
-        where: { kind, appVersionId: currentEnv.appVersionId },
+      return await manager.findOneOrFail(DataSource, {
+        where: { kind, appVersionId: currentEnv.appVersionId, type: DataSourceTypes.STATIC },
         relations: ['plugin', 'apps'],
       });
     });
@@ -111,7 +111,7 @@ export class DataSourcesService {
     manager: EntityManager
   ): Promise<DataSource> {
     const defaultDataSource = await manager.findOne(DataSource, {
-      where: { kind: `${kind}default`, appVersionId },
+      where: { kind, appVersionId, type: DataSourceTypes.STATIC },
     });
 
     if (defaultDataSource) {
@@ -130,8 +130,9 @@ export class DataSourcesService {
   ): Promise<DataSource> {
     const newDataSource = manager.create(DataSource, {
       name: `${kind}default`,
-      kind: `${kind}default`,
+      kind,
       appVersionId,
+      type: DataSourceTypes.STATIC,
       pluginId,
       createdAt: new Date(),
       updatedAt: new Date(),