diff --git a/server/src/services/app_import_export.service.ts b/server/src/services/app_import_export.service.ts index 0d564cdd82..b025a53b2e 100644 --- a/server/src/services/app_import_export.service.ts +++ b/server/src/services/app_import_export.service.ts @@ -1634,12 +1634,21 @@ export class AppImportExportService { // Entire function should be santised for Undefined values replaceTooljetDbTableIds(queryOptions, tooljetDatabaseMapping, organizationId: string) { - if (queryOptions?.operation === 'join_tables') - return this.replaceTooljetDbTableIdOnJoin(queryOptions, tooljetDatabaseMapping, organizationId); + let transformedQueryOptions; + if (Object.keys(queryOptions).includes('join_table')) { + transformedQueryOptions = this.replaceTooljetDbTableIdOnJoin( + queryOptions, + tooljetDatabaseMapping, + organizationId + ); + } + if (queryOptions?.operation === 'join_tables') { + return transformedQueryOptions; + } - const mappedTableId = tooljetDatabaseMapping[queryOptions.table_id]?.id; + const mappedTableId = tooljetDatabaseMapping[transformedQueryOptions.table_id]?.id; return { - ...queryOptions, + ...transformedQueryOptions, ...(mappedTableId && { table_id: mappedTableId }), ...(organizationId && { organization_id: organizationId }), }; @@ -1729,10 +1738,8 @@ export class AppImportExportService { return this.updateNewTableIdForFilter(condition.conditions, tooljetDatabaseMapping); } else { const { operator = '=', leftField = {}, rightField = {} } = { ...condition }; - if (leftField?.type && leftField.type === 'Column') - leftField['table'] = tooljetDatabaseMapping[leftField.table]?.id ?? leftField.table; - if (rightField?.type && rightField.type === 'Column') - rightField['table'] = tooljetDatabaseMapping[rightField.table]?.id ?? rightField.table; + if (leftField?.table) leftField['table'] = tooljetDatabaseMapping[leftField.table]?.id ?? leftField.table; + if (rightField?.table) rightField['table'] = tooljetDatabaseMapping[rightField.table]?.id ?? rightField.table; return { operator, leftField, rightField }; } }); diff --git a/server/src/services/import_export_resources.service.ts b/server/src/services/import_export_resources.service.ts index db2d15e20b..6c70c2e382 100644 --- a/server/src/services/import_export_resources.service.ts +++ b/server/src/services/import_export_resources.service.ts @@ -7,7 +7,6 @@ import { ImportResourcesDto } from '@dto/import-resources.dto'; import { AppsService } from './apps.service'; import { CloneResourcesDto } from '@dto/clone-resources.dto'; import { isEmpty } from 'lodash'; -import { transformTjdbImportDto } from 'src/helpers/tjdb_dto_transforms'; import { InjectEntityManager } from '@nestjs/typeorm'; import { EntityManager } from 'typeorm'; @@ -51,24 +50,14 @@ export class ImportExportResourcesService { } async import(user: User, importResourcesDto: ImportResourcesDto, cloning = false) { - const tableNameMapping = {}; + let tableNameMapping = {}; const imports = { app: [], tooljet_database: [] }; const importingVersion = importResourcesDto.tooljet_version; if (importResourcesDto.tooljet_database) { - for (const tjdbImportDto of importResourcesDto.tooljet_database) { - const transformedDto = transformTjdbImportDto(tjdbImportDto, importingVersion); - - const createdTable = await this.tooljetDbImportExportService.import( - importResourcesDto.organization_id, - transformedDto, - cloning - ); - tableNameMapping[tjdbImportDto.id] = createdTable; - imports.tooljet_database.push(createdTable); - } - - await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'"); + const res = await this.tooljetDbImportExportService.bulkImport(importResourcesDto, importingVersion, cloning); + tableNameMapping = res.tableNameMapping; + imports.tooljet_database = res.tooljet_database; } if (importResourcesDto.app) { diff --git a/server/src/services/tooljet_db.service.ts b/server/src/services/tooljet_db.service.ts index aebdb6082e..91c7f98a5a 100644 --- a/server/src/services/tooljet_db.service.ts +++ b/server/src/services/tooljet_db.service.ts @@ -56,12 +56,17 @@ export class TooljetDbService { private readonly tooljetDbManager: EntityManager ) {} - async perform(organizationId: string, action: string, params = {}) { + async perform( + organizationId: string, + action: string, + params = {}, + connectionManagers: Record = { appManager: this.manager, tjdbManager: this.tooljetDbManager } + ) { const actionHandler = this.getActionHandler(action); if (!actionHandler) { throw new BadRequestException('Action not defined'); } - return await actionHandler.call(this, organizationId, params); + return await actionHandler.call(this, organizationId, params, connectionManagers); } private getActionHandler(action: string): ((organizationId: string, params: any) => Promise) | undefined { @@ -84,11 +89,13 @@ export class TooljetDbService { private async viewTable( organizationId: string, - params + params, + connectionManagers: Record = { appManager: this.manager, tjdbManager: this.tooljetDbManager } ): Promise<{ foreign_keys: ForeignKeyDetails[]; columns: TableColumnSchema[] }> { const { table_name: tableName, id: id } = params; + const { appManager, tjdbManager } = connectionManagers; - const internalTable = await this.manager.findOne(InternalTable, { + const internalTable = await appManager.findOne(InternalTable, { where: { organizationId, ...(tableName && { tableName }), @@ -98,7 +105,7 @@ export class TooljetDbService { if (!internalTable) throw new NotFoundException('Internal table not found: ' + tableName); - let foreign_keys = await this.tooljetDbManager.query(` + let foreign_keys = await tjdbManager.query(` select pgc.confrelid::regclass as referenced_table_name, pgc.conname as constraint_name, @@ -139,7 +146,8 @@ export class TooljetDbService { const referenced_tables_info = await this.fetchAndCheckIfValidForeignKeyTables( referenced_table_list, organizationId, - 'TABLEID' + 'TABLEID', + appManager ); foreign_keys = foreign_keys.map((foreign_key_detail) => { @@ -150,7 +158,7 @@ export class TooljetDbService { }; }); - const columns = await this.tooljetDbManager.query(` + const columns = await tjdbManager.query(` SELECT c.COLUMN_NAME, c.DATA_TYPE, CASE @@ -227,7 +235,11 @@ export class TooljetDbService { return value; } - private async createTable(organizationId: string, params) { + private async createTable( + organizationId: string, + params, + connectionManagers: Record = { appManager: this.manager, tjdbManager: this.tooljetDbManager } + ) { const primaryKeyColumnList = params.columns .filter((column) => column.constraints_type.is_primary_key) .map((column) => column.column_name); @@ -235,8 +247,8 @@ export class TooljetDbService { if (isEmpty(primaryKeyColumnList)) throw new BadRequestException('Primary key is mandatory'); const { table_name: tableName, foreign_keys = [] } = params; - - const tableWithSameName = await this.manager.findOne(InternalTable, { + const { appManager, tjdbManager } = connectionManagers; + const tableWithSameName = await appManager.findOne(InternalTable, { tableName, organizationId, }); @@ -249,13 +261,15 @@ export class TooljetDbService { referenced_tables_info = await this.fetchAndCheckIfValidForeignKeyTables( referenced_table_list, organizationId, - 'TABLENAME' + 'TABLENAME', + appManager ); } const isFKfromCompositePK = await this.checkIfForeignKeyReferencedColumnsAreFromCompositePrimaryKey( foreign_keys, - organizationId + organizationId, + connectionManagers ); if (isFKfromCompositePK) @@ -263,11 +277,11 @@ export class TooljetDbService { 'Foreign key cannot be created as the referenced column is in the composite primary key.' ); - const queryRunner = this.manager.connection.createQueryRunner(); + const queryRunner = appManager?.queryRunner || appManager.connection.createQueryRunner(); + const tjdbQueryRunner = tjdbManager?.queryRunner || tjdbManager.connection.createQueryRunner(); + await queryRunner.connect(); await queryRunner.startTransaction(); - - const tjdbQueryRunner = this.tooljetDbManager.connection.createQueryRunner(); await tjdbQueryRunner.connect(); await tjdbQueryRunner.startTransaction(); @@ -288,15 +302,15 @@ export class TooljetDbService { }), }) ); - await tjdbQueryRunner.createPrimaryKey(internalTable.id, primaryKeyColumnList); - await queryRunner.commitTransaction(); await tjdbQueryRunner.commitTransaction(); await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'"); - await queryRunner.release(); - await tjdbQueryRunner.release(); + //@ts-expect-error queryRunner has property transactionDepth which is not defined in type EntityManager + if (!queryRunner?.transactionDepth || queryRunner.transactionDepth < 1) await queryRunner.release(); + //@ts-expect-error queryRunner has property transactionDepth which is not defined in type EntityManager + if (!tjdbQueryRunner?.transactionDepth || tjdbQueryRunner.transactionDepth < 1) await tjdbQueryRunner.release(); return { id: internalTable.id, table_name: tableName }; } catch (err) { await queryRunner.rollbackTransaction(); @@ -889,9 +903,10 @@ export class TooljetDbService { private async fetchAndCheckIfValidForeignKeyTables( referenced_table_list, organisation_id, - type: 'TABLEID' | 'TABLENAME' + type: 'TABLEID' | 'TABLENAME', + manager: EntityManager = this.manager ) { - const valid_referenced_table_details = await this.manager.find(InternalTable, { + const valid_referenced_table_details = await manager.find(InternalTable, { where: { organizationId: organisation_id, ...(type === 'TABLENAME' && { tableName: In(referenced_table_list) }), @@ -929,11 +944,15 @@ export class TooljetDbService { return referenced_tables_info; } - private async createForeignKey(organizationId: string, params) { + private async createForeignKey( + organizationId: string, + params, + connectionManagers: Record = { appManager: this.manager, tjdbManager: this.tooljetDbManager } + ) { const { table_name, foreign_keys } = params; + const { appManager, tjdbManager } = connectionManagers; if (!foreign_keys?.length) throw new BadRequestException('Foreign key details are missing'); - - const internalTable = await this.manager.findOne(InternalTable, { + const internalTable = await appManager.findOne(InternalTable, { where: { organizationId: organizationId, tableName: table_name }, }); if (!internalTable) throw new NotFoundException('Internal table not found: ' + table_name); @@ -943,12 +962,14 @@ export class TooljetDbService { referenced_tables_info = await this.fetchAndCheckIfValidForeignKeyTables( referenced_table_list, organizationId, - 'TABLENAME' + 'TABLENAME', + appManager ); const isFKfromCompositePK = await this.checkIfForeignKeyReferencedColumnsAreFromCompositePrimaryKey( foreign_keys, - organizationId + organizationId, + connectionManagers ); if (isFKfromCompositePK) @@ -956,19 +977,19 @@ export class TooljetDbService { 'Foreign key cannot be created as the referenced column is in the composite primary key.' ); - const tjdbQueryRunner = this.tooljetDbManager.connection.createQueryRunner(); + const tjdbQueryRunner = tjdbManager?.queryRunner || tjdbManager.connection.createQueryRunner(); await tjdbQueryRunner.connect(); await tjdbQueryRunner.startTransaction(); - try { const foreignKeys = this.prepareForeignKeyDetailsJSON(foreign_keys, referenced_tables_info).map( (foreignkeydetail) => new TableForeignKey({ ...foreignkeydetail }) ); await tjdbQueryRunner.createForeignKeys(internalTable.id, foreignKeys); - await tjdbQueryRunner.commitTransaction(); await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'"); - await tjdbQueryRunner.release(); + //@ts-expect-error queryRunner has property transactionDepth which is not defined in type EntityManager + if (!tjdbQueryRunner?.transactionDepth || tjdbQueryRunner.transactionDepth < 1) await tjdbQueryRunner.release(); + return { statusCode: 200, message: 'Foreign key relation created successfully!' }; } catch (err) { await tjdbQueryRunner.rollbackTransaction(); @@ -1085,12 +1106,20 @@ export class TooljetDbService { } } - private async checkIfForeignKeyReferencedColumnsAreFromCompositePrimaryKey(foreignKeys, organizationId) { + private async checkIfForeignKeyReferencedColumnsAreFromCompositePrimaryKey( + foreignKeys, + organizationId, + connectionManagers: Record = { appManager: this.manager, tjdbManager: this.tooljetDbManager } + ) { if (!foreignKeys.length) return; let isFKfromCompositePK = false; for (const foreignKeyDetails of foreignKeys) { const { referenced_table_name = '', referenced_column_names = [] } = foreignKeyDetails; - const referencedTableMetaData = await this.viewTable(organizationId, { table_name: referenced_table_name }); + const referencedTableMetaData = await this.viewTable( + organizationId, + { table_name: referenced_table_name }, + connectionManagers + ); const { columns = [] } = referencedTableMetaData; const pkColumnList = []; diff --git a/server/src/services/tooljet_db_import_export_service.ts b/server/src/services/tooljet_db_import_export_service.ts index bb5d4d47fc..6a1f174478 100644 --- a/server/src/services/tooljet_db_import_export_service.ts +++ b/server/src/services/tooljet_db_import_export_service.ts @@ -1,13 +1,23 @@ -import { Injectable, NotFoundException } from '@nestjs/common'; +import { Injectable, NotFoundException, Optional } from '@nestjs/common'; import { ExportTooljetDatabaseDto } from '@dto/export-resources.dto'; -import { ImportTooljetDatabaseDto } from '@dto/import-resources.dto'; +import { ImportResourcesDto, ImportTooljetDatabaseDto } from '@dto/import-resources.dto'; import { TooljetDbService } from './tooljet_db.service'; import { EntityManager } from 'typeorm'; import { InternalTable } from 'src/entities/internal_table.entity'; +import { transformTjdbImportDto } from 'src/helpers/tjdb_dto_transforms'; +import { InjectEntityManager } from '@nestjs/typeorm'; @Injectable() export class TooljetDbImportExportService { - constructor(private readonly tooljetDbService: TooljetDbService, private readonly manager: EntityManager) {} + constructor( + private readonly tooljetDbService: TooljetDbService, + private readonly manager: EntityManager, + // TODO: remove optional decorator when + // ENABLE_TOOLJET_DB flag is deprecated + @Optional() + @InjectEntityManager('tooljetDb') + private readonly tooljetDbManager: EntityManager + ) {} async export(organizationId: string, tjDbDto: ExportTooljetDatabaseDto) { const internalTable = await this.manager.findOne(InternalTable, { @@ -27,8 +37,75 @@ export class TooljetDbImportExportService { }; } - async import(organizationId: string, tjDbDto: ImportTooljetDatabaseDto, cloning = false) { - const internalTableWithSameNameExists = await this.manager.findOne(InternalTable, { + async bulkImport(importResourcesDto: ImportResourcesDto, importingVersion: string, cloning: boolean) { + const tableNameMapping = {}; + const tjdbDatabase = []; + const tableNameForeignKeyMapping = {}; + const transformedTableNameMapping = {}; + const queryRunner = this.manager.connection.createQueryRunner(); + const tjdbQueryRunner = this.tooljetDbManager.connection.createQueryRunner(); + const connectionManagers = { appManager: queryRunner.manager, tjdbManager: tjdbQueryRunner.manager }; + await tjdbQueryRunner.connect(); + await tjdbQueryRunner.startTransaction(); + await queryRunner.connect(); + await queryRunner.startTransaction(); + + try { + for (const tjdbImportDto of importResourcesDto.tooljet_database) { + const transformedDto = transformTjdbImportDto(tjdbImportDto, importingVersion); + const { foreign_keys } = transformedDto.schema; + const createdTable = await this.import( + importResourcesDto.organization_id, + transformedDto, + cloning, + connectionManagers + ); + transformedTableNameMapping[tjdbImportDto.table_name] = createdTable.table_name; + if (foreign_keys.length) tableNameForeignKeyMapping[createdTable.table_name] = foreign_keys; + tableNameMapping[tjdbImportDto.id] = createdTable; + tjdbDatabase.push(createdTable); + } + for (const tableName in tableNameForeignKeyMapping) { + const foreignKeys = tableNameForeignKeyMapping[tableName].map((ele) => { + return { + ...ele, + referenced_table_name: + transformedTableNameMapping?.[ele.referenced_table_name] || ele.referenced_table_name, + }; + }); + await this.tooljetDbService.perform( + importResourcesDto.organization_id, + 'create_foreign_key', + { + table_name: tableName, + foreign_keys: foreignKeys, + }, + connectionManagers + ); + } + + await tjdbQueryRunner.commitTransaction(); + await queryRunner.commitTransaction(); + await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'"); + return { tableNameMapping, tooljet_database: tjdbDatabase }; + } catch (err) { + await tjdbQueryRunner.rollbackTransaction(); + await queryRunner.rollbackTransaction(); + throw err; + } finally { + await tjdbQueryRunner.release(); + await queryRunner.release(); + } + } + + async import( + organizationId: string, + tjDbDto: ImportTooljetDatabaseDto, + cloning = false, + connectionManagers: Record = {} + ) { + const { appManager } = connectionManagers; + const internalTableWithSameNameExists = await appManager.findOne(InternalTable, { where: { tableName: tjDbDto.table_name, organizationId, @@ -49,10 +126,15 @@ export class TooljetDbImportExportService { // TODO: Add support for foreign keys const { columns } = tjDbDto.schema; - return await this.tooljetDbService.perform(organizationId, 'create_table', { - table_name: tableName, - ...{ columns, foreign_keys: [] }, - }); + return await this.tooljetDbService.perform( + organizationId, + 'create_table', + { + table_name: tableName, + ...{ columns, foreign_keys: [] }, + }, + connectionManagers + ); } async isTableColumnsSubset(internalTable: InternalTable, tjDbDto: ImportTooljetDatabaseDto): Promise {