Elgato_dark/ToolJet
1
0
Fork 0
mirror of https://github.com/ToolJet/ToolJet synced 2026-05-23 17:08:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Fix for missing foreign key constraint on app import (#9661)

Browse source 
* Import Foreign Key Fix

* Minor Changes

* Relocated Logic and Added a Transaction Wrapper for Rollback

* Shifted Entire Logic to bulkTableCreate

* Nested Transaction Queries Added

* Fallback Added

* App export break fix

---------

Co-authored-by: Shaurya Sharma <shauryasharma@Shauryas-MacBook-Air.local>
This commit is contained in:
Shaurya Sharma 2024-06-04 15:01:42 +05:30 committed by GitHub
parent 0f05b98aab
commit 35efc02a4c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 171 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
server/src/services/app_import_export.service.ts
View file

@ -1634,12 +1634,21 @@ export class AppImportExportService {
// Entire function should be santised for Undefined values
replaceTooljetDbTableIds(queryOptions, tooljetDatabaseMapping, organizationId: string) {
if (queryOptions?.operation === 'join_tables')
return this.replaceTooljetDbTableIdOnJoin(queryOptions, tooljetDatabaseMapping, organizationId);
let transformedQueryOptions;
if (Object.keys(queryOptions).includes('join_table')) {
transformedQueryOptions = this.replaceTooljetDbTableIdOnJoin(
queryOptions,
tooljetDatabaseMapping,
organizationId
);
}
if (queryOptions?.operation === 'join_tables') {
return transformedQueryOptions;
}
const mappedTableId = tooljetDatabaseMapping[queryOptions.table_id]?.id;
const mappedTableId = tooljetDatabaseMapping[transformedQueryOptions.table_id]?.id;
return {
...queryOptions,
...transformedQueryOptions,
...(mappedTableId && { table_id: mappedTableId }),
...(organizationId && { organization_id: organizationId }),
};
@ -1729,10 +1738,8 @@ export class AppImportExportService {
return this.updateNewTableIdForFilter(condition.conditions, tooljetDatabaseMapping);
} else {
const { operator = '=', leftField = {}, rightField = {} } = { ...condition };
if (leftField?.type && leftField.type === 'Column')
leftField['table'] = tooljetDatabaseMapping[leftField.table]?.id ?? leftField.table;
if (rightField?.type && rightField.type === 'Column')
rightField['table'] = tooljetDatabaseMapping[rightField.table]?.id ?? rightField.table;
if (leftField?.table) leftField['table'] = tooljetDatabaseMapping[leftField.table]?.id ?? leftField.table;
if (rightField?.table) rightField['table'] = tooljetDatabaseMapping[rightField.table]?.id ?? rightField.table;
return { operator, leftField, rightField };
}
});

19
server/src/services/import_export_resources.service.ts
View file

@ -7,7 +7,6 @@ import { ImportResourcesDto } from '@dto/import-resources.dto';
import { AppsService } from './apps.service';
import { CloneResourcesDto } from '@dto/clone-resources.dto';
import { isEmpty } from 'lodash';
import { transformTjdbImportDto } from 'src/helpers/tjdb_dto_transforms';
import { InjectEntityManager } from '@nestjs/typeorm';
import { EntityManager } from 'typeorm';
@ -51,24 +50,14 @@ export class ImportExportResourcesService {
}
async import(user: User, importResourcesDto: ImportResourcesDto, cloning = false) {
const tableNameMapping = {};
let tableNameMapping = {};
const imports = { app: [], tooljet_database: [] };
const importingVersion = importResourcesDto.tooljet_version;
if (importResourcesDto.tooljet_database) {
for (const tjdbImportDto of importResourcesDto.tooljet_database) {
const transformedDto = transformTjdbImportDto(tjdbImportDto, importingVersion);
const createdTable = await this.tooljetDbImportExportService.import(
importResourcesDto.organization_id,
transformedDto,
cloning
);
tableNameMapping[tjdbImportDto.id] = createdTable;
imports.tooljet_database.push(createdTable);
}
await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'");
const res = await this.tooljetDbImportExportService.bulkImport(importResourcesDto, importingVersion, cloning);
tableNameMapping = res.tableNameMapping;
imports.tooljet_database = res.tooljet_database;
}
if (importResourcesDto.app) {

93
server/src/services/tooljet_db.service.ts
View file

@ -56,12 +56,17 @@ export class TooljetDbService {
private readonly tooljetDbManager: EntityManager
) {}
async perform(organizationId: string, action: string, params = {}) {
async perform(
organizationId: string,
action: string,
params = {},
connectionManagers: Record<string, EntityManager> = { appManager: this.manager, tjdbManager: this.tooljetDbManager }
) {
const actionHandler = this.getActionHandler(action);
if (!actionHandler) {
throw new BadRequestException('Action not defined');
}
return await actionHandler.call(this, organizationId, params);
return await actionHandler.call(this, organizationId, params, connectionManagers);
}
private getActionHandler(action: string): ((organizationId: string, params: any) => Promise<any>) | undefined {
@ -84,11 +89,13 @@ export class TooljetDbService {
private async viewTable(
organizationId: string,
params
params,
connectionManagers: Record<string, EntityManager> = { appManager: this.manager, tjdbManager: this.tooljetDbManager }
): Promise<{ foreign_keys: ForeignKeyDetails[]; columns: TableColumnSchema[] }> {
const { table_name: tableName, id: id } = params;
const { appManager, tjdbManager } = connectionManagers;
const internalTable = await this.manager.findOne(InternalTable, {
const internalTable = await appManager.findOne(InternalTable, {
where: {
organizationId,
...(tableName && { tableName }),
@ -98,7 +105,7 @@ export class TooljetDbService {
if (!internalTable) throw new NotFoundException('Internal table not found: ' + tableName);
let foreign_keys = await this.tooljetDbManager.query(`
let foreign_keys = await tjdbManager.query(`
select
pgc.confrelid::regclass as referenced_table_name,
pgc.conname as constraint_name,
@ -139,7 +146,8 @@ export class TooljetDbService {
const referenced_tables_info = await this.fetchAndCheckIfValidForeignKeyTables(
referenced_table_list,
organizationId,
'TABLEID'
'TABLEID',
appManager
);
foreign_keys = foreign_keys.map((foreign_key_detail) => {
@ -150,7 +158,7 @@ export class TooljetDbService {
};
});
const columns = await this.tooljetDbManager.query(`
const columns = await tjdbManager.query(`
SELECT c.COLUMN_NAME,
c.DATA_TYPE,
CASE
@ -227,7 +235,11 @@ export class TooljetDbService {
return value;
}
private async createTable(organizationId: string, params) {
private async createTable(
organizationId: string,
params,
connectionManagers: Record<string, EntityManager> = { appManager: this.manager, tjdbManager: this.tooljetDbManager }
) {
const primaryKeyColumnList = params.columns
.filter((column) => column.constraints_type.is_primary_key)
.map((column) => column.column_name);
@ -235,8 +247,8 @@ export class TooljetDbService {
if (isEmpty(primaryKeyColumnList)) throw new BadRequestException('Primary key is mandatory');
const { table_name: tableName, foreign_keys = [] } = params;
const tableWithSameName = await this.manager.findOne(InternalTable, {
const { appManager, tjdbManager } = connectionManagers;
const tableWithSameName = await appManager.findOne(InternalTable, {
tableName,
organizationId,
});
@ -249,13 +261,15 @@ export class TooljetDbService {
referenced_tables_info = await this.fetchAndCheckIfValidForeignKeyTables(
referenced_table_list,
organizationId,
'TABLENAME'
'TABLENAME',
appManager
);
}
const isFKfromCompositePK = await this.checkIfForeignKeyReferencedColumnsAreFromCompositePrimaryKey(
foreign_keys,
organizationId
organizationId,
connectionManagers
);
if (isFKfromCompositePK)
@ -263,11 +277,11 @@ export class TooljetDbService {
'Foreign key cannot be created as the referenced column is in the composite primary key.'
);
const queryRunner = this.manager.connection.createQueryRunner();
const queryRunner = appManager?.queryRunner || appManager.connection.createQueryRunner();
const tjdbQueryRunner = tjdbManager?.queryRunner || tjdbManager.connection.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction();
const tjdbQueryRunner = this.tooljetDbManager.connection.createQueryRunner();
await tjdbQueryRunner.connect();
await tjdbQueryRunner.startTransaction();
@ -288,15 +302,15 @@ export class TooljetDbService {
}),
})
);
await tjdbQueryRunner.createPrimaryKey(internalTable.id, primaryKeyColumnList);
await queryRunner.commitTransaction();
await tjdbQueryRunner.commitTransaction();
await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'");
await queryRunner.release();
await tjdbQueryRunner.release();
//@ts-expect-error queryRunner has property transactionDepth which is not defined in type EntityManager
if (!queryRunner?.transactionDepth || queryRunner.transactionDepth < 1) await queryRunner.release();
//@ts-expect-error queryRunner has property transactionDepth which is not defined in type EntityManager
if (!tjdbQueryRunner?.transactionDepth || tjdbQueryRunner.transactionDepth < 1) await tjdbQueryRunner.release();
return { id: internalTable.id, table_name: tableName };
} catch (err) {
await queryRunner.rollbackTransaction();
@ -889,9 +903,10 @@ export class TooljetDbService {
private async fetchAndCheckIfValidForeignKeyTables(
referenced_table_list,
organisation_id,
type: 'TABLEID' | 'TABLENAME'
type: 'TABLEID' | 'TABLENAME',
manager: EntityManager = this.manager
) {
const valid_referenced_table_details = await this.manager.find(InternalTable, {
const valid_referenced_table_details = await manager.find(InternalTable, {
where: {
organizationId: organisation_id,
...(type === 'TABLENAME' && { tableName: In(referenced_table_list) }),
@ -929,11 +944,15 @@ export class TooljetDbService {
return referenced_tables_info;
}
private async createForeignKey(organizationId: string, params) {
private async createForeignKey(
organizationId: string,
params,
connectionManagers: Record<string, EntityManager> = { appManager: this.manager, tjdbManager: this.tooljetDbManager }
) {
const { table_name, foreign_keys } = params;
const { appManager, tjdbManager } = connectionManagers;
if (!foreign_keys?.length) throw new BadRequestException('Foreign key details are missing');
const internalTable = await this.manager.findOne(InternalTable, {
const internalTable = await appManager.findOne(InternalTable, {
where: { organizationId: organizationId, tableName: table_name },
});
if (!internalTable) throw new NotFoundException('Internal table not found: ' + table_name);
@ -943,12 +962,14 @@ export class TooljetDbService {
referenced_tables_info = await this.fetchAndCheckIfValidForeignKeyTables(
referenced_table_list,
organizationId,
'TABLENAME'
'TABLENAME',
appManager
);
const isFKfromCompositePK = await this.checkIfForeignKeyReferencedColumnsAreFromCompositePrimaryKey(
foreign_keys,
organizationId
organizationId,
connectionManagers
);
if (isFKfromCompositePK)
@ -956,19 +977,19 @@ export class TooljetDbService {
'Foreign key cannot be created as the referenced column is in the composite primary key.'
);
const tjdbQueryRunner = this.tooljetDbManager.connection.createQueryRunner();
const tjdbQueryRunner = tjdbManager?.queryRunner || tjdbManager.connection.createQueryRunner();
await tjdbQueryRunner.connect();
await tjdbQueryRunner.startTransaction();
try {
const foreignKeys = this.prepareForeignKeyDetailsJSON(foreign_keys, referenced_tables_info).map(
(foreignkeydetail) => new TableForeignKey({ ...foreignkeydetail })
);
await tjdbQueryRunner.createForeignKeys(internalTable.id, foreignKeys);
await tjdbQueryRunner.commitTransaction();
await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'");
await tjdbQueryRunner.release();
//@ts-expect-error queryRunner has property transactionDepth which is not defined in type EntityManager
if (!tjdbQueryRunner?.transactionDepth || tjdbQueryRunner.transactionDepth < 1) await tjdbQueryRunner.release();
return { statusCode: 200, message: 'Foreign key relation created successfully!' };
} catch (err) {
await tjdbQueryRunner.rollbackTransaction();
@ -1085,12 +1106,20 @@ export class TooljetDbService {
}
}
private async checkIfForeignKeyReferencedColumnsAreFromCompositePrimaryKey(foreignKeys, organizationId) {
private async checkIfForeignKeyReferencedColumnsAreFromCompositePrimaryKey(
foreignKeys,
organizationId,
connectionManagers: Record<string, EntityManager> = { appManager: this.manager, tjdbManager: this.tooljetDbManager }
) {
if (!foreignKeys.length) return;
let isFKfromCompositePK = false;
for (const foreignKeyDetails of foreignKeys) {
const { referenced_table_name = '', referenced_column_names = [] } = foreignKeyDetails;
const referencedTableMetaData = await this.viewTable(organizationId, { table_name: referenced_table_name });
const referencedTableMetaData = await this.viewTable(
organizationId,
{ table_name: referenced_table_name },
connectionManagers
);
const { columns = [] } = referencedTableMetaData;
const pkColumnList = [];

100
server/src/services/tooljet_db_import_export_service.ts
View file

@ -1,13 +1,23 @@
import { Injectable, NotFoundException } from '@nestjs/common';
import { Injectable, NotFoundException, Optional } from '@nestjs/common';
import { ExportTooljetDatabaseDto } from '@dto/export-resources.dto';
import { ImportTooljetDatabaseDto } from '@dto/import-resources.dto';
import { ImportResourcesDto, ImportTooljetDatabaseDto } from '@dto/import-resources.dto';
import { TooljetDbService } from './tooljet_db.service';
import { EntityManager } from 'typeorm';
import { InternalTable } from 'src/entities/internal_table.entity';
import { transformTjdbImportDto } from 'src/helpers/tjdb_dto_transforms';
import { InjectEntityManager } from '@nestjs/typeorm';
@Injectable()
export class TooljetDbImportExportService {
constructor(private readonly tooljetDbService: TooljetDbService, private readonly manager: EntityManager) {}
constructor(
private readonly tooljetDbService: TooljetDbService,
private readonly manager: EntityManager,
// TODO: remove optional decorator when
// ENABLE_TOOLJET_DB flag is deprecated
@Optional()
@InjectEntityManager('tooljetDb')
private readonly tooljetDbManager: EntityManager
) {}
async export(organizationId: string, tjDbDto: ExportTooljetDatabaseDto) {
const internalTable = await this.manager.findOne(InternalTable, {
@ -27,8 +37,75 @@ export class TooljetDbImportExportService {
};
}
async import(organizationId: string, tjDbDto: ImportTooljetDatabaseDto, cloning = false) {
const internalTableWithSameNameExists = await this.manager.findOne(InternalTable, {
async bulkImport(importResourcesDto: ImportResourcesDto, importingVersion: string, cloning: boolean) {
const tableNameMapping = {};
const tjdbDatabase = [];
const tableNameForeignKeyMapping = {};
const transformedTableNameMapping = {};
const queryRunner = this.manager.connection.createQueryRunner();
const tjdbQueryRunner = this.tooljetDbManager.connection.createQueryRunner();
const connectionManagers = { appManager: queryRunner.manager, tjdbManager: tjdbQueryRunner.manager };
await tjdbQueryRunner.connect();
await tjdbQueryRunner.startTransaction();
await queryRunner.connect();
await queryRunner.startTransaction();
try {
for (const tjdbImportDto of importResourcesDto.tooljet_database) {
const transformedDto = transformTjdbImportDto(tjdbImportDto, importingVersion);
const { foreign_keys } = transformedDto.schema;
const createdTable = await this.import(
importResourcesDto.organization_id,
transformedDto,
cloning,
connectionManagers
);
transformedTableNameMapping[tjdbImportDto.table_name] = createdTable.table_name;
if (foreign_keys.length) tableNameForeignKeyMapping[createdTable.table_name] = foreign_keys;
tableNameMapping[tjdbImportDto.id] = createdTable;
tjdbDatabase.push(createdTable);
}
for (const tableName in tableNameForeignKeyMapping) {
const foreignKeys = tableNameForeignKeyMapping[tableName].map((ele) => {
return {
...ele,
referenced_table_name:
transformedTableNameMapping?.[ele.referenced_table_name] || ele.referenced_table_name,
};
});
await this.tooljetDbService.perform(
importResourcesDto.organization_id,
'create_foreign_key',
{
table_name: tableName,
foreign_keys: foreignKeys,
},
connectionManagers
);
}
await tjdbQueryRunner.commitTransaction();
await queryRunner.commitTransaction();
await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'");
return { tableNameMapping, tooljet_database: tjdbDatabase };
} catch (err) {
await tjdbQueryRunner.rollbackTransaction();
await queryRunner.rollbackTransaction();
throw err;
} finally {
await tjdbQueryRunner.release();
await queryRunner.release();
}
}
async import(
organizationId: string,
tjDbDto: ImportTooljetDatabaseDto,
cloning = false,
connectionManagers: Record<string, EntityManager> = {}
) {
const { appManager } = connectionManagers;
const internalTableWithSameNameExists = await appManager.findOne(InternalTable, {
where: {
tableName: tjDbDto.table_name,
organizationId,
@ -49,10 +126,15 @@ export class TooljetDbImportExportService {
// TODO: Add support for foreign keys
const { columns } = tjDbDto.schema;
return await this.tooljetDbService.perform(organizationId, 'create_table', {
table_name: tableName,
...{ columns, foreign_keys: [] },
});
return await this.tooljetDbService.perform(
organizationId,
'create_table',
{
table_name: tableName,
...{ columns, foreign_keys: [] },
},
connectionManagers
);
}
async isTableColumnsSubset(internalTable: InternalTable, tjDbDto: ImportTooljetDatabaseDto): Promise<boolean> {