From 953c0d69e53a93cebb968d1fcf6ab3c70e0668cd Mon Sep 17 00:00:00 2001 From: Arpit Date: Fri, 20 Oct 2023 11:15:29 +0530 Subject: [PATCH] [appdef] fixes: migrations (#7951) * refactor migrations with batching * event actions: switch page should be mapped to correct new page id * fixes: importing json-schema app with multiple version: same components do not get render in the canvas * fixes: import/export of legecy apps * event actions mapping to correct page ids: migrations * fixes: migrations children not rendered in subcontainer components --- ...6-MigrateAppsDefinitionSchemaTransition.ts | 455 +++++++++++------- .../src/services/app_import_export.service.ts | 123 +++-- 2 files changed, 362 insertions(+), 216 deletions(-) diff --git a/server/data-migrations/1697473340856-MigrateAppsDefinitionSchemaTransition.ts b/server/data-migrations/1697473340856-MigrateAppsDefinitionSchemaTransition.ts index 20169cd23a..7cdf6942b7 100644 --- a/server/data-migrations/1697473340856-MigrateAppsDefinitionSchemaTransition.ts +++ b/server/data-migrations/1697473340856-MigrateAppsDefinitionSchemaTransition.ts @@ -1,4 +1,4 @@ -import { In, MigrationInterface, QueryRunner } from 'typeorm'; +import { In, MigrationInterface, QueryRunner, EntityManager } from 'typeorm'; import { AppVersion } from '../src/entities/app_version.entity'; import { Component } from 'src/entities/component.entity'; import { Page } from 'src/entities/page.entity'; @@ -6,10 +6,15 @@ import { Layout } from 'src/entities/layout.entity'; import { EventHandler, Target } from 'src/entities/event_handler.entity'; import { DataQuery } from 'src/entities/data_query.entity'; import { MigrationProgress, processDataInBatches } from 'src/helpers/utils.helper'; +import { v4 as uuid } from 'uuid'; + +interface AppResourceMappings { + pagesMapping: Record; + componentsMapping: Record; +} export class MigrateAppsDefinitionSchemaTransition1697473340856 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { - // let progress = 0; const entityManager = queryRunner.manager; const appVersionRepository = entityManager.getRepository(AppVersion); const appVersions = await appVersionRepository.find(); @@ -21,195 +26,279 @@ export class MigrateAppsDefinitionSchemaTransition1697473340856 implements Migra await processDataInBatches( entityManager, - async (entityManager, skip, take) => { + async (entityManager: EntityManager, skip: number, take: number) => { return entityManager.find(AppVersion, { where: { id: In(appVersions.map((appVersion) => appVersion.id)) }, take, skip, }); }, - async (entityManager, versions: AppVersion[]) => { - for (const version of versions) { - const definition = version['definition']; - - if (!definition) return; - - const dataQueriesRepository = entityManager.getRepository(DataQuery); - const dataQueries = await dataQueriesRepository.find({ - where: { appVersionId: version.id }, - }); - - let updateHomepageId = null; - - if (definition?.pages) { - for (const pageId of Object.keys(definition?.pages)) { - const page = definition.pages[pageId]; - const pageEvents = page.events || []; - const componentEvents = []; - const pagePositionInTheList = Object.keys(definition?.pages).indexOf(pageId); - const isHomepage = (definition['homePageId'] as any) === pageId; - const pageComponents = page.components; - const componentLayouts = []; - const mappedComponents = this.transformComponentData(pageComponents, componentEvents); - const newPage = entityManager.create(Page, { - name: page.name, - handle: page.handle, - appVersionId: version.id, - disabled: page.disabled || false, - hidden: page.hidden || false, - index: pagePositionInTheList, - }); - - const pageCreated = await entityManager.save(newPage); - mappedComponents.forEach((component) => { - component.page = pageCreated; - }); - - const savedComponents = await entityManager.save(Component, mappedComponents); - - savedComponents.forEach((component) => { - const componentLayout = pageComponents[component.id]['layouts']; - - if (componentLayout) { - for (const type in componentLayout) { - const layout = componentLayout[type]; - const newLayout = new Layout(); - newLayout.type = type; - newLayout.top = layout.top; - newLayout.left = layout.left; - newLayout.width = layout.width; - newLayout.height = layout.height; - newLayout.component = component; - componentLayouts.push(newLayout); - } - } - }); - - await entityManager.save(Layout, componentLayouts); - - if (pageEvents.length > 0) { - pageEvents.forEach(async (event, index) => { - const newEvent = { - name: event.eventId, - sourceId: pageCreated.id, - target: Target.page, - event: event, - index: pageEvents.index || index, - appVersionId: version.id, - }; - - await entityManager.save(EventHandler, newEvent); - }); - } - - componentEvents.forEach((eventObj) => { - if (eventObj.event?.length === 0) return; - - eventObj.event.forEach(async (event, index) => { - const newEvent = { - name: event.eventId, - sourceId: eventObj.componentId, - target: Target.component, - event: event, - index: eventObj.index || index, - appVersionId: version.id, - }; - - await entityManager.save(EventHandler, newEvent); - }); - }); - - savedComponents.forEach(async (component) => { - if (component.type === 'Table') { - const tableActions = component.properties?.actions?.value || []; - const tableColumns = component.properties?.columns?.value || []; - const tableActionAndColumnEvents = []; - - tableActions.forEach((action) => { - const actionEvents = action.events || []; - - actionEvents.forEach((event, index) => { - tableActionAndColumnEvents.push({ - name: event.eventId, - sourceId: component.id, - target: Target.tableAction, - event: { ...event, ref: action.name }, - index: event.index ?? index, - appVersionId: version.id, - }); - }); - }); - - tableColumns.forEach((column) => { - if (column?.columnType !== 'toggle') return; - const columnEvents = column.events || []; - - columnEvents.forEach((event, index) => { - tableActionAndColumnEvents.push({ - name: event.eventId, - sourceId: component.id, - target: Target.tableColumn, - event: { ...event, ref: column.name }, - index: event.index ?? index, - appVersionId: version.id, - }); - }); - }); - - await entityManager.save(EventHandler, tableActionAndColumnEvents); - } - }); - - if (isHomepage) { - updateHomepageId = pageCreated.id; - } - } - } - - for (const dataQuery of dataQueries) { - const queryEvents = dataQuery?.options?.events || []; - - if (queryEvents.length > 0) { - queryEvents.forEach(async (event, index) => { - const newEvent = { - name: event.eventId, - sourceId: dataQuery.id, - target: Target.dataQuery, - event: event, - index: queryEvents.index || index, - appVersionId: version.id, - }; - - await entityManager.save(EventHandler, newEvent); - }); - } - } - - migrationProgress.show(); - await entityManager.update( - AppVersion, - { id: version.id }, - { - homePageId: updateHomepageId, - showViewerNavigation: definition?.showViewerNavigation || true, - globalSettings: definition.globalSettings, - } - ); - } + async (entityManager: EntityManager, versions: AppVersion[]) => { + await this.processVersions(entityManager, versions, migrationProgress); }, batchSize ); } - private transformComponentData(data: object, componentEvents: any[]): Component[] { + private async processVersions( + entityManager: EntityManager, + versions: AppVersion[], + migrationProgress: MigrationProgress + ) { + for (const version of versions) { + const definition = version['definition']; + + if (!definition) return; + + const dataQueriesRepository = entityManager.getRepository(DataQuery); + const dataQueries = await dataQueriesRepository.find({ + where: { appVersionId: version.id }, + }); + + let updateHomepageId = null; + + const appResourceMappings: AppResourceMappings = { + pagesMapping: {}, + componentsMapping: {}, + }; + if (definition?.pages) { + for (const pageId of Object.keys(definition?.pages)) { + const page = definition.pages[pageId]; + const pagePositionInTheList = Object.keys(definition?.pages).indexOf(pageId); + const pageEvents = page.events || []; + const pageComponents = page.components; + + const isHomepage = (definition['homePageId'] as any) === pageId; + + const componentEvents = []; + const componentLayouts = []; + const transformedComponents = this.transformComponentData( + pageComponents, + componentEvents, + appResourceMappings.componentsMapping + ); + + const newPage = entityManager.create(Page, { + name: page.name, + handle: page.handle, + appVersionId: version.id, + disabled: page.disabled || false, + hidden: page.hidden || false, + index: pagePositionInTheList, + }); + + const pageCreated = await entityManager.save(newPage); + + appResourceMappings.pagesMapping[pageId] = pageCreated.id; + + transformedComponents.forEach((component) => { + component.page = pageCreated; + }); + + const savedComponents = await entityManager.save(Component, transformedComponents); + + for (const componentId in pageComponents) { + const componentLayout = pageComponents[componentId]['layouts']; + + if (componentLayout) { + for (const type in componentLayout) { + const layout = componentLayout[type]; + const newLayout = new Layout(); + newLayout.type = type; + newLayout.top = layout.top; + newLayout.left = layout.left; + newLayout.width = layout.width; + newLayout.height = layout.height; + newLayout.componentId = appResourceMappings.componentsMapping[componentId]; + + componentLayouts.push(newLayout); + } + } + } + + await entityManager.save(Layout, componentLayouts); + + if (pageEvents.length > 0) { + pageEvents.forEach(async (event, index) => { + const newEvent = { + name: event.eventId, + sourceId: pageCreated.id, + target: Target.page, + event: event, + index: pageEvents.index || index, + appVersionId: version.id, + }; + + await entityManager.save(EventHandler, newEvent); + }); + } + + componentEvents.forEach((eventObj) => { + if (eventObj.event?.length === 0) return; + + eventObj.event.forEach(async (event, index) => { + const newEvent = { + name: event.eventId, + sourceId: appResourceMappings.componentsMapping[eventObj.componentId], + target: Target.component, + event: event, + index: eventObj.index || index, + appVersionId: version.id, + }; + + await entityManager.save(EventHandler, newEvent); + }); + }); + + savedComponents.forEach(async (component) => { + if (component.type === 'Table') { + const tableActions = component.properties?.actions?.value || []; + const tableColumns = component.properties?.columns?.value || []; + const tableActionAndColumnEvents = []; + + tableActions.forEach((action) => { + const actionEvents = action.events || []; + + actionEvents.forEach((event, index) => { + tableActionAndColumnEvents.push({ + name: event.eventId, + sourceId: component.id, + target: Target.tableAction, + event: { ...event, ref: action.name }, + index: event.index ?? index, + appVersionId: version.id, + }); + }); + }); + + tableColumns.forEach((column) => { + if (column?.columnType !== 'toggle') return; + const columnEvents = column.events || []; + + columnEvents.forEach((event, index) => { + tableActionAndColumnEvents.push({ + name: event.eventId, + sourceId: component.id, + target: Target.tableColumn, + event: { ...event, ref: column.name }, + index: event.index ?? index, + appVersionId: version.id, + }); + }); + }); + + await entityManager.save(EventHandler, tableActionAndColumnEvents); + } + }); + + if (isHomepage) { + updateHomepageId = pageCreated.id; + } + } + } + + for (const dataQuery of dataQueries) { + const queryEvents = dataQuery?.options?.events || []; + + if (queryEvents.length > 0) { + queryEvents.forEach(async (event, index) => { + const newEvent = { + name: event.eventId, + sourceId: dataQuery.id, + target: Target.dataQuery, + event: event, + index: queryEvents.index || index, + appVersionId: version.id, + }; + + await entityManager.save(EventHandler, newEvent); + }); + } + } + + await entityManager.update( + AppVersion, + { id: version.id }, + { + homePageId: updateHomepageId, + showViewerNavigation: definition?.showViewerNavigation || true, + globalSettings: definition.globalSettings, + } + ); + + await this.updateEventActionsForNewVersionWithNewMappingIds( + entityManager, + version.id, + appResourceMappings.componentsMapping, + appResourceMappings.pagesMapping + ); + + migrationProgress.show(); + } + } + + async updateEventActionsForNewVersionWithNewMappingIds( + manager: EntityManager, + versionId: string, + oldComponentToNewComponentMapping: Record, + oldPageToNewPageMapping: Record + ) { + const allEvents = await manager.find(EventHandler, { + where: { appVersionId: versionId }, + }); + + for (const event of allEvents) { + const eventDefinition = event.event; + + if (eventDefinition?.actionId === 'switch-page') { + eventDefinition.pageId = oldPageToNewPageMapping[eventDefinition.pageId]; + } + + if (eventDefinition?.actionId === 'control-component') { + eventDefinition.componentId = oldComponentToNewComponentMapping[eventDefinition.componentId]; + } + + event.event = eventDefinition; + + await manager.save(event); + } + } + + private transformComponentData( + data: object, + componentEvents: any[], + componentsMapping: Record + ): Component[] { const transformedComponents: Component[] = []; + const allComponents = Object.keys(data).map((key) => { + return { + id: key, + ...data[key], + }; + }); + for (const componentId in data) { - const componentData = data[componentId]['component']; + const component = data[componentId]; + const componentData = component['component']; const transformedComponent: Component = new Component(); - transformedComponent.id = componentId; + let parentId = component.parent ? component.parent : null; + + const isParentTabOrCalendar = this.isChildOfTabsOrCalendar(component, allComponents, parentId); + + if (isParentTabOrCalendar) { + const childTabId = component.parent.split('-')[component.parent.split('-').length - 1]; + const _parentId = component?.parent?.split('-').slice(0, -1).join('-'); + const mappedParentId = componentsMapping[_parentId]; + + parentId = `${mappedParentId}-${childTabId}`; + } else { + parentId = componentsMapping[parentId]; + } + + transformedComponent.id = uuid(); transformedComponent.name = componentData.name; transformedComponent.type = componentData.component; transformedComponent.properties = componentData.definition.properties || {}; @@ -218,18 +307,34 @@ export class MigrateAppsDefinitionSchemaTransition1697473340856 implements Migra transformedComponent.general = componentData.definition.general || {}; transformedComponent.generalStyles = componentData.definition.generalStyles || {}; transformedComponent.displayPreferences = componentData.definition.others || {}; - transformedComponent.parent = data[componentId].parent || null; + transformedComponent.parent = component.parent ? parentId : null; + transformedComponents.push(transformedComponent); componentEvents.push({ componentId: componentId, event: componentData.definition.events, }); + componentsMapping[componentId] = transformedComponent.id; } return transformedComponents; } + isChildOfTabsOrCalendar = (component, allComponents = [], componentParentId = undefined) => { + if (componentParentId) { + const parentId = component?.parent?.split('-').slice(0, -1).join('-'); + + const parentComponent = allComponents.find((comp) => comp.id === parentId); + + if (parentComponent) { + return parentComponent.component.component === 'Tabs' || parentComponent.component.component === 'Calendar'; + } + } + + return false; + }; + public async down(queryRunner: QueryRunner): Promise { await queryRunner.query('DELETE FROM page'); await queryRunner.query('DELETE FROM component'); diff --git a/server/src/services/app_import_export.service.ts b/server/src/services/app_import_export.service.ts index 7c82f29172..6ce4dab66b 100644 --- a/server/src/services/app_import_export.service.ts +++ b/server/src/services/app_import_export.service.ts @@ -22,6 +22,7 @@ import { Page } from 'src/entities/page.entity'; import { Component } from 'src/entities/component.entity'; import { Layout } from 'src/entities/layout.entity'; import { EventHandler, Target } from 'src/entities/event_handler.entity'; +import { v4 as uuid } from 'uuid'; interface AppResourceMappings { defaultDataSourceIdMapping: Record; @@ -370,7 +371,11 @@ export class AppImportExportService { const pageComponents = page.components; - const mappedComponents = transformComponentData(pageComponents, componentEvents); + const mappedComponents = transformComponentData( + pageComponents, + componentEvents, + appResourceMappings.componentsMapping + ); const componentLayouts = []; @@ -387,14 +392,13 @@ export class AppImportExportService { appResourceMappings.pagesMapping[pageId] = pageCreated.id; mappedComponents.forEach((component) => { - appResourceMappings.componentsMapping[component.id] = component.id; component.page = pageCreated; }); const savedComponents = await manager.save(Component, mappedComponents); - savedComponents.forEach((component) => { - const componentLayout = pageComponents[component.id]['layouts']; + for (const componentId in pageComponents) { + const componentLayout = pageComponents[componentId]['layouts']; if (componentLayout) { for (const type in componentLayout) { @@ -405,12 +409,12 @@ export class AppImportExportService { newLayout.left = layout.left; newLayout.width = layout.width; newLayout.height = layout.height; - newLayout.component = component; + newLayout.componentId = appResourceMappings.componentsMapping[componentId]; componentLayouts.push(newLayout); } } - }); + } await manager.save(Layout, componentLayouts); @@ -435,14 +439,14 @@ export class AppImportExportService { if (eventObj.event?.length === 0) return; eventObj.event.forEach(async (event, index) => { - const newEvent = { + const newEvent = await manager.create(EventHandler, { name: event.eventId, - sourceId: eventObj.componentId, + sourceId: appResourceMappings.componentsMapping[eventObj.componentId], target: Target.component, event: event, index: eventObj.index || index, appVersionId: appResourceMappings.appVersionMapping[importingAppVersion.id], - }; + }); await manager.save(EventHandler, newEvent); }); @@ -504,20 +508,31 @@ export class AppImportExportService { homePageId: updateHomepageId, } ); + + await this.updateEventActionsForNewVersionWithNewMappingIds( + manager, + appResourceMappings.appVersionMapping[importingAppVersion.id], + appResourceMappings.dataQueryMapping, + appResourceMappings.componentsMapping, + appResourceMappings.pagesMapping, + isNormalizedAppDefinitionSchema + ); } } - const appVersionIds = Object.values(appResourceMappings.appVersionMapping); + if (isNormalizedAppDefinitionSchema) { + const appVersionIds = Object.values(appResourceMappings.appVersionMapping); - for (const appVersionId of appVersionIds) { - await this.updateEventActionsForNewVersionWithNewMappingIds( - manager, - appVersionId, - appResourceMappings.dataQueryMapping, - appResourceMappings.componentsMapping, - appResourceMappings.pagesMapping, - isNormalizedAppDefinitionSchema - ); + for (const appVersionId of appVersionIds) { + await this.updateEventActionsForNewVersionWithNewMappingIds( + manager, + appVersionId, + appResourceMappings.dataQueryMapping, + appResourceMappings.componentsMapping, + appResourceMappings.pagesMapping, + isNormalizedAppDefinitionSchema + ); + } } await this.setEditingVersionAsLatestVersion(manager, appResourceMappings.appVersionMapping, importingAppVersions); @@ -632,20 +647,6 @@ export class AppImportExportService { appResourceMappings.dataQueryMapping = dataQueryMapping; } - const isChildOfTabsOrCalendar = (component, allComponents = [], componentParentId = undefined) => { - if (componentParentId) { - const parentId = component?.parent?.split('-').slice(0, -1).join('-'); - - const parentComponent = allComponents.find((comp) => comp.id === parentId); - - if (parentComponent) { - return parentComponent.type === 'Tabs' || parentComponent.type === 'Calendar'; - } - } - - return false; - }; - const pagesOfAppVersion = importingPages.filter((page) => page.appVersionId === importingAppVersion.id); for (const page of pagesOfAppVersion) { @@ -675,7 +676,7 @@ export class AppImportExportService { let parentId = component.parent ? component.parent : null; - const isParentTabOrCalendar = isChildOfTabsOrCalendar(component, pageComponents, parentId); + const isParentTabOrCalendar = isChildOfTabsOrCalendar(component, pageComponents, parentId); //from here if (isParentTabOrCalendar) { const childTabId = component.parent.split('-')[component.parent.split('-').length - 1]; @@ -1456,8 +1457,6 @@ export class AppImportExportService { oldPageToNewPageMapping: Record, isNormalizedAppDefinitionSchema: boolean ) { - if (!isNormalizedAppDefinitionSchema) return; - const allEvents = await manager.find(EventHandler, { where: { appVersionId: versionId }, }); @@ -1465,7 +1464,7 @@ export class AppImportExportService { for (const event of allEvents) { const eventDefinition = event.event; - if (eventDefinition?.actionId === 'run-query') { + if (isNormalizedAppDefinitionSchema && eventDefinition?.actionId === 'run-query') { eventDefinition.queryId = oldDataQueryToNewMapping[eventDefinition.queryId]; } @@ -1495,14 +1494,41 @@ function convertSinglePageSchemaToMultiPageSchema(appParams: any) { return appParamsWithMultipageSchema; } -function transformComponentData(data: object, componentEvents: any[]): Component[] { +function transformComponentData( + data: object, + componentEvents: any[], + componentsMapping: Record +): Component[] { const transformedComponents: Component[] = []; + const allComponents = Object.keys(data).map((key) => { + return { + id: key, + ...data[key], + }; + }); + for (const componentId in data) { - const componentData = data[componentId]['component']; + const component = data[componentId]; + const componentData = component['component']; const transformedComponent: Component = new Component(); - transformedComponent.id = componentId; + + let parentId = component.parent ? component.parent : null; + + const isParentTabOrCalendar = isChildOfTabsOrCalendar(component, allComponents, parentId); + + if (isParentTabOrCalendar) { + const childTabId = component.parent.split('-')[component.parent.split('-').length - 1]; + const _parentId = component?.parent?.split('-').slice(0, -1).join('-'); + const mappedParentId = componentsMapping[_parentId]; + + parentId = `${mappedParentId}-${childTabId}`; + } else { + parentId = componentsMapping[parentId]; + } + + transformedComponent.id = uuid(); transformedComponent.name = componentData.name; transformedComponent.type = componentData.component; transformedComponent.properties = componentData.definition.properties || {}; @@ -1511,7 +1537,7 @@ function transformComponentData(data: object, componentEvents: any[]): Component transformedComponent.general = componentData.definition.general || {}; transformedComponent.generalStyles = componentData.definition.generalStyles || {}; transformedComponent.displayPreferences = componentData.definition.others || {}; - transformedComponent.parent = data[componentId].parent || null; + transformedComponent.parent = component.parent ? parentId : null; transformedComponents.push(transformedComponent); @@ -1519,7 +1545,22 @@ function transformComponentData(data: object, componentEvents: any[]): Component componentId: componentId, event: componentData.definition.events, }); + componentsMapping[componentId] = transformedComponent.id; } return transformedComponents; } + +const isChildOfTabsOrCalendar = (component, allComponents = [], componentParentId = undefined) => { + if (componentParentId) { + const parentId = component?.parent?.split('-').slice(0, -1).join('-'); + + const parentComponent = allComponents.find((comp) => comp.id === parentId); + + if (parentComponent) { + return parentComponent.component.component === 'Tabs' || parentComponent.component.component === 'Calendar'; + } + } + + return false; +};