[appdef] fixes: migrations (#7951)

* refactor migrations with batching

* event actions: switch page should be mapped to correct new page id

* fixes: importing json-schema app with multiple version: same components do not get render in the canvas

* fixes: import/export of legecy apps

* event actions mapping to correct page ids: migrations

* fixes: migrations  children not rendered in subcontainer components
This commit is contained in:
Arpit 2023-10-20 11:15:29 +05:30 committed by GitHub
parent c002faf2e8
commit 953c0d69e5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 362 additions and 216 deletions

View file

@ -1,4 +1,4 @@
import { In, MigrationInterface, QueryRunner } from 'typeorm';
import { In, MigrationInterface, QueryRunner, EntityManager } from 'typeorm';
import { AppVersion } from '../src/entities/app_version.entity';
import { Component } from 'src/entities/component.entity';
import { Page } from 'src/entities/page.entity';
@ -6,10 +6,15 @@ import { Layout } from 'src/entities/layout.entity';
import { EventHandler, Target } from 'src/entities/event_handler.entity';
import { DataQuery } from 'src/entities/data_query.entity';
import { MigrationProgress, processDataInBatches } from 'src/helpers/utils.helper';
import { v4 as uuid } from 'uuid';
interface AppResourceMappings {
pagesMapping: Record<string, string>;
componentsMapping: Record<string, string>;
}
export class MigrateAppsDefinitionSchemaTransition1697473340856 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
// let progress = 0;
const entityManager = queryRunner.manager;
const appVersionRepository = entityManager.getRepository(AppVersion);
const appVersions = await appVersionRepository.find();
@ -21,195 +26,279 @@ export class MigrateAppsDefinitionSchemaTransition1697473340856 implements Migra
await processDataInBatches(
entityManager,
async (entityManager, skip, take) => {
async (entityManager: EntityManager, skip: number, take: number) => {
return entityManager.find(AppVersion, {
where: { id: In(appVersions.map((appVersion) => appVersion.id)) },
take,
skip,
});
},
async (entityManager, versions: AppVersion[]) => {
for (const version of versions) {
const definition = version['definition'];
if (!definition) return;
const dataQueriesRepository = entityManager.getRepository(DataQuery);
const dataQueries = await dataQueriesRepository.find({
where: { appVersionId: version.id },
});
let updateHomepageId = null;
if (definition?.pages) {
for (const pageId of Object.keys(definition?.pages)) {
const page = definition.pages[pageId];
const pageEvents = page.events || [];
const componentEvents = [];
const pagePositionInTheList = Object.keys(definition?.pages).indexOf(pageId);
const isHomepage = (definition['homePageId'] as any) === pageId;
const pageComponents = page.components;
const componentLayouts = [];
const mappedComponents = this.transformComponentData(pageComponents, componentEvents);
const newPage = entityManager.create(Page, {
name: page.name,
handle: page.handle,
appVersionId: version.id,
disabled: page.disabled || false,
hidden: page.hidden || false,
index: pagePositionInTheList,
});
const pageCreated = await entityManager.save(newPage);
mappedComponents.forEach((component) => {
component.page = pageCreated;
});
const savedComponents = await entityManager.save(Component, mappedComponents);
savedComponents.forEach((component) => {
const componentLayout = pageComponents[component.id]['layouts'];
if (componentLayout) {
for (const type in componentLayout) {
const layout = componentLayout[type];
const newLayout = new Layout();
newLayout.type = type;
newLayout.top = layout.top;
newLayout.left = layout.left;
newLayout.width = layout.width;
newLayout.height = layout.height;
newLayout.component = component;
componentLayouts.push(newLayout);
}
}
});
await entityManager.save(Layout, componentLayouts);
if (pageEvents.length > 0) {
pageEvents.forEach(async (event, index) => {
const newEvent = {
name: event.eventId,
sourceId: pageCreated.id,
target: Target.page,
event: event,
index: pageEvents.index || index,
appVersionId: version.id,
};
await entityManager.save(EventHandler, newEvent);
});
}
componentEvents.forEach((eventObj) => {
if (eventObj.event?.length === 0) return;
eventObj.event.forEach(async (event, index) => {
const newEvent = {
name: event.eventId,
sourceId: eventObj.componentId,
target: Target.component,
event: event,
index: eventObj.index || index,
appVersionId: version.id,
};
await entityManager.save(EventHandler, newEvent);
});
});
savedComponents.forEach(async (component) => {
if (component.type === 'Table') {
const tableActions = component.properties?.actions?.value || [];
const tableColumns = component.properties?.columns?.value || [];
const tableActionAndColumnEvents = [];
tableActions.forEach((action) => {
const actionEvents = action.events || [];
actionEvents.forEach((event, index) => {
tableActionAndColumnEvents.push({
name: event.eventId,
sourceId: component.id,
target: Target.tableAction,
event: { ...event, ref: action.name },
index: event.index ?? index,
appVersionId: version.id,
});
});
});
tableColumns.forEach((column) => {
if (column?.columnType !== 'toggle') return;
const columnEvents = column.events || [];
columnEvents.forEach((event, index) => {
tableActionAndColumnEvents.push({
name: event.eventId,
sourceId: component.id,
target: Target.tableColumn,
event: { ...event, ref: column.name },
index: event.index ?? index,
appVersionId: version.id,
});
});
});
await entityManager.save(EventHandler, tableActionAndColumnEvents);
}
});
if (isHomepage) {
updateHomepageId = pageCreated.id;
}
}
}
for (const dataQuery of dataQueries) {
const queryEvents = dataQuery?.options?.events || [];
if (queryEvents.length > 0) {
queryEvents.forEach(async (event, index) => {
const newEvent = {
name: event.eventId,
sourceId: dataQuery.id,
target: Target.dataQuery,
event: event,
index: queryEvents.index || index,
appVersionId: version.id,
};
await entityManager.save(EventHandler, newEvent);
});
}
}
migrationProgress.show();
await entityManager.update(
AppVersion,
{ id: version.id },
{
homePageId: updateHomepageId,
showViewerNavigation: definition?.showViewerNavigation || true,
globalSettings: definition.globalSettings,
}
);
}
async (entityManager: EntityManager, versions: AppVersion[]) => {
await this.processVersions(entityManager, versions, migrationProgress);
},
batchSize
);
}
private transformComponentData(data: object, componentEvents: any[]): Component[] {
private async processVersions(
entityManager: EntityManager,
versions: AppVersion[],
migrationProgress: MigrationProgress
) {
for (const version of versions) {
const definition = version['definition'];
if (!definition) return;
const dataQueriesRepository = entityManager.getRepository(DataQuery);
const dataQueries = await dataQueriesRepository.find({
where: { appVersionId: version.id },
});
let updateHomepageId = null;
const appResourceMappings: AppResourceMappings = {
pagesMapping: {},
componentsMapping: {},
};
if (definition?.pages) {
for (const pageId of Object.keys(definition?.pages)) {
const page = definition.pages[pageId];
const pagePositionInTheList = Object.keys(definition?.pages).indexOf(pageId);
const pageEvents = page.events || [];
const pageComponents = page.components;
const isHomepage = (definition['homePageId'] as any) === pageId;
const componentEvents = [];
const componentLayouts = [];
const transformedComponents = this.transformComponentData(
pageComponents,
componentEvents,
appResourceMappings.componentsMapping
);
const newPage = entityManager.create(Page, {
name: page.name,
handle: page.handle,
appVersionId: version.id,
disabled: page.disabled || false,
hidden: page.hidden || false,
index: pagePositionInTheList,
});
const pageCreated = await entityManager.save(newPage);
appResourceMappings.pagesMapping[pageId] = pageCreated.id;
transformedComponents.forEach((component) => {
component.page = pageCreated;
});
const savedComponents = await entityManager.save(Component, transformedComponents);
for (const componentId in pageComponents) {
const componentLayout = pageComponents[componentId]['layouts'];
if (componentLayout) {
for (const type in componentLayout) {
const layout = componentLayout[type];
const newLayout = new Layout();
newLayout.type = type;
newLayout.top = layout.top;
newLayout.left = layout.left;
newLayout.width = layout.width;
newLayout.height = layout.height;
newLayout.componentId = appResourceMappings.componentsMapping[componentId];
componentLayouts.push(newLayout);
}
}
}
await entityManager.save(Layout, componentLayouts);
if (pageEvents.length > 0) {
pageEvents.forEach(async (event, index) => {
const newEvent = {
name: event.eventId,
sourceId: pageCreated.id,
target: Target.page,
event: event,
index: pageEvents.index || index,
appVersionId: version.id,
};
await entityManager.save(EventHandler, newEvent);
});
}
componentEvents.forEach((eventObj) => {
if (eventObj.event?.length === 0) return;
eventObj.event.forEach(async (event, index) => {
const newEvent = {
name: event.eventId,
sourceId: appResourceMappings.componentsMapping[eventObj.componentId],
target: Target.component,
event: event,
index: eventObj.index || index,
appVersionId: version.id,
};
await entityManager.save(EventHandler, newEvent);
});
});
savedComponents.forEach(async (component) => {
if (component.type === 'Table') {
const tableActions = component.properties?.actions?.value || [];
const tableColumns = component.properties?.columns?.value || [];
const tableActionAndColumnEvents = [];
tableActions.forEach((action) => {
const actionEvents = action.events || [];
actionEvents.forEach((event, index) => {
tableActionAndColumnEvents.push({
name: event.eventId,
sourceId: component.id,
target: Target.tableAction,
event: { ...event, ref: action.name },
index: event.index ?? index,
appVersionId: version.id,
});
});
});
tableColumns.forEach((column) => {
if (column?.columnType !== 'toggle') return;
const columnEvents = column.events || [];
columnEvents.forEach((event, index) => {
tableActionAndColumnEvents.push({
name: event.eventId,
sourceId: component.id,
target: Target.tableColumn,
event: { ...event, ref: column.name },
index: event.index ?? index,
appVersionId: version.id,
});
});
});
await entityManager.save(EventHandler, tableActionAndColumnEvents);
}
});
if (isHomepage) {
updateHomepageId = pageCreated.id;
}
}
}
for (const dataQuery of dataQueries) {
const queryEvents = dataQuery?.options?.events || [];
if (queryEvents.length > 0) {
queryEvents.forEach(async (event, index) => {
const newEvent = {
name: event.eventId,
sourceId: dataQuery.id,
target: Target.dataQuery,
event: event,
index: queryEvents.index || index,
appVersionId: version.id,
};
await entityManager.save(EventHandler, newEvent);
});
}
}
await entityManager.update(
AppVersion,
{ id: version.id },
{
homePageId: updateHomepageId,
showViewerNavigation: definition?.showViewerNavigation || true,
globalSettings: definition.globalSettings,
}
);
await this.updateEventActionsForNewVersionWithNewMappingIds(
entityManager,
version.id,
appResourceMappings.componentsMapping,
appResourceMappings.pagesMapping
);
migrationProgress.show();
}
}
async updateEventActionsForNewVersionWithNewMappingIds(
manager: EntityManager,
versionId: string,
oldComponentToNewComponentMapping: Record<string, unknown>,
oldPageToNewPageMapping: Record<string, unknown>
) {
const allEvents = await manager.find(EventHandler, {
where: { appVersionId: versionId },
});
for (const event of allEvents) {
const eventDefinition = event.event;
if (eventDefinition?.actionId === 'switch-page') {
eventDefinition.pageId = oldPageToNewPageMapping[eventDefinition.pageId];
}
if (eventDefinition?.actionId === 'control-component') {
eventDefinition.componentId = oldComponentToNewComponentMapping[eventDefinition.componentId];
}
event.event = eventDefinition;
await manager.save(event);
}
}
private transformComponentData(
data: object,
componentEvents: any[],
componentsMapping: Record<string, string>
): Component[] {
const transformedComponents: Component[] = [];
const allComponents = Object.keys(data).map((key) => {
return {
id: key,
...data[key],
};
});
for (const componentId in data) {
const componentData = data[componentId]['component'];
const component = data[componentId];
const componentData = component['component'];
const transformedComponent: Component = new Component();
transformedComponent.id = componentId;
let parentId = component.parent ? component.parent : null;
const isParentTabOrCalendar = this.isChildOfTabsOrCalendar(component, allComponents, parentId);
if (isParentTabOrCalendar) {
const childTabId = component.parent.split('-')[component.parent.split('-').length - 1];
const _parentId = component?.parent?.split('-').slice(0, -1).join('-');
const mappedParentId = componentsMapping[_parentId];
parentId = `${mappedParentId}-${childTabId}`;
} else {
parentId = componentsMapping[parentId];
}
transformedComponent.id = uuid();
transformedComponent.name = componentData.name;
transformedComponent.type = componentData.component;
transformedComponent.properties = componentData.definition.properties || {};
@ -218,18 +307,34 @@ export class MigrateAppsDefinitionSchemaTransition1697473340856 implements Migra
transformedComponent.general = componentData.definition.general || {};
transformedComponent.generalStyles = componentData.definition.generalStyles || {};
transformedComponent.displayPreferences = componentData.definition.others || {};
transformedComponent.parent = data[componentId].parent || null;
transformedComponent.parent = component.parent ? parentId : null;
transformedComponents.push(transformedComponent);
componentEvents.push({
componentId: componentId,
event: componentData.definition.events,
});
componentsMapping[componentId] = transformedComponent.id;
}
return transformedComponents;
}
isChildOfTabsOrCalendar = (component, allComponents = [], componentParentId = undefined) => {
if (componentParentId) {
const parentId = component?.parent?.split('-').slice(0, -1).join('-');
const parentComponent = allComponents.find((comp) => comp.id === parentId);
if (parentComponent) {
return parentComponent.component.component === 'Tabs' || parentComponent.component.component === 'Calendar';
}
}
return false;
};
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query('DELETE FROM page');
await queryRunner.query('DELETE FROM component');

View file

@ -22,6 +22,7 @@ import { Page } from 'src/entities/page.entity';
import { Component } from 'src/entities/component.entity';
import { Layout } from 'src/entities/layout.entity';
import { EventHandler, Target } from 'src/entities/event_handler.entity';
import { v4 as uuid } from 'uuid';
interface AppResourceMappings {
defaultDataSourceIdMapping: Record<string, string>;
@ -370,7 +371,11 @@ export class AppImportExportService {
const pageComponents = page.components;
const mappedComponents = transformComponentData(pageComponents, componentEvents);
const mappedComponents = transformComponentData(
pageComponents,
componentEvents,
appResourceMappings.componentsMapping
);
const componentLayouts = [];
@ -387,14 +392,13 @@ export class AppImportExportService {
appResourceMappings.pagesMapping[pageId] = pageCreated.id;
mappedComponents.forEach((component) => {
appResourceMappings.componentsMapping[component.id] = component.id;
component.page = pageCreated;
});
const savedComponents = await manager.save(Component, mappedComponents);
savedComponents.forEach((component) => {
const componentLayout = pageComponents[component.id]['layouts'];
for (const componentId in pageComponents) {
const componentLayout = pageComponents[componentId]['layouts'];
if (componentLayout) {
for (const type in componentLayout) {
@ -405,12 +409,12 @@ export class AppImportExportService {
newLayout.left = layout.left;
newLayout.width = layout.width;
newLayout.height = layout.height;
newLayout.component = component;
newLayout.componentId = appResourceMappings.componentsMapping[componentId];
componentLayouts.push(newLayout);
}
}
});
}
await manager.save(Layout, componentLayouts);
@ -435,14 +439,14 @@ export class AppImportExportService {
if (eventObj.event?.length === 0) return;
eventObj.event.forEach(async (event, index) => {
const newEvent = {
const newEvent = await manager.create(EventHandler, {
name: event.eventId,
sourceId: eventObj.componentId,
sourceId: appResourceMappings.componentsMapping[eventObj.componentId],
target: Target.component,
event: event,
index: eventObj.index || index,
appVersionId: appResourceMappings.appVersionMapping[importingAppVersion.id],
};
});
await manager.save(EventHandler, newEvent);
});
@ -504,20 +508,31 @@ export class AppImportExportService {
homePageId: updateHomepageId,
}
);
await this.updateEventActionsForNewVersionWithNewMappingIds(
manager,
appResourceMappings.appVersionMapping[importingAppVersion.id],
appResourceMappings.dataQueryMapping,
appResourceMappings.componentsMapping,
appResourceMappings.pagesMapping,
isNormalizedAppDefinitionSchema
);
}
}
const appVersionIds = Object.values(appResourceMappings.appVersionMapping);
if (isNormalizedAppDefinitionSchema) {
const appVersionIds = Object.values(appResourceMappings.appVersionMapping);
for (const appVersionId of appVersionIds) {
await this.updateEventActionsForNewVersionWithNewMappingIds(
manager,
appVersionId,
appResourceMappings.dataQueryMapping,
appResourceMappings.componentsMapping,
appResourceMappings.pagesMapping,
isNormalizedAppDefinitionSchema
);
for (const appVersionId of appVersionIds) {
await this.updateEventActionsForNewVersionWithNewMappingIds(
manager,
appVersionId,
appResourceMappings.dataQueryMapping,
appResourceMappings.componentsMapping,
appResourceMappings.pagesMapping,
isNormalizedAppDefinitionSchema
);
}
}
await this.setEditingVersionAsLatestVersion(manager, appResourceMappings.appVersionMapping, importingAppVersions);
@ -632,20 +647,6 @@ export class AppImportExportService {
appResourceMappings.dataQueryMapping = dataQueryMapping;
}
const isChildOfTabsOrCalendar = (component, allComponents = [], componentParentId = undefined) => {
if (componentParentId) {
const parentId = component?.parent?.split('-').slice(0, -1).join('-');
const parentComponent = allComponents.find((comp) => comp.id === parentId);
if (parentComponent) {
return parentComponent.type === 'Tabs' || parentComponent.type === 'Calendar';
}
}
return false;
};
const pagesOfAppVersion = importingPages.filter((page) => page.appVersionId === importingAppVersion.id);
for (const page of pagesOfAppVersion) {
@ -675,7 +676,7 @@ export class AppImportExportService {
let parentId = component.parent ? component.parent : null;
const isParentTabOrCalendar = isChildOfTabsOrCalendar(component, pageComponents, parentId);
const isParentTabOrCalendar = isChildOfTabsOrCalendar(component, pageComponents, parentId); //from here
if (isParentTabOrCalendar) {
const childTabId = component.parent.split('-')[component.parent.split('-').length - 1];
@ -1456,8 +1457,6 @@ export class AppImportExportService {
oldPageToNewPageMapping: Record<string, unknown>,
isNormalizedAppDefinitionSchema: boolean
) {
if (!isNormalizedAppDefinitionSchema) return;
const allEvents = await manager.find(EventHandler, {
where: { appVersionId: versionId },
});
@ -1465,7 +1464,7 @@ export class AppImportExportService {
for (const event of allEvents) {
const eventDefinition = event.event;
if (eventDefinition?.actionId === 'run-query') {
if (isNormalizedAppDefinitionSchema && eventDefinition?.actionId === 'run-query') {
eventDefinition.queryId = oldDataQueryToNewMapping[eventDefinition.queryId];
}
@ -1495,14 +1494,41 @@ function convertSinglePageSchemaToMultiPageSchema(appParams: any) {
return appParamsWithMultipageSchema;
}
function transformComponentData(data: object, componentEvents: any[]): Component[] {
function transformComponentData(
data: object,
componentEvents: any[],
componentsMapping: Record<string, string>
): Component[] {
const transformedComponents: Component[] = [];
const allComponents = Object.keys(data).map((key) => {
return {
id: key,
...data[key],
};
});
for (const componentId in data) {
const componentData = data[componentId]['component'];
const component = data[componentId];
const componentData = component['component'];
const transformedComponent: Component = new Component();
transformedComponent.id = componentId;
let parentId = component.parent ? component.parent : null;
const isParentTabOrCalendar = isChildOfTabsOrCalendar(component, allComponents, parentId);
if (isParentTabOrCalendar) {
const childTabId = component.parent.split('-')[component.parent.split('-').length - 1];
const _parentId = component?.parent?.split('-').slice(0, -1).join('-');
const mappedParentId = componentsMapping[_parentId];
parentId = `${mappedParentId}-${childTabId}`;
} else {
parentId = componentsMapping[parentId];
}
transformedComponent.id = uuid();
transformedComponent.name = componentData.name;
transformedComponent.type = componentData.component;
transformedComponent.properties = componentData.definition.properties || {};
@ -1511,7 +1537,7 @@ function transformComponentData(data: object, componentEvents: any[]): Component
transformedComponent.general = componentData.definition.general || {};
transformedComponent.generalStyles = componentData.definition.generalStyles || {};
transformedComponent.displayPreferences = componentData.definition.others || {};
transformedComponent.parent = data[componentId].parent || null;
transformedComponent.parent = component.parent ? parentId : null;
transformedComponents.push(transformedComponent);
@ -1519,7 +1545,22 @@ function transformComponentData(data: object, componentEvents: any[]): Component
componentId: componentId,
event: componentData.definition.events,
});
componentsMapping[componentId] = transformedComponent.id;
}
return transformedComponents;
}
const isChildOfTabsOrCalendar = (component, allComponents = [], componentParentId = undefined) => {
if (componentParentId) {
const parentId = component?.parent?.split('-').slice(0, -1).join('-');
const parentComponent = allComponents.find((comp) => comp.id === parentId);
if (parentComponent) {
return parentComponent.component.component === 'Tabs' || parentComponent.component.component === 'Calendar';
}
}
return false;
};