From 8d4b8aa56a4aff8aa381078744ec3a6049f19529 Mon Sep 17 00:00:00 2001 From: Kartik Gupta Date: Fri, 15 Nov 2024 11:46:34 +0530 Subject: [PATCH 1/3] form button to submit mapping issue on version creation and import export --- server/package.json | 2 + server/src/helpers/import_export.helpers.ts | 474 +++++++++++++++--- .../src/services/app_import_export.service.ts | 18 +- server/src/services/apps.service.ts | 22 +- server/src/services/page.service.ts | 10 +- 5 files changed, 408 insertions(+), 118 deletions(-) diff --git a/server/package.json b/server/package.json index 745a0e9610..4f66f7de3d 100644 --- a/server/package.json +++ b/server/package.json @@ -53,6 +53,8 @@ "@sentry/tracing": "6.17.6", "@tooljet/plugins": "../plugins", "@types/express-serve-static-core": "^4.19.5", + "acorn": "^8.13.0", + "acorn-walk": "^8.3.4", "bcrypt": "^5.0.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", diff --git a/server/src/helpers/import_export.helpers.ts b/server/src/helpers/import_export.helpers.ts index 2c98071159..b7aa926d34 100644 --- a/server/src/helpers/import_export.helpers.ts +++ b/server/src/helpers/import_export.helpers.ts @@ -1,39 +1,40 @@ +import * as acorn from 'acorn'; +import * as walk from 'acorn-walk'; + +function findExpression(input) { + const matches = []; + let startIdx = -1; + let braceCount = 0; + + for (let i = 0; i < input.length; i++) { + if (input[i] === '{' && input[i + 1] === '{' && braceCount === 0) { + startIdx = i; + braceCount = 2; + i++; // Skip the second '{' + } else if (input[i] === '{' && braceCount > 0) { + braceCount++; + } else if (input[i] === '}' && braceCount > 0) { + braceCount--; + if (braceCount === 0 && startIdx !== -1) { + matches.push({ + fullMatch: input.slice(startIdx, i + 1), + expression: input.slice(startIdx + 2, i - 1).trim(), + index: startIdx, + }); + startIdx = -1; + } + } + } + + return matches; +} + export function updateEntityReferences(node, resourceMapping: Record = {}) { if (typeof node === 'object') { for (const key in node) { let value = node[key]; if (typeof value === 'string' && value.includes('{{') && value.includes('}}')) { - const referenceExists = value; - - if (referenceExists) { - const matches = value.match(/{{(.*?)}}/g); - // gett all references {{entityName}} - if (matches) { - matches.forEach((match) => { - // remove curly braces and extract the entity "component.entityName.something" - const ref = match.slice(2, -2).trim(); - const entityName = ref.split('.')[1]; - - if (resourceMapping[entityName]) { - const newValue = value.replace(entityName, resourceMapping[entityName]); - - node[key] = newValue; - value = newValue; - } - }); - } else { - // kept this logic for fallback, although it should not be needed - const ref = value.replace('{{', '').replace('}}', ''); - - const entityName = ref.split('.')[1]; - - if (resourceMapping[entityName]) { - const newValue = value.replace(entityName, resourceMapping[entityName]); - - node[key] = newValue; - } - } - } + node[key] = extractAndReplaceReferencesFromString(value, resourceMapping, resourceMapping)?.valueWithId; } else if (typeof value === 'object') { value = updateEntityReferences(value, resourceMapping); } @@ -43,53 +44,372 @@ export function updateEntityReferences(node, resourceMapping: Record { - const ref = match.slice(2, -2).trim(); // Remove {{ and }} - const entityName = ref.split('.')[1]; - if (entityName && !allRefs.includes(entityName)) { - allRefs.push(entityName); - } - }); - } else { - // kept this logic for fallback, although it should not be needed - const ref = value.replace('{{', '').replace('}}', ''); - - const entityName = ref.split('.')[1]; - - allRefs.push(entityName); - } - } - } else if (typeof value === 'object') { - findAllEntityReferences(value, allRefs); - } - } - } - return allRefs; -} - export function isValidUUID(uuid) { const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; return uuidRegex.test(uuid); } + +export function extractAndReplaceReferencesFromString(input, componentIdNameMapping = {}, queryIdNameMapping = {}) { + // Quick check for relevant keywords + const regexForQuickCheck = + /\b(components|queries|globals|variables|page|parameters|secrets|constants)(?:\[\S*|\.\S*|\?\.\S*)/; + if (!regexForQuickCheck.test(input)) { + return { + allRefs: [], + valueWithId: input, + valueWithBrackets: input, + }; + } + + const relevantKeywords = /\b(components|queries|globals|variables|page|parameters|secrets|constants)\b/; + const expressionRegex = /{{(.*?)}}/gs; + const results = []; + let lastIndex = 0; + let replacedString = ''; + let bracketNotationString = ''; + + // Precompile the UUID regex + const uuidRegex = + /\b(components|queries)(\??\.|\??\.?\[['"]?)([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(['"]?\])?/g; + + let match; + if (input.startsWith('{{{') && input.endsWith('}}}')) { + input = input.replace(/\{\{(.*)\}\}/, '{{($1)}}'); + const matches = findExpression(input); + for (const match of matches) { + const { fullMatch, expression, index } = match; + + // Check if the expression contains relevant keywords + if (!relevantKeywords.test(expression)) { + replacedString += input.slice(lastIndex, index); + bracketNotationString += input.slice(lastIndex, index); + replacedString += fullMatch; + bracketNotationString += fullMatch; + lastIndex = index + fullMatch.length; + continue; + } + + try { + const { processedExpression, uuidMappings } = preprocessExpression( + expression, + uuidRegex, + componentIdNameMapping, + queryIdNameMapping + ); + const parsedResult = parseExpression( + processedExpression, + componentIdNameMapping, + queryIdNameMapping, + uuidMappings + ); + + replacedString += input.slice(lastIndex, index); + bracketNotationString += input.slice(lastIndex, index); + + const replacedExpression = replaceIdsInExpression( + processedExpression, + componentIdNameMapping, + queryIdNameMapping, + false, + uuidMappings + ); + const bracketNotationExpression = replaceIdsInExpression( + processedExpression, + componentIdNameMapping, + queryIdNameMapping, + true, + uuidMappings + ); + + replacedString += `{{${replacedExpression}}}`; + bracketNotationString += `{{${bracketNotationExpression}}}`; + + results.push({ + allRefs: parsedResult.references, + valueWithId: `{{${replacedExpression}}}`, + valueWithBrackets: `{{${bracketNotationExpression}}}`, + }); + } catch (error) { + replacedString += fullMatch; + bracketNotationString += fullMatch; + results.push({ + allRefs: [], + valueWithId: fullMatch, + valueWithBrackets: fullMatch, + }); + } + + lastIndex = index + fullMatch.length; + } + + replacedString += input.slice(lastIndex); + bracketNotationString += input.slice(lastIndex); + // remove the parentheses that were added + + return { + valueWithId: `{{${replacedString.slice(3, -3)}}}`, + valueWithBrackets: `{{${bracketNotationString.slice(3, -3)}}}`, + allRefs: results.flatMap((r) => r.allRefs), + }; + } + while ((match = expressionRegex.exec(input)) !== null) { + const fullMatch = match[0]; + const expression = match[1].trim(); + + // Check if the expression contains relevant keywords + if (!relevantKeywords.test(expression)) { + replacedString += input.slice(lastIndex, match.index); + bracketNotationString += input.slice(lastIndex, match.index); + replacedString += fullMatch; + bracketNotationString += fullMatch; + lastIndex = match.index + fullMatch.length; + continue; + } + + try { + const { processedExpression, uuidMappings } = preprocessExpression( + expression, + uuidRegex, + componentIdNameMapping, + queryIdNameMapping + ); + const parsedResult = parseExpression( + processedExpression, + componentIdNameMapping, + queryIdNameMapping, + uuidMappings + ); + + replacedString += input.slice(lastIndex, match.index); + bracketNotationString += input.slice(lastIndex, match.index); + + const replacedExpression = replaceIdsInExpression( + processedExpression, + componentIdNameMapping, + queryIdNameMapping, + false, + uuidMappings + ); + const bracketNotationExpression = replaceIdsInExpression( + processedExpression, + componentIdNameMapping, + queryIdNameMapping, + true, + uuidMappings + ); + + replacedString += `{{${replacedExpression}}}`; + bracketNotationString += `{{${bracketNotationExpression}}}`; + + results.push({ + allRefs: parsedResult.references, + valueWithId: `{{${replacedExpression}}}`, + valueWithBrackets: `{{${bracketNotationExpression}}}`, + }); + } catch (error) { + replacedString += fullMatch; + bracketNotationString += fullMatch; + results.push({ + allRefs: [], + valueWithId: fullMatch, + valueWithBrackets: fullMatch, + }); + } + + lastIndex = match.index + fullMatch.length; + } + + replacedString += input.slice(lastIndex); + bracketNotationString += input.slice(lastIndex); + + return { + allRefs: results.flatMap((r) => r.allRefs), + valueWithId: replacedString, + valueWithBrackets: bracketNotationString, + }; +} + +function preprocessExpression(expression, uuidRegex, componentIdNameMapping, queryIdNameMapping) { + const uuidMappings = {}; + let placeholderCounter = 0; + + const processedExpression = expression.replace(uuidRegex, (match, p1, p2, p3, p4) => { + const placeholder = `__UUID_PLACEHOLDER_${placeholderCounter}__`; + uuidMappings[placeholder] = (p1 === 'components' ? componentIdNameMapping[p3] : queryIdNameMapping[p3]) || p3; + placeholderCounter++; + return `${p1}${p2}${placeholder}${p4 || ''}`; + }); + + return { processedExpression, uuidMappings }; +} + +function replaceIdsInExpression( + expression, + componentIdNameMapping, + queryIdNameMapping, + useBracketNotation, + uuidMappings +) { + try { + const ast = acorn.parse(expression, { ecmaVersion: 2020 }); + const replacements = []; + + walk.simple(ast, { + MemberExpression(node) { + if ( + node.object.type === 'Identifier' && + (node.object.name === 'components' || node.object.name === 'queries') + ) { + const isComponent = node.object.name === 'components'; + const mapping = isComponent ? componentIdNameMapping : queryIdNameMapping; + + if (node.property.type === 'Identifier') { + const name = node.property.name; + const nameWithOptionalCheck = node.optional + ? useBracketNotation + ? `${node.object.name}?.` + : `${node.object.name}?` + : `${node.object.name}`; + if (mapping[name] || name.startsWith('__UUID_PLACEHOLDER_')) { + const start = node.start; + const end = node.end; + let replacement; + if (name.startsWith('__UUID_PLACEHOLDER_')) { + const actualName = uuidMappings[name]; + replacement = useBracketNotation + ? `${nameWithOptionalCheck}["${actualName}"]` + : `${nameWithOptionalCheck}.${actualName}`; + } else { + replacement = useBracketNotation + ? `${nameWithOptionalCheck}["${mapping[name]}"]` + : `${nameWithOptionalCheck}.${mapping[name]}`; + } + replacements.push({ start, end, replacement }); + } + } else if (node.property.type === 'Literal') { + const name = node.property.value as string; + const nameWithOptionalCheck = node.optional ? `${node.object.name}?.` : `${node.object.name}`; + if (mapping[name] || name.startsWith('__UUID_PLACEHOLDER_')) { + const start = node.start; + const end = node.end; + let replacement; + if (name.startsWith('__UUID_PLACEHOLDER_')) { + const actualName = uuidMappings[name]; + replacement = `${nameWithOptionalCheck}["${actualName}"]`; + } else { + replacement = `${nameWithOptionalCheck}["${mapping[name]}"]`; + } + replacements.push({ start, end, replacement }); + } + } + } + }, + }); + + if (replacements.length === 0) return expression; + + replacements.sort((a, b) => b.start - a.start); + + let result = expression; + for (const { start, end, replacement } of replacements) { + result = result.slice(0, start) + replacement + result.slice(end); + } + + return result; + } catch (error) { + return expression; + } +} + +function parseExpression(expression, componentIdNameMapping, queryIdNameMapping, uuidMappings) { + try { + const ast = acorn.parse(expression, { ecmaVersion: 2020 }); + const references = []; + const validRootObjects = { + components: true, + queries: true, + variables: true, + globals: true, + page: true, + }; + walk.simple(ast, { + MemberExpression: handleMemberExpression, + }); + + // eslint-disable-next-line no-inner-declarations + function handleMemberExpression(node) { + const reference = extractPath(node); + if (reference) references.push(reference); + } + + // eslint-disable-next-line no-inner-declarations + function extractPath(node) { + const path = []; + let current = node; + let rootObject = ''; + + while (current) { + if (current.type === 'Identifier') { + path.unshift(current.name); + if (validRootObjects[current.name]) { + rootObject = current.name; + break; + } + } else if (current.type === 'MemberExpression' || current.type === 'OptionalMemberExpression') { + if (current.computed) { + if ( + current.property.type === 'Literal' && + (typeof current.property.value === 'string' || typeof current.property.value === 'number') + ) { + path.unshift(current.property.value.toString()); + } else { + break; + } + } else { + path.unshift(current.property.name); + } + } else { + break; + } + current = current.object; + } + + if ( + (rootObject && (rootObject === 'queries' || rootObject === 'components') && path.length >= 3) || + ((rootObject === 'variables' || rootObject === 'globals') && path.length === 2) || + (rootObject === 'page' && path.length === 3) + ) { + return createReferenceObject(rootObject, path, uuidMappings, componentIdNameMapping, queryIdNameMapping); + } + return null; + } + + return { references }; + } catch (error) { + console.log(error); + return { references: [] }; + } +} + +function createReferenceObject(entityType, path, uuidMappings, componentIdNameMapping, queryIdNameMapping) { + let entityNameOrId, entityKey; + + if (entityType === 'components' || entityType === 'queries') { + entityNameOrId = path[1]; + entityKey = path[2]; + + if (entityNameOrId.startsWith('__UUID_PLACEHOLDER_')) { + entityNameOrId = uuidMappings[entityNameOrId]; + } else { + const mapping = entityType === 'components' ? componentIdNameMapping : queryIdNameMapping; + entityNameOrId = mapping[entityNameOrId] || entityNameOrId; + } + } else if (entityType === 'variables' || entityType === 'globals') { + entityKey = path[1]; + } else if (entityType === 'page') { + entityNameOrId = path[1]; + entityKey = path[2]; + } + + return { entityType, entityNameOrId, entityKey }; +} diff --git a/server/src/services/app_import_export.service.ts b/server/src/services/app_import_export.service.ts index 0cf6c7f83b..e799047435 100644 --- a/server/src/services/app_import_export.service.ts +++ b/server/src/services/app_import_export.service.ts @@ -30,7 +30,7 @@ import { Component } from 'src/entities/component.entity'; import { Layout } from 'src/entities/layout.entity'; import { EventHandler, Target } from 'src/entities/event_handler.entity'; import { v4 as uuid } from 'uuid'; -import { findAllEntityReferences, isValidUUID, updateEntityReferences } from 'src/helpers/import_export.helpers'; +import { updateEntityReferences } from 'src/helpers/import_export.helpers'; interface AppResourceMappings { defaultDataSourceIdMapping: Record; dataQueryMapping: Record; @@ -290,13 +290,7 @@ export class AppImportExportService { .getMany(); const toUpdateComponents = components.filter((component) => { - const entityReferencesInComponentDefinitions = findAllEntityReferences(component, []).filter( - (entity) => entity && isValidUUID(entity) - ); - - if (entityReferencesInComponentDefinitions.length > 0) { - return updateEntityReferences(component, mappings); - } + return updateEntityReferences(component, mappings); }); if (!isEmpty(toUpdateComponents)) { @@ -312,13 +306,7 @@ export class AppImportExportService { .getMany(); const toUpdateDataQueries = dataQueries.filter((dataQuery) => { - const entityReferencesInQueryOptions = findAllEntityReferences(dataQuery, []).filter( - (entity) => entity && isValidUUID(entity) - ); - - if (entityReferencesInQueryOptions.length > 0) { - return updateEntityReferences(dataQuery, mappings); - } + return updateEntityReferences(dataQuery, mappings); }); if (!isEmpty(toUpdateDataQueries)) { diff --git a/server/src/services/apps.service.ts b/server/src/services/apps.service.ts index 65cd02c410..ca10a6ff50 100644 --- a/server/src/services/apps.service.ts +++ b/server/src/services/apps.service.ts @@ -27,7 +27,7 @@ import { Component } from 'src/entities/component.entity'; import { EventHandler, Target } from 'src/entities/event_handler.entity'; import { VersionReleaseDto } from '@dto/version-release.dto'; -import { findAllEntityReferences, isValidUUID, updateEntityReferences } from 'src/helpers/import_export.helpers'; +import { updateEntityReferences } from 'src/helpers/import_export.helpers'; import { isEmpty, set } from 'lodash'; import { AppBase } from 'src/entities/app_base.entity'; import { LayoutDimensionUnits } from 'src/helpers/components.helper'; @@ -435,13 +435,7 @@ export class AppsService { .getMany(); const toUpdateComponents = components.filter((component) => { - const entityReferencesInComponentDefinitions = findAllEntityReferences(component, []).filter( - (entity) => entity && isValidUUID(entity) - ); - - if (entityReferencesInComponentDefinitions.length > 0) { - return updateEntityReferences(component, mappings); - } + return updateEntityReferences(component, mappings); }); if (!isEmpty(toUpdateComponents)) { @@ -457,13 +451,7 @@ export class AppsService { .getMany(); const toUpdateDataQueries = dataQueries.filter((dataQuery) => { - const entityReferencesInQueryOptions = findAllEntityReferences(dataQuery, []).filter( - (entity) => entity && isValidUUID(entity) - ); - - if (entityReferencesInQueryOptions.length > 0) { - return updateEntityReferences(dataQuery, mappings); - } + return updateEntityReferences(dataQuery, mappings); }); if (!isEmpty(toUpdateDataQueries)) { @@ -532,7 +520,7 @@ export class AppsService { let homePageId = prevHomePagePage; - let newComponents = []; + const newComponents = []; const newComponentLayouts = []; let oldComponentToNewComponentMapping = {}; const oldPageToNewPageMapping = {}; @@ -693,8 +681,6 @@ export class AppsService { await manager.save(newComponents); await manager.save(newComponentLayouts); - newComponents = []; - oldComponentToNewComponentMapping = {}; } await manager.update(AppVersion, { id: appVersion.id }, { homePageId }); diff --git a/server/src/services/page.service.ts b/server/src/services/page.service.ts index e67420c4f2..84a0db5ed5 100644 --- a/server/src/services/page.service.ts +++ b/server/src/services/page.service.ts @@ -11,7 +11,7 @@ import { EventsService } from './events_handler.service'; import { Component } from 'src/entities/component.entity'; import { Layout } from 'src/entities/layout.entity'; import { EventHandler } from 'src/entities/event_handler.entity'; -import { findAllEntityReferences, isValidUUID, updateEntityReferences } from 'src/helpers/import_export.helpers'; +import { updateEntityReferences } from 'src/helpers/import_export.helpers'; import { isEmpty } from 'class-validator'; import { PageHelperService } from '@apps/services/pages/service.helper'; import * as _ from 'lodash'; @@ -243,13 +243,7 @@ export class PageService { } const toUpdateComponents = clonedComponents.filter((component) => { - const entityReferencesInComponentDefinitions = findAllEntityReferences(component, []).filter( - (entity) => entity && isValidUUID(entity) - ); - - if (entityReferencesInComponentDefinitions.length > 0) { - return updateEntityReferences(component, componentsIdMap); - } + return updateEntityReferences(component, componentsIdMap); }); if (!isEmpty(toUpdateComponents)) { From bb928f2031a85649dc245b7e4f7e5738ab3523fa Mon Sep 17 00:00:00 2001 From: Kartik Gupta Date: Mon, 18 Nov 2024 12:39:36 +0530 Subject: [PATCH 2/3] fix bug breaking hash map --- server/src/services/apps.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/services/apps.service.ts b/server/src/services/apps.service.ts index ca10a6ff50..cd683968b8 100644 --- a/server/src/services/apps.service.ts +++ b/server/src/services/apps.service.ts @@ -522,7 +522,7 @@ export class AppsService { const newComponents = []; const newComponentLayouts = []; - let oldComponentToNewComponentMapping = {}; + const oldComponentToNewComponentMapping = {}; const oldPageToNewPageMapping = {}; const isChildOfTabsOrCalendar = (component, allComponents = [], componentParentId = undefined) => { From 022e475b6d8d7a402dcc9df32719950e812725f0 Mon Sep 17 00:00:00 2001 From: Kartik Gupta Date: Mon, 18 Nov 2024 12:48:52 +0530 Subject: [PATCH 3/3] use string manipulation instead of regex --- frontend/src/AppBuilder/_stores/ast.js | 3 ++- server/src/helpers/import_export.helpers.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/AppBuilder/_stores/ast.js b/frontend/src/AppBuilder/_stores/ast.js index f735228974..106298588f 100644 --- a/frontend/src/AppBuilder/_stores/ast.js +++ b/frontend/src/AppBuilder/_stores/ast.js @@ -54,7 +54,8 @@ export function extractAndReplaceReferencesFromString(input, componentIdNameMapp let match; if (input.startsWith('{{{') && input.endsWith('}}}')) { - input = input.replace(/\{\{(.*)\}\}/, '{{($1)}}'); + const inputContent = input.slice(3, -3); + input = `{{({${inputContent}})}}`; const matches = findExpression(input); for (const match of matches) { const { fullMatch, expression, index } = match; diff --git a/server/src/helpers/import_export.helpers.ts b/server/src/helpers/import_export.helpers.ts index b7aa926d34..1f1bc57b7e 100644 --- a/server/src/helpers/import_export.helpers.ts +++ b/server/src/helpers/import_export.helpers.ts @@ -74,7 +74,8 @@ export function extractAndReplaceReferencesFromString(input, componentIdNameMapp let match; if (input.startsWith('{{{') && input.endsWith('}}}')) { - input = input.replace(/\{\{(.*)\}\}/, '{{($1)}}'); + const inputContent = input.slice(3, -3); + input = `{{({${inputContent}})}}`; const matches = findExpression(input); for (const match of matches) { const { fullMatch, expression, index } = match;