From 09e0a94de1d1d34c0a6c6be0160338523aabea78 Mon Sep 17 00:00:00 2001 From: parthy007 Date: Fri, 9 May 2025 17:04:49 +0530 Subject: [PATCH] Add in-mem-cache setup to reuse auth code --- server/src/helpers/in_memory_cache.service.ts | 22 +++++ server/src/modules/data-sources/module.ts | 2 + .../src/modules/data-sources/util.service.ts | 97 +++++++++++-------- 3 files changed, 81 insertions(+), 40 deletions(-) create mode 100644 server/src/helpers/in_memory_cache.service.ts diff --git a/server/src/helpers/in_memory_cache.service.ts b/server/src/helpers/in_memory_cache.service.ts new file mode 100644 index 0000000000..b1cfde0184 --- /dev/null +++ b/server/src/helpers/in_memory_cache.service.ts @@ -0,0 +1,22 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class InMemoryCacheService { + private static cacheStore: Map> = new Map(); + + set(key: string, value: Promise): void { + InMemoryCacheService.cacheStore.set(key, value); + } + + get(key: string): Promise | undefined { + return InMemoryCacheService.cacheStore.get(key); + } + + has(key: string): boolean { + return InMemoryCacheService.cacheStore.has(key); + } + + clear(): void { + InMemoryCacheService.cacheStore.clear(); + } +} diff --git a/server/src/modules/data-sources/module.ts b/server/src/modules/data-sources/module.ts index f914bbcd2e..8c1692b3ef 100644 --- a/server/src/modules/data-sources/module.ts +++ b/server/src/modules/data-sources/module.ts @@ -12,6 +12,7 @@ import { AppsRepository } from '@modules/apps/repository'; import { TooljetDbModule } from '@modules/tooljet-db/module'; import { SessionModule } from '@modules/session/module'; import { SampleDBScheduler } from './schedulers/sample-db.scheduler'; +import { InMemoryCacheService } from '@helpers/in_memory_cache.service'; export class DataSourcesModule { static async register(configs?: { IS_GET_CONTEXT: boolean }): Promise { @@ -43,6 +44,7 @@ export class DataSourcesModule { SampleDataSourceService, FeatureAbilityFactory, SampleDBScheduler, + InMemoryCacheService, ], controllers: [DataSourcesController], exports: [DataSourcesUtilService, SampleDataSourceService, PluginsServiceSelector], diff --git a/server/src/modules/data-sources/util.service.ts b/server/src/modules/data-sources/util.service.ts index 03ad5ddd52..b66b97fe29 100644 --- a/server/src/modules/data-sources/util.service.ts +++ b/server/src/modules/data-sources/util.service.ts @@ -21,6 +21,7 @@ import { PluginsServiceSelector } from './services/plugin-selector.service'; import { OrganizationConstantsUtilService } from '@modules/organization-constants/util.service'; import { DataSourceOptions } from '@entities/data_source_options.entity'; import { IDataSourcesUtilService } from './interfaces/IUtilService'; +import { InMemoryCacheService } from '@helpers/in_memory_cache.service'; @Injectable() export class DataSourcesUtilService implements IDataSourcesUtilService { @@ -31,7 +32,8 @@ export class DataSourcesUtilService implements IDataSourcesUtilService { protected readonly licenseTermsService: LicenseTermsService, protected readonly encryptionService: EncryptionService, protected readonly pluginsServiceSelector: PluginsServiceSelector, - protected readonly organizationConstantsUtilService: OrganizationConstantsUtilService + protected readonly organizationConstantsUtilService: OrganizationConstantsUtilService, + protected readonly inMemoryCacheService: InMemoryCacheService ) {} async create(createArgumentsDto: CreateArgumentsDto, user: User): Promise { return await dbTransactionWrap(async (manager: EntityManager) => { @@ -135,7 +137,17 @@ export class DataSourcesUtilService implements IDataSourcesUtilService { const queryService = await this.pluginsServiceSelector.getService(plugin_id, provider); // const queryService = new allPlugins[provider](); - const accessDetails = await queryService.accessDetailsFrom(authCode, options, resetSecureData); + let accessDetailsPromise: Promise; + + const cacheKey = `${provider}_${authCode}`; + + if (this.inMemoryCacheService.has(cacheKey)) { + accessDetailsPromise = this.inMemoryCacheService.get(cacheKey); + } else { + accessDetailsPromise = queryService.accessDetailsFrom(authCode, options, resetSecureData); + this.inMemoryCacheService.set(cacheKey, accessDetailsPromise); + } + const accessDetails = await accessDetailsPromise; for (const row of accessDetails) { const option = {}; @@ -165,52 +177,56 @@ export class DataSourcesUtilService implements IDataSourcesUtilService { throw new BadRequestException('Cannot update configuration of sample data source'); } - await dbTransactionWrap(async (manager: EntityManager) => { - const isMultiEnvEnabled = await this.licenseTermsService.getLicenseTerms(LICENSE_FIELD.MULTI_ENVIRONMENT); - const envToUpdate = await this.appEnvironmentUtilService.get(organizationId, environmentId, false, manager); + try { + await dbTransactionWrap(async (manager: EntityManager) => { + const isMultiEnvEnabled = await this.licenseTermsService.getLicenseTerms(LICENSE_FIELD.MULTI_ENVIRONMENT); + const envToUpdate = await this.appEnvironmentUtilService.get(organizationId, environmentId, false, manager); - // if datasource is restapi then reset the token data - if (dataSource.kind === 'restapi') - options.push({ - key: 'tokenData', - value: undefined, - encrypted: false, - }); + // if datasource is restapi then reset the token data + if (dataSource.kind === 'restapi') + options.push({ + key: 'tokenData', + value: undefined, + encrypted: false, + }); - if (isMultiEnvEnabled) { - dataSource.options = ( - await this.appEnvironmentUtilService.getOptions(dataSourceId, organizationId, envToUpdate.id) - ).options; - - const newOptions = await this.parseOptionsForUpdate(dataSource, options, manager); - await this.appEnvironmentUtilService.updateOptions(newOptions, envToUpdate.id, dataSource.id, manager); - } else { - const allEnvs = await this.appEnvironmentUtilService.getAll(organizationId); - /* - Basic plan customer. lets update all environment options. - this will help us to run the queries successfully when the user buys enterprise plan - */ - - const newOptions = await this.parseOptionsForUpdate(dataSource, options, manager); - for (const env of allEnvs) { + if (isMultiEnvEnabled) { dataSource.options = ( - await this.appEnvironmentUtilService.getOptions(dataSourceId, organizationId, env.id) + await this.appEnvironmentUtilService.getOptions(dataSourceId, organizationId, envToUpdate.id) ).options; - await this.appEnvironmentUtilService.updateOptions(newOptions, env.id, dataSource.id, manager); + const newOptions = await this.parseOptionsForUpdate(dataSource, options, manager); + await this.appEnvironmentUtilService.updateOptions(newOptions, envToUpdate.id, dataSource.id, manager); + } else { + const allEnvs = await this.appEnvironmentUtilService.getAll(organizationId); + /* + Basic plan customer. lets update all environment options. + this will help us to run the queries successfully when the user buys enterprise plan + */ + + for (const env of allEnvs) { + dataSource.options = ( + await this.appEnvironmentUtilService.getOptions(dataSourceId, organizationId, env.id) + ).options; + const newOptions = await this.parseOptionsForUpdate(dataSource, options, manager); + + await this.appEnvironmentUtilService.updateOptions(newOptions, env.id, dataSource.id, manager); + } } - } - const updatableParams = { - id: dataSourceId, - name, - updatedAt: new Date(), - }; + const updatableParams = { + id: dataSourceId, + name, + updatedAt: new Date(), + }; - // Remove keys with undefined values - cleanObject(updatableParams); + // Remove keys with undefined values + cleanObject(updatableParams); - await manager.save(DataSource, updatableParams); - }); + await manager.save(DataSource, updatableParams); + }); + } finally { + this.inMemoryCacheService.clear(); + } } async decrypt(options: Record) { @@ -612,6 +628,7 @@ export class DataSourcesUtilService implements IDataSourcesUtilService { for (const key of Object.keys(options)) { const currentOption = options[key]?.['value']; + constantMatcher.lastIndex = 0; //! request options are nested arrays with constants and variables if (Array.isArray(currentOption)) {