diff --git a/frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx b/frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx index f25e72cb59..6175e84005 100644 --- a/frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx +++ b/frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx @@ -17,6 +17,8 @@ import { useNavigate } from 'react-router-dom'; import { deepClone } from '@/_helpers/utilities/utils.helpers'; import { BulkUploadPrimaryKey } from './BulkUploadPrimaryKey'; import BulkUpsertPrimaryKey from './BulkUpsertPrimaryKey'; +import { fetchEdition } from '@/modules/common/helpers/utils'; +import config from 'config'; import './styles.scss'; import CodeHinter from '@/AppBuilder/CodeEditor'; @@ -49,6 +51,21 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay const [bulkUpdatePrimaryKey, setBulkUpdatePrimaryKey] = useState(() => options['bulk_update_with_primary_key'] || {}); const [bulkUpsertPrimaryKey, setBulkUpsertPrimaryKey] = useState(() => options['bulk_upsert_with_primary_key'] || {}); + // Check if SQL mode should be disabled + const isSqlModeDisabled = () => { + // Check legacy environment variable for backward compatibility + if (window.public_config?.TJDB_SQL_MODE_DISABLE === 'true') { + return true; + } + + const edition = fetchEdition(config); + if (edition === 'cloud') { + return true; + } + + return false; + }; + const joinOptions = options['join_table']?.['joins'] || [ { conditions: { conditionsList: [{ leftField: { table: selectedTableId } }] } }, ]; @@ -557,7 +574,7 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay {/* table name dropdown */} - {window.public_config?.TJDB_SQL_MODE_DISABLE !== 'true' && ( + {!isSqlModeDisabled() && (
diff --git a/server/data-migrations/1721236971725-MoveToolJetDatabaseTablesFromPublicToTenantSchema.ts b/server/data-migrations/1721236971725-MoveToolJetDatabaseTablesFromPublicToTenantSchema.ts index bc5be2b2ec..8f0b5a98ae 100644 --- a/server/data-migrations/1721236971725-MoveToolJetDatabaseTablesFromPublicToTenantSchema.ts +++ b/server/data-migrations/1721236971725-MoveToolJetDatabaseTablesFromPublicToTenantSchema.ts @@ -5,6 +5,7 @@ import { InternalTable } from '@entities/internal_table.entity'; import { MigrationProgress, processDataInBatches } from '@helpers/migration.helper'; import { getEnvVars } from 'scripts/database-config-utils'; import { EncryptionService } from '@modules/encryption/service'; +import { TOOLJET_EDITIONS } from '@modules/app/constants'; import { createNewTjdbRole, createAndGrantSchemaPrivilege, @@ -20,6 +21,12 @@ const crypto = require('crypto'); export class MoveToolJetDatabaseTablesFromPublicToTenantSchema1721236971725 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { const envData = getEnvVars(); + const isSqlModeDisabled = envData.TJDB_SQL_MODE_DISABLE == 'true'; + const isCloud = envData.TOOLJET_EDITION == TOOLJET_EDITIONS.Cloud; + if (isSqlModeDisabled || isCloud) { + console.log('Skipping TJDB schema migration for SQL mode'); + return; + } const batchSize = 100; const entityManager = queryRunner.manager; const tooljetDbConnection = new DataSource({ diff --git a/server/src/helpers/tooljet_db.helper.ts b/server/src/helpers/tooljet_db.helper.ts index 67c9d8f3af..c53f0d5d0a 100644 --- a/server/src/helpers/tooljet_db.helper.ts +++ b/server/src/helpers/tooljet_db.helper.ts @@ -3,6 +3,8 @@ import { tooljetDbOrmconfig } from 'ormconfig'; import { OrganizationTjdbConfigurations } from 'src/entities/organization_tjdb_configurations.entity'; import { EntityManager, DataSource } from 'typeorm'; import { v4 as uuidv4 } from 'uuid'; +import { getTooljetEdition } from '@helpers/utils.helper'; +import { TOOLJET_EDITIONS } from '@modules/app/constants'; /** * Creates a custom tooljet database connection using a tenant user, for the respective workspace. @@ -40,6 +42,10 @@ export async function createTooljetDatabaseConnection( } export async function decryptTooljetDatabasePassword(password: string) { + if (isSQLModeDisabled()) { + return process.env.TOOLJET_DB_PASS; + } + const encryptionService = new EncryptionService(); const decryptedvalue = await encryptionService.decryptColumnValue( 'organization_tjdb_configurations', @@ -60,9 +66,20 @@ export async function encryptTooljetDatabasePassword(password: string) { } export function findTenantSchema(organisationId: string): string { + if (isSQLModeDisabled()) { + return 'public'; + } + return `workspace_${organisationId}`; } +// TODO: Cloud TJDB SQL mode is disabled: Use public schema for cloud edition +// This is because Postgrest doesn't handle loading large amount of schemas in memory +// We need to migrate to use Table based access control instead of schema based access control +export function isSQLModeDisabled(): boolean { + return process.env.TJDB_SQL_MODE_DISABLE === 'true' || getTooljetEdition() === TOOLJET_EDITIONS.Cloud; +} + export function concatSchemaAndTableName(schema: string, tableName: string) { return `${schema}` + '.' + `${tableName}`; } @@ -244,7 +261,7 @@ export function validateTjdbJSONBColumnInputs(jsonbColumnList: Array, in } else { inValidValueColumnsList.push(key); } - } catch (error) { + } catch { inValidValueColumnsList.push(key); } } diff --git a/server/src/modules/tooljet-db/helper.ts b/server/src/modules/tooljet-db/helper.ts index 5dab99a0d2..02642892b6 100644 --- a/server/src/modules/tooljet-db/helper.ts +++ b/server/src/modules/tooljet-db/helper.ts @@ -27,3 +27,31 @@ export async function reconfigurePostgrest( throw error; } } + +/** + * Cloud TJDB SQL Disabled: Postgrest configuration without schema synchronization + * Postgres schema for each workspace is not loaded into Postgrest. + */ +export async function reconfigurePostgrestWithoutSchemaSync( + tooljetDbManager: EntityManager, + options: { user: string; enableAggregates: boolean; statementTimeoutInSecs: number } +) { + try { + await tooljetDbManager.transaction(async (transactionalEntityManager) => { + await transactionalEntityManager.queryRunner.query('CREATE SCHEMA IF NOT EXISTS postgrest'); + await transactionalEntityManager.queryRunner.query(`GRANT USAGE ON SCHEMA postgrest to ${options.user}`); + await transactionalEntityManager.queryRunner.query(`create or replace function postgrest.pre_config() + returns void as $$ + select + set_config('pgrst.db_aggregates_enabled', '${options.enableAggregates}', false); + $$ language sql; + `); + await transactionalEntityManager.queryRunner.query( + `ALTER ROLE ${options.user} SET statement_timeout TO '${options.statementTimeoutInSecs}s'` + ); + }); + } catch (error) { + console.error('The tooljet database reconfiguration process encountered an error.', error); + throw error; + } +} diff --git a/server/src/modules/tooljet-db/module.ts b/server/src/modules/tooljet-db/module.ts index f0ccb9f602..a28c62f6dd 100644 --- a/server/src/modules/tooljet-db/module.ts +++ b/server/src/modules/tooljet-db/module.ts @@ -6,12 +6,15 @@ import { Logger } from 'nestjs-pino'; import { Credential } from '../../../src/entities/credential.entity'; import { InternalTable } from 'src/entities/internal_table.entity'; import { AppUser } from 'src/entities/app_user.entity'; -import { reconfigurePostgrest } from './helper'; +import { reconfigurePostgrest, reconfigurePostgrestWithoutSchemaSync } from './helper'; +import { getTooljetEdition } from '@helpers/utils.helper'; +import { TOOLJET_EDITIONS } from '@modules/app/constants'; import { TableCountGuard } from '@modules/licensing/guards/table.guard'; import { AbilityUtilService } from '@modules/ability/util.service'; import { RolesRepository } from '@modules/roles/repository'; import { FeatureAbilityFactory } from './ability'; import { SubModule } from '@modules/app/sub-module'; +import { isSQLModeDisabled } from '@helpers/tooljet_db.helper'; export class TooljetDbModule extends SubModule implements OnModuleInit { constructor( @@ -69,11 +72,20 @@ export class TooljetDbModule extends SubModule implements OnModuleInit { const statementTimeout = this.configService.get('TOOLJET_DB_STATEMENT_TIMEOUT') || 60000; const statementTimeoutInSecs = Number.isNaN(Number(statementTimeout)) ? 60 : Number(statementTimeout) / 1000; - await reconfigurePostgrest(this.tooljetDbManager, { - user: tooljtDbUser, - enableAggregates: true, - statementTimeoutInSecs: statementTimeoutInSecs, - }); + if (isSQLModeDisabled()) { + await reconfigurePostgrestWithoutSchemaSync(this.tooljetDbManager, { + user: tooljtDbUser, + enableAggregates: true, + statementTimeoutInSecs: statementTimeoutInSecs, + }); + } else { + await reconfigurePostgrest(this.tooljetDbManager, { + user: tooljtDbUser, + enableAggregates: true, + statementTimeoutInSecs: statementTimeoutInSecs, + }); + } + await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'"); } } diff --git a/server/src/modules/tooljet-db/services/postgrest-proxy.service.ts b/server/src/modules/tooljet-db/services/postgrest-proxy.service.ts index 3743d8ad38..0103595d96 100644 --- a/server/src/modules/tooljet-db/services/postgrest-proxy.service.ts +++ b/server/src/modules/tooljet-db/services/postgrest-proxy.service.ts @@ -8,7 +8,7 @@ import { ConfigService } from '@nestjs/config'; import { EventEmitter2 } from '@nestjs/event-emitter'; import got from 'got'; import { TooljetDbTableOperationsService } from './tooljet-db-table-operations.service'; -import { validateTjdbJSONBColumnInputs } from 'src/helpers/tooljet_db.helper'; +import { isSQLModeDisabled, validateTjdbJSONBColumnInputs } from 'src/helpers/tooljet_db.helper'; import { QueryError } from '@modules/data-sources/types'; import { PostgrestError, TooljetDatabaseError, TooljetDbActions } from '../types'; import { maybeSetSubPath } from '@helpers/utils.helper'; @@ -28,8 +28,16 @@ export class PostgrestProxyService { async proxy(req, res, next) { const organizationId = req.headers['tj-workspace-id'] || req.dataQuery?.app?.organizationId; - const dbUser = `user_${organizationId}`; - const dbSchema = `workspace_${organizationId}`; + const { dbUser, dbSchema } = isSQLModeDisabled() + ? { + dbUser: this.configService.get('TOOLJET_DB_USER'), + dbSchema: 'public', + } + : { + dbUser: `user_${organizationId}`, + dbSchema: `workspace_${organizationId}`, + }; + const authToken = 'Bearer ' + this.signJwtPayload(dbUser); req.url = await this.replaceTableNamesAtPlaceholder(req.url, organizationId); @@ -92,8 +100,16 @@ export class PostgrestProxyService { body: Record = {} ) { try { - const dbUser = `user_${headers['tj-workspace-id']}`; - const dbSchema = `workspace_${headers['tj-workspace-id']}`; + const { dbUser, dbSchema } = isSQLModeDisabled() + ? { + dbUser: this.configService.get('TOOLJET_DB_USER'), + dbSchema: 'public', + } + : { + dbUser: `user_${headers['tj-workspace-id']}`, + dbSchema: `workspace_${headers['tj-workspace-id']}`, + }; + const authToken = 'Bearer ' + this.signJwtPayload(dbUser); const updatedPath = replaceUrlForPostgrest(url); let postgrestUrl = (this.configService.get('PGRST_HOST') || 'http://localhost:3001') + updatedPath; diff --git a/server/src/modules/tooljet-db/services/tooljet-db-data-operations.service.ts b/server/src/modules/tooljet-db/services/tooljet-db-data-operations.service.ts index b0ef19d4a2..5aaca3f841 100644 --- a/server/src/modules/tooljet-db/services/tooljet-db-data-operations.service.ts +++ b/server/src/modules/tooljet-db/services/tooljet-db-data-operations.service.ts @@ -4,11 +4,13 @@ import { QueryService, QueryResult, QueryError } from '@tooljet/plugins/dist/pac import { TooljetDbTableOperationsService } from './tooljet-db-table-operations.service'; import { isEmpty } from 'lodash'; import { maybeSetSubPath } from 'src/helpers/utils.helper'; + import { AST, Parser } from 'node-sql-parser/build/postgresql'; import { createTooljetDatabaseConnection, decryptTooljetDatabasePassword, findTenantSchema, + isSQLModeDisabled, modifyTjdbErrorObject, } from 'src/helpers/tooljet_db.helper'; import { EntityManager, In, QueryFailedError } from 'typeorm'; @@ -169,10 +171,10 @@ export class TooljetDbDataOperationsService implements QueryService { ); if (groupByAndAggregateQueryList.length) query.push(`select=${groupByAndAggregateQueryList.join(',')}`); } - !isEmpty(whereQuery) && query.push(whereQuery); - !isEmpty(orderQuery) && query.push(orderQuery); - !isEmpty(limit) && query.push(`limit=${limit}`); - !isEmpty(offset) && query.push(`offset=${offset}`); + if (!isEmpty(whereQuery)) query.push(whereQuery); + if (!isEmpty(orderQuery)) query.push(orderQuery); + if (!isEmpty(limit)) query.push(`limit=${limit}`); + if (!isEmpty(offset)) query.push(`offset=${offset}`); } const headers = { 'data-query-id': queryOptions.id, 'tj-workspace-id': organizationId }; @@ -218,7 +220,7 @@ export class TooljetDbDataOperationsService implements QueryService { return Object.assign(acc, { [colOpts.column]: colOpts.value }); }, {}); - !isEmpty(whereQuery) && query.push(whereQuery); + if (!isEmpty(whereQuery)) query.push(whereQuery); const headers = { 'data-query-id': queryOptions.id, 'tj-workspace-id': organizationId }; const url = maybeSetSubPath(`/api/tooljet-db/proxy/${tableId}?` + query.join('&') + '&order=id'); @@ -251,8 +253,8 @@ export class TooljetDbDataOperationsService implements QueryService { throw new QueryError('An incorrect limit value.', 'Limit should be a valid integer', {}); } - !isEmpty(whereQuery) && query.push(whereQuery); - limit && limit !== '' && query.push(`limit=${limit}&order=id`); + if (!isEmpty(whereQuery)) query.push(whereQuery); + if (limit && limit !== '') query.push(`limit=${limit}&order=id`); const headers = { 'data-query-id': queryOptions.id, 'tj-workspace-id': organizationId }; const url = maybeSetSubPath(`/api/tooljet-db/proxy/${tableId}?` + query.join('&')); @@ -325,7 +327,7 @@ export class TooljetDbDataOperationsService implements QueryService { } async sqlExecution(queryOptions, context): Promise { - if (this.configService.get('TJDB_SQL_MODE_DISABLE') === 'true') + if (isSQLModeDisabled()) throw new QueryError('SQL execution is disabled', 'Contact Admin to enable SQL execution', {}); const { organization_id: organizationId } = context.app; diff --git a/server/src/modules/tooljet-db/services/tooljet-db-table-operations.service.ts b/server/src/modules/tooljet-db/services/tooljet-db-table-operations.service.ts index add57838ae..47ae973e4d 100644 --- a/server/src/modules/tooljet-db/services/tooljet-db-table-operations.service.ts +++ b/server/src/modules/tooljet-db/services/tooljet-db-table-operations.service.ts @@ -27,6 +27,7 @@ import { createTooljetDatabaseConnection, decryptTooljetDatabasePassword, grantTenantRoleToTjdbAdminRole, + isSQLModeDisabled, } from 'src/helpers/tooljet_db.helper'; import { OrganizationTjdbConfigurations } from 'src/entities/organization_tjdb_configurations.entity'; const crypto = require('crypto'); @@ -831,10 +832,10 @@ export class TooljetDbTableOperationsService { } async getTablesLimit(organizationId: string) { - const licenseTerms = await this.licenseTermsService.getLicenseTerms([ - LICENSE_FIELD.TABLE_COUNT, - LICENSE_FIELD.STATUS, - ], organizationId); + const licenseTerms = await this.licenseTermsService.getLicenseTerms( + [LICENSE_FIELD.TABLE_COUNT, LICENSE_FIELD.STATUS], + organizationId + ); return { tablesCount: generatePayloadForLimits( licenseTerms[LICENSE_FIELD.TABLE_COUNT] !== LICENSE_LIMIT.UNLIMITED @@ -851,9 +852,15 @@ export class TooljetDbTableOperationsService { const { joinQueryJson, dataQuery, user } = params; if (!Object.keys(joinQueryJson).length) throw new BadRequestException("Input can't be empty"); - const tjdbTenantConfigs = await this.manager.findOne(OrganizationTjdbConfigurations, { - where: { organizationId }, - }); + const tjdbTenantConfigs = isSQLModeDisabled() + ? { + pgUser: this.configService.get('TOOLJET_DB_USER'), + pgPassword: this.configService.get('TOOLJET_DB_PASS'), + } + : await this.manager.findOne(OrganizationTjdbConfigurations, { + where: { organizationId }, + }); + if (!tjdbTenantConfigs) throw new NotFoundException(`Tooljet database schema configuration doesn't exists`); // Gathering tables used, from Join coditions @@ -931,7 +938,7 @@ export class TooljetDbTableOperationsService { protected buildJoinQuery( queryJson, internalTableIdToNameMap, - // eslint-disable-next-line + tooljetDbTenantConnection: Connection ): SelectQueryBuilder { const queryBuilder: SelectQueryBuilder = tooljetDbTenantConnection.createQueryBuilder(); @@ -1524,6 +1531,8 @@ export class TooljetDbTableOperationsService { } async createTooljetDbTenantSchemaAndRole(organizationId: string, entityManager: EntityManager) { + if (isSQLModeDisabled()) return; + const dbUser = `user_${organizationId}`; const dbSchema = `workspace_${organizationId}`; const dbPassword = crypto.randomBytes(8).toString('hex');