Merge pull request #13223 from ToolJet/feat/optional-tjdb-sql-mode

Feature: Optional SQL mode for ToolJet Database

frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx

@@ -17,6 +17,8 @@ import { useNavigate } from 'react-router-dom';
 import { deepClone } from '@/_helpers/utilities/utils.helpers';
 import { BulkUploadPrimaryKey } from './BulkUploadPrimaryKey';
 import BulkUpsertPrimaryKey from './BulkUpsertPrimaryKey';
+import { fetchEdition } from '@/modules/common/helpers/utils';
+import config from 'config';
 
 import './styles.scss';
 import CodeHinter from '@/AppBuilder/CodeEditor';
@@ -49,6 +51,21 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
   const [bulkUpdatePrimaryKey, setBulkUpdatePrimaryKey] = useState(() => options['bulk_update_with_primary_key'] || {});
   const [bulkUpsertPrimaryKey, setBulkUpsertPrimaryKey] = useState(() => options['bulk_upsert_with_primary_key'] || {});
 
+  // Check if SQL mode should be disabled
+  const isSqlModeDisabled = () => {
+    // Check legacy environment variable for backward compatibility
+    if (window.public_config?.TJDB_SQL_MODE_DISABLE === 'true') {
+      return true;
+    }
+
+    const edition = fetchEdition(config);
+    if (edition === 'cloud') {
+      return true;
+    }
+
+    return false;
+  };
+
   const joinOptions = options['join_table']?.['joins'] || [
     { conditions: { conditionsList: [{ leftField: { table: selectedTableId } }] } },
   ];
@@ -557,7 +574,7 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
     <TooljetDatabaseContext.Provider value={value}>
       {/* table name dropdown */}
 
-      {window.public_config?.TJDB_SQL_MODE_DISABLE !== 'true' && (
+      {!isSqlModeDisabled() && (
         <div
           className={cx({ 'col-4': !isHorizontalLayout, 'd-flex tooljetdb-worflow-operations': isHorizontalLayout })}
         >

server/data-migrations/1721236971725-MoveToolJetDatabaseTablesFromPublicToTenantSchema.ts

@@ -5,6 +5,7 @@ import { InternalTable } from '@entities/internal_table.entity';
 import { MigrationProgress, processDataInBatches } from '@helpers/migration.helper';
 import { getEnvVars } from 'scripts/database-config-utils';
 import { EncryptionService } from '@modules/encryption/service';
+import { TOOLJET_EDITIONS } from '@modules/app/constants';
 import {
   createNewTjdbRole,
   createAndGrantSchemaPrivilege,
@@ -20,6 +21,12 @@ const crypto = require('crypto');
 export class MoveToolJetDatabaseTablesFromPublicToTenantSchema1721236971725 implements MigrationInterface {
   public async up(queryRunner: QueryRunner): Promise<void> {
     const envData = getEnvVars();
+    const isSqlModeDisabled = envData.TJDB_SQL_MODE_DISABLE == 'true';
+    const isCloud = envData.TOOLJET_EDITION == TOOLJET_EDITIONS.Cloud;
+    if (isSqlModeDisabled || isCloud) {
+      console.log('Skipping TJDB schema migration for SQL mode');
+      return;
+    }
     const batchSize = 100;
     const entityManager = queryRunner.manager;
     const tooljetDbConnection = new DataSource({

server/src/helpers/tooljet_db.helper.ts

@@ -3,6 +3,8 @@ import { tooljetDbOrmconfig } from 'ormconfig';
 import { OrganizationTjdbConfigurations } from 'src/entities/organization_tjdb_configurations.entity';
 import { EntityManager, DataSource } from 'typeorm';
 import { v4 as uuidv4 } from 'uuid';
+import { getTooljetEdition } from '@helpers/utils.helper';
+import { TOOLJET_EDITIONS } from '@modules/app/constants';
 
 /**
  * Creates a custom tooljet database connection using a tenant user, for the respective workspace.
@@ -40,6 +42,10 @@ export async function createTooljetDatabaseConnection(
 }
 
 export async function decryptTooljetDatabasePassword(password: string) {
+  if (isSQLModeDisabled()) {
+    return process.env.TOOLJET_DB_PASS;
+  }
+
   const encryptionService = new EncryptionService();
   const decryptedvalue = await encryptionService.decryptColumnValue(
     'organization_tjdb_configurations',
@@ -60,9 +66,20 @@ export async function encryptTooljetDatabasePassword(password: string) {
 }
 
 export function findTenantSchema(organisationId: string): string {
+  if (isSQLModeDisabled()) {
+    return 'public';
+  }
+
   return `workspace_${organisationId}`;
 }
 
+// TODO: Cloud TJDB SQL mode is disabled: Use public schema for cloud edition
+// This is because Postgrest doesn't handle loading large amount of schemas in memory
+// We need to migrate to use Table based access control instead of schema based access control
+export function isSQLModeDisabled(): boolean {
+  return process.env.TJDB_SQL_MODE_DISABLE === 'true' || getTooljetEdition() === TOOLJET_EDITIONS.Cloud;
+}
+
 export function concatSchemaAndTableName(schema: string, tableName: string) {
   return `${schema}` + '.' + `${tableName}`;
 }
@@ -244,7 +261,7 @@ export function validateTjdbJSONBColumnInputs(jsonbColumnList: Array<string>, in
         } else {
           inValidValueColumnsList.push(key);
         }
-      } catch (error) {
+      } catch {
         inValidValueColumnsList.push(key);
       }
     }

server/src/modules/tooljet-db/helper.ts

@@ -27,3 +27,31 @@ export async function reconfigurePostgrest(
     throw error;
   }
 }
+
+/**
+ * Cloud TJDB SQL Disabled: Postgrest configuration without schema synchronization
+ * Postgres schema for each workspace is not loaded into Postgrest.
+ */
+export async function reconfigurePostgrestWithoutSchemaSync(
+  tooljetDbManager: EntityManager,
+  options: { user: string; enableAggregates: boolean; statementTimeoutInSecs: number }
+) {
+  try {
+    await tooljetDbManager.transaction(async (transactionalEntityManager) => {
+      await transactionalEntityManager.queryRunner.query('CREATE SCHEMA IF NOT EXISTS postgrest');
+      await transactionalEntityManager.queryRunner.query(`GRANT USAGE ON SCHEMA postgrest to ${options.user}`);
+      await transactionalEntityManager.queryRunner.query(`create or replace function postgrest.pre_config()
+      returns void as $$
+        select
+          set_config('pgrst.db_aggregates_enabled', '${options.enableAggregates}', false);
+      $$ language sql;
+      `);
+      await transactionalEntityManager.queryRunner.query(
+        `ALTER ROLE ${options.user} SET statement_timeout TO '${options.statementTimeoutInSecs}s'`
+      );
+    });
+  } catch (error) {
+    console.error('The tooljet database reconfiguration process encountered an error.', error);
+    throw error;
+  }
+}

server/src/modules/tooljet-db/module.ts

@@ -6,12 +6,15 @@ import { Logger } from 'nestjs-pino';
 import { Credential } from '../../../src/entities/credential.entity';
 import { InternalTable } from 'src/entities/internal_table.entity';
 import { AppUser } from 'src/entities/app_user.entity';
-import { reconfigurePostgrest } from './helper';
+import { reconfigurePostgrest, reconfigurePostgrestWithoutSchemaSync } from './helper';
+import { getTooljetEdition } from '@helpers/utils.helper';
+import { TOOLJET_EDITIONS } from '@modules/app/constants';
 import { TableCountGuard } from '@modules/licensing/guards/table.guard';
 import { AbilityUtilService } from '@modules/ability/util.service';
 import { RolesRepository } from '@modules/roles/repository';
 import { FeatureAbilityFactory } from './ability';
 import { SubModule } from '@modules/app/sub-module';
+import { isSQLModeDisabled } from '@helpers/tooljet_db.helper';
 
 export class TooljetDbModule extends SubModule implements OnModuleInit {
   constructor(
@@ -69,11 +72,20 @@ export class TooljetDbModule extends SubModule implements OnModuleInit {
       const statementTimeout = this.configService.get('TOOLJET_DB_STATEMENT_TIMEOUT') || 60000;
       const statementTimeoutInSecs = Number.isNaN(Number(statementTimeout)) ? 60 : Number(statementTimeout) / 1000;
 
-      await reconfigurePostgrest(this.tooljetDbManager, {
-        user: tooljtDbUser,
-        enableAggregates: true,
-        statementTimeoutInSecs: statementTimeoutInSecs,
-      });
+      if (isSQLModeDisabled()) {
+        await reconfigurePostgrestWithoutSchemaSync(this.tooljetDbManager, {
+          user: tooljtDbUser,
+          enableAggregates: true,
+          statementTimeoutInSecs: statementTimeoutInSecs,
+        });
+      } else {
+        await reconfigurePostgrest(this.tooljetDbManager, {
+          user: tooljtDbUser,
+          enableAggregates: true,
+          statementTimeoutInSecs: statementTimeoutInSecs,
+        });
+      }
+
       await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'");
     }
   }

server/src/modules/tooljet-db/services/postgrest-proxy.service.ts

@@ -8,7 +8,7 @@ import { ConfigService } from '@nestjs/config';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import got from 'got';
 import { TooljetDbTableOperationsService } from './tooljet-db-table-operations.service';
-import { validateTjdbJSONBColumnInputs } from 'src/helpers/tooljet_db.helper';
+import { isSQLModeDisabled, validateTjdbJSONBColumnInputs } from 'src/helpers/tooljet_db.helper';
 import { QueryError } from '@modules/data-sources/types';
 import { PostgrestError, TooljetDatabaseError, TooljetDbActions } from '../types';
 import { maybeSetSubPath } from '@helpers/utils.helper';
@@ -28,8 +28,16 @@ export class PostgrestProxyService {
   async proxy(req, res, next) {
     const organizationId = req.headers['tj-workspace-id'] || req.dataQuery?.app?.organizationId;
 
-    const dbUser = `user_${organizationId}`;
-    const dbSchema = `workspace_${organizationId}`;
+    const { dbUser, dbSchema } = isSQLModeDisabled()
+      ? {
+          dbUser: this.configService.get<string>('TOOLJET_DB_USER'),
+          dbSchema: 'public',
+        }
+      : {
+          dbUser: `user_${organizationId}`,
+          dbSchema: `workspace_${organizationId}`,
+        };
+
     const authToken = 'Bearer ' + this.signJwtPayload(dbUser);
 
     req.url = await this.replaceTableNamesAtPlaceholder(req.url, organizationId);
@@ -92,8 +100,16 @@ export class PostgrestProxyService {
     body: Record<string, any> = {}
   ) {
     try {
-      const dbUser = `user_${headers['tj-workspace-id']}`;
-      const dbSchema = `workspace_${headers['tj-workspace-id']}`;
+      const { dbUser, dbSchema } = isSQLModeDisabled()
+        ? {
+            dbUser: this.configService.get<string>('TOOLJET_DB_USER'),
+            dbSchema: 'public',
+          }
+        : {
+            dbUser: `user_${headers['tj-workspace-id']}`,
+            dbSchema: `workspace_${headers['tj-workspace-id']}`,
+          };
+
       const authToken = 'Bearer ' + this.signJwtPayload(dbUser);
       const updatedPath = replaceUrlForPostgrest(url);
       let postgrestUrl = (this.configService.get<string>('PGRST_HOST') || 'http://localhost:3001') + updatedPath;

server/src/modules/tooljet-db/services/tooljet-db-data-operations.service.ts

@@ -4,11 +4,13 @@ import { QueryService, QueryResult, QueryError } from '@tooljet/plugins/dist/pac
 import { TooljetDbTableOperationsService } from './tooljet-db-table-operations.service';
 import { isEmpty } from 'lodash';
 import { maybeSetSubPath } from 'src/helpers/utils.helper';
+
 import { AST, Parser } from 'node-sql-parser/build/postgresql';
 import {
   createTooljetDatabaseConnection,
   decryptTooljetDatabasePassword,
   findTenantSchema,
+  isSQLModeDisabled,
   modifyTjdbErrorObject,
 } from 'src/helpers/tooljet_db.helper';
 import { EntityManager, In, QueryFailedError } from 'typeorm';
@@ -169,10 +171,10 @@ export class TooljetDbDataOperationsService implements QueryService {
           );
           if (groupByAndAggregateQueryList.length) query.push(`select=${groupByAndAggregateQueryList.join(',')}`);
         }
-        !isEmpty(whereQuery) && query.push(whereQuery);
-        !isEmpty(orderQuery) && query.push(orderQuery);
-        !isEmpty(limit) && query.push(`limit=${limit}`);
-        !isEmpty(offset) && query.push(`offset=${offset}`);
+        if (!isEmpty(whereQuery)) query.push(whereQuery);
+        if (!isEmpty(orderQuery)) query.push(orderQuery);
+        if (!isEmpty(limit)) query.push(`limit=${limit}`);
+        if (!isEmpty(offset)) query.push(`offset=${offset}`);
       }
 
       const headers = { 'data-query-id': queryOptions.id, 'tj-workspace-id': organizationId };
@@ -218,7 +220,7 @@ export class TooljetDbDataOperationsService implements QueryService {
       return Object.assign(acc, { [colOpts.column]: colOpts.value });
     }, {});
 
-    !isEmpty(whereQuery) && query.push(whereQuery);
+    if (!isEmpty(whereQuery)) query.push(whereQuery);
 
     const headers = { 'data-query-id': queryOptions.id, 'tj-workspace-id': organizationId };
     const url = maybeSetSubPath(`/api/tooljet-db/proxy/${tableId}?` + query.join('&') + '&order=id');
@@ -251,8 +253,8 @@ export class TooljetDbDataOperationsService implements QueryService {
       throw new QueryError('An incorrect limit value.', 'Limit should be a valid integer', {});
     }
 
-    !isEmpty(whereQuery) && query.push(whereQuery);
-    limit && limit !== '' && query.push(`limit=${limit}&order=id`);
+    if (!isEmpty(whereQuery)) query.push(whereQuery);
+    if (limit && limit !== '') query.push(`limit=${limit}&order=id`);
 
     const headers = { 'data-query-id': queryOptions.id, 'tj-workspace-id': organizationId };
     const url = maybeSetSubPath(`/api/tooljet-db/proxy/${tableId}?` + query.join('&'));
@@ -325,7 +327,7 @@ export class TooljetDbDataOperationsService implements QueryService {
   }
 
   async sqlExecution(queryOptions, context): Promise<QueryResult> {
-    if (this.configService.get<string>('TJDB_SQL_MODE_DISABLE') === 'true')
+    if (isSQLModeDisabled())
       throw new QueryError('SQL execution is disabled', 'Contact Admin to enable SQL execution', {});
 
     const { organization_id: organizationId } = context.app;

server/src/modules/tooljet-db/services/tooljet-db-table-operations.service.ts

@@ -27,6 +27,7 @@ import {
   createTooljetDatabaseConnection,
   decryptTooljetDatabasePassword,
   grantTenantRoleToTjdbAdminRole,
+  isSQLModeDisabled,
 } from 'src/helpers/tooljet_db.helper';
 import { OrganizationTjdbConfigurations } from 'src/entities/organization_tjdb_configurations.entity';
 const crypto = require('crypto');
@@ -831,10 +832,10 @@ export class TooljetDbTableOperationsService {
   }
 
   async getTablesLimit(organizationId: string) {
-    const licenseTerms = await this.licenseTermsService.getLicenseTerms([
-      LICENSE_FIELD.TABLE_COUNT,
-      LICENSE_FIELD.STATUS,
-    ], organizationId);
+    const licenseTerms = await this.licenseTermsService.getLicenseTerms(
+      [LICENSE_FIELD.TABLE_COUNT, LICENSE_FIELD.STATUS],
+      organizationId
+    );
     return {
       tablesCount: generatePayloadForLimits(
         licenseTerms[LICENSE_FIELD.TABLE_COUNT] !== LICENSE_LIMIT.UNLIMITED
@@ -851,9 +852,15 @@ export class TooljetDbTableOperationsService {
     const { joinQueryJson, dataQuery, user } = params;
     if (!Object.keys(joinQueryJson).length) throw new BadRequestException("Input can't be empty");
 
-    const tjdbTenantConfigs = await this.manager.findOne(OrganizationTjdbConfigurations, {
-      where: { organizationId },
-    });
+    const tjdbTenantConfigs = isSQLModeDisabled()
+      ? {
+          pgUser: this.configService.get<string>('TOOLJET_DB_USER'),
+          pgPassword: this.configService.get<string>('TOOLJET_DB_PASS'),
+        }
+      : await this.manager.findOne(OrganizationTjdbConfigurations, {
+          where: { organizationId },
+        });
+
     if (!tjdbTenantConfigs) throw new NotFoundException(`Tooljet database schema configuration doesn't exists`);
 
     // Gathering tables used, from Join coditions
@@ -931,7 +938,7 @@ export class TooljetDbTableOperationsService {
   protected buildJoinQuery(
     queryJson,
     internalTableIdToNameMap,
-    // eslint-disable-next-line
+
     tooljetDbTenantConnection: Connection
   ): SelectQueryBuilder<any> {
     const queryBuilder: SelectQueryBuilder<any> = tooljetDbTenantConnection.createQueryBuilder();
@@ -1524,6 +1531,8 @@ export class TooljetDbTableOperationsService {
   }
 
   async createTooljetDbTenantSchemaAndRole(organizationId: string, entityManager: EntityManager) {
+    if (isSQLModeDisabled()) return;
+
     const dbUser = `user_${organizationId}`;
     const dbSchema = `workspace_${organizationId}`;
     const dbPassword = crypto.randomBytes(8).toString('hex');