Feature: Make tooljetdb optional (#5062)

* make tooljetdb optional

* restrict routes when tooljetdb is disabled

* validate dbname before creating database

* fix comment

* format

.env.example

@@ -66,3 +66,4 @@ SSO_DISABLE_SIGNUPS=
 
 #ONBOARDING
 ENABLE_ONBOARDING_QUESTIONS_FOR_ALL_SIGN_UPS=
+ENABLE_TOOLJET_DB=

.github/workflows/render-preview-deploy.yml

@@ -57,6 +57,10 @@ jobs:
                 "key": "PG_DB",
                 "value": "${{ env.PR_NUMBER }}"
               },
+              {
+                "key": "ENABLE_TOOLJET_DB",
+                "value": "true"
+              },
               {
                 "key": "TOOLJET_DB",
                 "value": "TJDB_${{ env.PR_NUMBER }}"

deploy/docker/.env.example

@@ -42,6 +42,7 @@ SENTRY_DEBUG=
 # FEATURE TOGGLE
 COMMENT_FEATURE_ENABLE=
 ENABLE_MULTIPLAYER_EDITING=true
+ENABLE_TOOLJET_DB=
 
 #SSO
 SSO_DISABLE_SIGNUP=

docs/versioned_docs/version-2.0.0-beta/setup/env-vars.md

@@ -77,6 +77,14 @@ Use this environment variable to enable/disable the feature that allows users to
 | -------- | ---------------------- |
 | ENABLE_MARKETPLACE_FEATURE  | `true` or `false` |
 
+#### ToolJet Database feature enable ( optional )
+
+Use this environment variable to enable/disable the feature that allows users to work with inbuilt data store to build apps with.
+
+| variable          | value             |
+| ----------------- | ----------------- |
+| ENABLE_TOOLJET_DB | `true` or `false` |
+
 #### Server Host ( optional )
 
 You can specify a different server for backend if it is hosted on another server.

frontend/src/App/App.jsx

@@ -228,13 +228,15 @@ class App extends React.Component {
               switchDarkMode={this.switchDarkMode}
               darkMode={darkMode}
             />
-            <PrivateRoute
+            {window.public_config?.ENABLE_TOOLJET_DB == 'true' && (
-              exact
+              <PrivateRoute
-              path="/tooljet-database"
+                exact
-              component={TooljetDatabase}
+                path="/tooljet-database"
-              switchDarkMode={this.switchDarkMode}
+                component={TooljetDatabase}
-              darkMode={darkMode}
+                switchDarkMode={this.switchDarkMode}
-            />
+                darkMode={darkMode}
+              />
+            )}
             {window.public_config?.ENABLE_MARKETPLACE_FEATURE && (
               <AdminRoute
                 exact

frontend/src/_ui/Layout/index.jsx

@@ -52,34 +52,36 @@ function Layout({ children, switchDarkMode, darkMode }) {
                   </ToolTip>
                 </Link>
               </li>
-              <li className="text-center mt-3 cursor-pointer">
+              {window.public_config?.ENABLE_TOOLJET_DB == 'true' && (
-                <Link to="/tooljet-database">
+                <li className="text-center mt-3 cursor-pointer">
-                  <ToolTip message="Tooljet database" placement="right">
+                  <Link to="/tooljet-database">
-                    <svg
+                    <ToolTip message="Tooljet database" placement="right">
-                      className="layout-sidebar-icon"
+                      <svg
-                      width="32"
+                        className="layout-sidebar-icon"
-                      height="33"
-                      viewBox="0 0 32 33"
-                      fill="none"
-                      xmlns="http://www.w3.org/2000/svg"
-                    >
-                      <rect
-                        y="0.325684"
                         width="32"
-                        height="32"
+                        height="33"
-                        rx="4"
+                        viewBox="0 0 32 33"
-                        fill={router.pathname === '/tooljet-database' ? '#E6EDFE' : '#none'}
+                        fill="none"
-                      />
+                        xmlns="http://www.w3.org/2000/svg"
-                      <path
+                      >
-                        fillRule="evenodd"
+                        <rect
-                        clipRule="evenodd"
+                          y="0.325684"
-                        d="M10 9.32568C9.73478 9.32568 9.48043 9.43104 9.29289 9.61858C9.10536 9.80611 9 10.0605 9 10.3257V13.3257H13V9.32568H10ZM10 7.32568C9.20435 7.32568 8.44129 7.64175 7.87868 8.20436C7.31607 8.76697 7 9.53003 7 10.3257V16.3257C7 16.878 7.44772 17.3257 8 17.3257C8.55228 17.3257 9 16.878 9 16.3257V15.3257H13V16.3257C13 16.878 13.4477 17.3257 14 17.3257C14.5523 17.3257 15 16.878 15 16.3257V15.3257H23V22.3257C23 22.5909 22.8946 22.8453 22.7071 23.0328C22.5196 23.2203 22.2652 23.3257 22 23.3257H16C15.4477 23.3257 15 23.7734 15 24.3257C15 24.878 15.4477 25.3257 16 25.3257H22C22.7957 25.3257 23.5587 25.0096 24.1213 24.447C24.6839 23.8844 25 23.1213 25 22.3257V10.3257C25 9.53003 24.6839 8.76697 24.1213 8.20436C23.5587 7.64175 22.7957 7.32568 22 7.32568H10ZM15 9.32568V13.3257H23V10.3257C23 10.0605 22.8946 9.80611 22.7071 9.61858C22.5196 9.43104 22.2652 9.32568 22 9.32568H15ZM6 20.3257C6 19.2211 6.89543 18.3257 8 18.3257H12C13.1046 18.3257 14 19.2211 14 20.3257V24.3257C14 25.4303 13.1046 26.3257 12 26.3257H8C6.89543 26.3257 6 25.4303 6 24.3257V20.3257ZM8 20.3257V24.3257H12V20.3257H8Z"
+                          width="32"
-                        fill={router.pathname === '/tooljet-database' ? '#3E63DD' : '#C1C8CD'}
+                          height="32"
-                      />
+                          rx="4"
-                    </svg>
+                          fill={router.pathname === '/tooljet-database' ? '#E6EDFE' : '#none'}
-                  </ToolTip>
+                        />
-                </Link>
+                        <path
-              </li>
+                          fillRule="evenodd"
+                          clipRule="evenodd"
+                          d="M10 9.32568C9.73478 9.32568 9.48043 9.43104 9.29289 9.61858C9.10536 9.80611 9 10.0605 9 10.3257V13.3257H13V9.32568H10ZM10 7.32568C9.20435 7.32568 8.44129 7.64175 7.87868 8.20436C7.31607 8.76697 7 9.53003 7 10.3257V16.3257C7 16.878 7.44772 17.3257 8 17.3257C8.55228 17.3257 9 16.878 9 16.3257V15.3257H13V16.3257C13 16.878 13.4477 17.3257 14 17.3257C14.5523 17.3257 15 16.878 15 16.3257V15.3257H23V22.3257C23 22.5909 22.8946 22.8453 22.7071 23.0328C22.5196 23.2203 22.2652 23.3257 22 23.3257H16C15.4477 23.3257 15 23.7734 15 24.3257C15 24.878 15.4477 25.3257 16 25.3257H22C22.7957 25.3257 23.5587 25.0096 24.1213 24.447C24.6839 23.8844 25 23.1213 25 22.3257V10.3257C25 9.53003 24.6839 8.76697 24.1213 8.20436C23.5587 7.64175 22.7957 7.32568 22 7.32568H10ZM15 9.32568V13.3257H23V10.3257C23 10.0605 22.8946 9.80611 22.7071 9.61858C22.5196 9.43104 22.2652 9.32568 22 9.32568H15ZM6 20.3257C6 19.2211 6.89543 18.3257 8 18.3257H12C13.1046 18.3257 14 19.2211 14 20.3257V24.3257C14 25.4303 13.1046 26.3257 12 26.3257H8C6.89543 26.3257 6 25.4303 6 24.3257V20.3257ZM8 20.3257V24.3257H12V20.3257H8Z"
+                          fill={router.pathname === '/tooljet-database' ? '#3E63DD' : '#C1C8CD'}
+                        />
+                      </svg>
+                    </ToolTip>
+                  </Link>
+                </li>
+              )}
               <li className="text-center mt-3 cursor-pointer">
                 <Link to="/organization-settings">
                   <ToolTip message="Organization settings" placement="right">

frontend/webpack.config.js

@@ -158,6 +158,7 @@ module.exports = {
       apiUrl: `${stripTrailingSlash(API_URL[environment]) || ''}/api`,
       SERVER_IP: process.env.SERVER_IP,
       COMMENT_FEATURE_ENABLE: process.env.COMMENT_FEATURE_ENABLE ?? true,
+      ENABLE_TOOLJET_DB: process.env.ENABLE_TOOLJET_DB ?? true,
       ENABLE_MULTIPLAYER_EDITING: true,
       TOOLJET_MARKETPLACE_URL:
         process.env.TOOLJET_MARKETPLACE_URL || 'https://tooljet-plugins-production.s3.us-east-2.amazonaws.com',

render.yaml

@@ -15,6 +15,8 @@ services:
           property: connectionString
       - key: TOOLJET_DB
         value: 'tooljet_db'
+      - key: ENABLE_TOOLJET_DB
+        value: 'true'
       - key: TOOLJET_HOST
         fromService:
           type: web

server/migrations/1665507074072-CreateTooljetDb.ts

@@ -1,29 +0,0 @@
-import { getManager, MigrationInterface, QueryRunner } from 'typeorm';
-import { getEnvVars } from '../scripts/database-config-utils';
-
-// This migration is a prerequisite for tooljetDb and incase the PG_USER
-// dont have createdb privileges, this migration needs to be run for
-// internal storage to function
-export class CreateTooljetDb1665507074072 implements MigrationInterface {
-  public async up(queryRunner: QueryRunner): Promise<void> {
-    const data = getEnvVars();
-    if (data.PG_DB_OWNER !== 'false') {
-      // Database creation can't be made inside a transaction block which queryRunner is run inside.
-      // Thus we get a new connection to create the database first.
-      try {
-        await getManager().query(`CREATE DATABASE ${data.TOOLJET_DB} WITH OWNER ${data.PG_USER};`);
-      } catch (err) {
-        const errorMessage = `database "${data.TOOLJET_DB}" already exists`;
-        if (err.message.includes(errorMessage)) return;
-        throw err;
-      }
-    }
-  }
-
-  public async down(_queryRunner: QueryRunner): Promise<void> {
-    const data = getEnvVars();
-    if (data.PG_DB_OWNER !== 'false') {
-      await getManager().query(`DROP DATABASE ${data.TOOLJET_DB}`);
-    }
-  }
-}

server/scripts/create-database.ts

@@ -3,6 +3,7 @@ import * as dotenv from 'dotenv';
 import * as fs from 'fs';
 import { exec } from 'child_process';
 import { buildAndValidateDatabaseConfig } from './database-config-utils';
+import { isEmpty } from 'lodash';
 
 function createDatabaseFromFile(envPath: string): void {
   const result = dotenv.config({ path: envPath });
@@ -26,6 +27,11 @@ function createDatabase(): void {
     throw new Error(`Config validation error: ${error.message}`);
   }
 
+  if (envVars.PG_DB_OWNER === 'false') {
+    console.log('Skipping database creation');
+    return;
+  }
+
   const connectivityCheck = exec('command -v createdb');
 
   connectivityCheck.on('exit', function (signal) {
@@ -35,28 +41,37 @@ function createDatabase(): void {
     }
   });
 
-  if (envVars.PG_DB_OWNER === 'false') {
+  // Allow creating db based on cmd line arg
-    console.log('Skipping database creation');
+  const dbNameFromArg = process.argv[2];
-    return;
+  if (dbNameFromArg) return createDb(envVars, dbNameFromArg);
+
+  createDb(envVars, envVars.PG_DB);
+  if (process.env.ENABLE_TOOLJET_DB == 'true') {
+    createDb(envVars, envVars.TOOLJET_DB);
   }
+}
+
+function createDb(envVars, dbName) {
+  if (isEmpty(dbName)) throw 'Database name cannot be empty';
 
   const createdb =
     `PGPASSWORD="${envVars.PG_PASS}" createdb ` +
     `-h ${envVars.PG_HOST} ` +
     `-p ${envVars.PG_PORT} ` +
     `-U ${envVars.PG_USER} ` +
-    envVars.PG_DB;
+    dbName;
 
   exec(createdb, (err, _stdout, _stderr) => {
     if (!err) {
-      console.log(`Created database ${envVars.PG_DB}`);
+      console.log(`Created database ${dbName}`);
       return;
     }
 
-    const errorMessage = `database "${envVars.PG_DB}" already exists`;
+    const errorMessage = `database "${dbName}" already exists`;
 
     if (err.message.includes(errorMessage)) {
-      console.log(`Using database: ${envVars.PG_DB}`);
+      envVars.PG_DB == dbName && console.log(`Using PG_DB: ${dbName}`);
+      envVars.TOOLJET_DB == dbName && console.log(`Using TOOLJET_DB: ${dbName}`);
     } else {
       console.error(err);
       process.exit(1);
@@ -74,6 +89,5 @@ if (fs.existsSync(nodeEnvPath)) {
   createDatabaseFromFile(fallbackPath);
 } else {
   console.log('Picking up config from the environment');
-
   createDatabase();
 }

server/scripts/drop-database.ts

@@ -29,20 +29,28 @@ function dropDatabase(): void {
     }
   });
 
+  // Allow dropping db based on cmd line arg
+  const dbNameFromArg = process.argv[2];
+  if (dbNameFromArg) return dropDb(envVars, dbNameFromArg);
+
+  dropDb(envVars, envVars.PG_DB);
+}
+
+function dropDb(envVars, dbName) {
   const dropdb =
     `PGPASSWORD=${envVars.PG_PASS} dropdb ` +
     `-h ${envVars.PG_HOST} ` +
     `-p ${envVars.PG_PORT} ` +
     `-U ${envVars.PG_USER} ` +
-    envVars.PG_DB;
+    dbName;
 
   exec(dropdb, (err, _stdout, _stderr) => {
     if (!err) {
-      console.log(`Dropped database ${envVars.PG_DB}`);
+      console.log(`Dropped database ${dbName}`);
       return;
     }
 
-    const errorMessage = `database "${envVars.PG_DB}" does not exist`;
+    const errorMessage = `database "${dbName}" does not exist`;
 
     if (err.message.includes(errorMessage)) {
       console.log(errorMessage);

server/src/app.module.ts

@@ -94,7 +94,6 @@ const imports = [
   PluginsModule,
   EventsModule,
   AppEnvironmentsModule,
-  TooljetDbModule,
 ];
 
 if (process.env.SERVE_CLIENT !== 'false') {
@@ -145,6 +144,10 @@ if (process.env.COMMENT_FEATURE_ENABLE !== 'false') {
   imports.unshift(CommentModule, ThreadModule, CommentUsersModule);
 }
 
+if (process.env.ENABLE_TOOLJET_DB === 'true') {
+  imports.unshift(TooljetDbModule);
+}
+
 @Module({
   imports,
   controllers: [AppController],

server/src/services/app_config.service.ts

@@ -26,6 +26,7 @@ export class AppConfigService {
       'SUB_PATH',
       'DISABLE_MULTI_WORKSPACE',
       'ENABLE_MARKETPLACE_FEATURE',
+      'ENABLE_TOOLJET_DB',
     ];
   }