Elgato_dark/ToolJet
1
0
Fork 0
mirror of https://github.com/ToolJet/ToolJet synced 2026-05-23 17:08:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #12862 from ToolJet/fix/workspace-constants-mapping

Browse source 
Fix: Ensure secrets and constants mapped to encrypted fields are correctly imported with the app
This commit is contained in:
Adish M 2025-07-08 22:29:58 +05:30 committed by GitHub
commit 12537130b2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 493 additions and 236 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
cypress-tests/cypress/commands/commands.js
View file

@ -239,9 +239,9 @@ Cypress.Commands.add(
.invoke("text")
.then((text) => {
cy.wrap(subject).realType(createBackspaceText(text)),
{
delay: 0,
};
{
delay: 0,
};
});
}
);
@ -561,7 +561,7 @@ Cypress.Commands.add("installMarketplacePlugin", (pluginName) => {
}
});
function installPlugin (pluginName) {
function installPlugin(pluginName) {
cy.get('[data-cy="-list-item"]').eq(1).click();
cy.wait(1000);
@ -621,6 +621,7 @@ Cypress.Commands.add("uninstallMarketplacePlugin", (pluginName) => {
Cypress.Commands.add(
"verifyRequiredFieldValidation",
(fieldName, expectedColor) => {
cy.get(commonSelectors.textField(fieldName)).type("some text").clear();
cy.get(commonSelectors.textField(fieldName)).should(
"have.css",
"border-color",

40
cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/airTableHappyPath.cy.js
View file

@ -202,10 +202,10 @@ describe("Data source Airtable", () => {
);
cy.get(dataSourceSelector.queryPreviewButton).click();
cy.verifyToastMessage(
commonSelectors.toastMessage,
`Query (${data.dsName}) completed.`
);
// cy.verifyToastMessage(
// commonSelectors.toastMessage,
// `Query (${data.dsName}) completed.`
// );
// Verfiy Retrieve record operation
@ -225,10 +225,10 @@ describe("Data source Airtable", () => {
);
cy.get(dataSourceSelector.queryPreviewButton).click();
cy.verifyToastMessage(
commonSelectors.toastMessage,
`Query (${data.dsName}) completed.`
);
// cy.verifyToastMessage(
// commonSelectors.toastMessage,
// `Query (${data.dsName}) completed.`
// );
// Verfiy Create record operation
@ -251,10 +251,10 @@ describe("Data source Airtable", () => {
.realType('": {}', { force: true, delay: 0 });
cy.get(dataSourceSelector.queryPreviewButton).click();
cy.verifyToastMessage(
commonSelectors.toastMessage,
`Query (${data.dsName}) completed.`
);
// cy.verifyToastMessage(
// commonSelectors.toastMessage,
// `Query (${data.dsName}) completed.`
// );
// Verfiy Update record operation
@ -285,10 +285,10 @@ describe("Data source Airtable", () => {
.realType('"Phone Number": "555_98"', { force: true, delay: 0 });
cy.get(dataSourceSelector.queryPreviewButton).click();
cy.verifyToastMessage(
commonSelectors.toastMessage,
`Query (${data.queryName}) completed.`
);
// cy.verifyToastMessage(
// commonSelectors.toastMessage,
// `Query (${data.queryName}) completed.`
// );
// Verify Delete record operation
@ -337,10 +337,10 @@ describe("Data source Airtable", () => {
);
cy.get(dataSourceSelector.queryPreviewButton).click();
cy.verifyToastMessage(
commonSelectors.toastMessage,
`Query (${data.queryName}) completed.`
);
// cy.verifyToastMessage(
// commonSelectors.toastMessage,
// `Query (${data.queryName}) completed.`
// );
cy.apiDeleteApp(`${data.dsName}-airtable-app`);
cy.apiDeleteGDS(`cypress-${data.dsName}-airtable`);

2
cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/postgresHappyPath.cy.js
View file

@ -254,7 +254,7 @@ describe("Data sources", () => {
.and("be.disabled");
cy.get(dataSourceSelector.connectionAlertText).verifyVisibleElement(
"have.text",
"connect ECONNREFUSED 127.0.0.1:5432"
postgreSqlText.serverNotSuppotSsl
);
cy.apiDeleteGDS(`cypress-${data.dataSourceName}-postgresql`);

15
frontend/src/_components/DynamicForm.jsx
View file

@ -253,6 +253,8 @@ const DynamicForm = ({
}) => {
const source = schema?.source?.kind;
const darkMode = localStorage.getItem('darkMode') === 'true';
const workspaceConstant = options?.[key]?.workspace_constant;
const isWorkspaceConstant = !!workspaceConstant;
if (!options) return;
@ -264,7 +266,7 @@ const DynamicForm = ({
(options?.[key]?.encrypted !== undefined ? options?.[key].encrypted : encrypted) || type === 'password';
return {
type,
placeholder: useEncrypted ? '**************' : description,
placeholder: workspaceConstant ? workspaceConstant : useEncrypted ? '**************' : description,
className: `form-control${handleToggle(controller)} ${useEncrypted && 'dynamic-form-encrypted-field'}`,
style: { marginBottom: '0px !important' },
value: options?.[key]?.value || '',
@ -276,6 +278,7 @@ const DynamicForm = ({
workspaceVariables,
workspaceConstants: currentOrgEnvironmentConstants,
encrypted: useEncrypted,
isWorkspaceConstant: isWorkspaceConstant,
};
}
case 'toggle':
@ -509,10 +512,16 @@ const DynamicForm = ({
return;
}
const isEditing = computedProps[field]['disabled'];
const workspaceConstant = options?.[field]?.workspace_constant;
const isWorkspaceConstant = !!workspaceConstant;
if (isEditing) {
optionchanged(field, '');
if (isWorkspaceConstant) {
optionchanged(field, workspaceConstant);
} else {
optionchanged(field, '');
}
} else {
//Send old field value if editing mode disabled for encrypted fields
const newOptions = { ...options };
const oldFieldValue = selectedDataSource?.['options']?.[field];
if (oldFieldValue) {

109
frontend/src/_components/DynamicFormV2.jsx
View file

@ -8,7 +8,6 @@ import Headers from '@/_ui/HttpHeaders';
import Toggle from '@/_ui/Toggle';
import InputV3 from '@/_ui/Input-V3';
import { filter, find, isEmpty } from 'lodash';
import { ButtonSolid } from './AppButton';
import { useGlobalDataSourcesStatus } from '@/_stores/dataSourcesStore';
import { canDeleteDataSource, canUpdateDataSource } from '@/_helpers';
import { OverlayTrigger, Tooltip } from 'react-bootstrap';
@ -206,39 +205,63 @@ const DynamicFormV2 = ({
}
const processFields = (fieldsObject) => {
Object.keys(fieldsObject).forEach((key) => {
const field = fieldsObject[key];
const { widget, encrypted, key: propertyKey } = field;
const processNestedField = (field, propertyKey) => {
const { widget, encrypted } = field;
if (!canUpdateDataSource(selectedDataSource?.id) && !canDeleteDataSource()) {
encryptedFieldsProps[propertyKey] = {
disabled: !!selectedDataSource?.id,
};
} else if (!isDataSourceEditing) {
if (widget === 'password' || encrypted) {
encryptedFieldsProps[propertyKey] = {
disabled: true,
};
}
} else {
if ((widget === 'password' || encrypted) && !(propertyKey in computedProps)) {
const isEncryptedField =
widget === 'password-v3' ||
widget === 'password-v3-textarea' ||
widget === 'password' ||
encrypted ||
encryptedProperties.includes(propertyKey);
if (isEncryptedField) {
if (computedProps[propertyKey] !== undefined && computedProps[propertyKey].disabled === false) {
encryptedFieldsProps[propertyKey] = { disabled: false };
} else if (!isDataSourceEditing) {
encryptedFieldsProps[propertyKey] = { disabled: true };
} else if (!(propertyKey in computedProps)) {
encryptedFieldsProps[propertyKey] = {
disabled: !!selectedDataSource?.id,
};
}
}
};
// To check for nested dropdown-component-flip
if (widget === 'dropdown-component-flip') {
const selectedOption = options?.[field.key]?.value;
Object.keys(fieldsObject).forEach((key) => {
const field = fieldsObject[key];
if (field.commonFields) {
processFields(field.commonFields);
if (field.key) {
processNestedField(field, field.key);
}
// Check for nested structures and recursively process them
if (typeof field === 'object') {
if (field.widget === 'dropdown-component-flip') {
const selectedOption = options?.[field.key]?.value;
if (field.commonFields) {
Object.keys(field.commonFields).forEach((commonKey) => {
const commonField = field.commonFields[commonKey];
processNestedField(commonField, commonField.key);
});
}
if (selectedOption && fieldsObject[selectedOption]) {
processFields(fieldsObject[selectedOption]);
}
}
if (selectedOption && fieldsObject[selectedOption]) {
processFields(fieldsObject[selectedOption]);
}
// For other nested objects, recursively process them
Object.keys(field).forEach((subKey) => {
if (typeof field[subKey] === 'object' && field[subKey] !== null) {
if (field[subKey].widget || field[subKey].key) {
processNestedField(field[subKey], field[subKey].key);
} else {
processFields({ [subKey]: field[subKey] });
}
}
});
}
});
};
@ -264,6 +287,11 @@ const DynamicFormV2 = ({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedDataSource?.id, options, isDataSourceEditing]);
React.useEffect(() => {
const requiredFields = processAllOfConditions(schema, options);
setConditionallyRequiredProperties(requiredFields);
}, [options, processAllOfConditions, schema, selectedDataSource.id]);
const getElement = (type) => {
switch (type) {
case 'password':
@ -295,6 +323,8 @@ const DynamicFormV2 = ({
const currentValue = options?.[key]?.value;
const skipValidation =
(!hasUserInteracted && !showValidationErrors) || (!interactedFields.has(key) && !showValidationErrors);
const workspaceConstant = options?.[key]?.workspace_constant;
const isEditing = computedProps[key] && computedProps[key].disabled === false;
const handleOptionChange = (key, value, flag = true) => {
if (!hasUserInteracted) {
@ -309,10 +339,10 @@ const DynamicFormV2 = ({
case 'text':
case 'textarea': {
return {
key,
propertyKey: key,
widget,
label,
placeholder: isEncrypted ? '**************' : description,
placeholder: workspaceConstant ? workspaceConstant : isEncrypted ? '**************' : description,
className: cx('form-control', {
'dynamic-form-encrypted-field': isEncrypted,
}),
@ -321,20 +351,20 @@ const DynamicFormV2 = ({
value: currentValue || '',
onChange: (e) => optionchanged(key, e.target.value, true),
isGDS: true,
workspaceVariables: [],
workspaceConstants: [],
encrypted: isEncrypted,
onBlur,
workspaceVariables,
workspaceConstants: currentOrgEnvironmentConstants,
};
}
case 'password-v3':
case 'password-v3-textarea':
case 'text-v3': {
return {
key,
propertyKey: key,
widget,
label,
placeholder: isEncrypted ? '**************' : description,
placeholder: workspaceConstant ? workspaceConstant : isEncrypted ? '**************' : description,
className: cx('form-control', {
'dynamic-form-encrypted-field': isEncrypted,
}),
@ -343,8 +373,6 @@ const DynamicFormV2 = ({
value: currentValue || '',
onChange: (e) => handleOptionChange(key, e.target.value, true),
isGDS: true,
workspaceVariables: [],
workspaceConstants: [],
encrypted: isEncrypted,
onBlur,
isRequired: isRequired,
@ -356,6 +384,10 @@ const DynamicFormV2 = ({
? { valid: true, message: '' }
: { valid: null, message: '' }, // handle optional && encrypted fields
isDisabled: !canUpdateDataSource(selectedDataSource?.id) && !canDeleteDataSource(),
workspaceVariables,
workspaceConstants: currentOrgEnvironmentConstants,
isEditing: isEditing,
labelDisabled: false,
};
}
case 'react-component-headers': {
@ -411,11 +443,18 @@ const DynamicFormV2 = ({
if (!canUpdateDataSource(selectedDataSource?.id) && !canDeleteDataSource()) {
return;
}
const isEditing = computedProps[field]['disabled'];
const workspaceConstant = options?.[field]?.workspace_constant;
const isWorkspaceConstant = !!workspaceConstant;
if (isEditing) {
optionchanged(field, '');
if (isWorkspaceConstant) {
optionchanged(field, workspaceConstant);
} else {
optionchanged(field, '');
}
} else {
//Send old field value if editing mode disabled for encrypted fields
const newOptions = { ...options };
const oldFieldValue = selectedDataSource?.['options']?.[field];
if (oldFieldValue) {
@ -425,6 +464,7 @@ const DynamicFormV2 = ({
optionsChanged({ ...newOptions });
}
}
setComputedProps({
...computedProps,
[field]: {
@ -511,6 +551,7 @@ const DynamicFormV2 = ({
dataCy={uiProperties[key].key.replace(/_/g, '-')}
//to be removed after whole ui is same
isHorizontalLayout={isHorizontalLayout}
handleEncryptedFieldsToggle={handleEncryptedFieldsToggle}
/>
</div>
</div>

12
frontend/src/_ui/Input-V3/index.js
View file

@ -6,7 +6,7 @@ import { toast } from 'react-hot-toast';
import InputComponent from '@/components/ui/Input/Index';
const InputV3 = ({ helpText, ...props }) => {
const { workspaceVariables, workspaceConstants, value, widget, disabled, encrypted } = props;
const { workspaceVariables, workspaceConstants, value, widget, encrypted, onBlur } = props;
const [isFocused, setIsFocused] = useState(false);
const [isCopied, setIsCopied] = useState(false);
@ -37,6 +37,11 @@ const InputV3 = ({ helpText, ...props }) => {
<InputComponent
{...props}
value={value}
onFocus={() => setIsFocused(true)}
onBlur={(event) => {
setIsFocused(false);
onBlur(event);
}}
styles="tw-bg-transparent"
label={props.label}
placeholder={props.placeholder}
@ -49,6 +54,11 @@ const InputV3 = ({ helpText, ...props }) => {
{...props}
type="password"
value={value}
onFocus={() => setIsFocused(true)}
onBlur={(event) => {
setIsFocused(false);
onBlur(event);
}}
styles="tw-bg-transparent"
label={props.label}
placeholder={props.placeholder}

35
frontend/src/_ui/Input/index.js
View file

@ -5,20 +5,21 @@ import SolidIcon from '../Icon/SolidIcons';
import { toast } from 'react-hot-toast';
const Input = ({ helpText, onBlur, ...props }) => {
const { workspaceVariables, workspaceConstants, value, type, disabled, encrypted } = props;
const { workspaceVariables, workspaceConstants, value, type, disabled, encrypted, isWorkspaceConstant } = props;
const [isFocused, setIsFocused] = useState(false);
const [isCopied, setIsCopied] = useState(false);
const [showPasswordProps, setShowPasswordProps] = useState({
inputType: type,
iconType: 'eyedisable',
});
const [showPassword, setShowPassword] = useState(false);
const inputType = type === 'password' || encrypted ? (showPassword ? 'text' : 'password') : type;
const iconType = showPassword ? 'eye' : 'eyedisable';
useEffect(() => {
if (isWorkspaceConstant) {
setShowPassword(true);
}
}, [isWorkspaceConstant]);
const toggleShowPassword = () => {
if (inputType !== 'text') {
setShowPasswordProps({ inputType: 'text', iconType: 'eye' });
} else {
setShowPasswordProps({ inputType: 'password', iconType: 'eyedisable' });
}
setShowPassword(!showPassword);
};
const handleCopyToClipboard = async () => {
@ -36,12 +37,6 @@ const Input = ({ helpText, onBlur, ...props }) => {
}
};
useEffect(() => {
if (disabled && encrypted) setShowPasswordProps({ inputType: 'password', iconType: 'eyedisable' });
}, [disabled]);
const { inputType, iconType } = showPasswordProps;
return (
<div className="tj-app-input">
<div
@ -57,8 +52,10 @@ const Input = ({ helpText, onBlur, ...props }) => {
}}
/>
{(type === 'password' || encrypted) && (
<div onClick={!disabled && toggleShowPassword}>
{' '}
<div
onClick={!disabled ? toggleShowPassword : undefined}
style={{ cursor: !disabled ? 'pointer' : 'default' }}
>
<SolidIcon className="eye-icon" name={iconType} />
</div>
)}
@ -66,12 +63,10 @@ const Input = ({ helpText, onBlur, ...props }) => {
value &&
(!isCopied ? (
<div style={{ cursor: 'pointer' }} onClick={handleCopyToClipboard}>
{' '}
<SolidIcon className="copy-icon" name="copy" />
</div>
) : (
<div style={{ color: 'green' }}>
{' '}
<span>Copied!</span>
</div>
))}

29
frontend/src/components/ui/Input/CommonInput/Index.jsx
View file

@ -6,14 +6,26 @@ import { ButtonSolid } from '../../../../_components/AppButton';
import { generateCypressDataCy } from '../../../../modules/common/helpers/cypressHelpers.js';
const CommonInput = ({ label, helperText, disabled, required, onChange: change, ...restProps }) => {
const { type, encrypted, validation, isValidatedMessages, isDisabled } = restProps;
const {
propertyKey,
type,
encrypted,
validation,
isValidatedMessages,
isDisabled,
isEditing,
handleEncryptedFieldsToggle,
labelDisabled,
} = restProps;
const InputComponentType = type === 'number' ? NumberInput : TextInput;
const [isValid, setIsValid] = useState(null);
const [message, setMessage] = useState('');
const [isEditing, setIsEditing] = useState(false);
const isEncrypted = type === 'password' || encrypted;
const isWorkspaceConstant =
restProps.placeholder &&
(restProps.placeholder.includes('{{constants') || restProps.placeholder.includes('{{secrets'));
const handleChange = (e) => {
if (validation) {
@ -39,20 +51,12 @@ const CommonInput = ({ label, helperText, disabled, required, onChange: change,
}
}, [isValid, isValidatedMessages]);
const toggleEditing = () => {
if (isDisabled) return;
const willBeInEditMode = !isEditing;
setIsEditing(willBeInEditMode);
change({ target: { value: '' } });
};
return (
<div>
<div className="d-flex">
{label && (
<div className="tw-flex-shrink-0">
<InputLabel disabled={disabled} label={label} required={required} />
<InputLabel disabled={labelDisabled ?? disabled} label={label} required={required} />
</div>
)}
{type === 'password' && (
@ -65,7 +69,7 @@ const CommonInput = ({ label, helperText, disabled, required, onChange: change,
target="_blank"
rel="noreferrer"
disabled={isDisabled}
onClick={toggleEditing}
onClick={(e) => handleEncryptedFieldsToggle(e, propertyKey)}
data-cy={`button-${generateCypressDataCy(isEditing ? 'Cancel' : 'Edit')}`}
>
{isEditing ? 'Cancel' : 'Edit'}
@ -86,6 +90,7 @@ const CommonInput = ({ label, helperText, disabled, required, onChange: change,
required={required}
response={isValid}
onChange={handleChange}
isWorkspaceConstant={isWorkspaceConstant}
{...restProps}
/>
{helperText && (

101
frontend/src/components/ui/Input/Input.jsx
View file

@ -2,56 +2,65 @@ import * as React from 'react';
import { cn } from '@/lib/utils';
import { inputVariants } from './InputUtils/Variants';
import SolidIcon from '../../../_ui/Icon/SolidIcons';
import { useEffect } from 'react';
const Input = React.forwardRef(({ className, size, type, multiline, response, rows = 3, ...props }, ref) => {
const [isPasswordVisible, setIsPasswordVisible] = React.useState(false);
const isPasswordField = type === 'password';
const Input = React.forwardRef(
({ className, size, type, multiline, response, isWorkspaceConstant, rows = 3, ...props }, ref) => {
const [isPasswordVisible, setIsPasswordVisible] = React.useState(false);
const isPasswordField = type === 'password';
const togglePasswordVisibility = () => {
if (!props.disabled) {
setIsPasswordVisible((prev) => !prev);
}
};
const togglePasswordVisibility = () => {
if (!props.disabled) {
setIsPasswordVisible((prev) => !prev);
}
};
const validationClass = response === true ? 'valid-textarea' : response === false ? 'invalid-textarea' : '';
useEffect(() => {
if (isWorkspaceConstant) {
setIsPasswordVisible(true);
}
}, [isWorkspaceConstant]);
return (
<div className="design-component-inputs">
{multiline ? (
<textarea
className={cn(
`tw-relative tw-peer tw-flex tw-text-[12px]/[18px] tw-w-full tw-rounded-[8px] tw-border-[1px] tw-border-solid tw-bg-background-surface-layer-01 tw-py-[7px] tw-text-text-default focus-visible:tw-ring-[1px] focus-visible:tw-ring-offset-[1px] focus-visible:tw-ring-border-accent-strong focus-visible:tw-ring-offset-border-accent-strong focus-visible:tw-border-transparent disabled:tw-cursor-not-allowed ${props.styles}`,
className,
validationClass
)}
rows={rows}
ref={ref}
{...props}
/>
) : (
<input
type={isPasswordField && isPasswordVisible ? 'text' : type}
className={cn(
inputVariants({ size }),
`tw-peer tw-flex tw-text-[12px]/[18px] tw-w-full tw-rounded-[8px] tw-border-[1px] tw-border-solid tw-bg-background-surface-layer-01 tw-py-[7px] tw-text-text-default focus-visible:tw-ring-[1px] focus-visible:tw-ring-offset-[1px] focus-visible:tw-ring-border-accent-strong focus-visible:tw-ring-offset-border-accent-strong focus-visible:tw-border-transparent disabled:tw-cursor-not-allowed ${props.styles}`,
className
)}
ref={ref}
{...props}
/>
)}
{isPasswordField && !multiline && (
<div onClick={togglePasswordVisibility}>
{isPasswordVisible ? (
<SolidIcon className="eye-icon" name="eye" />
) : (
<SolidIcon className="eye-icon" name="eyedisable" />
)}
</div>
)}
</div>
);
});
const validationClass = response === true ? 'valid-textarea' : response === false ? 'invalid-textarea' : '';
return (
<div className="design-component-inputs">
{multiline ? (
<textarea
className={cn(
`tw-relative tw-peer tw-flex tw-text-[12px]/[18px] tw-w-full tw-rounded-[8px] tw-border-[1px] tw-border-solid tw-bg-background-surface-layer-01 tw-py-[7px] tw-text-text-default focus-visible:tw-ring-[1px] focus-visible:tw-ring-offset-[1px] focus-visible:tw-ring-border-accent-strong focus-visible:tw-ring-offset-border-accent-strong focus-visible:tw-border-transparent disabled:tw-cursor-not-allowed ${props.styles}`,
className,
validationClass
)}
rows={rows}
ref={ref}
{...props}
/>
) : (
<input
type={isPasswordField && isPasswordVisible ? 'text' : type}
className={cn(
inputVariants({ size }),
`tw-relative tw-peer tw-flex tw-text-[12px]/[18px] tw-w-full tw-rounded-[8px] tw-border-[1px] tw-border-solid tw-bg-background-surface-layer-01 tw-py-[7px] tw-text-text-default focus-visible:tw-ring-[1px] focus-visible:tw-ring-offset-[1px] focus-visible:tw-ring-border-accent-strong focus-visible:tw-ring-offset-border-accent-strong focus-visible:tw-border-transparent disabled:tw-cursor-not-allowed ${props.styles}`,
className
)}
ref={ref}
{...props}
/>
)}
{isPasswordField && !multiline && (
<div onClick={togglePasswordVisibility}>
{isPasswordVisible ? (
<SolidIcon className="eye-icon" name="eye" />
) : (
<SolidIcon className="eye-icon" name="eyedisable" />
)}
</div>
)}
</div>
);
}
);
Input.displayName = 'Input';
export { Input };

7
frontend/src/modules/dataSources/components/DataSourceManager/DataSourceManager.jsx
View file

@ -117,6 +117,9 @@ class DataSourceManagerComponent extends React.Component {
selectedDataSourceIcon: this.props.selectedDataSource?.plugin?.iconFile?.data,
connectionTestError: null,
datasourceName: this.props.selectedDataSource?.name,
validationMessages: {},
validationError: [],
showValidationErrors: false,
});
}
}
@ -146,6 +149,9 @@ class DataSourceManagerComponent extends React.Component {
dataSourceSchema: source.manifestFile?.data,
selectedDataSourcePluginId: source.id,
datasourceName: source.name,
validationMessages: {},
validationError: [],
showValidationErrors: false,
},
() => this.createDataSource()
);
@ -413,6 +419,7 @@ class DataSourceManagerComponent extends React.Component {
const ComponentToRender = isPlugin ? SourceComponent : SourceComponents[sourceComponentName] || SourceComponent;
return (
<ComponentToRender
key={this.state.selectedDataSource?.id}
dataSourceSchema={this.state.dataSourceSchema}
optionsChanged={(options = {}) => this.setState({ options })}
optionchanged={this.optionchanged}

6
marketplace/plugins/qdrant/lib/manifest.json
View file

@ -13,8 +13,7 @@
},
"options": {
"url": {
"type": "string",
"encrypted": false
"type": "string"
},
"apiKey": {
"type": "string",
@ -29,8 +28,7 @@
"key": "url",
"type": "text",
"description": "Enter your Qdrant URL.",
"helpText": "<a href='https://qdrant.tech/documentation/quickstart-cloud/#authenticate-via-sdks' target='_blank' rel='noreferrer'>REST URL</a> to authenticate the requests of the Qdrant instance.",
"encrypted": true
"helpText": "<a href='https://qdrant.tech/documentation/quickstart-cloud/#authenticate-via-sdks' target='_blank' rel='noreferrer'>REST URL</a> to authenticate the requests of the Qdrant instance."
},
"apiKey": {
"label": "API Key",

4
plugins/packages/common/lib/index.ts
View file

@ -6,6 +6,8 @@ import { QueryService } from './query_service.interface';
import {
isEmpty,
cacheConnection,
cacheConnectionWithConfiguration,
generateSourceOptionsHash,
getCachedConnection,
parseJson,
cleanSensitiveData,
@ -37,6 +39,8 @@ export {
User,
App,
cacheConnection,
generateSourceOptionsHash,
cacheConnectionWithConfiguration,
getCachedConnection,
parseJson,
isEmpty,

26
plugins/packages/common/lib/utils.helper.ts
View file

@ -1,6 +1,7 @@
import { QueryError } from './query.error';
import * as tls from 'tls';
import { readFileSync } from 'fs';
import crypto from 'crypto';
const CACHED_CONNECTIONS: any = {};
@ -17,8 +18,29 @@ export function cacheConnection(dataSourceId: string, connection: any): any {
CACHED_CONNECTIONS[dataSourceId] = { connection, updatedAt };
}
export function getCachedConnection(dataSourceId: string | number, dataSourceUpdatedAt: any): any {
const cachedData = CACHED_CONNECTIONS[dataSourceId];
export function generateSourceOptionsHash(sourceOptions) {
const sortedEntries = Object.entries(sourceOptions)
.filter(([_, value]) => value !== undefined && value !== null && value !== '')
.sort(([a], [b]) => a.localeCompare(b))
.map(([key, value]) => `${key}:${value}`)
.join('|');
return crypto.createHash('sha256').update(sortedEntries).digest('hex').substring(0, 16);
}
export function cacheConnectionWithConfiguration(dataSourceId: string, enhancedCacheKey: string, connection: any): any {
const updatedAt = new Date();
const allKeys = Object.keys(CACHED_CONNECTIONS);
const oldKeysForThisDatasource = allKeys.filter(
(key) => key.startsWith(`${dataSourceId}_`) && key !== enhancedCacheKey
);
oldKeysForThisDatasource.forEach((oldKey) => delete CACHED_CONNECTIONS[oldKey]);
CACHED_CONNECTIONS[enhancedCacheKey] = { connection, updatedAt };
}
export function getCachedConnection(cacheKey: string | number, dataSourceUpdatedAt: any): any {
const cachedData = CACHED_CONNECTIONS[cacheKey];
if (cachedData) {
const updatedAt = new Date(dataSourceUpdatedAt || null);

9
plugins/packages/mssql/lib/index.ts
View file

@ -4,7 +4,8 @@ import {
QueryError,
QueryResult,
QueryService,
cacheConnection,
cacheConnectionWithConfiguration,
generateSourceOptionsHash,
getCachedConnection,
} from '@tooljet-plugins/common';
import { SourceOptions, QueryOptions } from './types';
@ -143,13 +144,15 @@ export default class MssqlQueryService implements QueryService {
dataSourceUpdatedAt?: string
): Promise<Knex> {
if (checkCache) {
let connection = await getCachedConnection(dataSourceId, dataSourceUpdatedAt);
const optionsHash = generateSourceOptionsHash(sourceOptions);
const enhancedCacheKey = `${dataSourceId}_${optionsHash}`;
let connection = await getCachedConnection(enhancedCacheKey, dataSourceUpdatedAt);
if (connection) {
return connection;
} else {
connection = await this.buildConnection(sourceOptions);
dataSourceId && cacheConnection(dataSourceId, connection);
cacheConnectionWithConfiguration(dataSourceId, enhancedCacheKey, connection);
return connection;
}
} else {

15
plugins/packages/mysql/lib/index.ts
View file

@ -1,6 +1,7 @@
import knex, { Knex } from 'knex';
import {
cacheConnection,
cacheConnectionWithConfiguration,
generateSourceOptionsHash,
getCachedConnection,
ConnectionTestResult,
QueryService,
@ -145,13 +146,17 @@ export default class MysqlQueryService implements QueryService {
dataSourceUpdatedAt?: string
): Promise<Knex> {
if (checkCache) {
const cachedConnection = await getCachedConnection(dataSourceId, dataSourceUpdatedAt);
const optionsHash = generateSourceOptionsHash(sourceOptions);
const enhancedCacheKey = `${dataSourceId}_${optionsHash}`;
const cachedConnection = await getCachedConnection(enhancedCacheKey, dataSourceUpdatedAt);
if (cachedConnection) return cachedConnection;
const connection = await this.buildConnection(sourceOptions);
cacheConnectionWithConfiguration(dataSourceId, enhancedCacheKey, connection);
return connection;
}
const connection = await this.buildConnection(sourceOptions);
if (checkCache && dataSourceId) cacheConnection(dataSourceId, connection);
return connection;
return await this.buildConnection(sourceOptions);
}
buildBulkUpdateQuery(queryOptions: QueryOptions): string {

9
plugins/packages/oracledb/lib/index.ts
View file

@ -1,7 +1,8 @@
import { Knex, knex } from 'knex';
import oracledb from 'oracledb';
import {
cacheConnection,
cacheConnectionWithConfiguration,
generateSourceOptionsHash,
getCachedConnection,
ConnectionTestResult,
QueryService,
@ -118,13 +119,15 @@ export default class OracledbQueryService implements QueryService {
dataSourceUpdatedAt?: string
): Promise<any> {
if (checkCache) {
let connection = await getCachedConnection(dataSourceId, dataSourceUpdatedAt);
const optionsHash = generateSourceOptionsHash(sourceOptions);
const enhancedCacheKey = `${dataSourceId}_${optionsHash}`;
let connection = await getCachedConnection(enhancedCacheKey, dataSourceUpdatedAt);
if (connection) {
return connection;
} else {
connection = await this.buildConnection(sourceOptions);
dataSourceId && cacheConnection(dataSourceId, connection);
cacheConnectionWithConfiguration(dataSourceId, enhancedCacheKey, connection);
return connection;
}
} else {

15
plugins/packages/postgresql/lib/index.ts
View file

@ -1,6 +1,7 @@
import {
ConnectionTestResult,
cacheConnection,
cacheConnectionWithConfiguration,
generateSourceOptionsHash,
getCachedConnection,
QueryService,
QueryResult,
@ -145,13 +146,17 @@ export default class PostgresqlQueryService implements QueryService {
dataSourceUpdatedAt?: string
): Promise<Knex> {
if (checkCache) {
const cachedConnection = await getCachedConnection(dataSourceId, dataSourceUpdatedAt);
const optionsHash = generateSourceOptionsHash(sourceOptions);
const enhancedCacheKey = `${dataSourceId}_${optionsHash}`;
const cachedConnection = await getCachedConnection(enhancedCacheKey, dataSourceUpdatedAt);
if (cachedConnection) return cachedConnection;
const connection = await this.buildConnection(sourceOptions);
cacheConnectionWithConfiguration(dataSourceId, enhancedCacheKey, connection);
return connection;
}
const connection = await this.buildConnection(sourceOptions);
if (checkCache && dataSourceId) cacheConnection(dataSourceId, connection);
return connection;
return await this.buildConnection(sourceOptions);
}
buildBulkUpdateQuery(queryOptions: QueryOptions): string {

9
plugins/packages/snowflake/lib/index.ts
View file

@ -3,7 +3,8 @@ import {
QueryResult,
QueryService,
ConnectionTestResult,
cacheConnection,
cacheConnectionWithConfiguration,
generateSourceOptionsHash,
getCachedConnection,
} from '@tooljet-plugins/common';
import { SourceOptions, QueryOptions } from './types';
@ -93,13 +94,15 @@ export default class Snowflake implements QueryService {
dataSourceUpdatedAt?: string
): Promise<any> {
if (checkCache) {
let connection = await getCachedConnection(dataSourceId, dataSourceUpdatedAt);
const optionsHash = generateSourceOptionsHash(sourceOptions);
const enhancedCacheKey = `${dataSourceId}_${optionsHash}`;
let connection = await getCachedConnection(enhancedCacheKey, dataSourceUpdatedAt);
if (connection && (await connection.isValidAsync())) {
return connection;
} else {
connection = await this.buildConnection(sourceOptions);
await cacheConnection(dataSourceId, connection);
cacheConnectionWithConfiguration(dataSourceId, enhancedCacheKey, connection);
return connection;
}
} else {

2
server/src/modules/app/module.ts
View file

@ -56,6 +56,7 @@ import { SessionScheduler } from '@modules/session/scheduler';
import { AuditLogsClearScheduler } from '@modules/audit-logs/scheduler';
import { ModulesModule } from '@modules/modules/module';
import { EmailListenerModule } from '@modules/email-listener/module';
import { InMemoryCacheModule } from '@modules/inMemoryCache/module';
export class AppModule implements OnModuleInit {
static async register(configs: { IS_GET_CONTEXT: boolean }): Promise<DynamicModule> {
// Load static and dynamic modules
@ -116,6 +117,7 @@ export class AppModule implements OnModuleInit {
await CrmModule.register(configs),
await OrganizationPaymentModule.register(configs),
await EmailListenerModule.register(configs),
await InMemoryCacheModule.register(configs),
];
const conditionalImports = [];

1
server/src/modules/apps/services/app-import-export.service.ts
View file

@ -1897,6 +1897,7 @@ export class AppImportExportService {
key: key,
value: options[key]['value'],
encrypted: options[key]['encrypted'],
workspace_constant: options[key]['workspace_constant'],
};
});
}

2
server/src/modules/data-sources/module.ts
View file

@ -12,6 +12,7 @@ import { TooljetDbModule } from '@modules/tooljet-db/module';
import { OrganizationRepository } from '@modules/organizations/repository';
import { SessionModule } from '@modules/session/module';
import { SubModule } from '@modules/app/sub-module';
import { InMemoryCacheModule } from '@modules/inMemoryCache/module';
export class DataSourcesModule extends SubModule {
static async register(configs?: { IS_GET_CONTEXT: boolean }): Promise<DynamicModule> {
@ -40,6 +41,7 @@ export class DataSourcesModule extends SubModule {
await InstanceSettingsModule.register(configs),
await TooljetDbModule.register(configs),
await SessionModule.register(configs),
await InMemoryCacheModule.register(configs),
],
providers: [
DataSourcesService,

207
server/src/modules/data-sources/util.service.ts
View file

@ -21,6 +21,7 @@ import { PluginsServiceSelector } from './services/plugin-selector.service';
import { OrganizationConstantsUtilService } from '@modules/organization-constants/util.service';
import { DataSourceOptions } from '@entities/data_source_options.entity';
import { IDataSourcesUtilService } from './interfaces/IUtilService';
import { InMemoryCacheService } from '@modules/inMemoryCache/in-memory-cache.service';
@Injectable()
export class DataSourcesUtilService implements IDataSourcesUtilService {
@ -31,7 +32,8 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
protected readonly licenseTermsService: LicenseTermsService,
protected readonly encryptionService: EncryptionService,
protected readonly pluginsServiceSelector: PluginsServiceSelector,
protected readonly organizationConstantsUtilService: OrganizationConstantsUtilService
protected readonly organizationConstantsUtilService: OrganizationConstantsUtilService,
protected readonly inMemoryCacheService: InMemoryCacheService
) {}
async create(createArgumentsDto: CreateArgumentsDto, user: User): Promise<DataSource> {
return await dbTransactionWrap(async (manager: EntityManager) => {
@ -103,15 +105,25 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
for (const option of optionsWithOauth) {
if (option['encrypted']) {
const credential = await this.credentialService.create(
resetSecureData ? '' : option['value'] || '',
entityManager
);
if (option['workspace_constant']) {
const credential = await this.credentialService.create(option['workspace_constant'], entityManager);
parsedOptions[option['key']] = {
credential_id: credential.id,
encrypted: option['encrypted'],
};
parsedOptions[option['key']] = {
credential_id: credential.id,
workspace_constant: option['workspace_constant'],
encrypted: option['encrypted'],
};
} else {
const credential = await this.credentialService.create(
resetSecureData ? '' : option['value'] || '',
entityManager
);
parsedOptions[option['key']] = {
credential_id: credential.id,
encrypted: option['encrypted'],
};
}
} else {
parsedOptions[option['key']] = {
value: option['value'],
@ -135,7 +147,17 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
const queryService = await this.pluginsServiceSelector.getService(plugin_id, provider);
// const queryService = new allPlugins[provider]();
const accessDetails = await queryService.accessDetailsFrom(authCode, options, resetSecureData);
let accessDetailsPromise: Promise<any>;
const cacheKey = `${provider}_${authCode}`;
if (this.inMemoryCacheService.has(cacheKey)) {
accessDetailsPromise = this.inMemoryCacheService.get(cacheKey);
} else {
accessDetailsPromise = queryService.accessDetailsFrom(authCode, options, resetSecureData);
this.inMemoryCacheService.set(cacheKey, accessDetailsPromise);
}
const accessDetails = await accessDetailsPromise;
for (const row of accessDetails) {
const option = {};
@ -165,52 +187,58 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
throw new BadRequestException('Cannot update configuration of sample data source');
}
await dbTransactionWrap(async (manager: EntityManager) => {
const isMultiEnvEnabled = await this.licenseTermsService.getLicenseTerms(LICENSE_FIELD.MULTI_ENVIRONMENT, organizationId);
const envToUpdate = await this.appEnvironmentUtilService.get(organizationId, environmentId, false, manager);
try {
await dbTransactionWrap(async (manager: EntityManager) => {
const isMultiEnvEnabled = await this.licenseTermsService.getLicenseTerms(
LICENSE_FIELD.MULTI_ENVIRONMENT,
organizationId
);
const envToUpdate = await this.appEnvironmentUtilService.get(organizationId, environmentId, false, manager);
// if datasource is restapi then reset the token data
if (dataSource.kind === 'restapi')
options.push({
key: 'tokenData',
value: undefined,
encrypted: false,
});
// if datasource is restapi then reset the token data
if (dataSource.kind === 'restapi')
options.push({
key: 'tokenData',
value: undefined,
encrypted: false,
});
if (isMultiEnvEnabled) {
dataSource.options = (
await this.appEnvironmentUtilService.getOptions(dataSourceId, organizationId, envToUpdate.id)
).options;
const newOptions = await this.parseOptionsForUpdate(dataSource, options, manager);
await this.appEnvironmentUtilService.updateOptions(newOptions, envToUpdate.id, dataSource.id, manager);
} else {
const allEnvs = await this.appEnvironmentUtilService.getAll(organizationId);
/*
Basic plan customer. lets update all environment options.
this will help us to run the queries successfully when the user buys enterprise plan
*/
const newOptions = await this.parseOptionsForUpdate(dataSource, options, manager);
for (const env of allEnvs) {
if (isMultiEnvEnabled) {
dataSource.options = (
await this.appEnvironmentUtilService.getOptions(dataSourceId, organizationId, env.id)
await this.appEnvironmentUtilService.getOptions(dataSourceId, organizationId, envToUpdate.id)
).options;
await this.appEnvironmentUtilService.updateOptions(newOptions, env.id, dataSource.id, manager);
const newOptions = await this.parseOptionsForUpdate(dataSource, options, manager);
await this.appEnvironmentUtilService.updateOptions(newOptions, envToUpdate.id, dataSource.id, manager);
} else {
const allEnvs = await this.appEnvironmentUtilService.getAll(organizationId);
/*
Basic plan customer. lets update all environment options.
this will help us to run the queries successfully when the user buys enterprise plan
*/
for (const env of allEnvs) {
dataSource.options = (
await this.appEnvironmentUtilService.getOptions(dataSourceId, organizationId, env.id)
).options;
const newOptions = await this.parseOptionsForUpdate(dataSource, options, manager);
await this.appEnvironmentUtilService.updateOptions(newOptions, env.id, dataSource.id, manager);
}
}
}
const updatableParams = {
id: dataSourceId,
name,
updatedAt: new Date(),
};
const updatableParams = {
id: dataSourceId,
name,
updatedAt: new Date(),
};
// Remove keys with undefined values
cleanObject(updatableParams);
// Remove keys with undefined values
cleanObject(updatableParams);
await manager.save(DataSource, updatableParams);
});
await manager.save(DataSource, updatableParams);
});
} finally {
this.inMemoryCacheService.clear();
}
}
async decrypt(options: Record<string, any>) {
@ -233,33 +261,67 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
const optionsWithOauth = await this.parseOptionsForOauthDataSource(options);
const parsedOptions = {};
if (dataSource?.options) {
for (const key in dataSource.options) {
if (dataSource.options[key]?.workspace_constant) {
parsedOptions[key] = {
workspace_constant: dataSource.options[key].workspace_constant,
credential_id: dataSource.options[key].credential_id,
encrypted: dataSource.options[key].encrypted,
};
}
}
}
return await dbTransactionWrap(async (entityManager: EntityManager) => {
for (const option of optionsWithOauth) {
const key = option['key'];
const credentialValue = option['value'];
if (option['encrypted']) {
const existingCredentialId =
dataSource?.options &&
dataSource.options[option['key']] &&
dataSource.options[option['key']]['credential_id'];
dataSource?.options && dataSource.options[key] && dataSource.options[key]['credential_id'];
if (credentialValue && (credentialValue.includes('{{constants') || credentialValue.includes('{{secrets'))) {
if (!parsedOptions[key]) {
parsedOptions[key] = {};
}
parsedOptions[key].workspace_constant = credentialValue;
} else {
if (
existingCredentialId &&
credentialValue !== undefined &&
credentialValue !== (await this.credentialService.getValue(existingCredentialId))
) {
if (parsedOptions[key]) {
delete parsedOptions[key].workspace_constant;
}
}
}
if (existingCredentialId) {
(option['value'] || option['value'] === '') &&
(await this.credentialService.update(existingCredentialId, option['value'] || ''));
if (credentialValue !== undefined) {
await this.credentialService.update(existingCredentialId, credentialValue || '');
}
parsedOptions[option['key']] = {
credential_id: existingCredentialId,
encrypted: option['encrypted'],
};
if (!parsedOptions[key]) {
parsedOptions[key] = {};
}
parsedOptions[key].credential_id = existingCredentialId;
parsedOptions[key].encrypted = option['encrypted'];
} else {
const credential = await this.credentialService.create(option['value'] || '', entityManager);
const credential = await this.credentialService.create(credentialValue || '', entityManager);
parsedOptions[option['key']] = {
credential_id: credential.id,
encrypted: option['encrypted'],
};
if (!parsedOptions[key]) {
parsedOptions[key] = {};
}
parsedOptions[key].credential_id = credential.id;
parsedOptions[key].encrypted = option['encrypted'];
}
} else {
parsedOptions[option['key']] = {
value: option['value'],
parsedOptions[key] = {
value: credentialValue,
encrypted: false,
};
}
@ -412,6 +474,7 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
const credentialId = parsedOptions[key]?.['credential_id'];
if (credentialId) {
const encryptedKeyValue = await this.credentialService.getValue(credentialId);
constantMatcher.lastIndex = 0;
//check if encrypted key value is a constant
if (constantMatcher.test(encryptedKeyValue)) {
@ -485,7 +548,10 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
const envToUpdate = await this.appEnvironmentUtilService.get(organizationId, environmentId, false, manager);
const oldOptions = dataSource.options || {};
const updatedOptions = { ...oldOptions, ...parsedOptions };
const isMultiEnvEnabled = await this.licenseTermsService.getLicenseTerms(LICENSE_FIELD.MULTI_ENVIRONMENT,organizationId);
const isMultiEnvEnabled = await this.licenseTermsService.getLicenseTerms(
LICENSE_FIELD.MULTI_ENVIRONMENT,
organizationId
);
if (isMultiEnvEnabled) {
await this.appEnvironmentUtilService.updateOptions(updatedOptions, envToUpdate.id, dataSourceId, manager);
@ -581,7 +647,7 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
let errorObj = {};
try {
errorObj = JSON.parse(error);
} catch (err) {
} catch (error) {
errorObj['error_details'] = error;
}
@ -612,6 +678,7 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
for (const key of Object.keys(options)) {
const currentOption = options[key]?.['value'];
constantMatcher.lastIndex = 0;
//! request options are nested arrays with constants and variables
if (Array.isArray(currentOption)) {
@ -690,9 +757,9 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
if (existingAccessTokenCredentialId) {
await this.credentialService.update(existingAccessTokenCredentialId, accessTokenDetails['access_token']);
existingRefreshTokenCredentialId &&
accessTokenDetails['refresh_token'] &&
(await this.credentialService.update(existingRefreshTokenCredentialId, accessTokenDetails['refresh_token']));
if (existingRefreshTokenCredentialId && accessTokenDetails['refresh_token']) {
await this.credentialService.update(existingRefreshTokenCredentialId, accessTokenDetails['refresh_token']);
}
} else if (dataSourceId) {
const isMultiAuthEnabled = dataSourceOptions['multiple_auth_enabled']?.value;
const updatedTokenData = this.changeCurrentToken(

23
server/src/modules/inMemoryCache/in-memory-cache.service.ts Normal file
View file

@ -0,0 +1,23 @@
import { Injectable } from '@nestjs/common';
import { ICacheService } from './interfaces/IUtilService';
@Injectable()
export class InMemoryCacheService implements ICacheService {
private static cacheStore: Map<string, Promise<any>> = new Map();
set(key: string, value: Promise<any>): void {
InMemoryCacheService.cacheStore.set(key, value);
}
get(key: string): Promise<any> | undefined {
return InMemoryCacheService.cacheStore.get(key);
}
has(key: string): boolean {
return InMemoryCacheService.cacheStore.has(key);
}
clear(): void {
InMemoryCacheService.cacheStore.clear();
}
}

27
server/src/modules/inMemoryCache/interfaces/IUtilService.ts Normal file
View file

@ -0,0 +1,27 @@
export interface ICacheService {
/**
* Store a promise value in the cache with the given key
* @param key - The cache key
* @param value - The promise to cache
*/
set(key: string, value: Promise<any>): void;
/**
* Retrieve a cached promise by key
* @param key - The cache key
* @returns The cached promise or undefined if not found
*/
get(key: string): Promise<any> | undefined;
/**
* Check if a key exists in the cache
* @param key - The cache key
* @returns True if the key exists, false otherwise
*/
has(key: string): boolean;
/**
* Clear all cached entries
*/
clear(): void;
}

15
server/src/modules/inMemoryCache/module.ts Normal file
View file

@ -0,0 +1,15 @@
import { SubModule } from '@modules/app/sub-module';
import { DynamicModule } from '@nestjs/common';
export class InMemoryCacheModule extends SubModule {
static async register(configs?: { IS_GET_CONTEXT: boolean }): Promise<DynamicModule> {
const { InMemoryCacheService } = await this.getProviders(configs, 'inMemoryCache', ['in-memory-cache.service']);
return {
module: InMemoryCacheModule,
controllers: [],
providers: [InMemoryCacheService],
exports: [InMemoryCacheService],
};
}
}