Elgato_dark/ToolJet
1
0
Fork 0
mirror of https://github.com/ToolJet/ToolJet synced 2026-05-23 08:58:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Merge pull request #12708 from ToolJet/release/marketplace-sprint-10

Browse source 
Release: Marketplace Sprint 10

🚀 Features
- Added feature to add bulk upsert using PK in #12325 by @manishkushare
- Added feature to set default value to null while creating or editing FK column in #12704 by @manishkushare

🌟 Enhancements
- Enhanced plugin schema for validation and design component in #12655 by @parthy007
- Enhanced schema revamp for MySQL and MSSQL in #12661 by @parthy007
- Added metadata to QueryError and populated metadata in inspector on error in #12501 by @thesynthax
- Improved empty plugin state copywriting in #12588 by @thesynthax
- Added validation for encrypted fields with input reference on failure in #12541 by @parthy007

🛠️ Fixes
- Fix plugin copywriting and icon issues in #12664 by @parthy007
- Fix empty URL parameters in the GraphQL plugin in #12353 by @ganesh8056
- Fixed ToolJet database JSON column values not persisting in query builder in #12438 by @ganesh8056
This commit is contained in:
Akshay 2025-04-30 21:38:49 +05:30 committed by GitHub
commit dfad7a55f8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
57 changed files with 1668 additions and 714 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.version
View file

@ -1 +1 @@
3.11.0
3.12.0

0
cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/airTableHappyPath.cy.js → cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/airTableHappyPath.cy.skip.js
View file

0
cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/bigqueryHappyPath.cy.js → cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/bigqueryHappyPath.cy.skip.js
View file

0
cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/mongoDbHappyPath.cy.js → cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/mongoDbHappyPath.cy.skip.js
View file

0
cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/mysqlHappyPath.cy.js → cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/mysqlHappyPath.cy.skip.js
View file

0
cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/sqlServerHappyPath.cy.js → cypress-tests/cypress/e2e/happyPath/marketplace/commonTestcases/data-source/sqlServerHappyPath.cy.skip.js
View file

2
frontend/.version
View file

@ -1 +1 @@
3.11.0
3.12.0

6
frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/BulkUploadPrimaryKey.jsx
View file

@ -53,7 +53,11 @@ export const BulkUploadPrimaryKey = () => {
<div className="field flex-grow-1 minw-400-w-400">
<CodeHinter
type="basic"
initialValue={`{{${JSON.stringify(bulkUpdatePrimaryKey?.rows_update ?? [])}}}`}
initialValue={
bulkUpdatePrimaryKey?.rows_update
? `{{${JSON.stringify(bulkUpdatePrimaryKey?.rows_update ?? [])}}}`
: null
}
className="codehinter-plugins"
placeholder="{{ [ { 'column1': 'value', ... } ] }}"
onChange={(newValue) => {

78
frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/BulkUpsertPrimaryKey.jsx Normal file
View file

@ -0,0 +1,78 @@
import React, { useContext, useEffect } from 'react';
import { TooljetDatabaseContext } from '@/TooljetDatabase/index';
import { resolveReferences } from '@/AppBuilder/CodeEditor/utils';
import CodeHinter from '@/AppBuilder/CodeEditor';
export const BulkUpsertPrimaryKey = () => {
const {
columns,
bulkUpsertPrimaryKey,
handleBulkUpsertRowsOptionChanged,
handlePrimaryKeyOptionChangedForBulkUpsert,
} = useContext(TooljetDatabaseContext);
useEffect(() => {
const primaryKeys = columns.reduce((acc, column) => {
if (column?.keytype === 'PRIMARY KEY' || column?.isPrimaryKey) {
acc.push(column?.accessor);
}
return acc;
}, []);
if (primaryKeys.length > 0) {
handlePrimaryKeyOptionChangedForBulkUpsert(primaryKeys);
}
}, [columns]);
const handleRowsChange = (value) => {
handleBulkUpsertRowsOptionChanged(value);
};
return (
<div className="tab-content-wrapper tj-db-field-wrapper mt-2 d-flex flex-column custom-gap-16">
<div className="field-container d-flex tooljetdb-worflow-operations">
<label className="form-label flex-shrink-0">Primary key</label>
<div
className="field flex-grow-1 minw-400-w-400 px-1"
style={{ height: '28px', background: 'var(--controls-switch-tag)', borderRadius: '6px' }}
>
<input
type="text"
value={bulkUpsertPrimaryKey?.primary_key?.join(', ') || ''}
style={{
width: '100%',
height: '100%',
border: '0',
color: 'var(--text-placeholder)',
background: 'transparent',
}}
disabled
placeholder={''}
/>
</div>
</div>
<div className="field-container d-flex tooljetdb-worflow-operations">
<label className="form-label flex-shrink-0" data-cy="">
Rows to upsert
</label>
<div className="field flex-grow-1 minw-400-w-400">
<CodeHinter
type="basic"
initialValue={
bulkUpsertPrimaryKey?.rows
? typeof bulkUpsertPrimaryKey?.rows === 'string'
? bulkUpsertPrimaryKey?.rows
: JSON.stringify(bulkUpsertPrimaryKey?.rows)
: bulkUpsertPrimaryKey?.rows
}
className="codehinter-plugins"
placeholder="{{ [ { 'column1': 'value', ... } ] }}"
onChange={handleRowsChange}
/>
</div>
</div>
</div>
);
};
export default BulkUpsertPrimaryKey;

4
frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/RenderColumnUI.jsx
View file

@ -1,5 +1,4 @@
import CodeHinter from '@/AppBuilder/CodeEditor';
import { resolveReferences } from '@/Editor/CodeEditor/utils';
import { ButtonSolid } from '@/_ui/AppButton/AppButton';
import Trash from '@/_ui/Icon/solidIcons/Trash';
import React from 'react';
@ -43,8 +42,7 @@ const RenderColumnUI = ({
placeholder="key"
onChange={(newValue) => {
if (isJSonTypeColumn) {
const [_, __, resolvedValue] = resolveReferences(`{{${newValue}}}`);
handleValueChange(resolvedValue);
handleValueChange(newValue);
} else {
handleValueChange(newValue);
}

7
frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/SelectBox.jsx
View file

@ -220,7 +220,9 @@ function DataSourceSelect({
if (isFirstPageLoaded && offset >= totalRecords) return;
if (foreignKeys.length < 1) return;
setIsLoadingFKDetails(true);
const referencedColumns = foreignKeys.find((item) => item.column_names[0] === cellColumnName);
const referencedColumns = Array.isArray(foreignKeys)
? foreignKeys.find((item) => item.column_names[0] === cellColumnName)
: undefined;
if (!referencedColumns?.referenced_column_names?.length) return;
const selectQuery = new PostgrestQueryBuilder();
@ -709,7 +711,8 @@ const MenuList = ({
...props
}) => {
const menuListStyles = getStyles('menuList', props);
const referencedColumnDetails = foreignKeys?.find((item) => item.column_names[0] === cellColumnName);
const referencedColumnDetails =
Array.isArray(foreignKeys) && foreignKeys?.find((item) => item?.column_names[0] === cellColumnName);
const handleNavigateToReferencedTable = () => {
const data = {

22
frontend/src/AppBuilder/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx
View file

@ -16,6 +16,7 @@ import { getPrivateRoute } from '@/_helpers/routes';
import { useNavigate } from 'react-router-dom';
import { deepClone } from '@/_helpers/utilities/utils.helpers';
import { BulkUploadPrimaryKey } from './BulkUploadPrimaryKey';
import BulkUpsertPrimaryKey from './BulkUpsertPrimaryKey';
import './styles.scss';
import CodeHinter from '@/AppBuilder/CodeEditor';
@ -46,6 +47,7 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
const [tableForeignKeyInfo, setTableForeignKeyInfo] = useState({});
const [bulkUpdatePrimaryKey, setBulkUpdatePrimaryKey] = useState(() => options['bulk_update_with_primary_key'] || {});
const [bulkUpsertPrimaryKey, setBulkUpsertPrimaryKey] = useState(() => options['bulk_upsert_with_primary_key'] || {});
const joinOptions = options['join_table']?.['joins'] || [
{ conditions: { conditionsList: [{ leftField: { table: selectedTableId } }] } },
@ -196,6 +198,11 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [bulkUpdatePrimaryKey]);
useEffect(() => {
mounted && optionchanged('bulk_upsert_with_primary_key', bulkUpsertPrimaryKey);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [bulkUpsertPrimaryKey]);
useEffect(() => {
mounted && optionchanged('update_rows', updateRowsOptions);
// eslint-disable-next-line react-hooks/exhaustive-deps
@ -234,10 +241,18 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
setBulkUpdatePrimaryKey((prev) => ({ ...prev, rows_update: value }));
};
const handleBulkUpsertRowsOptionChanged = (value) => {
setBulkUpsertPrimaryKey((prev) => ({ ...prev, rows: value }));
};
const handlePrimaryKeyOptionChangedForBulkUpdate = (value) => {
setBulkUpdatePrimaryKey((prev) => ({ ...prev, primary_key: value }));
};
const handlePrimaryKeyOptionChangedForBulkUpsert = (value) => {
setBulkUpsertPrimaryKey((prev) => ({ ...prev, primary_key: value }));
};
const loadTableInformation = async (tableId, isNewTableAdded) => {
const tableDetails = findTableDetails(tableId);
if (tableDetails?.table_name && !tableInfo[tableDetails?.table_name]) {
@ -340,8 +355,11 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
tableForeignKeyInfo,
setTableForeignKeyInfo,
bulkUpdatePrimaryKey,
bulkUpsertPrimaryKey,
handleBulkUpdateWithPrimaryKeysRowsUpdateOptionChanged,
handleBulkUpsertRowsOptionChanged,
handlePrimaryKeyOptionChangedForBulkUpdate,
handlePrimaryKeyOptionChangedForBulkUpsert,
}),
[
organizationId,
@ -357,6 +375,7 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
joinOrderByOptions,
selectedTableId,
bulkUpdatePrimaryKey,
bulkUpsertPrimaryKey,
]
);
@ -517,6 +536,8 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
return JoinTable;
case 'bulk_update_with_primary_key':
return BulkUploadPrimaryKey;
case 'bulk_upsert_with_primary_key':
return BulkUpsertPrimaryKey;
}
};
@ -527,6 +548,7 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
{ label: 'Delete rows', value: 'delete_rows' },
{ label: 'Join tables', value: 'join_tables' },
{ label: 'Bulk update with primary key', value: 'bulk_update_with_primary_key' },
{ label: 'Bulk upsert with primary key', value: 'bulk_upsert_with_primary_key' },
];
const ComponentToRender = getComponent(operation);

1
frontend/src/AppBuilder/_stores/slices/queryPanelSlice.js
View file

@ -425,6 +425,7 @@ export const createQueryPanelSlice = (set, get) => ({
isLoading: false,
...(query.kind === 'restapi'
? {
metadata: data.metadata,
request: data.data.requestObject,
response: data.data.responseObject,
responseHeaders: data.data.responseHeaders,

7
frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/SelectBox.jsx
View file

@ -220,7 +220,9 @@ function DataSourceSelect({
if (isFirstPageLoaded && offset >= totalRecords) return;
if (foreignKeys.length < 1) return;
setIsLoadingFKDetails(true);
const referencedColumns = foreignKeys.find((item) => item.column_names[0] === cellColumnName);
const referencedColumns = Array.isArray(foreignKeys)
? foreignKeys.find((item) => item.column_names[0] === cellColumnName)
: undefined;
if (!referencedColumns?.referenced_column_names?.length) return;
const selectQuery = new PostgrestQueryBuilder();
@ -715,7 +717,8 @@ const MenuList = ({
...props
}) => {
const menuListStyles = getStyles('menuList', props);
const referencedColumnDetails = foreignKeys?.find((item) => item.column_names[0] === cellColumnName);
const referencedColumnDetails =
Array.isArray(foreignKeys) && foreignKeys.find((item) => item?.column_names[0] === cellColumnName);
const handleNavigateToReferencedTable = () => {
const data = {

2
frontend/src/MarketplacePage/InstalledPlugins.jsx
View file

@ -68,7 +68,7 @@ export const InstalledPlugins = () => {
})}
{!fetching && installedPlugins?.length === 0 && (
<div className="empty">
<p className="empty-title">No results found</p>
<p className="empty-title">No plugins added. Please add a plugin from the Marketplace.</p>
</div>
)}
</div>

136
frontend/src/TooljetDatabase/Forms/ColumnForm.jsx
View file

@ -342,8 +342,39 @@ const ColumnForm = ({
</div>
)}
<div className="mb-3 tj-app-input">
<div className="form-label" data-cy="default-value-input-field-label">
Default value
<div className="d-flex align-items-center justify-content-between">
<div className="form-label" data-cy="default-value-input-field-label">
Default value
</div>
{foreignKeyDetails?.length > 0 && isForeignKey && (
<ToolTip
message={'Set the default value for the column to Null'}
placement="top"
tooltipClassName="tootip-table"
show={foreignKeyDetails?.length > 0 && isForeignKey}
>
<div className="d-flex align-items-center custom-gap-4">
<span className="form-label">Set default value to Null</span>
<label className={`form-switch`}>
<input
className="form-check-input"
type="checkbox"
checked={defaultValue === null}
onChange={(e) => {
if (e.target.checked) {
setForeignKeyDefaultValue({ label: null, value: null });
setDefaultValue(null);
} else {
setForeignKeyDefaultValue({ label: '', value: '' });
setDefaultValue('');
}
}}
disabled={isNotNull}
/>
</label>
</div>
</ToolTip>
)}
</div>
<ToolTip
message={dataType === 'serial' ? 'Serial data type values cannot be modified' : null}
@ -351,7 +382,7 @@ const ColumnForm = ({
tooltipClassName="tootip-table"
show={dataType === 'serial'}
>
<div>
<div style={{ position: 'relative' }}>
{isTimestamp ? (
<DateTimePicker
timestamp={defaultValue}
@ -380,46 +411,65 @@ const ColumnForm = ({
disabled={dataType?.value === 'serial'}
/>
) : (
<DropDownSelect
buttonClasses="border border-end-1 foreignKeyAcces-container-drawer mb-2"
showPlaceHolder={true}
options={referenceTableDetails}
darkMode={darkMode}
emptyError={
<div className="dd-select-alert-error m-2 d-flex align-items-center">
<Information />
No data found
</div>
}
loader={
<>
<Skeleton height={22} width={396} className="skeleton" style={{ margin: '15px 50px 7px 7px' }} />
<Skeleton height={22} width={450} className="skeleton" style={{ margin: '7px 14px 7px 7px' }} />
<Skeleton height={22} width={396} className="skeleton" style={{ margin: '7px 50px 15px 7px' }} />
</>
}
isLoading={true}
value={foreignKeyDefaultValue}
foreignKeyAccessInRowForm={true}
disabled={dataType === 'serial'}
topPlaceHolder={dataType === 'serial' ? 'Auto-generated' : 'Enter a value'}
onChange={(value) => {
setForeignKeyDefaultValue(value);
setDefaultValue(value?.value);
}}
onAdd={true}
addBtnLabel={'Open referenced table'}
foreignKeys={foreignKeyDetails}
setReferencedColumnDetails={setReferencedColumnDetails}
scrollEventForColumnValues={true}
cellColumnName={columnName}
columnDataType={dataType?.value}
isCreateColumn={true}
/>
<>
<DropDownSelect
buttonClasses="border border-end-1 foreignKeyAcces-container-drawer mb-2"
showPlaceHolder={true}
options={referenceTableDetails}
darkMode={darkMode}
emptyError={
<div className="dd-select-alert-error m-2 d-flex align-items-center">
<Information />
No data found
</div>
}
loader={
<>
<Skeleton
height={22}
width={396}
className="skeleton"
style={{ margin: '15px 50px 7px 7px' }}
/>
<Skeleton height={22} width={450} className="skeleton" style={{ margin: '7px 14px 7px 7px' }} />
<Skeleton
height={22}
width={396}
className="skeleton"
style={{ margin: '7px 50px 15px 7px' }}
/>
</>
}
isLoading={true}
value={foreignKeyDefaultValue}
foreignKeyAccessInRowForm={true}
disabled={dataType === 'serial'}
topPlaceHolder={
dataType === 'serial'
? 'Auto-generated'
: foreignKeyDefaultValue?.value === null
? 'Null'
: 'Enter a value'
}
onChange={(value) => {
setForeignKeyDefaultValue(value);
setDefaultValue(value?.value);
}}
onAdd={true}
addBtnLabel={'Open referenced table'}
foreignKeys={foreignKeyDetails}
setReferencedColumnDetails={setReferencedColumnDetails}
scrollEventForColumnValues={true}
cellColumnName={columnName}
columnDataType={dataType?.value}
isCreateColumn={true}
/>
{defaultValue === null && <p className={darkMode === true ? 'null-tag-dark' : 'null-tag'}>Null</p>}
</>
)}
</div>
</ToolTip>
{isNotNull === true && dataType?.value !== 'serial' && rows.length > 0 && defaultValue.length <= 0 ? (
{isNotNull === true && dataType?.value !== 'serial' && defaultValue?.length <= 0 ? (
<span className="form-error-message">
Default value is required to populate this field in existing rows as NOT NULL constraint is added
</span>
@ -546,6 +596,10 @@ const ColumnForm = ({
checked={isNotNull}
onChange={(e) => {
setIsNotNull(e.target.checked);
if (e.target.checked && defaultValue === null) {
setForeignKeyDefaultValue({ label: '', value: '' });
setDefaultValue('');
}
}}
disabled={dataType?.value === 'serial'}
/>
@ -602,7 +656,7 @@ const ColumnForm = ({
shouldDisableCreateBtn={
isEmpty(columnName) ||
isEmpty(dataType) ||
(isNotNull === true && rows.length > 0 && isEmpty(defaultValue) && dataType?.value !== 'serial') ||
(dataType?.value !== 'serial' && isNotNull === true && isEmpty(defaultValue)) ||
disabledSaveButton
}
showToolTipForFkOnReadDocsSection={true}

230
frontend/src/TooljetDatabase/Forms/EditColumnForm.jsx
View file

@ -31,6 +31,8 @@ import DateTimePicker from '@/Editor/QueryManager/QueryEditors/TooljetDatabase/D
import { getLocalTimeZone, timeZonesWithOffsets } from '@/Editor/QueryManager/QueryEditors/TooljetDatabase/util';
import CodeHinter from '@/AppBuilder/CodeEditor';
import { resolveReferences } from '@/AppBuilder/CodeEditor/utils';
import Switch from '@/AppBuilder/CodeBuilder/Elements/Switch';
import PostgrestQueryBuilder from '@/_helpers/postgrestQueryBuilder';
const ColumnForm = ({
onClose,
@ -102,6 +104,68 @@ const ColumnForm = ({
const [foreignKeyDetails, setForeignKeyDetails] = useState([]);
// Add function to validate default value
const validateDefaultValue = async () => {
if (!isMatchingForeignKeyColumn(selectedColumn?.Header)) return;
try {
const referencedColumns = foreignKeys.find((item) => item.column_names[0] === selectedColumn?.Header);
if (!referencedColumns?.referenced_column_names?.length) {
setForeignKeyDefaultValue({
value: '',
label: '',
});
setDefaultValue('');
return;
}
const selectQuery = new PostgrestQueryBuilder();
selectQuery.select(referencedColumns.referenced_column_names[0]);
selectQuery.eq(referencedColumns.referenced_column_names[0], defaultValue);
const query = selectQuery.url.toString();
const { data = [], error } = await tooljetDatabaseService.findOne(
organizationId,
referencedColumns.referenced_table_id,
query
);
if (error) {
toast.error(error?.message ?? `Failed to validate default value`);
setForeignKeyDefaultValue({
value: '',
label: '',
});
setDefaultValue('');
return;
}
if (data.length === 0) {
setForeignKeyDefaultValue({
value: '',
label: '',
});
setDefaultValue('');
}
} catch (error) {
console.error('Error validating default value:', error);
setForeignKeyDefaultValue({
value: '',
label: '',
});
setDefaultValue('');
}
};
// Add useEffect to validate on mount
useEffect(() => {
if (isMatchingForeignKeyColumn(selectedColumn?.Header) && defaultValue) {
validateDefaultValue();
}
}, []);
useEffect(() => {
toast.dismiss();
setForeignKeyDetails(
@ -471,6 +535,11 @@ const ColumnForm = ({
setDisabledSaveButton(columnName === '');
}, [columnName]);
useEffect(() => {
const shouldDisableForNullValue = dataType?.value !== 'serial' && isNotNull === true && isEmpty(defaultValue);
setDisabledSaveButton(shouldDisableForNullValue);
}, [isNotNull, defaultValue, dataType]);
const handleInputError = (bool = false) => {
setDisabledSaveButton(bool);
};
@ -621,16 +690,52 @@ const ColumnForm = ({
</div>
)}
<div className="mb-3 tj-app-input">
<div className="form-label" data-cy="default-value-input-field-label">
Default value
<div className="d-flex align-items-center justify-content-between">
<div className="form-label" data-cy="default-value-input-field-label">
Default value
</div>
{isMatchingForeignKeyColumn(selectedColumn?.Header) && (
<ToolTip
message={
isNotNull
? 'Disable the NOT NULL constraint to set the default value to Null'
: 'Set the default value for the column to Null'
}
placement="top"
tooltipClassName="tootip-table"
show={isMatchingForeignKeyColumn(selectedColumn?.Header) || isNotNull}
>
<div className="d-flex align-items-center custom-gap-4">
<span className="form-label">Set default value to Null</span>
<label className={`form-switch`}>
<input
className="form-check-input"
type="checkbox"
checked={defaultValue === null}
onChange={(e) => {
if (e.target.checked) {
setForeignKeyDefaultValue({ label: null, value: null });
setDefaultValue(null);
} else {
setForeignKeyDefaultValue({ label: '', value: '' });
setDefaultValue('');
}
}}
disabled={isNotNull}
/>
</label>
</div>
</ToolTip>
)}
</div>
<ToolTip
message={selectedColumn?.dataType === 'serial' ? 'Serial data type values cannot be modified' : null}
placement="top"
tooltipClassName="tootip-table"
show={selectedColumn?.dataType === 'serial'}
>
<div>
<div style={{ position: 'relative' }}>
{isTimestamp ? (
<DateTimePicker
timestamp={defaultValue}
@ -655,57 +760,76 @@ const ColumnForm = ({
disabled={selectedColumn?.dataType === 'serial'}
/>
) : (
<DropDownSelect
buttonClasses="border border-end-1 foreignKeyAcces-container-drawer mb-2"
showPlaceHolder={true}
options={referenceTableDetails}
darkMode={darkMode}
emptyError={
<div className="dd-select-alert-error m-2 d-flex align-items-center">
<Information />
No data found
</div>
}
loader={
<>
<Skeleton
height={22}
width={396}
className="skeleton"
style={{ margin: '15px 50px 7px 7px' }}
/>
<Skeleton height={22} width={450} className="skeleton" style={{ margin: '7px 14px 7px 7px' }} />
<Skeleton
height={22}
width={396}
className="skeleton"
style={{ margin: '7px 50px 15px 7px' }}
/>
</>
}
isLoading={true}
value={foreignKeyDefaultValue}
foreignKeyAccessInRowForm={true}
disabled={
selectedColumn?.dataType === 'serial' || selectedColumn.constraints_type.is_primary_key === true
}
topPlaceHolder={selectedColumn?.dataType === 'serial' ? 'Auto-generated' : 'Enter a value'}
onChange={(value) => {
setForeignKeyDefaultValue(value);
setDefaultValue(value?.value);
}}
onAdd={true}
addBtnLabel={'Open referenced table'}
foreignKeys={foreignKeys}
setReferencedColumnDetails={setReferencedColumnDetails}
scrollEventForColumnValues={true}
cellColumnName={selectedColumn?.Header}
columnDataType={dataType}
isEditColumn={true}
/>
<>
<DropDownSelect
buttonClasses="border border-end-1 foreignKeyAcces-container-drawer mb-2"
showPlaceHolder={true}
options={referenceTableDetails}
darkMode={darkMode}
emptyError={
<div className="dd-select-alert-error m-2 d-flex align-items-center">
<Information />
No data found
</div>
}
loader={
<>
<Skeleton
height={22}
width={396}
className="skeleton"
style={{ margin: '15px 50px 7px 7px' }}
/>
<Skeleton
height={22}
width={450}
className="skeleton"
style={{ margin: '7px 14px 7px 7px' }}
/>
<Skeleton
height={22}
width={396}
className="skeleton"
style={{ margin: '7px 50px 15px 7px' }}
/>
</>
}
isLoading={true}
value={foreignKeyDefaultValue}
foreignKeyAccessInRowForm={true}
disabled={
selectedColumn?.dataType === 'serial' || selectedColumn.constraints_type.is_primary_key === true
}
topPlaceHolder={
selectedColumn?.dataType === 'serial'
? 'Auto-generated'
: foreignKeyDefaultValue?.value === null || defaultValue === null
? 'Null'
: 'Enter a value'
}
onChange={(value) => {
setForeignKeyDefaultValue(value);
setDefaultValue(value?.value);
}}
onAdd={true}
addBtnLabel={'Open referenced table'}
foreignKeys={foreignKeys}
setReferencedColumnDetails={setReferencedColumnDetails}
scrollEventForColumnValues={true}
cellColumnName={selectedColumn?.Header}
columnDataType={dataType}
isEditColumn={true}
/>
{defaultValue === null && <p className={darkMode === true ? 'null-tag-dark' : 'null-tag'}>Null</p>}
</>
)}
</div>
</ToolTip>
{isNotNull === true && dataType?.value !== 'serial' && defaultValue?.length <= 0 ? (
<span className="form-error-message">
Default value is required to populate this field in existing rows as NOT NULL constraint is added
</span>
) : null}
{isNotNull === true &&
selectedColumn?.dataType !== 'serial' &&
rows.length > 0 &&
@ -866,6 +990,10 @@ const ColumnForm = ({
checked={isNotNull}
onChange={(e) => {
setIsNotNull(e.target.checked);
if (e.target.checked && defaultValue === null) {
setForeignKeyDefaultValue({ label: '', value: '' });
setDefaultValue('');
}
}}
disabled={selectedColumn?.dataType === 'serial' || selectedColumn?.constraints_type?.is_primary_key}
/>

4
frontend/src/TooljetDatabase/Forms/EditRowForm.jsx
View file

@ -177,7 +177,9 @@ const EditRowForm = ({
}
function isMatchingForeignKeyColumnDetails(columnHeader) {
const matchingColumn = foreignKeys.find((foreignKey) => foreignKey.column_names[0] === columnHeader);
const matchingColumn = Array.isArray(foreignKeys)
? foreignKeys.find((foreignKey) => foreignKey.column_names[0] === columnHeader)
: undefined;
return matchingColumn;
}

4
frontend/src/TooljetDatabase/Forms/RowForm.jsx
View file

@ -151,7 +151,9 @@ const RowForm = ({
}
function isMatchingForeignKeyColumnDetails(columnHeader) {
const matchingColumn = foreignKeys.find((foreignKey) => foreignKey.column_names[0] === columnHeader);
const matchingColumn = Array.isArray(foreignKeys)
? foreignKeys.find((foreignKey) => foreignKey.column_names[0] === columnHeader)
: undefined;
return matchingColumn;
}

4
frontend/src/TooljetDatabase/Table/index.jsx
View file

@ -958,7 +958,9 @@ const Table = ({ collapseSidebar }) => {
}
function isMatchingForeignKeyColumnDetails(columnHeader) {
const matchingColumn = foreignKeys.find((foreignKey) => foreignKey.column_names[0] === columnHeader);
const matchingColumn = Array.isArray(foreignKeys)
? foreignKeys.find((foreignKey) => foreignKey.column_names[0] === columnHeader)
: undefined;
return matchingColumn;
}

129
frontend/src/_components/DynamicFormV2.jsx
View file

@ -30,6 +30,8 @@ const DynamicFormV2 = ({
validationMessages,
setValidationMessages,
clearValidationMessages,
showValidationErrors,
clearValidationErrorBanner,
}) => {
const uiProperties = schema['tj:ui:properties'] || {};
const dsm = React.useMemo(() => new DataSourceSchemaManager(schema), [schema]);
@ -89,18 +91,97 @@ const DynamicFormV2 = ({
React.useEffect(() => {
if (!hasUserInteracted) return;
const { valid, errors } = dsm.validateData(options);
if (valid) {
clearValidationMessages();
} else {
setValidationMessages(errors, schema);
const requiredFields = errors
.filter((error) => error.keyword === 'required')
.map((error) => error.params.missingProperty);
setConditionallyRequiredProperties(requiredFields);
const timeout = setTimeout(() => {
validateOptions();
}, 300);
return () => clearTimeout(timeout);
}, [options, hasUserInteracted, validateOptions]);
const validateOptions = React.useCallback(async () => {
try {
const { valid, errors } = await dsm.validateData(options);
const conditionallyRequiredFields = processAllOfConditions(schema, options);
setConditionallyRequiredProperties(conditionallyRequiredFields);
if (valid) {
clearValidationMessages();
clearValidationErrorBanner();
} else {
setValidationMessages(errors, schema, interactedFields);
}
} catch (error) {
console.error('Validation error:', error);
}
}, [options]);
}, [
dsm,
options,
processAllOfConditions,
schema,
clearValidationMessages,
clearValidationErrorBanner,
setValidationMessages,
interactedFields,
]);
const processAllOfConditions = React.useCallback((schema, options, path = []) => {
let requiredFields = [];
if (schema.allOf) {
schema.allOf.forEach((condition) => {
if (condition.if && condition.then) {
const conditionMatches = Object.entries(condition.if.properties || {}).every(([propName, propCondition]) => {
const propertyPath = [...path, propName];
let currentValue = options;
for (const segment of propertyPath) {
if (!currentValue || typeof currentValue !== 'object') {
return false;
}
currentValue = currentValue[segment]?.value;
}
return propCondition.const === currentValue;
});
if (conditionMatches) {
if (condition.then.required) {
requiredFields = [...requiredFields, ...condition.then.required];
}
if (condition.then.allOf) {
const nestedRequired = processAllOfConditions({ allOf: condition.then.allOf }, options, path);
requiredFields = [...requiredFields, ...nestedRequired];
}
if (condition.then.properties) {
Object.entries(condition.then.properties).forEach(([propName, propSchema]) => {
if (propSchema.allOf) {
const nestedRequired = processAllOfConditions({ allOf: propSchema.allOf }, options, [
...path,
propName,
]);
requiredFields = [...requiredFields, ...nestedRequired];
}
});
}
}
}
});
}
return requiredFields;
}, []);
React.useEffect(() => {
if (showValidationErrors) {
setHasUserInteracted(true);
const allFieldKeys = Object.keys(options);
setInteractedFields(new Set(allFieldKeys));
}
}, [showValidationErrors, options]);
React.useEffect(() => {
const prevDataSourceId = prevDataSourceIdRef.current;
@ -189,6 +270,7 @@ const DynamicFormV2 = ({
return Input;
case 'password-v3':
case 'text-v3':
case 'password-v3-textarea':
return InputV3;
case 'textarea':
return Textarea;
@ -210,8 +292,10 @@ const DynamicFormV2 = ({
const isRequired = required || conditionallyRequiredProperties.includes(key);
const isEncrypted = widget === 'password-v3' || encryptedProperties.includes(key);
const currentValue = options?.[key]?.value;
const skipValidation =
(!hasUserInteracted && !showValidationErrors) || (!interactedFields.has(key) && !showValidationErrors);
const handleOptionChange = (key, value, flag) => {
const handleOptionChange = (key, value, flag = true) => {
if (!hasUserInteracted) {
setHasUserInteracted(true);
}
@ -243,6 +327,7 @@ const DynamicFormV2 = ({
};
}
case 'password-v3':
case 'password-v3-textarea':
case 'text-v3': {
return {
key,
@ -262,14 +347,13 @@ const DynamicFormV2 = ({
encrypted: isEncrypted,
onBlur,
isRequired: isRequired,
isValidatedMessages:
!hasUserInteracted || !interactedFields.has(key)
? { valid: null, message: '' } // skip validation for initial render and untouched elements
: validationMessages[key]
? { valid: false, message: validationMessages[key] }
: isRequired && !isEncrypted
? { valid: true, message: '' }
: { valid: null, message: '' }, // handle optional && encrypted fields
isValidatedMessages: skipValidation
? { valid: null, message: '' } // skip validation for initial render and untouched elements
: validationMessages[key]
? { valid: false, message: validationMessages[key] }
: isRequired
? { valid: true, message: '' }
: { valid: null, message: '' }, // handle optional && encrypted fields
isDisabled: !canUpdateDataSource(selectedDataSource?.id) && !canDeleteDataSource(),
};
}
@ -285,7 +369,7 @@ const DynamicFormV2 = ({
options: isRenderedAsQueryEditor
? options?.[key] ?? schema?.defaults?.[key]
: options?.[key]?.value ?? schema?.defaults?.[key]?.value,
optionchanged,
handleOptionChange,
isRenderedAsQueryEditor,
workspaceConstants: currentOrgEnvironmentConstants,
isDisabled: !canUpdateDataSource(selectedDataSource?.id) && !canDeleteDataSource(),
@ -298,14 +382,14 @@ const DynamicFormV2 = ({
return {
defaultChecked: currentValue,
checked: currentValue,
onChange: (e) => optionchanged(key, e.target.checked),
onChange: (e) => handleOptionChange(key, e.target.checked, true),
};
case 'dropdown':
case 'dropdown-component-flip':
return {
options: list,
value: options?.[key]?.value || options?.[key],
onChange: (value) => optionchanged(key, value),
onChange: (value) => handleOptionChange(key, value, true),
width: width || '100%',
encrypted: options?.[key]?.encrypted,
};
@ -404,6 +488,7 @@ const DynamicFormV2 = ({
{label &&
widget !== 'text-v3' &&
widget !== 'password-v3' &&
widget !== 'password-v3-textarea' &&
renderLabel(label, uiProperties[key].tooltip)}
</div>
)}

22
frontend/src/_helpers/dataSourceSchemaManager.js
View file

@ -1,3 +1,4 @@
import { datasourceService } from '@/_services';
import Ajv2020 from 'ajv';
const ajvOptions = {
@ -18,13 +19,18 @@ export default class DataSourceSchemaManager {
this.validate = this.ajv.compile(this.schema);
}
validateData(options) {
const data = this._convertDataSourceOptionsToData(options);
const valid = this.validate(data);
if (!valid) {
return { valid: false, errors: this.validate.errors };
async validateData(options) {
const decryptedOptions = await datasourceService.getDecryptedOptions(options);
const data = this._convertDataSourceOptionsToData(decryptedOptions);
try {
const valid = this.validate(data);
if (!valid) {
return { valid: false, errors: this.validate.errors };
}
return { valid: true, errors: [] };
} catch (error) {
console.log('Validtion error: ', error);
}
return { valid: true, errors: [] };
}
getDefaults(options = {}) {
@ -95,10 +101,6 @@ export default class DataSourceSchemaManager {
result[key] = value;
}
// Add a dummy value to pass validation for encrypted keys
if (this.getEncryptedProperties().includes(key)) {
result[key] = 'REDACTED';
}
return result;
}, {});
}

11
frontend/src/_services/datasource.service.js
View file

@ -11,8 +11,19 @@ export const datasourceService = {
save,
fetchOauth2BaseUrl,
testSampleDb,
getDecryptedOptions,
};
function getDecryptedOptions(options) {
const requestOptions = {
method: 'POST',
headers: authHeader(),
credentials: 'include',
body: JSON.stringify(options),
};
return fetch(`${config.apiUrl}/data-sources/decrypt`, requestOptions).then(handleResponse);
}
function getAll(appVersionId, environment_id, includeStaticSources = false) {
const requestOptions = { method: 'GET', headers: authHeader(), credentials: 'include' };
let searchParams = new URLSearchParams(

15
frontend/src/_styles/theme.scss
View file

@ -12345,6 +12345,17 @@ tbody {
}
}
.design-component-inputs textarea {
&.valid-textarea {
border: 1.5px solid #519b62 !important;
}
&.invalid-textarea {
border: 1.5px solid #e26367 !important;
}
}
}
.tj-app-input-wrapper {
@ -13141,6 +13152,10 @@ tbody {
background-color: var(--slate3) !important;
}
textarea:disabled {
background-color: var(--slate3) !important;
}
.react-select__control--is-disabled {
background-color: var(--slate3) !important;
}

52
frontend/src/_ui/HttpHeaders/index.js
View file

@ -1,13 +1,14 @@
import React from "react";
import _ from "lodash";
import QueryEditor from "./QueryEditor";
import SourceEditor from "./SourceEditor";
import { deepClone } from "@/_helpers/utilities/utils.helpers";
import React from 'react';
import _ from 'lodash';
import QueryEditor from './QueryEditor';
import SourceEditor from './SourceEditor';
import { deepClone } from '@/_helpers/utilities/utils.helpers';
export default ({
getter,
options = [["", ""]],
options = [['', '']],
optionchanged,
handleOptionChange,
isRenderedAsQueryEditor,
workspaceConstants,
isDisabled,
@ -16,27 +17,46 @@ export default ({
dataCy,
}) => {
function addNewKeyValuePair(options) {
const newPairs = [...options, ["", ""]];
optionchanged(getter, newPairs);
const newPairs = [...options, ['', '']];
if (handleOptionChange) {
handleOptionChange(getter, newPairs, true);
} else {
optionchanged(getter, newPairs);
}
}
function removeKeyValuePair(index) {
const newOptions = [...options];
newOptions.splice(index, 1);
optionchanged(getter, newOptions);
if (handleOptionChange) {
handleOptionChange(getter, newOptions, true);
} else {
optionchanged(getter, newOptions);
}
}
function keyValuePairValueChanged(value, keyIndex, index) {
if (!isRenderedAsQueryEditor) {
const newOptions = deepClone(options);
newOptions[index][keyIndex] = value;
options.length - 1 === index
? addNewKeyValuePair(newOptions)
: optionchanged(getter, newOptions);
if (options.length - 1 === index) {
addNewKeyValuePair(newOptions);
} else {
if (handleOptionChange) {
handleOptionChange(getter, newOptions, true);
} else {
optionchanged(getter, newOptions);
}
}
} else {
let newOptions = deepClone(options);
newOptions[index][keyIndex] = value;
optionchanged(getter, newOptions);
if (handleOptionChange) {
handleOptionChange(getter, newOptions, true);
} else {
optionchanged(getter, newOptions);
}
}
}
@ -53,10 +73,6 @@ export default ({
return isRenderedAsQueryEditor ? (
<QueryEditor {...commonProps} />
) : (
<SourceEditor
{...commonProps}
workspaceConstants={workspaceConstants}
width={width}
/>
<SourceEditor {...commonProps} workspaceConstants={workspaceConstants} width={width} />
);
};

3
frontend/src/_ui/Input-V3/index.js
View file

@ -43,7 +43,7 @@ const InputV3 = ({ helpText, ...props }) => {
required={props.isRequired}
/>
)}
{(widget === 'password-v3' || encrypted) && (
{(widget === 'password-v3' || widget === 'password-v3-textarea' || encrypted) && (
<div style={{ flex: '1' }}>
<InputComponent
{...props}
@ -53,6 +53,7 @@ const InputV3 = ({ helpText, ...props }) => {
label={props.label}
placeholder={props.placeholder}
required={props.isRequired}
multiline={widget === 'password-v3-textarea'}
/>
</div>
)}

17
frontend/src/components/ui/Input/CommonInput/Index.jsx
View file

@ -32,21 +32,28 @@ const CommonInput = ({ label, helperText, disabled, required, onChange: change,
}
}, [isValidatedMessages]);
useEffect(() => {
if (isValid === true && (!isValidatedMessages || isValidatedMessages.valid === null)) {
setIsValid(true);
}
}, [isValid, isValidatedMessages]);
const toggleEditing = () => {
if (isDisabled) return;
const willBeInEditMode = !isEditing;
setIsEditing(willBeInEditMode);
if (willBeInEditMode) {
change({ target: { value: '' } });
}
change({ target: { value: '' } });
};
return (
<div>
<div className="d-flex">
{label && <InputLabel disabled={disabled} label={label} required={required} />}
{label && (
<div className="tw-flex-shrink-0">
<InputLabel disabled={disabled} label={label} required={required} />
</div>
)}
{type === 'password' && (
<div className="d-flex justify-content-between w-100">
<div className="mx-1 col">

5
frontend/src/components/ui/Input/CommonInput/TextInput.jsx
View file

@ -14,14 +14,14 @@ const TextInput = ({
readOnly,
...restProps
}) => {
const inputStyle = `tw-border-border-default placeholder:tw-text-text-placeholder tw-font-normal disabled:tw-bg-[#CCD1D5]/30 ${
const inputStyle = `placeholder:tw-text-text-placeholder tw-font-normal disabled:tw-bg-[#CCD1D5]/30 ${
leadingIcon ? (size === 'small' ? 'tw-pl-[32px]' : 'tw-pl-[34px]') : 'tw-pl-[12px]'
} ${trailingAction ? (size === 'small' ? 'tw-pr-[40px]' : 'tw-pr-[44px]') : 'tw-pr-[12px]'} ${
response === true
? '!tw-border-border-success-strong focus-visible:!tw-ring-0 focus-visible:!tw-ring-offset-0 focus-visible:!tw-border-border-success-strong'
: response === false
? '!tw-border-border-danger-strong focus-visible:!tw-ring-0 focus-visible:!tw-ring-offset-0 focus-visible:!tw-border-border-danger-strong'
: ''
: 'tw-border-border-default'
}`;
return (
@ -39,6 +39,7 @@ const TextInput = ({
size={size}
placeholder={disabled && readOnly ? readOnly : placeholder}
disabled={disabled}
response={response}
{...restProps}
className={inputStyle}
/>

43
frontend/src/components/ui/Input/Input.jsx
View file

@ -3,7 +3,7 @@ import { cn } from '@/lib/utils';
import { inputVariants } from './InputUtils/Variants';
import SolidIcon from '../../../_ui/Icon/SolidIcons';
const Input = React.forwardRef(({ className, size, type, ...props }, ref) => {
const Input = React.forwardRef(({ className, size, type, multiline, response, rows = 3, ...props }, ref) => {
const [isPasswordVisible, setIsPasswordVisible] = React.useState(false);
const isPasswordField = type === 'password';
@ -13,19 +13,34 @@ const Input = React.forwardRef(({ className, size, type, ...props }, ref) => {
}
};
const validationClass = response === true ? 'valid-textarea' : response === false ? 'invalid-textarea' : '';
return (
<>
<input
type={isPasswordField && isPasswordVisible ? 'text' : type}
className={cn(
inputVariants({ size }),
`tw-relative tw-peer tw-flex tw-text-[12px]/[18px] tw-w-full tw-rounded-[8px] tw-border-[1px] tw-border-solid tw-bg-background-surface-layer-01 tw-py-[7px] tw-text-text-default focus-visible:tw-ring-[1px] focus-visible:tw-ring-offset-[1px] focus-visible:tw-ring-border-accent-strong focus-visible:tw-ring-offset-border-accent-strong focus-visible:tw-border-transparent disabled:tw-cursor-not-allowed ${props.styles}`,
className
)}
ref={ref}
{...props}
/>
{isPasswordField && (
<div className="design-component-inputs">
{multiline ? (
<textarea
className={cn(
`tw-relative tw-peer tw-flex tw-text-[12px]/[18px] tw-w-full tw-rounded-[8px] tw-border-[1px] tw-border-solid tw-bg-background-surface-layer-01 tw-py-[7px] tw-text-text-default focus-visible:tw-ring-[1px] focus-visible:tw-ring-offset-[1px] focus-visible:tw-ring-border-accent-strong focus-visible:tw-ring-offset-border-accent-strong focus-visible:tw-border-transparent disabled:tw-cursor-not-allowed ${props.styles}`,
className,
validationClass
)}
rows={rows}
ref={ref}
{...props}
/>
) : (
<input
type={isPasswordField && isPasswordVisible ? 'text' : type}
className={cn(
inputVariants({ size }),
`tw-relative tw-peer tw-flex tw-text-[12px]/[18px] tw-w-full tw-rounded-[8px] tw-border-[1px] tw-border-solid tw-bg-background-surface-layer-01 tw-py-[7px] tw-text-text-default focus-visible:tw-ring-[1px] focus-visible:tw-ring-offset-[1px] focus-visible:tw-ring-border-accent-strong focus-visible:tw-ring-offset-border-accent-strong focus-visible:tw-border-transparent disabled:tw-cursor-not-allowed ${props.styles}`,
className
)}
ref={ref}
{...props}
/>
)}
{isPasswordField && !multiline && (
<div onClick={togglePasswordVisibility}>
{isPasswordVisible ? (
<SolidIcon className="eye-icon" name="eye" />
@ -34,7 +49,7 @@ const Input = React.forwardRef(({ className, size, type, ...props }, ref) => {
)}
</div>
)}
</>
</div>
);
});
Input.displayName = 'Input';

25
frontend/src/modules/dataSources/components/DataSourceManager/DataSourceManager.jsx
View file

@ -84,6 +84,7 @@ class DataSourceManagerComponent extends React.Component {
creatingApp: false,
validationError: [],
validationMessages: {},
showValidationErrors: false,
};
}
@ -219,11 +220,13 @@ class DataSourceManagerComponent extends React.Component {
dataSourceMeta,
dataSourceSchema,
validationMessages,
validationError,
} = this.state;
if (!isEmpty(validationMessages)) {
const validationMessageArray = Object.values(validationMessages);
this.setState({ validationError: validationMessageArray });
this.setState({ validationError: validationMessageArray, showValidationErrors: true });
toast.error(
this.props.t(
'editor.queryManager.dataSourceManager.toast.error.validationFailed',
@ -362,7 +365,7 @@ class DataSourceManagerComponent extends React.Component {
this.setState({ suggestingDatasources: true, activeDatasourceList: '#' });
};
setValidationMessages = (errors, schema) => {
setValidationMessages = (errors, schema, interactedFields) => {
const errorMap = errors.reduce((acc, error) => {
// Get property name from either required error or dataPath
const property =
@ -379,10 +382,19 @@ class DataSourceManagerComponent extends React.Component {
return acc;
}, {});
this.setState({ validationMessages: errorMap });
const filteredValidationBanner = interactedFields
? Object.keys(this.state.validationMessages)
.filter((key) => interactedFields.has(key))
.reduce((result, key) => {
result.push(this.state.validationMessages[key]);
return result;
}, [])
: Object.values(this.state.validationMessages);
this.setState({ validationError: filteredValidationBanner });
};
renderSourceComponent = (kind, isPlugin = false) => {
const { options, isSaving } = this.state;
const { options, isSaving, showValidationErrors } = this.state;
const sourceComponentName = kind?.charAt(0).toUpperCase() + kind?.slice(1);
const ComponentToRender = isPlugin ? SourceComponent : SourceComponents[sourceComponentName] || SourceComponent;
@ -402,6 +414,8 @@ class DataSourceManagerComponent extends React.Component {
setValidationMessages={this.setValidationMessages}
clearValidationMessages={() => this.setState({ validationMessages: {} })}
setDefaultOptions={this.setDefaultOptions}
showValidationErrors={showValidationErrors}
clearValidationErrorBanner={() => this.setState({ validationError: [] })}
/>
);
};
@ -901,6 +915,7 @@ class DataSourceManagerComponent extends React.Component {
addingDataSource,
datasourceName,
validationError,
validationMessages,
} = this.state;
const isPlugin = dataSourceSchema ? true : false;
const createSelectedDataSource = (dataSource) => {
@ -910,7 +925,9 @@ class DataSourceManagerComponent extends React.Component {
const sampleDBmodalBodyStyle = isSampleDb ? { paddingBottom: '0px', borderBottom: '1px solid #E6E8EB' } : {};
const sampleDBmodalFooterStyle = isSampleDb ? { paddingTop: '8px' } : {};
const isSaveDisabled = selectedDataSource
? deepEqual(options, selectedDataSource?.options, ['encrypted']) && selectedDataSource?.name === datasourceName
? (deepEqual(options, selectedDataSource?.options, ['encrypted']) &&
selectedDataSource?.name === datasourceName) ||
!isEmpty(validationMessages)
: true;
this.props.setGlobalDataSourceStatus({ isEditing: !isSaveDisabled });
const docLink = isSampleDb

2
marketplace/plugins/azurerepos/lib/manifest.json
View file

@ -4,7 +4,7 @@
"description": "A schema defining Azurerepos datasource",
"type": "api",
"source": {
"name": "Azurerepos",
"name": "Azure Repos",
"kind": "azurerepos",
"exposedVariables": {
"isLoading": false,

78
marketplace/plugins/pinecone/lib/icon.svg
View file

@ -1,72 +1,10 @@
<?xml version="1.0" encoding="iso-8859-1"?>
<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 512 512" style="enable-background:new 0 0 512 512;" xml:space="preserve">
<g>
<g>
<g>
<rect x="445.176" y="222.066" width="15.208" height="32.606"/>
<path d="M465.275,103.536V55.237c0-10.89-8.859-19.75-19.749-19.75H292.721c-10.89,0-19.75,8.86-19.75,19.75v48.298h-38.498
V55.237c0-10.89-8.86-19.75-19.75-19.75H61.918c-10.891,0-19.75,8.86-19.75,19.75v48.298H0v372.977h512V103.536H465.275z
M288.178,55.237c0-2.505,2.038-4.542,4.542-4.542h152.805c2.504,0,4.541,2.038,4.541,4.542v48.298H422.47V64.268h-15.208v39.268
H288.178V55.237z M57.374,55.237c0-2.505,2.038-4.542,4.542-4.542h152.804c2.504,0,4.542,2.038,4.542,4.542v48.298h-29.126
V64.268h-15.208v39.268H57.374V55.237z M496.792,461.305h-36.404V269.298H445.18v192.007H15.208V118.743h26.959h192.305h38.498
h192.305h31.517V461.305z"/>
<path d="M385.777,275.757c-1.028-0.078-2.067-0.118-3.092-0.118c-10.329,0-20.117,3.942-27.562,11.096
c-0.054,0.051-0.105,0.101-0.159,0.15v-81.028h-81.027c0.05-0.054,0.099-0.105,0.149-0.158
c7.838-8.156,11.84-19.33,10.978-30.656c-1.449-19.026-16.24-34.353-35.171-36.443c-1.494-0.164-3.008-0.248-4.5-0.248
c-21.939,0-39.787,17.848-39.787,39.788c0,10.414,3.994,20.254,11.25,27.719h-81.031v99.334h7.604
c6.248,0,12.237-2.492,16.431-6.835c5.122-5.303,12.213-8.006,19.621-7.435c11.548,0.878,21.213,10.196,22.481,21.673
c0.787,7.121-1.381,13.948-6.105,19.223c-4.661,5.204-11.337,8.189-18.317,8.189c-6.631,0-12.846-2.602-17.502-7.328
c-3.186-3.232-7.177-5.463-11.542-6.449c-0.757-0.172-1.534-0.259-2.307-0.259c-5.715,0-10.364,4.639-10.364,10.343v88.681
h88.991c5.703,0,10.343-4.639,10.343-10.342c0-6.248-2.492-12.238-6.836-16.432c-5.285-5.103-7.994-12.256-7.435-19.622
c0.879-11.548,10.197-21.212,21.675-22.48c0.943-0.105,1.895-0.157,2.831-0.157c13.553,0,24.58,11.027,24.58,24.58
c0,6.606-2.585,12.805-7.281,17.456c-4.555,4.514-7.065,10.396-7.065,16.565c0,5.753,4.68,10.433,10.434,10.433h88.9v-81.031
c7.465,7.255,17.304,11.25,27.719,11.25c11.299,0,22.105-4.83,29.646-13.251c7.537-8.415,11.142-19.728,9.893-31.037
C420.13,291.995,404.803,277.205,385.777,275.757z M401.001,331.817c-4.661,5.205-11.338,8.19-18.317,8.19
c-6.457,0-12.557-2.486-17.174-7c-4.843-4.737-11.287-7.344-18.148-7.344h-7.604v84.126h-67.227
c0.262-0.342,0.551-0.671,0.868-0.987c7.6-7.528,11.785-17.564,11.785-28.259c0-21.94-17.848-39.788-39.788-39.788
c-1.491,0-3.005,0.084-4.5,0.248c-18.931,2.093-33.722,17.419-35.17,36.442c-0.905,11.898,3.482,23.458,12.036,31.718
c0.202,0.195,0.391,0.404,0.569,0.625h-67.295v-64.691c7.3,6.542,16.618,10.117,26.507,10.117
c11.298,0,22.104-4.83,29.646-13.251c7.537-8.415,11.142-19.728,9.892-31.038c-2.092-18.93-17.419-33.72-36.444-35.168
c-1.027-0.078-2.068-0.118-3.092-0.118c-9.906,0-19.212,3.571-26.508,10.116v-64.689l84.126-0.005v-7.604
c0-6.856-2.608-13.299-7.344-18.144c-4.515-4.618-7.001-10.716-7.001-17.174c0-13.553,11.026-24.58,24.579-24.58
c0.936,0,1.888,0.053,2.831,0.157c11.478,1.267,20.796,10.932,21.675,22.481c0.541,7.117-1.867,13.851-6.78,18.964
c-4.831,5.026-7.491,11.525-7.491,18.3v7.604h84.126v84.126h7.604c6.775,0,13.273-2.66,18.301-7.491
c5.061-4.863,11.901-7.314,18.963-6.78c11.548,0.878,21.213,10.196,22.481,21.675
C407.893,319.717,405.725,326.542,401.001,331.817z"/>
</g>
</g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
</g>
<g>
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_127_41)">
<path d="M10.544 19.6692C11.1746 19.6692 11.6858 20.1804 11.6858 20.8109C11.6858 21.4415 11.1746 21.9527 10.544 21.9527C9.91343 21.9527 9.40225 21.4415 9.40225 20.8109C9.40225 20.1804 9.91343 19.6692 10.544 19.6692ZM16.8785 17.586L18.0067 17.9203L17.1111 20.9431C17.0328 21.2072 16.7818 21.3817 16.507 21.3629L16.2303 21.344L16.2236 21.3495L13.3492 21.1478L13.4292 19.9738L15.3604 20.1048L14.0961 18.2811L15.0632 17.6106L16.3291 19.4366L16.8785 17.586ZM4.5981 15.4964L5.77194 15.5795L5.63444 17.5103L7.45487 16.2445L8.12689 17.2105L6.30408 18.4777L8.16464 19.0259L7.83207 20.1547L4.79504 19.2599C4.52989 19.1818 4.35487 18.9297 4.3744 18.654L4.5981 15.4964ZM12.1772 13.8522L14.2957 16.41L13.3409 17.2008L12.0801 15.6786L11.6713 18.019L10.45 17.8055L10.8592 15.4615L9.16178 16.4623L8.53246 15.3941L11.3851 13.7135C11.6474 13.559 11.983 13.6178 12.1772 13.8522ZM18.1045 12.1361L19.1628 11.5603L20.7045 14.394C20.8396 14.6422 20.7839 14.951 20.5707 15.1364L20.3525 15.3249L18.1268 17.2619L17.3362 16.3528L18.8325 15.051L16.6037 14.6495L16.8175 13.4638L19.0448 13.8651L18.1045 12.1361ZM5.62376 9.17356L6.42099 10.0769L4.94518 11.3785L7.17825 11.7683L6.97127 12.9552L4.73574 12.5651L5.69598 14.301L4.64174 14.8842L3.07531 12.0526C2.93854 11.8053 2.99198 11.4963 3.20381 11.3093L5.62376 9.17356ZM13.172 8.24164L15.2834 10.7926L14.3283 11.5831L13.0677 10.0602L12.659 12.4013L11.4377 12.1878L11.8454 9.85288L10.1545 10.8441L9.52761 9.77445L12.1249 8.25162L12.1275 8.24173L12.1389 8.24322L12.381 8.10209C12.6432 7.9484 12.9782 8.0075 13.172 8.24164ZM15.8928 6.98711L16.3546 5.87432L19.3415 7.11388C19.6023 7.22208 19.7529 7.49679 19.7039 7.77478L19.6542 8.05129L19.1441 10.9527L17.9576 10.7437L18.2995 8.80149L16.3178 9.89673L15.7354 8.84203L17.7209 7.74589L15.8928 6.98711ZM10.2026 4.79426L10.2539 5.99796L8.28078 6.08229L9.76896 7.78298L8.86226 8.57634L7.37158 6.87311L7.02428 8.81762L5.83831 8.60549L6.40668 5.4278C6.45635 5.15008 6.69212 4.94404 6.97398 4.93202L7.2532 4.91953L7.25947 4.91499L10.2026 4.79426ZM14.2388 2.22722L16.3406 4.79423L15.3813 5.57968L14.1338 4.05586L13.7131 6.39386L12.4929 6.17428L12.9143 3.83101L11.2111 4.82682L10.5856 3.75636L13.4464 2.08471C13.7096 1.93089 14.0456 1.99132 14.2388 2.22722Z" fill="#201D1E"/>
</g>
<defs>
<clipPath id="clip0_127_41">
<rect width="17.7778" height="20" fill="white" transform="translate(3 2)"/>
</clipPath>
</defs>
</svg>

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 3.6 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

24
plugins/packages/airtable/lib/index.ts
View file

@ -1,4 +1,4 @@
import { QueryError, QueryResult, QueryService } from '@tooljet-plugins/common';
import { QueryError, QueryResult, QueryService, ConnectionTestResult } from '@tooljet-plugins/common';
import { SourceOptions, QueryOptions } from './types';
import { sanitizeSortPairs } from '@tooljet-plugins/common';
import got, { Headers } from 'got';
@ -167,4 +167,26 @@ export default class AirtableQueryService implements QueryService {
data: result,
};
}
async testConnection(sourceOptions: SourceOptions): Promise<ConnectionTestResult> {
try {
const apiToken = sourceOptions.personal_access_token;
const response = await got('https://api.airtable.com/v0/meta/whoami', {
headers: this.authHeader(apiToken),
});
const responseBody = JSON.parse(response.body);
if (responseBody && responseBody.id) {
return { status: 'ok' };
}
throw new Error('Invalid response from Airtable');
} catch (error) {
if (error.response?.statusCode === 401) {
throw new Error('Authentication failed: Invalid personal access token');
}
throw error;
}
}
}

46
plugins/packages/airtable/lib/manifest.json
View file

@ -2,38 +2,34 @@
"$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json",
"title": "Airtable datasource",
"description": "A schema defining airtable datasource",
"type": "api",
"source": {
"type": "object",
"tj:version": "1.0.0",
"tj:source": {
"name": "Airtable",
"kind": "airtable",
"exposedVariables": {
"isLoading": false,
"data": {},
"rawData": {}
},
"options": {
"personal_access_token": {
"type": "string",
"encrypted": true
}
},
"customTesting": true
},
"defaults": {
"personal_access_token": {
"value": ""
}
"type": "api"
},
"properties": {
"personal_access_token": {
"label": "Personal access token",
"key": "personal_access_token",
"type": "password",
"description": "Personal access token for airtable",
"helpText": "For generating personal access token, visit: <a href='https://airtable.com/account' target='_blank' rel='noreferrer'>Airtable account page</a>"
"type": "string",
"title": "Personal access token",
"description": "Personal access token for airtable"
}
},
"tj:encrypted": [
"personal_access_token"
],
"required": [
"personal_access_token"
]
],
"tj:ui:properties": {
"personal_access_token": {
"$ref": "#/properties/personal_access_token",
"key": "personal_access_token",
"label": "Personal access token",
"description": "Personal access token for airtable",
"widget": "password-v3-textarea",
"required": true
}
}
}

42
plugins/packages/bigquery/lib/manifest.json
View file

@ -2,36 +2,34 @@
"$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json",
"title": "Bigquery datasource",
"description": "A schema defining BigQuery datasource",
"type": "database",
"source": {
"type": "object",
"tj:version": "1.0.0",
"tj:source": {
"name": "BigQuery",
"kind": "bigquery",
"exposedVariables": {
"isLoading": false,
"data": {},
"rawData": {}
},
"options": {
"private_key": {
"encrypted": true
}
}
},
"defaults": {
"private_key": {
"value": ""
}
"type": "database"
},
"properties": {
"private_key": {
"label": "Private key",
"key": "private_key",
"type": "textarea",
"encrypted": true,
"type": "string",
"title": "Private key",
"description": "Enter JSON private key for service account"
}
},
"tj:encrypted": [
"private_key"
],
"required": [
"private_key"
]
],
"tj:ui:properties": {
"private_key": {
"$ref": "#/properties/private_key",
"key": "private_key",
"label": "Private key",
"description": "Enter JSON private key for service account",
"widget": "password-v3-textarea",
"required": true
}
}
}

4
plugins/packages/common/lib/query.error.ts
View file

@ -1,11 +1,13 @@
export class QueryError extends Error {
data: Record<string, unknown>;
description: any;
constructor(message: string | undefined, description: any, data: Record<string, unknown>) {
metadata?: unknown;
constructor(message: string | undefined, description: unknown, data: Record<string, unknown>, metadata?: unknown) {
super(message);
this.name = this.constructor.name;
this.data = data;
this.description = description;
this.metadata = metadata;
console.log(this.description);
}

25
plugins/packages/common/lib/utils.helper.ts
View file

@ -83,14 +83,26 @@ export const sanitizeHeaders = (
hasDataSource = true
): { [k: string]: string } => {
const cleanHeaders = (headers) => headers.filter(([k, _]) => k !== '').map(([k, v]) => [k.trim(), v]);
const filterValidHeaderEntries = (headers) => {
return headers.filter(([_, value]) => {
if (value == null) return false;
if (typeof value === 'string') return true;
if (Array.isArray(value) && value.every((v) => typeof v === 'string')) return true;
return false;
});
};
const _queryHeaders = cleanHeaders(queryOptions.headers || []);
const queryHeaders = Object.fromEntries(_queryHeaders);
const processHeaders = (rawHeaders) => {
const cleaned = cleanHeaders(rawHeaders || []);
const validHeaders = filterValidHeaderEntries(cleaned);
return Object.fromEntries(validHeaders);
};
const queryHeaders = processHeaders(queryOptions.headers || []);
if (!hasDataSource) return queryHeaders;
const _sourceHeaders = cleanHeaders(sourceOptions.headers || []);
const sourceHeaders = Object.fromEntries(_sourceHeaders);
const sourceHeaders = processHeaders(sourceOptions.headers || []);
return { ...queryHeaders, ...sourceHeaders };
};
@ -121,8 +133,11 @@ export const sanitizeSearchParams = (sourceOptions: any, queryOptions: any, hasD
});
if (!hasDataSource) return _urlParams;
const sanitisedUrlParamsFromSourceOptions = (sourceOptions.url_params || []).filter((o) => {
return o.some((e) => !isEmpty(e));
});
const urlParams = _urlParams.concat(sourceOptions.url_params || []);
const urlParams = _urlParams.concat(sanitisedUrlParamsFromSourceOptions || []);
return urlParams;
};

323
plugins/packages/mongodb/lib/manifest.json
View file

@ -2,82 +2,164 @@
"$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json",
"title": "Mongodb datasource",
"description": "A schema defining mongodb datasource",
"type": "database",
"source": {
"type": "object",
"tj:version": "1.0.0",
"tj:source": {
"name": "MongoDB",
"kind": "mongodb",
"exposedVariables": {
"isLoading": false,
"data": {},
"rawData": {}
},
"options": {
"host": {
"type": "string"
},
"port": {
"type": "string"
},
"username": {
"type": "string"
},
"password": {
"type": "string",
"encrypted": true
},
"connection_type": {
"type": "options"
},
"connection_string": {
"type": "string",
"encrypted": true
},
"ca_cert": {
"encrypted": true
},
"client_key": {
"encrypted": true
},
"client_cert": {
"encrypted": true
},
"root_cert": {
"encrypted": true
}
}
},
"defaults": {
"database": {
"value": ""
},
"host": {
"value": "localhost"
},
"port": {
"value": 27017
},
"username": {
"value": ""
},
"password": {
"value": ""
},
"connection_type": {
"value": "manual"
},
"connection_string": {
"value": ""
},
"tls_certificate": {
"value": "none"
}
"type": "database"
},
"properties": {
"connection_type": {
"label": "Connection type",
"key": "connection_type",
"type": "dropdown-component-flip",
"type": "string",
"title": "Connection type",
"description": "Single select dropdown for connection_type",
"enum": [
"manual",
"string"
],
"default": "manual"
},
"host": {
"type": "string",
"title": "Host",
"description": "Enter host",
"default": "localhost"
},
"port": {
"type": "number",
"title": "Port",
"description": "Enter port",
"default": 27017
},
"database": {
"type": "string",
"title": "Database name",
"description": "Name of the database"
},
"username": {
"type": "string",
"title": "Username",
"description": "Enter username"
},
"password": {
"type": "string",
"title": "Password",
"description": "Enter password"
},
"tls_certificate": {
"type": "string",
"title": "TLS/SSL Certificate",
"description": "Single select dropdown for choosing certificates",
"enum": [
"ca_certificate",
"client_certificate",
"none"
],
"default": "none"
},
"connection_string": {
"type": "string",
"title": "Connection string",
"description": "mongodb+srv://tooljet:<password>@cluster0.i1vq4.mongodb.net/mydb?retryWrites=true&w=majority"
},
"ca_cert": {
"type": "string",
"title": "CA Cert",
"description": "Enter CA certificate"
},
"client_key": {
"type": "string",
"title": "Client Key",
"description": "Enter client key"
},
"client_cert": {
"type": "string",
"title": "Client Cert",
"description": "Enter client certificate"
}
},
"tj:encrypted": [
"password",
"ca_cert",
"client_key",
"client_cert",
"connection_string"
],
"required": [
"connection_type"
],
"allOf": [
{
"if": {
"properties": {
"connection_type": {
"const": "manual"
}
}
},
"then": {
"required": [
"host",
"port",
"tls_certificate"
],
"allOf": [
{
"if": {
"properties": {
"tls_certificate": {
"const": "ca_certificate"
}
}
},
"then": {
"required": [
"ca_cert"
]
}
},
{
"if": {
"properties": {
"tls_certificate": {
"const": "client_certificate"
}
}
},
"then": {
"required": [
"client_key",
"client_cert",
"ca_cert"
]
}
}
]
}
},
{
"if": {
"properties": {
"connection_type": {
"const": "string"
}
}
},
"then": {
"required": [
"connection_string"
]
}
}
],
"tj:ui:properties": {
"connection_type": {
"$ref": "#/properties/connection_type",
"key": "connection_type",
"label": "Connection type",
"description": "Single select dropdown for connection_type",
"widget": "dropdown-component-flip",
"list": [
{
"name": "Manual connection",
@ -91,88 +173,107 @@
},
"manual": {
"tls_certificate": {
"label": "TLS/SSL Certificate",
"$ref": "#/properties/tls_certificate",
"key": "tls_certificate",
"type": "dropdown-component-flip",
"label": "TLS/SSL Certificate",
"description": "Single select dropdown for choosing certificates",
"widget": "dropdown-component-flip",
"list": [
{ "value": "ca_certificate", "name": "CA certificate" },
{ "value": "client_certificate", "name": "Client certificate" },
{ "value": "none", "name": "None" }
{
"value": "ca_certificate",
"name": "CA certificate"
},
{
"value": "client_certificate",
"name": "Client certificate"
},
{
"value": "none",
"name": "None"
}
],
"commonFields":{
"commonFields": {
"host": {
"label": "Host",
"$ref": "#/properties/host",
"key": "host",
"type": "text",
"description": "Enter host"
"label": "Host",
"description": "Enter host",
"widget": "text-v3",
"required": true
},
"port": {
"label": "Port",
"$ref": "#/properties/port",
"key": "port",
"type": "text",
"description": "Enter port"
"label": "Port",
"description": "Enter port",
"widget": "text-v3",
"required": true
},
"database": {
"label": "Database Name",
"$ref": "#/properties/database",
"key": "database",
"type": "text",
"description": "Name of the database"
"label": "Database name",
"description": "Name of the database",
"widget": "text-v3"
},
"username": {
"label": "Username",
"$ref": "#/properties/username",
"key": "username",
"type": "text",
"description": "Enter username"
"label": "Username",
"description": "Enter username",
"widget": "text-v3"
},
"password": {
"label": "Password",
"$ref": "#/properties/password",
"key": "password",
"type": "password",
"description": "Enter password"
"label": "Password",
"description": "Enter password",
"widget": "password-v3",
"required": false
}
}
},
"ca_certificate": {
"ca_cert": {
"label": "CA Cert",
"$ref": "#/properties/ca_cert",
"key": "ca_cert",
"type": "textarea",
"encrypted": true,
"description": "Enter CA certificate"
"label": "CA Cert",
"description": "Enter CA certificate",
"widget": "password-v3-textarea"
}
},
"client_certificate": {
"client_key": {
"label": "Client Key",
"$ref": "#/properties/client_key",
"key": "client_key",
"type": "textarea",
"encrypted": true,
"description": "Enter client key"
"label": "Client Key",
"description": "Enter client key",
"widget": "password-v3-textarea"
},
"client_cert": {
"label": "Client Cert",
"$ref": "#/properties/client_cert",
"key": "client_cert",
"type": "textarea",
"encrypted": true,
"description": "Enter client certificate"
"label": "Client Cert",
"description": "Enter client certificate",
"widget": "password-v3-textarea"
},
"ca_cert": {
"label": "CA Cert",
"$ref": "#/properties/ca_cert",
"key": "ca_cert",
"type": "textarea",
"encrypted": true,
"description": "Enter CA certificate"
"label": "CA Cert",
"description": "Enter CA certificate",
"widget": "password-v3-textarea"
}
}
},
"string": {
"connection_string": {
"label": "Connection string",
"$ref": "#/properties/connection_string",
"key": "connection_string",
"type": "text",
"encrypted": true,
"description": "mongodb+srv://tooljet:<password>@cluster0.i1vq4.mongodb.net/mydb?retryWrites=true&w=majority"
"label": "Connection string",
"description": "mongodb+srv://tooljet:<password>@cluster0.i1vq4.mongodb.net/mydb?retryWrites=true&w=majority",
"widget": "password-v3-textarea",
"required": true
}
}
}

176
plugins/packages/mssql/lib/manifest.json
View file

@ -2,116 +2,128 @@
"$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json",
"title": "Mssql datasource",
"description": "A schema defining mssql datasource",
"type": "database",
"source": {
"type": "object",
"tj:version": "1.0.0",
"tj:source": {
"name": "SQL Server",
"kind": "mssql",
"exposedVariables": {
"isLoading": false,
"data": {},
"rawData": {}
},
"options": {
"host": {
"type": "string"
},
"instanceName": {
"type": "string"
},
"port": {
"type": "string"
},
"database": {
"type": "string"
},
"username": {
"type": "string"
},
"password": {
"type": "string",
"encrypted": true
},
"connection_options": {
"type": "array"
}
}
},
"defaults": {
"host": {
"value": "localhost"
},
"instanceName": {
"value": ""
},
"port": {
"value": 1433
},
"database": {
"value": ""
},
"username": {
"value": ""
},
"password": {
"value": ""
},
"azure": {
"value": false
}
"type": "database"
},
"properties": {
"host": {
"label": "Host",
"key": "host",
"type": "text",
"description": "Enter host"
"type": "string",
"title": "Host",
"description": "Enter host",
"default": "localhost"
},
"instanceName": {
"label": "Instance",
"key": "instance_name",
"type": "text",
"type": "string",
"title": "Instance",
"description": "Enter the name of the database instance"
},
"port": {
"label": "Port",
"key": "port",
"type": "text",
"description": "Enter port"
"type": "number",
"title": "Port",
"description": "Enter port",
"default": 1433
},
"database": {
"label": "Database Name",
"key": "database",
"type": "text",
"type": "string",
"title": "Database name",
"description": "Name of the database"
},
"username": {
"label": "Username",
"key": "username",
"type": "text",
"type": "string",
"title": "Username",
"description": "Enter username"
},
"password": {
"label": "Password",
"key": "password",
"type": "password",
"type": "string",
"title": "Password",
"description": "Enter password"
},
"connection_options": {
"label": "Connection Options",
"key": "connection_options",
"type": "react-component-headers"
"type": "array",
"title": "Connection Options",
"description": "Connection options"
},
"azure": {
"label": "Azure (encrypt connection)",
"key": "azure",
"type": "toggle",
"description": "Toggle for azure"
"type": "boolean",
"title": "Azure (encrypt connection)",
"description": "Toggle for azure",
"default": false
}
},
"tj:encrypted": [
"password"
],
"required": [
"host",
"port",
"database",
"username",
"password"
]
],
"tj:ui:properties": {
"host": {
"$ref": "#/properties/host",
"key": "host",
"label": "Host",
"description": "Enter host",
"widget": "text-v3",
"required": true
},
"instanceName": {
"$ref": "#/properties/instanceName",
"key": "instance_name",
"label": "Instance",
"description": "Enter the name of the database instance",
"widget": "text"
},
"port": {
"$ref": "#/properties/port",
"key": "port",
"label": "Port",
"description": "Enter port",
"widget": "text-v3",
"required": true
},
"database": {
"$ref": "#/properties/database",
"key": "database",
"label": "Database name",
"description": "Name of the database",
"widget": "text-v3",
"required": true
},
"username": {
"$ref": "#/properties/username",
"key": "username",
"label": "Username",
"description": "Enter username",
"widget": "text-v3",
"required": true
},
"password": {
"$ref": "#/properties/password",
"key": "password",
"label": "Password",
"description": "Enter password",
"widget": "password-v3",
"required": true
},
"connection_options": {
"$ref": "#/properties/connection_options",
"key": "connection_options",
"label": "Connection Options",
"widget": "react-component-headers"
},
"azure": {
"$ref": "#/properties/azure",
"key": "azure",
"label": "Azure (encrypt connection)",
"description": "Toggle for azure",
"widget": "toggle"
}
}
}

342
plugins/packages/mysql/lib/manifest.json
View file

@ -2,87 +2,175 @@
"$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json",
"title": "Mysql datasource",
"description": "A schema defining mysql datasource",
"type": "database",
"source": {
"type": "object",
"tj:version": "1.0.0",
"tj:source": {
"name": "MySQL",
"kind": "mysql",
"exposedVariables": {
"isLoading": false,
"data": {},
"rawData": {}
},
"options": {
"connection_type": {
"type": "string"
},
"host": {
"type": "string"
},
"port": {
"type": "string"
},
"database": {
"type": "string"
},
"socket_path": {
"type": "string"
},
"username": {
"type": "string"
},
"password": {
"type": "string",
"encrypted": true
},
"ca_cert": {
"encrypted": true
},
"client_key": {
"encrypted": true
},
"client_cert": {
"encrypted": true
},
"root_cert": {
"encrypted": true
}
}
},
"defaults": {
"connection_type": {
"value": "hostname"
},
"host": {
"value": "localhost"
},
"port": {
"value": 3306
},
"database": {
"value": ""
},
"socket": {
"value": ""
},
"username": {
"value": ""
},
"password": {
"value": ""
},
"ssl_enabled": {
"value": false
},
"ssl_certificate": {
"value": "none"
}
"type": "database"
},
"properties": {
"connection_type": {
"type": "string",
"title": "Connection type",
"description": "Single select dropdown for connection type",
"enum": [
"hostname",
"socket_path"
],
"default": "hostname"
},
"host": {
"type": "string",
"title": "Host",
"description": "Enter host",
"default": "localhost"
},
"port": {
"type": "number",
"title": "Port",
"description": "Enter port",
"default": 3306
},
"database": {
"type": "string",
"title": "Database name",
"description": "Name of the database"
},
"socket_path": {
"type": "string",
"title": "Socket path",
"description": "Enter the socket path"
},
"username": {
"type": "string",
"title": "Username",
"description": "Enter username"
},
"password": {
"type": "string",
"title": "Password",
"description": "Enter password"
},
"ssl_enabled": {
"type": "boolean",
"title": "SSL",
"description": "Toggle for ssl_enabled",
"default": false
},
"ssl_certificate": {
"label": "SSL certificate",
"key": "ssl_certificate",
"type": "dropdown-component-flip",
"type": "string",
"title": "SSL certificate",
"description": "Single select dropdown for choosing certificates",
"enum": [
"ca_certificate",
"self_signed",
"none"
],
"default": "none"
},
"ca_cert": {
"type": "string",
"title": "CA cert",
"description": "Enter ca certificate"
},
"client_key": {
"type": "string",
"title": "Client key",
"description": "Enter client key"
},
"client_cert": {
"type": "string",
"title": "Client cert",
"description": "Enter client certificate"
},
"root_cert": {
"type": "string",
"title": "Root cert",
"description": "Enter root certificate"
}
},
"tj:encrypted": [
"password",
"ca_cert",
"client_key",
"client_cert",
"root_cert"
],
"required": [
"connection_type",
"username",
"password",
"database"
],
"allOf": [
{
"if": {
"properties": {
"connection_type": {
"const": "hostname"
}
}
},
"then": {
"required": [
"host",
"port"
]
}
},
{
"if": {
"properties": {
"connection_type": {
"const": "socket_path"
}
}
},
"then": {
"required": [
"socket_path"
]
}
},
{
"if": {
"properties": {
"ssl_certificate": {
"const": "ca_certificate"
}
}
},
"then": {
"required": [
"ca_cert"
]
}
},
{
"if": {
"properties": {
"ssl_certificate": {
"const": "self_signed"
}
}
},
"then": {
"required": [
"client_key",
"client_cert",
"root_cert"
]
}
}
],
"tj:ui:properties": {
"ssl_certificate": {
"$ref": "#/properties/ssl_certificate",
"key": "ssl_certificate",
"label": "SSL certificate",
"description": "Single select dropdown for choosing certificates",
"widget": "dropdown-component-flip",
"list": [
{
"value": "ca_certificate",
@ -99,10 +187,11 @@
],
"commonFields": {
"connection_type": {
"label": "Connection type",
"$ref": "#/properties/connection_type",
"key": "connection_type",
"type": "dropdown-component-flip",
"label": "Connection type",
"description": "Single select dropdown for connection type",
"widget": "dropdown-component-flip",
"list": [
{
"value": "hostname",
@ -115,92 +204,99 @@
],
"commonFields": {
"username": {
"label": "Username",
"$ref": "#/properties/username",
"key": "username",
"type": "text",
"description": "Enter username"
"label": "Username",
"description": "Enter username",
"widget": "text-v3",
"required": true
},
"password": {
"label": "Password",
"$ref": "#/properties/password",
"key": "password",
"type": "password",
"description": "Enter password"
"label": "Password",
"description": "Enter password",
"widget": "password-v3",
"required": true
},
"database": {
"label": "Database name",
"$ref": "#/properties/database",
"key": "database",
"type": "text",
"description": "Name of the database"
"label": "Database name",
"description": "Name of the database",
"widget": "text-v3",
"required": true
}
}
},
"hostname": {
"host": {
"label": "Host",
"$ref": "#/properties/host",
"key": "host",
"type": "text",
"description": "Enter host"
"label": "Host",
"description": "Enter host",
"widget": "text-v3",
"required": true
},
"port": {
"label": "Port",
"$ref": "#/properties/port",
"key": "port",
"type": "text",
"description": "Enter port"
"label": "Port",
"description": "Enter port",
"widget": "text-v3",
"required": true
},
"ssl_enabled": {
"label": "SSL",
"$ref": "#/properties/ssl_enabled",
"key": "ssl_enabled",
"type": "toggle",
"description": "Toggle for ssl_enabled"
"label": "SSL",
"description": "Toggle for ssl_enabled",
"widget": "toggle"
}
},
"socket_path": {
"socket": {
"socket_path": {
"$ref": "#/properties/socket_path",
"key": "socket_path",
"label": "Socket path",
"type": "text",
"name": "socket_path",
"description": "Enter the socket path"
"description": "Enter the socket path",
"widget": "text-v3",
"required": true
}
}
}
},
"ca_certificate": {
"ca_cert": {
"label": "CA cert",
"$ref": "#/properties/ca_cert",
"key": "ca_cert",
"type": "textarea",
"encrypted": true,
"description": "Enter ca certificate"
"label": "CA cert",
"description": "Enter ca certificate",
"widget": "password-v3-textarea"
}
},
"self_signed": {
"client_key": {
"label": "Client key",
"$ref": "#/properties/client_key",
"key": "client_key",
"type": "textarea",
"encrypted": true,
"description": "Enter client key"
"label": "Client key",
"description": "Enter client key",
"widget": "password-v3-textarea"
},
"client_cert": {
"label": "Client cert",
"$ref": "#/properties/client_cert",
"key": "client_cert",
"type": "textarea",
"encrypted": true,
"description": "Enter client certificate"
"label": "Client cert",
"description": "Enter client certificate",
"widget": "password-v3-textarea"
},
"root_cert": {
"label": "Root cert",
"$ref": "#/properties/root_cert",
"key": "root_cert",
"type": "textarea",
"encrypted": true,
"description": "Enter root certificate"
"label": "Root cert",
"description": "Enter root certificate",
"widget": "password-v3-textarea"
}
}
},
"required": [
"connection_type",
"port",
"username",
"password"
]
}
}
}

9
plugins/packages/nocodb/lib/manifest.json
View file

@ -1,11 +1,10 @@
{
"$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json",
"title": "Nocodb datasource",
"description": "A schema defining Nocodb datasource",
"type": "api",
"source": {
"name": "Nocodb",
"name": "NocoDB",
"kind": "nocodb",
"exposedVariables": {
"isLoading": false,
@ -15,14 +14,14 @@
"options": {
"api_token": {
"encrypted": true
}
}
},
"customTesting": true
},
"defaults": {
"api_token": {
"value": ""
}
}
},
"properties": {
"host": {
@ -61,4 +60,4 @@
"required": [
"api_token"
]
}
}

1
plugins/packages/postgresql/lib/index.ts
View file

@ -113,7 +113,6 @@ export default class PostgresqlQueryService implements QueryService {
} else if (sourceOptions.connection_type === 'string' && sourceOptions.connection_string) {
connectionConfig = {
connectionString: sourceOptions.connection_string,
ssl: this.getSslConfig(sourceOptions),
};
}
const connectionOptions: Knex.Config = {

11
plugins/packages/postgresql/lib/manifest.json
View file

@ -269,7 +269,7 @@
"key": "ca_cert",
"label": "CA Cert",
"description": "Enter ca certificate",
"widget": "textarea"
"widget": "password-v3-textarea"
}
},
"self_signed": {
@ -278,22 +278,21 @@
"key": "client_key",
"label": "Client Key",
"description": "Enter client key",
"widget": "textarea"
"widget": "password-v3-textarea"
},
"client_cert": {
"$ref": "#/properties/client_cert",
"key": "client_cert",
"label": "Client Cert",
"description": "Enter client certificate",
"widget": "textarea"
"widget": "password-v3-textarea"
},
"root_cert": {
"$ref": "#/properties/root_cert",
"key": "root_cert",
"label": "Root Cert",
"description": "Enter root certificate",
"widget": "textarea",
"required": true
"widget": "password-v3-textarea"
}
}
},
@ -303,7 +302,7 @@
"key": "connection_string",
"label": "Connection string",
"description": "postgres://username:password@hostname:port/database?sslmode=require",
"widget": "text",
"widget": "password-v3-textarea",
"required": true
}
}

36
plugins/packages/restapi/lib/index.ts
View file

@ -193,6 +193,7 @@ export default class RestapiQueryService implements QueryService {
let result = {};
let requestObject = {};
let responseObject = {};
let metadata = {};
try {
const response = await got(url, requestOptions);
@ -217,24 +218,39 @@ export default class RestapiQueryService implements QueryService {
if (error instanceof HTTPError) {
const requestUrl = error?.request?.options?.url?.origin + error?.request?.options?.url?.pathname;
const requestHeaders = cleanSensitiveData(error?.request?.options?.headers, ['authorization']);
result = {
requestObject: {
requestUrl,
requestHeaders,
requestObject = {
requestUrl: requestUrl,
requestHeaders: requestHeaders,
requestParams: urrl.parse(error.request.requestUrl, true).query,
},
responseObject: {
statusCode: error.response.statusCode,
responseBody: error.response.body,
},
};
responseObject = {
statusCode: error.response.statusCode,
responseBody: error.response.body,
headers: redactHeaders(error.response.headers),
}
metadata = {
request: requestObject,
response: responseObject,
};
// TODO: Need to remove the request/response related information in result in next MAJOR release.
// This is now shared in `metadata` key. Keeping this here for backward compatibility.
result = {
requestObject: requestObject,
responseObject: responseObject,
responseHeaders: error.response.headers,
};
}
if (sourceOptions['auth_type'] === 'oauth2' && error?.response?.statusCode == 401) {
throw new OAuthUnauthorizedClientError('Unauthorized status from API server', error.message, result);
}
throw new QueryError('Query could not be completed', error.message, result);
throw new QueryError('Query could not be completed', error.message, result, metadata);
}
return {

2
server/.version
View file

@ -1 +1 @@
3.11.0
3.12.0

60
server/src/assets/marketplace/plugins.json
View file

@ -1,6 +1,6 @@
[
{
"name": "plivo",
"name": "Plivo",
"description": "Plugin for Plivo APIs",
"version": "1.0.0",
"id": "plivo",
@ -22,7 +22,9 @@
"id": "openai",
"author": "Tooljet",
"timestamp": "Mon, 10 Apr 2023 06:33:21 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "AWS Textract",
@ -35,7 +37,7 @@
{
"name": "HarperDB",
"repo": "",
"description": "Plugin for HarperDB data source",
"description": "Plugin to store and query data from HarperDB",
"version": "1.0.0",
"id": "harperdb",
"author": "Tooljet",
@ -82,7 +84,7 @@
"timestamp": "Thu, 29 Feb 2024 09:46:21 GMT"
},
{
"name": "salesforce",
"name": "Salesforce",
"description": "API plugin from salesforce",
"version": "1.0.0",
"id": "salesforce",
@ -91,7 +93,7 @@
},
{
"name": "Presto",
"description": "Plugin for PrestoDB data source",
"description": "Plugin for open source SQL query engine",
"version": "1.0.0",
"id": "presto",
"author": "Tooljet",
@ -107,7 +109,7 @@
},
{
"name": "Jira",
"description": "Plugin for Jira data source",
"description": "Integrate with Jira for projects management",
"version": "1.0.0",
"id": "jira",
"author": "Tooljet",
@ -120,16 +122,20 @@
"id": "portkey",
"author": "Portkey",
"timestamp": "Sat, 29 Jun 2024 09:40:13 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "Pinecone",
"description": "Plugin for Pinecone Vector DB",
"description": "Plugin to store and manage data in Vector DB",
"version": "1.0.0",
"id": "pinecone",
"author": "Tooljet",
"timestamp": "Mon, 28 Oct 2024 08:08:28 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "Cohere",
@ -138,7 +144,9 @@
"id": "cohere",
"author": "Tooljet",
"timestamp": "Tue, 21 Jan 2025 05:09:30 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "Mistral",
@ -147,7 +155,9 @@
"id": "mistral_ai",
"author": "Tooljet",
"timestamp": "Tue, 21 Jan 2025 06:35:01 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "Hugging Face",
@ -156,7 +166,9 @@
"id": "hugging_face",
"author": "Tooljet",
"timestamp": "Thu, 23 Jan 2025 06:44:25 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "Gemini",
@ -165,7 +177,9 @@
"id": "gemini",
"author": "Tooljet",
"timestamp": "Fri, 17 Jan 2025 18:04:48 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "Anthropic",
@ -174,16 +188,20 @@
"id": "anthropic",
"author": "Tooljet",
"timestamp": "Mon, 20 Jan 2025 08:04:46 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "Qdrant",
"description": "Plugin for Qdrant APIs",
"description": "Plugin to store and manage data in Vector DB",
"version": "1.0.0",
"id": "qdrant",
"author": "Tooljet",
"timestamp": "Tue, 10 Dec 2024 02:11:32 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "Weaviate DB",
@ -192,14 +210,16 @@
"id": "weaviate",
"author": "Tooljet",
"timestamp": "Tue, 21 Jan 2025 16:55:28 GMT",
"tags": ["AI"]
"tags": [
"AI"
]
},
{
"name": "azurerepos",
"description": "api plugin from azurerepos",
"name": "Azure Repos",
"description": "Plugin for managing Azure repositories",
"version": "1.0.0",
"id": "azurerepos",
"author": "Tooljet",
"timestamp": "Mon, 23 Dec 2024 11:57:30 GMT"
}
]
]

21
server/src/helpers/tooljet_db.helper.ts
View file

@ -226,11 +226,28 @@ export function modifyTjdbErrorObject(error) {
* @returns - Column names with invalid JSON data.
*/
export function validateTjdbJSONBColumnInputs(jsonbColumnList: Array<string>, inputValues) {
const body = { ...inputValues };
const inValidValueColumnsList = [];
Object.entries(inputValues).forEach(([key, value]) => {
if (jsonbColumnList.includes(key)) {
if (typeof value !== 'object') inValidValueColumnsList.push(key);
try {
const parsedValue = typeof value === 'string' ? JSON.parse(value) : value;
const isJson =
typeof parsedValue === 'object' &&
parsedValue !== null &&
!Array.isArray(parsedValue) &&
Object.prototype.toString.call(parsedValue) === '[object Object]';
if (isJson || Array.isArray(parsedValue) || value === null) {
body[key] = parsedValue;
} else {
inValidValueColumnsList.push(key);
}
} catch (error) {
inValidValueColumnsList.push(key);
}
}
});
return inValidValueColumnsList;
return { inValidValueColumnsList, updatedRequestBody: body };
}

1
server/src/modules/data-queries/service.ts
View file

@ -174,6 +174,7 @@ export class DataQueriesService implements IDataQueriesService {
message: error.message,
description: error.description,
data: error.data,
metadata: error.metadata,
};
} else {
console.log(error);

7
server/src/modules/data-sources/controller.ts
View file

@ -128,4 +128,11 @@ export class DataSourcesController implements IDataSourcesController {
await this.dataSourcesService.authorizeOauth2(id, environmentId, authorizeDataSourceOauthDto, user);
return;
}
@InitFeature(FEATURE_KEY.AUTHORIZE)
@UseGuards(FeatureAbilityGuard)
@Post('decrypt')
async decryptOptions(@Body() options: Record<string, any>) {
return await this.dataSourcesService.decryptOptions(options);
}
}

2
server/src/modules/data-sources/interfaces/IController.ts
View file

@ -44,4 +44,6 @@ export interface IDataSourcesController {
environmentId: string,
authorizeDataSourceOauthDto: AuthorizeDataSourceOauthDto
): Promise<void>;
decryptOptions(options: Record<string, any>): Promise<any>;
}

4
server/src/modules/data-sources/service.ts
View file

@ -164,6 +164,10 @@ export class DataSourcesService implements IDataSourcesService {
return;
}
async decryptOptions(options: Record<string, any>) {
return await this.dataSourcesUtilService.decrypt(options);
}
async delete(dataSourceId: string, user: User) {
const dataSource = await this.dataSourcesRepository.findById(dataSourceId);
if (!dataSource) {

15
server/src/modules/data-sources/util.service.ts
View file

@ -214,6 +214,21 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
});
}
async decrypt(options: Record<string, any>) {
const decryptedOptions = { ...options };
for (const [key, value] of Object.entries(options)) {
if (value?.credential_id) {
decryptedOptions[key] = {
...value,
value: await this.credentialService.getValue(value.credential_id),
};
}
}
return decryptedOptions;
}
async parseOptionsForUpdate(dataSource: DataSource, options: Array<object>, manager: EntityManager) {
if (!options) return {};

17
server/src/modules/tooljet-db/services/postgrest-proxy.service.ts
View file

@ -70,7 +70,8 @@ export class PostgrestProxyService {
}
if (['PATCH', 'POST'].includes(req.method)) {
await this.validateJSONBInputs(organizationId, internalTable.tableName, req.body);
const updatedRequestBody = await this.validateJSONBInputs(organizationId, internalTable.tableName, req.body);
req.body = { ...req.body, ...updatedRequestBody };
}
return this.httpProxy(req, res, next);
@ -117,7 +118,12 @@ export class PostgrestProxyService {
}
if (['PATCH', 'POST'].includes(method)) {
await this.validateJSONBInputs(headers['tj-workspace-id'], internalTable.tableName, body);
const updatedRequestBody = await this.validateJSONBInputs(
headers['tj-workspace-id'],
internalTable.tableName,
body
);
body = { ...body, ...updatedRequestBody };
}
const reqHeaders = {
@ -266,14 +272,19 @@ export class PostgrestProxyService {
.map((column) => column.column_name);
if (jsonbColumns.length) {
const inValidJsonbColumns = validateTjdbJSONBColumnInputs(jsonbColumns, body);
const { inValidValueColumnsList: inValidJsonbColumns, updatedRequestBody } = validateTjdbJSONBColumnInputs(
jsonbColumns,
body
);
if (inValidJsonbColumns.length) {
throw new HttpException(
`Expected JSON values in the following columns : ${inValidJsonbColumns.join(', ')}`,
400
);
}
return updatedRequestBody;
}
return body;
}
}

192
server/src/modules/tooljet-db/services/tooljet-db-bulk-upload.service.ts
View file

@ -260,16 +260,18 @@ export class TooljetDbBulkUploadService {
}
convertToDataType(columnValue: string, supportedDataType: TooljetDatabaseDataTypes) {
if (!columnValue) return null;
if (!columnValue && supportedDataType !== TJDB.boolean) return null;
switch (supportedDataType) {
case TJDB.boolean:
if (typeof columnValue === 'boolean') return columnValue;
return this.convertBoolean(columnValue);
case TJDB.integer:
case TJDB.double_precision:
case TJDB.bigint:
return this.convertNumber(columnValue, supportedDataType);
case TJDB.jsonb:
if (typeof columnValue !== 'string') return columnValue;
return JSON.parse(columnValue);
default:
return columnValue;
@ -368,4 +370,192 @@ export class TooljetDbBulkUploadService {
};
}
}
async bulkUpsertRowsWithPrimaryKey(
rows: Record<string, any>[],
tableId: string,
primaryKeyColumns: string[],
organizationId: string
): Promise<{ status: string; inserted: number; updated: number; rows: any[]; error?: string }> {
const rowsToUpsert = [...rows];
if (isEmpty(rowsToUpsert)) {
return {
status: 'failed',
error: 'No rows provided for upsert operation',
inserted: 0,
updated: 0,
rows: [],
};
}
if (rowsToUpsert.length > this.MAX_ROW_COUNT) {
return {
status: 'failed',
error: `Row count cannot be greater than ${this.MAX_ROW_COUNT}`,
inserted: 0,
updated: 0,
rows: [],
};
}
const internalTable = await this.manager.findOne(InternalTable, {
where: { organizationId, id: tableId },
});
if (!internalTable) {
return {
status: 'failed',
error: 'Table not found',
inserted: 0,
updated: 0,
rows: [],
};
}
const result = await this.tableOperationsService.perform(organizationId, 'view_table', {
id: tableId,
});
const tableColumns = result?.columns || [];
// Check if pk columns exist in the table
const nonExistentColumns = primaryKeyColumns.filter((pk) => !tableColumns.some((col) => col.column_name === pk));
if (nonExistentColumns.length > 0) {
return {
status: 'failed',
error: `Primary key columns not found in table: ${nonExistentColumns.join(', ')}`,
inserted: 0,
updated: 0,
rows: [],
};
}
// Check if columns are actually primary keys
const nonPrimaryKeyColumns = primaryKeyColumns.filter(
(pk) => !tableColumns.some((col) => col.column_name === pk && col.constraints_type.is_primary_key)
);
if (nonPrimaryKeyColumns.length > 0) {
return {
status: 'failed',
error: `Columns are not primary keys: ${nonPrimaryKeyColumns.join(', ')}`,
inserted: 0,
updated: 0,
rows: [],
};
}
const serialTypeColumns = tableColumns
.filter((col) => col.data_type === 'integer' && /^nextval\(/.test(col.column_default))
.map((col) => col.column_name);
const serialPrimaryKeys = [];
const nonSerialPrimaryKeys = [];
primaryKeyColumns.forEach((pk) =>
serialTypeColumns.includes(pk) ? serialPrimaryKeys.push(pk) : nonSerialPrimaryKeys.push(pk)
);
// Group rows by the exact set of provided keys
const rowGroups = new Map<string, Record<string, any>[]>();
for (const row of rowsToUpsert) {
// Ensure required non-serial PKs are provided
const missingNonSerialPKs = nonSerialPrimaryKeys.filter((pk) => row[pk] === undefined);
if (missingNonSerialPKs.length > 0) {
return {
status: 'failed',
error: `Missing required non-serial primary key values: ${missingNonSerialPKs.join(', ')}`,
inserted: 0,
updated: 0,
rows: [],
};
}
// Create a group key based on the sorted keys present in this row.
const keys = Object.keys(row).sort();
const groupKey = keys.join(',');
if (!rowGroups.has(groupKey)) {
rowGroups.set(groupKey, []);
}
rowGroups.get(groupKey)!.push(row);
}
// Process each group separately – each group has a consistent set of keys.
const tenantSchema = findTenantSchema(organizationId);
let totalInserted = 0;
let totalUpdated = 0;
const allResultRows: any[] = [];
try {
for (const [groupKey, groupRows] of rowGroups.entries()) {
// The provided columns for this group are exactly the keys in the group.
const providedColumns = groupKey.split(','); // sorted keys
const columnsQuoted = providedColumns.map((col) => `"${col}"`);
// Build the VALUES clause for this group
let parameterIndex = 1;
const allValueSets: string[] = [];
const allPlaceholders: any[] = [];
for (const row of groupRows) {
const valueSet: string[] = [];
for (const col of providedColumns) {
// Since the group is built by the row's own keys, if the row doesn't have a key, that column won't be in providedColumns.
if (Object.prototype.hasOwnProperty.call(row, col)) {
if (row[col] === 'DEFAULT') {
valueSet.push('DEFAULT');
} else {
valueSet.push(`$${parameterIndex++}`);
allPlaceholders.push(row[col]);
}
} else {
valueSet.push('DEFAULT');
}
}
allValueSets.push(`(${valueSet.join(', ')})`);
}
// Determine if this group omits any serial PK
const omittedSerialPKs = serialPrimaryKeys.filter((pk) => !providedColumns.includes(pk));
let queryText: string;
if (omittedSerialPKs.length > 0) {
// For groups omitting serial PKs, use plain INSERT (so PostgreSQL auto-generates those values)
queryText = `
INSERT INTO "${tenantSchema}"."${tableId}" (${columnsQuoted.join(', ')})
VALUES ${allValueSets.join(', ')}
RETURNING *, true as inserted;
`;
} else {
// For groups with all primary keys provided, use UPSERT
const conflictTarget = primaryKeyColumns.map((col) => `"${col}"`).join(', ');
// Update only non-PK columns
const updateColumns = providedColumns.filter((col) => !primaryKeyColumns.includes(col));
const onConflictUpdates = updateColumns.map((col) => `"${col}" = EXCLUDED."${col}"`).join(',\n ');
queryText = `
INSERT INTO "${tenantSchema}"."${tableId}" (${columnsQuoted.join(', ')})
VALUES ${allValueSets.join(', ')}
ON CONFLICT (${conflictTarget})
DO UPDATE SET
${onConflictUpdates}
RETURNING *, (xmax = 0) as inserted;
`;
}
const result = await this.tooljetDbManager.query(queryText, allPlaceholders);
totalInserted += result.filter((row: any) => row.inserted).length;
totalUpdated += result.length - result.filter((row: any) => row.inserted).length;
allResultRows.push(...result.map(({ inserted, ...row }: any) => row));
}
return {
status: 'ok',
inserted: totalInserted,
updated: totalUpdated,
rows: allResultRows,
};
} catch (error) {
return {
status: 'failed',
error: error.message,
inserted: 0,
updated: 0,
rows: [],
};
}
}
}

66
server/src/modules/tooljet-db/services/tooljet-db-data-operations.service.ts
View file

@ -58,6 +58,8 @@ export class TooljetDbDataOperationsService implements QueryService {
return this.sqlExecution(queryOptions, context);
case 'bulk_update_with_primary_key':
return this.bulkUpdateWithPrimaryKey(queryOptions, context);
case 'bulk_upsert_with_primary_key':
return this.bulkUpsertUsingPrimaryKey(queryOptions, context);
default:
return {
status: 'failed',
@ -577,6 +579,70 @@ export class TooljetDbDataOperationsService implements QueryService {
return query;
}
async bulkUpsertUsingPrimaryKey(queryOptions, context): Promise<QueryResult> {
if (hasNullValueInFilters(queryOptions, 'bulk_upsert_with_primary_key')) {
return {
status: 'failed',
errorMessage: 'Null value comparison not allowed. To check null values, please use IS operator instead.',
data: {},
};
}
try {
const { table_id: tableId, bulk_upsert_with_primary_key: bulkUpsertOptions } = queryOptions;
const { primary_key: primaryKeyColumns, rows: rowsToUpsert } = bulkUpsertOptions;
const { organization_id: organizationId } = context.app;
// Validate input
if (!Array.isArray(rowsToUpsert) || rowsToUpsert.length === 0) {
return {
status: 'failed',
errorMessage: 'No rows provided for upsert operation',
data: {},
};
}
if (!Array.isArray(primaryKeyColumns) || primaryKeyColumns.length === 0) {
return {
status: 'failed',
errorMessage: 'No primary key columns specified',
data: {},
};
}
// Perform bulk upsert
const result = await this.tooljetDbBulkUploadService.bulkUpsertRowsWithPrimaryKey(
rowsToUpsert,
tableId,
primaryKeyColumns,
organizationId
);
if (result.status === 'failed') {
return {
status: 'failed',
errorMessage: result.error,
data: {},
};
}
return {
status: 'ok',
data: {
inserted: result.inserted,
updated: result.updated,
data: result.rows,
},
};
} catch (error) {
return {
status: 'failed',
errorMessage: error.message,
data: {},
};
}
}
}
function hasNullValueInFilters(queryOptions, operation) {

3
server/src/modules/tooljet-db/types.ts
View file

@ -73,7 +73,8 @@ export type TooljetDbActions =
| 'view_tables'
| 'sql_execution'
| 'bulk_upload'
| 'proxy_postgrest';
| 'proxy_postgrest'
| 'bulk_upsert_with_primary_key';
type ErrorCodeMappingItem = Partial<Record<TooljetDbActions | 'default', string>>;
type ErrorCodeMapping = {

42
server/src/modules/versions/services/create.service.ts
View file

@ -6,7 +6,7 @@ import { DataSource } from '@entities/data_source.entity';
import { DataSourceOptions } from '@entities/data_source_options.entity';
import { EventHandler, Target } from '@entities/event_handler.entity';
import { dbTransactionWrap } from '@helpers/database.helper';
import { EntityManager, In } from 'typeorm';
import { EntityManager } from 'typeorm';
import { Credential } from 'src/entities/credential.entity';
import * as uuid from 'uuid';
import { Page } from '@entities/page.entity';
@ -22,8 +22,6 @@ import { DataSourcesRepository } from '@modules/data-sources/repository';
import { DataQueryRepository } from '@modules/data-queries/repository';
import { AppEnvironmentUtilService } from '@modules/app-environments/util.service';
import { IVersionsCreateService } from '../interfaces/services/ICreateService';
import { PagePermission } from '@entities/page_permissions.entity';
import { PageUser } from '@entities/page_users.entity';
@Injectable()
export class VersionsCreateService implements IVersionsCreateService {
@ -403,44 +401,6 @@ export class VersionsCreateService implements IVersionsCreateService {
homePageId = savedPage.id;
}
const oldPermissions = await manager.find(PagePermission, {
where: { pageId: page.id },
});
const newPermissions = oldPermissions.map((permission) => {
return manager.create(PagePermission, {
...permission,
id: undefined,
pageId: oldPageToNewPageMapping[permission.pageId],
});
});
await manager.save(PagePermission, newPermissions);
const permissionIdMap = new Map<string, string>();
oldPermissions.forEach((oldPerm, index) => {
const newPerm = newPermissions[index];
permissionIdMap.set(oldPerm.id, newPerm.id);
});
const oldPermissionIds = oldPermissions.map((p) => p.id);
const oldPageUsers = await manager.find(PageUser, {
where: {
pagePermissionsId: In(oldPermissionIds),
},
});
const newPageUsers = oldPageUsers.map((pu) =>
manager.create(PageUser, {
...pu,
id: undefined,
pagePermissionsId: permissionIdMap.get(pu.pagePermissionsId),
})
);
await manager.save(PageUser, newPageUsers);
const pageEvents = allEvents.filter((event) => event.sourceId === page.id);
pageEvents.forEach(async (event, index) => {