handleHeaderClick(props.id)}
+ style={{ cursor: 'pointer' }}
+ >
+
+ );
+};
+
+export default DataSourceSelect;
diff --git a/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx b/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx
index 88e67e896d..5c698701fb 100644
--- a/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx
+++ b/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/ToolJetDbOperations.jsx
@@ -7,23 +7,28 @@ import { CreateRow } from './CreateRow';
import { UpdateRows } from './UpdateRows';
import { DeleteRows } from './DeleteRows';
import { toast } from 'react-hot-toast';
-import Select from '@/_ui/Select';
import { queryManagerSelectComponentStyle } from '@/_ui/Select/styles';
import { useMounted } from '@/_hooks/use-mount';
import { useCurrentState } from '@/_stores/currentStateStore';
+import { JoinTable } from './JoinTable';
+import { cloneDeep, difference } from 'lodash';
+import DropDownSelect from './DropDownSelect';
+import { getPrivateRoute } from '@/_helpers/routes';
+import { useNavigate } from 'react-router-dom';
const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLayout }) => {
const computeSelectStyles = (darkMode, width) => {
return queryManagerSelectComponentStyle(darkMode, width);
};
const currentState = useCurrentState();
+ const navigate = useNavigate();
const { current_organization_id: organizationId } = authenticationService.currentSessionValue;
const mounted = useMounted();
const [operation, setOperation] = useState(options['operation'] || '');
const [columns, setColumns] = useState([]);
const [tables, setTables] = useState([]);
+ const [tableInfo, setTableInfo] = useState({});
const [selectedTableId, setSelectedTableId] = useState(options['table_id']);
- const [selectedTableName, setSelectedTableName] = useState(null);
const [listRowsOptions, setListRowsOptions] = useState(() => options['list_rows'] || {});
const [updateRowsOptions, setUpdateRowsOptions] = useState(
options['update_rows'] || { columns: {}, where_filters: {} }
@@ -33,6 +38,100 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
limit: 1,
}
);
+ const [joinTableOptions, setJoinTableOptions] = useState(options['join_table'] || {});
+
+ const joinOptions = options['join_table']?.['joins'] || [
+ { conditions: { conditionsList: [{ leftField: { table: selectedTableId } }] } },
+ ];
+
+ const setJoinOptions = (values) => {
+ const tableSet = new Set();
+ (values || []).forEach((join) => {
+ const { table, conditions } = join;
+ tableSet.add(table);
+ conditions?.conditionsList?.forEach((condition) => {
+ const { leftField, rightField } = condition;
+ if (leftField?.table) {
+ tableSet.add(leftField?.table);
+ }
+ if (rightField?.table) {
+ tableSet.add(rightField?.table);
+ }
+ });
+ });
+ tableSet.add(selectedTableId);
+
+ setJoinTableOptions((prevJoinOptions) => {
+ const { conditions, order_by = [], joins: currJoins, fields: currFields = [] } = prevJoinOptions;
+ const conditionsList = cloneDeep(conditions?.conditionsList || []);
+ const newConditionsList = conditionsList.filter((condition) => {
+ const { leftField } = condition || {};
+ if (tableSet.has(leftField?.table)) {
+ return true;
+ }
+ return false;
+ });
+ const newOrderBy = order_by.filter((order) => tableSet.has(order.table));
+
+ //getting old states
+ const currTableSet = new Set();
+ (currJoins || []).forEach((join) => {
+ const { table, conditions } = join;
+ currTableSet.add(table);
+ conditions?.conditionsList?.forEach((condition) => {
+ const { leftField, rightField } = condition;
+ if (leftField?.table) {
+ currTableSet.add(leftField?.table);
+ }
+ if (rightField?.table) {
+ currTableSet.add(rightField?.table);
+ }
+ });
+ });
+ currTableSet.add(selectedTableId);
+ const newTables = difference([...tableSet], [...currTableSet]);
+ const newFields = newTables.reduce(
+ (acc, newTable) => [
+ ...acc,
+ ...(tableInfo[newTable]
+ ? tableInfo[newTable].map((col) => ({
+ name: col.Header,
+ table: newTable,
+ }))
+ : []),
+ ],
+ []
+ );
+
+ const updatedFields = [...currFields.filter((field) => tableSet.has(field.table)), ...newFields];
+ newTables.forEach((tableId) => tableId && loadTableInformation(tableId, true));
+
+ return {
+ ...prevJoinOptions,
+ joins: values,
+ conditions: {
+ ...(conditions?.operator && { operator: conditions.operator }),
+ conditionsList: newConditionsList,
+ },
+ order_by: newOrderBy,
+ fields: updatedFields,
+ };
+ });
+ };
+
+ const joinOrderByOptions = options?.['join_table']?.['order_by'] || [];
+ const setJoinOrderByOptions = (values) => {
+ if (values.length) {
+ setJoinTableOptions((prevJoinOptions) => {
+ return {
+ ...prevJoinOptions,
+ order_by: values,
+ };
+ });
+ } else {
+ deleteJoinTableOptions('order_by');
+ }
+ };
useEffect(() => {
fetchTables();
@@ -40,17 +139,30 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
}, []);
useEffect(() => {
- if (tables.length > 0) {
- const tableInfo = tables.find((table) => table.table_id == selectedTableId);
- tableInfo && setSelectedTableName(tableInfo.table_name);
- }
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [tables]);
+ const tableSet = new Set();
+ const joinOptions = options['join_table']?.['joins'];
+ (joinOptions || []).forEach((join) => {
+ const { table, conditions } = join;
+ tableSet.add(table);
+ conditions?.conditionsList?.forEach((condition) => {
+ const { leftField, rightField } = condition;
+ if (leftField?.table) {
+ tableSet.add(leftField?.table);
+ }
+ if (rightField?.table) {
+ tableSet.add(rightField?.table);
+ }
+ });
+ });
+
+ const tables = [...tableSet];
+ tables.forEach((tableId) => tableId && loadTableInformation(tableId));
+ }, [options['join_table']?.['joins'], tables]);
useEffect(() => {
- selectedTableName && fetchTableInformation(selectedTableName);
+ selectedTableId && fetchTableInformation(selectedTableId);
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [selectedTableName]);
+ }, [selectedTableId]);
useEffect(() => {
if (mounted) {
@@ -77,6 +189,11 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [updateRowsOptions]);
+ useEffect(() => {
+ mounted && optionchanged('join_table', joinTableOptions);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [joinTableOptions]);
+
const handleOptionsChange = (optionsChanged, value) => {
setListRowsOptions((prev) => ({ ...prev, [optionsChanged]: value }));
};
@@ -97,6 +214,66 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
setDeleteRowsOptions((prev) => ({ ...prev, limit: limit }));
};
+ const loadTableInformation = async (tableId, isNewTableAdded) => {
+ const tableDetails = findTableDetails(tableId);
+ if (tableDetails?.table_name && !tableInfo[tableDetails?.table_name]) {
+ const { table_name } = tableDetails;
+ const { data } = await tooljetDatabaseService.viewTable(organizationId, table_name);
+
+ setTableInfo((info) => ({
+ ...info,
+ [table_name]: data?.result.map(({ column_name, data_type, keytype, ...rest }) => ({
+ Header: column_name,
+ accessor: column_name,
+ dataType: data_type,
+ isPrimaryKey: keytype?.toLowerCase() === 'primary key',
+ ...rest,
+ })),
+ }));
+
+ if (isNewTableAdded) {
+ setJoinTableOptions((joinOptions) => {
+ const { fields } = joinOptions;
+ const newFields = cloneDeep(fields).filter((field) => field.table !== tableId);
+ newFields.push(
+ ...(data?.result
+ ? data.result.map((col) => ({
+ name: col.column_name,
+ table: tableId,
+ // alias: `${tableId}_${col.column_name}`,
+ }))
+ : [])
+ );
+
+ return {
+ ...joinOptions,
+ fields: newFields,
+ };
+ });
+ }
+ }
+ };
+
+ const joinTableOptionsChange = (optionsChanged, value) => {
+ setJoinTableOptions((prev) => ({ ...prev, [optionsChanged]: value }));
+ };
+
+ const deleteJoinTableOptions = (optionToDelete) => {
+ setJoinTableOptions((prev) => {
+ const prevOptions = { ...prev };
+ if (prevOptions[optionToDelete]) delete prevOptions[optionToDelete];
+ return prevOptions;
+ });
+ };
+
+ const findTableDetails = (tableId) => {
+ return tables.find((table) => table.table_id == tableId);
+ };
+
+ const findTableDetailsByName = (tableName) => {
+ return tables.find((table) => table.table_name == tableName);
+ };
+
const value = useMemo(
() => ({
organizationId,
@@ -106,8 +283,6 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
setColumns,
selectedTableId,
setSelectedTableId,
- selectedTableName,
- setSelectedTableName,
listRowsOptions,
setListRowsOptions,
limitOptionChanged,
@@ -117,16 +292,31 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
deleteOperationLimitOptionChanged,
updateRowsOptions,
handleUpdateRowsOptionsChange,
+ joinTableOptions,
+ joinTableOptionsChange,
+ tableInfo,
+ loadTableInformation,
+ joinOptions,
+ setJoinOptions,
+ joinOrderByOptions,
+ setJoinOrderByOptions,
+ deleteJoinTableOptions,
+ findTableDetails,
+ findTableDetailsByName,
}),
[
organizationId,
tables,
columns,
- selectedTableName,
- selectedTableId,
listRowsOptions,
deleteRowsOptions,
updateRowsOptions,
+ joinTableOptions,
+ tableInfo,
+ loadTableInformation,
+ joinOptions,
+ joinOrderByOptions,
+ selectedTableId,
]
);
@@ -139,42 +329,72 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
}
if (Array.isArray(data?.result)) {
- const selectedTableInfo = data.result.find((table) => table.id === options['table_id']);
-
- selectedTableInfo && setSelectedTableId(selectedTableInfo.id);
setTables(
data.result.map((table) => {
return { table_name: table.table_name, table_id: table.id };
}) || []
);
+ const selectedTableInfo = data.result.find((table) => table.id === options['table_id']);
+ if (selectedTableInfo) {
+ setSelectedTableId(selectedTableInfo.id);
+ fetchTableInformation(selectedTableInfo.id);
+ }
}
};
- const fetchTableInformation = async (table) => {
- const { error, data } = await tooljetDatabaseService.viewTable(organizationId, table);
+ /**
+ * TODO: This function to be removed and replaced with loadTableInformation function everywhere
+ */
+ const fetchTableInformation = async (tableId, isNewTableAdded) => {
+ const tableDetails = findTableDetails(tableId);
+ if (tableDetails?.table_name) {
+ const { table_name } = tableDetails;
+ const { error, data } = await tooljetDatabaseService.viewTable(organizationId, table_name);
- if (error) {
- toast.error(error?.message ?? 'Failed to fetch table information');
- return;
- }
+ if (error) {
+ toast.error(error?.message ?? 'Failed to fetch table information');
+ return;
+ }
- if (data?.result?.length > 0) {
- setColumns(
- data?.result.map(({ column_name, data_type, keytype, ...rest }) => ({
+ if (data?.result?.length > 0) {
+ const columnList = data?.result.map(({ column_name, data_type, keytype, ...rest }) => ({
Header: column_name,
accessor: column_name,
dataType: data_type,
isPrimaryKey: keytype?.toLowerCase() === 'primary key',
...rest,
- }))
- );
+ }));
+ setColumns(columnList);
+ setTableInfo((prevTableInfo) => ({ ...prevTableInfo, [table_name]: columnList }));
+
+ if (isNewTableAdded) {
+ setJoinTableOptions((joinOptions) => {
+ const { fields } = joinOptions;
+ const newFields = cloneDeep(fields).filter((field) => field.table !== tableId);
+ newFields.push(
+ ...(data?.result
+ ? data.result.map((col) => ({
+ name: col.column_name,
+ table: tableId,
+ // alias: `${tableId}_${col.column_name}`,
+ }))
+ : [])
+ );
+
+ return {
+ ...joinOptions,
+ fields: newFields,
+ };
+ });
+ }
+ }
}
};
const generateListForDropdown = (tableList) => {
return tableList.map((tableMap) =>
Object.fromEntries([
- ['name', tableMap.table_name],
+ ['label', tableMap.table_name],
['value', tableMap.table_id],
])
);
@@ -182,11 +402,34 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
const handleTableNameSelect = (tableId) => {
setSelectedTableId(tableId);
- const { table_name: tableName } = tables.find((t) => t.table_id === tableId);
- tableName && setSelectedTableName(tableName);
-
+ fetchTableInformation(tableId, true);
optionchanged('organization_id', organizationId);
optionchanged('table_id', tableId);
+
+ setJoinTableOptions(() => {
+ return {
+ joins: [
+ {
+ id: new Date().getTime(),
+ conditions: {
+ operator: 'AND',
+ conditionsList: [
+ {
+ operator: '=',
+ leftField: { table: tableId },
+ },
+ ],
+ },
+ joinType: 'INNER',
+ },
+ ],
+ from: {
+ name: tableId,
+ type: 'Table',
+ },
+ fields: [],
+ };
+ });
};
const getComponent = () => {
@@ -199,9 +442,19 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
return UpdateRows;
case 'delete_rows':
return DeleteRows;
+ case 'join_tables':
+ return JoinTable;
}
};
+ const tooljetDbOperationList = [
+ { label: 'List rows', value: 'list_rows' },
+ { label: 'Create row', value: 'create_row' },
+ { label: 'Update rows', value: 'update_rows' },
+ { label: 'Delete rows', value: 'delete_rows' },
+ { label: 'Join tables', value: 'join_tables' },
+ ];
+
const ComponentToRender = getComponent(operation);
return (
@@ -210,15 +463,17 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
Table name
-
-
+ handleTableNameSelect(value)}
- width="100%"
- // useMenuPortal={false}
- useCustomStyles={true}
- styles={computeSelectStyles(darkMode, '100%')}
+ darkMode={darkMode}
+ onChange={(value) => {
+ value?.value && handleTableNameSelect(value?.value);
+ }}
+ onAdd={() => navigate(getPrivateRoute('database'))}
+ addBtnLabel={'Add new table'}
+ value={generateListForDropdown(tables).find((val) => val?.value === selectedTableId)}
/>
@@ -231,20 +486,15 @@ const ToolJetDbOperations = ({ optionchanged, options, darkMode, isHorizontalLay
className={cx({ 'col-4': !isHorizontalLayout, 'd-flex': isHorizontalLayout })}
>
Operations
-
-
setOperation(value)}
- width="100%"
- // useMenuPortal={false}
- useCustomStyles={true}
- styles={computeSelectStyles(darkMode, '100%')}
+
+ {
+ value?.value && setOperation(value?.value);
+ }}
+ value={tooljetDbOperationList.find((val) => val?.value === operation)}
/>
diff --git a/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/operations.js b/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/operations.js
index 9d82f95fe3..99fb9baac3 100644
--- a/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/operations.js
+++ b/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/operations.js
@@ -1,4 +1,4 @@
-import { tooljetDatabaseService } from '@/_services';
+import { tooljetDatabaseService, authenticationService } from '@/_services';
import { isEmpty } from 'lodash';
import PostgrestQueryBuilder from '@/_helpers/postgrestQueryBuilder';
import { resolveReferences } from '@/_helpers/utils';
@@ -18,6 +18,8 @@ async function perform(dataQuery, currentState) {
return updateRows(dataQuery, currentState);
case 'delete_rows':
return deleteRows(dataQuery, currentState);
+ case 'join_tables':
+ return joinTables(dataQuery, currentState);
default:
return {
@@ -173,3 +175,79 @@ async function deleteRows(dataQuery, currentState) {
const headers = { 'data-query-id': dataQuery.id };
return await tooljetDatabaseService.deleteRows(headers, tableId, query.join('&'));
}
+
+// Function:- To valid Empty fields in JSON ( Works for Nested JSON too )
+// function validateInputJsonHasEmptyFields(input) {
+// let isValid = true;
+
+// if (isEmpty(input)) return false;
+// if (Array.isArray(input)) {
+// let isIncludesInvalidJson = input
+// .map((eachValue) => {
+// let isValidJson = validateInputJsonHasEmptyFields(eachValue);
+// return isValidJson;
+// })
+// .includes(false);
+// if (isIncludesInvalidJson) isValid = false;
+// }
+
+// if (typeof input === 'object') {
+// let isIncludesInvalidJson = Object.entries(input)
+// .map(([key, value]) => {
+// let isValidJson = validateInputJsonHasEmptyFields(value);
+// return isValidJson;
+// })
+// .includes(false);
+// if (isIncludesInvalidJson) isValid = false;
+// }
+
+// return isValid;
+// }
+
+async function joinTables(dataQuery, currentState) {
+ const organizationId = authenticationService.currentSessionValue.current_organization_id;
+ const queryOptions = dataQuery.options;
+ const resolvedOptions = resolveReferences(queryOptions, currentState);
+ const { join_table = {} } = resolvedOptions;
+
+ // Empty Input is restricted
+ if (Object.keys(join_table).length === 0) {
+ return {
+ status: 'failed',
+ statusText: 'failed',
+ message: `Input can't be empty`,
+ description: 'Empty inputs are not allowed',
+ data: {},
+ };
+ }
+
+ const sanitizedJoinTableJson = { ...join_table };
+ // If mandatory fields ( Select, JOin & From section ), are empty throw error
+ let mandatoryFieldsButEmpty = [];
+ if (!sanitizedJoinTableJson?.fields.length) mandatoryFieldsButEmpty.push('Select');
+ if (sanitizedJoinTableJson?.from && !Object.keys(sanitizedJoinTableJson?.from).length)
+ mandatoryFieldsButEmpty.push('From');
+ // if (join_table?.joins && !validateInputJsonHasEmptyFields(join_table?.joins)) mandatoryFieldsButEmpty.push('Joins');
+ if (mandatoryFieldsButEmpty.length) {
+ return {
+ status: 'failed',
+ statusText: 'failed',
+ message: `Empty values are found in the following section - ${mandatoryFieldsButEmpty.join(', ')}.`,
+ description: 'Mandatory fields are not empty',
+ data: {},
+ };
+ }
+
+ // If non-mandatory fields ( Filter & Sort ) are empty - remove the particular field
+ if (
+ sanitizedJoinTableJson?.conditions &&
+ (!Object.keys(sanitizedJoinTableJson?.conditions)?.length ||
+ !sanitizedJoinTableJson?.conditions?.conditionsList?.length)
+ ) {
+ delete sanitizedJoinTableJson.conditions;
+ }
+ if (sanitizedJoinTableJson?.order_by && !sanitizedJoinTableJson?.order_by.length)
+ delete sanitizedJoinTableJson.order_by;
+
+ return await tooljetDatabaseService.joinTables(organizationId, sanitizedJoinTableJson);
+}
diff --git a/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/util.js b/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/util.js
index f26545d3fe..777cc19d4b 100644
--- a/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/util.js
+++ b/frontend/src/Editor/QueryManager/QueryEditors/TooljetDatabase/util.js
@@ -40,3 +40,26 @@ export const isOperatorOptions = [
{ value: 'null', label: 'null' },
{ value: 'notNull', label: 'not null' },
];
+
+export const filterOperatorOptions = [
+ { label: 'equals', value: '=' },
+ { label: 'greater than', value: '>' },
+ { label: 'greater than or equal', value: '>=' },
+ { label: 'less than', value: '<' },
+ { label: 'less than or equal', value: '<=' },
+ { label: 'not equal', value: '!=' },
+ { label: 'like', value: 'LIKE' },
+ { label: 'not like', value: 'NOT LIKE' },
+ { label: 'ilike', value: 'ILIKE' },
+ { label: 'not ilike', value: 'NOT ILIKE' },
+ { label: 'match', value: '~' },
+ { label: 'imatch', value: '~*' },
+ { label: 'in', value: 'IN' },
+ { label: 'not in', value: 'NOT IN' },
+ { label: 'is', value: 'IS' },
+];
+
+export const nullOperatorOptions = [
+ { label: 'null', value: 'NULL' },
+ { label: 'not null', value: 'NOT NULL' },
+];
diff --git a/frontend/src/MarketplacePage/InstalledPlugins.jsx b/frontend/src/MarketplacePage/InstalledPlugins.jsx
index 1066c8552c..0b11923c0e 100644
--- a/frontend/src/MarketplacePage/InstalledPlugins.jsx
+++ b/frontend/src/MarketplacePage/InstalledPlugins.jsx
@@ -109,19 +109,33 @@ const InstalledPluginCard = ({ plugin, marketplacePlugin, fetchPlugins, isDevMod
toast.success(`${capitalizeFirstLetter(name)} reloaded`);
};
+ const pluginDeleteMessage = (
+ <>
+ Deleting
{capitalizeFirstLetter(name)} plugin will result in the permanent removal of all
+ associated datasources and its dataqueries. This action cannot be undone. Are you sure you wish to proceed with
+ the deletion?
+
+ );
+
return (
<>
-
+
@@ -163,7 +177,7 @@ const InstalledPluginCard = ({ plugin, marketplacePlugin, fetchPlugins, isDevMod
)}
-
+
diff --git a/frontend/src/MarketplacePage/MarketplaceCard.jsx b/frontend/src/MarketplacePage/MarketplaceCard.jsx
index 5a6146cdde..e31f388101 100644
--- a/frontend/src/MarketplacePage/MarketplaceCard.jsx
+++ b/frontend/src/MarketplacePage/MarketplaceCard.jsx
@@ -45,7 +45,7 @@ export const MarketplaceCard = ({ id, name, repo, description, version, isInstal
return (
-
+
@@ -57,7 +57,7 @@ export const MarketplaceCard = ({ id, name, repo, description, version, isInstal
{description}
-
+
v{version}
diff --git a/frontend/src/TooljetDatabase/Drawers/BulkUploadDrawer/FileDropzone.jsx b/frontend/src/TooljetDatabase/Drawers/BulkUploadDrawer/FileDropzone.jsx
new file mode 100644
index 0000000000..a3e11336fb
--- /dev/null
+++ b/frontend/src/TooljetDatabase/Drawers/BulkUploadDrawer/FileDropzone.jsx
@@ -0,0 +1,90 @@
+import React, { useState } from 'react';
+import { useDropzone } from 'react-dropzone';
+import BulkIcon from '@/_ui/Icon/BulkIcons';
+import { toast } from 'react-hot-toast';
+import SolidIcon from '@/_ui/Icon/SolidIcons';
+
+export function FileDropzone({ handleClick, hiddenFileInput, errors, handleFileChange, onButtonClick, onDrop }) {
+ const [fileData, setFileData] = useState();
+ const { getRootProps, getInputProps, isDragActive, acceptedFiles } = useDropzone({
+ accept: { parsedFileType: ['text/csv'] },
+ onDrop,
+ noClick: true,
+ onDropRejected: (files) => {
+ if (Math.round(files[0].size / 1024) > 2 * 1024) {
+ handleFileChange(files[0]);
+ } else {
+ toast.error('Please upload a CSV file');
+ }
+ },
+ maxFiles: 1,
+ onFileDialogCancel: () => {
+ toast.error('Please upload a CSV file');
+ },
+ noKeyboard: true,
+ });
+
+ return (
+
+ );
+}
diff --git a/frontend/src/TooljetDatabase/Drawers/BulkUploadDrawer/index.jsx b/frontend/src/TooljetDatabase/Drawers/BulkUploadDrawer/index.jsx
new file mode 100644
index 0000000000..346cc365ec
--- /dev/null
+++ b/frontend/src/TooljetDatabase/Drawers/BulkUploadDrawer/index.jsx
@@ -0,0 +1,144 @@
+import React, { useState, useContext, useCallback, useRef } from 'react';
+import Drawer from '@/_ui/Drawer';
+import { toast } from 'react-hot-toast';
+import { TooljetDatabaseContext } from '../../index';
+import { ButtonSolid } from '@/_ui/AppButton/AppButton';
+import { FileDropzone } from './FileDropzone';
+import SolidIcon from '@/_ui/Icon/SolidIcons';
+
+function BulkUploadDrawer({
+ isBulkUploadDrawerOpen,
+ setIsBulkUploadDrawerOpen,
+ bulkUploadFile,
+ handleBulkUploadFileChange,
+ handleBulkUpload,
+ isBulkUploading,
+ errors,
+}) {
+ const [isDownloadingTemplate, setIsDownloadingTemplate] = useState(false);
+ const { columns, selectedTable } = useContext(TooljetDatabaseContext);
+ const hiddenFileInput = useRef(null);
+
+ const onDrop = useCallback((acceptedFiles) => {
+ const file = acceptedFiles[0];
+ if (Math.round(file.size / 1024) > 2 * 1024) {
+ toast.error('File size cannot exceed more than 2MB');
+ } else {
+ handleBulkUploadFileChange(file);
+ }
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, []);
+
+ const handleTemplateDownload = () => {
+ setIsDownloadingTemplate(true);
+
+ return setTimeout(() => {
+ // Create a CSV content string with the column names as the header row
+ const headerRow = columns.map((col) => col.Header).join(',');
+ const csvContent = [headerRow].join('\n');
+ // Create a Blob with the CSV content
+ const blob = new Blob([csvContent], { type: 'text/csv' });
+ // Create a temporary URL for the Blob
+ const href = URL.createObjectURL(blob);
+ // Create a link element to trigger the download
+ const link = document.createElement('a');
+ link.href = href;
+ link.download = `${selectedTable.table_name}.csv`;
+ // Trigger the download
+ link.click();
+
+ setIsDownloadingTemplate(false);
+
+ // Clean up
+ document.body.removeChild(link);
+ window.URL.revokeObjectURL(href);
+ }, 500);
+ };
+
+ const handleClick = () => {
+ hiddenFileInput.current.click();
+ };
+
+ return (
+ <>
+
setIsBulkUploadDrawerOpen(!isBulkUploadDrawerOpen)}
+ className={`ghost-black-operation ${isBulkUploadDrawerOpen ? 'open' : ''}`}
+ >
+
+ Bulk upload data
+
+
+
+
setIsBulkUploadDrawerOpen(false)}
+ position="right"
+ drawerStyle={{ 'overflow-y': 'hidden' }}
+ >
+
+
+
+ Bulk upload data
+
+
+
+
+
+
+
+
+
+
+ Download the template to add your data or format your file in the same as the template. ToolJet
+ won’t be able to recognise files in any other format.
+
+
+ Download Template
+
+
+
+
+
+
+
+
+
+ setIsBulkUploadDrawerOpen(false)}>
+ Cancel
+
+ 0 || errors.server.length > 0}
+ data-cy={`save-changes-button`}
+ onClick={handleBulkUpload}
+ fill="#fff"
+ leftIcon="floppydisk"
+ loading={isBulkUploading}
+ >
+ Upload data
+
+
+
+
+ );
+}
+export default BulkUploadDrawer;
diff --git a/frontend/src/TooljetDatabase/Drawers/CreateColumnDrawer/index.jsx b/frontend/src/TooljetDatabase/Drawers/CreateColumnDrawer/index.jsx
index 9009d65aa0..f92f240478 100644
--- a/frontend/src/TooljetDatabase/Drawers/CreateColumnDrawer/index.jsx
+++ b/frontend/src/TooljetDatabase/Drawers/CreateColumnDrawer/index.jsx
@@ -13,7 +13,7 @@ const CreateColumnDrawer = ({ setIsCreateColumnDrawerOpen, isCreateColumnDrawerO
<>
setIsCreateColumnDrawerOpen(!isCreateColumnDrawerOpen)}
- className={`add-new-column-btn ghost-black-operation ${isCreateColumnDrawerOpen && 'open'}`}
+ className={`ghost-black-operation ${isCreateColumnDrawerOpen ? 'open' : ''}`}
data-cy="add-new-column-button"
>
Add new row
+
+ Add new row
+
setIsCreateRowDrawerOpen(false)} position="right">
{
- tooljetDatabaseService.findOne(organizationId, selectedTable.id).then(({ headers, data = [], error }) => {
- if (error) {
- toast.error(error?.message ?? `Failed to fetch table "${selectedTable.table_name}"`);
- return;
- }
+ tooljetDatabaseService
+ .findOne(organizationId, selectedTable.id, 'order=id.desc')
+ .then(({ headers, data = [], error }) => {
+ if (error) {
+ toast.error(error?.message ?? `Failed to fetch table "${selectedTable.table_name}"`);
+ return;
+ }
- if (Array.isArray(data) && data?.length > 0) {
- const totalContentRangeRecords = headers['content-range'].split('/')[1] || 0;
- setTotalRecords(totalContentRangeRecords);
- setSelectedTableData(data);
- }
- });
+ if (Array.isArray(data) && data?.length > 0) {
+ const totalContentRangeRecords = headers['content-range'].split('/')[1] || 0;
+ setTotalRecords(totalContentRangeRecords);
+ setSelectedTableData(data);
+ }
+ });
setIsCreateRowDrawerOpen(false);
}}
onClose={() => setIsCreateRowDrawerOpen(false)}
diff --git a/frontend/src/TooljetDatabase/Drawers/EditRowDrawer/index.jsx b/frontend/src/TooljetDatabase/Drawers/EditRowDrawer/index.jsx
index 6152f7fdba..564cd10359 100644
--- a/frontend/src/TooljetDatabase/Drawers/EditRowDrawer/index.jsx
+++ b/frontend/src/TooljetDatabase/Drawers/EditRowDrawer/index.jsx
@@ -12,14 +12,21 @@ const EditRowDrawer = ({ isCreateRowDrawerOpen, setIsCreateRowDrawerOpen }) => {
<>
setIsCreateRowDrawerOpen(!isCreateRowDrawerOpen)}
- className={`edit-row-btn border-0 ghost-black-operation ${isCreateRowDrawerOpen && 'open'}`}
+ className={`ghost-black-operation ${isCreateRowDrawerOpen ? 'open' : ''}`}
>
-
+ {/*
@@ -30,18 +37,20 @@ const EditRowDrawer = ({ isCreateRowDrawerOpen, setIsCreateRowDrawerOpen }) => {
setIsCreateRowDrawerOpen(false)} position="right">
{
- tooljetDatabaseService.findOne(organizationId, selectedTable.id).then(({ headers, data = [], error }) => {
- if (error) {
- toast.error(error?.message ?? `Failed to fetch table "${selectedTable.table_name}"`);
- return;
- }
+ tooljetDatabaseService
+ .findOne(organizationId, selectedTable.id, 'order=id.desc')
+ .then(({ headers, data = [], error }) => {
+ if (error) {
+ toast.error(error?.message ?? `Failed to fetch table "${selectedTable.table_name}"`);
+ return;
+ }
- if (Array.isArray(data) && data?.length > 0) {
- const totalContentRangeRecords = headers['content-range'].split('/')[1] || 0;
- setTotalRecords(totalContentRangeRecords);
- setSelectedTableData(data);
- }
- });
+ if (Array.isArray(data) && data?.length > 0) {
+ const totalContentRangeRecords = headers['content-range'].split('/')[1] || 0;
+ setTotalRecords(totalContentRangeRecords);
+ setSelectedTableData(data);
+ }
+ });
setIsCreateRowDrawerOpen(false);
}}
onClose={() => setIsCreateRowDrawerOpen(false)}
diff --git a/frontend/src/TooljetDatabase/ExportSchema/ExportSchema.jsx b/frontend/src/TooljetDatabase/ExportSchema/ExportSchema.jsx
deleted file mode 100644
index 943100bfe1..0000000000
--- a/frontend/src/TooljetDatabase/ExportSchema/ExportSchema.jsx
+++ /dev/null
@@ -1,13 +0,0 @@
-import React from 'react';
-import SolidIcon from '@/_ui/Icon/SolidIcons';
-
-function ExportSchema({ onClick }) {
- return (
-
-
- );
-}
-
-export default ExportSchema;
diff --git a/frontend/src/TooljetDatabase/Filter/index.jsx b/frontend/src/TooljetDatabase/Filter/index.jsx
index 2708e3a4a9..c59c74eae7 100644
--- a/frontend/src/TooljetDatabase/Filter/index.jsx
+++ b/frontend/src/TooljetDatabase/Filter/index.jsx
@@ -4,7 +4,7 @@ import OverlayTrigger from 'react-bootstrap/OverlayTrigger';
import Popover from 'react-bootstrap/Popover';
import { FilterForm } from '../Forms/FilterForm';
import { isEmpty } from 'lodash';
-import { pluralize } from '@/_helpers/utils';
+// import { pluralize } from '@/_helpers/utils';
import { useMounted } from '@/_hooks/use-mount';
import SolidIcon from '@/_ui/Icon/SolidIcons';
@@ -94,9 +94,9 @@ const Filter = ({ filters, setFilters, handleBuildFilterQuery, resetFilterQuery
>
ed by {pluralize(Object.values(filters).filter(checkIsFilterObjectEmpty).length, 'column')}
- )}
+ )} */}
diff --git a/frontend/src/TooljetDatabase/Forms/EditRowForm.jsx b/frontend/src/TooljetDatabase/Forms/EditRowForm.jsx
index ec9583d25f..d2ab699180 100644
--- a/frontend/src/TooljetDatabase/Forms/EditRowForm.jsx
+++ b/frontend/src/TooljetDatabase/Forms/EditRowForm.jsx
@@ -155,6 +155,7 @@ const RenderElement = ({ columnName, dataType, isPrimaryKey, defaultValue, value
switch (dataType) {
case 'character varying':
case 'integer':
+ case 'bigint':
case 'serial':
case 'double precision':
return (
diff --git a/frontend/src/TooljetDatabase/Forms/RowForm.jsx b/frontend/src/TooljetDatabase/Forms/RowForm.jsx
index 79df84b77e..f43afd9891 100644
--- a/frontend/src/TooljetDatabase/Forms/RowForm.jsx
+++ b/frontend/src/TooljetDatabase/Forms/RowForm.jsx
@@ -51,6 +51,7 @@ const RowForm = ({ onCreate, onClose }) => {
switch (dataType) {
case 'character varying':
case 'integer':
+ case 'bigint':
case 'serial':
case 'double precision':
return (
diff --git a/frontend/src/TooljetDatabase/Sort/index.jsx b/frontend/src/TooljetDatabase/Sort/index.jsx
index b400f5eb4b..12c190f768 100644
--- a/frontend/src/TooljetDatabase/Sort/index.jsx
+++ b/frontend/src/TooljetDatabase/Sort/index.jsx
@@ -3,7 +3,7 @@ import cx from 'classnames';
import OverlayTrigger from 'react-bootstrap/OverlayTrigger';
import Popover from 'react-bootstrap/Popover';
import { SortForm } from '../Forms/SortForm';
-import { pluralize } from '@/_helpers/utils';
+// import { pluralize } from '@/_helpers/utils';
import { isEmpty } from 'lodash';
import { useMounted } from '@/_hooks/use-mount';
import SolidIcon from '@/_ui/Icon/SolidIcons';
@@ -102,9 +102,9 @@ const Sort = ({ filters, setFilters, handleBuildSortQuery, resetSortQuery }) =>
fill={areFiltersApplied ? '#46A758' : show ? '#3E63DD' : '#889096'}
/>
Sort
- {areFiltersApplied && (
+ {/* {areFiltersApplied && (
ed by {pluralize(Object.values(filters).filter(checkIsFilterObjectEmpty).length, 'column')}
- )}
+ )} */}
);
diff --git a/frontend/src/TooljetDatabase/Table/index.jsx b/frontend/src/TooljetDatabase/Table/index.jsx
index 5fee5fb8a1..93f1ad37cd 100644
--- a/frontend/src/TooljetDatabase/Table/index.jsx
+++ b/frontend/src/TooljetDatabase/Table/index.jsx
@@ -57,7 +57,7 @@ const Table = ({ openCreateRowDrawer, openCreateColumnDrawer }) => {
};
const fetchTableData = (queryParams = '', pagesize = 50, pagecount = 1) => {
- const defaultQueryParams = `limit=${pagesize}&offset=${(pagecount - 1) * pagesize}`;
+ const defaultQueryParams = `limit=${pagesize}&offset=${(pagecount - 1) * pagesize}&order=id.desc`;
let params = queryParams ? queryParams : defaultQueryParams;
setLoading(true);
@@ -109,6 +109,8 @@ const Table = ({ openCreateRowDrawer, openCreateColumnDrawer }) => {
switch (type) {
case 'integer':
return 'int';
+ case 'bigint':
+ return 'bigint';
case 'character varying':
return 'varchar';
case 'boolean':
diff --git a/frontend/src/TooljetDatabase/TableListItem/ActionsPopover/index.jsx b/frontend/src/TooljetDatabase/TableListItem/ActionsPopover/index.jsx
index b5c6228553..73fd16ebc8 100644
--- a/frontend/src/TooljetDatabase/TableListItem/ActionsPopover/index.jsx
+++ b/frontend/src/TooljetDatabase/TableListItem/ActionsPopover/index.jsx
@@ -7,8 +7,10 @@ import EditIcon from './Icons/Edit.svg';
import DeleteIcon from './Icons/Delete.svg';
import SolidIcon from '@/_ui/Icon/SolidIcons';
-export const ListItemPopover = ({ onEdit, onDelete, darkMode }) => {
- const [open, setOpen] = React.useState(false);
+export const ListItemPopover = ({ onEdit, onDelete, darkMode, handleExportTable, onMenuToggle }) => {
+ const closeMenu = () => {
+ document.body.click();
+ };
const popover = (
@@ -20,14 +22,30 @@ export const ListItemPopover = ({ onEdit, onDelete, darkMode }) => {
{
- setOpen(false);
+ onClick={(event) => {
+ event.stopPropagation();
+ closeMenu();
onEdit();
}}
>
Edit
+
+
+
+
{
+ closeMenu();
+ handleExportTable();
+ }}
+ >
+ Export table
+
+
{/*
@@ -38,7 +56,14 @@ export const ListItemPopover = ({ onEdit, onDelete, darkMode }) => {
-
+
{
+ closeMenu();
+ onDelete();
+ }}
+ >
Delete
@@ -47,27 +72,12 @@ export const ListItemPopover = ({ onEdit, onDelete, darkMode }) => {
);
return (
-
-
{
- setOpen(isOpen);
- }}
- show={open}
- rootClose
- trigger="click"
- placement="bottom"
- overlay={popover}
- transition={false}
- >
+
+
-
-
+
+
);
};
diff --git a/frontend/src/TooljetDatabase/TableListItem/index.jsx b/frontend/src/TooljetDatabase/TableListItem/index.jsx
index 5c1e9fea40..960216a7b8 100644
--- a/frontend/src/TooljetDatabase/TableListItem/index.jsx
+++ b/frontend/src/TooljetDatabase/TableListItem/index.jsx
@@ -1,8 +1,7 @@
-import React, { useState, useContext } from 'react';
+import React, { useState, useContext, useCallback, useEffect } from 'react';
import cx from 'classnames';
-
import { toast } from 'react-hot-toast';
-import { tooljetDatabaseService } from '@/_services';
+import { tooljetDatabaseService, appService } from '@/_services';
import { ListItemPopover } from './ActionsPopover';
import { TooljetDatabaseContext } from '../index';
import { ToolTip } from '@/_components';
@@ -14,11 +13,41 @@ export const ListItem = ({ active, onClick, text = '', onDeleteCallback }) => {
const { organizationId, columns, selectedTable, setTables, setSelectedTable } = useContext(TooljetDatabaseContext);
const [isEditTableDrawerOpen, setIsEditTableDrawerOpen] = useState(false);
const darkMode = localStorage.getItem('darkMode') === 'true';
+ const [isHovered, setIsHovered] = useState(false);
+ const [showDropDownMenu, setShowDropDownMenu] = useState(false);
+ const [focused, setFocused] = useState(false);
function updateSelectedTable(tableObj) {
setSelectedTable(tableObj);
}
+ const handleExportTable = () => {
+ appService
+ .exportResource({
+ tooljet_database: [{ table_id: selectedTable.id }],
+ organization_id: organizationId,
+ })
+ .then((data) => {
+ const tableName = selectedTable.table_name.replace(/\s+/g, '-').toLowerCase();
+ const fileName = `${tableName}-export-${new Date().getTime()}`;
+ // simulate link click download
+ const json = JSON.stringify(data, null, 2);
+ const blob = new Blob([json], { type: 'application/json' });
+ const href = URL.createObjectURL(blob);
+ const link = document.createElement('a');
+ link.href = href;
+ link.download = fileName + '.json';
+ document.body.appendChild(link);
+ link.click();
+ document.body.removeChild(link);
+ })
+ .catch(() => {
+ toast.error('Could not export table.', {
+ position: 'top-center',
+ });
+ });
+ };
+
const handleDeleteTable = async () => {
const shouldDelete = confirm(`Are you sure you want to delete the table "${text}"?`);
if (shouldDelete) {
@@ -39,8 +68,23 @@ export const ListItem = ({ active, onClick, text = '', onDeleteCallback }) => {
return acc;
}, {});
+ const onMenuToggle = useCallback(
+ (status) => {
+ setShowDropDownMenu(!!status);
+ !status && !isHovered && setFocused(false);
+ },
+ [isHovered]
+ );
+
+ useEffect(() => {
+ !showDropDownMenu && setFocused(!!isHovered);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [isHovered]);
+
return (
setIsHovered(true)}
+ onMouseLeave={() => setIsHovered(false)}
className={cx(
'table-list-item mb-1 rounded-3 d-inline-flex align-items-center justify-content-between h-4 list-group-item cursor-pointer list-group-item-action border-0 py-1',
{
@@ -59,7 +103,21 @@ export const ListItem = ({ active, onClick, text = '', onDeleteCallback }) => {
{text}
-
setIsEditTableDrawerOpen(true)} onDelete={handleDeleteTable} darkMode={darkMode} />
+ {focused && (
+
+ {
+ setShowDropDownMenu(false);
+ setIsEditTableDrawerOpen(true);
+ }}
+ onDelete={handleDeleteTable}
+ darkMode={darkMode}
+ handleExportTable={handleExportTable}
+ onMenuToggle={onMenuToggle}
+ />
+
+ )}
+
{
const {
@@ -27,11 +28,65 @@ const TooljetDatabasePage = ({ totalTables }) => {
sortFilters,
setSortFilters,
organizationId,
+ setTotalRecords,
+ setSelectedTableData,
} = useContext(TooljetDatabaseContext);
const [isCreateRowDrawerOpen, setIsCreateRowDrawerOpen] = useState(false);
+ const [isBulkUploadDrawerOpen, setIsBulkUploadDrawerOpen] = useState(false);
const [isEditRowDrawerOpen, setIsEditRowDrawerOpen] = useState(false);
const [isCreateColumnDrawerOpen, setIsCreateColumnDrawerOpen] = useState(false);
+ const [bulkUploadFile, setBulkUploadFile] = useState(null);
+ const [isBulkUploading, setIsBulkUploading] = useState(false);
+ const [errors, setErrors] = useState({ client: [], server: [] });
+ const [uploadResult, setUploadResult] = useState(null);
+
+ useEffect(() => {
+ setErrors({ client: [], server: [] });
+ handleFileValidation();
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [bulkUploadFile]);
+
+ useEffect(() => {
+ if (!isBulkUploadDrawerOpen) {
+ setErrors({ client: [], server: [] });
+ setBulkUploadFile(null);
+ }
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [isBulkUploadDrawerOpen]);
+
+ useEffect(() => {
+ if (isEmpty(selectedTable)) return;
+
+ const reloadTableData = async () => {
+ const { headers, data, error } = await tooljetDatabaseService.findOne(
+ organizationId,
+ selectedTable.id,
+ 'order=id.desc'
+ );
+
+ if (error) {
+ toast.error(error?.message ?? 'Something went wrong');
+ return;
+ }
+ const totalRecords = headers['content-range'].split('/')[1] || 0;
+
+ if (Array.isArray(data)) {
+ setTotalRecords(totalRecords);
+ setSelectedTableData(data);
+ }
+ };
+
+ setIsBulkUploading(false);
+ setBulkUploadFile(null);
+ setIsBulkUploadDrawerOpen(false);
+ setQueryFilters({});
+ resetFilterQuery();
+ setSortFilters({});
+ resetSortQuery();
+ reloadTableData();
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [uploadResult]);
const EmptyState = () => {
return (
@@ -56,31 +111,54 @@ const TooljetDatabasePage = ({ totalTables }) => {
);
};
- const exportTable = () => {
- appService
- .exportResource({
- tooljet_database: [{ table_id: selectedTable.id }],
- organization_id: organizationId,
- })
- .then((data) => {
- const tableName = selectedTable.table_name.replace(/\s+/g, '-').toLowerCase();
- const fileName = `${tableName}-export-${new Date().getTime()}`;
- // simulate link click download
- const json = JSON.stringify(data, null, 2);
- const blob = new Blob([json], { type: 'application/json' });
- const href = URL.createObjectURL(blob);
- const link = document.createElement('a');
- link.href = href;
- link.download = fileName + '.json';
- document.body.appendChild(link);
- link.click();
- document.body.removeChild(link);
- })
- .catch(() => {
- toast.error('Could not export table.', {
- position: 'top-center',
- });
+ const handleFileValidation = () => {
+ const fileValidationErrors = [];
+
+ if (bulkUploadFile && bulkUploadFile.size / 1024 > 2 * 1024) {
+ fileValidationErrors.push('File size cannot exceed 2mb');
+ }
+
+ setErrors({ server: [], client: fileValidationErrors });
+ };
+
+ const handleBulkUpload = async (event) => {
+ event.preventDefault();
+ setErrors({ client: [], server: [] });
+ setIsBulkUploading(true);
+
+ const formData = new FormData();
+ formData.append('file', bulkUploadFile);
+ try {
+ const { error, data } = await tooljetDatabaseService.bulkUpload(
+ organizationId,
+ selectedTable.table_name,
+ formData
+ );
+
+ if (error) {
+ setErrors({ ...errors, ...{ server: error.message } });
+ setIsBulkUploading(false);
+ toast.error('Upload failed!', { position: 'top-center' });
+ return;
+ }
+
+ const { processed_rows: processedRows, rows_inserted: rowsInserted, rows_updated: rowsUpdated } = data.result;
+ const toastMessage =
+ `${pluralize(rowsInserted, 'new row')} added, ` + `${pluralize(rowsUpdated, 'row')} updated.`;
+
+ toast.success(toastMessage, {
+ position: 'top-center',
});
+
+ setUploadResult({ processedRows, rowsInserted, rowsUpdated });
+ } catch (error) {
+ toast.error(error.errors, { position: 'top-center' });
+ setIsBulkUploading(false);
+ }
+ };
+
+ const handleBulkUploadFileChange = (file) => {
+ setBulkUploadFile(file);
};
return (
@@ -92,28 +170,15 @@ const TooljetDatabasePage = ({ totalTables }) => {
<>
-
+
-
+
{
isCreateRowDrawerOpen={isEditRowDrawerOpen}
setIsCreateRowDrawerOpen={setIsEditRowDrawerOpen}
/>
+
+
diff --git a/frontend/src/TooljetDatabase/constants.js b/frontend/src/TooljetDatabase/constants.js
index 5409c8584e..d5f8701b7f 100644
--- a/frontend/src/TooljetDatabase/constants.js
+++ b/frontend/src/TooljetDatabase/constants.js
@@ -1,6 +1,7 @@
export const dataTypes = [
{ value: 'character varying', label: 'varchar' },
{ value: 'integer', label: 'int' },
+ { value: 'bigint', label: 'bigint' },
{ value: 'double precision', label: 'float' },
{ value: 'boolean', label: 'boolean' },
];
diff --git a/frontend/src/TooljetDatabase/usePostgrestQueryBuilder.jsx b/frontend/src/TooljetDatabase/usePostgrestQueryBuilder.jsx
index 7fca6489cf..19f375eba0 100644
--- a/frontend/src/TooljetDatabase/usePostgrestQueryBuilder.jsx
+++ b/frontend/src/TooljetDatabase/usePostgrestQueryBuilder.jsx
@@ -25,10 +25,14 @@ export const usePostgrestQueryBuilder = ({ organizationId, selectedTable, setSel
};
const updateSelectedTableData = async () => {
+ const sortQuery = isEmpty(postgrestQueryBuilder.current.sortQuery.url.toString())
+ ? 'order=id.desc'
+ : postgrestQueryBuilder.current.sortQuery.url.toString();
+
const query =
postgrestQueryBuilder.current.filterQuery.url.toString() +
'&' +
- postgrestQueryBuilder.current.sortQuery.url.toString() +
+ sortQuery +
'&' +
postgrestQueryBuilder.current.paginationQuery.url.toString();
diff --git a/frontend/src/_components/ConfirmDialog.jsx b/frontend/src/_components/ConfirmDialog.jsx
index 4dc30e59a3..66c3f8b66b 100644
--- a/frontend/src/_components/ConfirmDialog.jsx
+++ b/frontend/src/_components/ConfirmDialog.jsx
@@ -17,6 +17,7 @@ export function ConfirmDialog({
cancelButtonText = 'Cancel',
backdropClassName,
onCloseIconClick,
+ footerStyle,
}) {
darkMode = darkMode ?? (localStorage.getItem('darkMode') === 'true' || false);
const [showModal, setShow] = useState(show);
@@ -73,7 +74,7 @@ export function ConfirmDialog({
{message}
-
+
{cancelButtonText ?? t('globals.cancel', 'Cancel')}
diff --git a/frontend/src/_helpers/appUtils.js b/frontend/src/_helpers/appUtils.js
index a2fefd4e3e..578ca8bb92 100644
--- a/frontend/src/_helpers/appUtils.js
+++ b/frontend/src/_helpers/appUtils.js
@@ -889,6 +889,7 @@ export function previewQuery(_ref, query, calledFromQuery = false, parameters =
setPreviewLoading(false);
setPreviewData(finalData);
}
+ let queryStatusCode = data?.status ?? null;
const queryStatus =
query.kind === 'tooljetdb'
? data.statusText
@@ -896,23 +897,29 @@ export function previewQuery(_ref, query, calledFromQuery = false, parameters =
? data?.data?.status ?? 'ok'
: data.status;
- switch (queryStatus) {
- case 'Bad Request':
- case 'failed': {
+ switch (true) {
+ // Note: Need to move away from statusText -> statusCode
+ case queryStatus === 'Bad Request' ||
+ queryStatus === 'Not Found' ||
+ queryStatus === 'Unprocessable Entity' ||
+ queryStatus === 'failed' ||
+ queryStatusCode === 400 ||
+ queryStatusCode === 404 ||
+ queryStatusCode === 422: {
const err = query.kind == 'tooljetdb' ? data?.error || data : _.isEmpty(data.data) ? data : data.data;
toast.error(`${err.message}`);
break;
}
- case 'needs_oauth': {
+ case queryStatus === 'needs_oauth': {
const url = data.data.auth_url; // Backend generates and return sthe auth url
fetchOAuthToken(url, query.data_source_id);
break;
}
- case 'ok':
- case 'OK':
- case 'Created':
- case 'Accepted':
- case 'No Content': {
+ case queryStatus === 'ok' ||
+ queryStatus === 'OK' ||
+ queryStatus === 'Created' ||
+ queryStatus === 'Accepted' ||
+ queryStatus === 'No Content': {
toast(`Query ${'(' + query.name + ') ' || ''}completed.`, {
icon: '🚀',
});
@@ -997,15 +1004,45 @@ export function runQuery(_ref, queryId, queryName, confirmed = undefined, mode =
fetchOAuthToken(url, dataQuery['data_source_id'] || dataQuery['dataSourceId']);
}
+ let queryStatusCode = data?.status ?? null;
const promiseStatus =
query.kind === 'tooljetdb'
? data.statusText
: query.kind === 'runpy'
? data?.data?.status ?? 'ok'
: data.status;
-
- if (promiseStatus === 'failed' || promiseStatus === 'Bad Request') {
- const errorData = query.kind === 'runpy' ? data.data : data;
+ // Note: Need to move away from statusText -> statusCode
+ if (
+ promiseStatus === 'failed' ||
+ promiseStatus === 'Bad Request' ||
+ promiseStatus === 'Not Found' ||
+ promiseStatus === 'Unprocessable Entity' ||
+ queryStatusCode === 400 ||
+ queryStatusCode === 404 ||
+ queryStatusCode === 422
+ ) {
+ let errorData = {};
+ switch (query.kind) {
+ case 'runpy':
+ errorData = data.data;
+ break;
+ case 'tooljetdb':
+ if (data?.error) {
+ errorData = {
+ message: data?.error?.message || 'Something went wrong',
+ description: data?.error?.message || 'Something went wrong',
+ status: data?.statusText || 'Failed',
+ data: data?.error || {},
+ };
+ } else {
+ errorData = data;
+ }
+ break;
+ default:
+ errorData = data;
+ break;
+ }
+ // errorData = query.kind === 'runpy' ? data.data : data;
useCurrentStateStore.getState().actions.setErrors({
[queryName]: {
type: 'query',
diff --git a/frontend/src/_helpers/http-client.js b/frontend/src/_helpers/http-client.js
index f12ed47b06..cb27800a69 100644
--- a/frontend/src/_helpers/http-client.js
+++ b/frontend/src/_helpers/http-client.js
@@ -34,7 +34,7 @@ class HttpClient {
const endpoint = urlJoin(this.host, this.namespace, url);
const options = {
method,
- headers: this.headers,
+ headers: { ...this.headers },
credentials: 'include',
};
let session = authenticationService.currentSessionValue;
@@ -48,9 +48,13 @@ class HttpClient {
}
options.headers['tj-workspace-id'] = session?.current_organization_id;
+
if (data) {
- options.body = JSON.stringify(data);
+ // fetch library generates content type with boundary for form data
+ data instanceof FormData && delete options.headers['content-type'];
+ options.body = data instanceof FormData ? data : JSON.stringify(data);
}
+
const response = await fetch(endpoint, options);
const payload = {
status: response.status,
diff --git a/frontend/src/_services/tooljetDatabase.service.js b/frontend/src/_services/tooljetDatabase.service.js
index 8ff35752af..3663ff7a18 100644
--- a/frontend/src/_services/tooljetDatabase.service.js
+++ b/frontend/src/_services/tooljetDatabase.service.js
@@ -4,30 +4,34 @@ const tooljetAdapter = new HttpClient();
function findOne(headers, tableId, query = '') {
tooljetAdapter.headers = { ...tooljetAdapter.headers, ...headers };
- return tooljetAdapter.get(`/tooljet_db/proxy/${tableId}?${query}`, headers);
+ return tooljetAdapter.get(`/tooljet-db/proxy/${tableId}?${query}`, headers);
}
function findAll(organizationId) {
- return tooljetAdapter.get(`/tooljet_db/organizations/${organizationId}/tables`);
+ return tooljetAdapter.get(`/tooljet-db/organizations/${organizationId}/tables`);
}
function createTable(organizationId, tableName, columns) {
- return tooljetAdapter.post(`/tooljet_db/organizations/${organizationId}/table`, {
+ return tooljetAdapter.post(`/tooljet-db/organizations/${organizationId}/table`, {
table_name: tableName,
columns,
});
}
function viewTable(organizationId, tableName) {
- return tooljetAdapter.get(`/tooljet_db/organizations/${organizationId}/table/${tableName}`);
+ return tooljetAdapter.get(`/tooljet-db/organizations/${organizationId}/table/${tableName}`);
+}
+
+function bulkUpload(organizationId, tableName, file) {
+ return tooljetAdapter.post(`/tooljet-db/organizations/${organizationId}/table/${tableName}/bulk-upload`, file);
}
function createRow(headers, tableId, data) {
- return tooljetAdapter.post(`/tooljet_db/proxy/${tableId}`, data, headers);
+ return tooljetAdapter.post(`/tooljet-db/proxy/${tableId}`, data, headers);
}
function createColumn(organizationId, tableId, columnName, dataType, defaultValue) {
- return tooljetAdapter.post(`/tooljet_db/organizations/${organizationId}/table/${tableId}/column`, {
+ return tooljetAdapter.post(`/tooljet-db/organizations/${organizationId}/table/${tableId}/column`, {
column: {
column_name: columnName,
data_type: dataType,
@@ -37,7 +41,7 @@ function createColumn(organizationId, tableId, columnName, dataType, defaultValu
}
function updateTable(organizationId, tableName, columns) {
- return tooljetAdapter.patch(`/tooljet_db/${organizationId}/perform`, {
+ return tooljetAdapter.patch(`/tooljet-db/${organizationId}/perform`, {
action: 'update_table',
table_name: tableName,
columns,
@@ -45,7 +49,7 @@ function updateTable(organizationId, tableName, columns) {
}
function renameTable(organizationId, tableName, newTableName) {
- return tooljetAdapter.patch(`/tooljet_db/organizations/${organizationId}/table/${tableName}`, {
+ return tooljetAdapter.patch(`/tooljet-db/organizations/${organizationId}/table/${tableName}`, {
action: 'rename_table',
table_name: tableName,
new_table_name: newTableName,
@@ -53,19 +57,23 @@ function renameTable(organizationId, tableName, newTableName) {
}
function updateRows(headers, tableId, data, query = '') {
- return tooljetAdapter.patch(`/tooljet_db/proxy/${tableId}?${query}`, data, headers);
+ return tooljetAdapter.patch(`/tooljet-db/proxy/${tableId}?${query}`, data, headers);
}
function deleteRows(headers, tableId, query = '') {
- return tooljetAdapter.delete(`/tooljet_db/proxy/${tableId}?${query}`, headers);
+ return tooljetAdapter.delete(`/tooljet-db/proxy/${tableId}?${query}`, headers);
}
function deleteColumn(organizationId, tableName, columnName) {
- return tooljetAdapter.delete(`/tooljet_db/organizations/${organizationId}/table/${tableName}/column/${columnName}`);
+ return tooljetAdapter.delete(`/tooljet-db/organizations/${organizationId}/table/${tableName}/column/${columnName}`);
}
function deleteTable(organizationId, tableName) {
- return tooljetAdapter.delete(`/tooljet_db/organizations/${organizationId}/table/${tableName}`);
+ return tooljetAdapter.delete(`/tooljet-db/organizations/${organizationId}/table/${tableName}`);
+}
+
+function joinTables(organizationId, data) {
+ return tooljetAdapter.post(`tooljet-db/organizations/${organizationId}/join`, data);
}
export const tooljetDatabaseService = {
@@ -81,4 +89,6 @@ export const tooljetDatabaseService = {
deleteColumn,
deleteTable,
renameTable,
+ bulkUpload,
+ joinTables,
};
diff --git a/frontend/src/_styles/queryManager.scss b/frontend/src/_styles/queryManager.scss
index ccfe586acd..1378e93e68 100644
--- a/frontend/src/_styles/queryManager.scss
+++ b/frontend/src/_styles/queryManager.scss
@@ -496,6 +496,11 @@ $border-radius: 4px;
width: 34px;
}
+ .delete-field-option-dark {
+ background-color: #272822 !important;
+ border-color: #272822 !important;
+ }
+
.code-hinter.codehinter-default-input {
border: 1px solid transparent !important;
}
@@ -1525,6 +1530,51 @@ $border-radius: 4px;
overflow-wrap: break-word;
}
+.dd-select-control-chevron {
+ position: absolute;
+ right: 0;
+}
+
+.dd-select-value-badge {
+ border-radius: 6px;
+ background: var(--slate3) !important;
+ text-transform: none;
+ color: var(--slate12) !important;
+}
+
+.dd-select-alert-error {
+ border-radius: 6px;
+ border: 1px solid var(--tomato-05, #FDD8D3);
+ background: var(--tomato-02, #FFF8F7);
+ padding: 8px;
+ font-size: 10px;
+ font-weight: 400;
+ color: var(--tomato-09, #E54D2E);
+ svg {
+ height: 17px;
+ width: 17px;
+ margin-right: 2px;
+ }
+ svg>path{
+ fill:var(--tomato-09, #E54D2E);
+ opacity: 0.75;
+ }
+}
+
+.tdb-join-filtersection {
+ .codehinter-plugins {
+ border: 0 !important;
+ }
+}
+
+.tdb-dropdown-btn {
+ &:active {
+ border: 1px solid var(--indigo-09, #3E63DD) !important;
+ background: var(--indigo2, #F8FAFF);
+ box-shadow: 0px 0px 0px 1px #C6D4F9
+ }
+}
+
.copilot-section-header {
background-color: var(--slate2);
border: 1px solid var(--slate5);
diff --git a/frontend/src/_styles/theme.scss b/frontend/src/_styles/theme.scss
index 5eaf15fb4d..fa21410e7e 100644
--- a/frontend/src/_styles/theme.scss
+++ b/frontend/src/_styles/theme.scss
@@ -4609,16 +4609,16 @@ input[type="text"] {
color: $white !important;
.modal-title {
- color: $white !important;
+ color: $white !important;
}
.tj-version-wrap-sub-footer {
- background-color: $bg-dark-light !important;
+ background-color: $bg-dark-light !important;
border-top: 1px solid #3A3F42 !important;
p {
- color: $white !important;
+ color: $white !important;
}
}
@@ -4629,7 +4629,9 @@ input[type="text"] {
}
.modal-header {
- background-color: $bg-dark-light !important;
+ background-color: $bg-dark-light !important;
+ color: $white !important;
+ border-bottom: 2px solid #3A3F42 !important;
}
.btn-close {
@@ -6583,9 +6585,9 @@ input.hide-input-arrows {
box-shadow: 0px 12px 16px -4px rgba(16, 24, 40, 0.08), 0px 4px 6px -2px rgba(16, 24, 40, 0.03);
border-radius: 4px;
border: 1px solid var(--slate3) !important;
- left: 109px !important;
- top: 8px !important;
- position: absolute !important;
+ // left: 109px !important;
+ // top: 8px !important;
+ // position: absolute !important;
.card-body,
@@ -6603,9 +6605,9 @@ input.hide-input-arrows {
box-shadow: 0px 12px 16px -4px rgba(16, 24, 40, 0.08), 0px 4px 6px -2px rgba(16, 24, 40, 0.03);
border-radius: 4px;
border: 1px solid var(--slate3) !important;
- left: 193px !important;
- top: 10px !important;
- position: absolute !important;
+ // left: 193px !important;
+ // top: 10px !important;
+ // position: absolute !important;
.card-body,
@@ -6909,14 +6911,6 @@ tbody {
.delete-row-btn {
max-width: 140px;
}
-
- .table-list-item-popover {
- display: none;
- }
-
- .table-list-item:hover .table-list-item-popover {
- display: block;
- }
}
.apploader {
@@ -8725,7 +8719,7 @@ tbody {
}
}
-.tj-db-operaions-header {
+.tj-db-operations-header {
height: 48px;
padding: 0 !important;
display: flex;
@@ -8734,19 +8728,19 @@ tbody {
.row {
margin-left: 0px;
+ width: 98%;
}
- .col {
+ .col-8 {
padding-left: 0px;
display: flex;
- gap: 8px;
+ gap: 12px;
align-items: center;
}
}
.add-new-column-btn {
margin-left: 16px;
- width: 144px !important;
height: 28px;
border-radius: 6px;
padding: 0 !important;
@@ -8759,7 +8753,7 @@ tbody {
}
.tj-db-filter-btn {
- width: 81px;
+ width: 100%;
height: 28px;
border-radius: 6px;
background: transparent;
@@ -8777,7 +8771,7 @@ tbody {
justify-content: center !important;
align-items: center !important;
padding: 4px 16px !important;
- width: 171px !important;
+ width: 100% !important;
height: 28px !important;
background: var(--grass2) !important;
border-radius: 6px !important;
@@ -8785,19 +8779,22 @@ tbody {
.tj-db-filter-btn-active,
.tj-db-sort-btn-active {
- width: 81px !important;
+ display: flex !important;
+ flex-direction: row !important;
+ justify-content: center !important;
+ align-items: center !important;
+ padding: 4px 16px !important;
+ width: 100% !important;
height: 28px !important;
- background: var(--indigo4) !important;
- border: 1px solid var(--indigo9) !important;
border-radius: 6px !important;
- justify-content: center;
+ background: var(--indigo4) !important;
+ //border: 1px solid var(--indigo9) !important;
color: var(--indigo9) !important;
}
.tj-db-header-add-new-row-btn {
- width: 125px;
height: 28px;
- background: var(--indigo3);
+ background: transparent;
border-radius: 6px !important;
display: flex;
flex-direction: row;
@@ -8806,13 +8803,13 @@ tbody {
gap: 6px;
border: none;
- span {
- color: var(--indigo9);
+ padding: span {
+ //color: var(--indigo9);
}
}
.tj-db-sort-btn {
- width: 75px;
+ width: 100%;
height: 28px;
background: transparent;
color: var(--slate12);
@@ -8820,6 +8817,7 @@ tbody {
display: flex;
align-items: center;
justify-content: center;
+ margin: 0
}
.edit-row-btn {
@@ -8828,6 +8826,7 @@ tbody {
border: none;
display: flex;
align-items: center;
+ justify-content: center;
}
.workspace-variable-header {
@@ -9687,7 +9686,6 @@ tbody {
padding: 60px 0px;
gap: 36px;
width: 486px;
- height: 244px;
border: 2px dashed var(--indigo9);
border-radius: 6px;
align-items: center;
@@ -10425,6 +10423,7 @@ tbody {
.upload-user-form span.file-upload-error {
color: var(--tomato10) !important;
+ margin-top: 12px 0px 0px 0px;
}
.tj-onboarding-phone-input {
@@ -10613,7 +10612,7 @@ tbody {
}
.export-table-button {
- width: 135px;
+
display: flex;
align-items: center;
justify-content: center;
@@ -10621,7 +10620,7 @@ tbody {
#global-settings-popover.theme-dark {
- background-color: $bg-dark-light !important;
+ background-color: $bg-dark-light !important;
border: 1px solid #2B2F31;
.global-popover-text {
@@ -10629,7 +10628,7 @@ tbody {
}
.maximum-canvas-width-input-select {
- background-color: $bg-dark-light !important;
+ background-color: $bg-dark-light !important;
border: 1px solid #324156;
color: $white;
}
@@ -10924,6 +10923,13 @@ tbody {
border: 1px solid var(--slate3);
box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05);
border-radius: 6px;
+
+ .card-body-alignment {
+ min-height: 145px;
+ display: flex;
+ flex-direction: column;
+ justify-content: space-between;
+ }
}
.template-source-name {
@@ -11581,6 +11587,14 @@ tbody {
.custom-gap-6{
gap:6px
}
+
+// ToolJet Database buttons
+
+.ghost-black-operation {
+ border: 1px solid transparent !important;
+ padding: 4px 10px;
+}
+
.custom-gap-2{
gap:2px
}
@@ -11599,4 +11613,4 @@ tbody {
border-bottom: none !important;
}
}
-}
\ No newline at end of file
+}
diff --git a/frontend/src/_ui/AppButton/AppButton.scss b/frontend/src/_ui/AppButton/AppButton.scss
index 7f61321213..9276abd7a9 100644
--- a/frontend/src/_ui/AppButton/AppButton.scss
+++ b/frontend/src/_ui/AppButton/AppButton.scss
@@ -64,13 +64,14 @@
.tj-primary-btn {
border-radius: 6px;
border: 1px solid rgba(255, 255, 255, 0.00);
- background: var(--indigo9) ;
+ background: var(--indigo9);
box-shadow: 0px 2px 0px 0px rgba(0, 0, 0, 0.04);
color: #FDFDFE;
+
&:hover {
border: 1px solid rgba(255, 255, 255, 0.00);
- background: var(--indigo10) ;
- color: #FDFDFE;
+ background: var(--indigo10);
+ color: #FDFDFE;
}
&:focus-visible {
@@ -93,6 +94,7 @@
color: var(--indigo9) !important;
border: 1px solid rgba(255, 255, 255, 0.00) !important;
background: var(--indigo3, #F0F4FF) !important;
+
&:hover {
color: var(--indigo10) !important;
border: 1px solid rgba(255, 255, 255, 0.00) !important;
@@ -115,7 +117,8 @@
/* Focus rings/Indigo/light */
box-shadow: 0px 0px 0px 4px var(--indigo6);
- }}
+ }
+}
.tj-tertiary-btn {
color: var(--slate12);
@@ -128,22 +131,27 @@
background: var(--slate4, #ECEEF0) !important;
}
- &:active,&:focus,&.always-active-btn {
+ &:active,
+ &:focus,
+ &.always-active-btn {
border: 1px solid var(--slate7, #D7DBDF) !important;
background: var(--slate5) !important;
color: var(--slate12);
border: 1px solid var(--slate7);
}
+
&:active {
- background: transparent;
+ background: var(--slate1);
box-shadow: none;
color: var(--slate12) !important;
}
&:focus-visible {
- color: var(--slate11) !important;
- box-shadow: 0px 0px 0px 4px var(--slate6) !important;
+ background: var(--slate1);
+ color: var(--slate11);
+ outline: 1px solid var(--slate8);
+ box-shadow: 0px 0px 0px 4px var(--slate6);
outline: none;
border: 1px solid var(--slate8, #C1C8CD) !important;
background: var(--slate1, #FBFCFD) !important;
@@ -188,7 +196,8 @@
color: var(--slate11);
border: 1px solid rgba(255, 255, 255, 0.00) !important;
}
- &:focus-visible{
+
+ &:focus-visible {
color: var(--slate11) !important;
// background: var(--base);
// border: none;
@@ -197,13 +206,17 @@
border: 1px solid rgba(255, 255, 255, 0.00) !important;
background: var(--slate1, #FBFCFD) !important;
}
- &:active,&:focus,&.always-active-btn {
+
+ &:active,
+ &:focus,
+ &.always-active-btn {
color: var(--slate12) !important;
border: 1px solid rgba(255, 255, 255, 0.00) !important;
background: var(--slate5, #E6E8EB) !important;
box-shadow: none !important;
}
- &:disabled{
+
+ &:disabled {
background: rgba(255, 255, 255, 0.00) !important;
border: 1px solid rgba(255, 255, 255, 0.00) !important;
color: var(--slate8) !important;
diff --git a/frontend/src/_ui/Drawer/index.jsx b/frontend/src/_ui/Drawer/index.jsx
index f99f28830e..006db171cf 100644
--- a/frontend/src/_ui/Drawer/index.jsx
+++ b/frontend/src/_ui/Drawer/index.jsx
@@ -23,6 +23,7 @@ const Drawer = ({
onClose,
position = 'left',
removeWhenClosed = true,
+ drawerStyle,
}) => {
const bodyRef = useRef(document.querySelector('body'));
const portalRootRef = useRef(document.getElementById('tooljet-drawer-root') || createPortalRoot());
@@ -79,7 +80,7 @@ const Drawer = ({
return createPortal(
-
+
-
+
{children}
diff --git a/frontend/src/_ui/Icon/solidIcons/FullOuterJoin.jsx b/frontend/src/_ui/Icon/solidIcons/FullOuterJoin.jsx
new file mode 100644
index 0000000000..afa2c6bc99
--- /dev/null
+++ b/frontend/src/_ui/Icon/solidIcons/FullOuterJoin.jsx
@@ -0,0 +1,19 @@
+import React from 'react';
+
+const FullOuterJoin = ({ fill = '#889096', width = '25', className = '', viewBox = '0 0 25 25' }) => (
+
+
+);
+
+export default FullOuterJoin;
diff --git a/frontend/src/_ui/Icon/solidIcons/Information.jsx b/frontend/src/_ui/Icon/solidIcons/Information.jsx
index e98032b19a..ca4d6281a5 100644
--- a/frontend/src/_ui/Icon/solidIcons/Information.jsx
+++ b/frontend/src/_ui/Icon/solidIcons/Information.jsx
@@ -1,6 +1,6 @@
import React from 'react';
-const Information = ({ fill = '#C1C8CD', width = '25', className = '', viewBox = '0 0 25 25' }) => (
+const Information = ({ fill = '#C1C8CD', width = '25', className = '', viewBox = '0 0 25 25', style }) => (
(
+
+
+);
+
+export default InnerJoinIcon;
diff --git a/frontend/src/_ui/Icon/solidIcons/LeftOuterJoinIcon.jsx b/frontend/src/_ui/Icon/solidIcons/LeftOuterJoinIcon.jsx
new file mode 100644
index 0000000000..61bac8786d
--- /dev/null
+++ b/frontend/src/_ui/Icon/solidIcons/LeftOuterJoinIcon.jsx
@@ -0,0 +1,19 @@
+import React from 'react';
+
+const LeftOuterJoinIcon = ({ fill = '#889096', width = '25', className = '', viewBox = '0 0 25 25' }) => (
+
+
+);
+
+export default LeftOuterJoinIcon;
diff --git a/frontend/src/_ui/Icon/solidIcons/ReloadError.jsx b/frontend/src/_ui/Icon/solidIcons/ReloadError.jsx
new file mode 100644
index 0000000000..055525eb4e
--- /dev/null
+++ b/frontend/src/_ui/Icon/solidIcons/ReloadError.jsx
@@ -0,0 +1,24 @@
+import React from 'react';
+
+const ReloadError = ({ fill = '#E54D2E', width = '16', height = '17', className = '', viewBox = '0 0 16 17' }) => (
+
+
+
+
+);
+
+export default ReloadError;
diff --git a/frontend/src/_ui/Icon/solidIcons/Remove.jsx b/frontend/src/_ui/Icon/solidIcons/Remove.jsx
index e952871856..aeb0e1fe11 100644
--- a/frontend/src/_ui/Icon/solidIcons/Remove.jsx
+++ b/frontend/src/_ui/Icon/solidIcons/Remove.jsx
@@ -1,6 +1,6 @@
import React from 'react';
-const Remove = ({ fill = '#C1C8CD', width = '25', className = '', viewBox = '0 0 25 25' }) => (
+const Remove = ({ fill = '#C1C8CD', width = '25', className = '', viewBox = '0 0 25 25', style }) => (
(
+
+
+);
+
+export default RightOuterJoin;
diff --git a/frontend/src/_ui/Icon/solidIcons/Tick.jsx b/frontend/src/_ui/Icon/solidIcons/Tick.jsx
index 97fe9588ab..f0ef0e8cc7 100644
--- a/frontend/src/_ui/Icon/solidIcons/Tick.jsx
+++ b/frontend/src/_ui/Icon/solidIcons/Tick.jsx
@@ -1,6 +1,6 @@
import React from 'react';
-const Tick = ({ fill = '#C1C8CD', width = '25', className = '', viewBox = '0 0 25 25' }) => (
+const Tick = ({ fill = '#C1C8CD', width = '25', className = '', viewBox = '0 0 25 25', style }) => (
{
return |
;
case 'sadrectangle':
diff --git a/server/.version b/server/.version
index cf8690732f..ef0f38abe1 100644
--- a/server/.version
+++ b/server/.version
@@ -1 +1 @@
-2.18.0
+2.19.0
diff --git a/server/src/controllers/tooljet_db.controller.ts b/server/src/controllers/tooljet_db.controller.ts
index 956ecce253..05fa0a7f9d 100644
--- a/server/src/controllers/tooljet_db.controller.ts
+++ b/server/src/controllers/tooljet_db.controller.ts
@@ -1,4 +1,21 @@
-import { All, Controller, Req, Res, Next, UseGuards, Get, Post, Body, Param, Delete, Patch } from '@nestjs/common';
+import {
+ All,
+ Controller,
+ Req,
+ Res,
+ Next,
+ UseGuards,
+ Get,
+ Post,
+ Body,
+ Param,
+ Delete,
+ Patch,
+ UseInterceptors,
+ UploadedFile,
+ BadRequestException,
+} from '@nestjs/common';
+import { Express } from 'express';
import { JwtAuthGuard } from 'src/modules/auth/jwt-auth.guard';
import { ActiveWorkspaceGuard } from 'src/modules/auth/active-workspace.guard';
import { TooljetDbService } from '@services/tooljet_db.service';
@@ -10,12 +27,17 @@ import { Action, TooljetDbAbility } from 'src/modules/casl/abilities/tooljet-db-
import { TooljetDbGuard } from 'src/modules/casl/tooljet-db.guard';
import { CreatePostgrestTableDto, RenamePostgrestTableDto, PostgrestTableColumnDto } from '@dto/tooljet-db.dto';
import { OrganizationAuthGuard } from 'src/modules/auth/organization-auth.guard';
+import { FileInterceptor } from '@nestjs/platform-express';
+import { TooljetDbBulkUploadService } from '@services/tooljet_db_bulk_upload.service';
-@Controller('tooljet_db')
+const MAX_CSV_FILE_SIZE = 1024 * 1024 * 2; // 2MB
+
+@Controller('tooljet-db')
export class TooljetDbController {
constructor(
private readonly tooljetDbService: TooljetDbService,
- private readonly postgrestProxyService: PostgrestProxyService
+ private readonly postgrestProxyService: PostgrestProxyService,
+ private readonly tooljetDbBulkUploadService: TooljetDbBulkUploadService
) {}
@All('/proxy/*')
@@ -97,4 +119,31 @@ export class TooljetDbController {
const result = await this.tooljetDbService.perform(organizationId, 'drop_column', params);
return decamelizeKeys({ result });
}
+
+ @UseInterceptors(FileInterceptor('file'))
+ @Post('/organizations/:organizationId/table/:tableName/bulk-upload')
+ async bulkUpload(
+ @Param('organizationId') organizationId,
+ @Param('tableName') tableName,
+ @UploadedFile() file: Express.Multer.File
+ ) {
+ if (file.size > MAX_CSV_FILE_SIZE) {
+ throw new BadRequestException('File size cannot be greater than 2MB');
+ }
+ const result = await this.tooljetDbBulkUploadService.perform(organizationId, tableName, file.buffer);
+
+ return decamelizeKeys({ result });
+ }
+
+ @Post('/organizations/:organizationId/join')
+ @UseGuards(TooljetDbGuard)
+ @CheckPolicies((ability: TooljetDbAbility) => ability.can(Action.JoinTables, 'all'))
+ async joinTables(@Body() joinQueryJsonDto: any, @Param('organizationId') organizationId) {
+ const params = {
+ joinQueryJson: { ...joinQueryJsonDto },
+ };
+
+ const result = await this.tooljetDbService.perform(organizationId, 'join_tables', params);
+ return decamelizeKeys({ result });
+ }
}
diff --git a/server/src/dto/tooljet-db.dto.ts b/server/src/dto/tooljet-db.dto.ts
index 7a79482d3f..3ccc737608 100644
--- a/server/src/dto/tooljet-db.dto.ts
+++ b/server/src/dto/tooljet-db.dto.ts
@@ -49,7 +49,7 @@ export class MatchTypeConstraint implements ValidatorConstraintInterface {
return typeof value === 'string';
}
- if (relatedType === 'integer' || relatedType === 'double precision') {
+ if (relatedType === 'integer' || relatedType === 'bigint' || relatedType === 'double precision') {
const isInt = Number.isInteger(value);
const isFloat = !Number.isInteger(value) && !isNaN(value);
return isInt || isFloat;
diff --git a/server/src/modules/casl/abilities/tooljet-db-ability.factory.ts b/server/src/modules/casl/abilities/tooljet-db-ability.factory.ts
index a29e135d65..739355577a 100644
--- a/server/src/modules/casl/abilities/tooljet-db-ability.factory.ts
+++ b/server/src/modules/casl/abilities/tooljet-db-ability.factory.ts
@@ -13,6 +13,8 @@ export enum Action {
DropTable = 'dropTable',
AddColumn = 'addColumn',
DropColumn = 'dropColumn',
+ BulkUpload = 'bulkUpload',
+ JoinTables = 'joinTables',
}
type Subjects = 'all';
@@ -36,6 +38,7 @@ export class TooljetDbAbilityFactory {
can(Action.AddColumn, 'all');
can(Action.DropColumn, 'all');
can(Action.RenameTable, 'all');
+ can(Action.BulkUpload, 'all');
}
if (isPublicAppRequest || isUserLoggedin) {
@@ -44,6 +47,7 @@ export class TooljetDbAbilityFactory {
can(Action.ViewTables, 'all');
can(Action.ViewTable, 'all');
+ can(Action.JoinTables, 'all');
return build({
detectSubjectType: (item) => item as ExtractSubjectType,
diff --git a/server/src/modules/import_export_resources/import_export_resources.module.ts b/server/src/modules/import_export_resources/import_export_resources.module.ts
index a920608b23..df40321a3f 100644
--- a/server/src/modules/import_export_resources/import_export_resources.module.ts
+++ b/server/src/modules/import_export_resources/import_export_resources.module.ts
@@ -20,6 +20,7 @@ import { AppsService } from '@services/apps.service';
import { App } from 'src/entities/app.entity';
import { AppVersion } from 'src/entities/app_version.entity';
import { AppUser } from 'src/entities/app_user.entity';
+import { PostgrestProxyService } from '@services/postgrest_proxy.service';
const imports = [
PluginsModule,
@@ -45,6 +46,7 @@ if (process.env.ENABLE_TOOLJET_DB === 'true') {
PluginsHelper,
AppsService,
CredentialsService,
+ PostgrestProxyService,
],
})
export class ImportExportResourcesModule {}
diff --git a/server/src/modules/tooljet_db/tooljet_db.module.ts b/server/src/modules/tooljet_db/tooljet_db.module.ts
index fb98e5d95a..eb7c554ebe 100644
--- a/server/src/modules/tooljet_db/tooljet_db.module.ts
+++ b/server/src/modules/tooljet_db/tooljet_db.module.ts
@@ -7,10 +7,17 @@ import { TooljetDbService } from '@services/tooljet_db.service';
import { CredentialsService } from '@services/credentials.service';
import { EncryptionService } from '@services/encryption.service';
import { PostgrestProxyService } from '@services/postgrest_proxy.service';
+import { TooljetDbBulkUploadService } from '@services/tooljet_db_bulk_upload.service';
@Module({
imports: [TypeOrmModule.forFeature([Credential]), CaslModule],
controllers: [TooljetDbController],
- providers: [TooljetDbService, PostgrestProxyService, EncryptionService, CredentialsService],
+ providers: [
+ TooljetDbService,
+ TooljetDbBulkUploadService,
+ PostgrestProxyService,
+ EncryptionService,
+ CredentialsService,
+ ],
})
export class TooljetDbModule {}
diff --git a/server/src/services/postgrest_proxy.service.ts b/server/src/services/postgrest_proxy.service.ts
index edd818fc49..215db8df1d 100644
--- a/server/src/services/postgrest_proxy.service.ts
+++ b/server/src/services/postgrest_proxy.service.ts
@@ -26,7 +26,7 @@ export class PostgrestProxyService {
private httpProxy = proxy(this.configService.get('PGRST_HOST'), {
proxyReqPathResolver: function (req) {
- const path = '/api/tooljet_db';
+ const path = '/api/tooljet-db';
const pathRegex = new RegExp(`${maybeSetSubPath(path)}/proxy`);
const parts = req.url.split('?');
const queryString = parts[1];
@@ -85,7 +85,7 @@ export class PostgrestProxyService {
return urlBeingReplaced;
}
- private async findOrFailAllInternalTableFromTableNames(requestedTableNames: Array, organizationId: string) {
+ async findOrFailAllInternalTableFromTableNames(requestedTableNames: Array, organizationId: string) {
const internalTables = await this.manager.find(InternalTable, {
where: {
organizationId,
diff --git a/server/src/services/tooljet_db.service.ts b/server/src/services/tooljet_db.service.ts
index 24c6f5cca0..0b422e71a4 100644
--- a/server/src/services/tooljet_db.service.ts
+++ b/server/src/services/tooljet_db.service.ts
@@ -1,8 +1,21 @@
-import { BadRequestException, Injectable, NotFoundException, Optional } from '@nestjs/common';
-import { EntityManager } from 'typeorm';
+import { BadRequestException, HttpException, Injectable, NotFoundException, Optional } from '@nestjs/common';
+import { EntityManager, In, QueryFailedError } from 'typeorm';
import { InjectEntityManager } from '@nestjs/typeorm';
import { InternalTable } from 'src/entities/internal_table.entity';
-import { isString } from 'lodash';
+import { isString, isEmpty } from 'lodash';
+
+export type TableColumnSchema = {
+ column_name: string;
+ data_type: SupportedDataTypes;
+ column_default: string | null;
+ character_maximum_length: number | null;
+ numeric_precision: number | null;
+ is_nullable: 'YES' | 'NO';
+ constraint_type: string | null;
+ keytype: string | null;
+};
+
+export type SupportedDataTypes = 'character varying' | 'integer' | 'bigint' | 'serial' | 'double precision' | 'boolean';
@Injectable()
export class TooljetDbService {
@@ -10,7 +23,7 @@ export class TooljetDbService {
private readonly manager: EntityManager,
@Optional()
@InjectEntityManager('tooljetDb')
- private tooljetDbManager: EntityManager
+ private readonly tooljetDbManager: EntityManager
) {}
async perform(organizationId: string, action: string, params = {}) {
@@ -29,12 +42,14 @@ export class TooljetDbService {
return await this.dropColumn(organizationId, params);
case 'rename_table':
return await this.renameTable(organizationId, params);
+ case 'join_tables':
+ return await this.joinTable(organizationId, params);
default:
throw new BadRequestException('Action not defined');
}
}
- private async viewTable(organizationId: string, params) {
+ private async viewTable(organizationId: string, params): Promise {
const { table_name: tableName, id: id } = params;
const internalTable = await this.manager.findOne(InternalTable, {
@@ -232,4 +247,210 @@ export class TooljetDbService {
await this.tooljetDbManager.query("NOTIFY pgrst, 'reload schema'");
return result;
}
+
+ private async joinTable(organizationId: string, params) {
+ const { joinQueryJson } = params;
+ if (!Object.keys(joinQueryJson).length) throw new BadRequestException("Input can't be empty");
+
+ // Gathering tables used, from Join coditions
+ const tableSet = new Set();
+ const joinOptions = joinQueryJson?.['joins'];
+ (joinOptions || []).forEach((join) => {
+ const { table, conditions } = join;
+ tableSet.add(table);
+ conditions?.conditionsList?.forEach((condition) => {
+ const { leftField, rightField } = condition;
+ if (leftField?.table) {
+ tableSet.add(leftField?.table);
+ }
+ if (rightField?.table) {
+ tableSet.add(rightField?.table);
+ }
+ });
+ });
+
+ const tables = [...tableSet].map((tableId: string) => ({
+ name: tableId,
+ type: 'Table',
+ }));
+
+ if (!tables?.length) throw new BadRequestException('Tables are not chosen');
+
+ const tableIdList: Array = tables
+ .filter((table) => table.type === 'Table')
+ .map((filteredTable) => filteredTable.name);
+
+ const internalTables = await this.findOrFailInternalTableFromTableId(tableIdList, organizationId);
+ const internalTableIdToNameMap = tableIdList.reduce((acc, tableId) => {
+ return {
+ ...acc,
+ [tableId]: internalTables.find((table) => table.id === tableId).tableName,
+ };
+ }, {});
+
+ const finalQuery = await this.buildJoinQuery(organizationId, joinQueryJson, internalTableIdToNameMap);
+
+ try {
+ return await this.tooljetDbManager.query(finalQuery);
+ } catch (error) {
+ // custom error handling - for Query error
+ if (error instanceof QueryFailedError) {
+ let customErrorMessage: string = (error as QueryFailedError).message;
+ Object.entries(internalTableIdToNameMap).forEach(([key, value]) => {
+ customErrorMessage = customErrorMessage.replace(key, value as string);
+ });
+ throw new HttpException(customErrorMessage, 422);
+ }
+ throw error;
+ }
+ }
+
+ private async buildJoinQuery(_organizationId: string, queryJson, internalTableIdToNameMap) {
+ // Pending: For Subquery, Alias is its table name. Need to handle it on Internal Table details mapping
+ // Pending: SELECT Statement - Nested params --> SUM( price * quantity )
+
+ // @description: Only SELECT & FROM statement is Mandatory, else is Optional
+ let finalQuery = ``;
+ finalQuery += `SELECT ${await this.constructSelectStatement(queryJson.fields, internalTableIdToNameMap)}`;
+ finalQuery += `\nFROM ${await this.constructFromStatement(queryJson, internalTableIdToNameMap)}`;
+ if (queryJson?.joins?.length)
+ finalQuery += `\n${await this.constructJoinStatements(queryJson.joins, internalTableIdToNameMap)}`;
+ if (
+ queryJson?.conditions &&
+ Object.keys(queryJson?.conditions).length &&
+ queryJson?.conditions?.conditionsList.length
+ )
+ finalQuery += `\nWHERE ${await this.constructWhereStatement(queryJson.conditions, internalTableIdToNameMap)}`;
+ if (queryJson?.group_by?.length)
+ finalQuery += `\nGROUP BY ${await this.constructGroupByStatement(queryJson.group_by, internalTableIdToNameMap)}`;
+ if (queryJson?.having && Object.keys(queryJson?.having).length)
+ finalQuery += `\nHAVING ${await this.constructWhereStatement(queryJson.having, internalTableIdToNameMap)}`;
+ if (queryJson?.order_by?.length)
+ finalQuery += `\nORDER BY ${await this.constructOrderByStatement(queryJson.order_by, internalTableIdToNameMap)}`;
+ if (queryJson?.limit && queryJson?.limit.length) finalQuery += `\nLIMIT ${queryJson.limit}`;
+ if (queryJson?.offset && queryJson?.offset.length) finalQuery += `\nOFFSET ${queryJson.offset}`;
+
+ return finalQuery;
+ }
+
+ // Assuming tableId is being passed, tableName to tableId mapping is removed
+ private constructSelectStatement(selectStatementInputList, internalTableIdToNameMap) {
+ if (selectStatementInputList.length) {
+ const selectQueryFields = selectStatementInputList
+ .map((field) => {
+ let fieldExpression = ``;
+ if (field.function) fieldExpression += `${field.function}(`;
+ fieldExpression += `${field.table ? '"' + field.table + '"' + '.' : ''}${field.name}`;
+ if (field.function) fieldExpression += `)`;
+ if (field.alias) {
+ fieldExpression += ` AS ${field.alias}`;
+ } else {
+ // By Default Alias has been added here for tooljetdb join flow
+ fieldExpression += ` AS ${internalTableIdToNameMap[field.table]}_${field.name}`;
+ }
+ return fieldExpression;
+ })
+ .join(', ');
+ return selectQueryFields;
+ }
+
+ throw new BadRequestException('Select statement is empty');
+ }
+
+ private constructFromStatement(queryJson, _internalTableIdToNameMap) {
+ const { from } = queryJson;
+ if (from.name) {
+ return `${'"' + from.name + '"'} ${from.alias ? from.alias : ''}`;
+ }
+
+ throw new BadRequestException('From table is not selected');
+ }
+
+ private constructJoinStatements(joinsInputList, internalTableIdToNameMap) {
+ const joinStatementOutput = joinsInputList
+ .map((joinCondition) => {
+ const { table, joinType, conditions } = joinCondition;
+ return `${joinType} JOIN ${'"' + table + '"'} ${
+ joinCondition.alias ? joinCondition.alias : ''
+ } ON ${this.constructWhereStatement(conditions, internalTableIdToNameMap)}`;
+ })
+ .join('\n');
+ return joinStatementOutput;
+ }
+
+ private constructWhereStatement(whereStatementConditions, internalTableIdToNameMap) {
+ const { operator = 'AND', conditionsList = [] } = whereStatementConditions;
+ const whereConditionOutput = conditionsList
+ .map((condition) => {
+ // @description: Recursive call to build - Sub-condition
+ if (condition.conditions)
+ return `(${this.constructWhereStatement(condition.conditions, internalTableIdToNameMap)})`;
+ // @description: Building a Condition for 'WHERE & HAVING statements' - LHS, operator and RHS
+ // @description: In LHS & RHS it is not mandatory to provide table name, but column name is mandatory
+ // @description: In LHS & RHS - We get function only in HAVING statement
+ const { operator, leftField, rightField } = condition;
+ // @desc: When 'IS' operator is choosed, 'NULL' & 'NOT NULL' keywords will be provided as value and it should not be converted to string
+ const keywords = ['NULL', 'NOT NULL'];
+
+ let leftSideInput = ``;
+ if (leftField.type === 'Value') {
+ const dontAddQuotes =
+ (keywords.includes(leftField.value) && operator === 'IS') || operator === 'IN' || operator === 'NOT IN';
+
+ leftSideInput += dontAddQuotes ? leftField.value : this.addQuotesIfString(leftField.value);
+ } else {
+ if (leftField.function) leftSideInput += `${leftField.function}(`;
+ leftSideInput += `${leftField.table ? '"' + leftField.table + '"' + '.' : ''}${leftField.columnName}`;
+ if (leftField.function) leftSideInput += `)`;
+ }
+
+ let rightSideInput = ``;
+ if (rightField.type === 'Value') {
+ const dontAddQuotes =
+ (keywords.includes(rightField.value) && operator === 'IS') || operator === 'IN' || operator === 'NOT IN';
+
+ rightSideInput += dontAddQuotes ? rightField.value : this.addQuotesIfString(rightField.value);
+ } else {
+ if (rightField.function) rightSideInput += `${rightField.function}(`;
+ rightSideInput += `${rightField.table ? '"' + rightField.table + '"' + '.' : ''}${rightField.columnName}`;
+ if (rightField.function) rightSideInput += `)`;
+ }
+
+ return `${leftSideInput} ${operator} ${rightSideInput}`;
+ })
+ .join(` ${operator} `);
+ return whereConditionOutput;
+ }
+
+ private constructGroupByStatement(groupByInputList, _internalTableIdToNameMap) {
+ return groupByInputList
+ .map((groupByInput) => `${'"' + groupByInput.table + '"'}.${groupByInput.columnName}`)
+ .join(', ');
+ }
+
+ private constructOrderByStatement(orderByInputList, internalTableIdToNameMap) {
+ // @description: For "ORDER BY" statement table field is optional. But column_name & order_by direction is mandatory
+ return orderByInputList
+ .map((orderByInput) => {
+ const { columnName, direction } = orderByInput;
+ return `${orderByInput.table ? '"' + orderByInput.table + '"' + '.' : ''}${columnName} ${direction}`;
+ })
+ .join(`, `);
+ }
+
+ private async findOrFailInternalTableFromTableId(requestedTableIdList: Array, organizationId: string) {
+ const internalTables = await this.manager.find(InternalTable, {
+ where: {
+ organizationId,
+ id: In(requestedTableIdList),
+ },
+ });
+
+ const obtainedTableNames = internalTables.map((t) => t.id);
+ const tableNamesNotInOrg = requestedTableIdList.filter((tableId) => !obtainedTableNames.includes(tableId));
+
+ if (isEmpty(tableNamesNotInOrg)) return internalTables;
+
+ throw new NotFoundException('Some tables are not found');
+ }
}
diff --git a/server/src/services/tooljet_db_bulk_upload.service.ts b/server/src/services/tooljet_db_bulk_upload.service.ts
new file mode 100644
index 0000000000..74536e00e0
--- /dev/null
+++ b/server/src/services/tooljet_db_bulk_upload.service.ts
@@ -0,0 +1,200 @@
+import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common';
+import { EntityManager } from 'typeorm';
+import { InternalTable } from 'src/entities/internal_table.entity';
+import * as csv from 'fast-csv';
+import { SupportedDataTypes, TableColumnSchema, TooljetDbService } from './tooljet_db.service';
+import { InjectEntityManager } from '@nestjs/typeorm';
+import { isEmpty } from 'lodash';
+import { pipeline } from 'stream/promises';
+import { PassThrough } from 'stream';
+
+const MAX_ROW_COUNT = 1000;
+
+@Injectable()
+export class TooljetDbBulkUploadService {
+ constructor(
+ private readonly manager: EntityManager,
+ @InjectEntityManager('tooljetDb')
+ private readonly tooljetDbManager: EntityManager,
+ private readonly tooljetDbService: TooljetDbService
+ ) {}
+
+ async perform(organizationId: string, tableName: string, fileBuffer: Buffer) {
+ const internalTable = await this.manager.findOne(InternalTable, {
+ select: ['id'],
+ where: { organizationId, tableName },
+ });
+
+ if (!internalTable) {
+ throw new NotFoundException(`Table ${tableName} not found`);
+ }
+
+ const internalTableColumnSchema = await this.tooljetDbService.perform(organizationId, 'view_table', {
+ table_name: tableName,
+ });
+
+ return await this.bulkUploadCsv(internalTable.id, internalTableColumnSchema, fileBuffer);
+ }
+
+ async bulkUploadCsv(
+ internalTableId: string,
+ internalTableColumnSchema: TableColumnSchema[],
+ fileBuffer: Buffer
+ ): Promise<{ processedRows: number; rowsInserted: number; rowsUpdated: number }> {
+ const csvStream = csv.parseString(fileBuffer.toString(), {
+ headers: true,
+ ignoreEmpty: true,
+ strictColumnHandling: true,
+ discardUnmappedColumns: true,
+ });
+ const rowsToInsert = [];
+ const rowsToUpdate = [];
+ const idstoUpdate = new Set();
+ let rowsProcessed = 0;
+
+ const passThrough = new PassThrough();
+
+ csvStream
+ .on('headers', (headers) => this.validateHeadersAsColumnSubset(internalTableColumnSchema, headers, csvStream))
+ .transform((row) => this.validateAndParseColumnDataType(internalTableColumnSchema, row, rowsProcessed, csvStream))
+ .on('data', (row) => {
+ rowsProcessed++;
+ if (row.id) {
+ if (idstoUpdate.has(row.id)) {
+ throw new BadRequestException(`Duplicate 'id' value found on row[${rowsProcessed + 1}]: ${row.id}`);
+ }
+
+ idstoUpdate.add(row.id);
+ rowsToUpdate.push(row);
+ } else {
+ rowsToInsert.push(row);
+ }
+ })
+ .on('error', (error) => {
+ csvStream.destroy();
+ passThrough.emit('error', new BadRequestException(error));
+ })
+ .on('end', () => {
+ passThrough.emit('end');
+ });
+
+ await pipeline(passThrough, csvStream);
+
+ await this.tooljetDbManager.transaction(async (tooljetDbManager) => {
+ await this.bulkInsertRows(tooljetDbManager, rowsToInsert, internalTableId);
+ await this.bulkUpdateRows(tooljetDbManager, rowsToUpdate, internalTableId);
+ });
+
+ return { processedRows: rowsProcessed, rowsInserted: rowsToInsert.length, rowsUpdated: rowsToUpdate.length };
+ }
+
+ async bulkUpdateRows(tooljetDbManager: EntityManager, rowsToUpdate: unknown[], internalTableId: string) {
+ if (isEmpty(rowsToUpdate)) return;
+
+ const updateQueries = rowsToUpdate.map((row) => {
+ const columnNames = Object.keys(rowsToUpdate[0]);
+ const setClauses = columnNames
+ .map((column) => {
+ return `${column} = $${columnNames.indexOf(column) + 1}`;
+ })
+ .join(', ');
+
+ return {
+ text: `UPDATE "${internalTableId}" SET ${setClauses} WHERE id = $${columnNames.indexOf('id') + 1}`,
+ values: columnNames.map((column) => row[column]),
+ };
+ });
+
+ for (const updateQuery of updateQueries) {
+ await tooljetDbManager.query(updateQuery.text, updateQuery.values);
+ }
+ }
+
+ async bulkInsertRows(tooljetDbManager: EntityManager, rowsToInsert: unknown[], internalTableId: string) {
+ if (isEmpty(rowsToInsert)) return;
+
+ const insertQueries = rowsToInsert.map((row, index) => {
+ return {
+ text: `INSERT INTO "${internalTableId}" (${Object.keys(row).join(', ')}) VALUES (${Object.values(row).map(
+ (_, index) => `$${index + 1}`
+ )})`,
+ values: Object.values(row),
+ };
+ });
+
+ for (const insertQuery of insertQueries) {
+ await tooljetDbManager.query(insertQuery.text, insertQuery.values);
+ }
+ }
+
+ async validateHeadersAsColumnSubset(
+ internalTableColumnSchema: TableColumnSchema[],
+ headers: string[],
+ csvStream: csv.CsvParserStream, csv.ParserRow>
+ ) {
+ const internalTableColumns = new Set(internalTableColumnSchema.map((c) => c.column_name));
+ const columnsInCsv = new Set(headers);
+ const isSubset = (subset: Set, superset: Set) => [...subset].every((item) => superset.has(item));
+
+ if (!isSubset(columnsInCsv, internalTableColumns)) {
+ const columnsNotIntable = [...columnsInCsv].filter((element) => !internalTableColumns.has(element));
+
+ csvStream.emit('error', `Columns ${columnsNotIntable.join(',')} not found in table`);
+ }
+ }
+
+ validateAndParseColumnDataType(
+ internalTableColumnSchema: TableColumnSchema[],
+ row: unknown,
+ rowsProcessed: number,
+ csvStream: csv.CsvParserStream, csv.ParserRow>
+ ) {
+ if (rowsProcessed >= MAX_ROW_COUNT) csvStream.emit('error', `Row count cannot be greater than ${MAX_ROW_COUNT}`);
+
+ try {
+ const columnsInCsv = Object.keys(row);
+ const transformedRow = columnsInCsv.reduce((result, columnInCsv) => {
+ const columnDetails = internalTableColumnSchema.find((colDetails) => colDetails.column_name === columnInCsv);
+ const convertedValue = this.validateDataType(row[columnInCsv], columnDetails.data_type);
+
+ if (convertedValue) result[columnInCsv] = this.validateDataType(row[columnInCsv], columnDetails.data_type);
+
+ return result;
+ }, {});
+
+ return transformedRow;
+ } catch (error) {
+ csvStream.emit('error', `Data type error at row[${rowsProcessed + 1}]: ${error}`);
+ }
+ }
+
+ validateDataType(columnValue: string, supportedDataType: SupportedDataTypes) {
+ if (!columnValue) return null;
+
+ switch (supportedDataType) {
+ case 'boolean':
+ return this.validateBoolean(columnValue);
+ case 'integer':
+ case 'double precision':
+ case 'bigint':
+ return this.validateNumber(columnValue, supportedDataType);
+ default:
+ return columnValue;
+ }
+ }
+
+ validateBoolean(str: string) {
+ const parsedString = str.toLowerCase().trim();
+ if (parsedString === 'true' || parsedString === 'false') return str;
+
+ throw `${str} is not a valid boolean string`;
+ }
+
+ validateNumber(str: string, dataType: 'integer' | 'bigint' | 'double precision') {
+ if (dataType === 'integer' && !isNaN(parseInt(str, 10))) return str;
+ if (dataType === 'double precision' && !isNaN(parseFloat(str))) return str;
+ if (dataType === 'bigint' && typeof BigInt(str) === 'bigint') return str;
+
+ throw `${str} is not a valid ${dataType}`;
+ }
+}