[Feature] :: Global datasources (#5504)

* add: columns and migrations for data queries and sources

* add: migrations for app environments

* fix: datasources and queries api

* fix: import apis

* add: radixui colors

* create: global datasource page

* fix: version creation not including global datasources queries

* fix: version deletion failure

* fix: ui and other bugs

* add: check for abilities on global ds

* fix: bugs

* fix: existing test cases

* fix: migration and bugs

* fix: rest api oauthorize bugs

* hide: add button for local ds

* fix: query bugs

* fix: new organization environment creation

* fix: local ds label showing for new apps

* fix: on page load queries for preview app and published app

* fix: import bugs from v1

* fix: merge conflicts

* fix: import apis

* fix: apss with mulit envs

* fix: ui bugs

* fix: environments not being created on db:seed

* fix: ui bugs

* fix: route settings for global datasources

* fix: customer dashboard template

* fix: local ds queries not being saved

* fix: runpy issues

* changes: ui

* fix: migration issues

* fix: ui

* hide datasources when no local datasources

* fix: test cases

* fix: unit test cases and global queries on app import/export

* cleanup

* add: package-lock file

* undo: migration rename

* cleanup

* fix: ui bugs

* migration fixes

* fix: dark mode issues

* fix: change datasource failing on query create mode

* fix: workspace selector issues

* fix: clickoutside for change scope option

* migration changes

* fix: open api issue

* reverting configs changes

* [Fix] Global datasources & Environment Id issue (#5830)

* fix: oauth env id issue

* code changes

---------

Co-authored-by: gsmithun4 <gsmithun4@gmail.com>
Co-authored-by: Muhsin Shah <muhsinshah21@gmail.com>
This commit is contained in:
vjaris42 2023-03-24 21:41:21 +05:30 committed by GitHub
parent 4cd8839d44
commit bb9a211e55
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
74 changed files with 3594 additions and 1352 deletions

View file

@ -0,0 +1,4 @@
<svg width="40" height="41" viewBox="0 0 40 41" fill="none" xmlns="http://www.w3.org/2000/svg">
<path opacity="0.4" d="M28.1405 9.02475H23.2353C22.0134 9.03283 20.8532 8.48926 20.0789 7.54585L18.4636 5.31272C17.7022 4.36103 16.5421 3.81492 15.3219 3.83382H11.8543C5.63023 3.83382 3.33325 7.4867 3.33325 13.6981V20.4123C3.32553 21.1507 36.6593 21.1497 36.6615 20.4123V18.4601C36.6911 12.2488 34.4534 9.02475 28.1405 9.02475Z" fill="#3E63DD"/>
<path d="M28.1256 9.02476C30.3003 8.85658 32.4587 9.51124 34.1718 10.8586C34.3691 11.0259 34.5525 11.2089 34.7201 11.4058C35.2534 12.0293 35.6654 12.7464 35.9353 13.5206C36.4663 15.1118 36.7121 16.7838 36.6614 18.4601V27.2152C36.6593 27.9527 36.6048 28.689 36.4984 29.4188C36.2958 30.7067 35.8427 31.9425 35.1647 33.0569C34.8531 33.5951 34.4747 34.092 34.0385 34.5358C32.0638 36.3481 29.4415 37.2915 26.7623 37.1534H13.2176C10.5341 37.2905 7.90761 36.3475 5.9266 34.5358C5.49559 34.0911 5.1222 33.5943 4.81517 33.0569C4.14117 31.9433 3.69773 30.7063 3.51108 29.4188C3.3924 28.6902 3.33325 27.9533 3.33325 27.2152V18.4601C3.33325 17.7289 3.37254 16.9983 3.45181 16.2714C3.49626 15.9312 3.6 15.6059 3.6 15.2805C3.75043 14.4033 4.02485 13.5518 4.41505 12.7516C5.57094 10.2818 7.942 9.02476 11.8246 9.02476H28.1256ZM28.5258 23.6511H11.6171C10.856 23.6511 10.239 24.2668 10.239 25.0264C10.239 25.786 10.856 26.4018 11.6171 26.4018H28.422C28.7901 26.4178 29.1493 26.2861 29.4194 26.036C29.6894 25.7859 29.8479 25.4383 29.8595 25.0708C29.8803 24.7478 29.7739 24.4294 29.5631 24.1835C29.3205 23.853 28.9362 23.6557 28.5258 23.6511Z" fill="#3E63DD"/>
</svg>

After

Width:  |  Height:  |  Size: 1.6 KiB

View file

@ -0,0 +1,5 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect x="0.5" y="0.5" width="19" height="19" rx="3.5" fill="white"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M9 6.5C9 5.94772 9.44772 5.5 10 5.5C10.5523 5.5 11 5.94772 11 6.5C11 7.05228 10.5523 7.5 10 7.5C9.44772 7.5 9 7.05228 9 6.5ZM9 10C9 9.44772 9.44772 9 10 9C10.5523 9 11 9.44772 11 10C11 10.5523 10.5523 11 10 11C9.44772 11 9 10.5523 9 10ZM9 13.5C9 12.9477 9.44772 12.5 10 12.5C10.5523 12.5 11 12.9477 11 13.5C11 14.0523 10.5523 14.5 10 14.5C9.44772 14.5 9 14.0523 9 13.5Z" fill="#121212"/>
<rect x="0.5" y="0.5" width="19" height="19" rx="3.5" stroke="#D7DBDF"/>
</svg>

After

Width:  |  Height:  |  Size: 681 B

View file

@ -13,6 +13,7 @@
"@dnd-kit/utilities": "^3.2.1",
"@emoji-mart/data": "^1.1.2",
"@emoji-mart/react": "^1.1.1",
"@radix-ui/colors": "^0.1.8",
"@radix-ui/react-popover": "^1.0.3",
"@react-google-maps/api": "^2.18.1",
"@sentry/react": "^7.37.2",
@ -24436,6 +24437,11 @@
"url": "https://opencollective.com/popperjs"
}
},
"node_modules/@radix-ui/colors": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/@radix-ui/colors/-/colors-0.1.8.tgz",
"integrity": "sha512-jwRMXYwC0hUo0mv6wGpuw254Pd9p/R6Td5xsRpOmaWkUHlooNWqVcadgyzlRumMq3xfOTXwJReU0Jv+EIy4Jbw=="
},
"node_modules/@radix-ui/primitive": {
"version": "1.0.0",
"license": "MIT",
@ -49935,6 +49941,11 @@
"@popperjs/core": {
"version": "2.11.6"
},
"@radix-ui/colors": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/@radix-ui/colors/-/colors-0.1.8.tgz",
"integrity": "sha512-jwRMXYwC0hUo0mv6wGpuw254Pd9p/R6Td5xsRpOmaWkUHlooNWqVcadgyzlRumMq3xfOTXwJReU0Jv+EIy4Jbw=="
},
"@radix-ui/primitive": {
"version": "1.0.0",
"requires": {

View file

@ -8,6 +8,7 @@
"@dnd-kit/utilities": "^3.2.1",
"@emoji-mart/data": "^1.1.2",
"@emoji-mart/react": "^1.1.1",
"@radix-ui/colors": "^0.1.8",
"@radix-ui/react-popover": "^1.0.3",
"@react-google-maps/api": "^2.18.1",
"@sentry/react": "^7.37.2",
@ -206,4 +207,4 @@
"jsx"
]
}
}
}

View file

@ -18,6 +18,7 @@ import { SettingsPage } from '../SettingsPage/SettingsPage';
import { ForgotPassword } from '@/ForgotPassword';
import { ResetPassword } from '@/ResetPassword';
import { MarketplacePage } from '@/MarketplacePage';
import { GlobalDatasources } from '@/GlobalDatasources';
import { lt } from 'semver';
import Toast from '@/_ui/Toast';
import { VerificationSuccessInfoScreen } from '@/SuccessInfoScreen';
@ -71,7 +72,7 @@ class AppComponent extends React.Component {
};
render() {
const { updateAvailable, darkMode } = this.state;
const { updateAvailable, darkMode, currentUser } = this.state;
let toastOptions = {
style: {
wordBreak: 'break-all',
@ -92,7 +93,7 @@ class AppComponent extends React.Component {
return (
<>
<div className={`main-wrapper ${darkMode ? 'theme-dark' : ''}`} data-cy="main-wrapper">
<div className={`main-wrapper ${darkMode ? 'theme-dark dark-theme' : ''}`} data-cy="main-wrapper">
{updateAvailable && (
<div className="alert alert-info alert-dismissible" role="alert">
<h3 className="mb-1">Update available</h3>
@ -216,6 +217,15 @@ class AppComponent extends React.Component {
}
/>
)}
<Route
exact
path="/global-datasources"
element={
<PrivateRoute>
<GlobalDatasources switchDarkMode={this.switchDarkMode} darkMode={darkMode} />
</PrivateRoute>
}
/>
{window.public_config?.ENABLE_MARKETPLACE_FEATURE === 'true' && (
<Route
exact

View file

@ -1,5 +1,5 @@
import React from 'react';
import { datasourceService, authenticationService, pluginsService } from '@/_services';
import { datasourceService, authenticationService, pluginsService, globalDatasourceService } from '@/_services';
import { Modal, Button, Tab, Row, Col, ListGroup } from 'react-bootstrap';
import { toast } from 'react-hot-toast';
import { getSvgIcon } from '@/_helpers/appUtils';
@ -53,6 +53,9 @@ class DataSourceManagerComponent extends React.Component {
filteredDatasources: [],
activeDatasourceList: '#alldatasources',
suggestingDatasources: false,
scope: props?.scope,
modalProps: props?.modalProps ?? {},
showBackButton: props?.showBackButton ?? true,
};
}
@ -155,6 +158,7 @@ class DataSourceManagerComponent extends React.Component {
const kind = selectedDataSource.kind;
const pluginId = selectedDataSourcePluginId;
const appVersionId = this.props.editingVersionId;
const scope = this.state?.scope || selectedDataSource?.scope;
const parsedOptions = Object.keys(options).map((key) => {
const keyMeta = selectedDataSource.options[key];
@ -166,28 +170,45 @@ class DataSourceManagerComponent extends React.Component {
};
});
if (name.trim() !== '') {
let service = scope === 'global' ? globalDatasourceService : datasourceService;
if (selectedDataSource.id) {
this.setState({ isSaving: true });
datasourceService.save(selectedDataSource.id, appId, name, parsedOptions).then(() => {
this.setState({ isSaving: false });
this.hideModal();
toast.success(
this.props.t('editor.queryManager.dataSourceManager.toast.success.dataSourceSaved', 'Datasource Saved'),
{ position: 'top-center' }
);
this.props.dataSourcesChanged();
});
service
.save(selectedDataSource.id, name, parsedOptions, appId)
.then(() => {
this.setState({ isSaving: false });
this.hideModal();
toast.success(
this.props.t('editor.queryManager.dataSourceManager.toast.success.dataSourceSaved', 'Datasource Saved'),
{ position: 'top-center' }
);
this.props.dataSourcesChanged();
this.props.globalDataSourcesChanged();
})
.catch(({ error }) => {
this.setState({ isSaving: false });
this.hideModal();
error && toast.error(error, { position: 'top-center' });
});
} else {
this.setState({ isSaving: true });
datasourceService.create(appId, appVersionId, pluginId, name, kind, parsedOptions).then(() => {
this.setState({ isSaving: false });
this.hideModal();
toast.success(
this.props.t('editor.queryManager.dataSourceManager.toast.success.dataSourceAdded', 'Datasource Added'),
{ position: 'top-center' }
);
this.props.dataSourcesChanged();
});
service
.create(pluginId, name, kind, parsedOptions, appId, appVersionId, scope)
.then(() => {
this.setState({ isSaving: false });
this.hideModal();
toast.success(
this.props.t('editor.queryManager.dataSourceManager.toast.success.dataSourceAdded', 'Datasource Added'),
{ position: 'top-center' }
);
this.props.dataSourcesChanged();
this.props.globalDataSourcesChanged();
})
.catch(({ error }) => {
this.setState({ isSaving: false });
this.hideModal();
error && toast.error(error, { position: 'top-center' });
});
}
} else {
toast.error(
@ -313,6 +334,7 @@ class DataSourceManagerComponent extends React.Component {
onClear={this.handleBackToAllDatasources}
queryString={this.state.queryString}
activeDatasourceList={this.state.activeDatasourceList}
scope={this.state.scope}
/>
</div>
{datasources.map((datasource) => (
@ -600,9 +622,11 @@ class DataSourceManagerComponent extends React.Component {
contentClassName={this.props.darkMode ? 'theme-dark' : ''}
animation={false}
onExit={this.onExit}
container={this.props.container}
{...this.props.modalProps}
>
<Modal.Header className="justify-content-start">
{selectedDataSource && (
{selectedDataSource && this.props.showBackButton && (
<div
className={`back-btn me-3 ${this.props.darkMode ? 'dark' : ''}`}
role="button"
@ -869,7 +893,7 @@ const EmptyStateContainer = ({
);
};
const SearchBoxContainer = ({ onChange, onClear, queryString, activeDatasourceList, dataCy }) => {
const SearchBoxContainer = ({ onChange, onClear, queryString, activeDatasourceList, dataCy, scope }) => {
const [searchText, setSearchText] = React.useState(queryString ?? '');
const { t } = useTranslation();
const handleChange = (e) => {
@ -900,12 +924,18 @@ const SearchBoxContainer = ({ onChange, onClear, queryString, activeDatasourceLi
if (searchText === '') {
onClear();
}
let element = document.querySelector('.input-icon .form-control:not(:first-child)');
if (scope === 'global') {
element = document.querySelector('.input-icon .form-control');
}
if (searchText) {
document.querySelector('.input-icon .form-control:not(:first-child)').style.paddingLeft = '0.5rem';
element.style.paddingLeft = '0.5rem';
}
return () => {
document.querySelector('.input-icon .form-control:not(:first-child)').style.paddingLeft = '2.5rem';
element.style.paddingLeft = '2.5rem';
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [searchText]);

View file

@ -6,6 +6,7 @@ import {
authenticationService,
appVersionService,
orgEnvironmentVariableService,
globalDatasourceService,
} from '@/_services';
import { DndProvider } from 'react-dnd';
import { HTML5Backend } from 'react-dnd-html5-backend';
@ -122,6 +123,7 @@ class EditorComponent extends React.Component {
appId,
editingVersion: null,
loadingDataSources: true,
loadingGlobalDataSources: true,
loadingDataQueries: true,
showLeftSidebar: true,
showComments: false,
@ -290,6 +292,23 @@ class EditorComponent extends React.Component {
);
};
fetchGlobalDataSources = () => {
this.setState(
{
loadingGlobalDataSources: true,
},
() => {
const { organization_id: organizationId } = this.state.currentUser;
globalDatasourceService.getAll(organizationId).then((data) =>
this.setState({
globalDataSources: data.data_sources,
loadingGlobalDataSources: false,
})
);
}
);
};
fetchDataQueries = () => {
this.setState(
{
@ -447,6 +466,7 @@ class EditorComponent extends React.Component {
this.fetchDataSources();
this.fetchDataQueries();
this.fetchGlobalDataSources();
initEditorWalkThrough();
};
@ -493,6 +513,10 @@ class EditorComponent extends React.Component {
}
};
globalDataSourcesChanged = () => {
this.fetchGlobalDataSources();
};
/**
* https://developer.mozilla.org/en-US/docs/Web/API/WebSocket/readyState
*/
@ -1734,6 +1758,7 @@ class EditorComponent extends React.Component {
appId,
slug,
dataSources,
globalDataSources = [],
loadingDataQueries,
dataQueries,
loadingDataSources,
@ -1831,8 +1856,10 @@ class EditorComponent extends React.Component {
appId={appId}
darkMode={this.props.darkMode}
dataSources={this.state.dataSources}
globalDataSources={globalDataSources}
dataSourcesChanged={this.dataSourcesChanged}
dataQueriesChanged={this.dataQueriesChanged}
globalDataSourcesChanged={this.globalDataSourcesChanged}
onZoomChanged={this.onZoomChanged}
toggleComments={this.toggleComments}
switchDarkMode={this.changeDarkMode}
@ -2096,6 +2123,7 @@ class EditorComponent extends React.Component {
}
toggleQueryEditor={toggleQueryEditor}
dataSources={dataSources}
globalDataSources={globalDataSources}
dataQueries={dataQueries}
mode={editingQuery ? 'edit' : 'create'}
selectedQuery={selectedQuery}

View file

@ -1,17 +1,20 @@
/* eslint-disable import/no-named-as-default */
import React from 'react';
import { Link } from 'react-router-dom';
import { LeftSidebarItem } from './SidebarItem';
import { Button, HeaderSection } from '@/_ui/LeftSidebar';
import { HeaderSection } from '@/_ui/LeftSidebar';
import { DataSourceManager } from '../DataSourceManager';
import { DataSourceTypes } from '../DataSourceManager/SourceComponents';
import { getSvgIcon } from '@/_helpers/appUtils';
import { datasourceService } from '@/_services';
import { datasourceService, globalDatasourceService, authenticationService } from '@/_services';
import { ConfirmDialog } from '@/_components';
import toast from 'react-hot-toast';
import { useTranslation } from 'react-i18next';
import Popover from '@/_ui/Popover';
import { Popover as PopoverBS, OverlayTrigger } from 'react-bootstrap';
// eslint-disable-next-line import/no-unresolved
import TrashIcon from '@assets/images/icons/query-trash-icon.svg';
import VerticalIcon from '@assets/images/icons/vertical-menu.svg';
export const LeftSidebarDataSources = ({
appId,
@ -20,7 +23,9 @@ export const LeftSidebarDataSources = ({
setSelectedSidebarItem,
darkMode,
dataSources = [],
globalDataSources = [],
dataSourcesChanged,
globalDataSourcesChanged,
dataQueriesChanged,
toggleDataSourceManagerModal,
showDataSourceManagerModal,
@ -30,6 +35,8 @@ export const LeftSidebarDataSources = ({
const [isDeleteModalVisible, setDeleteModalVisibility] = React.useState(false);
const [isDeletingDatasource, setDeletingDatasource] = React.useState(false);
const { admin } = authenticationService.currentUserValue;
const deleteDataSource = (selectedSource) => {
setSelectedDataSource(selectedSource);
setDeleteModalVisibility(true);
@ -45,6 +52,7 @@ export const LeftSidebarDataSources = ({
setDeletingDatasource(false);
setSelectedDataSource(null);
dataSourcesChanged();
globalDataSourcesChanged();
dataQueriesChanged();
})
.catch(({ error }) => {
@ -59,6 +67,20 @@ export const LeftSidebarDataSources = ({
setSelectedDataSource(null);
};
const changeScope = (dataSource) => {
globalDatasourceService
.convertToGlobal(dataSource.id)
.then(() => {
dataSourcesChanged();
globalDataSourcesChanged();
toast.success('Data Source scope changed');
})
.catch(({ error }) => {
setSelectedDataSource(null);
toast.error(error);
});
};
const getSourceMetaData = (dataSource) => {
if (dataSource.pluginId) {
const srcMeta = dataSource.plugin?.manifestFile?.data.source || undefined;
@ -69,19 +91,55 @@ export const LeftSidebarDataSources = ({
return DataSourceTypes.find((source) => source.kind === dataSource.kind);
};
const renderDataSource = (dataSource, idx) => {
const RenderDataSource = ({ dataSource, idx, convertToGlobal, showDeleteIcon = true, enableEdit = true }) => {
const [isConversionVisible, setConversionVisible] = React.useState(false);
const sourceMeta = getSourceMetaData(dataSource);
const icon = getSvgIcon(sourceMeta?.kind?.toLowerCase(), 24, 24, dataSource?.plugin?.iconFile?.data);
const convertToGlobalDataSource = (dataSource) => {
setConversionVisible(false);
changeScope(dataSource);
};
React.useEffect(() => {
const handleClickOutside = (event) => {
if (isConversionVisible && event.target.closest('.popover-change-scope') === null) {
setConversionVisible(false);
}
};
document.addEventListener('mousedown', handleClickOutside);
return () => {
document.removeEventListener('mousedown', handleClickOutside);
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [JSON.stringify({ dataSource, isConversionVisible })]);
const popover = (
<PopoverBS key={dataSource.id} id="popover-change-scope">
<PopoverBS.Body key={dataSource.id} className={`${darkMode && 'theme-dark popover-dark-themed'}`}>
<div className={`row cursor-pointer`}>
<div className="col text-truncate cursor-pointer" onClick={() => convertToGlobalDataSource(dataSource)}>
Change scope
</div>
</div>
</PopoverBS.Body>
</PopoverBS>
);
return (
<div className="row mb-3 ds-list-item" key={idx}>
<div
role="button"
onClick={() => {
setSelectedDataSource(dataSource);
toggleDataSourceManagerModal(true);
}}
onClick={
enableEdit
? () => {
setSelectedDataSource(dataSource);
toggleDataSourceManagerModal(true);
}
: null
}
className="col d-flex align-items-center"
>
{icon}
@ -89,13 +147,30 @@ export const LeftSidebarDataSources = ({
{dataSource.name}
</span>
</div>
<div className="col-auto">
<button className="btn btn-sm p-1 ds-delete-btn" onClick={() => deleteDataSource(dataSource)}>
<div>
<TrashIcon width="14" height="14" />
</div>
</button>
</div>
{showDeleteIcon && (
<div className="col-auto">
<button className="btn btn-sm p-1 ds-delete-btn" onClick={() => deleteDataSource(dataSource)}>
<div>
<TrashIcon width="14" height="14" />
</div>
</button>
</div>
)}
{convertToGlobal && admin && (
<div className="col-auto">
<OverlayTrigger
rootClose={false}
show={isConversionVisible}
trigger="click"
placement="bottom"
overlay={popover}
>
<div onClick={() => setConversionVisible(!isConversionVisible)}>
<VerticalIcon />
</div>
</OverlayTrigger>
</div>
)}
</div>
);
};
@ -103,8 +178,9 @@ export const LeftSidebarDataSources = ({
const popoverContent = (
<LeftSidebarDataSources.Container
darkMode={darkMode}
renderDataSource={renderDataSource}
RenderDataSource={RenderDataSource}
dataSources={dataSources}
globalDataSources={globalDataSources}
toggleDataSourceManagerModal={toggleDataSourceManagerModal}
/>
);
@ -147,52 +223,49 @@ export const LeftSidebarDataSources = ({
}}
editingVersionId={editingVersionId}
dataSourcesChanged={dataSourcesChanged}
globalDataSourcesChanged={globalDataSourcesChanged}
selectedDataSource={selectedDataSource}
/>
</>
);
};
const LeftSidebarDataSourcesContainer = ({
darkMode,
renderDataSource,
dataSources = [],
toggleDataSourceManagerModal,
}) => {
const LeftSidebarDataSourcesContainer = ({ darkMode, RenderDataSource, dataSources = [], globalDataSources = [] }) => {
const { t } = useTranslation();
return (
<div>
<HeaderSection darkMode={darkMode}>
<HeaderSection.PanelHeader title="Datasources">
<div className="d-flex justify-content-end float-right" style={{ maxWidth: 48 }}>
<Button
styles={{ width: '28px', padding: 0 }}
onClick={() => toggleDataSourceManagerModal(true)}
darkMode={darkMode}
size="sm"
>
<Button.Content iconSrc={'assets/images/icons/plus.svg'} direction="left" />
</Button>
</div>
</HeaderSection.PanelHeader>
<HeaderSection.PanelHeader title="Datasources"></HeaderSection.PanelHeader>
</HeaderSection>
<div className="card-body pb-5">
<div className="d-flex w-100">
{dataSources.length === 0 ? (
<center
onClick={() => toggleDataSourceManagerModal(true)}
className="p-2 color-primary cursor-pointer"
data-cy="add-datasource-link"
>
{t(`leftSidebar.Sources.addDataSource`, '+ add data source')}
</center>
) : (
<div className="mt-2 w-100" data-cy="datasource-Label">
{dataSources?.map((source, idx) => renderDataSource(source, idx))}
</div>
)}
<div className="d-flex w-100 flex-column align-items-start">
<div className="d-flex flex-column w-100">
{dataSources.length ? (
<>
<div className="tj-text-sm my-2 datasources-category">Local Datasources</div>
<div className="mt-2 w-100" data-cy="datasource-Label">
{dataSources?.map((source, idx) => (
<RenderDataSource
key={idx}
dataSource={source}
idx={idx}
convertToGlobal={true}
showDeleteIcon={true}
/>
))}
</div>
</>
) : null}
</div>
</div>
</div>
<div className="add-datasource-btn w-100 p-3">
<Link to="/global-datasources">
<div className="p-2 color-primary cursor-pointer">
{t(`leftSidebar.Sources.addDataSource`, '+ add data source')}
</div>
</Link>
</div>
</div>
);
};

View file

@ -21,7 +21,9 @@ export const LeftSidebar = forwardRef((props, ref) => {
components,
toggleComments,
dataSources = [],
globalDataSources = [],
dataSourcesChanged,
globalDataSourcesChanged,
dataQueriesChanged,
errorLogs,
appVersionsId,
@ -110,19 +112,23 @@ export const LeftSidebar = forwardRef((props, ref) => {
dataSources={dataSources}
popoverContentHeight={popoverContentHeight}
/>
<LeftSidebarDataSources
darkMode={darkMode}
selectedSidebarItem={selectedSidebarItem}
setSelectedSidebarItem={handleSelectedSidebarItem}
appId={appId}
editingVersionId={appVersionsId}
dataSources={dataSources}
dataSourcesChanged={dataSourcesChanged}
dataQueriesChanged={dataQueriesChanged}
toggleDataSourceManagerModal={toggleDataSourceManagerModal}
showDataSourceManagerModal={showDataSourceManagerModal}
popoverContentHeight={popoverContentHeight}
/>
{dataSources?.length > 0 && (
<LeftSidebarDataSources
darkMode={darkMode}
selectedSidebarItem={selectedSidebarItem}
setSelectedSidebarItem={handleSelectedSidebarItem}
appId={appId}
editingVersionId={appVersionsId}
dataSources={dataSources}
globalDataSources={globalDataSources}
dataSourcesChanged={dataSourcesChanged}
globalDataSourcesChanged={globalDataSourcesChanged}
dataQueriesChanged={dataQueriesChanged}
toggleDataSourceManagerModal={toggleDataSourceManagerModal}
showDataSourceManagerModal={showDataSourceManagerModal}
popoverContentHeight={popoverContentHeight}
/>
)}
{config.COMMENT_FEATURE_ENABLE && (
<LeftSidebarComment
appVersionsId={appVersionsId}

View file

@ -0,0 +1,19 @@
import React from 'react';
import Select from '@/_ui/Select';
export const ChangeDataSource = ({ dataSources, onChange, value, selectedQuery }) => {
return (
<Select
className="px-4"
options={dataSources
.filter((ds) => ds.kind === selectedQuery?.kind)
.map((ds) => ({ label: ds.name, value: ds.id }))}
value={value.id}
onChange={(value) => {
const dataSource = dataSources.find((ds) => ds.id === value);
onChange(dataSource);
}}
useMenuPortal={true}
/>
);
};

View file

@ -13,6 +13,7 @@ function DataSourceLister({
handleBackButton,
darkMode,
dataSourceModalHandler,
showAddDatasourceBtn = true,
}) {
const [allSources, setAllSources] = useState([...dataSources, ...staticDataSources]);
const { t } = useTranslation();
@ -65,10 +66,12 @@ function DataSourceLister({
</div>
);
})}
<div className="query-datasource-card" style={computedStyles} onClick={dataSourceModalHandler}>
<AddIcon style={{ height: 25, width: 25, marginTop: '-3px' }} />
<p>{t('editor.queryManager.addDatasource', 'Add datasource')}</p>
</div>
{showAddDatasourceBtn && (
<div className="query-datasource-card" style={computedStyles} onClick={dataSourceModalHandler}>
<AddIcon style={{ height: 25, width: 25, marginTop: '-3px' }} />
<p>{t('editor.queryManager.addDatasource', 'Add datasource')}</p>
</div>
)}
</div>
);
}

View file

@ -18,6 +18,7 @@ import cx from 'classnames';
// eslint-disable-next-line import/no-unresolved
import { diff } from 'deep-object-diff';
import { CustomToggleSwitch } from './CustomToggleSwitch';
import { ChangeDataSource } from './ChangeDataSource';
const queryNameRegex = new RegExp('^[A-Za-z0-9_-]*$');
@ -58,7 +59,9 @@ class QueryManagerComponent extends React.Component {
const selectedQuery = props.selectedQuery;
const dataSourceId = selectedQuery?.data_source_id;
const source = props.dataSources.find((datasource) => datasource.id === dataSourceId);
const source = [...props.dataSources, ...props.globalDataSources].find(
(datasource) => datasource.id === dataSourceId
);
const selectedDataSource =
paneHeightChanged || queryPaneDragged ? this.state.selectedDataSource : props.selectedDataSource;
const dataSourceMeta = selectedQuery?.pluginId
@ -73,6 +76,7 @@ class QueryManagerComponent extends React.Component {
{
appId: props.appId,
dataSources: props.dataSources,
globalDataSources: props.globalDataSources,
dataQueries: dataQueries,
appDefinition: props.appDefinition,
mode: props.mode,
@ -117,7 +121,9 @@ class QueryManagerComponent extends React.Component {
shouldRunQuery: props.mode === 'edit' ? this.state.isFieldsChanged : this.props.isSourceSelected,
},
() => {
let source = props.dataSources.find((datasource) => datasource.id === selectedQuery?.data_source_id);
let source = [...props.dataSources, ...props.globalDataSources].find(
(datasource) => datasource.id === selectedQuery?.data_source_id
);
if (selectedQuery?.kind === 'restapi') {
if (!selectedQuery.data_source_id) {
source = { kind: 'restapi', id: 'null', name: 'REST API' };
@ -491,9 +497,32 @@ class QueryManagerComponent extends React.Component {
}
};
changeDataSourceQueryAssociation = (selectedDataSource, selectedQuery) => {
this.setState({
selectedDataSource: selectedDataSource,
isUpdating: true,
});
dataqueryService
.changeQueryDataSource(selectedQuery?.id, selectedDataSource.id)
.then(() => {
this.props.dataQueriesChanged();
this.setState({
isUpdating: false,
});
toast.success('Data source changed');
})
.catch((error) => {
toast.error(error);
this.setState({
isUpdating: false,
});
});
};
render() {
const {
dataSources,
globalDataSources,
selectedDataSource,
mode,
options,
@ -733,12 +762,30 @@ class QueryManagerComponent extends React.Component {
changeDataSource={this.changeDataSource}
handleBackButton={this.handleBackButton}
darkMode={this.props.darkMode}
showAddDatasourceBtn={false}
dataSourceModalHandler={this.props.dataSourceModalHandler}
/>
)}
</div>
)}
{dataSources && mode === 'create' && !this.state.isSourceSelected && (
<div className="datasource-picker">
{!this.state.isSourceSelected && <label className="form-label col-md-3">Global Datasources</label>}{' '}
{!this.state.isSourceSelected && (
<DataSourceLister
dataSources={globalDataSources}
staticDataSources={[]}
changeDataSource={this.changeDataSource}
handleBackButton={this.handleBackButton}
darkMode={this.props.darkMode}
dataSourceModalHandler={this.props.dataSourceModalHandler}
showAddDatasourceBtn={false}
/>
)}
</div>
)}
{selectedDataSource && (
<div style={{ padding: '0 32px' }}>
<div>
@ -887,6 +934,25 @@ class QueryManagerComponent extends React.Component {
}
/>
</div>
{mode === 'edit' && (
<div className="mt-2 pb-4">
<div
className={`border-top query-manager-border-color px-4 hr-text-left py-2 ${
this.props.darkMode ? 'color-white' : 'color-light-slate-12'
}`}
>
Change Datasource
</div>
<ChangeDataSource
dataSources={[...globalDataSources, ...this.props.dataSources]}
value={selectedDataSource}
selectedQuery={selectedQuery}
onChange={(selectedDataSource) => {
this.changeDataSourceQueryAssociation(selectedDataSource, selectedQuery);
}}
/>
</div>
)}
</div>
)}
</div>

View file

@ -0,0 +1,22 @@
import React, { useContext } from 'react';
import { GlobalDataSourcesContext } from '../index';
import { List } from '../List';
export const CreateDataSourceModal = () => {
const { handleModalVisibility } = useContext(GlobalDataSourcesContext);
return (
<div className="col border-end">
<div className="p-3">
<button
className="add-datasource-btn btn btn-primary active w-100"
type="button"
onClick={handleModalVisibility}
>
Add new datasource
</button>
</div>
<List />
</div>
);
};

View file

@ -0,0 +1,94 @@
import React, { useContext, useRef, useState, useEffect } from 'react';
import cx from 'classnames';
import { Sidebar } from '../Sidebar';
import { GlobalDataSourcesContext } from '..';
import { DataSourceManager } from '../../Editor/DataSourceManager';
import DataSourceFolder from '@assets/images/icons/datasource-folder.svg';
export const GlobalDataSourcesPage = ({ darkMode }) => {
const containerRef = useRef(null);
const [modalProps, setModalProps] = useState({
backdrop: true,
dialogClassName: 'datasource-edit-modal',
enforceFocus: false,
});
const {
dataSources,
setSelectedDataSource,
selectedDataSource,
fetchDataSources,
showDataSourceManagerModal,
toggleDataSourceManagerModal,
handleModalVisibility,
isEditing,
setEditing,
} = useContext(GlobalDataSourcesContext);
useEffect(() => {
if (selectedDataSource) {
setModalProps({ ...modalProps, backdrop: false });
} else {
setModalProps({ ...modalProps, backdrop: true });
}
}, [selectedDataSource]);
const handleHideModal = () => {
if (dataSources?.length) {
if (!isEditing) {
setEditing(true);
setSelectedDataSource(dataSources[0]);
} else {
setSelectedDataSource(null);
setEditing(true);
toggleDataSourceManagerModal(false);
}
} else {
handleModalVisibility();
}
};
return (
<div className="row gx-0">
<Sidebar />
<div
ref={containerRef}
className={cx('col animation-fade datasource-modal-container', {
'bg-light-gray': !darkMode,
})}
>
{containerRef && containerRef?.current && (
<DataSourceManager
showBackButton={selectedDataSource ? false : true}
showDataSourceManagerModal={showDataSourceManagerModal}
darkMode={darkMode}
hideModal={handleHideModal}
scope="global"
dataSourcesChanged={fetchDataSources}
selectedDataSource={selectedDataSource}
modalProps={modalProps}
container={selectedDataSource ? containerRef?.current : null}
/>
)}
{!selectedDataSource && isEditing && (
<div className="main-empty-container">
<div className="icon-container">
<DataSourceFolder />
</div>
<div className="heading tj-text-lg mt-2">Datasource 101</div>
<div className="sub-heading text-secondary tj-text-md mt-2">
Connect your app with REST API, PGSQL, MongoDB, Stripe and 40+ other datasources
</div>
<button
className="add-datasource-btn btn btn-primary active w-100 mt-3"
type="button"
onClick={handleModalVisibility}
>
Add new datasource
</button>
</div>
)}
</div>
</div>
);
};

View file

@ -0,0 +1,6 @@
<svg width="13" height="14" viewBox="0 0 13 14" fill="none" xmlns="http://www.w3.org/2000/svg">
<path
d="M2.61247 13.2459C2.25691 13.2459 1.94858 13.1154 1.68747 12.8543C1.42636 12.5932 1.2958 12.2848 1.2958 11.9293V2.59595H1.11247C0.923579 2.59595 0.765245 2.53206 0.637467 2.40428C0.50969 2.2765 0.445801 2.11817 0.445801 1.92928C0.445801 1.74039 0.50969 1.58206 0.637467 1.45428C0.765245 1.3265 0.923579 1.26261 1.11247 1.26261H3.9958C3.9958 1.07373 4.05969 0.915392 4.18747 0.787614C4.31524 0.659836 4.47358 0.595947 4.66247 0.595947H8.02913C8.21802 0.595947 8.37913 0.662614 8.51247 0.795947C8.6458 0.929281 8.71247 1.08484 8.71247 1.26261H11.5791C11.768 1.26261 11.9264 1.3265 12.0541 1.45428C12.1819 1.58206 12.2458 1.74039 12.2458 1.92928C12.2458 2.11817 12.1819 2.2765 12.0541 2.40428C11.9264 2.53206 11.768 2.59595 11.5791 2.59595H11.3958V11.9293C11.3958 12.2848 11.2652 12.5932 11.0041 12.8543C10.743 13.1154 10.4347 13.2459 10.0791 13.2459H2.61247ZM2.61247 2.59595V11.9293H10.0791V2.59595H2.61247ZM4.3458 9.99595C4.3458 10.1515 4.40136 10.2848 4.51247 10.3959C4.62358 10.5071 4.75691 10.5626 4.91247 10.5626C5.07913 10.5626 5.21802 10.5071 5.32913 10.3959C5.44025 10.2848 5.4958 10.1515 5.4958 9.99595V4.51261C5.4958 4.34595 5.43747 4.20428 5.3208 4.08761C5.20413 3.97095 5.06802 3.91261 4.91247 3.91261C4.7458 3.91261 4.60969 3.97095 4.50413 4.08761C4.39858 4.20428 4.3458 4.34595 4.3458 4.51261V9.99595ZM7.1958 9.99595C7.1958 10.1515 7.25413 10.2848 7.3708 10.3959C7.48747 10.5071 7.62358 10.5626 7.77913 10.5626C7.9458 10.5626 8.08469 10.5071 8.1958 10.3959C8.30691 10.2848 8.36247 10.1515 8.36247 9.99595V4.51261C8.36247 4.34595 8.30413 4.20428 8.18747 4.08761C8.0708 3.97095 7.93469 3.91261 7.77913 3.91261C7.61247 3.91261 7.47358 3.97095 7.36247 4.08761C7.25136 4.20428 7.1958 4.34595 7.1958 4.51261V9.99595ZM2.61247 2.59595V11.9293V2.59595Z"
fill="#EB1414"
/>
</svg>

After

Width:  |  Height:  |  Size: 1.9 KiB

View file

@ -0,0 +1,57 @@
import React, { useContext } from 'react';
import cx from 'classnames';
import { GlobalDataSourcesContext } from '..';
import { DataSourceTypes } from '../../Editor/DataSourceManager/SourceComponents';
import { getSvgIcon } from '@/_helpers/appUtils';
import DeleteIcon from '../Icons/DeleteIcon.svg';
export const ListItem = ({ dataSource, key, active, onDelete }) => {
const { setSelectedDataSource, toggleDataSourceManagerModal } = useContext(GlobalDataSourcesContext);
const getSourceMetaData = (dataSource) => {
if (dataSource.plugin_id) {
return dataSource.plugin?.manifest_file?.data.source;
}
return DataSourceTypes.find((source) => source.kind === dataSource.kind);
};
const sourceMeta = getSourceMetaData(dataSource);
const icon = getSvgIcon(sourceMeta.kind.toLowerCase(), 24, 24, dataSource?.plugin?.icon_file?.data);
const focusModal = () => {
const element = document.getElementsByClassName('form-control-plaintext form-control-plaintext-sm')[0];
element.focus();
};
return (
<div
key={key}
className={cx('tj-text-sm mx-3 p-2 rounded-3 mb-2 datasources-list', {
'datasources-list-item': active,
})}
>
<div
role="button"
onClick={() => {
setSelectedDataSource(dataSource);
toggleDataSourceManagerModal(true);
focusModal();
}}
className="col d-flex align-items-center"
>
{icon}
<span className="font-400" style={{ paddingLeft: 5 }}>
{dataSource.name}
</span>
</div>
<div className="col-auto">
<button className="btn btn-sm ds-delete-btn" onClick={() => onDelete(dataSource)}>
<div>
<DeleteIcon width="14" height="14" />
</div>
</button>
</div>
</div>
);
};

View file

@ -0,0 +1,108 @@
import React, { useContext, useEffect, useState } from 'react';
import { toast } from 'react-hot-toast';
import { GlobalDataSourcesContext } from '..';
import Skeleton from 'react-loading-skeleton';
import { ListItem } from '../LIstItem';
import { ConfirmDialog } from '@/_components';
import { globalDatasourceService } from '@/_services';
import EmptyFoldersIllustration from '@assets/images/icons/no-queries-added.svg';
export const List = () => {
const { dataSources, fetchDataSources, selectedDataSource, setSelectedDataSource, toggleDataSourceManagerModal } =
useContext(GlobalDataSourcesContext);
const [loading, setLoading] = useState(true);
const [isDeletingDatasource, setDeletingDatasource] = useState(false);
const [isDeleteModalVisible, setDeleteModalVisibility] = React.useState(false);
const darkMode = localStorage.getItem('darkMode') === 'true';
useEffect(() => {
fetchDataSources(true)
.then(() => {
setLoading(false);
})
.catch((e) => {
setLoading(false);
toast.error('Failed to fetch datasources');
return;
});
}, []);
const deleteDataSource = (selectedSource) => {
toggleDataSourceManagerModal(false);
setSelectedDataSource(selectedSource);
setDeleteModalVisibility(true);
};
const executeDataSourceDeletion = () => {
setDeleteModalVisibility(false);
setDeletingDatasource(true);
globalDatasourceService
.deleteDataSource(selectedDataSource.id)
.then(() => {
toast.success('Data Source Deleted');
setDeletingDatasource(false);
setSelectedDataSource(null);
fetchDataSources(true);
})
.catch(({ error }) => {
setDeletingDatasource(false);
setSelectedDataSource(null);
toast.error(error);
});
};
const cancelDeleteDataSource = () => {
setDeleteModalVisibility(false);
setSelectedDataSource(null);
};
const EmptyState = () => {
return (
<div
style={{
transform: 'translateY(80%)',
}}
className="d-flex justify-content-center align-items-center flex-column mt-3"
>
<div className="mb-4">
<EmptyFoldersIllustration />
</div>
<div className="tj-text-md text-secondary">No datasources added</div>
</div>
);
};
return (
<>
<div className="list-group mb-3">
{loading && <Skeleton count={3} height={22} />}
{!loading && (
<div className="mt-2 w-100" data-cy="datasource-Label">
{dataSources?.length ? (
dataSources?.map((source, idx) => (
<ListItem
dataSource={source}
key={idx}
active={selectedDataSource?.id === source?.id}
onDelete={deleteDataSource}
/>
))
) : (
<EmptyState />
)}
</div>
)}
</div>
<ConfirmDialog
show={isDeleteModalVisible}
message={'You will lose all the queries created from this data source. Do you really want to delete?'}
confirmButtonLoading={isDeletingDatasource}
onConfirm={() => executeDataSourceDeletion()}
onCancel={() => cancelDeleteDataSource()}
darkMode={darkMode}
/>
</>
);
};

View file

@ -0,0 +1,10 @@
import React from 'react';
import { CreateDataSourceModal } from '../CreateDataSourceModal';
export const Sidebar = () => {
return (
<div className="global-datasources-sidebar col border-bottom">
<CreateDataSourceModal />
</div>
);
};

View file

@ -0,0 +1,72 @@
import React, { createContext, useMemo, useState, useEffect } from 'react';
import { useNavigate } from 'react-router-dom';
import Layout from '@/_ui/Layout';
import { globalDatasourceService } from '@/_services';
import { GlobalDataSourcesPage } from './GlobalDataSourcesPage';
export const GlobalDataSourcesContext = createContext({
showDataSourceManagerModal: false,
toggleDataSourceManagerModal: () => {},
selectedDataSource: null,
setSelectedDataSource: () => {},
});
export const GlobalDatasources = (props) => {
const { organization_id, admin } = JSON.parse(localStorage.getItem('currentUser')) || {};
const [organizationId, setOrganizationId] = useState(organization_id);
const [selectedDataSource, setSelectedDataSource] = useState(null);
const [dataSources, setDataSources] = useState([]);
const [showDataSourceManagerModal, toggleDataSourceManagerModal] = useState(false);
const [isEditing, setEditing] = useState(true);
const navigate = useNavigate();
useEffect(() => {
if (!admin) {
navigate('/');
}
}, [admin]);
const fetchDataSources = async (resetSelection = false) => {
globalDatasourceService
.getAll(organizationId)
.then((data) => {
setDataSources([...(data.data_sources ?? [])]);
if (data.data_sources.length && resetSelection) {
setSelectedDataSource(data.data_sources[0]);
toggleDataSourceManagerModal(true);
}
})
.catch(() => setDataSources([]));
};
const handleModalVisibility = () => {
setSelectedDataSource(null);
setEditing(false);
toggleDataSourceManagerModal(true);
};
const value = useMemo(
() => ({
selectedDataSource,
setSelectedDataSource,
fetchDataSources,
dataSources,
showDataSourceManagerModal,
toggleDataSourceManagerModal,
handleModalVisibility,
isEditing,
setEditing,
}),
[selectedDataSource, dataSources, showDataSourceManagerModal, isEditing]
);
return (
<Layout switchDarkMode={props.switchDarkMode} darkMode={props.darkMode}>
<GlobalDataSourcesContext.Provider value={value}>
<div className="page-wrapper">
<GlobalDataSourcesPage darkMode={props.darkMode} />
</div>
</GlobalDataSourcesContext.Provider>
</Layout>
);
};

View file

@ -254,6 +254,7 @@ const DynamicForm = ({
auth_url: options.auth_url?.value,
custom_auth_params: options.custom_auth_params?.value,
custom_query_params: options.custom_query_params?.value,
spec: options.spec?.value,
};
default:
return {};

View file

@ -8,6 +8,7 @@ export const dataqueryService = {
update,
del,
preview,
changeQueryDataSource,
};
function getAll(appVersionId) {
@ -65,3 +66,11 @@ function preview(query, options, versionId) {
const requestOptions = { method: 'POST', headers: authHeader(), body: JSON.stringify(body) };
return fetch(`${config.apiUrl}/data_queries/preview`, requestOptions).then(handleResponse);
}
function changeQueryDataSource(id, dataSourceId) {
const body = {
data_source_id: dataSourceId,
};
const requestOptions = { method: 'PUT', headers: authHeader(), body: JSON.stringify(body) };
return fetch(`${config.apiUrl}/data_queries/${id}/data_source`, requestOptions).then(handleResponse);
}

View file

@ -17,25 +17,25 @@ function getAll(appVersionId) {
return fetch(`${config.apiUrl}/data_sources?` + searchParams, requestOptions).then(handleResponse);
}
function create(app_id, app_version_id, plugin_id, name, kind, options) {
function create(plugin_id, name, kind, options, app_id, app_version_id) {
const body = {
app_id,
app_version_id,
plugin_id,
name,
kind,
options,
app_id,
app_version_id,
};
const requestOptions = { method: 'POST', headers: authHeader(), body: JSON.stringify(body) };
return fetch(`${config.apiUrl}/data_sources`, requestOptions).then(handleResponse);
}
function save(id, app_id, name, options) {
function save(id, name, options, app_id) {
const body = {
app_id,
name,
options,
app_id,
};
const requestOptions = { method: 'PUT', headers: authHeader(), body: JSON.stringify(body) };

View file

@ -0,0 +1,49 @@
import config from 'config';
import { authHeader, handleResponse } from '@/_helpers';
export const globalDatasourceService = {
create,
getAll,
save,
deleteDataSource,
convertToGlobal,
};
function getAll(organizationId) {
const requestOptions = { method: 'GET', headers: authHeader() };
let searchParams = new URLSearchParams(`organization_id=${organizationId}`);
return fetch(`${config.apiUrl}/v2/data_sources?` + searchParams, requestOptions).then(handleResponse);
}
function create(plugin_id, name, kind, options, app_id, app_version_id, scope) {
const body = {
plugin_id,
name,
kind,
options,
scope,
};
const requestOptions = { method: 'POST', headers: authHeader(), body: JSON.stringify(body) };
return fetch(`${config.apiUrl}/v2/data_sources`, requestOptions).then(handleResponse);
}
function save(id, name, options) {
const body = {
name,
options,
};
const requestOptions = { method: 'PUT', headers: authHeader(), body: JSON.stringify(body) };
return fetch(`${config.apiUrl}/v2/data_sources/${id}`, requestOptions).then(handleResponse);
}
function deleteDataSource(id) {
const requestOptions = { method: 'DELETE', headers: authHeader() };
return fetch(`${config.apiUrl}/v2/data_sources/${id}`, requestOptions).then(handleResponse);
}
function convertToGlobal(id) {
const requestOptions = { method: 'POST', headers: authHeader() };
return fetch(`${config.apiUrl}/v2/data_sources/${id}/scope`, requestOptions).then(handleResponse);
}

View file

@ -17,3 +17,4 @@ export * from './groupPermission.service';
export * from './plugins.service';
export * from './marketplace.service';
export * from './tooljetDatabase.service';
export * from './globalDatasource.service';

View file

@ -312,4 +312,10 @@ $btn-dark-color: #FFFFFF;
height: 100%;
overflow: visible;
fill: #919eab;
}
#popover-change-scope {
border: 1px solid rgba(101, 109, 119, 0.16);
box-shadow: 0px 3px 2px rgba(0, 0, 0, 0.25);
}

View file

@ -0,0 +1,71 @@
@import "@radix-ui/colors/slate.css";
@import "@radix-ui/colors/indigo.css";
@import "@radix-ui/colors/tomato.css";
@import "@radix-ui/colors/grass.css";
@import "@radix-ui/colors/orange.css";
@import "@radix-ui/colors/crimson.css";
@import "@radix-ui/colors/pink.css";
@import "@radix-ui/colors/plum.css";
@import "@radix-ui/colors/purple.css";
@import "@radix-ui/colors/violet.css";
@import "@radix-ui/colors/blue.css";
@import "@radix-ui/colors/cyan.css";
@import "@radix-ui/colors/teal.css";
@import "@radix-ui/colors/green.css";
@import "@radix-ui/colors/brown.css";
@import "@radix-ui/colors/red.css";
@import "@radix-ui/colors/slateDark.css";
@import "@radix-ui/colors/indigoDark.css";
@import "@radix-ui/colors/tomatoDark.css";
@import "@radix-ui/colors/grassDark.css";
@import "@radix-ui/colors/orangeDark.css";
@import "@radix-ui/colors/crimsonDark.css";
@import "@radix-ui/colors/pinkDark.css";
@import "@radix-ui/colors/plumDark.css";
@import "@radix-ui/colors/purpleDark.css";
@import "@radix-ui/colors/violetDark.css";
@import "@radix-ui/colors/blueDark.css";
@import "@radix-ui/colors/cyanDark.css";
@import "@radix-ui/colors/tealDark.css";
@import "@radix-ui/colors/greenDark.css";
@import "@radix-ui/colors/brownDark.css";
@import "@radix-ui/colors/redDark.css";
// Bright Colors
@import "@radix-ui/colors/sky.css";
@import "@radix-ui/colors/mint.css";
@import "@radix-ui/colors/lime.css";
@import "@radix-ui/colors/yellow.css";
@import "@radix-ui/colors/amber.css";
@import "@radix-ui/colors/skyDark.css";
@import "@radix-ui/colors/mintDark.css";
@import "@radix-ui/colors/limeDark.css";
@import "@radix-ui/colors/yellowDark.css";
@import "@radix-ui/colors/amberDark.css";
// Grays
@import "@radix-ui/colors/grayDark.css";
@import "@radix-ui/colors/mauveDark.css";
@import "@radix-ui/colors/sageDark.css";
@import "@radix-ui/colors/oliveDark.css";
@import "@radix-ui/colors/sandDark.css";
@import "@radix-ui/colors/gray.css";
@import "@radix-ui/colors/mauve.css";
@import "@radix-ui/colors/sage.css";
@import "@radix-ui/colors/olive.css";
@import "@radix-ui/colors/sand.css";
// Metals
@import "@radix-ui/colors/gold.css";
@import "@radix-ui/colors/bronze.css";
@import "@radix-ui/colors/goldDark.css";
@import "@radix-ui/colors/bronzeDark.css";
:root {
--base: white;
}
.dark-theme {
/* Remap your colors for dark mode */
--base: #121212;
}

View file

@ -0,0 +1,87 @@
@import "./typography.scss";
@import "./designtheme.scss";
.global-datasources-sidebar {
height: calc(100vh - 48px);
max-width: 288px;
overflow-y: scroll;
.add-datasource-btn {
height: 32px;
background: var(--indigo9);
}
.datasources-list {
display: flex;
&:hover {
background-color: var(--indigo2);
.ds-delete-btn {
display: block;
}
}
.ds-delete-btn {
display: none;
border: none;
background: none;
}
}
.datasources-list-item {
background-color: var(--indigo3);
}
}
.main-empty-container {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
margin: auto;
width: 300px;
height: 100%;
.icon-container {
background-color: var(--indigo3);
border-radius: 15px;
padding: 12px;
}
.heading {
color: var(--slate12);
font-weight: 600;
}
.sub-heading {
text-align: center;
}
.add-datasource-btn {
background-color: var(--indigo3);
color: var(--indigo9);
}
}
.datasource-modal-container {
position: relative;
.modal {
position: absolute;
}
.modal-content {
border: 1px solid var(--slate5);
.input-icon {
&:hover {
input {
padding-right: 2rem !important;
}
.input-icon-addon {
justify-content: flex-end;
}
}
}
}
}

View file

@ -478,8 +478,14 @@
top: 8px;
}
.sidebar-h-100-popover {
position: relative;
height: 100vh;
margin-top: 0px;
.add-datasource-btn {
position: absolute;
bottom: 3rem;
}
}
.sidebar-h-100-popover-inspector {
min-width: 422px;

View file

@ -5,6 +5,11 @@
@import "./queryManager.scss";
@import "./onboarding.scss";
@import "./components.scss";
@import "./global-datasources.scss";
@import "./typography.scss";
@import "./designtheme.scss";
@import url('https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;1,100;1,200;1,400;1,500;1,600;1,700&display=swap');
// variables
$border-radius: 4px;
@ -142,7 +147,7 @@ button {
.resizer-select,
.resizer-active {
border: solid 1px $primary !important;
border: solid 1px $primary !important;
.top-right,
.top-left,
@ -622,7 +627,7 @@ button {
.list-group.list-group-transparent.dark .all-apps-link,
.list-group-item-action.dark.active {
background-color: $dark-background !important;
background-color: $dark-background !important;
}
}
@ -1215,7 +1220,7 @@ button {
.select-search-dark input {
width: 224px !important;
height: 32px !important;
border-radius: $border-radius !important;
border-radius: $border-radius !important;
}
}
@ -1226,7 +1231,7 @@ button {
.select-search__value input,
.select-search-dark input {
height: 32px !important;
border-radius: $border-radius !important;
border-radius: $border-radius !important;
}
}
@ -1288,7 +1293,7 @@ button {
-moz-appearance: none;
appearance: none;
// border-radius: 0;
border-radius: $border-radius !important;
border-radius: $border-radius !important;
transition: border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;
}
@ -1530,7 +1535,7 @@ button {
height: 36px;
width: 100%;
padding: 0 16px;
background-color: $dark-background !important;
background-color: $dark-background !important;
color: #fff !important;
border: none;
outline: none;
@ -3601,7 +3606,7 @@ input[type="text"] {
.nav-tabs .nav-link.active {
font-weight: 400 !important;
color: $primary !important;
color: $primary !important;
}
.empty {
@ -4120,7 +4125,7 @@ input[type="text"] {
.tabs-inspector.dark {
.nav-link.active {
border-bottom: 1px solid $primary !important;
border-bottom: 1px solid $primary !important;
}
}
@ -4336,7 +4341,7 @@ input[type="text"] {
}
input {
border-radius: $border-radius !important;
border-radius: $border-radius !important;
padding-left: 1.75rem !important;
}
}
@ -4463,11 +4468,11 @@ input[type="text"] {
}
.modal-content.home-modal-component.dark {
background-color: $bg-dark-light !important;
color: $white !important;
background-color: $bg-dark-light !important;
color: $white !important;
.modal-header {
background-color: $bg-dark-light !important;
background-color: $bg-dark-light !important;
}
.btn-close {
@ -4475,22 +4480,22 @@ input[type="text"] {
}
.form-control {
border-color: $border-grey-dark !important;
border-color: $border-grey-dark !important;
color: inherit;
}
input {
background-color: $bg-dark-light !important;
background-color: $bg-dark-light !important;
}
.form-select {
background-color: $bg-dark !important;
color: $white !important;
border-color: $border-grey-dark !important;
background-color: $bg-dark !important;
color: $white !important;
border-color: $border-grey-dark !important;
}
.text-muted {
color: $white !important;
color: $white !important;
}
}
@ -4783,7 +4788,7 @@ div#driver-page-overlay {
}
.dark-theme-walkthrough#driver-popover-item {
background-color: $bg-dark-light !important;
background-color: $bg-dark-light !important;
border-color: rgba(101, 109, 119, 0.16) !important;
.driver-popover-title {
@ -4791,7 +4796,7 @@ div#driver-page-overlay {
}
.driver-popover-tip {
border-color: transparent transparent transparent $bg-dark-light !important;
border-color: transparent transparent transparent $bg-dark-light !important;
}
.driver-popover-description {
@ -4823,7 +4828,7 @@ div#driver-page-overlay {
.driver-next-btn,
.driver-prev-btn {
color: $primary !important;
color: $primary !important;
}
.driver-disabled {
@ -5168,7 +5173,7 @@ div#driver-page-overlay {
}
.selected-node {
border-color: $primary-light !important;
border-color: $primary-light !important;
}
.json-tree-icon-container .selected-node>svg:first-child {
@ -5258,7 +5263,7 @@ div#driver-page-overlay {
}
.selected-node {
border-color: $primary-light !important;
border-color: $primary-light !important;
}
.selected-node .group-object-container .badge {
@ -5525,7 +5530,7 @@ div#driver-page-overlay {
//Kanban board
.kanban-container.dark-themed {
background-color: $bg-dark-light !important;
background-color: $bg-dark-light !important;
.kanban-column {
.card-header {
@ -5571,7 +5576,7 @@ div#driver-page-overlay {
}
.dnd-card.card.card-dark {
background-color: $bg-dark !important;
background-color: $bg-dark !important;
}
}
@ -7457,6 +7462,10 @@ tbody {
}
}
.datasources-category {
color: var(--slate10);
}
.react-tooltip {
font-size: .765625rem !important;
}

View file

@ -0,0 +1,127 @@
//Typography
.tj-text {
font-family: 'IBM Plex Sans';
font-style: normal;
color: #121212;
margin: 0;
padding: 0;
}
.tj-header-lg {
font-weight: 500;
font-size: 52px;
line-height: 56px;
letter-spacing: -0.02em;
margin: 0;
padding: 0;
}
.tj-header-sm {
font-weight: 500;
font-size: 44px;
line-height: 48px;
letter-spacing: -0.02em;
margin: 0;
padding: 0;
}
.tj-header-h1 {
font-weight: 500;
font-size: 40px;
line-height: 48px;
letter-spacing: -0.02em;
}
.tj-header-h2 {
font-weight: 500;
font-size: 36px;
line-height: 44px;
letter-spacing: -0.02em;
margin: 0;
padding: 0;
}
.tj-header-h3 {
font-weight: 500;
font-size: 32px;
line-height: 40px;
letter-spacing: -0.02em;
}
.tj-header-h4 {
font-weight: 500;
font-size: 32px;
line-height: 40px;
letter-spacing: -0.02em;
margin: 0;
padding: 0;
}
.tj-header-h5 {
font-weight: 500;
font-size: 32px;
line-height: 40px;
letter-spacing: -0.02em;
margin: 0;
padding: 0;
}
.tj-header-h6 {
font-weight: 500;
font-size: 32px;
line-height: 40px;
letter-spacing: -0.02em;
margin: 0;
padding: 0;
}
// paragraph texts
.tj-text-lg {
font-weight: 400;
font-size: 18px;
line-height: 28px;
margin: 0;
padding: 0;
}
.tj-text-md {
font-weight: 400;
font-size: 14px;
line-height: 24px;
margin: 0;
padding: 0;
}
.tj-text-sm {
font-weight: 400;
font-size: 14px;
line-height: 20px;
margin: 0;
padding: 0;
}
.tj-text-xsm {
font-weight: 400;
font-size: 12px;
line-height: 20px;
margin: 0;
padding: 0;
}
.tj-text-xxsm {
font-weight: 400;
font-size: 10px;
line-height: 16px;
margin: 0;
padding: 0;
}
.tj-para-md {
font-weight: 500;
font-size: 16px;
line-height: 24px;
margin: 0;
padding: 0;
}

View file

@ -27,4 +27,5 @@ const routes = [
// { path: '/', breadcrumb: 'Apps' },
{ path: '/database', breadcrumb: 'Tables', props: { dataCy: 'tables-page-header' } },
{ path: '/workspace-settings', breadcrumb: 'Workspace settings' },
{ path: '/global-datasources', breadcrumb: 'Global Datasources' },
];

View file

@ -119,6 +119,36 @@ function Layout({ children, switchDarkMode, darkMode }) {
</ToolTip>
</Link>
</li>
{admin && (
<li className="text-center mt-3 cursor-pointer">
<Link to="/global-datasources">
<ToolTip message="Global Datasources" placement="right">
<svg
className="layout-sidebar-icon"
width="32"
height="33"
viewBox="0 0 32 33"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<rect
y="0.325684"
width="32"
height="32"
rx="4"
fill={router.pathname === '/global-datasources' ? '#E6EDFE' : '#none'}
/>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M9.29209 9.58052C9.01022 9.83042 9 9.97447 9 10C9 10.0255 9.01022 10.1696 9.29209 10.4195C9.57279 10.6683 10.036 10.9381 10.6943 11.185C12.0034 11.6759 13.879 12 16 12C18.121 12 19.9966 11.6759 21.3057 11.185C21.964 10.9381 22.4272 10.6683 22.7079 10.4195C22.9898 10.1696 23 10.0255 23 10C23 9.97447 22.9898 9.83042 22.7079 9.58052C22.4272 9.33166 21.964 9.06185 21.3057 8.81501C19.9966 8.32409 18.121 8 16 8C13.879 8 12.0034 8.32409 10.6943 8.81501C10.036 9.06185 9.57279 9.33166 9.29209 9.58052ZM23 12.6162C22.6909 12.7798 22.3574 12.9266 22.008 13.0576C20.4217 13.6525 18.2973 14 16 14C13.7027 14 11.5783 13.6525 9.99202 13.0576C9.64262 12.9266 9.3091 12.7798 9 12.6162V16C9 16.0187 9.00689 16.1594 9.28011 16.4067C9.55297 16.6538 10.0136 16.9298 10.6943 17.185C12.0524 17.6943 13.9615 18 16 18C18.0385 18 19.9476 17.6943 21.3057 17.185C21.9864 16.9298 22.447 16.6538 22.7199 16.4067C22.9931 16.1594 23 16.0187 23 16V12.6162ZM23 18.6158C22.6937 18.7782 22.3609 18.9253 22.008 19.0576C20.3656 19.6736 18.205 20 16 20C13.795 20 11.6344 19.6736 9.99202 19.0576C9.6391 18.9253 9.30634 18.7782 9 18.6158V22C9 22.0187 9.00689 22.1594 9.28011 22.4067C9.55297 22.6538 10.0136 22.9298 10.6943 23.185C12.0524 23.6943 13.9615 24 16 24C18.0385 24 19.9476 23.6943 21.3057 23.185C21.9864 22.9298 22.447 22.6538 22.7199 22.4067C22.9931 22.1594 23 22.0187 23 22V18.6158ZM25 22C25 22.777 24.5855 23.4156 24.0622 23.8894C23.5385 24.3634 22.8276 24.7503 22.008 25.0576C20.3656 25.6736 18.205 26 16 26C13.795 26 11.6344 25.6736 9.99202 25.0576C9.17237 24.7503 8.46146 24.3634 7.93782 23.8894C7.41454 23.4156 7 22.777 7 22V10C7 9.19711 7.43749 8.55194 7.96527 8.08401C8.49422 7.61504 9.20256 7.2384 9.99202 6.94235C11.5783 6.34749 13.7027 6 16 6C18.2973 6 20.4217 6.34749 22.008 6.94235C22.7974 7.2384 23.5058 7.61504 24.0347 8.08401C24.5625 8.55194 25 9.19711 25 10V22Z"
fill={router.pathname === '/global-datasources' ? '#3E63DD' : '#C1C8CD'}
/>
</svg>
</ToolTip>
</Link>
</li>
)}
<li className="m-auto">
<NotificationCenter darkMode={darkMode} />
<Profile switchDarkMode={switchDarkMode} darkMode={darkMode} />

View file

@ -25,6 +25,7 @@ const OpenApi = ({
grant_type,
scopes,
auth_url,
spec,
}) => {
const [securities, setSecurities] = useState([]);
const [loadingSpec, setLoadingSpec] = useState(false);
@ -41,6 +42,11 @@ const OpenApi = ({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [auth_key, securities]);
useEffect(() => {
spec && setSecurities(resolveSecurities(spec));
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [spec]);
const validateDef = () => {
if (definition) {
setLoadingSpec(true);
@ -49,7 +55,6 @@ const OpenApi = ({
.parseOpenapiSpec(definition, format)
.then((result) => {
optionchanged('spec', result);
setSecurities(resolveSecurities(result));
setLoadingSpec(false);
})
.catch((err) => {

2481
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -27,15 +27,14 @@ export class moveDataSourceOptionsToEnvironment1669054493160 implements Migratio
const encryptionService = this.nestApp.get(EncryptionService);
for (const { name, isDefault } of defaultAppEnvironments) {
const environment: AppEnvironment = await entityManager.save(
entityManager.create(AppEnvironment, {
name,
isDefault,
appVersionId: appVersion.id,
})
const environment: AppEnvironment = await entityManager.query(
'insert into app_environments (name, is_default, app_version_id, created_at, updated_at) values ($1, $2, $3, $4, $4) returning *',
[name, isDefault, appVersion.id, new Date()]
);
// Get all data sources under app
const dataSources = await entityManager.query('select * from data_sources where app_id = $1', [appVersion.appId]);
// Get all data sources under app version
const dataSources = await entityManager.query('select * from data_sources where app_version_id = $1', [
appVersion.id,
]);
if (dataSources?.length) {
for (const dataSource of dataSources) {

View file

@ -2,7 +2,6 @@ import { MigrationInterface, QueryRunner, TableForeignKey } from 'typeorm';
export class removeRepetitionInDataSourceAndQuery1669919175280 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
// TODO: next version add : await queryRunner.dropColumn('data_sources', 'options');
await this.dropForeignKey('data_sources', 'app_id', queryRunner);
await this.dropForeignKey('data_queries', 'app_id', queryRunner);
await this.dropForeignKey('data_queries', 'app_version_id', queryRunner);

View file

@ -0,0 +1,26 @@
import { MigrationInterface, QueryRunner, TableColumn, TableForeignKey } from 'typeorm';
export class AddAppVersionIdColumnToDataQueries1675368628629 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.addColumn(
'data_queries',
new TableColumn({
name: 'app_version_id',
type: 'uuid',
isNullable: true,
})
);
await queryRunner.createForeignKey(
'data_queries',
new TableForeignKey({
columnNames: ['app_version_id'],
referencedColumnNames: ['id'],
referencedTableName: 'app_versions',
onDelete: 'CASCADE',
})
);
}
public async down(queryRunner: QueryRunner): Promise<void> {}
}

View file

@ -0,0 +1,37 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class BackfillAppVersionToDataQueries1675368628727 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const entityManager = queryRunner.manager;
let progress = 0;
const appVersions = await entityManager.query(
'select distinct(data_sources.app_version_id) from data_queries inner join data_sources on data_queries.data_source_id = data_sources.id'
);
const queryToAppVersionMap = await entityManager.query(
'select data_queries.id, data_sources.app_version_id from data_queries inner join data_sources on data_queries.data_source_id = data_sources.id'
);
console.log(`App versions found : ${appVersions?.length || 0}`);
if (queryToAppVersionMap?.length) {
for (const { app_version_id: appVersion } of appVersions) {
progress++;
console.log(
`BackfillAppVersionToDataQueries1675368628727 Progress ${Math.round((progress / appVersions.length) * 100)} %`
);
const queries = queryToAppVersionMap?.filter((query) => query.app_version_id === appVersion);
if (queries?.length) {
await entityManager.query(
`update data_queries set app_version_id = $1 where id IN(${queries.map((dq) => `'${dq.id}'`)?.join()})`,
[appVersion]
);
}
}
}
}
public async down(queryRunner: QueryRunner): Promise<void> {}
}

View file

@ -0,0 +1,36 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
// This migration is to cleanup data source options which are added on v2 migration, and conflicts with global data source
export class CleanupDataSourceOptionData1675844361117 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const entityManager = queryRunner.manager;
let progress = 0;
await queryRunner.dropColumn('data_sources', 'options');
const dataSources = await entityManager.query('select id from data_sources');
for (const { id } of dataSources) {
progress++;
console.log(
`CleanupDataSourceOptionData1675844361117 Progress ${Math.round((progress / dataSources.length) * 100)} %`
);
const idsToDelete = await entityManager.query(
`select data_source_options.id from data_source_options
inner join data_sources on data_source_options.data_source_id = data_sources.id
inner join app_environments on data_source_options.environment_id = app_environments.id
where data_sources.app_version_id != app_environments.app_version_id and data_source_options.data_source_id = $1`,
[id]
);
if (idsToDelete && idsToDelete.length > 0) {
await entityManager.query(
`delete from data_source_options where id IN(${idsToDelete.map((ids) => `'${ids.id}'`).join()})`
);
}
}
}
public async down(queryRunner: QueryRunner): Promise<void> {}
}

View file

@ -0,0 +1,74 @@
import { AppEnvironment } from 'src/entities/app_environments.entity';
import { Organization } from 'src/entities/organization.entity';
import { defaultAppEnvironments } from 'src/helpers/utils.helper';
import { MigrationInterface, QueryRunner } from 'typeorm';
export class MigrateEnvironmentsUnderWorkspace1675844361118 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const entityManager = queryRunner.manager;
let progress = 0;
const organizations = await entityManager.find(Organization, {
select: ['id', 'name'],
});
//Insert new environments under workspace
for (const org of organizations) {
progress++;
console.log(
`MigrateEnvironmentsUnderWorkspace1675844361118 Progress ${Math.round(
(progress / organizations.length) * 100
)} %`
);
const newMappingForEnvironments = {};
console.log(`Performing environment migration for ${org.name}: ${org.id}`);
for (const { name, isDefault } of defaultAppEnvironments) {
console.log(`Current Environment name: ${name}`);
const environment: AppEnvironment = await entityManager.save(
entityManager.create(AppEnvironment, {
name,
isDefault,
organizationId: org.id,
})
);
newMappingForEnvironments[name] = {
...newMappingForEnvironments[name],
[org.id]: environment.id,
};
//Retrieve old environments under app_versions
const oldMappingForEnvironments = {};
const envs = await queryRunner.query(
`select app_environments.id from app_versions inner join apps on apps.organization_id = $1
inner join app_environments on app_environments.app_version_id = app_versions.id
where app_versions.app_id = apps.id and app_environments.name=$2`,
[org.id, name]
);
oldMappingForEnvironments[name] = {
...oldMappingForEnvironments[name],
[org.id]: envs,
};
//Update datasources options from old and new mapping
if (oldMappingForEnvironments[name][org.id] && oldMappingForEnvironments[name][org.id].length > 0) {
await queryRunner.query(
`update data_source_options set environment_id = $1
where environment_id IN (${oldMappingForEnvironments[name][org.id]
.map((env) => `'${env.id}'`)
?.join()})`,
[newMappingForEnvironments[name][org.id]]
);
}
}
console.log(`Env migration completed for organization: ${org.name}: ${org.id}`);
}
//Drop app_version_id column as it is no longer needed
await queryRunner.dropColumn('app_environments', 'app_version_id');
}
public async down(queryRunner: QueryRunner): Promise<void> {}
}

View file

@ -8,6 +8,7 @@ import { EntityManager, MigrationInterface, QueryRunner } from 'typeorm';
export class BackfillRunpyDatasources1676545162064 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const entityManager = queryRunner.manager;
let progress = 0;
const allVersions = await entityManager
.createQueryBuilder()
@ -16,6 +17,11 @@ export class BackfillRunpyDatasources1676545162064 implements MigrationInterface
.getRawMany();
for (const version of allVersions) {
progress++;
console.log(
`BackfillRunpyDatasources1676545162064 Progress ${Math.round((progress / allVersions.length) * 100)} %`
);
await this.createDefaultVersionAndAttachQueries(entityManager, version);
}
}

View file

@ -0,0 +1,27 @@
import { AppEnvironment } from 'src/entities/app_environments.entity';
import { MigrationInterface, QueryRunner, TableForeignKey } from 'typeorm';
export class AlterOrganizationIdInAppEnvironments1677822012965 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const entityManager = queryRunner.manager;
//Delete old app_environments which are not under organizations
await entityManager.delete(AppEnvironment, { organizationId: null });
//Add not null constrain to organization_id column
await queryRunner.query('alter table app_environments alter column organization_id set not null');
//Add Foreign key and delete constraints on organization_id column
await queryRunner.createForeignKey(
'app_environments',
new TableForeignKey({
columnNames: ['organization_id'],
referencedColumnNames: ['id'],
referencedTableName: 'organizations',
onDelete: 'CASCADE',
})
);
}
public async down(queryRunner: QueryRunner): Promise<void> {}
}

View file

@ -0,0 +1,21 @@
import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm';
export class AddScopeColumnToDataSources1675179628075 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.addColumn(
'data_sources',
new TableColumn({
name: 'scope',
type: 'enum',
enumName: 'scope',
enum: ['local', 'global'],
default: `'local'`,
isNullable: false,
})
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.dropColumn('data_sources', 'scope');
}
}

View file

@ -0,0 +1,16 @@
import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm';
export class AddOrganizationIdInAppEnvironments1675842611112 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.addColumn(
'app_environments',
new TableColumn({
name: 'organization_id',
type: 'uuid',
isNullable: true,
})
);
}
public async down(queryRunner: QueryRunner): Promise<void> {}
}

View file

@ -0,0 +1,11 @@
import { dropForeignKey } from 'src/helpers/utils.helper';
import { MigrationInterface, QueryRunner } from 'typeorm';
export class DropAppVersionIdInAppEnvironments1677820920004 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await dropForeignKey('app_environments', 'app_version_id', queryRunner);
await queryRunner.query('alter table app_environments alter column app_version_id drop not null');
}
public async down(queryRunner: QueryRunner): Promise<void> {}
}

View file

@ -10,6 +10,7 @@ import {
UseGuards,
ForbiddenException,
BadRequestException,
Put,
} from '@nestjs/common';
import { JwtAuthGuard } from '../../src/modules/auth/jwt-auth.guard';
import { decamelizeKeys } from 'humps';
@ -24,7 +25,8 @@ import { decode } from 'js-base64';
import { dbTransactionWrap } from 'src/helpers/utils.helper';
import { EntityManager } from 'typeorm';
import { DataSource } from 'src/entities/data_source.entity';
import { DataSourceTypes } from 'src/helpers/data_source.constants';
import { DataSourceScopes, DataSourceTypes } from 'src/helpers/data_source.constants';
import { App } from 'src/entities/app.entity';
@Controller('data_queries')
export class DataQueriesController {
@ -87,6 +89,7 @@ export class DataQueriesController {
} = dataQueryDto;
let dataSource: DataSource;
let app: App;
if (!dataSourceId && !(kind === 'restapi' || kind === 'runjs' || kind === 'tooljetdb' || kind === 'runpy')) {
throw new BadRequestException();
@ -94,10 +97,22 @@ export class DataQueriesController {
return dbTransactionWrap(async (manager: EntityManager) => {
if (!dataSourceId && (kind === 'restapi' || kind === 'runjs' || kind === 'tooljetdb' || kind === 'runpy')) {
dataSource = await this.dataSourcesService.findDefaultDataSource(kind, appVersionId, pluginId, manager);
dataSource = await this.dataSourcesService.findDefaultDataSource(
kind,
appVersionId,
pluginId,
user.organizationId,
manager
);
}
dataSource = await this.dataSourcesService.findOne(dataSource?.id || dataSourceId, manager);
if (dataSource.scope === DataSourceScopes.GLOBAL) {
app = await this.appsService.findAppFromVersion(appVersionId);
} else {
app = await this.dataSourcesService.findApp(dataSource?.id || dataSourceId, manager);
}
const app = await this.dataSourcesService.findApp(dataSource?.id || dataSourceId, manager);
const ability = await this.appsAbilityFactory.appsActions(user, app.id);
if (!ability.can('createQuery', app)) {
@ -105,7 +120,13 @@ export class DataQueriesController {
}
// todo: pass the whole dto instead of indv. values
const dataQuery = await this.dataQueriesService.create(name, options, dataSource?.id || dataSourceId, manager);
const dataQuery = await this.dataQueriesService.create(
name,
options,
dataSource?.id || dataSourceId,
appVersionId,
manager
);
return decamelizeKeys(dataQuery);
});
}
@ -116,9 +137,9 @@ export class DataQueriesController {
const { name, options } = updateDataQueryDto;
const dataQuery = await this.dataQueriesService.findOne(dataQueryId);
const ability = await this.appsAbilityFactory.appsActions(user, dataQuery.dataSource.app.id);
const ability = await this.appsAbilityFactory.appsActions(user, dataQuery.app.id);
if (!ability.can('updateQuery', dataQuery.dataSource.app)) {
if (!ability.can('updateQuery', dataQuery.app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
}
@ -130,9 +151,9 @@ export class DataQueriesController {
@Delete(':id')
async delete(@User() user, @Param('id') dataQueryId) {
const dataQuery = await this.dataQueriesService.findOne(dataQueryId);
const ability = await this.appsAbilityFactory.appsActions(user, dataQuery.dataSource.app.id);
const ability = await this.appsAbilityFactory.appsActions(user, dataQuery.app.id);
if (!ability.can('deleteQuery', dataQuery.dataSource.app)) {
if (!ability.can('deleteQuery', dataQuery.app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
}
@ -153,9 +174,9 @@ export class DataQueriesController {
const dataQuery = await this.dataQueriesService.findOne(dataQueryId);
if (user) {
const ability = await this.appsAbilityFactory.appsActions(user, dataQuery.dataSource.app.id);
const ability = await this.appsAbilityFactory.appsActions(user, dataQuery.app.id);
if (!ability.can('runQuery', dataQuery.dataSource.app)) {
if (!ability.can('runQuery', dataQuery.app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
}
}
@ -195,6 +216,8 @@ export class DataQueriesController {
) {
const { options, query, app_version_id: appVersionId } = updateDataQueryDto;
const app = await this.appsService.findAppFromVersion(appVersionId);
if (!(query['data_source_id'] || appVersionId || environmentId)) {
throw new BadRequestException('Data source id or app version id or environment id is mandatory');
}
@ -202,14 +225,15 @@ export class DataQueriesController {
const kind = query ? query['kind'] : null;
const dataQueryEntity = {
...query,
app,
dataSource: query['data_source_id']
? await this.dataSourcesService.findOne(query['data_source_id'])
: await this.dataSourcesService.findDefaultDataSourceByKind(kind, appVersionId, environmentId),
: await this.dataSourcesService.findDefaultDataSourceByKind(kind, appVersionId),
};
const ability = await this.appsAbilityFactory.appsActions(user, dataQueryEntity.dataSource.app.id);
const ability = await this.appsAbilityFactory.appsActions(user, app.id);
if (!ability.can('previewQuery', dataQueryEntity.dataSource.app)) {
if (!ability.can('previewQuery', app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
}
@ -238,4 +262,19 @@ export class DataQueriesController {
return result;
}
@UseGuards(JwtAuthGuard)
@Put(':id/data_source')
async changeQueryDataSource(@User() user, @Param('id') queryId, @Body() updateDataQueryDto: UpdateDataQueryDto) {
const { data_source_id: dataSourceId } = updateDataQueryDto;
const dataQuery = await this.dataQueriesService.findOne(queryId);
const ability = await this.appsAbilityFactory.appsActions(user, dataQuery.app.id);
if (!ability.can('updateQuery', dataQuery.app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
}
await this.dataQueriesService.changeQueryDataSource(queryId, dataSourceId);
return;
}
}

View file

@ -16,6 +16,7 @@ import { decamelizeKeys } from 'humps';
import { DataSourcesService } from '../../src/services/data_sources.service';
import { AppsService } from '@services/apps.service';
import { AppsAbilityFactory } from 'src/modules/casl/abilities/apps-ability.factory';
import { GlobalDataSourceAbilityFactory } from 'src/modules/casl/abilities/global-datasource-ability.factory';
import { DataQueriesService } from '@services/data_queries.service';
import {
AuthorizeDataSourceOauthDto,
@ -26,12 +27,15 @@ import {
} from '@dto/data-source.dto';
import { decode } from 'js-base64';
import { User } from 'src/decorators/user.decorator';
import { DataSourceScopes } from 'src/helpers/data_source.constants';
import { DataSource } from 'src/entities/data_source.entity';
@Controller('data_sources')
export class DataSourcesController {
constructor(
private appsService: AppsService,
private appsAbilityFactory: AppsAbilityFactory,
private globalDataSourceAbilityFactory: GlobalDataSourceAbilityFactory,
private dataSourcesService: DataSourcesService,
private dataQueriesService: DataQueriesService
) {}
@ -43,10 +47,10 @@ export class DataSourcesController {
const ability = await this.appsAbilityFactory.appsActions(user, app.id);
if (!ability.can('getDataSources', app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
throw new ForbiddenException('You do not have permissions to perform this action');
}
const dataSources = await this.dataSourcesService.all(query);
const dataSources = await this.dataSourcesService.all(query, user.organizationId);
for (const dataSource of dataSources) {
if (dataSource.pluginId) {
dataSource.plugin.iconFile.data = dataSource.plugin.iconFile.data.toString('utf8');
@ -71,10 +75,18 @@ export class DataSourcesController {
const ability = await this.appsAbilityFactory.appsActions(user, app.id);
if (!ability.can('createDataSource', app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
throw new ForbiddenException('You do not have permissions to perform this action');
}
const dataSource = await this.dataSourcesService.create(name, kind, options, appVersionId, pluginId, environmentId);
const dataSource = await this.dataSourcesService.create(
name,
kind,
options,
appVersionId,
user.organizationId,
pluginId,
environmentId
);
return decamelizeKeys(dataSource);
}
@ -94,10 +106,10 @@ export class DataSourcesController {
const ability = await this.appsAbilityFactory.appsActions(user, app.id);
if (!ability.can('updateDataSource', app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
throw new ForbiddenException('You do not have permissions to perform this action');
}
await this.dataSourcesService.update(dataSourceId, name, options, environmentId);
await this.dataSourcesService.update(dataSourceId, user.organizationId, name, options, environmentId);
return;
}
@ -110,7 +122,7 @@ export class DataSourcesController {
const ability = await this.appsAbilityFactory.appsActions(user, app.id);
if (!ability.can('deleteDataSource', app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
throw new ForbiddenException('You do not have permissions to perform this action');
}
const result = await this.dataSourcesService.delete(dataSourceId);
@ -148,14 +160,22 @@ export class DataSourcesController {
const dataSource = await this.dataSourcesService.findOneByEnvironment(dataSourceId, environmentId);
const { app } = dataSource;
const ability = await this.appsAbilityFactory.appsActions(user, app.id);
if (dataSource.scope === DataSourceScopes.GLOBAL) {
const ability = await this.globalDataSourceAbilityFactory.globalDataSourceActions(user);
if (!ability.can('authorizeOauthForSource', app)) {
throw new ForbiddenException('you do not have permissions to perform this action');
if (!ability.can('authorizeOauthForSource', DataSource)) {
throw new ForbiddenException('You do not have permissions to perform this actio');
}
} else {
const { app } = dataSource;
const ability = await this.appsAbilityFactory.appsActions(user, app.id);
if (!ability.can('authorizeOauthForSource', app)) {
throw new ForbiddenException('You do not have permissions to perform this actions');
}
}
await this.dataQueriesService.authorizeOauth2(dataSource, code, user.id, environmentId);
await this.dataQueriesService.authorizeOauth2(dataSource, code, user.id, environmentId, user.organizationId);
return;
}
}

View file

@ -0,0 +1,126 @@
import {
Controller,
ForbiddenException,
Get,
Body,
Param,
Post,
Delete,
Put,
Query,
UseGuards,
BadRequestException,
} from '@nestjs/common';
import { JwtAuthGuard } from '../../src/modules/auth/jwt-auth.guard';
import { decamelizeKeys } from 'humps';
import { GlobalDataSourceAbilityFactory } from 'src/modules/casl/abilities/global-datasource-ability.factory';
import { DataQueriesService } from '@services/data_queries.service';
import { DataSourcesService } from '@services/data_sources.service';
import { CreateDataSourceDto, UpdateDataSourceDto } from '@dto/data-source.dto';
import { decode } from 'js-base64';
import { User } from 'src/decorators/user.decorator';
import { DataSource } from 'src/entities/data_source.entity';
import { DataSourceScopes } from 'src/helpers/data_source.constants';
@Controller({
path: 'data_sources',
version: '2',
})
export class GlobalDataSourcesController {
constructor(
private globalDataSourceAbilityFactory: GlobalDataSourceAbilityFactory,
private dataSourcesService: DataSourcesService,
private dataQueriesService: DataQueriesService
) {}
@UseGuards(JwtAuthGuard)
@Get()
async fetchGlobalDataSources(@User() user, @Query() query) {
const dataSources = await this.dataSourcesService.all(query, user.organizationId, DataSourceScopes.GLOBAL);
for (const dataSource of dataSources) {
if (dataSource.pluginId) {
dataSource.plugin.iconFile.data = dataSource.plugin.iconFile.data.toString('utf8');
dataSource.plugin.manifestFile.data = JSON.parse(decode(dataSource.plugin.manifestFile.data.toString('utf8')));
dataSource.plugin.operationsFile.data = JSON.parse(
decode(dataSource.plugin.operationsFile.data.toString('utf8'))
);
}
}
return decamelizeKeys({ data_sources: dataSources }, function (key, convert, options) {
const checkForKeysAsPath = /^(\/{0,1}(?!\/))[A-Za-z0-9/\-_]+(.([a-zA-Z]+))?$/gm;
return checkForKeysAsPath.test(key) ? key : convert(key, options);
});
}
@UseGuards(JwtAuthGuard)
@Post()
async createGlobalDataSources(@User() user, @Body() createDataSourceDto: CreateDataSourceDto) {
const { kind, name, options, app_version_id: appVersionId, plugin_id: pluginId, scope } = createDataSourceDto;
const ability = await this.globalDataSourceAbilityFactory.globalDataSourceActions(user);
if (!ability.can('createGlobalDataSource', DataSource)) {
throw new ForbiddenException('You do not have permissions to perform this action');
}
const dataSource = await this.dataSourcesService.create(
name,
kind,
options,
appVersionId,
user.organizationId,
scope,
pluginId
);
return decamelizeKeys(dataSource);
}
@UseGuards(JwtAuthGuard)
@Put(':id')
async update(
@User() user,
@Param('id') dataSourceId,
@Query('environment_id') environmentId,
@Body() updateDataSourceDto: UpdateDataSourceDto
) {
const ability = await this.globalDataSourceAbilityFactory.globalDataSourceActions(user);
if (!ability.can('updateGlobalDataSource', DataSource)) {
throw new ForbiddenException('You do not have permissions to perform this action');
}
const { name, options } = updateDataSourceDto;
await this.dataSourcesService.update(dataSourceId, user.organizationId, name, options, environmentId);
return;
}
@UseGuards(JwtAuthGuard)
@Delete(':id')
async delete(@User() user, @Param('id') dataSourceId) {
const ability = await this.globalDataSourceAbilityFactory.globalDataSourceActions(user);
if (!ability.can('deleteGlobalDataSource', DataSource)) {
throw new ForbiddenException('You do not have permissions to perform this action');
}
const result = await this.dataSourcesService.delete(dataSourceId);
if (result.affected == 1) {
return;
} else {
throw new BadRequestException();
}
}
@UseGuards(JwtAuthGuard)
@Post(':id/scope')
async convertToGlobal(@User() user, @Param('id') dataSourceId) {
const ability = await this.globalDataSourceAbilityFactory.globalDataSourceActions(user);
if (!ability.can('updateGlobalDataSource', DataSource)) {
throw new ForbiddenException('You do not have permissions to perform this action');
}
await this.dataSourcesService.convertToGlobalSource(dataSourceId, user.organizationId);
return;
}
}

View file

@ -5,6 +5,7 @@ import { PartialType } from '@nestjs/mapped-types';
export class CreateDataSourceDto {
@IsUUID()
@IsOptional()
app_version_id: string;
@IsUUID()
@ -23,6 +24,9 @@ export class CreateDataSourceDto {
@IsDefined()
options: any;
@IsOptional()
scope: string;
}
export class UpdateDataSourceDto extends PartialType(CreateDataSourceDto) {}

View file

@ -8,16 +8,16 @@ import {
JoinColumn,
Unique,
} from 'typeorm';
import { AppVersion } from './app_version.entity';
import { Organization } from './organization.entity';
@Entity({ name: 'app_environments' })
@Unique(['appVersionId', 'name'])
@Unique(['name'])
export class AppEnvironment {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ name: 'app_version_id' })
appVersionId: string;
@Column({ name: 'organization_id' })
organizationId: string;
@Column({ name: 'name' })
name: string;
@ -31,7 +31,7 @@ export class AppEnvironment {
@UpdateDateColumn({ default: () => 'now()', name: 'updated_at' })
updatedAt: Date;
@ManyToOne(() => AppVersion, (appVersion) => appVersion.id)
@JoinColumn({ name: 'app_version_id' })
appVersion: AppVersion;
@ManyToOne(() => Organization, (organization) => organization.id)
@JoinColumn({ name: 'organization_id' })
organization: Organization;
}

View file

@ -8,12 +8,9 @@ import {
JoinColumn,
BaseEntity,
Unique,
ManyToMany,
JoinTable,
OneToMany,
} from 'typeorm';
import { App } from './app.entity';
import { AppEnvironment } from './app_environments.entity';
import { DataQuery } from './data_query.entity';
import { DataSource } from './data_source.entity';
@ -45,20 +42,6 @@ export class AppVersion extends BaseEntity {
@OneToMany(() => DataSource, (dataSource) => dataSource.appVersion)
dataSources: DataSource[];
@OneToMany(() => AppEnvironment, (appEnv) => appEnv.appVersion, { onDelete: 'CASCADE' })
appEnvironments: AppEnvironment[];
@ManyToMany(() => DataQuery)
@JoinTable({
name: 'data_sources',
joinColumn: {
name: 'app_version_id',
referencedColumnName: 'id',
},
inverseJoinColumn: {
name: 'id',
referencedColumnName: 'dataSourceId',
},
})
@OneToMany(() => DataQuery, (dataQuery) => dataQuery.appVersion)
dataQueries: DataQuery[];
}

View file

@ -11,6 +11,8 @@ import {
ManyToMany,
AfterLoad,
} from 'typeorm';
import { App } from './app.entity';
import { AppVersion } from './app_version.entity';
import { DataSource } from './data_source.entity';
import { Plugin } from './plugin.entity';
@ -28,6 +30,9 @@ export class DataQuery extends BaseEntity {
@Column({ name: 'data_source_id' })
dataSourceId: string;
@Column({ name: 'app_version_id' })
appVersionId: string;
@CreateDateColumn({ default: () => 'now()', name: 'created_at' })
createdAt: Date;
@ -38,6 +43,10 @@ export class DataQuery extends BaseEntity {
@JoinColumn({ name: 'data_source_id' })
dataSource: DataSource;
@ManyToOne(() => AppVersion, (appVersion) => appVersion.id, { onDelete: 'CASCADE' })
@JoinColumn({ name: 'app_version_id' })
appVersion: AppVersion;
@ManyToMany(() => Plugin)
@JoinTable({
name: 'data_sources',
@ -56,6 +65,22 @@ export class DataQuery extends BaseEntity {
kind: string;
@ManyToMany(() => App)
@JoinTable({
name: 'app_versions',
joinColumn: {
name: 'id',
referencedColumnName: 'appVersionId',
},
inverseJoinColumn: {
name: 'app_id',
referencedColumnName: 'id',
},
})
apps: App[];
app: App;
@AfterLoad()
updatePlugin() {
if (this.plugins?.length) this.plugin = this.plugins[0];
@ -65,4 +90,9 @@ export class DataQuery extends BaseEntity {
updateKind() {
this.kind = this.dataSource?.kind;
}
@AfterLoad()
updateApp() {
if (this.apps?.length) this.app = this.apps[0];
}
}

View file

@ -48,6 +48,9 @@ export class DataSource extends BaseEntity {
@Column({ name: 'organization_id' })
organizationId: string;
@Column({ type: 'enum', enumName: 'scope', enum: ['local', 'global'], default: 'local' })
scope: string;
@CreateDateColumn({ default: () => 'now()', name: 'created_at' })
createdAt: Date;

View file

@ -12,6 +12,7 @@ import { GroupPermission } from './group_permission.entity';
import { SSOConfigs } from './sso_config.entity';
import { OrganizationUser } from './organization_user.entity';
import { InternalTable } from './internal_table.entity';
import { AppEnvironment } from './app_environments.entity';
@Entity({ name: 'organizations' })
export class Organization extends BaseEntity {
@ -46,6 +47,10 @@ export class Organization extends BaseEntity {
@OneToMany(() => OrganizationUser, (organizationUser) => organizationUser.organization)
organizationUsers: OrganizationUser[];
@OneToMany(() => AppEnvironment, (appEnvironment) => appEnvironment.organization, { onDelete: 'CASCADE' })
@JoinColumn({ name: 'organization_id' })
appEnvironments: AppEnvironment[];
@OneToMany(() => InternalTable, (internalTable) => internalTable.organization)
internalTable: InternalTable[];
}

View file

@ -2,3 +2,8 @@ export enum DataSourceTypes {
STATIC = 'static',
DEFAULT = 'default',
}
export enum DataSourceScopes {
LOCAL = 'local',
GLOBAL = 'global',
}

View file

@ -88,3 +88,9 @@ export function validateDefaultValue(value: any, params: any) {
if (data_type === 'boolean') return value || 'false';
return value;
}
export async function dropForeignKey(tableName: string, columnName: string, queryRunner) {
const table = await queryRunner.getTable(tableName);
const foreignKey = table.foreignKeys.find((fk) => fk.columnNames.indexOf(columnName) !== -1);
await queryRunner.dropForeignKey(tableName, foreignKey);
}

View file

@ -7,7 +7,7 @@ import * as helmet from 'helmet';
import { Logger } from 'nestjs-pino';
import { urlencoded, json } from 'express';
import { AllExceptionsFilter } from './all-exceptions-filter';
import { RequestMethod, ValidationPipe } from '@nestjs/common';
import { RequestMethod, ValidationPipe, VersioningType, VERSION_NEUTRAL } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { bootstrap as globalAgentBootstrap } from 'global-agent';
import { join } from 'path';
@ -87,6 +87,11 @@ async function bootstrap() {
app.use(urlencoded({ extended: true, limit: '50mb', parameterLimit: 1000000 }));
app.useStaticAssets(join(__dirname, 'assets'), { prefix: (UrlPrefix ? UrlPrefix : '/') + 'assets' });
app.enableVersioning({
type: VersioningType.URI,
defaultVersion: VERSION_NEUTRAL,
});
const port = parseInt(process.env.PORT) || 3000;
await app.listen(port, '0.0.0.0', function () {

View file

@ -16,7 +16,7 @@ export class QueryAuthGuard extends AuthGuard('jwt') {
const apiUrl = maybeSetSubPath('/api/data_queries/:id/run');
if (request.route.path === apiUrl) {
const dataQuery = await this.dataQueriesService.findOne(request.params.id);
const app = dataQuery.dataSource.app;
const app = dataQuery.app;
if (app.isPublic === true) {
return true;

View file

@ -0,0 +1,46 @@
import { User } from 'src/entities/user.entity';
import { InferSubjects, AbilityBuilder, Ability, AbilityClass, ExtractSubjectType } from '@casl/ability';
import { Injectable } from '@nestjs/common';
import { UsersService } from 'src/services/users.service';
import { DataSource } from 'src/entities/data_source.entity';
type Actions =
| 'createGlobalDataSource'
| 'updateGlobalDataSource'
| 'deleteGlobalDataSource'
| 'authorizeOauthForSource';
type Subjects = InferSubjects<typeof User | typeof DataSource> | 'all';
export type GlobalDataSourcesAbility = Ability<[Actions, Subjects]>;
@Injectable()
export class GlobalDataSourceAbilityFactory {
constructor(private usersService: UsersService) {}
async globalDataSourceActions(user: User) {
const { can, build } = new AbilityBuilder<Ability<[Actions, Subjects]>>(
Ability as AbilityClass<GlobalDataSourcesAbility>
);
if (await this.usersService.userCan(user, 'create', 'GlobalDataSource')) {
can('createGlobalDataSource', DataSource);
}
if (await this.usersService.userCan(user, 'update', 'GlobalDataSource')) {
can('updateGlobalDataSource', DataSource);
}
if (await this.usersService.userCan(user, 'delete', 'GlobalDataSource')) {
can('deleteGlobalDataSource', DataSource);
}
if (await this.usersService.userCan(user, 'create', 'GlobalDataSource')) {
can('authorizeOauthForSource', DataSource);
}
return build({
detectSubjectType: (item) => item.constructor as ExtractSubjectType<Subjects>,
});
}
}

View file

@ -17,6 +17,7 @@ import { FoldersAbilityFactory } from './abilities/folders-ability.factory';
import { FilesService } from '@services/files.service';
import { OrgEnvironmentVariablesAbilityFactory } from './abilities/org-environment-variables-ability.factory';
import { TooljetDbAbilityFactory } from './abilities/tooljet-db-ability.factory';
import { GlobalDataSourceAbilityFactory } from './abilities/global-datasource-ability.factory';
@Module({
imports: [TypeOrmModule.forFeature([User, File, Organization, OrganizationUser, App])],
@ -33,6 +34,7 @@ import { TooljetDbAbilityFactory } from './abilities/tooljet-db-ability.factory'
FoldersAbilityFactory,
OrgEnvironmentVariablesAbilityFactory,
TooljetDbAbilityFactory,
GlobalDataSourceAbilityFactory,
],
exports: [
CaslAbilityFactory,
@ -43,6 +45,7 @@ import { TooljetDbAbilityFactory } from './abilities/tooljet-db-ability.factory'
FoldersAbilityFactory,
OrgEnvironmentVariablesAbilityFactory,
TooljetDbAbilityFactory,
GlobalDataSourceAbilityFactory,
],
})
export class CaslModule {}

View file

@ -1,6 +1,7 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { DataSourcesController } from '../../../src/controllers/data_sources.controller';
import { GlobalDataSourcesController } from '@controllers/global_data_sources.controller';
import { DataSourcesService } from '../../../src/services/data_sources.service';
import { DataSource } from '../../../src/entities/data_source.entity';
import { CredentialsService } from '../../../src/services/credentials.service';
@ -63,6 +64,6 @@ import { AppEnvironmentService } from '@services/app_environments.service';
PluginsHelper,
AppEnvironmentService,
],
controllers: [DataSourcesController],
controllers: [DataSourcesController, GlobalDataSourcesController],
})
export class DataSourcesModule {}

View file

@ -1,32 +1,32 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { AppEnvironment } from 'src/entities/app_environments.entity';
import { dbTransactionWrap } from 'src/helpers/utils.helper';
import { dbTransactionWrap, defaultAppEnvironments } from 'src/helpers/utils.helper';
import { DataSourceOptions } from 'src/entities/data_source_options.entity';
@Injectable()
export class AppEnvironmentService {
async get(appVersionId: string, id?: string, manager?: EntityManager): Promise<AppEnvironment> {
async get(organizationId: string, id?: string, manager?: EntityManager): Promise<AppEnvironment> {
return await dbTransactionWrap(async (manager: EntityManager) => {
if (!id) {
return await manager.findOneOrFail(AppEnvironment, { where: { appVersionId, isDefault: true } });
return await manager.findOneOrFail(AppEnvironment, { where: { organizationId, isDefault: true } });
}
return await manager.findOneOrFail(AppEnvironment, { where: { id, appVersionId } });
return await manager.findOneOrFail(AppEnvironment, { where: { id, organizationId } });
}, manager);
}
async getOptions(dataSourceId: string, versionId?: string, environmentId?: string): Promise<DataSourceOptions> {
async getOptions(dataSourceId: string, organizationId: string, environmentId?: string): Promise<DataSourceOptions> {
return await dbTransactionWrap(async (manager: EntityManager) => {
let envId: string = environmentId;
if (!environmentId) {
envId = (await this.get(versionId, null, manager)).id;
envId = (await this.get(organizationId, null, manager)).id;
}
return await manager.findOneOrFail(DataSourceOptions, { where: { environmentId: envId, dataSourceId } });
});
}
async create(
appVersionId: string,
organizationId: string,
name: string,
isDefault = false,
manager?: EntityManager
@ -36,7 +36,7 @@ export class AppEnvironmentService {
AppEnvironment,
manager.create(AppEnvironment, {
name,
appVersionId,
organizationId,
isDefault,
createdAt: new Date(),
updatedAt: new Date(),
@ -45,9 +45,9 @@ export class AppEnvironmentService {
}, manager);
}
async getAll(appVersionId: string, manager?: EntityManager): Promise<AppEnvironment[]> {
async getAll(organizationId: string, manager?: EntityManager): Promise<AppEnvironment[]> {
return await dbTransactionWrap(async (manager: EntityManager) => {
return await manager.find(AppEnvironment, { where: { appVersionId } });
return await manager.find(AppEnvironment, { where: { organizationId } });
}, manager);
}
@ -64,9 +64,26 @@ export class AppEnvironmentService {
}, manager);
}
async createDataSourceInAllEnvironments(appVersionId: string, dataSourceId: string, manager?: EntityManager) {
async createDefaultEnvironments(organizationId: string, manager?: EntityManager) {
await dbTransactionWrap(async (manager: EntityManager) => {
const allEnvs = await this.getAll(appVersionId, manager);
await Promise.all(
defaultAppEnvironments.map(async (en) => {
const env = manager.create(AppEnvironment, {
organizationId: organizationId,
name: en.name,
isDefault: en.isDefault,
createdAt: new Date(),
updatedAt: new Date(),
});
await manager.save(env);
})
);
}, manager);
}
async createDataSourceInAllEnvironments(organizationId: string, dataSourceId: string, manager?: EntityManager) {
await dbTransactionWrap(async (manager: EntityManager) => {
const allEnvs = await this.getAll(organizationId, manager);
const allEnvOptions = allEnvs.map((env) =>
manager.create(DataSourceOptions, {
environmentId: env.id,

View file

@ -14,7 +14,8 @@ import { AppEnvironment } from 'src/entities/app_environments.entity';
import { DataSourceOptions } from 'src/entities/data_source_options.entity';
import { AppEnvironmentService } from './app_environments.service';
import { convertAppDefinitionFromSinglePageToMultiPage } from '../../lib/single-page-to-and-from-multipage-definition-conversion';
import { DataSourceTypes } from 'src/helpers/data_source.constants';
import { DataSourceScopes, DataSourceTypes } from 'src/helpers/data_source.constants';
import { Organization } from 'src/entities/organization.entity';
@Injectable()
export class AppImportExportService {
@ -61,8 +62,8 @@ export class AppImportExportService {
const appEnvironments = await manager
.createQueryBuilder(AppEnvironment, 'app_environments')
.where('app_environments.appVersionId IN(:...versionId)', {
versionId: appVersions.map((v) => v.id),
.where('app_environments.organizationId = :organizationId', {
organizationId: user.organizationId,
})
.orderBy('app_environments.createdAt', 'ASC')
.getMany();
@ -70,6 +71,15 @@ export class AppImportExportService {
let dataQueries: DataQuery[] = [];
let dataSourceOptions: DataSourceOptions[] = [];
const globalQueries: DataQuery[] = await manager
.createQueryBuilder(DataQuery, 'data_query')
.leftJoinAndSelect('data_query.dataSource', 'dataSource')
.where('data_query.appVersionId IN(:...versionId)', {
versionId: appVersions.map((v) => v.id),
})
.andWhere('dataSource.scope = :scope', { scope: DataSourceScopes.GLOBAL })
.getMany();
if (dataSources?.length) {
dataQueries = await manager
.createQueryBuilder(DataQuery, 'data_queries')
@ -88,6 +98,10 @@ export class AppImportExportService {
.getMany();
}
if (globalQueries?.length) {
dataQueries = [...dataQueries, ...globalQueries];
}
appToExport['dataQueries'] = dataQueries;
appToExport['dataSources'] = dataSources;
appToExport['appVersions'] = appVersions;
@ -125,7 +139,7 @@ export class AppImportExportService {
await dbTransactionWrap(async (manager) => {
importedApp = await this.createImportedAppForUser(manager, schemaUnifiedAppParams, user);
await this.buildImportedAppAssociations(manager, importedApp, schemaUnifiedAppParams);
await this.buildImportedAppAssociations(manager, importedApp, schemaUnifiedAppParams, user);
await this.createAdminGroupPermissions(manager, importedApp);
});
@ -153,7 +167,7 @@ export class AppImportExportService {
return importedApp;
}
async buildImportedAppAssociations(manager: EntityManager, importedApp: App, appParams: any) {
async buildImportedAppAssociations(manager: EntityManager, importedApp: App, appParams: any, user: User) {
const dataSourceMapping = {};
const defaultDataSourceIdMapping = {};
const dataQueryMapping = {};
@ -183,23 +197,34 @@ export class AppImportExportService {
await manager.update(App, importedApp, { currentVersionId: version.id });
// Create default data sources
const defaultDataSourceIds = await this.createDefaultDataSourceForVersion(version.id, [], manager);
const envIdArray = [];
await Promise.all(
defaultAppEnvironments.map(async (en) => {
const env = manager.create(AppEnvironment, {
appVersionId: version.id,
name: en.name,
isDefault: en.isDefault,
createdAt: new Date(),
updatedAt: new Date(),
});
await manager.save(env);
envIdArray.push(env.id);
})
const defaultDataSourceIds = await this.createDefaultDataSourceForVersion(
user.organizationId,
version.id,
[],
manager
);
let envIdArray: string[] = [];
const organization: Organization = await manager.findOne(Organization, {
where: { id: user.organizationId },
relations: ['appEnvironments'],
});
envIdArray = [...organization.appEnvironments.map((env) => env.id)];
if (!envIdArray.length) {
await Promise.all(
defaultAppEnvironments.map(async (en) => {
const env = manager.create(AppEnvironment, {
organizationId: user.organizationId,
name: en.name,
isDefault: en.isDefault,
createdAt: new Date(),
updatedAt: new Date(),
});
await manager.save(env);
envIdArray.push(env.id);
})
);
}
for (const source of dataSources) {
const convertedOptions = this.convertToArrayOfKeyValuePairs(source.options);
@ -237,6 +262,7 @@ export class AppImportExportService {
name: query.name,
options: query.options,
dataSourceId: !dataSourceId ? defaultDataSourceIds[query.kind] : dataSourceId,
appVersionId: query.appVersionId,
});
await manager.save(newQuery);
dataQueryMapping[query.id] = newQuery.id;
@ -270,25 +296,14 @@ export class AppImportExportService {
});
await manager.save(version);
if (!appEnvironments?.length) {
// v1
const envIdArray = [];
await Promise.all(
defaultAppEnvironments.map(async (en) => {
const env = manager.create(AppEnvironment, {
appVersionId: version.id,
name: en.name,
isDefault: en.isDefault,
createdAt: new Date(),
updatedAt: new Date(),
});
await manager.save(env);
envIdArray.push(env.id);
})
);
let envIdArray: string[] = [];
const organization: Organization = await manager.findOne(Organization, {
where: { id: user.organizationId },
relations: ['appEnvironments'],
});
envIdArray = [...organization.appEnvironments.map((env) => env.id)];
appDefaultEnvironmentMapping[appVersion.id] = envIdArray;
}
appDefaultEnvironmentMapping[appVersion.id] = envIdArray;
if (appVersion.id == appParams.currentVersionId) {
currentVersionId = version.id;
@ -299,17 +314,28 @@ export class AppImportExportService {
// associate App environments for each of the app versions
for (const appVersion of appVersions) {
for (const appEnvironment of appEnvironments?.filter((ae) => ae.appVersionId === appVersion.id)) {
const env = manager.create(AppEnvironment, {
appVersionId: appVersionMapping[appEnvironment.appVersionId],
name: appEnvironment.name,
isDefault: appEnvironment.isDefault,
createdAt: new Date(),
updatedAt: new Date(),
});
await manager.save(env);
const currentOrgEnvironments = await this.appEnvironmentService.getAll(user.organizationId, manager);
appEnvironmentMapping[appEnvironment.id] = env.id;
if (!appEnvironments?.length) {
currentOrgEnvironments.map((env) => (appEnvironmentMapping[env.id] = env.id));
} else {
//For apps imported on v2 where organizationId not available
for (const currentOrgEnv of currentOrgEnvironments) {
const appEnvironment = appEnvironments.filter((appEnv) => appEnv.name === currentOrgEnv.name)[0];
if (appEnvironment) {
appEnvironmentMapping[appEnvironment.id] = currentOrgEnv.id;
} else {
const env = manager.create(AppEnvironment, {
organizationId: user.organizationId,
name: currentOrgEnv.name,
isDefault: currentOrgEnv.isDefault,
createdAt: new Date(),
updatedAt: new Date(),
});
await manager.save(env);
appEnvironmentMapping[env.id] = env.id;
}
}
}
const dsKindsToCreate = [];
@ -333,6 +359,7 @@ export class AppImportExportService {
if (dsKindsToCreate.length > 0) {
// Create default data sources
defaultDataSourceIdMapping[appVersion.id] = await this.createDefaultDataSourceForVersion(
user.organizationId,
appVersionMapping[appVersion.id],
dsKindsToCreate,
manager
@ -341,6 +368,9 @@ export class AppImportExportService {
let dataSourcesToIterate = dataSources; // 0.9.0 -> add all data sources & queries to all versions
let dataQueriesToIterate = dataQueries;
const globalQueriesToIterate = dataQueries?.filter(
(dq) => dq.dataSource?.scope === DataSourceScopes.GLOBAL && dq.appVersionId === appVersion.id
);
if (dataSources[0]?.appVersionId || dataQueries[0]?.appVersionId) {
// v1 - Data queries without dataSourceId present
@ -365,7 +395,6 @@ export class AppImportExportService {
await Promise.all(
appDefaultEnvironmentMapping[appVersion.id].map(async (envId) => {
const newOptions = await this.dataSourcesService.parseOptionsForCreate(convertedOptions, true, manager);
const dsOption = manager.create(DataSourceOptions, {
environmentId: envId,
dataSourceId: newSource.id,
@ -378,25 +407,27 @@ export class AppImportExportService {
);
}
for (const dataSourceOption of dataSourceOptions?.filter((dso) => dso.dataSourceId === source.id)) {
const convertedOptions = this.convertToArrayOfKeyValuePairs(dataSourceOption.options);
const newOptions = await this.dataSourcesService.parseOptionsForCreate(convertedOptions, true, manager);
const dsOption = manager.create(DataSourceOptions, {
options: newOptions,
environmentId: appEnvironmentMapping[dataSourceOption.environmentId],
dataSourceId: newSource.id,
createdAt: new Date(),
updatedAt: new Date(),
});
await manager.save(dsOption);
for (const dataSourceOption of dataSourceOptions.filter((dso) => dso.dataSourceId === source.id)) {
if (dataSourceOption?.environmentId in appEnvironmentMapping) {
const convertedOptions = this.convertToArrayOfKeyValuePairs(dataSourceOption.options);
const newOptions = await this.dataSourcesService.parseOptionsForCreate(convertedOptions, true, manager);
const dsOption = manager.create(DataSourceOptions, {
options: newOptions,
environmentId: appEnvironmentMapping[dataSourceOption.environmentId],
dataSourceId: newSource.id,
createdAt: new Date(),
updatedAt: new Date(),
});
await manager.save(dsOption);
}
}
for (const query of dataQueries?.filter((dq) => dq.dataSourceId === source.id)) {
for (const query of dataQueries.filter((dq) => dq.dataSourceId === source.id)) {
const newQuery = manager.create(DataQuery, {
name: query.name,
options: query.options,
dataSourceId: newSource.id,
appVersionId: appVersionMapping[appVersion.id],
});
await manager.save(newQuery);
dataQueryMapping[query.id] = newQuery.id;
@ -410,6 +441,19 @@ export class AppImportExportService {
name: query.name,
options: query.options,
dataSourceId: defaultDataSourceIdMapping[appVersion.id][query.kind],
appVersionId: appVersionMapping[appVersion.id],
});
await manager.save(newQuery);
dataQueryMapping[query.id] = newQuery.id;
newDataQueries.push(newQuery);
}
for (const query of globalQueriesToIterate) {
const newQuery = manager.create(DataQuery, {
name: query.name,
options: query.options,
dataSourceId: query.dataSourceId,
appVersionId: appVersionMapping[appVersion.id],
});
await manager.save(newQuery);
dataQueryMapping[query.id] = newQuery.id;
@ -435,6 +479,7 @@ export class AppImportExportService {
}
async createDefaultDataSourceForVersion(
organizationId: string,
versionId: string,
kinds: string[] = ['restapi', 'runjs', 'tooljetdb'],
manager: EntityManager
@ -444,7 +489,7 @@ export class AppImportExportService {
for (const defaultSource of kinds) {
const dataSource = await this.dataSourcesService.createDefaultDataSource(defaultSource, versionId, null, manager);
response[defaultSource] = dataSource.id;
await this.appEnvironmentService.createDataSourceInAllEnvironments(versionId, dataSource.id, manager);
await this.appEnvironmentService.createDataSourceInAllEnvironments(organizationId, dataSource.id, manager);
}
return response;
}

View file

@ -20,6 +20,7 @@ import { AppEnvironment } from 'src/entities/app_environments.entity';
import { DataSourceOptions } from 'src/entities/data_source_options.entity';
import { AppEnvironmentService } from './app_environments.service';
import { decode } from 'js-base64';
import { DataSourceScopes } from 'src/helpers/data_source.constants';
@Injectable()
export class AppsService {
@ -90,7 +91,7 @@ export class AppsService {
.createQueryBuilder(DataQuery, 'data_query')
.innerJoin('data_query.dataSource', 'data_source')
.addSelect('data_source.kind')
.where('data_source.appVersionId = :appVersionId', { appVersionId })
.where('data_query.appVersionId = :appVersionId', { appVersionId })
.getMany();
});
}
@ -254,10 +255,11 @@ export class AppsService {
): Promise<AppVersion> {
return await dbTransactionWrap(async (manager: EntityManager) => {
let versionFrom: AppVersion;
const { organizationId } = user;
if (versionFromId) {
versionFrom = await manager.findOneOrFail(AppVersion, {
where: { id: versionFromId },
relations: ['appEnvironments', 'dataSources', 'dataSources.dataQueries', 'dataSources.dataSourceOptions'],
relations: ['dataSources', 'dataSources.dataQueries', 'dataSources.dataSourceOptions'],
});
}
@ -286,7 +288,7 @@ export class AppsService {
})
);
await this.createNewDataSourcesAndQueriesForVersion(manager, appVersion, versionFrom);
await this.createNewDataSourcesAndQueriesForVersion(manager, appVersion, versionFrom, organizationId);
return appVersion;
}, manager);
}
@ -307,12 +309,19 @@ export class AppsService {
async createNewDataSourcesAndQueriesForVersion(
manager: EntityManager,
appVersion: AppVersion,
versionFrom: AppVersion
versionFrom: AppVersion,
organizationId: string
) {
const oldDataQueryToNewMapping = {};
let appEnvironments: AppEnvironment[] = await this.appEnvironmentService.getAll(organizationId, manager);
if (!appEnvironments?.length) {
await this.createEnvironments(defaultAppEnvironments, manager, organizationId);
appEnvironments = await this.appEnvironmentService.getAll(organizationId, manager);
}
if (!versionFrom) {
await this.createEnvironments(defaultAppEnvironments, manager, appVersion);
//create default data sources
for (const defaultSource of ['restapi', 'runjs', 'tooljetdb']) {
const dataSource = await this.dataSourcesService.createDefaultDataSource(
@ -321,14 +330,18 @@ export class AppsService {
null,
manager
);
await this.appEnvironmentService.createDataSourceInAllEnvironments(appVersion.id, dataSource.id, manager);
await this.appEnvironmentService.createDataSourceInAllEnvironments(organizationId, dataSource.id, manager);
}
} else {
const appEnvironments: AppEnvironment[] = versionFrom?.appEnvironments;
const globalQueries: DataQuery[] = await manager
.createQueryBuilder(DataQuery, 'data_query')
.leftJoinAndSelect('data_query.dataSource', 'dataSource')
.where('data_query.appVersionId = :appVersionId', { appVersionId: versionFrom?.id })
.andWhere('dataSource.scope = :scope', { scope: DataSourceScopes.GLOBAL })
.getMany();
const dataSources = versionFrom?.dataSources;
const dataSourceMapping = {};
const newDataQueries = [];
if (dataSources?.length && appEnvironments?.length) {
if (dataSources?.length) {
for (const dataSource of dataSources) {
const dataSourceParams: Partial<DataSource> = {
name: dataSource.name,
@ -340,27 +353,54 @@ export class AppsService {
dataSourceMapping[dataSource.id] = newDataSource.id;
const dataQueries = versionFrom?.dataSources?.find((ds) => ds.id === dataSource.id).dataQueries;
const newDataQueries = [];
for (const dataQuery of dataQueries) {
const dataQueryParams = {
name: dataQuery.name,
options: dataQuery.options,
dataSourceId: newDataSource.id,
appVersionId: appVersion.id,
};
const newQuery = await manager.save(manager.create(DataQuery, dataQueryParams));
oldDataQueryToNewMapping[dataQuery.id] = newQuery.id;
newDataQueries.push(newQuery);
}
for (const newQuery of newDataQueries) {
const newOptions = this.replaceDataQueryOptionsWithNewDataQueryIds(
newQuery.options,
oldDataQueryToNewMapping
);
newQuery.options = newOptions;
await manager.save(newQuery);
}
}
for (const newQuery of newDataQueries) {
const newOptions = this.replaceDataQueryOptionsWithNewDataQueryIds(
newQuery.options,
oldDataQueryToNewMapping
);
newQuery.options = newOptions;
await manager.save(newQuery);
if (globalQueries?.length) {
for (const globalQuery of globalQueries) {
const dataQueryParams = {
name: globalQuery.name,
options: globalQuery.options,
dataSourceId: globalQuery.dataSourceId,
appVersionId: appVersion.id,
};
const newDataQueries = [];
const newQuery = await manager.save(manager.create(DataQuery, dataQueryParams));
oldDataQueryToNewMapping[globalQuery.id] = newQuery.id;
newDataQueries.push(newQuery);
for (const newQuery of newDataQueries) {
const newOptions = this.replaceDataQueryOptionsWithNewDataQueryIds(
newQuery.options,
oldDataQueryToNewMapping
);
newQuery.options = newOptions;
await manager.save(newQuery);
}
}
}
appVersion.definition = this.replaceDataQueryIdWithinDefinitions(
@ -370,12 +410,6 @@ export class AppsService {
await manager.save(appVersion);
for (const appEnvironment of appEnvironments) {
const newAppEnvironment = await this.appEnvironmentService.create(
appVersion.id,
appEnvironment.name,
appEnvironment.isDefault,
manager
);
for (const dataSource of dataSources) {
const dataSourceOption = await manager.findOneOrFail(DataSourceOptions, {
where: { dataSourceId: dataSource.id, environmentId: appEnvironment.id },
@ -389,20 +423,18 @@ export class AppsService {
manager.create(DataSourceOptions, {
options: newOptions,
dataSourceId: dataSourceMapping[dataSource.id],
environmentId: newAppEnvironment.id,
environmentId: appEnvironment.id,
})
);
}
}
} else {
await this.createEnvironments(appEnvironments, manager, appVersion);
}
}
}
private async createEnvironments(appEnvironments: any[], manager: EntityManager, appVersion: AppVersion) {
private async createEnvironments(appEnvironments: any[], manager: EntityManager, organizationId: string) {
for (const appEnvironment of appEnvironments) {
await this.appEnvironmentService.create(appVersion.id, appEnvironment.name, appEnvironment.isDefault, manager);
await this.appEnvironmentService.create(organizationId, appEnvironment.name, appEnvironment.isDefault, manager);
}
}

View file

@ -14,6 +14,7 @@ import { EncryptionService } from './encryption.service';
import { App } from 'src/entities/app.entity';
import { AppEnvironmentService } from './app_environments.service';
import { dbTransactionWrap } from 'src/helpers/utils.helper';
import { DataSourceScopes } from 'src/helpers/data_source.constants';
@Injectable()
export class DataQueriesService {
@ -32,7 +33,7 @@ export class DataQueriesService {
async findOne(dataQueryId: string): Promise<DataQuery> {
return await this.dataQueriesRepository.findOne({
where: { id: dataQueryId },
relations: ['dataSource', 'dataSource.apps', 'plugins'],
relations: ['dataSource', 'apps', 'dataSource.apps', 'plugins'],
});
}
@ -47,16 +48,24 @@ export class DataQueriesService {
.leftJoinAndSelect('plugins.iconFile', 'iconFile')
.leftJoinAndSelect('plugins.manifestFile', 'manifestFile')
.where('data_source.appVersionId = :appVersionId', { appVersionId })
.where('data_query.app_version_id = :appVersionId', { appVersionId })
.orderBy('data_query.createdAt', 'DESC')
.getMany();
});
}
async create(name: string, options: object, dataSourceId: string, manager: EntityManager): Promise<DataQuery> {
async create(
name: string,
options: object,
dataSourceId: string,
appVersionId: string,
manager: EntityManager
): Promise<DataQuery> {
const newDataQuery = manager.create(DataQuery, {
name,
options,
dataSourceId,
appVersionId,
createdAt: new Date(),
updatedAt: new Date(),
});
@ -102,18 +111,15 @@ export class DataQueriesService {
async runQuery(user: User, dataQuery: any, queryOptions: object, environmentId?: string): Promise<object> {
const dataSource: DataSource = dataQuery?.dataSource;
const app: App = dataSource?.app;
const app: App = dataQuery?.app;
if (!(dataSource && app)) {
throw new UnauthorizedException();
}
const dataSourceOptions = await this.appEnvironmentService.getOptions(
dataSource.id,
dataSource.appVersionId,
environmentId
);
const organizationId = user ? user.organizationId : app.organizationId;
const dataSourceOptions = await this.appEnvironmentService.getOptions(dataSource.id, organizationId, environmentId);
dataSource.options = dataSourceOptions.options;
const organizationId = user ? user.organizationId : app.organizationId;
let { sourceOptions, parsedQueryOptions, service } = await this.fetchServiceAndParsedParams(
dataSource,
dataQuery,
@ -190,11 +196,12 @@ export class DataQueriesService {
dataSource.options,
dataSource.id,
user?.id,
user?.organizationId,
environmentId
);
const dataSourceOptions = await this.appEnvironmentService.getOptions(
dataSource.id,
dataSource.appVersionId,
user.organizationId,
environmentId
);
dataSource.options = dataSourceOptions.options;
@ -322,7 +329,13 @@ export class DataQueriesService {
};
/* This function fetches access token from authorization code */
async authorizeOauth2(dataSource: DataSource, code: string, userId: string, environmentId?: string): Promise<void> {
async authorizeOauth2(
dataSource: DataSource,
code: string,
userId: string,
environmentId?: string,
organizationId?: string
): Promise<void> {
const sourceOptions = await this.parseSourceOptions(dataSource.options);
const isMultiAuthEnabled = dataSource.options['multiple_auth_enabled']?.value;
const newToken = await this.fetchOAuthToken(sourceOptions, code, userId, isMultiAuthEnabled);
@ -341,7 +354,7 @@ export class DataQueriesService {
},
];
await this.dataSourcesService.updateOptions(dataSource.id, tokenOptions, environmentId);
await this.dataSourcesService.updateOptions(dataSource.id, tokenOptions, organizationId, environmentId);
return;
}
@ -442,4 +455,25 @@ export class DataQueriesService {
}
return object;
}
async changeQueryDataSource(queryId: string, dataSourceId: string) {
return await dbTransactionWrap(async (manager: EntityManager) => {
return await manager.save(DataQuery, {
id: queryId,
dataSourceId: dataSourceId,
updatedAt: new Date(),
});
});
}
async getGlobalQueriesByAppVersion(appVersionId: string, manager: EntityManager) {
return await dbTransactionWrap(async (manager: EntityManager) => {
return await manager
.createQueryBuilder(DataQuery, 'data_query')
.leftJoinAndSelect('data_query.dataSource', 'dataSource')
.where('data_query.appVersionId = :appVersionId', { appVersionId })
.andWhere('dataSource.scope = :scope', { scope: DataSourceScopes.GLOBAL })
.getMany();
}, manager);
}
}

View file

@ -1,5 +1,5 @@
import allPlugins from '@tooljet/plugins/dist/server';
import { Injectable, NotImplementedException } from '@nestjs/common';
import { Injectable, NotAcceptableException, NotImplementedException } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { EntityManager, getManager, Repository } from 'typeorm';
import { DataSource } from '../../src/entities/data_source.entity';
@ -8,8 +8,7 @@ import { cleanObject, dbTransactionWrap } from 'src/helpers/utils.helper';
import { PluginsHelper } from '../helpers/plugins.helper';
import { AppEnvironmentService } from './app_environments.service';
import { App } from 'src/entities/app.entity';
import { AppEnvironment } from 'src/entities/app_environments.entity';
import { DataSourceTypes } from 'src/helpers/data_source.constants';
import { DataSourceScopes, DataSourceTypes } from 'src/helpers/data_source.constants';
@Injectable()
export class DataSourcesService {
@ -21,15 +20,20 @@ export class DataSourcesService {
private dataSourcesRepository: Repository<DataSource>
) {}
async all(query: object): Promise<DataSource[]> {
async all(
query: object,
organizationId: string,
scope: DataSourceScopes = DataSourceScopes.LOCAL
): Promise<DataSource[]> {
const { app_version_id: appVersionId, environmentId }: any = query;
let selectedEnvironmentId = environmentId;
return await dbTransactionWrap(async (manager: EntityManager) => {
if (!environmentId) {
selectedEnvironmentId = (await this.appEnvironmentService.get(appVersionId, null, manager))?.id;
selectedEnvironmentId = (await this.appEnvironmentService.get(organizationId, null, manager))?.id;
}
const result = await manager
const query = await manager
.createQueryBuilder(DataSource, 'data_source')
.innerJoinAndSelect('data_source.dataSourceOptions', 'data_source_options')
.leftJoinAndSelect('data_source.plugin', 'plugin')
@ -37,9 +41,19 @@ export class DataSourcesService {
.leftJoinAndSelect('plugin.manifestFile', 'manifestFile')
.leftJoinAndSelect('plugin.operationsFile', 'operationsFile')
.where('data_source_options.environmentId = :selectedEnvironmentId', { selectedEnvironmentId })
.andWhere('data_source.appVersionId = :appVersionId', { appVersionId })
.andWhere('data_source.type != :staticType', { staticType: DataSourceTypes.STATIC })
.getMany();
.andWhere('data_source.type != :staticType', { staticType: DataSourceTypes.STATIC });
if (scope === DataSourceScopes.GLOBAL) {
query
.andWhere('data_source.organization_id = :organizationId', { organizationId })
.andWhere('data_source.scope = :scope', { scope: DataSourceScopes.GLOBAL });
} else {
query
.andWhere('data_source.appVersionId = :appVersionId', { appVersionId })
.andWhere('data_source.scope = :scope', { scope: DataSourceScopes.LOCAL });
}
const result = await query.getMany();
//remove tokenData from restapi datasources
const dataSources = result?.map((ds) => {
@ -62,26 +76,39 @@ export class DataSourcesService {
});
}
async findOne(dataSourceId: string): Promise<DataSource> {
return await this.dataSourcesRepository.findOneOrFail({
where: { id: dataSourceId },
relations: ['plugin', 'apps', 'dataSourceOptions'],
});
async findOne(dataSourceId: string, manager?: EntityManager): Promise<DataSource> {
return await dbTransactionWrap(async (manager: EntityManager) => {
return await manager.findOneOrFail(DataSource, {
where: { id: dataSourceId },
relations: ['plugin', 'apps', 'dataSourceOptions'],
});
}, manager);
}
async findOneByEnvironment(dataSourceId: string, environmentId?: string): Promise<DataSource> {
const dataSource = await this.dataSourcesRepository.findOneOrFail({
where: { id: dataSourceId },
relations: ['plugin', 'apps', 'dataSourceOptions'],
relations: ['plugin', 'apps', 'dataSourceOptions', 'appVersion', 'appVersion.app'],
});
if (!environmentId) {
const env = await this.appEnvironmentService.get(dataSource.appVersionId, environmentId);
environmentId = env?.id;
const dsOrganizationId = dataSource.organizationId || dataSource.appVersion.app.organizationId;
if (!environmentId && dataSource.dataSourceOptions?.length > 1) {
//fix for env id issue when importing cloud/enterprise apps to CE
if (dataSource.dataSourceOptions?.length > 1) {
const env = await this.appEnvironmentService.get(dsOrganizationId, null);
environmentId = env?.id;
} else {
throw new NotAcceptableException('Environment id should not be empty');
}
}
if (environmentId) {
dataSource.options = (await this.appEnvironmentService.getOptions(dataSourceId, null, environmentId)).options;
dataSource.options = (
await this.appEnvironmentService.getOptions(dataSourceId, dsOrganizationId, environmentId)
).options;
} else {
dataSource.options = dataSource.dataSourceOptions?.[0]?.options || {};
}
return dataSource;
}
@ -96,13 +123,10 @@ export class DataSourcesService {
).app;
}
async findDefaultDataSourceByKind(kind: string, appVersionId?: string, environmentId?: string) {
async findDefaultDataSourceByKind(kind: string, appVersionId: string) {
return await dbTransactionWrap(async (manager: EntityManager) => {
const currentEnv = environmentId
? await manager.findOneOrFail(AppEnvironment, { where: { id: environmentId } })
: await manager.findOneOrFail(AppEnvironment, { where: { isDefault: true, appVersionId } });
return await manager.findOneOrFail(DataSource, {
where: { kind, appVersionId: currentEnv.appVersionId, type: DataSourceTypes.STATIC },
where: { kind, appVersionId: appVersionId, type: DataSourceTypes.STATIC },
relations: ['plugin', 'apps'],
});
});
@ -112,6 +136,7 @@ export class DataSourcesService {
kind: string,
appVersionId: string,
pluginId: string,
organizationId: string,
manager: EntityManager
): Promise<DataSource> {
const defaultDataSource = await manager.findOne(DataSource, {
@ -122,7 +147,7 @@ export class DataSourcesService {
return defaultDataSource;
}
const dataSource = await this.createDefaultDataSource(kind, appVersionId, pluginId, manager);
await this.appEnvironmentService.createDataSourceInAllEnvironments(appVersionId, dataSource.id, manager);
await this.appEnvironmentService.createDataSourceInAllEnvironments(organizationId, dataSource.id, manager);
return dataSource;
}
@ -148,7 +173,9 @@ export class DataSourcesService {
name: string,
kind: string,
options: Array<object>,
appVersionId: string,
appVersionId?: string,
organizationId?: string,
scope: string = DataSourceScopes.LOCAL,
pluginId?: string,
environmentId?: string
): Promise<DataSource> {
@ -158,16 +185,18 @@ export class DataSourcesService {
kind,
appVersionId,
pluginId,
organizationId,
scope,
createdAt: new Date(),
updatedAt: new Date(),
});
const dataSource = await manager.save(newDataSource);
// Creating empty options mapping
await this.appEnvironmentService.createDataSourceInAllEnvironments(appVersionId, dataSource.id, manager);
await this.appEnvironmentService.createDataSourceInAllEnvironments(organizationId, dataSource.id, manager);
// Find the environment to be updated
const envToUpdate = await this.appEnvironmentService.get(appVersionId, environmentId, manager);
const envToUpdate = await this.appEnvironmentService.get(organizationId, environmentId, manager);
await this.appEnvironmentService.updateOptions(
await this.parseOptionsForCreate(options, false, manager),
@ -177,7 +206,7 @@ export class DataSourcesService {
);
// Find other environments to be updated
const allEnvs = await this.appEnvironmentService.getAll(appVersionId, manager);
const allEnvs = await this.appEnvironmentService.getAll(organizationId, manager);
if (allEnvs?.length) {
const envsToUpdate = allEnvs.filter((env) => env.id !== envToUpdate.id);
@ -196,11 +225,17 @@ export class DataSourcesService {
});
}
async update(dataSourceId: string, name: string, options: Array<object>, environmentId?: string): Promise<void> {
async update(
dataSourceId: string,
organizationId: string,
name: string,
options: Array<object>,
environmentId?: string
): Promise<void> {
const dataSource = await this.findOne(dataSourceId);
await dbTransactionWrap(async (manager: EntityManager) => {
const envToUpdate = await this.appEnvironmentService.get(dataSource.appVersionId, environmentId, manager);
const envToUpdate = await this.appEnvironmentService.get(organizationId, environmentId, manager);
// if datasource is restapi then reset the token data
if (dataSource.kind === 'restapi')
@ -210,7 +245,9 @@ export class DataSourcesService {
encrypted: false,
});
dataSource.options = (await this.appEnvironmentService.getOptions(dataSourceId, null, envToUpdate.id)).options;
dataSource.options = (
await this.appEnvironmentService.getOptions(dataSourceId, organizationId, envToUpdate.id)
).options;
await this.appEnvironmentService.updateOptions(
await this.parseOptionsForUpdate(dataSource, options),
@ -236,11 +273,16 @@ export class DataSourcesService {
}
/* This function merges new options with the existing options */
async updateOptions(dataSourceId: string, optionsToMerge: any, environmentId?: string): Promise<void> {
async updateOptions(
dataSourceId: string,
optionsToMerge: any,
organizationId: string,
environmentId?: string
): Promise<void> {
await dbTransactionWrap(async (manager: EntityManager) => {
const dataSource = await manager.findOneOrFail(DataSource, dataSourceId, { relations: ['dataSourceOptions'] });
const parsedOptions = await this.parseOptionsForUpdate(dataSource, optionsToMerge);
const envToUpdate = await this.appEnvironmentService.get(dataSource.appVersionId, environmentId, manager);
const envToUpdate = await this.appEnvironmentService.get(organizationId, environmentId, manager);
const oldOptions = dataSource.dataSourceOptions?.[0]?.options || {};
const updatedOptions = { ...oldOptions, ...parsedOptions };
@ -391,6 +433,7 @@ export class DataSourcesService {
dataSourceOptions: object,
dataSourceId: string,
userId: string,
organizationId: string,
environmentId?: string
) {
const existingAccessTokenCredentialId =
@ -418,10 +461,22 @@ export class DataSourcesService {
encrypted: false,
},
];
await this.updateOptions(dataSourceId, tokenOptions, environmentId);
await this.updateOptions(dataSourceId, tokenOptions, organizationId, environmentId);
}
}
async convertToGlobalSource(datasourceId: string, organizationId: string) {
return await dbTransactionWrap(async (manager: EntityManager) => {
return await manager.save(DataSource, {
id: datasourceId,
updatedAt: new Date(),
appVersionId: null,
organizationId,
scope: DataSourceScopes.GLOBAL,
});
});
}
getAuthUrl(provider: string, sourceOptions?: any): { url: string } {
const service = new allPlugins[provider]();
return { url: service.authUrl(sourceOptions) };

View file

@ -24,6 +24,7 @@ import {
} from 'src/helpers/user_lifecycle';
import { decamelize } from 'humps';
import { Response } from 'express';
import { AppEnvironmentService } from './app_environments.service';
const MAX_ROW_COUNT = 500;
@ -56,6 +57,7 @@ export class OrganizationsService {
private usersService: UsersService,
private organizationUserService: OrganizationUsersService,
private groupPermissionService: GroupPermissionsService,
private appEnvironmentService: AppEnvironmentService,
private encryptionService: EncryptionService,
private emailService: EmailService,
private configService: ConfigService
@ -78,6 +80,8 @@ export class OrganizationsService {
})
);
await this.appEnvironmentService.createDefaultEnvironments(organization.id, manager);
const createdGroupPermissions: GroupPermission[] = await this.createDefaultGroupPermissionsForOrganization(
organization,
manager

View file

@ -4,8 +4,10 @@ import { User } from '../entities/user.entity';
import { Organization } from '../entities/organization.entity';
import { OrganizationUser } from '../entities/organization_user.entity';
import { GroupPermission } from 'src/entities/group_permission.entity';
import { AppEnvironment } from 'src/entities/app_environments.entity';
import { UserGroupPermission } from 'src/entities/user_group_permission.entity';
import { USER_STATUS, WORKSPACE_USER_STATUS } from 'src/helpers/user_lifecycle';
import { defaultAppEnvironments } from 'src/helpers/utils.helper';
@Injectable()
export class SeedsService {
@ -53,6 +55,8 @@ export class SeedsService {
status: WORKSPACE_USER_STATUS.ACTIVE,
});
await this.createDefaultEnvironments(organization.id, manager);
await manager.save(organizationUser);
await this.createDefaultUserGroups(manager, user);
@ -93,4 +97,19 @@ export class SeedsService {
await manager.save(userGroupPermission);
}
async createDefaultEnvironments(organizationId: string, manager: EntityManager) {
await Promise.all(
defaultAppEnvironments.map(async (en) => {
const env = manager.create(AppEnvironment, {
organizationId: organizationId,
name: en.name,
isDefault: en.isDefault,
createdAt: new Date(),
updatedAt: new Date(),
});
await manager.save(env);
})
);
}
}

View file

@ -255,6 +255,7 @@ export class UsersService {
case 'User':
case 'Plugin':
case 'GlobalDataSource':
return await this.hasGroup(user, 'admin');
case 'Thread':

File diff suppressed because one or more lines are too long

View file

@ -417,9 +417,9 @@ describe('apps controller', () => {
user: adminUserData.user,
});
const version = await createApplicationVersion(app, application);
await createApplicationVersion(app, application);
await createAppEnvironments(app, version.id);
await createAppEnvironments(app, adminUserData.user.organizationId);
let response = await request(app.getHttpServer())
.post(`/api/apps/${application.id}/clone`)
@ -931,7 +931,7 @@ describe('apps controller', () => {
appVersion: version,
});
const appEnvironments = await createAppEnvironments(app, version.id);
const appEnvironments = await createAppEnvironments(app, adminUserData.user.organizationId);
await createDataSourceOption(app, {
dataSource,
@ -1484,9 +1484,9 @@ describe('apps controller', () => {
slug: 'foo',
});
const version = await createApplicationVersion(app, application);
await createApplicationVersion(app, application);
await createAppEnvironments(app, version.id);
await createAppEnvironments(app, adminUserData.user.organizationId);
// setup app permissions for developer
const developerUserGroup = await getRepository(GroupPermission).findOneOrFail({

View file

@ -119,7 +119,9 @@ describe('data queries controller', () => {
email: 'another@tooljet.io',
groups: ['all_users', 'admin'],
});
const { application, dataSource } = await generateAppDefaults(app, adminUserData.user, { isQueryNeeded: false });
const { application, dataSource, appVersion } = await generateAppDefaults(app, adminUserData.user, {
isQueryNeeded: false,
});
// setup app permissions for developer
const developerUserGroup = await getRepository(GroupPermission).findOneOrFail({
@ -136,6 +138,7 @@ describe('data queries controller', () => {
for (const userData of [adminUserData, developerUserData]) {
const dataQuery = await createDataQuery(app, {
dataSource,
appVersion,
options: {
method: 'get',
url: 'https://api.github.com/repos/tooljet/tooljet/stargazers',
@ -160,6 +163,7 @@ describe('data queries controller', () => {
for (const userData of [anotherOrgAdminUserData, viewerUserData]) {
const dataQuery = await createDataQuery(app, {
dataSource,
appVersion,
options: {
method: 'get',
url: 'https://api.github.com/repos/tooljet/tooljet/stargazers',
@ -230,6 +234,7 @@ describe('data queries controller', () => {
await createDataQuery(app, {
dataSource,
appVersion,
kind: 'restapi',
options: { method: 'get' },
});
@ -268,6 +273,7 @@ describe('data queries controller', () => {
await createDataQuery(app, {
dataSource,
appVersion,
kind: 'restapi',
options: { method: 'get' },
});

View file

@ -90,7 +90,7 @@ describe('AppImportExportService', () => {
isPublic: true,
});
const appVersion1 = await createApplicationVersion(nestApp, application, { name: 'v1', definition: {} });
await createAppEnvironments(nestApp, appVersion1.id);
await createAppEnvironments(nestApp, adminUser.organizationId);
const dataSource1 = await createDataSource(nestApp, {
appVersion: appVersion1,
kind: 'test_kind',
@ -105,7 +105,6 @@ describe('AppImportExportService', () => {
name: 'v2',
definition: { hello: 'world' },
});
await createAppEnvironments(nestApp, appVersion2.id);
const dataSource2 = await createDataSource(nestApp, {
appVersion: appVersion2,
kind: 'test_kind',
@ -260,7 +259,7 @@ describe('AppImportExportService', () => {
const appVersion = importedApp.appVersions[0];
expect(appVersion.appId).toEqual(importedApp.id);
const dataSource = importedApp['dataSources'][0];
const dataSource = importedApp['dataSources'].reverse()[0];
expect(dataSource['appVersionId']).toEqual(appVersion.id);
const dataQuery = importedApp['dataQueries'][0];

View file

@ -146,7 +146,7 @@ export async function createApplicationVersion(nestApp, application, { name = 'v
);
}
export async function createAppEnvironments(nestApp, appVersionId): Promise<AppEnvironment[]> {
export async function createAppEnvironments(nestApp, organizationId): Promise<AppEnvironment[]> {
let appEnvironmentRepository: Repository<AppEnvironment>;
appEnvironmentRepository = nestApp.get('AppEnvironmentRepository');
@ -154,7 +154,7 @@ export async function createAppEnvironments(nestApp, appVersionId): Promise<AppE
defaultAppEnvironments.map(async (env) => {
return await appEnvironmentRepository.save(
appEnvironmentRepository.create({
appVersionId,
organizationId,
name: env.name,
isDefault: env.isDefault,
})
@ -463,7 +463,7 @@ export async function createDataSource(
return dataSource;
}
export async function createDataQuery(nestApp, { name = 'defaultquery', dataSource, options }: any) {
export async function createDataQuery(nestApp, { name = 'defaultquery', dataSource, appVersion, options }: any) {
let dataQueryRepository: Repository<DataQuery>;
dataQueryRepository = nestApp.get('DataQueryRepository');
@ -472,6 +472,7 @@ export async function createDataQuery(nestApp, { name = 'defaultquery', dataSour
options,
name,
dataSource,
appVersion,
createdAt: new Date(),
updatedAt: new Date(),
})
@ -700,7 +701,7 @@ export const generateAppDefaults = async (
});
const appVersion = await createApplicationVersion(app, application);
const appEnvironments = await createAppEnvironments(app, appVersion.id);
const appEnvironments = await createAppEnvironments(app, user.organizationId);
let dataQuery: any;
let dataSource: any;
@ -715,6 +716,7 @@ export const generateAppDefaults = async (
if (isQueryNeeded) {
dataQuery = await createDataQuery(app, {
dataSource,
appVersion,
options: {
method: 'get',
url: 'https://api.github.com/repos/tooljet/tooljet/stargazers',