Merge pull request #12375 from ToolJet/feat/server-side-resolver

Server side query resolver
This commit is contained in:
Johnson Cherian 2025-04-10 16:32:20 +05:30 committed by GitHub
commit 0373afda76
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 164 additions and 42 deletions

View file

@ -7,6 +7,7 @@ import { keymap } from '@codemirror/view';
import { completionKeymap, acceptCompletion, autocompletion, completionStatus } from '@codemirror/autocomplete';
import { python } from '@codemirror/lang-python';
import { sql } from '@codemirror/lang-sql';
import _ from 'lodash';
import { sass, sassCompletionSource } from '@codemirror/lang-sass';
import { okaidia } from '@uiw/codemirror-theme-okaidia';
import { githubLight } from '@uiw/codemirror-theme-github';
@ -21,6 +22,7 @@ import useStore from '@/AppBuilder/_stores/store';
import { shallow } from 'zustand/shallow';
import { search, searchKeymap, searchPanelOpen } from '@codemirror/search';
import { handleSearchPanel, SearchBtn } from './SearchBox';
import { isInsideParent } from './utils';
const langSupport = Object.freeze({
javascript: javascript(),
@ -51,8 +53,15 @@ const MultiLineCodeEditor = (props) => {
renderCopilot,
} = props;
const replaceIdsWithName = useStore((state) => state.replaceIdsWithName, shallow);
const wrapperRef = useRef(null);
const getSuggestions = useStore((state) => state.getSuggestions, shallow);
const getServerSideGlobalSuggestions = useStore((state) => state.getServerSideGlobalSuggestions, shallow);
const isInsideQueryPane = !!document.querySelector('.code-hinter-wrapper')?.closest('.query-details');
const isInsideQueryManager = useMemo(
() => isInsideParent(wrapperRef?.current, 'query-manager'),
[wrapperRef.current]
);
const context = useContext(CodeHinterContext);
@ -100,9 +109,16 @@ const MultiLineCodeEditor = (props) => {
const hints = getSuggestions();
const serverHints = getServerSideGlobalSuggestions(isInsideQueryManager);
const allHints = {
...hints,
appHints: [...hints.appHints, ...serverHints],
};
let JSLangHints = [];
if (lang === 'javascript') {
JSLangHints = Object.keys(hints['jsHints'])
JSLangHints = Object.keys(allHints['jsHints'])
.map((key) => {
return hints['jsHints'][key]['methods'].map((hint) => ({
hint: hint,
@ -120,7 +136,7 @@ const MultiLineCodeEditor = (props) => {
});
}
const appHints = hints['appHints'];
const appHints = allHints['appHints'];
let autoSuggestionList = appHints.filter((suggestion) => {
return suggestion.hint.includes(nearestSubstring);
@ -229,6 +245,7 @@ const MultiLineCodeEditor = (props) => {
<div
className={`code-hinter-wrapper position-relative ${isInsideQueryPane ? 'code-editor-query-panel' : ''}`}
style={{ width: '100%' }}
ref={wrapperRef}
>
<div className={`${className} ${darkMode && 'cm-codehinter-dark-themed'}`}>
<SearchBtn view={editorView} />

View file

@ -96,6 +96,7 @@ export const PreviewBox = ({
const [largeDataset, setLargeDataset] = useState(false);
const globals = useStore((state) => state.getAllExposedValues().constants || {}, shallow);
const secrets = useStore((state) => state.getSecrets(), shallow);
const globalServerConstantsRegex = /^\{\{.*globals\.server.*\}\}$/;
const getPreviewContent = (content, type) => {
if (content === undefined || content === null) return currentValue;
@ -118,11 +119,11 @@ export const PreviewBox = ({
let previewContent = resolvedValue;
let isGlobalConstant = currentValue && currentValue.includes('{{constants.');
let isSecretConstant = currentValue && currentValue.includes('{{secrets.');
const isServerConstant = currentValue && currentValue.match(globalServerConstantsRegex);
let invalidConstants = null;
let undefinedError = null;
if (isGlobalConstant || isSecretConstant) {
invalidConstants = verifyConstant(currentValue, globals, secrets);
console.log('invalidConstants', invalidConstants);
}
if (invalidConstants?.length) {
undefinedError = { type: 'Invalid constants' };
@ -197,7 +198,11 @@ export const PreviewBox = ({
const errValue = ifCoersionErrorHasCircularDependency(_resolveValue);
setError({
message: isSecretError ? 'secrets cannot be used in apps' : _error,
message: isServerConstant
? 'Server variables cannot be used in apps'
: isSecretError
? 'secrets cannot be used in apps'
: _error,
value: isSecretError
? 'Undefined'
: jsErrorType === 'Invalid'
@ -222,6 +227,7 @@ export const PreviewBox = ({
isWorkspaceVariable={isWorkspaceVariable}
isSecretConstant={isSecretConstant || false}
isLargeDataset={largeDataset}
isServerConstant={isServerConstant}
/>
<CodeHinter.PopupIcon
callback={() => copyToClipboard(error ? error?.value : content)}
@ -240,8 +246,11 @@ const RenderResolvedValue = ({
withValidation,
isWorkspaceVariable,
isSecretConstant = false,
isServerConstant = false,
isLargeDataset,
}) => {
const isServerSideGlobalEnabled = useStore((state) => !!state?.license?.featureAccess?.serverSideGlobal, shallow);
const computeCoersionPreview = (resolvedValue, coersionData) => {
if (coersionData?.typeBeforeCoercion === coersionData?.typeAfterCoercion) return resolvedValue;
@ -264,7 +273,11 @@ const RenderResolvedValue = ({
}`
: previewType;
const previewContent = isSecretConstant
const previewContent = isServerConstant
? isServerSideGlobalEnabled
? 'Server variables would be resolved at runtime'
: 'Server variables are only available in paid plans'
: isSecretConstant
? 'Values of secret constants are hidden'
: !withValidation
? resolvedValue

View file

@ -3,7 +3,7 @@ import React, { useEffect, useRef, useState } from 'react';
import { PreviewBox } from './PreviewBox';
import { ToolTip } from '@/Editor/Inspector/Elements/Components/ToolTip';
import { useTranslation } from 'react-i18next';
import { camelCase, isEmpty, noop } from 'lodash';
import { camelCase, isEmpty, noop, get } from 'lodash';
import CodeMirror from '@uiw/react-codemirror';
import { javascript } from '@codemirror/lang-javascript';
import { autocompletion, completionKeymap, completionStatus, acceptCompletion } from '@codemirror/autocomplete';
@ -12,7 +12,7 @@ import { keymap } from '@codemirror/view';
import FxButton from '../CodeBuilder/Elements/FxButton';
import cx from 'classnames';
import { DynamicFxTypeRenderer } from './DynamicFxTypeRenderer';
import { resolveReferences } from './utils';
import { isInsideParent, resolveReferences } from './utils';
import { okaidia } from '@uiw/codemirror-theme-okaidia';
import { githubLight } from '@uiw/codemirror-theme-github';
import { getAutocompletion } from './autocompleteExtensionConfig';
@ -161,6 +161,7 @@ const SingleLineCodeEditor = ({ componentName, fieldMeta = {}, componentId, ...r
componentName={componentName}
setShowPreview={setShowPreview}
showPreview={showPreview}
wrapperRef={wrapperRef}
showSuggestions={showSuggestions}
{...restProps}
/>
@ -194,11 +195,25 @@ const EditorInput = ({
previewRef,
setShowPreview,
onInputChange,
wrapperRef,
showSuggestions,
}) => {
const getServerSideGlobalSuggestions = useStore((state) => state.getServerSideGlobalSuggestions, shallow);
const getSuggestions = useStore((state) => state.getSuggestions, shallow);
const isInsideQueryManager = useMemo(
() => isInsideParent(wrapperRef?.current, 'query-manager'),
[wrapperRef.current]
);
function autoCompleteExtensionConfig(context) {
const hints = getSuggestions();
const serverHints = getServerSideGlobalSuggestions(isInsideQueryManager);
const allHints = {
...hints,
appHints: [...hints.appHints, ...serverHints],
};
let word = context.matchBefore(/\w*/);
const totalReferences = (context.state.doc.toString().match(/{{/g) || []).length;
@ -229,7 +244,7 @@ const EditorInput = ({
queryInput = '{{' + currentWord + '}}';
}
let completions = getAutocompletion(queryInput, validationType, hints, totalReferences, originalQueryInput);
let completions = getAutocompletion(queryInput, validationType, allHints, totalReferences, originalQueryInput);
return {
from: word.from,
@ -239,7 +254,7 @@ const EditorInput = ({
}
// eslint-disable-next-line react-hooks/exhaustive-deps
const overRideFunction = React.useCallback((context) => autoCompleteExtensionConfig(context), []);
const overRideFunction = React.useCallback((context) => autoCompleteExtensionConfig(context), [isInsideQueryManager]);
const autoCompleteConfig = autocompletion({
override: [overRideFunction],
@ -409,11 +424,11 @@ const EditorInput = ({
extensions={
showSuggestions
? [
javascript({ jsx: lang === 'jsx' }),
autoCompleteConfig,
keymap.of([...customKeyMaps]),
customTabKeymap,
]
javascript({ jsx: lang === 'jsx' }),
autoCompleteConfig,
keymap.of([...customKeyMaps]),
customTabKeymap,
]
: [javascript({ jsx: lang === 'jsx' })]
}
onChange={(val) => {
@ -485,9 +500,8 @@ const DynamicEditorBridge = (props) => {
<ToolTip
label={t(`widget.commonProperties.${camelCase(paramLabel)}`, paramLabel)}
meta={fieldMeta}
labelClass={`tj-text-xsm color-slate12 ${codeShow ? 'mb-2' : 'mb-0'} ${
darkMode && 'color-whitish-darkmode'
}`}
labelClass={`tj-text-xsm color-slate12 ${codeShow ? 'mb-2' : 'mb-0'} ${darkMode && 'color-whitish-darkmode'
}`}
/>
</div>
)}
@ -495,9 +509,8 @@ const DynamicEditorBridge = (props) => {
<div style={{ marginBottom: codeShow ? '0.5rem' : '0px' }} className={`d-flex align-items-center ${fxClass}`}>
{paramLabel !== 'Type' && isFxNotRequired === undefined && (
<div
className={`col-auto pt-0 fx-common fx-button-container ${
(isEventManagerParam || codeShow) && 'show-fx-button-container'
}`}
className={`col-auto pt-0 fx-common fx-button-container ${(isEventManagerParam || codeShow) && 'show-fx-button-container'
}`}
>
<FxButton
active={codeShow}

View file

@ -30,6 +30,17 @@ function traverseAST(node, callback) {
}
}
export const isInsideParent = (element, className) => {
while (element) {
if (element.classList?.contains(className)) {
console.log('element.classList', element.classList);
return true;
}
element = element.parentElement;
}
return false;
};
function getMethods(type) {
const arrayMethods = Object.getOwnPropertyNames(Array.prototype).filter(
(p) => typeof Array.prototype[p] === 'function'

View file

@ -36,4 +36,27 @@ export const createCodeHinterSlice = (set, get) => ({
setSuggestions({ appHints: suggestionList, jsHints: jsHints });
},
getSuggestions: () => get().suggestions,
getServerSideGlobalSuggestions: (isInsideQueryManager) => {
const isServerSideGlobalEnabled = !!get()?.license?.featureAccess?.serverSideGlobal;
const serverHints = [];
const hints = get().getSuggestions();
if (isInsideQueryManager && isServerSideGlobalEnabled) {
serverHints.push({ hint: 'globals.server', type: 'Object' });
hints?.appHints?.forEach((appHint) => {
if (appHint?.hint?.startsWith('globals.currentUser')) {
const key = appHint?.hint?.replace('globals.currentUser', 'globals.server.currentUser');
console.log({
hint: key,
type: appHint?.type,
});
serverHints.push({
hint: key,
type: appHint?.type,
});
}
});
}
return serverHints;
},
});

View file

@ -122,7 +122,7 @@ module.exports = {
'@cloud/modules': emptyModulePath,
},
},
devtool: environment === 'development' ? 'eval-source-map' : 'hidden-source-map',
devtool: environment === 'development' ? 'source-map' : 'hidden-source-map',
module: {
rules: [
{

@ -1 +1 @@
Subproject commit 683647f83d3efeeadbe69c40b8e8dd5ba4e8ea06
Subproject commit 381d9771d1e237285a362f0c75bffe68f6de707f

View file

@ -22,7 +22,8 @@ export interface IDataQueriesUtilService {
dataQuery: any,
queryOptions: object,
organization_id: string,
environmentId?: string
environmentId?: string,
user?: User
): Promise<{
service: any;
sourceOptions: object;
@ -31,5 +32,11 @@ export interface IDataQueriesUtilService {
setCookiesBackToClient(response: Response, responseHeaders: any): void;
parseQueryOptions(object: any, options: object, organization_id: string, environmentId?: string): Promise<object>;
parseQueryOptions(
object: any,
options: object,
organization_id: string,
environmentId?: string,
user?: User
): Promise<object>;
}

View file

@ -89,7 +89,8 @@ export class DataQueriesUtilService implements IDataQueriesUtilService {
dataQuery,
queryOptions,
organizationId,
environmentId
environmentId,
user
);
queryStatus.setOptions(parsedQueryOptions);
@ -217,7 +218,8 @@ export class DataQueriesUtilService implements IDataQueriesUtilService {
dataQuery,
queryOptions,
organizationId,
environmentId
environmentId,
user
));
queryStatus.setOptions(parsedQueryOptions);
result = await service.run(
@ -291,18 +293,27 @@ export class DataQueriesUtilService implements IDataQueriesUtilService {
}
}
async fetchServiceAndParsedParams(dataSource, dataQuery, queryOptions, organization_id, environmentId = undefined) {
async fetchServiceAndParsedParams(
dataSource,
dataQuery,
queryOptions,
organization_id,
environmentId = undefined,
user = undefined
) {
const sourceOptions = await this.dataSourceUtilService.parseSourceOptions(
dataSource.options,
organization_id,
environmentId
environmentId,
user
);
const parsedQueryOptions = await this.parseQueryOptions(
dataQuery.options,
queryOptions,
organization_id,
environmentId
environmentId,
user
);
const service = await this.pluginsSelectorService.getService(dataSource.pluginId, dataSource.kind);
@ -368,7 +379,8 @@ export class DataQueriesUtilService implements IDataQueriesUtilService {
object: any,
options: object,
organization_id: string,
environmentId?: string
environmentId?: string,
user?: User
): Promise<object> {
const stack: any[] = [{ obj: object, key: null, parent: null }];
@ -406,12 +418,14 @@ export class DataQueriesUtilService implements IDataQueriesUtilService {
// b: Handle {{constants.}} or {{secrets.}}
if (
(typeof resolvedValue === 'string' && resolvedValue.includes('{{constants.')) ||
resolvedValue.includes('{{secrets.')
resolvedValue.includes('{{secrets.') ||
resolvedValue.includes('{{globals.server.')
) {
const resolvingConstant = await this.dataSourceUtilService.resolveConstants(
resolvedValue,
organization_id,
environmentId
environmentId,
user
);
resolvedValue = resolvingConstant;
if (parent && key !== null) {

View file

@ -34,7 +34,7 @@ export interface IDataSourcesUtilService {
parseOptionsForOauthDataSource(options: Array<object>, resetSecureData?: boolean): Promise<Array<object>>;
resolveConstants(value: string, organizationId: string, environmentId: string): Promise<string>;
resolveConstants(value: string, organizationId: string, environmentId: string, user?: User): Promise<string>;
resolveKeyValuePair(element: any, organizationId: string, environmentId: string): Promise<any>;

View file

@ -10,6 +10,7 @@ import { InstanceSettingsModule } from '@modules/instance-settings/module';
import { VersionRepository } from '@modules/versions/repository';
import { AppsRepository } from '@modules/apps/repository';
import { TooljetDbModule } from '@modules/tooljet-db/module';
import { SessionModule } from '@modules/session/module';
export class DataSourcesModule {
static async register(configs?: { IS_GET_CONTEXT: boolean }): Promise<DynamicModule> {
@ -28,6 +29,7 @@ export class DataSourcesModule {
await OrganizationConstantModule.register(configs),
await InstanceSettingsModule.register(configs),
await TooljetDbModule.register(configs),
await SessionModule.register(configs),
],
providers: [
DataSourcesService,

View file

@ -302,8 +302,9 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
return dataSource;
}
async resolveConstants(str: string, organizationId: string, environmentId: string): Promise<string> {
async resolveConstants(str: string, organizationId: string, environmentId: string, user?: User): Promise<string> {
const regex = /\{\{(constants|secrets)\.(.*?)\}\}/g;
const matches = Array.from(str.matchAll(regex));
if (matches.length === 0) return str;
@ -353,7 +354,7 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
}
async resolveValue(value, organization_id, environment_id) {
const constantMatcher = /{{constants|secrets\..+?}}/g;
const constantMatcher = /{{constants|secrets|globals.server\..+?}}/g;
if (typeof value === 'string' && constantMatcher.test(value)) {
return await this.resolveConstants(value, organization_id, environment_id);
@ -371,7 +372,7 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
const parsedOptions = JSON.parse(JSON.stringify(options));
// need to match if currentOption is a contant, {{constants.psql_db}
const constantMatcher = /{{constants|secrets\..+?}}/g;
const constantMatcher = /{{constants|secrets|globals.server\..+?}}/g;
for (const key of Object.keys(parsedOptions)) {
let currentOption = parsedOptions[key]?.['value'];
@ -590,10 +591,10 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
return options;
}
async parseSourceOptions(options: any, organizationId: string, environmentId: string): Promise<object> {
async parseSourceOptions(options: any, organizationId: string, environmentId: string, user?: User): Promise<object> {
// For adhoc queries such as REST API queries, source options will be null
if (!options) return {};
const constantMatcher = /\{\{(constants|secrets)\..*?\}\}/g;
const constantMatcher = /\{\{(constants|secrets|globals.server)\..*?\}\}/g;
for (const key of Object.keys(options)) {
const currentOption = options[key]?.['value'];
@ -609,7 +610,7 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
constantMatcher.lastIndex = 0;
if (constantMatcher.test(inner)) {
const resolved = await this.resolveConstants(inner, organizationId, environmentId);
const resolved = await this.resolveConstants(inner, organizationId, environmentId, user);
curr[j] = resolved;
}
}
@ -618,7 +619,7 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
}
if (constantMatcher.test(currentOption)) {
const resolved = await this.resolveConstants(currentOption, organizationId, environmentId);
const resolved = await this.resolveConstants(currentOption, organizationId, environmentId, user);
options[key]['value'] = resolved;
}
}
@ -633,7 +634,7 @@ export class DataSourcesUtilService implements IDataSourcesUtilService {
const value = await this.credentialService.getValue(credentialId);
if (value.includes('{{constants') || value.includes('{{secrets')) {
const resolved = await this.resolveConstants(value, organizationId, environmentId);
const resolved = await this.resolveConstants(value, organizationId, environmentId, user);
parsedOptions[key] = resolved;
continue;
} else {

View file

@ -15,6 +15,7 @@ export default class LicenseBase {
private _isCustomStyling: boolean;
private _isWhiteLabelling: boolean;
private _isCustomThemes: boolean;
private _isServerSideGlobal: boolean;
private _isMultiEnvironment: boolean;
private _isMultiPlayerEdit: boolean;
private _isComments: boolean;
@ -49,6 +50,7 @@ export default class LicenseBase {
this._isCustomStyling = true;
this._isWhiteLabelling = true;
this._isCustomThemes = true;
this._isServerSideGlobal = true;
this._isLicenseValid = true;
this._isMultiEnvironment = true;
this._isAi = true;
@ -88,6 +90,7 @@ export default class LicenseBase {
this._isCustomStyling = this.getFeatureValue('customStyling');
this._isWhiteLabelling = this.getFeatureValue('whiteLabelling');
this._isCustomThemes = this.getFeatureValue('customThemes');
this._isServerSideGlobal = this.getFeatureValue('serverSideGlobal');
this._isMultiEnvironment = this.getFeatureValue('multiEnvironment');
this._isMultiPlayerEdit = this.getFeatureValue('multiPlayerEdit');
this._isComments = this.getFeatureValue('comments');
@ -256,6 +259,13 @@ export default class LicenseBase {
return this._isCustomThemes;
}
public get serverSideGlobal(): boolean {
if (this.IsBasicPlan) {
return !!BASIC_PLAN_TERMS.features?.serverSideGlobal;
}
return this._isServerSideGlobal;
}
public get multiPlayerEdit(): boolean {
if (this.IsBasicPlan) {
return !!BASIC_PLAN_TERMS.features?.multiPlayerEdit;
@ -298,6 +308,7 @@ export default class LicenseBase {
customStyling: this.customStyling,
whiteLabelling: this.whiteLabelling,
customThemes: this.customThemes,
serverSideGlobal: this.serverSideGlobal,
multiEnvironment: this.multiEnvironment,
multiPlayerEdit: this.multiPlayerEdit,
gitSync: this.gitSync,
@ -326,6 +337,7 @@ export default class LicenseBase {
samlEnabled: this.saml,
customStylingEnabled: this.customStyling,
customThemesEnabled: this.customThemes,
serverSideGlobalEnabled: this.serverSideGlobal,
multiEnvironmentEnabled: this.multiEnvironment,
multiPlayerEditEnabled: this.multiPlayerEdit,
commentsEnabled: this.comments,

View file

@ -25,6 +25,7 @@ export const BASIC_PLAN_TERMS: Partial<Terms> = {
gitSync: false,
comments: false,
customThemes: false,
serverSideGlobal: false,
ai: true,
},
domains: [],

View file

@ -104,6 +104,7 @@ export enum LICENSE_FIELD {
CUSTOM_STYLE = 'customStylingEnabled',
WHITE_LABEL = 'whitelabellingEnabled',
CUSTOM_THEMES = 'customThemeEnabled',
SERVER_SIDE_GLOBAL = 'serverSideGlobalEnabled',
AUDIT_LOGS = 'auditLogsEnabled',
MAX_DURATION_FOR_AUDIT_LOGS = 'maxDaysForAuditLogs',
MULTI_ENVIRONMENT = 'multiEnvironmentEnabled',

View file

@ -59,6 +59,9 @@ export function getLicenseFieldValue(type: LICENSE_FIELD, licenseInstance: Licen
case LICENSE_FIELD.CUSTOM_THEMES:
return licenseInstance.customThemes;
case LICENSE_FIELD.SERVER_SIDE_GLOBAL:
return licenseInstance.serverSideGlobal;
case LICENSE_FIELD.AUDIT_LOGS:
return licenseInstance.auditLogs;

View file

@ -27,6 +27,7 @@ export interface Terms {
gitSync?: boolean;
comments?: boolean;
customThemes?: boolean;
serverSideGlobal?: boolean;
ai?: boolean;
};
type?: LICENSE_TYPE;

View file

@ -1,6 +1,7 @@
export enum OrganizationConstantType {
GLOBAL = 'Global',
SECRET = 'Secret',
SERVER = 'Server',
}
export enum FEATURE_KEY {

View file

@ -20,7 +20,9 @@ export class OrganizationUsersModule {
const { OrganizationUsersController } = await import(
`${await getImportPath(IS_GET_CONTEXT)}/organization-users/controller`
);
const { OrganizationUsersService } = await import(`${await getImportPath(IS_GET_CONTEXT)}/organization-users/service`);
const { OrganizationUsersService } = await import(
`${await getImportPath(IS_GET_CONTEXT)}/organization-users/service`
);
const { OrganizationUsersUtilService } = await import(
`${await getImportPath(IS_GET_CONTEXT)}/organization-users/util.service`
);