fix(ai-builder): Use placeholders for user-provided values instead of hardcoding fake addresses (#28407)

This commit is contained in:
Albert Alises 2026-04-13 15:29:31 +02:00 committed by GitHub
parent 6217d08ce9
commit 39c6217109
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 223 additions and 9 deletions

View file

@ -200,7 +200,7 @@ Always pass \`conversationContext\` when spawning background agents (\`build-wor
**Credentials**: Call \`list-credentials\` first to know what's available. Build the workflow immediately — the builder auto-resolves available credentials and auto-mocks missing ones. Planned builder tasks handle their own verification and credential finalization flow.
**Post-build flow** (for direct builds via \`build-workflow-with-agent\`):
1. Builder finishes check if the workflow has mocked credentials, missing parameters, or unconfigured triggers.
1. Builder finishes check if the workflow has mocked credentials, missing parameters, unresolved placeholders, or unconfigured triggers.
2. If yes call \`setup-workflow\` with the workflowId so the user can configure them through the setup UI.
3. When \`setup-workflow\` returns \`deferred: true\`, respect the user's decision — do not retry with \`setup-credentials\` or any other setup tool. The user chose to set things up later.
4. Ask the user if they want to test the workflow.

View file

@ -27,13 +27,23 @@ import {
WORKFLOW_SDK_PATTERNS,
} from '@n8n/workflow-sdk/prompts/sdk-reference';
// ── Shared placeholder guidance (single source of truth) ────────────────────
// prettier-ignore
const PLACEHOLDER_RULE =
"**Do NOT use `placeholder()` for discoverable resources** (spreadsheet IDs, calendar IDs, channel IDs, folder IDs) — resolve real IDs via `explore-node-resources` or create them via setup workflows. For **user-provided values** that cannot be discovered or created (email recipients, phone numbers, custom URLs, notification targets), use `placeholder('descriptive hint')` so the setup wizard prompts the user after the build. Never hardcode fake values like `user@example.com`.";
// prettier-ignore
const PLACEHOLDER_ESCALATION =
'When the user says "send me", "email me", "notify me", or similar and you don\'t know their specific address, use `placeholder(\'Your email address\')` for the recipient field rather than hardcoding a fake address like `user@example.com`. The setup wizard will collect this from the user after the build.';
// ── Shared SDK reference sections ────────────────────────────────────────────
const SDK_CODE_RULES = `## SDK Code Rules
- Do NOT specify node positions they are auto-calculated by the layout engine.
- For credentials, see the credential rules in your specific workflow process section below.
- **Do NOT use \`placeholder()\`** — always resolve real resource IDs via \`explore-node-resources\` or create resources via setup workflows. If a resource truly cannot be created (external system), use a descriptive string comment like \`'NEEDS: Slack channel #engineering'\` and explain in your summary.
- ${PLACEHOLDER_RULE}
- Use \`expr('{{ $json.field }}')\` for n8n expressions. Variables MUST be inside \`{{ }}\`.
- Do NOT use \`as const\` assertions — the workflow parser only supports JavaScript syntax, not TypeScript-only features. Just use plain string literals.
- Use string values directly for discriminator fields like \`resource\` and \`operation\` (e.g., \`resource: 'message'\` not \`resource: 'message' as const\`).
@ -445,6 +455,7 @@ When called with failure details for an existing workflow, start from the pre-lo
## Escalation
- If you are stuck or need information only a human can provide (e.g., a chat ID, API key, external resource name), use the \`ask-user\` tool to ask a clear question.
- Do NOT retry the same failing approach more than twice ask the user instead.
- ${PLACEHOLDER_ESCALATION}
## Mandatory Process
1. **Research**: If the workflow fits a known category (notification, chatbot, scheduling, data_transformation, etc.), call \`get-suggested-nodes\` first for curated recommendations. Then use \`search-nodes\` for service-specific nodes (use short service names: "Gmail", "Slack", not "send email SMTP"). The results include \`discriminators\` (available resources and operations) for nodes that need them. Then call \`get-node-type-definition\` with the appropriate resource/operation to get the TypeScript schema with exact parameter names and types. **Pay attention to @builderHint annotations** in search results and type definitions — they prevent common configuration mistakes.
@ -631,7 +642,7 @@ Replace \`CHUNK_WORKFLOW_ID\` with the actual ID returned by \`submit-workflow\`
## Setup Workflows (Create Missing Resources)
**NEVER use \`placeholder()\` or hardcoded placeholder strings like "YOUR_SPREADSHEET_ID".** If a resource doesn't exist, create it.
${PLACEHOLDER_RULE}
When \`explore-node-resources\` returns no results for a required resource:
@ -648,6 +659,7 @@ When called with failure details for an existing workflow, start from the pre-lo
## Escalation
- If you are stuck or need information only a human can provide (e.g., a chat ID, API key, external resource name), use the \`ask-user\` tool to ask a clear question.
- Do NOT retry the same failing approach more than twice ask the user instead.
- ${PLACEHOLDER_ESCALATION}
## Sandbox Isolation
@ -714,7 +726,7 @@ n8n normalizes column names to snake_case (e.g., \`dayName\` → \`day_name\`).
4. **Resolve real resource IDs**: Check the node schemas from step 3 for parameters with \`searchListMethod\` or \`loadOptionsMethod\`. For EACH one, call \`explore-node-resources\` with the node type, method name, and the matching credential from step 1 to discover real resource IDs.
- **This is mandatory for: calendars, spreadsheets, channels, folders, models, databases, and any other list-based parameter.** Do NOT assume values like "primary", "default", or "General" always look up the real ID.
- Example: Google Calendar's \`calendar\` parameter uses \`searchListMethod: getCalendars\`. Call \`explore-node-resources\` with \`methodName: "getCalendars"\` to get the actual calendar ID (e.g., "user@example.com"), not "primary".
- **NEVER use \`placeholder()\` or fake IDs.** If a resource doesn't exist, build a setup workflow to create it (see "Setup Workflows" section).
- **Never use \`placeholder()\` or fake IDs for discoverable resources.** Create them via a setup workflow instead (see "Setup Workflows" section). For user-provided values, follow the placeholder rules in "SDK Code Rules".
- If the resource can't be created via n8n (e.g., Slack channels), explain clearly in your summary what the user needs to set up.
5. **Write workflow code** to \`${workspaceRoot}/src/workflow.ts\`.

View file

@ -94,12 +94,13 @@ function buildOutcome(
taskId,
workflowId: attempt.workflowId,
submitted: true,
triggerType: detectTriggerType(attempt),
triggerType: attempt.hasUnresolvedPlaceholders ? 'trigger_only' : detectTriggerType(attempt),
needsUserInput: false,
mockedNodeNames: attempt.mockedNodeNames,
mockedCredentialTypes: attempt.mockedCredentialTypes,
mockedCredentialsByNode: attempt.mockedCredentialsByNode,
verificationPinData: attempt.verificationPinData,
hasUnresolvedPlaceholders: attempt.hasUnresolvedPlaceholders,
summary: finalText,
};
}

View file

@ -5,6 +5,7 @@
* Separated from the tool definition so the tool stays a thin suspend/resume
* state machine, and this logic is testable independently.
*/
import { hasPlaceholderDeep } from '@n8n/utils';
import type { IDataObject, NodeJSON, DisplayOptions } from '@n8n/workflow-sdk';
import { matchesDisplayOptions } from '@n8n/workflow-sdk';
import { nanoid } from 'nanoid';
@ -67,6 +68,13 @@ export async function buildSetupRequests(
.catch(() => ({}));
}
// Also treat placeholder values as parameter issues so the setup wizard surfaces them
for (const [paramName, paramValue] of Object.entries(parameters)) {
if (!parameterIssues[paramName] && hasPlaceholderDeep(paramValue)) {
parameterIssues[paramName] = ['Contains a placeholder value - please provide the real value'];
}
}
// Build editable parameter definitions for parameters that have issues
let editableParameters: SetupRequest['editableParameters'];
if (Object.keys(parameterIssues).length > 0 && nodeDesc?.properties) {

View file

@ -8,6 +8,7 @@
import { createTool } from '@mastra/core/tools';
import type { Workspace } from '@mastra/core/workspace';
import { hasPlaceholderDeep } from '@n8n/utils';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { validateWorkflow, layoutWorkflowJSON } from '@n8n/workflow-sdk';
import { createHash, randomUUID } from 'node:crypto';
@ -36,6 +37,8 @@ export interface SubmitWorkflowAttempt {
mockedCredentialsByNode?: Record<string, string[]>;
/** Verification-only pin data — scoped to this build, never persisted to workflow. */
verificationPinData?: Record<string, Array<Record<string, unknown>>>;
/** Whether any node parameters contain unresolved placeholder values. */
hasUnresolvedPlaceholders?: boolean;
errors?: string[];
}
@ -346,6 +349,10 @@ export function createSubmitWorkflowTool(
(n) => n.type?.endsWith?.('Trigger') || n.type?.endsWith?.('trigger'),
);
const triggerNodeTypes = triggers.map((t) => t.type).filter(Boolean);
// Scan node parameters for unresolved placeholder values
const hasPlaceholders = (json.nodes ?? []).some((n) => hasPlaceholderDeep(n.parameters));
await reportAttempt({
success: true,
workflowId: savedId,
@ -359,6 +366,7 @@ export function createSubmitWorkflowTool(
hasMockedCredentials && Object.keys(mockResult.verificationPinData).length > 0
? mockResult.verificationPinData
: undefined,
hasUnresolvedPlaceholders: hasPlaceholders || undefined,
});
return {
success: true,

View file

@ -79,6 +79,30 @@ describe('formatWorkflowLoopGuidance', () => {
const result = formatWorkflowLoopGuidance(action);
expect(result).toContain('"unknown"');
});
it('should trigger setup-workflow guidance when hasUnresolvedPlaceholders is true (no mocked credentials)', () => {
const action: WorkflowLoopAction = {
type: 'done',
summary: 'Built with placeholders',
workflowId: 'wf-ph-1',
hasUnresolvedPlaceholders: true,
};
const result = formatWorkflowLoopGuidance(action);
expect(result).toContain('setup-workflow');
expect(result).toContain('wf-ph-1');
});
it('should trigger setup-workflow guidance when both mocked credentials and placeholders exist', () => {
const action: WorkflowLoopAction = {
type: 'done',
summary: 'Built with mocks and placeholders',
mockedCredentialTypes: ['gmailOAuth2'],
hasUnresolvedPlaceholders: true,
workflowId: 'wf-ph-2',
};
const result = formatWorkflowLoopGuidance(action);
expect(result).toContain('setup-workflow');
});
});
// ── verify ─────────────────────────────────────────────────────────────────

View file

@ -148,6 +148,38 @@ describe('handleBuildOutcome', () => {
expect(attempt.attempt).toBe(2);
});
it('persists hasUnresolvedPlaceholders in state and done action', () => {
const state = makeState();
const outcome = makeOutcome({
workflowId: 'wf_123',
triggerType: 'trigger_only',
hasUnresolvedPlaceholders: true,
});
const { state: next, action } = handleBuildOutcome(state, [], outcome);
expect(next.hasUnresolvedPlaceholders).toBe(true);
expect(action.type).toBe('done');
if (action.type === 'done') {
expect(action.hasUnresolvedPlaceholders).toBe(true);
}
});
it('does not set hasUnresolvedPlaceholders when not present in outcome', () => {
const state = makeState();
const outcome = makeOutcome({
workflowId: 'wf_123',
triggerType: 'trigger_only',
});
const { state: next, action } = handleBuildOutcome(state, [], outcome);
expect(next.hasUnresolvedPlaceholders).toBeUndefined();
if (action.type === 'done') {
expect(action.hasUnresolvedPlaceholders).toBeUndefined();
}
});
});
// ── handleVerificationVerdict ───────────────────────────────────────────────
@ -175,6 +207,22 @@ describe('handleVerificationVerdict', () => {
expect(action.type).toBe('done');
});
it('passes through hasUnresolvedPlaceholders from state on verified', () => {
const state = makeState({
phase: 'verifying',
workflowId: 'wf_123',
hasUnresolvedPlaceholders: true,
});
const verdict = makeVerdict({ verdict: 'verified' });
const { action } = handleVerificationVerdict(state, [], verdict);
expect(action.type).toBe('done');
if (action.type === 'done') {
expect(action.hasUnresolvedPlaceholders).toBe(true);
}
});
it('transitions to blocked on needs_user_input', () => {
const state = makeState({ phase: 'verifying', workflowId: 'wf_123' });
const verdict = makeVerdict({

View file

@ -10,7 +10,7 @@ export function formatWorkflowLoopGuidance(
): string {
switch (action.type) {
case 'done': {
if (action.mockedCredentialTypes?.length) {
if (action.mockedCredentialTypes?.length || action.hasUnresolvedPlaceholders) {
return (
'Workflow verified successfully with temporary mock data. ' +
`Call \`setup-workflow\` with workflowId "${action.workflowId ?? 'unknown'}" ` +

View file

@ -75,11 +75,13 @@ export function handleBuildOutcome(
outcome.mockedCredentialTypes && outcome.mockedCredentialTypes.length > 0
? outcome.mockedCredentialTypes
: undefined;
const hasUnresolvedPlaceholders = outcome.hasUnresolvedPlaceholders ?? undefined;
const updatedState: WorkflowLoopState = {
...state,
workflowId: outcome.workflowId ?? state.workflowId,
lastTaskId: outcome.taskId,
mockedCredentialTypes: mockedCredentialTypes ?? state.mockedCredentialTypes,
hasUnresolvedPlaceholders: hasUnresolvedPlaceholders ?? state.hasUnresolvedPlaceholders,
};
if (outcome.triggerType === 'trigger_only') {
@ -90,6 +92,7 @@ export function handleBuildOutcome(
workflowId: outcome.workflowId,
summary: outcome.summary,
mockedCredentialTypes,
hasUnresolvedPlaceholders: updatedState.hasUnresolvedPlaceholders,
},
attempt,
};
@ -138,6 +141,7 @@ export function handleVerificationVerdict(
workflowId: verdict.workflowId,
summary: verdict.summary,
mockedCredentialTypes: state.mockedCredentialTypes,
hasUnresolvedPlaceholders: state.hasUnresolvedPlaceholders,
},
attempt,
};
@ -152,6 +156,7 @@ export function handleVerificationVerdict(
workflowId: verdict.workflowId,
summary: verdict.summary,
mockedCredentialTypes: state.mockedCredentialTypes,
hasUnresolvedPlaceholders: state.hasUnresolvedPlaceholders,
},
attempt,
};

View file

@ -29,6 +29,8 @@ export const workflowLoopStateSchema = z.object({
rebuildAttempts: z.number().int().min(0),
/** Credential types that were mocked during build (persisted across phases). */
mockedCredentialTypes: z.array(z.string()).optional(),
/** Whether the submitted workflow contains unresolved placeholder values (persisted across phases). */
hasUnresolvedPlaceholders: z.boolean().optional(),
});
export type WorkflowLoopPhase = z.infer<typeof workflowLoopPhaseSchema>;
@ -80,6 +82,8 @@ export const workflowBuildOutcomeSchema = z.object({
mockedCredentialsByNode: z.record(z.array(z.string())).optional(),
/** Verification-only pin data — scoped to this build, never persisted to workflow. */
verificationPinData: z.record(z.array(z.record(z.unknown()))).optional(),
/** Whether any node parameters contain unresolved placeholder values. */
hasUnresolvedPlaceholders: z.boolean().optional(),
summary: z.string(),
});
@ -124,5 +128,11 @@ export type WorkflowLoopAction =
diagnosis: string;
patch?: Record<string, unknown>;
}
| { type: 'done'; workflowId?: string; summary: string; mockedCredentialTypes?: string[] }
| {
type: 'done';
workflowId?: string;
summary: string;
mockedCredentialTypes?: string[];
hasUnresolvedPlaceholders?: boolean;
}
| { type: 'blocked'; reason: string };

View file

@ -10,3 +10,4 @@ export * from './sort/sortByProperty';
export * from './string/truncate';
export * from './files/sanitize';
export * from './files/path';
export * from './placeholder';

View file

@ -0,0 +1,56 @@
import { describe, expect, it } from 'vitest';
import { isPlaceholderString, hasPlaceholderDeep } from './placeholder';
describe('isPlaceholderString', () => {
it('returns true for a valid placeholder sentinel', () => {
expect(isPlaceholderString('<__PLACEHOLDER_VALUE__Your email address__>')).toBe(true);
});
it('returns false for a regular string', () => {
expect(isPlaceholderString('user@example.com')).toBe(false);
});
it('returns false for non-string values', () => {
expect(isPlaceholderString(42)).toBe(false);
expect(isPlaceholderString(null)).toBe(false);
expect(isPlaceholderString(undefined)).toBe(false);
expect(isPlaceholderString({})).toBe(false);
});
it('returns false for partial matches', () => {
expect(isPlaceholderString('<__PLACEHOLDER_VALUE__no suffix')).toBe(false);
expect(isPlaceholderString('no prefix__>')).toBe(false);
});
});
describe('hasPlaceholderDeep', () => {
it('detects placeholder in a flat string', () => {
expect(hasPlaceholderDeep('<__PLACEHOLDER_VALUE__hint__>')).toBe(true);
});
it('returns false for a regular string', () => {
expect(hasPlaceholderDeep('hello')).toBe(false);
});
it('detects placeholder nested in an array', () => {
expect(hasPlaceholderDeep(['a', '<__PLACEHOLDER_VALUE__hint__>'])).toBe(true);
});
it('detects placeholder nested in an object', () => {
expect(hasPlaceholderDeep({ to: '<__PLACEHOLDER_VALUE__email__>' })).toBe(true);
});
it('detects deeply nested placeholder', () => {
expect(hasPlaceholderDeep({ a: { b: [{ c: '<__PLACEHOLDER_VALUE__x__>' }] } })).toBe(true);
});
it('returns false when no placeholders exist', () => {
expect(hasPlaceholderDeep({ a: { b: [1, 'hello', null] } })).toBe(false);
});
it('returns false for null and undefined', () => {
expect(hasPlaceholderDeep(null)).toBe(false);
expect(hasPlaceholderDeep(undefined)).toBe(false);
});
});

View file

@ -0,0 +1,21 @@
const PLACEHOLDER_PREFIX = '<__PLACEHOLDER_VALUE__';
const PLACEHOLDER_SUFFIX = '__>';
/** Check if a value is a placeholder sentinel string (format: `<__PLACEHOLDER_VALUE__hint__>`). */
export function isPlaceholderString(value: unknown): boolean {
return (
typeof value === 'string' &&
value.startsWith(PLACEHOLDER_PREFIX) &&
value.endsWith(PLACEHOLDER_SUFFIX)
);
}
/** Recursively check if a value (string, array, or object) contains any placeholder sentinel strings. */
export function hasPlaceholderDeep(value: unknown): boolean {
if (typeof value === 'string') return isPlaceholderString(value);
if (Array.isArray(value)) return value.some(hasPlaceholderDeep);
if (value !== null && typeof value === 'object') {
return Object.values(value as Record<string, unknown>).some(hasPlaceholderDeep);
}
return false;
}

View file

@ -1,11 +1,17 @@
import type { ComputedRef, Ref } from 'vue';
import { ref } from 'vue';
import { hasPlaceholderDeep } from '@n8n/utils';
import { NodeHelpers, type INodeProperties } from 'n8n-workflow';
import { useNodeTypesStore } from '@/app/stores/nodeTypes.store';
import { useWorkflowsStore } from '@/app/stores/workflows.store';
import type { IUpdateInformation } from '@/Interface';
import { isNestedParam, isParamValueSet, type SetupCard } from '../instanceAiWorkflowSetup.utils';
/** Check if the original node parameter value was a placeholder sentinel. */
function isOriginalValuePlaceholder(req: SetupCard['nodes'][0], paramName: string): boolean {
return hasPlaceholderDeep(req.node.parameters[paramName]);
}
export function useSetupCardParameters(
cards: ComputedRef<SetupCard[]>,
trackedParamNames: Ref<Map<string, Set<string>>>,
@ -112,6 +118,10 @@ export function useSetupCardParameters(
if (isParamValueSet(val)) {
merged[paramName] = val;
hasValues = true;
} else if (isOriginalValuePlaceholder(req, paramName)) {
// Explicitly send empty string to clear the placeholder sentinel on the backend
merged[paramName] = '';
hasValues = true;
}
}
if (Object.keys(merged).length > 0) {

View file

@ -1,6 +1,7 @@
import type { Ref } from 'vue';
import { computed, ref, watch } from 'vue';
import type { InstanceAiWorkflowSetupNode } from '@n8n/api-types';
import { hasPlaceholderDeep } from '@n8n/utils';
import { NodeConnectionTypes } from 'n8n-workflow';
import { useNodeTypesStore } from '@/app/stores/nodeTypes.store';
import { useWorkflowsStore } from '@/app/stores/workflows.store';
@ -309,6 +310,8 @@ export function useSetupCards(
if (storeNode) {
const liveIssues = getNodeParametersIssues(nodeTypesStore, storeNode);
if (Object.keys(liveIssues).length > 0) return false;
// Also check for remaining placeholder values in node parameters
if (hasPlaceholderDeep(storeNode.parameters)) return false;
}
}
}

View file

@ -1,4 +1,5 @@
import type { InstanceAiWorkflowSetupNode } from '@n8n/api-types';
import { isPlaceholderString } from '@n8n/utils';
import type { INodeProperties } from 'n8n-workflow';
import { isResourceLocatorValue } from 'n8n-workflow';
import type { INodeUi } from '@/Interface';
@ -65,11 +66,17 @@ export function credGroupKey(req: InstanceAiWorkflowSetupNode): string {
return credType;
}
/** Check if a parameter value is meaningfully set (not empty, null, or an empty resource locator). */
/** Check if a parameter value is meaningfully set (not empty, null, placeholder, or an empty resource locator). */
export function isParamValueSet(val: unknown): boolean {
if (val === undefined || val === null || val === '') return false;
if (isPlaceholderString(val)) return false;
if (isResourceLocatorValue(val)) {
return val.value !== '' && val.value !== null && val.value !== undefined;
return (
val.value !== '' &&
val.value !== null &&
val.value !== undefined &&
!isPlaceholderString(val.value)
);
}
return true;
}