mirror of
https://github.com/lobehub/lobehub
synced 2026-04-21 17:47:27 +00:00
🔨 chore: some ai image optimization (#8543)
This commit is contained in:
parent
225d3b6ed5
commit
82ca0074d4
33 changed files with 763 additions and 331 deletions
|
|
@ -1,36 +0,0 @@
|
|||
# 添加新的 AI 图像模型
|
||||
|
||||
## 兼容 openai 请求格式的模型
|
||||
|
||||
指的是可以使用 openai SDK 进行请求,并且请求参数和和返回值和 dall-e 以及 gpt-image-x 系列一致。
|
||||
|
||||
以智谱的 CogView-4 为例,它是一个兼容 openai 请求格式的模型,可以按照以下步骤添加:
|
||||
|
||||
1. 在对应的 ai models 文件 `src/config/aiModels/zhipu.ts` 中,添加模型配置,例如:
|
||||
|
||||
```ts
|
||||
const zhipuImageModels: AIImageModelCard[] = [
|
||||
// 添加模型配置
|
||||
// https://bigmodel.cn/dev/howuse/image-generation-model/cogview-4
|
||||
{
|
||||
description:
|
||||
'CogView-4 是智谱首个支持生成汉字的开源文生图模型,在语义理解、图像生成质量、中英文字生成能力等方面全面提升,支持任意长度的中英双语输入,能够生成在给定范围内的任意分辨率图像。',
|
||||
displayName: 'CogView-4',
|
||||
enabled: true,
|
||||
id: 'cogview-4',
|
||||
parameters: {
|
||||
prompt: {
|
||||
default: '',
|
||||
},
|
||||
size: {
|
||||
default: '1024x1024',
|
||||
enum: ['1024x1024', '768x1344', '864x1152', '1344x768', '1152x864', '1440x720', '720x1440'],
|
||||
},
|
||||
},
|
||||
releasedAt: '2025-03-04',
|
||||
type: 'image',
|
||||
},
|
||||
];
|
||||
```
|
||||
|
||||
2. 执行下 `npx i18n` 命令,更新模型描述的翻译文件
|
||||
162
docs/development/basic/add-new-image-model.mdx
Normal file
162
docs/development/basic/add-new-image-model.mdx
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
# Adding New Image Models
|
||||
|
||||
> Learn more about the AI image generation modal design in the [AI Image Generation Modal Design Discussion](https://github.com/lobehub/lobe-chat/discussions/7442)
|
||||
|
||||
## Parameter Standardization
|
||||
|
||||
All image generation models must use the standard parameters defined in `src/libs/standard-parameters/index.ts`. This ensures parameter consistency across different Providers, creating a more unified user experience.
|
||||
|
||||
**Supported Standard Parameters**:
|
||||
|
||||
- `prompt` (required): Text prompt for image generation
|
||||
- `aspectRatio`: Aspect ratio (e.g., "16:9", "1:1")
|
||||
- `width` / `height`: Image dimensions
|
||||
- `size`: Preset dimensions (e.g., "1024x1024")
|
||||
- `seed`: Random seed
|
||||
- `steps`: Generation steps
|
||||
- `cfg`: Guidance scale
|
||||
- For other parameters, please check the source file
|
||||
|
||||
## OpenAI Compatible Models
|
||||
|
||||
These models can be requested using the OpenAI SDK, with request parameters and return values consistent with DALL-E and GPT-Image-X series.
|
||||
|
||||
Taking Zhipu's CogView-4 as an example, which is an OpenAI-compatible model, you can add it by adding the model configuration in the corresponding AI models file `src/config/aiModels/zhipu.ts`:
|
||||
|
||||
```ts
|
||||
const zhipuImageModels: AIImageModelCard[] = [
|
||||
// Add model configuration
|
||||
// https://bigmodel.cn/dev/howuse/image-generation-model/cogview-4
|
||||
{
|
||||
description:
|
||||
'CogView-4 is the first open-source text-to-image model from Zhipu that supports Chinese character generation, with comprehensive improvements in semantic understanding, image generation quality, and Chinese-English text generation capabilities.',
|
||||
displayName: 'CogView-4',
|
||||
enabled: true,
|
||||
id: 'cogview-4',
|
||||
parameters: {
|
||||
prompt: {
|
||||
default: '',
|
||||
},
|
||||
size: {
|
||||
default: '1024x1024',
|
||||
enum: ['1024x1024', '768x1344', '864x1152', '1344x768', '1152x864', '1440x720', '720x1440'],
|
||||
},
|
||||
},
|
||||
releasedAt: '2025-03-04',
|
||||
type: 'image',
|
||||
},
|
||||
];
|
||||
```
|
||||
|
||||
## Non-OpenAI Compatible Models
|
||||
|
||||
For image generation models that are not compatible with OpenAI format, you need to implement a custom `createImage` method. There are two main implementation approaches:
|
||||
|
||||
### Method 1: Using OpenAI Compatible Factory
|
||||
|
||||
Most Providers use `openaiCompatibleFactory` for OpenAI compatibility. You can pass in a custom `createImage` function (reference [PR #8534](https://github.com/lobehub/lobe-chat/pull/8534)).
|
||||
|
||||
**Implementation Steps**:
|
||||
|
||||
1. **Read Provider documentation and standard parameter definitions**
|
||||
- Review the Provider's image generation API documentation to understand request and response formats
|
||||
- Read `src/libs/standard-parameters/index.ts` to understand supported parameters
|
||||
- Add image model configuration in the corresponding AI models file
|
||||
|
||||
2. **Implement custom createImage method**
|
||||
- Create a standalone image generation function that accepts standard parameters
|
||||
- Convert standard parameters to Provider-specific format
|
||||
- Call the Provider's image generation API
|
||||
- Return a unified response format (imageUrl and optional width/height)
|
||||
|
||||
3. **Add tests**
|
||||
- Write unit tests covering success scenarios
|
||||
- Test various error cases and edge conditions
|
||||
|
||||
**Code Example**:
|
||||
|
||||
```ts
|
||||
// src/libs/model-runtime/provider-name/createImage.ts
|
||||
export const createProviderImage = async (
|
||||
payload: ImageGenerationPayload,
|
||||
options: any,
|
||||
): Promise<ImageGenerationResponse> => {
|
||||
const { model, prompt, ...params } = payload;
|
||||
|
||||
// Call Provider's native API
|
||||
const result = await callProviderAPI({
|
||||
model,
|
||||
prompt,
|
||||
// Convert parameter format
|
||||
custom_param: params.width,
|
||||
// ...
|
||||
});
|
||||
|
||||
// Return unified format
|
||||
return {
|
||||
created: Date.now(),
|
||||
data: [{ url: result.imageUrl }],
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
```ts
|
||||
// src/libs/model-runtime/provider-name/index.ts
|
||||
export const LobeProviderAI = openaiCompatibleFactory({
|
||||
constructorOptions: {
|
||||
// ... other configurations
|
||||
},
|
||||
createImage: createProviderImage, // Pass custom implementation
|
||||
provider: ModelProvider.ProviderName,
|
||||
});
|
||||
```
|
||||
|
||||
### Method 2: Direct Implementation in Provider Class
|
||||
|
||||
If your Provider has an independent class implementation, you can directly add the `createImage` method in the class (reference [PR #8503](https://github.com/lobehub/lobe-chat/pull/8503)).
|
||||
|
||||
**Implementation Steps**:
|
||||
|
||||
1. **Read Provider documentation and standard parameter definitions**
|
||||
- Review the Provider's image generation API documentation
|
||||
- Read `src/libs/standard-parameters/index.ts`
|
||||
- Add image model configuration in the corresponding AI models file
|
||||
|
||||
2. **Implement createImage method in Provider class**
|
||||
- Add the `createImage` method directly in the class
|
||||
- Handle parameter conversion and API calls
|
||||
- Return a unified response format
|
||||
|
||||
3. **Add tests**
|
||||
- Write comprehensive test cases for the new method
|
||||
|
||||
**Code Example**:
|
||||
|
||||
```ts
|
||||
// src/libs/model-runtime/provider-name/index.ts
|
||||
export class LobeProviderAI {
|
||||
async createImage(
|
||||
payload: ImageGenerationPayload,
|
||||
options?: ChatStreamCallbacks,
|
||||
): Promise<ImageGenerationResponse> {
|
||||
const { model, prompt, ...params } = payload;
|
||||
|
||||
// Call native API and handle response
|
||||
const result = await this.client.generateImage({
|
||||
model,
|
||||
prompt,
|
||||
// Parameter conversion
|
||||
});
|
||||
|
||||
return {
|
||||
created: Date.now(),
|
||||
data: [{ url: result.url }],
|
||||
};
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Important Notes
|
||||
|
||||
- **Testing Requirements**: Add comprehensive unit tests for custom implementations, ensuring coverage of success scenarios and various error cases
|
||||
- **Error Handling**: Use `AgentRuntimeError` consistently for error wrapping to maintain error message consistency
|
||||
162
docs/development/basic/add-new-image-model.zh-CN.mdx
Normal file
162
docs/development/basic/add-new-image-model.zh-CN.mdx
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
# 添加新的图像模型
|
||||
|
||||
> 了解更多关于 AI 绘画模态的设计,请参考 [AI 绘画模态设计讨论](https://github.com/lobehub/lobe-chat/discussions/7442)
|
||||
|
||||
## 参数标准化
|
||||
|
||||
所有图像生成模型都必须使用 `src/libs/standard-parameters/index.ts` 中定义的标准参数。这确保了不同 Provider 之间的参数一致性,让用户体验更加统一。
|
||||
|
||||
**支持的标准参数**:
|
||||
|
||||
- `prompt` (必需):生成图像的提示词
|
||||
- `aspectRatio`:宽高比(如 "16:9", "1:1")
|
||||
- `width` / `height`:图像宽高
|
||||
- `size`:预设尺寸(如 "1024x1024")
|
||||
- `seed`:随机种子
|
||||
- `steps`:生成步数
|
||||
- `cfg`:引导缩放
|
||||
- 其他参数请查看源文件
|
||||
|
||||
## 兼容 OpenAI 请求格式的模型
|
||||
|
||||
指的是可以使用 openai SDK 进行请求,并且请求参数和和返回值和 dall-e 以及 gpt-image-x 系列一致。
|
||||
|
||||
以智谱的 CogView-4 为例,它是一个兼容 openai 请求格式的模型。你只需要在对应的 ai models 文件 `src/config/aiModels/zhipu.ts` 中,添加模型配置,例如:
|
||||
|
||||
```ts
|
||||
const zhipuImageModels: AIImageModelCard[] = [
|
||||
// 添加模型配置
|
||||
// https://bigmodel.cn/dev/howuse/image-generation-model/cogview-4
|
||||
{
|
||||
description:
|
||||
'CogView-4 是智谱首个支持生成汉字的开源文生图模型,在语义理解、图像生成质量、中英文字生成能力等方面全面提升,支持任意长度的中英双语输入,能够生成在给定范围内的任意分辨率图像。',
|
||||
displayName: 'CogView-4',
|
||||
enabled: true,
|
||||
id: 'cogview-4',
|
||||
parameters: {
|
||||
prompt: {
|
||||
default: '',
|
||||
},
|
||||
size: {
|
||||
default: '1024x1024',
|
||||
enum: ['1024x1024', '768x1344', '864x1152', '1344x768', '1152x864', '1440x720', '720x1440'],
|
||||
},
|
||||
},
|
||||
releasedAt: '2025-03-04',
|
||||
type: 'image',
|
||||
},
|
||||
];
|
||||
```
|
||||
|
||||
## 不兼容 OpenAI 请求格式的模型
|
||||
|
||||
对于不兼容 OpenAI 格式的图像生成模型,需要实现自定义的 `createImage` 方法。有两种主要实现方式:
|
||||
|
||||
### 方式一:使用 OpenAI Compatible Factory
|
||||
|
||||
大部分 Provider 都使用 `openaiCompatibleFactory` 来兼容 OpenAI,可以通过传入自定义的 `createImage` 函数(参考 [PR #8534](https://github.com/lobehub/lobe-chat/pull/8534))。
|
||||
|
||||
**实现步骤**:
|
||||
|
||||
1. **阅读 Provider 官方文档和标准参数定义**
|
||||
- 查看 Provider 的图像生成 API 文档,了解请求格式和响应格式
|
||||
- 阅读 `src/libs/standard-parameters/index.ts`,了解支持的参数
|
||||
- 在对应的 ai models 文件中增加 image model 配置
|
||||
|
||||
2. **实现自定义的 createImage 方法**
|
||||
- 创建独立的图像生成函数,接受标准生图参数
|
||||
- 将标准参数转换为 Provider 特定的格式
|
||||
- 调用 Provider 的生图接口
|
||||
- 返回统一格式的响应(imageUrl 和可选的宽高)
|
||||
|
||||
3. **补充测试**
|
||||
- 编写单元测试覆盖成功场景
|
||||
- 测试各种错误情况和边界条件
|
||||
|
||||
**代码示例**:
|
||||
|
||||
```ts
|
||||
// src/libs/model-runtime/provider-name/createImage.ts
|
||||
export const createProviderImage = async (
|
||||
payload: ImageGenerationPayload,
|
||||
options: any,
|
||||
): Promise<ImageGenerationResponse> => {
|
||||
const { model, prompt, ...params } = payload;
|
||||
|
||||
// 调用 Provider 的原生 API
|
||||
const result = await callProviderAPI({
|
||||
model,
|
||||
prompt,
|
||||
// 转换参数格式
|
||||
custom_param: params.width,
|
||||
// ...
|
||||
});
|
||||
|
||||
// 返回统一格式
|
||||
return {
|
||||
created: Date.now(),
|
||||
data: [{ url: result.imageUrl }],
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
```ts
|
||||
// src/libs/model-runtime/provider-name/index.ts
|
||||
export const LobeProviderAI = openaiCompatibleFactory({
|
||||
constructorOptions: {
|
||||
// ... 其他配置
|
||||
},
|
||||
createImage: createProviderImage, // 传入自定义实现
|
||||
provider: ModelProvider.ProviderName,
|
||||
});
|
||||
```
|
||||
|
||||
### 方式二:在 Provider 类中直接实现
|
||||
|
||||
如果你的 Provider 有独立的类实现,可以直接在类中添加 `createImage` 方法(参考 [PR #8503](https://github.com/lobehub/lobe-chat/pull/8503))。
|
||||
|
||||
**实现步骤**:
|
||||
|
||||
1. **阅读 Provider 官方文档和标准参数定义**
|
||||
- 查看 Provider 的图像生成 API 文档
|
||||
- 阅读 `src/libs/standard-parameters/index.ts`
|
||||
- 在对应的 ai models 文件中增加 image model 配置
|
||||
|
||||
2. **在 Provider 类中实现 createImage 方法**
|
||||
- 直接在类中添加 `createImage` 方法
|
||||
- 处理参数转换和 API 调用
|
||||
- 返回统一格式的响应
|
||||
|
||||
3. **补充测试**
|
||||
- 为新方法编写完整的测试用例
|
||||
|
||||
**代码示例**:
|
||||
|
||||
```ts
|
||||
// src/libs/model-runtime/provider-name/index.ts
|
||||
export class LobeProviderAI {
|
||||
async createImage(
|
||||
payload: ImageGenerationPayload,
|
||||
options?: ChatStreamCallbacks,
|
||||
): Promise<ImageGenerationResponse> {
|
||||
const { model, prompt, ...params } = payload;
|
||||
|
||||
// 调用原生 API 并处理响应
|
||||
const result = await this.client.generateImage({
|
||||
model,
|
||||
prompt,
|
||||
// 参数转换
|
||||
});
|
||||
|
||||
return {
|
||||
created: Date.now(),
|
||||
data: [{ url: result.url }],
|
||||
};
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 重要注意事项
|
||||
|
||||
- **测试要求**:为自定义实现添加完整的单元测试,确保覆盖成功场景和各种错误情况
|
||||
- **错误处理**:统一使用 `AgentRuntimeError` 进行错误封装,保持错误信息的一致性
|
||||
|
|
@ -56,7 +56,7 @@ tags:
|
|||
<Image alt={'Enter API Key'} inStep src={'https://hub-apac-1.lobeobjects.space/docs/fa056feecba0133c76abe1ad12706c05.png'} />
|
||||
|
||||
- Paste the API key you obtained.
|
||||
- Choose a Fal model (e.g. `fal-ai/flux-pro`, `fal-ai/kling-video`, `fal-ai/hidream-i1-fast`) for image or video generation.
|
||||
- Choose a Fal model (e.g. `Flux.1 Schnell`, `Flux.1 Kontext Dev`) for image or video generation.
|
||||
|
||||
<Image alt={'Select Fal model for media generation'} inStep src={'https://hub-apac-1.lobeobjects.space/docs/7560502f31b8500032922103fc22e69b.png'} />
|
||||
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ tags:
|
|||
<Image alt={'填入 API 密钥'} inStep src={'https://hub-apac-1.lobeobjects.space/docs/fa056feecba0133c76abe1ad12706c05.png'} />
|
||||
|
||||
- 粘贴获取到的 API Key;
|
||||
- 选择一个 Fal 模型(如 `fal-ai/flux-pro`、`fal-ai/kling-video`、`fal-ai/hidream-i1-fast`)用于图像或视频生成。
|
||||
- 选择一个 Fal 模型(如 `Flux.1 Schnell`、`Flux.1 Kontext Dev`)用于图像或视频生成。
|
||||
|
||||
<Image alt={'选择 Fal 模型进行媒体生成'} inStep src={'https://hub-apac-1.lobeobjects.space/docs/7560502f31b8500032922103fc22e69b.png'} />
|
||||
|
||||
|
|
|
|||
130
package.json
130
package.json
|
|
@ -120,28 +120,28 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@ant-design/icons": "^5.6.1",
|
||||
"@ant-design/pro-components": "^2.8.7",
|
||||
"@ant-design/pro-components": "^2.8.10",
|
||||
"@anthropic-ai/sdk": "^0.56.0",
|
||||
"@auth/core": "^0.38.0",
|
||||
"@aws-sdk/client-bedrock-runtime": "^3.821.0",
|
||||
"@aws-sdk/client-s3": "^3.821.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.821.0",
|
||||
"@aws-sdk/client-bedrock-runtime": "^3.848.0",
|
||||
"@aws-sdk/client-s3": "^3.850.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.850.0",
|
||||
"@azure-rest/ai-inference": "1.0.0-beta.5",
|
||||
"@azure/core-auth": "^1.9.0",
|
||||
"@azure/core-auth": "^1.10.0",
|
||||
"@cfworker/json-schema": "^4.1.1",
|
||||
"@clerk/localizations": "^3.16.3",
|
||||
"@clerk/nextjs": "^6.21.0",
|
||||
"@clerk/themes": "^2.2.48",
|
||||
"@clerk/localizations": "^3.20.1",
|
||||
"@clerk/nextjs": "^6.25.4",
|
||||
"@clerk/themes": "^2.3.3",
|
||||
"@codesandbox/sandpack-react": "^2.20.0",
|
||||
"@cyntler/react-doc-viewer": "^1.17.0",
|
||||
"@electric-sql/pglite": "0.2.17",
|
||||
"@fal-ai/client": "^1.5.0",
|
||||
"@fal-ai/client": "^1.6.1",
|
||||
"@formkit/auto-animate": "^0.8.2",
|
||||
"@google/genai": "^1.6.0",
|
||||
"@google/genai": "^1.10.0",
|
||||
"@huggingface/inference": "^2.8.1",
|
||||
"@icons-pack/react-simple-icons": "9.6.0",
|
||||
"@khmyznikov/pwa-install": "0.3.9",
|
||||
"@langchain/community": "^0.3.45",
|
||||
"@langchain/community": "^0.3.49",
|
||||
"@lobechat/electron-client-ipc": "workspace:*",
|
||||
"@lobechat/electron-server-ipc": "workspace:*",
|
||||
"@lobechat/file-loaders": "workspace:*",
|
||||
|
|
@ -150,29 +150,29 @@
|
|||
"@lobehub/charts": "^2.0.0",
|
||||
"@lobehub/chat-plugin-sdk": "^1.32.4",
|
||||
"@lobehub/chat-plugins-gateway": "^1.9.0",
|
||||
"@lobehub/icons": "^2.2.0",
|
||||
"@lobehub/market-sdk": "^0.22.2",
|
||||
"@lobehub/icons": "^2.17.0",
|
||||
"@lobehub/market-sdk": "^0.22.7",
|
||||
"@lobehub/tts": "^2.0.1",
|
||||
"@lobehub/ui": "^2.7.3",
|
||||
"@modelcontextprotocol/sdk": "^1.12.1",
|
||||
"@neondatabase/serverless": "^1.0.0",
|
||||
"@next/third-parties": "^15.3.3",
|
||||
"@lobehub/ui": "^2.7.4",
|
||||
"@modelcontextprotocol/sdk": "^1.16.0",
|
||||
"@neondatabase/serverless": "^1.0.1",
|
||||
"@next/third-parties": "^15.4.3",
|
||||
"@react-spring/web": "^9.7.5",
|
||||
"@sentry/nextjs": "^7.120.3",
|
||||
"@serwist/next": "^9.0.14",
|
||||
"@serwist/next": "^9.1.1",
|
||||
"@t3-oss/env-nextjs": "^0.12.0",
|
||||
"@tanstack/react-query": "^5.79.0",
|
||||
"@trpc/client": "^11.2.0",
|
||||
"@trpc/next": "^11.1.4",
|
||||
"@trpc/react-query": "^11.2.0",
|
||||
"@trpc/server": "^11.2.0",
|
||||
"@tanstack/react-query": "^5.83.0",
|
||||
"@trpc/client": "^11.4.3",
|
||||
"@trpc/next": "^11.4.3",
|
||||
"@trpc/react-query": "^11.4.3",
|
||||
"@trpc/server": "^11.4.3",
|
||||
"@vercel/analytics": "^1.5.0",
|
||||
"@vercel/edge-config": "^1.4.0",
|
||||
"@vercel/functions": "^2.1.0",
|
||||
"@vercel/functions": "^2.2.4",
|
||||
"@vercel/speed-insights": "^1.2.0",
|
||||
"@xterm/xterm": "^5.5.0",
|
||||
"ahooks": "^3.8.5",
|
||||
"antd": "^5.26.4",
|
||||
"ahooks": "^3.9.0",
|
||||
"antd": "^5.26.6",
|
||||
"antd-style": "^3.7.1",
|
||||
"brotli-wasm": "^3.0.1",
|
||||
"chroma-js": "^3.1.2",
|
||||
|
|
@ -187,12 +187,12 @@
|
|||
"epub2": "^3.0.2",
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"file-type": "^20.5.0",
|
||||
"framer-motion": "^12.15.0",
|
||||
"framer-motion": "^12.23.6",
|
||||
"gpt-tokenizer": "^2.9.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"html-to-text": "^9.0.5",
|
||||
"i18next": "^24.2.3",
|
||||
"i18next-browser-languagedetector": "^8.1.0",
|
||||
"i18next-browser-languagedetector": "^8.2.0",
|
||||
"i18next-resources-to-backend": "^1.2.1",
|
||||
"idb-keyval": "^6.2.2",
|
||||
"immer": "^10.1.1",
|
||||
|
|
@ -200,17 +200,17 @@
|
|||
"js-sha256": "^0.11.1",
|
||||
"jsonl-parse-stringify": "^1.0.3",
|
||||
"keyv": "^4.5.4",
|
||||
"langchain": "^0.3.27",
|
||||
"langfuse": "^3.37.3",
|
||||
"langfuse-core": "^3.37.3",
|
||||
"langchain": "^0.3.30",
|
||||
"langfuse": "^3.38.4",
|
||||
"langfuse-core": "^3.38.4",
|
||||
"lodash-es": "^4.17.21",
|
||||
"lucide-react": "^0.525.0",
|
||||
"mammoth": "^1.9.1",
|
||||
"markdown-to-txt": "^2.0.1",
|
||||
"mdast-util-to-markdown": "^2.1.2",
|
||||
"modern-screenshot": "^4.6.0",
|
||||
"modern-screenshot": "^4.6.5",
|
||||
"nanoid": "^5.1.5",
|
||||
"next": "~15.3.3",
|
||||
"next": "~15.3.5",
|
||||
"next-auth": "5.0.0-beta.29",
|
||||
"next-mdx-remote": "^5.0.0",
|
||||
"nextjs-toploader": "^3.8.16",
|
||||
|
|
@ -218,7 +218,7 @@
|
|||
"numeral": "^2.0.6",
|
||||
"nuqs": "^2.4.3",
|
||||
"officeparser": "5.1.1",
|
||||
"oidc-provider": "^9.2.0",
|
||||
"oidc-provider": "^9.4.0",
|
||||
"ollama": "^0.5.16",
|
||||
"openai": "^4.104.0",
|
||||
"openapi-fetch": "^0.9.8",
|
||||
|
|
@ -226,41 +226,41 @@
|
|||
"path-browserify-esm": "^1.0.6",
|
||||
"pdf-parse": "^1.1.1",
|
||||
"pdfjs-dist": "4.8.69",
|
||||
"pg": "^8.16.0",
|
||||
"pg": "^8.16.3",
|
||||
"pino": "^9.7.0",
|
||||
"plaiceholder": "^3.0.0",
|
||||
"polished": "^4.3.1",
|
||||
"posthog-js": "^1.249.0",
|
||||
"posthog-js": "^1.257.2",
|
||||
"pure-rand": "^7.0.1",
|
||||
"pwa-install-handler": "^2.6.2",
|
||||
"query-string": "^9.2.0",
|
||||
"query-string": "^9.2.2",
|
||||
"random-words": "^2.0.1",
|
||||
"react": "^19.1.0",
|
||||
"react-confetti": "^6.4.0",
|
||||
"react-dom": "^19.1.0",
|
||||
"react-fast-marquee": "^1.6.5",
|
||||
"react-hotkeys-hook": "^5.1.0",
|
||||
"react-i18next": "^15.5.2",
|
||||
"react-layout-kit": "^1.9.1",
|
||||
"react-i18next": "^15.6.1",
|
||||
"react-layout-kit": "^1.9.2",
|
||||
"react-lazy-load": "^4.0.1",
|
||||
"react-pdf": "^9.2.1",
|
||||
"react-rnd": "^10.5.2",
|
||||
"react-scan": "^0.3.4",
|
||||
"react-virtuoso": "^4.12.8",
|
||||
"react-scan": "^0.3.6",
|
||||
"react-virtuoso": "^4.13.0",
|
||||
"react-wrap-balancer": "^1.1.1",
|
||||
"remark": "^15.0.1",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"remark-html": "^16.0.1",
|
||||
"request-filtering-agent": "^2.0.1",
|
||||
"resolve-accept-language": "^3.1.11",
|
||||
"resolve-accept-language": "^3.1.12",
|
||||
"rtl-detect": "^1.1.2",
|
||||
"semver": "^7.7.2",
|
||||
"sharp": "^0.34.2",
|
||||
"shiki": "^3.4.2",
|
||||
"sharp": "^0.34.3",
|
||||
"shiki": "^3.8.1",
|
||||
"stripe": "^16.12.0",
|
||||
"superjson": "^2.2.2",
|
||||
"svix": "^1.66.0",
|
||||
"swr": "^2.3.3",
|
||||
"svix": "^1.69.0",
|
||||
"swr": "^2.3.4",
|
||||
"systemjs": "^6.15.1",
|
||||
"tokenx": "^0.4.1",
|
||||
"ts-md5": "^1.3.1",
|
||||
|
|
@ -270,12 +270,12 @@
|
|||
"url-join": "^5.0.0",
|
||||
"use-merge-value": "^1.2.0",
|
||||
"uuid": "^11.1.0",
|
||||
"ws": "^8.18.2",
|
||||
"ws": "^8.18.3",
|
||||
"y-protocols": "^1.0.6",
|
||||
"y-webrtc": "^10.3.0",
|
||||
"yaml": "^2.8.0",
|
||||
"yjs": "^13.6.27",
|
||||
"zod": "^3.25.48",
|
||||
"zod": "^3.25.76",
|
||||
"zustand": "5.0.4",
|
||||
"zustand-utils": "^2.1.0"
|
||||
},
|
||||
|
|
@ -285,10 +285,10 @@
|
|||
"@huggingface/tasks": "^0.15.9",
|
||||
"@lobehub/i18n-cli": "^1.25.1",
|
||||
"@lobehub/lint": "^1.26.2",
|
||||
"@lobehub/market-types": "^1.11.2",
|
||||
"@lobehub/seo-cli": "^1.6.0",
|
||||
"@next/bundle-analyzer": "^15.3.3",
|
||||
"@next/eslint-plugin-next": "^15.3.3",
|
||||
"@lobehub/market-types": "^1.11.4",
|
||||
"@lobehub/seo-cli": "^1.7.0",
|
||||
"@next/bundle-analyzer": "^15.4.3",
|
||||
"@next/eslint-plugin-next": "^15.4.3",
|
||||
"@peculiar/webcrypto": "^1.5.0",
|
||||
"@prettier/sync": "^0.6.1",
|
||||
"@semantic-release/exec": "^6.0.3",
|
||||
|
|
@ -302,14 +302,14 @@
|
|||
"@types/fs-extra": "^11.0.4",
|
||||
"@types/ip": "^1.1.3",
|
||||
"@types/json-schema": "^7.0.15",
|
||||
"@types/lodash": "^4.17.17",
|
||||
"@types/lodash": "^4.17.20",
|
||||
"@types/lodash-es": "^4.17.12",
|
||||
"@types/node": "^22.15.29",
|
||||
"@types/node": "^22.16.5",
|
||||
"@types/numeral": "^2.0.5",
|
||||
"@types/oidc-provider": "^9.1.1",
|
||||
"@types/pg": "^8.15.4",
|
||||
"@types/react": "^19.1.6",
|
||||
"@types/react-dom": "^19.1.5",
|
||||
"@types/react": "^19.1.8",
|
||||
"@types/react-dom": "^19.1.6",
|
||||
"@types/rtl-detect": "^1.0.3",
|
||||
"@types/semver": "^7.7.0",
|
||||
"@types/systemjs": "^6.15.3",
|
||||
|
|
@ -318,23 +318,23 @@
|
|||
"@types/uuid": "^10.0.0",
|
||||
"@types/ws": "^8.18.1",
|
||||
"@typescript/native-preview": "7.0.0-dev.20250711.1",
|
||||
"@vitest/coverage-v8": "^3.1.4",
|
||||
"@vitest/coverage-v8": "^3.2.4",
|
||||
"ajv-keywords": "^5.1.0",
|
||||
"commitlint": "^19.8.1",
|
||||
"consola": "^3.4.2",
|
||||
"cross-env": "^7.0.3",
|
||||
"crypto-js": "^4.2.0",
|
||||
"dbdocs": "^0.14.4",
|
||||
"dotenv": "^16.5.0",
|
||||
"dotenv": "^16.6.1",
|
||||
"dpdm-fast": "1.0.7",
|
||||
"drizzle-dbml-generator": "^0.10.0",
|
||||
"drizzle-kit": "^0.31.0",
|
||||
"drizzle-kit": "^0.31.4",
|
||||
"eslint": "^8.57.1",
|
||||
"eslint-plugin-mdx": "^3.4.2",
|
||||
"eslint-plugin-mdx": "^3.6.2",
|
||||
"fake-indexeddb": "^6.0.1",
|
||||
"fs-extra": "^11.3.0",
|
||||
"glob": "^11.0.2",
|
||||
"happy-dom": "^17.5.6",
|
||||
"glob": "^11.0.3",
|
||||
"happy-dom": "^17.6.3",
|
||||
"husky": "^9.1.7",
|
||||
"just-diff": "^6.0.2",
|
||||
"lint-staged": "^15.5.2",
|
||||
|
|
@ -346,13 +346,13 @@
|
|||
"node-gyp": "^11.2.0",
|
||||
"openapi-typescript": "^7.8.0",
|
||||
"p-map": "^7.0.3",
|
||||
"prettier": "^3.5.3",
|
||||
"prettier": "^3.6.2",
|
||||
"remark-cli": "^12.0.1",
|
||||
"remark-frontmatter": "^5.0.0",
|
||||
"remark-mdx": "^3.1.0",
|
||||
"remark-parse": "^11.0.0",
|
||||
"semantic-release": "^21.1.2",
|
||||
"serwist": "^9.0.14",
|
||||
"serwist": "^9.1.1",
|
||||
"stylelint": "^15.11.0",
|
||||
"tsx": "~4.19.4",
|
||||
"type-fest": "^4.41.0",
|
||||
|
|
@ -360,7 +360,7 @@
|
|||
"unified": "^11.0.5",
|
||||
"unist-util-visit": "^5.0.0",
|
||||
"vite": "^5.4.19",
|
||||
"vitest": "^3.2.0",
|
||||
"vitest": "^3.2.4",
|
||||
"vitest-canvas-mock": "^0.3.3"
|
||||
},
|
||||
"packageManager": "pnpm@10.10.0",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { CSSProperties, memo, useCallback } from 'react';
|
|||
import { useTranslation } from 'react-i18next';
|
||||
import { Flexbox } from 'react-layout-kit';
|
||||
|
||||
import { MAX_SEED } from '@/libs/standard-parameters/meta-schema';
|
||||
import { MAX_SEED } from '@/libs/standard-parameters/index';
|
||||
import { generateUniqueSeeds } from '@/utils/number';
|
||||
|
||||
export interface SeedNumberInputProps {
|
||||
|
|
|
|||
|
|
@ -14,12 +14,13 @@ import { useTranslation } from 'react-i18next';
|
|||
import { Flexbox } from 'react-layout-kit';
|
||||
|
||||
import InvalidAPIKey from '@/components/InvalidAPIKey';
|
||||
import { RuntimeImageGenParams } from '@/libs/standard-parameters/meta-schema';
|
||||
import { RuntimeImageGenParams } from '@/libs/standard-parameters/index';
|
||||
import { useImageStore } from '@/store/image';
|
||||
import { AsyncTaskErrorType } from '@/types/asyncTask';
|
||||
import { GenerationBatch } from '@/types/generation';
|
||||
|
||||
import { GenerationItem } from './GenerationItem';
|
||||
import { ReferenceImages } from './ReferenceImages';
|
||||
|
||||
const useStyles = createStyles(({ cx, css, token }) => ({
|
||||
batchActions: cx(
|
||||
|
|
@ -132,8 +133,15 @@ export const GenerationBatchItem = memo<GenerationBatchItemProps>(({ batch }) =>
|
|||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Block className={styles.container} gap={8} variant="borderless">
|
||||
// Calculate total number of reference images
|
||||
const referenceImageCount =
|
||||
(batch.config?.imageUrl ? 1 : 0) + (batch.config?.imageUrls?.length || 0);
|
||||
|
||||
const isSingleImageLayout = referenceImageCount === 1;
|
||||
|
||||
// Content for prompt and metadata
|
||||
const promptAndMetadata = (
|
||||
<>
|
||||
<Markdown variant={'chat'}>{batch.prompt}</Markdown>
|
||||
<Flexbox gap={4} horizontal justify="space-between" style={{ marginBottom: 10 }}>
|
||||
<Flexbox gap={4} horizontal>
|
||||
|
|
@ -146,6 +154,34 @@ export const GenerationBatchItem = memo<GenerationBatchItemProps>(({ batch }) =>
|
|||
<Tag>{t('generation.metadata.count', { count: batch.generations.length })}</Tag>
|
||||
</Flexbox>
|
||||
</Flexbox>
|
||||
</>
|
||||
);
|
||||
|
||||
return (
|
||||
<Block className={styles.container} gap={8} variant="borderless">
|
||||
{isSingleImageLayout ? (
|
||||
// Single image layout: horizontal arrangement with vertical centering
|
||||
<Flexbox align="center" gap={16} horizontal>
|
||||
<ReferenceImages
|
||||
imageUrl={batch.config?.imageUrl}
|
||||
imageUrls={batch.config?.imageUrls}
|
||||
layout="single"
|
||||
/>
|
||||
<Flexbox flex={1} gap={8}>
|
||||
{promptAndMetadata}
|
||||
</Flexbox>
|
||||
</Flexbox>
|
||||
) : (
|
||||
// Multiple images or no images: vertical arrangement
|
||||
<>
|
||||
<ReferenceImages
|
||||
imageUrl={batch.config?.imageUrl}
|
||||
imageUrls={batch.config?.imageUrls}
|
||||
layout="multiple"
|
||||
/>
|
||||
{promptAndMetadata}
|
||||
</>
|
||||
)}
|
||||
<Grid maxItemWidth={200} ref={imageGridRef} rows={batch.generations.length || 4}>
|
||||
{batch.generations.map((generation) => (
|
||||
<GenerationItem
|
||||
|
|
|
|||
|
|
@ -0,0 +1,122 @@
|
|||
'use client';
|
||||
|
||||
import { createStyles } from 'antd-style';
|
||||
import { memo } from 'react';
|
||||
import { Flexbox } from 'react-layout-kit';
|
||||
|
||||
import ImageItem from '@/components/ImageItem';
|
||||
|
||||
const useStyles = createStyles(({ css, token }) => ({
|
||||
container: css`
|
||||
gap: 8px;
|
||||
margin-block-end: 12px;
|
||||
`,
|
||||
image: css`
|
||||
overflow: hidden;
|
||||
flex-shrink: 0;
|
||||
|
||||
width: 60px;
|
||||
height: 60px;
|
||||
border-radius: ${token.borderRadius}px;
|
||||
`,
|
||||
imageSingle: css`
|
||||
position: relative;
|
||||
transform: rotate(-3deg);
|
||||
|
||||
flex-shrink: 0;
|
||||
|
||||
width: 64px;
|
||||
height: 64px;
|
||||
|
||||
transition: transform 0.2s ease;
|
||||
|
||||
&::before {
|
||||
content: '';
|
||||
|
||||
position: absolute;
|
||||
z-index: -1;
|
||||
inset: -4px;
|
||||
|
||||
border: 1px solid ${token.colorBorder};
|
||||
border-radius: ${token.borderRadius}px;
|
||||
|
||||
background: ${token.colorBgContainer};
|
||||
box-shadow: 0 2px 8px ${token.colorBgMask};
|
||||
}
|
||||
|
||||
&:hover {
|
||||
transform: rotate(-1deg) scale(1.05);
|
||||
}
|
||||
`,
|
||||
imageSingleInner: css`
|
||||
overflow: hidden;
|
||||
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border-radius: ${token.borderRadiusSM}px;
|
||||
|
||||
background: ${token.colorBgLayout};
|
||||
`,
|
||||
}));
|
||||
|
||||
interface ReferenceImagesProps {
|
||||
imageUrl?: string | null;
|
||||
imageUrls?: string[];
|
||||
layout?: 'single' | 'multiple';
|
||||
}
|
||||
|
||||
export const ReferenceImages = memo<ReferenceImagesProps>(({ imageUrl, imageUrls, layout }) => {
|
||||
const { styles } = useStyles();
|
||||
|
||||
// Collect all images
|
||||
const allImages: string[] = [];
|
||||
if (imageUrl) {
|
||||
allImages.push(imageUrl);
|
||||
}
|
||||
if (imageUrls && imageUrls.length > 0) {
|
||||
allImages.push(...imageUrls);
|
||||
}
|
||||
|
||||
// Don't render if no images
|
||||
if (allImages.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Single image layout (no label, with frame effect)
|
||||
if (layout === 'single' && allImages.length === 1) {
|
||||
return (
|
||||
<div className={styles.imageSingle}>
|
||||
<div className={styles.imageSingleInner}>
|
||||
<ImageItem
|
||||
alt="Reference image"
|
||||
preview={{
|
||||
src: allImages[0],
|
||||
}}
|
||||
style={{ height: '100%', width: '100%' }}
|
||||
url={allImages[0]}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Multiple images layout
|
||||
return (
|
||||
<Flexbox className={styles.container} horizontal wrap="wrap">
|
||||
{allImages.map((url, index) => (
|
||||
<div className={styles.image} key={`${url}-${index}`}>
|
||||
<ImageItem
|
||||
alt={`Reference image ${index + 1}`}
|
||||
preview={{
|
||||
src: url,
|
||||
}}
|
||||
style={{ height: '100%', width: '100%' }}
|
||||
url={url}
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</Flexbox>
|
||||
);
|
||||
});
|
||||
|
||||
ReferenceImages.displayName = 'ReferenceImages';
|
||||
|
|
@ -1,9 +1,13 @@
|
|||
import { ModelParamsSchema } from '@/libs/standard-parameters';
|
||||
import { AIImageModelCard } from '@/types/aiModel';
|
||||
|
||||
import { fluxKontextDevParamsSchema } from '../paramsSchemas/fal/flux-kontext-dev';
|
||||
import { fluxProKontextParamsSchema } from '../paramsSchemas/fal/flux-pro-kontext';
|
||||
import { fluxSchnellParamsSchema } from '../paramsSchemas/fal/flux-schnell';
|
||||
import { imagen4ParamsSchema } from '../paramsSchemas/fal/imagen4';
|
||||
export const fluxSchnellParamsSchema: ModelParamsSchema = {
|
||||
height: { default: 1024, max: 1536, min: 512, step: 1 },
|
||||
prompt: { default: '' },
|
||||
seed: { default: null },
|
||||
steps: { default: 4, max: 12, min: 1 },
|
||||
width: { default: 1024, max: 1536, min: 512, step: 1 },
|
||||
};
|
||||
|
||||
const falImageModels: AIImageModelCard[] = [
|
||||
{
|
||||
|
|
@ -11,7 +15,12 @@ const falImageModels: AIImageModelCard[] = [
|
|||
displayName: 'FLUX.1 Kontext Dev',
|
||||
enabled: true,
|
||||
id: 'flux-kontext/dev',
|
||||
parameters: fluxKontextDevParamsSchema,
|
||||
parameters: {
|
||||
imageUrl: { default: null },
|
||||
prompt: { default: '' },
|
||||
seed: { default: null },
|
||||
steps: { default: 28, max: 50, min: 10 },
|
||||
},
|
||||
releasedAt: '2025-06-28',
|
||||
type: 'image',
|
||||
},
|
||||
|
|
@ -21,7 +30,15 @@ const falImageModels: AIImageModelCard[] = [
|
|||
displayName: 'FLUX.1 Kontext [pro]',
|
||||
enabled: true,
|
||||
id: 'flux-pro/kontext',
|
||||
parameters: fluxProKontextParamsSchema,
|
||||
parameters: {
|
||||
aspectRatio: {
|
||||
default: '1:1',
|
||||
enum: ['21:9', '16:9', '4:3', '3:2', '1:1', '2:3', '3:4', '9:16', '9:21'],
|
||||
},
|
||||
imageUrl: { default: null },
|
||||
prompt: { default: '' },
|
||||
seed: { default: null },
|
||||
},
|
||||
releasedAt: '2025-05-01',
|
||||
type: 'image',
|
||||
},
|
||||
|
|
@ -41,7 +58,14 @@ const falImageModels: AIImageModelCard[] = [
|
|||
enabled: true,
|
||||
id: 'imagen4/preview',
|
||||
organization: 'Deepmind',
|
||||
parameters: imagen4ParamsSchema,
|
||||
parameters: {
|
||||
aspectRatio: {
|
||||
default: '1:1',
|
||||
enum: ['1:1', '16:9', '9:16', '3:4', '4:3'],
|
||||
},
|
||||
prompt: { default: '' },
|
||||
seed: { default: null },
|
||||
},
|
||||
releasedAt: '2025-05-21',
|
||||
type: 'image',
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { gptImage1ParamsSchema } from '@/config/paramsSchemas/openai/gpt-image-1';
|
||||
import { ModelParamsSchema } from '@/libs/standard-parameters';
|
||||
import {
|
||||
AIChatModelCard,
|
||||
AIEmbeddingModelCard,
|
||||
|
|
@ -8,6 +8,15 @@ import {
|
|||
AITTSModelCard,
|
||||
} from '@/types/aiModel';
|
||||
|
||||
export const gptImage1ParamsSchema: ModelParamsSchema = {
|
||||
imageUrls: { default: [] },
|
||||
prompt: { default: '' },
|
||||
size: {
|
||||
default: 'auto',
|
||||
enum: ['auto', '1024x1024', '1536x1024', '1024x1536'],
|
||||
},
|
||||
};
|
||||
|
||||
export const openaiChatModels: AIChatModelCard[] = [
|
||||
{
|
||||
abilities: {
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
import { type ModelParamsSchema } from '@/libs/standard-parameters/meta-schema';
|
||||
|
||||
export const fluxKontextDevParamsSchema: ModelParamsSchema = {
|
||||
imageUrl: { default: null },
|
||||
prompt: { default: '' },
|
||||
seed: { default: null },
|
||||
steps: { default: 28, max: 50, min: 10 },
|
||||
};
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
import { type ModelParamsSchema } from '@/libs/standard-parameters/meta-schema';
|
||||
|
||||
export const fluxProKontextParamsSchema: ModelParamsSchema = {
|
||||
aspectRatio: {
|
||||
default: '1:1',
|
||||
enum: ['21:9', '16:9', '4:3', '3:2', '1:1', '2:3', '3:4', '9:16', '9:21'],
|
||||
},
|
||||
imageUrl: { default: null },
|
||||
prompt: { default: '' },
|
||||
seed: { default: null },
|
||||
};
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
import { type ModelParamsSchema } from '@/libs/standard-parameters/meta-schema';
|
||||
|
||||
export const fluxSchnellParamsSchema: ModelParamsSchema = {
|
||||
height: { default: 1024, max: 1536, min: 512, step: 1 },
|
||||
prompt: { default: '' },
|
||||
seed: { default: null },
|
||||
steps: { default: 4, max: 12, min: 1 },
|
||||
width: { default: 1024, max: 1536, min: 512, step: 1 },
|
||||
};
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
import { type ModelParamsSchema } from '@/libs/standard-parameters/meta-schema';
|
||||
|
||||
export const imagen4ParamsSchema: ModelParamsSchema = {
|
||||
aspectRatio: {
|
||||
default: '1:1',
|
||||
enum: ['1:1', '16:9', '9:16', '3:4', '4:3'],
|
||||
},
|
||||
prompt: { default: '' },
|
||||
seed: { default: null },
|
||||
};
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
import { type ModelParamsSchema } from '@/libs/standard-parameters/meta-schema';
|
||||
|
||||
export const gptImage1ParamsSchema: ModelParamsSchema = {
|
||||
imageUrls: { default: [] },
|
||||
prompt: { default: '' },
|
||||
size: {
|
||||
default: 'auto',
|
||||
enum: ['auto', '1024x1024', '1536x1024', '1024x1536'],
|
||||
},
|
||||
};
|
||||
|
|
@ -67,7 +67,6 @@ const ImageTopicPanel = memo<PropsWithChildren>(({ children }) => {
|
|||
flex: 'none',
|
||||
height: '100%',
|
||||
minWidth: 80,
|
||||
padding: 16,
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import debug from 'debug';
|
|||
import { pick } from 'lodash-es';
|
||||
import { ClientOptions } from 'openai';
|
||||
|
||||
import { RuntimeImageGenParamsValue } from '@/libs/standard-parameters/meta-schema';
|
||||
import { RuntimeImageGenParamsValue } from '@/libs/standard-parameters/index';
|
||||
|
||||
import { LobeRuntimeAI } from '../BaseAI';
|
||||
import { AgentRuntimeErrorType } from '../error';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { RuntimeImageGenParams } from '@/libs/standard-parameters/meta-schema';
|
||||
import { RuntimeImageGenParams } from '@/libs/standard-parameters/index';
|
||||
|
||||
export type CreateImagePayload = {
|
||||
model: string;
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import OpenAI, { ClientOptions } from 'openai';
|
|||
import { Stream } from 'openai/streaming';
|
||||
|
||||
import { LOBE_DEFAULT_MODEL_LIST } from '@/config/aiModels';
|
||||
import { RuntimeImageGenParamsValue } from '@/libs/standard-parameters/meta-schema';
|
||||
import { RuntimeImageGenParamsValue } from '@/libs/standard-parameters/index';
|
||||
import type { ChatModelCard } from '@/types/llm';
|
||||
import { getModelPropertyWithFallback } from '@/utils/getFallbackModelProperty';
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import {
|
|||
type RuntimeImageGenParams,
|
||||
extractDefaultValues,
|
||||
validateModelParamsSchema,
|
||||
} from './meta-schema';
|
||||
} from './index';
|
||||
|
||||
describe('meta-schema', () => {
|
||||
describe('ModelParamsMetaSchema', () => {
|
||||
|
|
@ -1 +1,152 @@
|
|||
export * from './meta-schema';
|
||||
import type { Simplify } from 'type-fest';
|
||||
import { z } from 'zod';
|
||||
|
||||
export const MAX_SEED = 2 ** 31 - 1;
|
||||
|
||||
// 定义顶层的元规范 - 平铺结构
|
||||
export const ModelParamsMetaSchema = z.object({
|
||||
aspectRatio: z
|
||||
.object({
|
||||
default: z.string(),
|
||||
description: z.string().optional(),
|
||||
enum: z.array(z.string()),
|
||||
type: z.literal('string').optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
cfg: z
|
||||
.object({
|
||||
default: z.number(),
|
||||
description: z.string().optional(),
|
||||
max: z.number(),
|
||||
min: z.number(),
|
||||
step: z.number(),
|
||||
type: z.literal('number').optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
height: z
|
||||
.object({
|
||||
default: z.number(),
|
||||
description: z.string().optional(),
|
||||
max: z.number(),
|
||||
min: z.number(),
|
||||
step: z.number().optional().default(1),
|
||||
type: z.literal('number').optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
imageUrl: z
|
||||
.object({
|
||||
default: z.string().nullable().optional(),
|
||||
description: z.string().optional(),
|
||||
type: z.tuple([z.literal('string'), z.literal('null')]).optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
imageUrls: z
|
||||
.object({
|
||||
default: z.array(z.string()),
|
||||
description: z.string().optional(),
|
||||
type: z.literal('array').optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
/**
|
||||
* Prompt 是唯一一个每个模型都有的参数
|
||||
*/
|
||||
prompt: z.object({
|
||||
default: z.string().optional().default(''),
|
||||
description: z.string().optional(),
|
||||
type: z.literal('string').optional(),
|
||||
}),
|
||||
|
||||
seed: z
|
||||
.object({
|
||||
default: z.number().nullable().default(null),
|
||||
description: z.string().optional(),
|
||||
max: z.number().optional().default(MAX_SEED),
|
||||
min: z.number().optional().default(0),
|
||||
type: z.tuple([z.literal('number'), z.literal('null')]).optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
size: z
|
||||
.object({
|
||||
default: z.string(),
|
||||
description: z.string().optional(),
|
||||
enum: z.array(z.string()),
|
||||
type: z.literal('string').optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
steps: z
|
||||
.object({
|
||||
default: z.number(),
|
||||
description: z.string().optional(),
|
||||
max: z.number(),
|
||||
min: z.number(),
|
||||
step: z.number().optional().default(1),
|
||||
type: z.literal('number').optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
width: z
|
||||
.object({
|
||||
default: z.number(),
|
||||
description: z.string().optional(),
|
||||
max: z.number(),
|
||||
min: z.number(),
|
||||
step: z.number().optional().default(1),
|
||||
type: z.literal('number').optional(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
// 导出推断出的类型,供定义对象使用
|
||||
export type ModelParamsSchema = z.input<typeof ModelParamsMetaSchema>;
|
||||
export type ModelParamsOutputSchema = z.output<typeof ModelParamsMetaSchema>;
|
||||
export type ModelParamsKeys = Simplify<keyof ModelParamsOutputSchema>;
|
||||
|
||||
type TypeMapping<T> = T extends 'string'
|
||||
? string
|
||||
: T extends 'number'
|
||||
? number
|
||||
: T extends ['number', 'null']
|
||||
? number | null
|
||||
: T extends ['string', 'null']
|
||||
? string | null
|
||||
: T extends 'string'
|
||||
? string
|
||||
: T extends 'boolean'
|
||||
? boolean
|
||||
: never;
|
||||
type TypeType<K extends ModelParamsKeys> = NonNullable<ModelParamsOutputSchema[K]>['type'];
|
||||
type DefaultType<K extends ModelParamsKeys> = NonNullable<ModelParamsOutputSchema[K]>['default'];
|
||||
type _StandardImageGenerationParameters<P extends ModelParamsKeys = ModelParamsKeys> = {
|
||||
[key in P]: NonNullable<TypeType<key>> extends 'array'
|
||||
? DefaultType<key>
|
||||
: TypeMapping<TypeType<key>>;
|
||||
};
|
||||
|
||||
export type RuntimeImageGenParams = Pick<_StandardImageGenerationParameters, 'prompt'> &
|
||||
Partial<Omit<_StandardImageGenerationParameters, 'prompt'>>;
|
||||
export type RuntimeImageGenParamsKeys = keyof RuntimeImageGenParams;
|
||||
export type RuntimeImageGenParamsValue = RuntimeImageGenParams[RuntimeImageGenParamsKeys];
|
||||
|
||||
// 验证函数
|
||||
export function validateModelParamsSchema(paramsSchema: unknown): ModelParamsOutputSchema {
|
||||
return ModelParamsMetaSchema.parse(paramsSchema);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从参数定义对象提取默认值
|
||||
*/
|
||||
export function extractDefaultValues(paramsSchema: ModelParamsSchema) {
|
||||
// 部分默认值从 ModelParamsMetaSchema 中获取
|
||||
const schemaWithDefault = ModelParamsMetaSchema.parse(paramsSchema);
|
||||
return Object.fromEntries(
|
||||
Object.entries(schemaWithDefault).map(([key, value]) => {
|
||||
return [key, value.default];
|
||||
}),
|
||||
) as RuntimeImageGenParams;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,147 +0,0 @@
|
|||
import type { Simplify } from 'type-fest';
|
||||
import { z } from 'zod';
|
||||
|
||||
export const MAX_SEED = 2 ** 31 - 1;
|
||||
|
||||
// 定义顶层的元规范 - 平铺结构
|
||||
export const ModelParamsMetaSchema = z.object({
|
||||
aspectRatio: z
|
||||
.object({
|
||||
default: z.string(),
|
||||
description: z.string().optional(),
|
||||
enum: z.array(z.string()),
|
||||
type: z.literal('string').optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
cfg: z
|
||||
.object({
|
||||
default: z.number(),
|
||||
description: z.string().optional(),
|
||||
max: z.number(),
|
||||
min: z.number(),
|
||||
step: z.number(),
|
||||
type: z.literal('number').optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
height: z
|
||||
.object({
|
||||
default: z.number(),
|
||||
description: z.string().optional(),
|
||||
max: z.number(),
|
||||
min: z.number(),
|
||||
step: z.number().optional().default(1),
|
||||
type: z.literal('number').optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
imageUrl: z
|
||||
.object({
|
||||
default: z.string().nullable().optional(),
|
||||
description: z.string().optional(),
|
||||
type: z.tuple([z.literal('string'), z.literal('null')]).optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
imageUrls: z
|
||||
.object({
|
||||
default: z.array(z.string()),
|
||||
description: z.string().optional(),
|
||||
type: z.literal('array').optional(),
|
||||
})
|
||||
.optional(),
|
||||
/**
|
||||
* Prompt 是唯一一个每个模型都有的参数
|
||||
*/
|
||||
prompt: z.object({
|
||||
default: z.string().optional().default(''),
|
||||
description: z.string().optional(),
|
||||
type: z.literal('string').optional(),
|
||||
}),
|
||||
seed: z
|
||||
.object({
|
||||
default: z.number().nullable().default(null),
|
||||
description: z.string().optional(),
|
||||
max: z.number().optional().default(MAX_SEED),
|
||||
min: z.number().optional().default(0),
|
||||
type: z.tuple([z.literal('number'), z.literal('null')]).optional(),
|
||||
})
|
||||
.optional(),
|
||||
size: z
|
||||
.object({
|
||||
default: z.string(),
|
||||
description: z.string().optional(),
|
||||
enum: z.array(z.string()),
|
||||
type: z.literal('string').optional(),
|
||||
})
|
||||
.optional(),
|
||||
steps: z
|
||||
.object({
|
||||
default: z.number(),
|
||||
description: z.string().optional(),
|
||||
max: z.number(),
|
||||
min: z.number(),
|
||||
step: z.number().optional().default(1),
|
||||
type: z.literal('number').optional(),
|
||||
})
|
||||
.optional(),
|
||||
width: z
|
||||
.object({
|
||||
default: z.number(),
|
||||
description: z.string().optional(),
|
||||
max: z.number(),
|
||||
min: z.number(),
|
||||
step: z.number().optional().default(1),
|
||||
type: z.literal('number').optional(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
// 导出推断出的类型,供定义对象使用
|
||||
export type ModelParamsSchema = z.input<typeof ModelParamsMetaSchema>;
|
||||
export type ModelParamsOutputSchema = z.output<typeof ModelParamsMetaSchema>;
|
||||
export type ModelParamsKeys = Simplify<keyof ModelParamsOutputSchema>;
|
||||
|
||||
type TypeMapping<T> = T extends 'string'
|
||||
? string
|
||||
: T extends 'number'
|
||||
? number
|
||||
: T extends ['number', 'null']
|
||||
? number | null
|
||||
: T extends ['string', 'null']
|
||||
? string | null
|
||||
: T extends 'string'
|
||||
? string
|
||||
: T extends 'boolean'
|
||||
? boolean
|
||||
: never;
|
||||
type TypeType<K extends ModelParamsKeys> = NonNullable<ModelParamsOutputSchema[K]>['type'];
|
||||
type DefaultType<K extends ModelParamsKeys> = NonNullable<ModelParamsOutputSchema[K]>['default'];
|
||||
type _StandardImageGenerationParameters<P extends ModelParamsKeys = ModelParamsKeys> = {
|
||||
[key in P]: NonNullable<TypeType<key>> extends 'array'
|
||||
? DefaultType<key>
|
||||
: TypeMapping<TypeType<key>>;
|
||||
};
|
||||
|
||||
export type RuntimeImageGenParams = Pick<_StandardImageGenerationParameters, 'prompt'> &
|
||||
Partial<Omit<_StandardImageGenerationParameters, 'prompt'>>;
|
||||
export type RuntimeImageGenParamsKeys = keyof RuntimeImageGenParams;
|
||||
export type RuntimeImageGenParamsValue = RuntimeImageGenParams[RuntimeImageGenParamsKeys];
|
||||
|
||||
// 验证函数
|
||||
export function validateModelParamsSchema(paramsSchema: unknown): ModelParamsOutputSchema {
|
||||
return ModelParamsMetaSchema.parse(paramsSchema);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从参数定义对象提取默认值
|
||||
*/
|
||||
export function extractDefaultValues(paramsSchema: ModelParamsSchema) {
|
||||
// 部分默认值从 ModelParamsMetaSchema 中获取
|
||||
const schemaWithDefault = ModelParamsMetaSchema.parse(paramsSchema);
|
||||
return Object.fromEntries(
|
||||
Object.entries(schemaWithDefault).map(([key, value]) => {
|
||||
return [key, value.default];
|
||||
}),
|
||||
) as RuntimeImageGenParams;
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ import { ASYNC_TASK_TIMEOUT, AsyncTaskModel } from '@/database/models/asyncTask'
|
|||
import { FileModel } from '@/database/models/file';
|
||||
import { GenerationModel } from '@/database/models/generation';
|
||||
import { AgentRuntimeErrorType } from '@/libs/model-runtime/error';
|
||||
import { RuntimeImageGenParams } from '@/libs/standard-parameters/meta-schema';
|
||||
import { RuntimeImageGenParams } from '@/libs/standard-parameters/index';
|
||||
import { asyncAuthedProcedure, asyncRouter as router } from '@/libs/trpc/async';
|
||||
import { initAgentRuntimeWithUserPayload } from '@/server/modules/AgentRuntime';
|
||||
import { GenerationService } from '@/server/services/generation';
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import { act, renderHook } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { fluxSchnellParamsSchema } from '@/config/paramsSchemas/fal/flux-schnell';
|
||||
import { fluxSchnellParamsSchema } from '@/config/aiModels/fal';
|
||||
import {
|
||||
ModelParamsSchema,
|
||||
RuntimeImageGenParams,
|
||||
extractDefaultValues,
|
||||
} from '@/libs/standard-parameters/meta-schema';
|
||||
} from '@/libs/standard-parameters/index';
|
||||
import { useImageStore } from '@/store/image';
|
||||
import { AIImageModelCard } from '@/types/aiModel';
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import {
|
|||
RuntimeImageGenParamsKeys,
|
||||
RuntimeImageGenParamsValue,
|
||||
extractDefaultValues,
|
||||
} from '@/libs/standard-parameters/meta-schema';
|
||||
} from '@/libs/standard-parameters/index';
|
||||
import { aiProviderSelectors, getAiInfraStoreState } from '@/store/aiInfra';
|
||||
import { AIImageModelCard } from '@/types/aiModel';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { act, renderHook } from '@testing-library/react';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { fluxSchnellParamsSchema } from '@/config/paramsSchemas/fal/flux-schnell';
|
||||
import { ModelParamsSchema, RuntimeImageGenParams } from '@/libs/standard-parameters/meta-schema';
|
||||
import { fluxSchnellParamsSchema } from '@/config/aiModels/fal';
|
||||
import { ModelParamsSchema, RuntimeImageGenParams } from '@/libs/standard-parameters/index';
|
||||
import { useImageStore } from '@/store/image';
|
||||
import { AIImageModelCard } from '@/types/aiModel';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
import { useCallback, useMemo } from 'react';
|
||||
|
||||
import { DEFAULT_ASPECT_RATIO, PRESET_ASPECT_RATIOS } from '@/const/image';
|
||||
import {
|
||||
RuntimeImageGenParams,
|
||||
RuntimeImageGenParamsKeys,
|
||||
} from '@/libs/standard-parameters/meta-schema';
|
||||
import { RuntimeImageGenParams, RuntimeImageGenParamsKeys } from '@/libs/standard-parameters/index';
|
||||
|
||||
import { useImageStore } from '../../store';
|
||||
import { imageGenerationConfigSelectors } from './selectors';
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
/* eslint-disable sort-keys-fix/sort-keys-fix, typescript-sort-keys/interface */
|
||||
import { gptImage1ParamsSchema } from '@/config/paramsSchemas/openai/gpt-image-1';
|
||||
import { gptImage1ParamsSchema } from '@/config/aiModels/openai';
|
||||
import { ModelProvider } from '@/libs/model-runtime/types/type';
|
||||
import {
|
||||
ModelParamsSchema,
|
||||
RuntimeImageGenParams,
|
||||
extractDefaultValues,
|
||||
} from '@/libs/standard-parameters/meta-schema';
|
||||
} from '@/libs/standard-parameters/index';
|
||||
|
||||
export const DEFAULT_AI_IMAGE_PROVIDER = ModelProvider.OpenAI;
|
||||
export const DEFAULT_AI_IMAGE_MODEL = 'gpt-image-1';
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { gptImage1ParamsSchema } from '@/config/paramsSchemas/openai/gpt-image-1';
|
||||
import { ModelParamsSchema, RuntimeImageGenParams } from '@/libs/standard-parameters/meta-schema';
|
||||
import { gptImage1ParamsSchema } from '@/config/aiModels/openai';
|
||||
import { ModelParamsSchema, RuntimeImageGenParams } from '@/libs/standard-parameters/index';
|
||||
import { ImageStore } from '@/store/image';
|
||||
import { initialState } from '@/store/image/initialState';
|
||||
import { AIImageModelCard } from '@/types/aiModel';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { RuntimeImageGenParamsKeys } from '@/libs/standard-parameters/meta-schema';
|
||||
import { RuntimeImageGenParamsKeys } from '@/libs/standard-parameters/index';
|
||||
|
||||
import { GenerationConfigState } from './initialState';
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ export type GenerationAsset = ImageGenerationAsset;
|
|||
|
||||
export interface GenerationConfig {
|
||||
prompt: string;
|
||||
imageUrl?: string | null;
|
||||
imageUrls?: string[];
|
||||
width?: number;
|
||||
height?: number;
|
||||
|
|
|
|||
Loading…
Reference in a new issue