Skip to content

Commit af0243c

Browse files
arvinxxarvinxx
authored
✨ feat: support gemini image output in chat (#6956)
* support gemini image * fix lint * fix * fix upload bug * support google token count * update credit * update i18n
1 parent 8495844 commit af0243c

File tree

42 files changed

+551
-86
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+551
-86
lines changed

‎locales/ar/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash يقدم ميزات وتحسينات من الجيل التالي، بما في ذلك سرعة فائقة، واستخدام أدوات أصلية، وتوليد متعدد الوسائط، ونافذة سياق تصل إلى 1M توكن."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "نموذج جمنيس 2.0 فلاش، تم تحسينه لتحقيق أهداف مثل الكفاءة من حيث التكلفة وانخفاض الكمون."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "نموذج جمنّي 2.0 فلاش هو نسخة معدلة، تم تحسينها لتحقيق الكفاءة من حيث التكلفة والحد من التأخير."
838841
},

‎locales/bg-BG/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash предлага следващо поколение функции и подобрения, включително изключителна скорост, нативна употреба на инструменти, многомодално генериране и контекстен прозорец от 1M токена."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash моделна вариация, оптимизирана за икономичност и ниска латентност."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flash е вариант на модела, оптимизиран за икономичност и ниска латентност."
838841
},

‎locales/de-DE/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash bietet nächste Generation Funktionen und Verbesserungen, einschließlich außergewöhnlicher Geschwindigkeit, nativer Werkzeugnutzung, multimodaler Generierung und einem Kontextfenster von 1M Tokens."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash-Modellvariante, die auf Kosteneffizienz und niedrige Latenz optimiert ist."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flash ist eine Modellvariante, die auf Kosteneffizienz und niedrige Latenz optimiert ist."
838841
},

‎locales/en-US/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash offers next-generation features and improvements, including exceptional speed, native tool usage, multimodal generation, and a 1M token context window."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash model variant optimized for cost-effectiveness and low latency."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flash is a variant of the model optimized for cost-effectiveness and low latency."
838841
},

‎locales/es-ES/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash ofrece funciones y mejoras de próxima generación, incluyendo velocidad excepcional, uso de herramientas nativas, generación multimodal y una ventana de contexto de 1M tokens."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Variante del modelo Gemini 2.0 Flash, optimizada para objetivos como la rentabilidad y la baja latencia."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Variante del modelo Gemini 2.0 Flash, optimizada para objetivos como la rentabilidad y la baja latencia."
838841
},

‎locales/fa-IR/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash ویژگی‌ها و بهبودهای نسل بعدی را ارائه می‌دهد، از جمله سرعت عالی، استفاده از ابزارهای بومی، تولید چندرسانه‌ای و پنجره متن 1M توکن."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "مدل متغیر Gemini 2.0 Flash که برای بهینه‌سازی هزینه و تأخیر کم طراحی شده است."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "مدل متغیر Gemini 2.0 Flash برای بهینه‌سازی هزینه و تأخیر کم طراحی شده است."
838841
},

‎locales/fr-FR/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash propose des fonctionnalités et des améliorations de nouvelle génération, y compris une vitesse exceptionnelle, l'utilisation d'outils natifs, la génération multimodale et une fenêtre de contexte de 1M tokens."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Modèle variant Gemini 2.0 Flash, optimisé pour des objectifs tels que le rapport coût-efficacité et la faible latence."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Une variante du modèle Gemini 2.0 Flash, optimisée pour des objectifs tels que le rapport coût-efficacité et la faible latence."
838841
},

‎locales/it-IT/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash offre funzionalità e miglioramenti di nuova generazione, tra cui velocità eccezionale, utilizzo di strumenti nativi, generazione multimodale e una finestra di contesto di 1M token."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash è una variante del modello ottimizzata per obiettivi come il rapporto costo-efficacia e la bassa latenza."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flash è una variante del modello Flash, ottimizzata per obiettivi come il rapporto costo-efficacia e la bassa latenza."
838841
},

‎locales/ja-JP/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flashは、卓越した速度、ネイティブツールの使用、マルチモーダル生成、1Mトークンのコンテキストウィンドウを含む次世代の機能と改善を提供します。"
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash モデルのバリアントで、コスト効率と低遅延などの目標に最適化されています。"
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flashモデルのバリアントで、コスト効率と低遅延などの目標に最適化されています。"
838841
},

‎locales/ko-KR/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash는 뛰어난 속도, 원주율 도구 사용, 다중 모달 생성 및 1M 토큰 문맥 창을 포함한 차세대 기능과 개선 사항을 제공합니다."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash 모델 변형으로, 비용 효율성과 저지연 등의 목표를 위해 최적화되었습니다."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 플래시 모델 변형으로, 비용 효율성과 낮은 지연 시간 등의 목표를 위해 최적화되었습니다."
838841
},

‎locales/nl-NL/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash biedt next-gen functies en verbeteringen, waaronder uitstekende snelheid, native toolgebruik, multimodale generatie en een contextvenster van 1M tokens."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash modelvariant, geoptimaliseerd voor kosteneffectiviteit en lage latentie."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flash is een modelvariant die is geoptimaliseerd voor kosteneffectiviteit en lage latentie."
838841
},

‎locales/pl-PL/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash oferuje funkcje i ulepszenia nowej generacji, w tym doskonałą prędkość, natywne korzystanie z narzędzi, generowanie multimodalne oraz okno kontekstowe o długości 1M tokenów."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash to wariant modelu, zoptymalizowany pod kątem efektywności kosztowej i niskiego opóźnienia."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flash to wariant modelu, zoptymalizowany pod kątem efektywności kosztowej i niskiego opóźnienia."
838841
},

‎locales/pt-BR/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash oferece funcionalidades e melhorias de próxima geração, incluindo velocidade excepcional, uso nativo de ferramentas, geração multimodal e uma janela de contexto de 1M tokens."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Variante do modelo Gemini 2.0 Flash, otimizada para custo-benefício e baixa latência."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Variante do modelo Gemini 2.0 Flash, otimizada para custo-benefício e baixa latência."
838841
},

‎locales/ru-RU/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash предлагает функции следующего поколения и улучшения, включая выдающуюся скорость, использование встроенных инструментов, многомодальную генерацию и контекстное окно на 1M токенов."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Модельный вариант Gemini 2.0 Flash, оптимизированный для достижения таких целей, как экономическая эффективность и низкая задержка."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Модельный вариант Gemini 2.0 Flash, оптимизированный для достижения таких целей, как экономическая эффективность и низкая задержка."
838841
},

‎locales/tr-TR/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash, mükemmel hız, yerel araç kullanımı, çok modlu üretim ve 1M token bağlam penceresi dahil olmak üzere bir sonraki nesil özellikler ve iyileştirmeler sunar."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash modeli varyantı, maliyet etkinliği ve düşük gecikme gibi hedefler için optimize edilmiştir."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flash model varyantı, maliyet etkinliği ve düşük gecikme gibi hedefler için optimize edilmiştir."
838841
},

‎locales/vi-VN/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash cung cấp các tính năng và cải tiến thế hệ tiếp theo, bao gồm tốc độ vượt trội, sử dụng công cụ bản địa, tạo đa phương tiện và cửa sổ ngữ cảnh 1M token."
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Biến thể mô hình Gemini 2.0 Flash, được tối ưu hóa cho hiệu quả chi phí và độ trễ thấp."
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Biến thể mô hình Gemini 2.0 Flash được tối ưu hóa cho hiệu quả chi phí và độ trễ thấp."
838841
},

‎locales/zh-CN/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash 提供下一代功能和改进，包括卓越的速度、原生工具使用、多模态生成和1M令牌上下文窗口。"
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash 模型变体，针对成本效益和低延迟等目标进行了优化。"
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flash 模型变体，针对成本效益和低延迟等目标进行了优化。"
838841
},

‎locales/zh-TW/models.json

+3
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,9 @@
833833
"gemini-2.0-flash-001": {
834834
"description": "Gemini 2.0 Flash 提供下一代功能和改進，包括卓越的速度、原生工具使用、多模態生成和1M令牌上下文窗口。"
835835
},
836+
"gemini-2.0-flash-exp": {
837+
"description": "Gemini 2.0 Flash 模型變體，針對成本效益和低延遲等目標進行了優化。"
838+
},
836839
"gemini-2.0-flash-lite": {
837840
"description": "Gemini 2.0 Flash 模型變體，針對成本效益和低延遲等目標進行了優化。"
838841
},

‎src/config/aiModels/google.ts

+17
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,23 @@ const googleChatModels: AIChatModelCard[] = [
9292
releasedAt: '2025-02-05',
9393
type: 'chat',
9494
},
95+
{
96+
abilities: {
97+
vision: true,
98+
},
99+
contextWindowTokens: 32_768,
100+
description: 'Gemini 2.0 Flash 模型变体，针对成本效益和低延迟等目标进行了优化。',
101+
displayName: 'Gemini 2.0 Flash Exp',
102+
enabled: true,
103+
id: 'gemini-2.0-flash-exp',
104+
maxOutput: 8192,
105+
pricing: {
106+
input: 0,
107+
output: 0,
108+
},
109+
releasedAt: '2025-02-05',
110+
type: 'chat',
111+
},
95112
{
96113
abilities: {
97114
vision: true,

‎src/database/server/models/message.ts

+20-9
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
CreateMessageParams,
2222
MessageItem,
2323
ModelRankItem,
24+
UpdateMessageParams,
2425
} from '@/types/message';
2526
import { merge } from '@/utils/merge';
2627
import { today } from '@/utils/time';
@@ -497,15 +498,25 @@ export class MessageModel {
497498
};
498499
// **************** Update *************** //
499500

500-
update = async (id: string, message: Partial<MessageItem>) => {
501-
return this.db
502-
.update(messages)
503-
.set({
504-
...message,
505-
// TODO: need a better way to handle this
506-
role: message.role as any,
507-
})
508-
.where(and(eq(messages.id, id), eq(messages.userId, this.userId)));
501+
update = async (id: string, { imageList, ...message }: Partial<UpdateMessageParams>) => {
502+
return this.db.transaction(async (trx) => {
503+
// 1. insert message files
504+
if (imageList && imageList.length > 0) {
505+
await trx
506+
.insert(messagesFiles)
507+
.values(imageList.map((file) => ({ fileId: file.id, messageId: id })));
508+
}
509+
510+
return trx
511+
.update(messages)
512+
.set({
513+
...message,
514+
// TODO: need a better way to handle this
515+
// TODO: but I forget why 🤡
516+
role: message.role as any,
517+
})
518+
.where(and(eq(messages.id, id), eq(messages.userId, this.userId)));
519+
});
509520
};
510521

511522
updatePluginState = async (id: string, state: Record<string, any>) => {

‎src/features/AlertBanner/CloudBanner.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ const CloudBanner = memo<{ mobile?: boolean }>(({ mobile }) => {
6161
<b>{t('alert.cloud.title', { name: LOBE_CHAT_CLOUD })}:</b>
6262
<span>
6363
{t(mobile ? 'alert.cloud.descOnMobile' : 'alert.cloud.desc', {
64-
credit: new Intl.NumberFormat('en-US').format(450_000),
64+
credit: new Intl.NumberFormat('en-US').format(500_000),
6565
name: LOBE_CHAT_CLOUD,
6666
})}
6767
</span>

‎src/features/Conversation/Messages/Assistant/index.tsx

+4-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { ReactNode, memo } from 'react';
22
import { Flexbox } from 'react-layout-kit';
33

44
import { LOADING_FLAT } from '@/const/message';
5+
import ImageFileListViewer from '@/features/Conversation/Messages/User/ImageFileListViewer';
56
import { useChatStore } from '@/store/chat';
67
import { aiChatSelectors, chatSelectors } from '@/store/chat/selectors';
78
import { ChatMessage } from '@/types/message';
@@ -17,7 +18,7 @@ export const AssistantMessage = memo<
1718
ChatMessage & {
1819
editableContent: ReactNode;
1920
}
20-
>(({ id, tools, content, chunksList, search, ...props }) => {
21+
>(({ id, tools, content, chunksList, search, imageList, ...props }) => {
2122
const editing = useChatStore(chatSelectors.isMessageEditing(id));
2223
const generating = useChatStore(chatSelectors.isMessageGenerating(id));
2324

@@ -28,6 +29,7 @@ export const AssistantMessage = memo<
2829
const isIntentUnderstanding = useChatStore(aiChatSelectors.isIntentUnderstanding(id));
2930

3031
const showSearch = !!search && !!search.citations?.length;
32+
const showImageItems = !!imageList && imageList.length > 0;
3133

3234
// remove \n to avoid empty content
3335
// refs: https://github.com/lobehub/lobe-chat/pull/6153
@@ -64,6 +66,7 @@ export const AssistantMessage = memo<
6466
/>
6567
)
6668
)}
69+
{showImageItems && <ImageFileListViewer items={imageList} />}
6770
{tools && (
6871
<Flexbox gap={8}>
6972
{tools.map((toolCall, index) => (

‎src/features/Conversation/Messages/User/index.tsx

+4-4
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,16 @@ export const UserMessage = memo<
1212
ChatMessage & {
1313
editableContent: ReactNode;
1414
}
15-
>(({ id, editableContent, content, ...res }) => {
15+
>(({ id, editableContent, content, imageList, fileList }) => {
1616
if (content === LOADING_FLAT) return <BubblesLoading />;
1717

1818
return (
1919
<Flexbox gap={8} id={id}>
2020
{editableContent}
21-
{res.imageList && res.imageList?.length > 0 && <ImageFileListViewer items={res.imageList} />}
22-
{res.fileList && res.fileList?.length > 0 && (
21+
{imageList && imageList?.length > 0 && <ImageFileListViewer items={imageList} />}
22+
{fileList && fileList?.length > 0 && (
2323
<div style={{ marginTop: 8 }}>
24-
<FileListViewer items={res.fileList} />
24+
<FileListViewer items={fileList} />
2525
</div>
2626
)}
2727
</Flexbox>

‎src/libs/agent-runtime/google/index.ts

+8-2
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ import { StreamingResponse } from '../utils/response';
3131
import { GoogleGenerativeAIStream, convertIterableToStream } from '../utils/streams';
3232
import { parseDataUri } from '../utils/uriParser';
3333

34+
const modelsOffSafetySettings = new Set(['gemini-2.0-flash-exp']);
35+
const modelsWithModalities = new Set(['gemini-2.0-flash-exp']);
36+
3437
export interface GoogleModelCard {
3538
displayName: string;
3639
inputTokenLimit: number;
@@ -50,8 +53,7 @@ enum HarmBlockThreshold {
5053
}
5154

5255
function getThreshold(model: string): HarmBlockThreshold {
53-
const useOFF = ['gemini-2.0-flash-exp'];
54-
if (useOFF.includes(model)) {
56+
if (modelsOffSafetySettings.has(model)) {
5557
return 'OFF' as HarmBlockThreshold; // https://discuss.ai.google.dev/t/59352
5658
}
5759
return HarmBlockThreshold.BLOCK_NONE;
@@ -94,6 +96,10 @@ export class LobeGoogleAI implements LobeRuntimeAI {
9496
{
9597
generationConfig: {
9698
maxOutputTokens: payload.max_tokens,
99+
// @ts-expect-error - Google SDK 0.24.0 doesn't have this property for now with
100+
response_modalities: modelsWithModalities.has(model)
101+
? ['Text', 'Image']
102+
: undefined,
97103
temperature: payload.temperature,
98104
topP: payload.top_p,
99105
},

0 commit comments

Comments
 (0)