diff --git a/api/core/memory/token_buffer_memory.py b/api/core/memory/token_buffer_memory.py index 282cd9b36f..81d08dc885 100644 --- a/api/core/memory/token_buffer_memory.py +++ b/api/core/memory/token_buffer_memory.py @@ -3,7 +3,6 @@ from typing import Optional from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.file import file_manager -from core.file.models import FileType from core.model_manager import ModelInstance from core.model_runtime.entities import ( AssistantPromptMessage, @@ -103,12 +102,11 @@ class TokenBufferMemory: prompt_message_contents: list[PromptMessageContent] = [] prompt_message_contents.append(TextPromptMessageContent(data=message.query)) for file in file_objs: - if file.type in {FileType.IMAGE, FileType.AUDIO}: - prompt_message = file_manager.to_prompt_message_content( - file, - image_detail_config=detail, - ) - prompt_message_contents.append(prompt_message) + prompt_message = file_manager.to_prompt_message_content( + file, + image_detail_config=detail, + ) + prompt_message_contents.append(prompt_message) prompt_messages.append(UserPromptMessage(content=prompt_message_contents)) diff --git a/api/core/model_runtime/entities/message_entities.py b/api/core/model_runtime/entities/message_entities.py index a7e3db0032..e86fb37522 100644 --- a/api/core/model_runtime/entities/message_entities.py +++ b/api/core/model_runtime/entities/message_entities.py @@ -49,7 +49,7 @@ class PromptMessageFunction(BaseModel): function: PromptMessageTool -class PromptMessageContentType(Enum): +class PromptMessageContentType(str, Enum): """ Enum class for prompt message content type. """ diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-001.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-001.yaml index 2e68fa8e6f..43f4e4787d 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-001.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-001.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 1048576 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-002.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-002.yaml index 9f44504e89..7b9add6af1 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-002.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-002.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 1048576 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0827.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0827.yaml index a3da9095e1..d6de82012e 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0827.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0827.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 1048576 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0924.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0924.yaml index 19373e4993..23b8d318fc 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0924.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0924.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 1048576 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-exp-0827.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-exp-0827.yaml index ca1f0b39b2..9762706cd7 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-exp-0827.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-exp-0827.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 1048576 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-latest.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-latest.yaml index 24e8c3a74f..b9739d068e 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-latest.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-latest.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 1048576 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash.yaml index fa3e814fc3..d8ab4efc91 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 1048576 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-001.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-001.yaml index da125e6fab..05184823e4 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-001.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-001.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 2097152 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-002.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-002.yaml index f683e54d3b..548fe6ddb2 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-002.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-002.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 2097152 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0801.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0801.yaml index c67c156bdb..defab26acf 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0801.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0801.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 2097152 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0827.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0827.yaml index 56059fd799..9cbc889f17 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0827.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0827.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 2097152 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-latest.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-latest.yaml index ec376f3186..e5aefcdb99 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-latest.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-latest.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 2097152 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro.yaml index 8394cdfb56..00bd3e8d99 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 2097152 diff --git a/api/core/model_runtime/model_providers/google/llm/gemini-exp-1114.yaml b/api/core/model_runtime/model_providers/google/llm/gemini-exp-1114.yaml index 2d4965ad25..0515e706c2 100644 --- a/api/core/model_runtime/model_providers/google/llm/gemini-exp-1114.yaml +++ b/api/core/model_runtime/model_providers/google/llm/gemini-exp-1114.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 32767 diff --git a/api/core/model_runtime/model_providers/google/llm/llm.py b/api/core/model_runtime/model_providers/google/llm/llm.py index 754f056ac1..77e0801b63 100644 --- a/api/core/model_runtime/model_providers/google/llm/llm.py +++ b/api/core/model_runtime/model_providers/google/llm/llm.py @@ -16,6 +16,7 @@ from PIL import Image from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import ( AssistantPromptMessage, + DocumentPromptMessageContent, ImagePromptMessageContent, PromptMessage, PromptMessageContentType, @@ -35,6 +36,21 @@ from core.model_runtime.errors.invoke import ( from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel +GOOGLE_AVAILABLE_MIMETYPE = [ + "application/pdf", + "application/x-javascript", + "text/javascript", + "application/x-python", + "text/x-python", + "text/plain", + "text/html", + "text/css", + "text/md", + "text/csv", + "text/xml", + "text/rtf", +] + class GoogleLargeLanguageModel(LargeLanguageModel): def _invoke( @@ -370,6 +386,12 @@ class GoogleLargeLanguageModel(LargeLanguageModel): raise ValueError(f"Failed to fetch image data from url {message_content.data}, {ex}") blob = {"inline_data": {"mime_type": mime_type, "data": base64_data}} glm_content["parts"].append(blob) + elif c.type == PromptMessageContentType.DOCUMENT: + message_content = cast(DocumentPromptMessageContent, c) + if message_content.mime_type not in GOOGLE_AVAILABLE_MIMETYPE: + raise ValueError(f"Unsupported mime type {message_content.mime_type}") + blob = {"inline_data": {"mime_type": message_content.mime_type, "data": message_content.data}} + glm_content["parts"].append(blob) return glm_content elif isinstance(message, AssistantPromptMessage): diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max-0809.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max-0809.yaml index 50e10226a5..94b6666d05 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max-0809.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max-0809.yaml @@ -6,6 +6,7 @@ model_type: llm features: - vision - agent-thought + - video model_properties: mode: chat context_size: 32000 diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max.yaml index 21b127f56c..b6172c1cbc 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max.yaml @@ -6,6 +6,7 @@ model_type: llm features: - vision - agent-thought + - video model_properties: mode: chat context_size: 32000 diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus-0809.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus-0809.yaml index 67b2b2ebdd..0be4b68f4f 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus-0809.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus-0809.yaml @@ -6,6 +6,7 @@ model_type: llm features: - vision - agent-thought + - video model_properties: mode: chat context_size: 32768 diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus.yaml index f55764c6c0..6c8a8121c6 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus.yaml @@ -6,6 +6,7 @@ model_type: llm features: - vision - agent-thought + - video model_properties: mode: chat context_size: 8000 diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml index 91550ceee8..dbda18b888 100644 --- a/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml +++ b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml @@ -6,6 +6,7 @@ model_properties: mode: chat features: - vision + - video parameter_rules: - name: temperature use_template: temperature diff --git a/api/core/workflow/nodes/llm/exc.py b/api/core/workflow/nodes/llm/exc.py index b5207d5573..6599221691 100644 --- a/api/core/workflow/nodes/llm/exc.py +++ b/api/core/workflow/nodes/llm/exc.py @@ -26,9 +26,15 @@ class NoPromptFoundError(LLMNodeError): """Raised when no prompt is found in the LLM configuration.""" -class NotSupportedPromptTypeError(LLMNodeError): - """Raised when the prompt type is not supported.""" +class TemplateTypeNotSupportError(LLMNodeError): + def __init__(self, *, type_name: str): + super().__init__(f"Prompt type {type_name} is not supported.") class MemoryRolePrefixRequiredError(LLMNodeError): """Raised when memory role prefix is required for completion model.""" + + +class FileTypeNotSupportError(LLMNodeError): + def __init__(self, *, type_name: str): + super().__init__(f"{type_name} type is not supported by this model") diff --git a/api/core/workflow/nodes/llm/node.py b/api/core/workflow/nodes/llm/node.py index 0cb53ee9d3..2529c76942 100644 --- a/api/core/workflow/nodes/llm/node.py +++ b/api/core/workflow/nodes/llm/node.py @@ -65,6 +65,7 @@ from .entities import ( ModelConfig, ) from .exc import ( + FileTypeNotSupportError, InvalidContextStructureError, InvalidVariableTypeError, LLMModeRequiredError, @@ -72,7 +73,7 @@ from .exc import ( MemoryRolePrefixRequiredError, ModelNotExistError, NoPromptFoundError, - NotSupportedPromptTypeError, + TemplateTypeNotSupportError, VariableNotFoundError, ) @@ -621,9 +622,7 @@ class LLMNode(BaseNode[LLMNodeData]): prompt_content = prompt_messages[0].content.replace("#sys.query#", user_query) prompt_messages[0].content = prompt_content else: - errmsg = f"Prompt type {type(prompt_template)} is not supported" - logger.warning(errmsg) - raise NotSupportedPromptTypeError(errmsg) + raise TemplateTypeNotSupportError(type_name=str(type(prompt_template))) if vision_enabled and user_files: file_prompts = [] @@ -671,7 +670,7 @@ class LLMNode(BaseNode[LLMNodeData]): and ModelFeature.AUDIO not in model_config.model_schema.features ) ): - continue + raise FileTypeNotSupportError(type_name=content_item.type) prompt_message_content.append(content_item) if len(prompt_message_content) == 1 and prompt_message_content[0].type == PromptMessageContentType.TEXT: prompt_message.content = prompt_message_content[0].data diff --git a/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py b/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py index a1f9ece0d1..9a24d35a1f 100644 --- a/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py @@ -400,59 +400,6 @@ def test_fetch_prompt_messages__basic(faker, llm_node, model_config): ) }, ), - LLMNodeTestScenario( - description="Prompt template with variable selector of File without vision feature", - user_query=fake_query, - user_files=[], - vision_enabled=True, - vision_detail=fake_vision_detail, - features=[], - window_size=fake_window_size, - prompt_template=[ - LLMNodeChatModelMessage( - text="{{#input.image#}}", - role=PromptMessageRole.USER, - edition_type="basic", - ), - ], - expected_messages=mock_history[fake_window_size * -2 :] + [UserPromptMessage(content=fake_query)], - file_variables={ - "input.image": File( - tenant_id="test", - type=FileType.IMAGE, - filename="test1.jpg", - transfer_method=FileTransferMethod.REMOTE_URL, - remote_url=fake_remote_url, - ) - }, - ), - LLMNodeTestScenario( - description="Prompt template with variable selector of File with video file and vision feature", - user_query=fake_query, - user_files=[], - vision_enabled=True, - vision_detail=fake_vision_detail, - features=[ModelFeature.VISION], - window_size=fake_window_size, - prompt_template=[ - LLMNodeChatModelMessage( - text="{{#input.image#}}", - role=PromptMessageRole.USER, - edition_type="basic", - ), - ], - expected_messages=mock_history[fake_window_size * -2 :] + [UserPromptMessage(content=fake_query)], - file_variables={ - "input.image": File( - tenant_id="test", - type=FileType.VIDEO, - filename="test1.mp4", - transfer_method=FileTransferMethod.REMOTE_URL, - remote_url=fake_remote_url, - extension="mp4", - ) - }, - ), ] for scenario in test_scenarios: diff --git a/web/app/components/app/configuration/config-vision/index.tsx b/web/app/components/app/configuration/config-vision/index.tsx index 23f00d46d8..f30d3e4a0a 100644 --- a/web/app/components/app/configuration/config-vision/index.tsx +++ b/web/app/components/app/configuration/config-vision/index.tsx @@ -12,34 +12,46 @@ import ConfigContext from '@/context/debug-configuration' // import { Resolution } from '@/types/app' import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks' import Switch from '@/app/components/base/switch' -import type { FileUpload } from '@/app/components/base/features/types' +import { SupportUploadFileTypes } from '@/app/components/workflow/types' const ConfigVision: FC = () => { const { t } = useTranslation() - const { isShowVisionConfig } = useContext(ConfigContext) + const { isShowVisionConfig, isAllowVideoUpload } = useContext(ConfigContext) const file = useFeatures(s => s.features.file) const featuresStore = useFeaturesStore() - const handleChange = useCallback((data: FileUpload) => { + const isImageEnabled = file?.allowed_file_types?.includes(SupportUploadFileTypes.image) ?? false + + const handleChange = useCallback((value: boolean) => { const { features, setFeatures, } = featuresStore!.getState() const newFeatures = produce(features, (draft) => { - draft.file = { - ...draft.file, - enabled: data.enabled, - image: { - enabled: data.enabled, - detail: data.image?.detail, - transfer_methods: data.image?.transfer_methods, - number_limits: data.image?.number_limits, - }, + if (value) { + draft.file!.allowed_file_types = Array.from(new Set([ + ...(draft.file?.allowed_file_types || []), + SupportUploadFileTypes.image, + ...(isAllowVideoUpload ? [SupportUploadFileTypes.video] : []), + ])) + } + else { + draft.file!.allowed_file_types = draft.file!.allowed_file_types?.filter( + type => type !== SupportUploadFileTypes.image && (isAllowVideoUpload ? type !== SupportUploadFileTypes.video : true), + ) + } + + if (draft.file) { + draft.file.enabled = (draft.file.allowed_file_types?.length ?? 0) > 0 + draft.file.image = { + ...(draft.file.image || {}), + enabled: value, + } } }) setFeatures(newFeatures) - }, [featuresStore]) + }, [featuresStore, isAllowVideoUpload]) if (!isShowVisionConfig) return null @@ -89,11 +101,8 @@ const ConfigVision: FC = () => {
handleChange({ - ...(file || {}), - enabled: value, - })} + defaultValue={isImageEnabled} + onChange={handleChange} size='md' /> diff --git a/web/app/components/app/configuration/config/config-document.tsx b/web/app/components/app/configuration/config/config-document.tsx new file mode 100644 index 0000000000..1ac6da0dd8 --- /dev/null +++ b/web/app/components/app/configuration/config/config-document.tsx @@ -0,0 +1,78 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback } from 'react' +import { useTranslation } from 'react-i18next' +import produce from 'immer' +import { useContext } from 'use-context-selector' + +import { Document } from '@/app/components/base/icons/src/vender/features' +import Tooltip from '@/app/components/base/tooltip' +import ConfigContext from '@/context/debug-configuration' +import { SupportUploadFileTypes } from '@/app/components/workflow/types' +import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks' +import Switch from '@/app/components/base/switch' + +const ConfigDocument: FC = () => { + const { t } = useTranslation() + const file = useFeatures(s => s.features.file) + const featuresStore = useFeaturesStore() + const { isShowDocumentConfig } = useContext(ConfigContext) + + const isDocumentEnabled = file?.allowed_file_types?.includes(SupportUploadFileTypes.document) ?? false + + const handleChange = useCallback((value: boolean) => { + const { + features, + setFeatures, + } = featuresStore!.getState() + + const newFeatures = produce(features, (draft) => { + if (value) { + draft.file!.allowed_file_types = Array.from(new Set([ + ...(draft.file?.allowed_file_types || []), + SupportUploadFileTypes.document, + ])) + } + else { + draft.file!.allowed_file_types = draft.file!.allowed_file_types?.filter( + type => type !== SupportUploadFileTypes.document, + ) + } + if (draft.file) + draft.file.enabled = (draft.file.allowed_file_types?.length ?? 0) > 0 + }) + setFeatures(newFeatures) + }, [featuresStore]) + + if (!isShowDocumentConfig) + return null + + return ( +
+
+
+ +
+
+
+
{t('appDebug.feature.documentUpload.title')}
+ + {t('appDebug.feature.documentUpload.description')} +
+ } + /> +
+
+
+ +
+ + ) +} +export default React.memo(ConfigDocument) diff --git a/web/app/components/app/configuration/config/index.tsx b/web/app/components/app/configuration/config/index.tsx index 8687079931..39fdd502ef 100644 --- a/web/app/components/app/configuration/config/index.tsx +++ b/web/app/components/app/configuration/config/index.tsx @@ -7,6 +7,7 @@ import { useFormattingChangedDispatcher } from '../debug/hooks' import DatasetConfig from '../dataset-config' import HistoryPanel from '../config-prompt/conversation-history/history-panel' import ConfigVision from '../config-vision' +import ConfigDocument from './config-document' import AgentTools from './agent/agent-tools' import ConfigContext from '@/context/debug-configuration' import ConfigPrompt from '@/app/components/app/configuration/config-prompt' @@ -82,6 +83,8 @@ const Config: FC = () => { + + {/* Chat History */} {isAdvancedMode && isChatApp && modelModeType === ModelModeType.completion && ( { } const isShowVisionConfig = !!currModel?.features?.includes(ModelFeatureEnum.vision) - + const isShowDocumentConfig = !!currModel?.features?.includes(ModelFeatureEnum.document) + const isAllowVideoUpload = !!currModel?.features?.includes(ModelFeatureEnum.video) // *** web app features *** const featuresData: FeaturesData = useMemo(() => { return { @@ -472,7 +473,7 @@ const Configuration: FC = () => { transfer_methods: modelConfig.file_upload?.image?.transfer_methods || ['local_file', 'remote_url'], }, enabled: !!(modelConfig.file_upload?.enabled || modelConfig.file_upload?.image?.enabled), - allowed_file_types: modelConfig.file_upload?.allowed_file_types || [SupportUploadFileTypes.image, SupportUploadFileTypes.video], + allowed_file_types: modelConfig.file_upload?.allowed_file_types || [], allowed_file_extensions: modelConfig.file_upload?.allowed_file_extensions || [...FILE_EXTS[SupportUploadFileTypes.image], ...FILE_EXTS[SupportUploadFileTypes.video]].map(ext => `.${ext}`), allowed_file_upload_methods: modelConfig.file_upload?.allowed_file_upload_methods || modelConfig.file_upload?.image?.transfer_methods || ['local_file', 'remote_url'], number_limits: modelConfig.file_upload?.number_limits || modelConfig.file_upload?.image?.number_limits || 3, @@ -861,6 +862,8 @@ const Configuration: FC = () => { isShowVisionConfig, visionConfig, setVisionConfig: handleSetVisionConfig, + isAllowVideoUpload, + isShowDocumentConfig, rerankSettingModalOpen, setRerankSettingModalOpen, }} diff --git a/web/app/components/base/icons/assets/vender/features/document.svg b/web/app/components/base/icons/assets/vender/features/document.svg new file mode 100644 index 0000000000..dca0e91a52 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/features/document.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/features/Document.json b/web/app/components/base/icons/src/vender/features/Document.json new file mode 100644 index 0000000000..fdd08d5254 --- /dev/null +++ b/web/app/components/base/icons/src/vender/features/Document.json @@ -0,0 +1,23 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "xmlns": "http://www.w3.org/2000/svg", + "viewBox": "0 0 24 24", + "fill": "currentColor" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M20 22H4C3.44772 22 3 21.5523 3 21V3C3 2.44772 3.44772 2 4 2H20C20.5523 2 21 2.44772 21 3V21C21 21.5523 20.5523 22 20 22ZM7 6V10H11V6H7ZM7 12V14H17V12H7ZM7 16V18H17V16H7ZM13 7V9H17V7H13Z" + }, + "children": [] + } + ] + }, + "name": "Document" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/features/Document.tsx b/web/app/components/base/icons/src/vender/features/Document.tsx new file mode 100644 index 0000000000..84bf3a2f10 --- /dev/null +++ b/web/app/components/base/icons/src/vender/features/Document.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Document.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'Document' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/features/index.ts b/web/app/components/base/icons/src/vender/features/index.ts index 2b8cb17c94..f246732226 100644 --- a/web/app/components/base/icons/src/vender/features/index.ts +++ b/web/app/components/base/icons/src/vender/features/index.ts @@ -7,3 +7,4 @@ export { default as Microphone01 } from './Microphone01' export { default as TextToAudio } from './TextToAudio' export { default as VirtualAssistant } from './VirtualAssistant' export { default as Vision } from './Vision' +export { default as Document } from './Document' diff --git a/web/app/components/header/account-setting/model-provider-page/declarations.ts b/web/app/components/header/account-setting/model-provider-page/declarations.ts index 8a84376bea..02f178c52f 100644 --- a/web/app/components/header/account-setting/model-provider-page/declarations.ts +++ b/web/app/components/header/account-setting/model-provider-page/declarations.ts @@ -52,6 +52,9 @@ export enum ModelFeatureEnum { multiToolCall = 'multi-tool-call', agentThought = 'agent-thought', vision = 'vision', + video = 'video', + document = 'document', + audio = 'audio', } export enum ModelFeatureTextEnum { @@ -59,6 +62,9 @@ export enum ModelFeatureTextEnum { multiToolCall = 'Multi Tool Call', agentThought = 'Agent Thought', vision = 'Vision', + video = 'Video', + document = 'Document', + audio = 'Audio', } export enum ModelStatusEnum { diff --git a/web/context/debug-configuration.ts b/web/context/debug-configuration.ts index bce93a58b2..61508278d0 100644 --- a/web/context/debug-configuration.ts +++ b/web/context/debug-configuration.ts @@ -97,6 +97,8 @@ type IDebugConfiguration = { isShowVisionConfig: boolean visionConfig: VisionSettings setVisionConfig: (visionConfig: VisionSettings, noNotice?: boolean) => void + isAllowVideoUpload: boolean + isShowDocumentConfig: boolean rerankSettingModalOpen: boolean setRerankSettingModalOpen: (rerankSettingModalOpen: boolean) => void } @@ -244,6 +246,8 @@ const DebugConfigurationContext = createContext({ transfer_methods: [TransferMethod.remote_url], }, setVisionConfig: () => { }, + isAllowVideoUpload: false, + isShowDocumentConfig: false, rerankSettingModalOpen: false, setRerankSettingModalOpen: () => { }, }) diff --git a/web/i18n/en-US/app-debug.ts b/web/i18n/en-US/app-debug.ts index e17afc38bf..266da820a0 100644 --- a/web/i18n/en-US/app-debug.ts +++ b/web/i18n/en-US/app-debug.ts @@ -218,6 +218,10 @@ const translation = { enableText: 'Features Enabled', manage: 'Manage', }, + documentUpload: { + title: 'Document', + description: 'Enable Document will allows the model to take in documents and answer questions about them.', + }, }, codegen: { title: 'Code Generator', diff --git a/web/i18n/zh-Hans/app-debug.ts b/web/i18n/zh-Hans/app-debug.ts index 9e21945755..4e3f18ad7f 100644 --- a/web/i18n/zh-Hans/app-debug.ts +++ b/web/i18n/zh-Hans/app-debug.ts @@ -218,6 +218,10 @@ const translation = { enableText: '功能已开启', manage: '管理', }, + documentUpload: { + title: '文档', + description: '启用文档后，模型可以接收文档并回答关于它们的问题。', + }, }, codegen: { title: '代码生成器',