From a10b0db1022dd43d6736a28b873865433c0cb608 Mon Sep 17 00:00:00 2001 From: JzoNg Date: Thu, 12 Sep 2024 14:13:45 +0800 Subject: [PATCH] vision setting --- .../app/configuration/config-vision/index.tsx | 30 +- .../config-vision/param-config-content.tsx | 137 +++++++ .../config-vision/param-config.tsx | 41 ++ .../configuration/debug/chat-user-input.tsx | 1 - .../debug/debug-with-multiple-model/index.tsx | 30 +- .../debug/debug-with-single-model/index.tsx | 4 +- .../components/app/configuration/index.tsx | 2 +- .../prompt-value-panel/index.tsx | 2 +- web/app/components/base/chat/chat/index.tsx | 24 +- .../moderation/moderation-setting-modal.tsx | 376 ------------------ .../feature-panel/opening-statement/index.tsx | 321 --------------- web/app/components/base/features/types.ts | 6 +- 12 files changed, 251 insertions(+), 723 deletions(-) create mode 100644 web/app/components/app/configuration/config-vision/param-config-content.tsx create mode 100644 web/app/components/app/configuration/config-vision/param-config.tsx delete mode 100644 web/app/components/base/features/feature-panel/moderation/moderation-setting-modal.tsx delete mode 100644 web/app/components/base/features/feature-panel/opening-statement/index.tsx diff --git a/web/app/components/app/configuration/config-vision/index.tsx b/web/app/components/app/configuration/config-vision/index.tsx index ae388615c0..7666fcb834 100644 --- a/web/app/components/app/configuration/config-vision/index.tsx +++ b/web/app/components/app/configuration/config-vision/index.tsx @@ -4,12 +4,15 @@ import React, { useCallback } from 'react' import { useTranslation } from 'react-i18next' import produce from 'immer' import { useContext } from 'use-context-selector' +import ParamConfig from './param-config' import { Vision } from '@/app/components/base/icons/src/vender/features' import Tooltip from '@/app/components/base/tooltip' -import OptionCard from '@/app/components/workflow/nodes/_base/components/option-card' +// import OptionCard from '@/app/components/workflow/nodes/_base/components/option-card' import ConfigContext from '@/context/debug-configuration' -import { Resolution } from '@/types/app' +// import { Resolution } from '@/types/app' import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks' +import Switch from '@/app/components/base/switch' +import type { FileUpload } from '@/app/components/base/features/types' const ConfigVision: FC = () => { const { t } = useTranslation() @@ -17,7 +20,7 @@ const ConfigVision: FC = () => { const file = useFeatures(s => s.features.file) const featuresStore = useFeaturesStore() - const handleChange = useCallback((resolution: Resolution) => { + const handleChange = useCallback((data: FileUpload) => { const { features, setFeatures, @@ -26,7 +29,8 @@ const ConfigVision: FC = () => { const newFeatures = produce(features, (draft) => { draft.file = { ...draft.file, - image: { detail: resolution }, + enabled: data.enabled, + image: { detail: data.image?.detail }, } }) setFeatures(newFeatures) @@ -53,7 +57,7 @@ const ConfigVision: FC = () => { />
-
+ {/*
{t('appDebug.vision.visionSettings.resolution')}
{
} /> -
-
+
*/} + {/*
{ selected={file?.image?.detail === Resolution.low} onSelect={() => handleChange(Resolution.low)} /> -
+
*/} + +
+ handleChange({ + ...(file || {}), + enabled: value, + })} + size='md' + /> ) diff --git a/web/app/components/app/configuration/config-vision/param-config-content.tsx b/web/app/components/app/configuration/config-vision/param-config-content.tsx new file mode 100644 index 0000000000..97195b24d7 --- /dev/null +++ b/web/app/components/app/configuration/config-vision/param-config-content.tsx @@ -0,0 +1,137 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback } from 'react' +import { useTranslation } from 'react-i18next' +import produce from 'immer' +import OptionCard from '@/app/components/workflow/nodes/_base/components/option-card' +import { Resolution, TransferMethod } from '@/types/app' +import ParamItem from '@/app/components/base/param-item' +import Tooltip from '@/app/components/base/tooltip' +import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks' +import type { FileUpload } from '@/app/components/base/features/types' + +const MIN = 1 +const MAX = 6 +const ParamConfigContent: FC = () => { + const { t } = useTranslation() + const file = useFeatures(s => s.features.file) + const featuresStore = useFeaturesStore() + + const handleChange = useCallback((data: FileUpload) => { + const { + features, + setFeatures, + } = featuresStore!.getState() + + const newFeatures = produce(features, (draft) => { + draft.file = { + ...draft.file, + image: { detail: data.image?.detail }, + allowed_file_upload_methods: data.allowed_file_upload_methods, + number_limits: data.number_limits, + } + }) + setFeatures(newFeatures) + }, [featuresStore]) + + return ( +
+
{t('appDebug.vision.visionSettings.title')}
+
+
+
+
{t('appDebug.vision.visionSettings.resolution')}
+ + {t('appDebug.vision.visionSettings.resolutionTooltip').split('\n').map(item => ( +
{item}
+ ))} +
+ } + /> +
+
+ handleChange({ + ...file, + image: { detail: Resolution.high }, + })} + /> + handleChange({ + ...file, + image: { detail: Resolution.low }, + })} + /> +
+
+
+
{t('appDebug.vision.visionSettings.uploadMethod')}
+
+ handleChange({ + ...file, + allowed_file_upload_methods: [TransferMethod.local_file, TransferMethod.remote_url], + })} + /> + handleChange({ + ...file, + allowed_file_upload_methods: [TransferMethod.local_file], + })} + /> + handleChange({ + ...file, + allowed_file_upload_methods: [TransferMethod.remote_url], + })} + /> +
+
+
+ { + if (!value) + return + + handleChange({ + ...file, + number_limits: value, + }) + }} + /> +
+
+ + ) +} + +export default React.memo(ParamConfigContent) diff --git a/web/app/components/app/configuration/config-vision/param-config.tsx b/web/app/components/app/configuration/config-vision/param-config.tsx new file mode 100644 index 0000000000..8c63879391 --- /dev/null +++ b/web/app/components/app/configuration/config-vision/param-config.tsx @@ -0,0 +1,41 @@ +'use client' +import type { FC } from 'react' +import { memo, useState } from 'react' +import { useTranslation } from 'react-i18next' +import ParamConfigContent from './param-config-content' +import cn from '@/utils/classnames' +import { Settings01 } from '@/app/components/base/icons/src/vender/line/general' +import { + PortalToFollowElem, + PortalToFollowElemContent, + PortalToFollowElemTrigger, +} from '@/app/components/base/portal-to-follow-elem' + +const ParamsConfig: FC = () => { + const { t } = useTranslation() + const [open, setOpen] = useState(false) + + return ( + + setOpen(v => !v)}> +
+ +
{t('appDebug.voice.settings')}
+
+ + +
+ +
+ + + ) +} +export default memo(ParamsConfig) diff --git a/web/app/components/app/configuration/debug/chat-user-input.tsx b/web/app/components/app/configuration/debug/chat-user-input.tsx index 8dec6635bc..cda41917e3 100644 --- a/web/app/components/app/configuration/debug/chat-user-input.tsx +++ b/web/app/components/app/configuration/debug/chat-user-input.tsx @@ -49,7 +49,6 @@ const ChatUserInput = ({ return (
- {/* ##TODO## file_upload */} {promptVariables.map(({ key, name, type, options, max_length, required }, index) => (
{ const { mode, - isShowVisionConfig, + // isShowVisionConfig, } = useDebugConfigurationContext() const speech2text = useFeatures(s => s.features.speech2text) const file = useFeatures(s => s.features.file) @@ -32,6 +34,15 @@ const DebugWithMultipleModel = () => { const { eventEmitter } = useEventEmitterContextContext() const isChatMode = mode === 'chat' || mode === 'agent-chat' + const visionConfig = useMemo(() => { + return { + enabled: file?.enabled || false, + detail: file?.image?.detail || Resolution.high, + number_limits: file?.number_limits || 3, + transfer_methods: file?.allowed_file_upload_methods || [TransferMethod.local_file, TransferMethod.remote_url], + } + }, [file]) + const handleSend = useCallback((message: string, files?: VisionFile[]) => { if (checkCanSend && !checkCanSend()) return @@ -95,7 +106,7 @@ const DebugWithMultipleModel = () => { } }, [twoLine, threeLine, fourLine]) - const setShowAppConfigureFeaturesModal = useAppStore(s => s.setShowAppConfigureFeaturesModal) + // const setShowAppConfigureFeaturesModal = useAppStore(s => s.setShowAppConfigureFeaturesModal) return (
@@ -128,14 +139,19 @@ const DebugWithMultipleModel = () => {
{isChatMode && (
- + {/* + /> */}
)}
diff --git a/web/app/components/app/configuration/debug/debug-with-single-model/index.tsx b/web/app/components/app/configuration/debug/debug-with-single-model/index.tsx index 7fdb3e2046..456a77ebbb 100644 --- a/web/app/components/app/configuration/debug/debug-with-single-model/index.tsx +++ b/web/app/components/app/configuration/debug/debug-with-single-model/index.tsx @@ -41,7 +41,7 @@ const DebugWithSingleModel = forwardRef s.features) @@ -142,7 +142,7 @@ const DebugWithSingleModel = forwardRef { = ({
))} - {/* ##TODO## file_upload */} {visionConfig?.enabled && (
{t('common.imageUploader.imageUpload')}
@@ -204,6 +203,7 @@ const PromptValuePanel: FC = ({
diff --git a/web/app/components/base/chat/chat/index.tsx b/web/app/components/base/chat/chat/index.tsx index 6e305553ba..0f95e8c97d 100644 --- a/web/app/components/base/chat/chat/index.tsx +++ b/web/app/components/base/chat/chat/index.tsx @@ -6,6 +6,7 @@ import { memo, useCallback, useEffect, + useMemo, useRef, useState, } from 'react' @@ -21,7 +22,7 @@ import type { import type { ThemeBuilder } from '../embedded-chatbot/theme/theme-context' import Question from './question' import Answer from './answer' -// import ChatInput from './chat-input' +import ChatInput from './chat-input' import ChatInputArea from './chat-input-area' import TryToAsk from './try-to-ask' import { ChatContextProvider } from './context' @@ -33,6 +34,7 @@ import AgentLogModal from '@/app/components/base/agent-log-modal' import PromptLogModal from '@/app/components/base/prompt-log-modal' import { useStore as useAppStore } from '@/app/components/app/store' import type { AppData } from '@/models/share' +import { Resolution, TransferMethod } from '@/types/app' export type ChatProps = { appData?: AppData @@ -182,6 +184,15 @@ const Chat: FC = ({ const hasTryToAsk = config?.suggested_questions_after_answer?.enabled && !!suggestedQuestions?.length && onSend + const visionConfig = useMemo(() => { + return { + enabled: (config?.file_upload as any)?.enabled || false, + detail: (config?.file_upload as any)?.image?.detail || Resolution.high, + number_limits: (config?.file_upload as any)?.number_limits || 3, + transfer_methods: (config?.file_upload as any)?.allowed_file_upload_methods || [TransferMethod.local_file, TransferMethod.remote_url], + } + }, [config?.file_upload]) + return ( = ({ ) } { - !noChatInput && ( + !noChatInput && showFileUpload && ( = ({ speechToTextConfig={config?.speech_to_text} onSend={onSend} theme={themeBuilder?.theme} - // noSpacing={noSpacing} /> ) } + {!noChatInput && !showFileUpload && ( + + )} {showPromptLogModal && !hideLogModal && ( diff --git a/web/app/components/base/features/feature-panel/moderation/moderation-setting-modal.tsx b/web/app/components/base/features/feature-panel/moderation/moderation-setting-modal.tsx deleted file mode 100644 index 635506c053..0000000000 --- a/web/app/components/base/features/feature-panel/moderation/moderation-setting-modal.tsx +++ /dev/null @@ -1,376 +0,0 @@ -import type { ChangeEvent, FC } from 'react' -import { - memo, - useState, -} from 'react' -import useSWR from 'swr' -import { useContext } from 'use-context-selector' -import { useTranslation } from 'react-i18next' -import ModerationContent from './moderation-content' -import FormGeneration from './form-generation' -import ApiBasedExtensionSelector from '@/app/components/header/account-setting/api-based-extension-page/selector' -import Modal from '@/app/components/base/modal' -import Button from '@/app/components/base/button' -import { BookOpen01 } from '@/app/components/base/icons/src/vender/line/education' -import type { ModerationConfig, ModerationContentConfig } from '@/models/debug' -import { useToastContext } from '@/app/components/base/toast' -import { - fetchCodeBasedExtensionList, - fetchModelProviders, -} from '@/service/common' -import type { CodeBasedExtensionItem } from '@/models/common' -import I18n from '@/context/i18n' -import { LanguagesSupported } from '@/i18n/language' -import { InfoCircle } from '@/app/components/base/icons/src/vender/line/general' -import { useModalContext } from '@/context/modal-context' -import { CustomConfigurationStatusEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' - -const systemTypes = ['openai_moderation', 'keywords', 'api'] - -type Provider = { - key: string - name: string - form_schema?: CodeBasedExtensionItem['form_schema'] -} - -type ModerationSettingModalProps = { - data: ModerationConfig - onCancel: () => void - onSave: (moderationConfig: ModerationConfig) => void -} - -const ModerationSettingModal: FC = ({ - data, - onCancel, - onSave, -}) => { - const { t } = useTranslation() - const { notify } = useToastContext() - const { locale } = useContext(I18n) - const { data: modelProviders, isLoading, mutate } = useSWR('/workspaces/current/model-providers', fetchModelProviders) - const [localeData, setLocaleData] = useState(data) - const { setShowAccountSettingModal } = useModalContext() - const handleOpenSettingsModal = () => { - setShowAccountSettingModal({ - payload: 'provider', - onCancelCallback: () => { - mutate() - }, - }) - } - const { data: codeBasedExtensionList } = useSWR( - '/code-based-extension?module=moderation', - fetchCodeBasedExtensionList, - ) - const openaiProvider = modelProviders?.data.find(item => item.provider === 'openai') - const systemOpenaiProviderEnabled = openaiProvider?.system_configuration.enabled - const systemOpenaiProviderQuota = systemOpenaiProviderEnabled ? openaiProvider?.system_configuration.quota_configurations.find(item => item.quota_type === openaiProvider.system_configuration.current_quota_type) : undefined - const systemOpenaiProviderCanUse = systemOpenaiProviderQuota?.is_valid - const customOpenaiProvidersCanUse = openaiProvider?.custom_configuration.status === CustomConfigurationStatusEnum.active - const isOpenAIProviderConfigured = customOpenaiProvidersCanUse || systemOpenaiProviderCanUse - const providers: Provider[] = [ - { - key: 'openai_moderation', - name: t('appDebug.feature.moderation.modal.provider.openai'), - }, - { - key: 'keywords', - name: t('appDebug.feature.moderation.modal.provider.keywords'), - }, - { - key: 'api', - name: t('common.apiBasedExtension.selector.title'), - }, - ...( - codeBasedExtensionList - ? codeBasedExtensionList.data.map((item) => { - return { - key: item.name, - name: locale === 'zh-Hans' ? item.label['zh-Hans'] : item.label['en-US'], - form_schema: item.form_schema, - } - }) - : [] - ), - ] - - const currentProvider = providers.find(provider => provider.key === localeData.type) - - const handleDataTypeChange = (type: string) => { - let config: undefined | Record - const currProvider = providers.find(provider => provider.key === type) - - if (systemTypes.findIndex(t => t === type) < 0 && currProvider?.form_schema) { - config = currProvider?.form_schema.reduce((prev, next) => { - prev[next.variable] = next.default - return prev - }, {} as Record) - } - setLocaleData({ - ...localeData, - type, - config, - }) - } - - const handleDataKeywordsChange = (e: ChangeEvent) => { - const value = e.target.value - - const arr = value.split('\n').reduce((prev: string[], next: string) => { - if (next !== '') - prev.push(next.slice(0, 100)) - if (next === '' && prev[prev.length - 1] !== '') - prev.push(next) - - return prev - }, []) - - setLocaleData({ - ...localeData, - config: { - ...localeData.config, - keywords: arr.slice(0, 100).join('\n'), - }, - }) - } - - const handleDataContentChange = (contentType: string, contentConfig: ModerationContentConfig) => { - setLocaleData({ - ...localeData, - config: { - ...localeData.config, - [contentType]: contentConfig, - }, - }) - } - - const handleDataApiBasedChange = (apiBasedExtensionId: string) => { - setLocaleData({ - ...localeData, - config: { - ...localeData.config, - api_based_extension_id: apiBasedExtensionId, - }, - }) - } - - const handleDataExtraChange = (extraValue: Record) => { - setLocaleData({ - ...localeData, - config: { - ...localeData.config, - ...extraValue, - }, - }) - } - - const formatData = (originData: ModerationConfig) => { - const { enabled, type, config } = originData - const { inputs_config, outputs_config } = config! - const params: Record = {} - - if (type === 'keywords') - params.keywords = config?.keywords - - if (type === 'api') - params.api_based_extension_id = config?.api_based_extension_id - - if (systemTypes.findIndex(t => t === type) < 0 && currentProvider?.form_schema) { - currentProvider.form_schema.forEach((form) => { - params[form.variable] = config?.[form.variable] - }) - } - - return { - type, - enabled, - config: { - inputs_config: inputs_config || { enabled: false }, - outputs_config: outputs_config || { enabled: false }, - ...params, - }, - } - } - - const handleSave = () => { - if (localeData.type === 'openai_moderation' && !isOpenAIProviderConfigured) - return - - if (!localeData.config?.inputs_config?.enabled && !localeData.config?.outputs_config?.enabled) { - notify({ type: 'error', message: t('appDebug.feature.moderation.modal.content.condition') }) - return - } - - if (localeData.type === 'keywords' && !localeData.config.keywords) { - notify({ type: 'error', message: t('appDebug.errorMessage.valueOfVarRequired', { key: locale !== LanguagesSupported[1] ? 'keywords' : '关键词' }) }) - return - } - - if (localeData.type === 'api' && !localeData.config.api_based_extension_id) { - notify({ type: 'error', message: t('appDebug.errorMessage.valueOfVarRequired', { key: locale !== LanguagesSupported[1] ? 'API Extension' : 'API 扩展' }) }) - return - } - - if (systemTypes.findIndex(t => t === localeData.type) < 0 && currentProvider?.form_schema) { - for (let i = 0; i < currentProvider.form_schema.length; i++) { - if (!localeData.config?.[currentProvider.form_schema[i].variable] && currentProvider.form_schema[i].required) { - notify({ - type: 'error', - message: t('appDebug.errorMessage.valueOfVarRequired', { key: locale !== LanguagesSupported[1] ? currentProvider.form_schema[i].label['en-US'] : currentProvider.form_schema[i].label['zh-Hans'] }), - }) - return - } - } - } - - if (localeData.config.inputs_config?.enabled && !localeData.config.inputs_config.preset_response && localeData.type !== 'api') { - notify({ type: 'error', message: t('appDebug.feature.moderation.modal.content.errorMessage') }) - return - } - - if (localeData.config.outputs_config?.enabled && !localeData.config.outputs_config.preset_response && localeData.type !== 'api') { - notify({ type: 'error', message: t('appDebug.feature.moderation.modal.content.errorMessage') }) - return - } - - onSave(formatData(localeData)) - } - - return ( - { }} - className='!p-8 !pb-6 !mt-14 !max-w-none !w-[640px]' - > -
- {t('appDebug.feature.moderation.modal.title')} -
-
-
- {t('appDebug.feature.moderation.modal.provider.title')} -
-
- { - providers.map(provider => ( -
handleDataTypeChange(provider.key)} - > -
- {provider.name} -
- )) - } -
- { - !isLoading && !isOpenAIProviderConfigured && localeData.type === 'openai_moderation' && ( -
- -
- {t('appDebug.feature.moderation.modal.openaiNotConfig.before')} - -  {t('common.settings.provider')}  - - {t('appDebug.feature.moderation.modal.openaiNotConfig.after')} -
-
- ) - } -
- { - localeData.type === 'keywords' && ( -
-
{t('appDebug.feature.moderation.modal.provider.keywords')}
-
{t('appDebug.feature.moderation.modal.keywords.tip')}
-
- -
- ) - : ( -
- )} - {renderQuestions()} - ) : ( -
{t('appDebug.openingStatement.noDataPlaceHolder')}
- )} - - {isShowConfirmAddVar && ( - - )} - -
- - ) -} -export default React.memo(OpeningStatement) diff --git a/web/app/components/base/features/types.ts b/web/app/components/base/features/types.ts index 0b959c4aa4..3307f12cda 100644 --- a/web/app/components/base/features/types.ts +++ b/web/app/components/base/features/types.ts @@ -1,4 +1,4 @@ -import type { TransferMethod, TtsAutoPlay } from '@/types/app' +import type { Resolution, TransferMethod, TtsAutoPlay } from '@/types/app' export type EnabledOrDisabled = { enabled?: boolean @@ -30,13 +30,13 @@ export type SensitiveWordAvoidance = EnabledOrDisabled & { export type FileUpload = { image?: EnabledOrDisabled & { - detail?: 'high' | 'low' + detail?: Resolution number_limits?: number transfer_methods?: TransferMethod[] } allowed_file_types?: string[] allowed_file_extensions?: string[] - allowed_file_upload_methods?: string[] + allowed_file_upload_methods?: TransferMethod[] number_limits?: number } & EnabledOrDisabled