From 6b158272466bf6fb02c8fb245f93c1bcb6dbaae9 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Mon, 13 Nov 2023 22:32:39 +0800 Subject: [PATCH] feat: [frontend] support vision (#1518) Co-authored-by: Joel --- web/app/components/app/chat/index.tsx | 83 +- .../components/app/chat/question/index.tsx | 8 +- web/app/components/app/chat/type.ts | 3 +- .../app/configuration/config-model/index.tsx | 6 +- .../app/configuration/config-var/index.tsx | 2 +- .../app/configuration/config-vision/index.tsx | 60 ++ .../config-vision/param-config-content.tsx | 132 +++ .../config-vision/param-config.tsx | 41 + .../config-vision/radio-group/index.tsx | 40 + .../radio-group/style.module.css | 24 + .../app/configuration/config/index.tsx | 3 + .../configuration/dataset-config/index.tsx | 2 +- .../app/configuration/debug/index.tsx | 66 +- .../components/app/configuration/index.tsx | 43 +- .../prompt-value-panel/index.tsx | 24 + web/app/components/app/log/list.tsx | 17 +- web/app/components/app/log/var-panel.tsx | 34 +- .../base/auto-height-textarea/index.tsx | 7 +- .../assets/vender/line/general/link-03.svg | 8 + .../assets/vender/line/general/upload-03.svg | 10 + .../assets/vender/line/images/image-plus.svg | 5 + .../icons/assets/vender/solid/general/eye.svg | 4 + .../icons/src/vender/line/general/Link03.json | 57 ++ .../icons/src/vender/line/general/Link03.tsx | 16 + .../src/vender/line/general/Upload03.json | 66 ++ .../src/vender/line/general/Upload03.tsx | 16 + .../icons/src/vender/line/general/index.ts | 2 + .../src/vender/line/images/ImagePlus.json | 39 + .../src/vender/line/images/ImagePlus.tsx | 16 + .../icons/src/vender/line/images/index.ts | 1 + .../icons/src/vender/solid/general/Eye.json | 37 + .../icons/src/vender/solid/general/Eye.tsx | 16 + .../icons/src/vender/solid/general/index.ts | 1 + .../components/base/image-gallery/index.tsx | 84 ++ .../base/image-gallery/style.module.css | 22 + .../image-uploader/chat-image-uploader.tsx | 150 +++ .../components/base/image-uploader/hooks.ts | 81 ++ .../base/image-uploader/image-link-input.tsx | 50 + .../base/image-uploader/image-list.tsx | 129 +++ .../base/image-uploader/image-preview.tsx | 31 + .../text-generation-image-uploader.tsx | 148 +++ .../base/image-uploader/uploader.tsx | 100 ++ .../components/base/image-uploader/utils.ts | 36 + web/app/components/base/param-item/index.tsx | 73 ++ .../base/param-item/score-threshold-item.tsx | 54 ++ .../components/base/param-item/top-k-item.tsx | 48 + web/app/components/base/slider/index.tsx | 6 +- .../develop/template/template.en.mdx | 93 +- .../develop/template/template.zh.mdx | 93 +- .../develop/template/template_chat.en.mdx | 93 +- .../develop/template/template_chat.zh.mdx | 93 +- .../explore/universal-chat/index.tsx | 7 +- .../api-based-extension-page/item.tsx | 1 - .../model-page/declarations.ts | 1 + web/app/components/share/chat/index.tsx | 57 +- web/app/components/share/chatbot/index.tsx | 48 +- .../share/text-generation/index.tsx | 20 +- .../share/text-generation/result/index.tsx | 24 +- .../share/text-generation/run-once/index.tsx | 24 + web/context/debug-configuration.ts | 14 +- web/i18n/lang/app-debug.en.ts | 20 + web/i18n/lang/app-debug.zh.ts | 20 + web/i18n/lang/app-log.en.ts | 1 + web/i18n/lang/app-log.zh.ts | 1 + web/i18n/lang/common.en.ts | 12 +- web/i18n/lang/common.zh.ts | 30 +- web/models/common.ts | 1 + web/models/log.ts | 3 + web/package.json | 1 + web/service/base.ts | 24 +- web/service/share.ts | 4 + web/service/universal-chat.ts | 4 + web/types/app.ts | 41 + web/yarn.lock | 867 +++++++++++++----- 74 files changed, 3159 insertions(+), 339 deletions(-) create mode 100644 web/app/components/app/configuration/config-vision/index.tsx create mode 100644 web/app/components/app/configuration/config-vision/param-config-content.tsx create mode 100644 web/app/components/app/configuration/config-vision/param-config.tsx create mode 100644 web/app/components/app/configuration/config-vision/radio-group/index.tsx create mode 100644 web/app/components/app/configuration/config-vision/radio-group/style.module.css create mode 100644 web/app/components/base/icons/assets/vender/line/general/link-03.svg create mode 100644 web/app/components/base/icons/assets/vender/line/general/upload-03.svg create mode 100644 web/app/components/base/icons/assets/vender/line/images/image-plus.svg create mode 100644 web/app/components/base/icons/assets/vender/solid/general/eye.svg create mode 100644 web/app/components/base/icons/src/vender/line/general/Link03.json create mode 100644 web/app/components/base/icons/src/vender/line/general/Link03.tsx create mode 100644 web/app/components/base/icons/src/vender/line/general/Upload03.json create mode 100644 web/app/components/base/icons/src/vender/line/general/Upload03.tsx create mode 100644 web/app/components/base/icons/src/vender/line/images/ImagePlus.json create mode 100644 web/app/components/base/icons/src/vender/line/images/ImagePlus.tsx create mode 100644 web/app/components/base/icons/src/vender/line/images/index.ts create mode 100644 web/app/components/base/icons/src/vender/solid/general/Eye.json create mode 100644 web/app/components/base/icons/src/vender/solid/general/Eye.tsx create mode 100644 web/app/components/base/image-gallery/index.tsx create mode 100644 web/app/components/base/image-gallery/style.module.css create mode 100644 web/app/components/base/image-uploader/chat-image-uploader.tsx create mode 100644 web/app/components/base/image-uploader/hooks.ts create mode 100644 web/app/components/base/image-uploader/image-link-input.tsx create mode 100644 web/app/components/base/image-uploader/image-list.tsx create mode 100644 web/app/components/base/image-uploader/image-preview.tsx create mode 100644 web/app/components/base/image-uploader/text-generation-image-uploader.tsx create mode 100644 web/app/components/base/image-uploader/uploader.tsx create mode 100644 web/app/components/base/image-uploader/utils.ts create mode 100644 web/app/components/base/param-item/index.tsx create mode 100644 web/app/components/base/param-item/score-threshold-item.tsx create mode 100644 web/app/components/base/param-item/top-k-item.tsx diff --git a/web/app/components/app/chat/index.tsx b/web/app/components/app/chat/index.tsx index fe8ff5fa43..94eeb955c4 100644 --- a/web/app/components/app/chat/index.tsx +++ b/web/app/components/app/chat/index.tsx @@ -1,6 +1,7 @@ 'use client' import type { FC, ReactNode } from 'react' import React, { useEffect, useLayoutEffect, useRef, useState } from 'react' +import Textarea from 'rc-textarea' import { useContext } from 'use-context-selector' import cn from 'classnames' import Recorder from 'js-audio-recorder' @@ -10,9 +11,8 @@ import type { DisplayScene, FeedbackFunc, IChatItem, SubmitAnnotationFunc } from import { TryToAskIcon, stopIcon } from './icon-component' import Answer from './answer' import Question from './question' -import Tooltip from '@/app/components/base/tooltip' +import TooltipPlus from '@/app/components/base/tooltip-plus' import { ToastContext } from '@/app/components/base/toast' -import AutoHeightTextarea from '@/app/components/base/auto-height-textarea' import Button from '@/app/components/base/button' import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' import VoiceInput from '@/app/components/base/voice-input' @@ -20,6 +20,10 @@ import { Microphone01 } from '@/app/components/base/icons/src/vender/line/mediaA import { Microphone01 as Microphone01Solid } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices' import { XCircle } from '@/app/components/base/icons/src/vender/solid/general' import type { DataSet } from '@/models/datasets' +import ChatImageUploader from '@/app/components/base/image-uploader/chat-image-uploader' +import ImageList from '@/app/components/base/image-uploader/image-list' +import { TransferMethod, type VisionFile, type VisionSettings } from '@/types/app' +import { useImageFiles } from '@/app/components/base/image-uploader/hooks' export type IChatProps = { configElem?: React.ReactNode @@ -37,7 +41,7 @@ export type IChatProps = { onFeedback?: FeedbackFunc onSubmitAnnotation?: SubmitAnnotationFunc checkCanSend?: () => boolean - onSend?: (message: string) => void + onSend?: (message: string, files: VisionFile[]) => void displayScene?: DisplayScene useCurrentUserAvatar?: boolean isResponsing?: boolean @@ -54,6 +58,7 @@ export type IChatProps = { dataSets?: DataSet[] isShowCitationHitInfo?: boolean isShowPromptLog?: boolean + visionConfig?: VisionSettings } const Chat: FC = ({ @@ -83,9 +88,19 @@ const Chat: FC = ({ dataSets, isShowCitationHitInfo, isShowPromptLog, + visionConfig, }) => { const { t } = useTranslation() const { notify } = useContext(ToastContext) + const { + files, + onUpload, + onRemove, + onReUpload, + onImageLinkLoadError, + onImageLinkLoadSuccess, + onClear, + } = useImageFiles() const isUseInputMethod = useRef(false) const [query, setQuery] = React.useState('') @@ -114,9 +129,18 @@ const Chat: FC = ({ const handleSend = () => { if (!valid() || (checkCanSend && !checkCanSend())) return - onSend(query) - if (!isResponsing) - setQuery('') + onSend(query, files.filter(file => file.progress !== -1).map(fileItem => ({ + type: 'image', + transfer_method: fileItem.type, + url: fileItem.url, + upload_file_id: fileItem.fileId, + }))) + if (!files.find(item => item.type === TransferMethod.local_file && !item.fileId)) { + if (files.length) + onClear() + if (!isResponsing) + setQuery('') + } } const handleKeyUp = (e: React.KeyboardEvent) => { @@ -198,6 +222,8 @@ const Chat: FC = ({ item={item} isShowPromptLog={isShowPromptLog} isResponsing={isResponsing} + // ['https://placekitten.com/360/360', 'https://placekitten.com/360/640'] + imgSrcs={(item.message_files && item.message_files?.length > 0) ? item.message_files.map(item => item.url) : []} /> ) })} @@ -246,18 +272,42 @@ const Chat: FC = ({ ) } -
- + { + visionConfig?.enabled && ( + <> +
+ = visionConfig.number_limits} + /> +
+
+
+ +
+ + ) + } +