From a3a5a9966f23dd67ecf6f9076cf4ced108e57ca3 Mon Sep 17 00:00:00 2001 From: balibabu Date: Wed, 14 Aug 2024 17:26:47 +0800 Subject: [PATCH] feat: Supports chatting with files/images #1880 (#1943) ### What problem does this PR solve? feat: Supports chatting with files/images #1880 ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- .../indented-tree}/indented-tree.tsx | 0 web/src/components/indented-tree/modal.tsx | 31 +++++ web/src/components/message-input/index.tsx | 129 ++++++++++++++++++ web/src/components/message-item/index.tsx | 83 ++++++++++- web/src/hooks/chunk-hooks.ts | 7 +- web/src/hooks/common-hooks.tsx | 12 +- web/src/hooks/document-hooks.ts | 37 +++++ web/src/interfaces/database/chat.ts | 1 + web/src/interfaces/database/document.ts | 35 +++++ .../components/knowledge-graph/modal.tsx | 6 +- web/src/pages/chat/chat-container/index.tsx | 18 ++- web/src/pages/chat/hooks.ts | 31 +++-- web/src/pages/force-graph/index.tsx | 3 +- web/src/pages/force-graph/input-upload.tsx | 118 ++++++++++++++++ web/src/services/knowledge-service.ts | 5 + web/src/utils/api.ts | 2 +- web/src/utils/document-util.ts | 6 +- 17 files changed, 487 insertions(+), 37 deletions(-) rename web/src/{pages/add-knowledge/components/knowledge-chunk/components/knowledge-graph => components/indented-tree}/indented-tree.tsx (100%) create mode 100644 web/src/components/indented-tree/modal.tsx create mode 100644 web/src/components/message-input/index.tsx create mode 100644 web/src/interfaces/database/document.ts create mode 100644 web/src/pages/force-graph/input-upload.tsx diff --git a/web/src/pages/add-knowledge/components/knowledge-chunk/components/knowledge-graph/indented-tree.tsx b/web/src/components/indented-tree/indented-tree.tsx similarity index 100% rename from web/src/pages/add-knowledge/components/knowledge-chunk/components/knowledge-graph/indented-tree.tsx rename to web/src/components/indented-tree/indented-tree.tsx diff --git a/web/src/components/indented-tree/modal.tsx b/web/src/components/indented-tree/modal.tsx new file mode 100644 index 000000000..a54b6c70b --- /dev/null +++ b/web/src/components/indented-tree/modal.tsx @@ -0,0 +1,31 @@ +import { useFetchKnowledgeGraph } from '@/hooks/chunk-hooks'; +import { Modal } from 'antd'; +import { useTranslation } from 'react-i18next'; +import IndentedTree from './indented-tree'; + +import { IModalProps } from '@/interfaces/common'; + +const IndentedTreeModal = ({ + documentId, + visible, + hideModal, +}: IModalProps & { documentId: string }) => { + const { data } = useFetchKnowledgeGraph(documentId); + const { t } = useTranslation(); + + return ( + +
+ +
+ + ); +}; + +export default IndentedTreeModal; diff --git a/web/src/components/message-input/index.tsx b/web/src/components/message-input/index.tsx new file mode 100644 index 000000000..6ef01a776 --- /dev/null +++ b/web/src/components/message-input/index.tsx @@ -0,0 +1,129 @@ +import { Authorization } from '@/constants/authorization'; +import { useTranslate } from '@/hooks/common-hooks'; +import { getAuthorization } from '@/utils/authorization-util'; +import { PlusOutlined } from '@ant-design/icons'; +import type { GetProp, UploadFile } from 'antd'; +import { Button, Flex, Input, Upload, UploadProps } from 'antd'; +import get from 'lodash/get'; +import { ChangeEventHandler, useCallback, useState } from 'react'; + +type FileType = Parameters>[0]; + +interface IProps { + disabled: boolean; + value: string; + sendDisabled: boolean; + sendLoading: boolean; + onPressEnter(documentIds: string[]): Promise; + onInputChange: ChangeEventHandler; + conversationId: string; +} + +const getBase64 = (file: FileType): Promise => + new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file as any); + reader.onload = () => resolve(reader.result as string); + reader.onerror = (error) => reject(error); + }); + +const MessageInput = ({ + disabled, + value, + onPressEnter, + sendDisabled, + sendLoading, + onInputChange, + conversationId, +}: IProps) => { + const { t } = useTranslate('chat'); + + const [fileList, setFileList] = useState([ + // { + // uid: '-1', + // name: 'image.png', + // status: 'done', + // url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png', + // }, + // { + // uid: '-xxx', + // percent: 50, + // name: 'image.png', + // status: 'uploading', + // url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png', + // }, + // { + // uid: '-5', + // name: 'image.png', + // status: 'error', + // }, + ]); + + const handlePreview = async (file: UploadFile) => { + if (!file.url && !file.preview) { + file.preview = await getBase64(file.originFileObj as FileType); + } + + // setPreviewImage(file.url || (file.preview as string)); + // setPreviewOpen(true); + }; + + const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) => { + console.log('🚀 ~ newFileList:', newFileList); + setFileList(newFileList); + }; + + const handlePressEnter = useCallback(async () => { + const ids = fileList.reduce((pre, cur) => { + return pre.concat(get(cur, 'response.data', [])); + }, []); + + await onPressEnter(ids); + setFileList([]); + }, [fileList, onPressEnter]); + + const uploadButton = ( + + ); + + return ( + + + {t('send')} + + } + onPressEnter={handlePressEnter} + onChange={onInputChange} + /> + + {fileList.length >= 8 ? null : uploadButton} + + + ); +}; + +export default MessageInput; diff --git a/web/src/components/message-item/index.tsx b/web/src/components/message-item/index.tsx index 2f17e3460..c5a5af958 100644 --- a/web/src/components/message-item/index.tsx +++ b/web/src/components/message-item/index.tsx @@ -1,15 +1,17 @@ import { ReactComponent as AssistantIcon } from '@/assets/svg/assistant.svg'; import { MessageType } from '@/constants/chat'; -import { useTranslate } from '@/hooks/common-hooks'; +import { useSetModalState, useTranslate } from '@/hooks/common-hooks'; import { useSelectFileThumbnails } from '@/hooks/knowledge-hooks'; import { IReference, Message } from '@/interfaces/database/chat'; import { IChunk } from '@/interfaces/database/knowledge'; import classNames from 'classnames'; -import { useMemo } from 'react'; +import { memo, useCallback, useEffect, useMemo, useState } from 'react'; +import { useFetchDocumentInfosByIds } from '@/hooks/document-hooks'; import MarkdownContent from '@/pages/chat/markdown-content'; -import { getExtension } from '@/utils/document-util'; -import { Avatar, Flex, List } from 'antd'; +import { getExtension, isImage } from '@/utils/document-util'; +import { Avatar, Button, Flex, List } from 'antd'; +import IndentedTreeModal from '../indented-tree/modal'; import NewDocumentLink from '../new-document-link'; import SvgIcon from '../svg-icon'; import styles from './index.less'; @@ -32,8 +34,13 @@ const MessageItem = ({ clickDocumentButton, }: IProps) => { const isAssistant = item.role === MessageType.Assistant; + const isUser = item.role === MessageType.User; const { t } = useTranslate('chat'); const fileThumbnails = useSelectFileThumbnails(); + const { data: documentList, setDocumentIds } = useFetchDocumentInfosByIds(); + console.log('🚀 ~ documentList:', documentList); + const { visible, hideModal, showModal } = useSetModalState(); + const [clickedDocumentId, setClickedDocumentId] = useState(''); const referenceDocumentList = useMemo(() => { return reference?.doc_aggs ?? []; @@ -47,6 +54,21 @@ const MessageItem = ({ return loading ? text?.concat('~~2$$') : text; }, [item.content, loading, t]); + const handleUserDocumentClick = useCallback( + (id: string) => () => { + setClickedDocumentId(id); + showModal(); + }, + [showModal], + ); + + useEffect(() => { + const ids = item?.doc_ids ?? []; + if (ids.length) { + setDocumentIds(ids); + } + }, [item.doc_ids, setDocumentIds]); + return (
)} + {isUser && documentList.length > 0 && ( + { + const fileThumbnail = fileThumbnails[item.id]; + const fileExtension = getExtension(item.name); + return ( + + + {fileThumbnail ? ( + + ) : ( + + )} + + {isImage(fileExtension) ? ( + + {item.name} + + ) : ( + + )} + + + ); + }} + /> + )}
+ {visible && ( + + )} ); }; -export default MessageItem; +export default memo(MessageItem); diff --git a/web/src/hooks/chunk-hooks.ts b/web/src/hooks/chunk-hooks.ts index 0a5cc6c22..deee3aada 100644 --- a/web/src/hooks/chunk-hooks.ts +++ b/web/src/hooks/chunk-hooks.ts @@ -207,12 +207,13 @@ export const useFetchChunk = (chunkId?: string): ResponseType => { return data; }; -export const useFetchKnowledgeGraph = (): ResponseType => { - const { documentId } = useGetKnowledgeSearchParams(); - +export const useFetchKnowledgeGraph = ( + documentId: string, +): ResponseType => { const { data } = useQuery({ queryKey: ['fetchKnowledgeGraph', documentId], initialData: true, + enabled: !!documentId, gcTime: 0, queryFn: async () => { const data = await kbService.knowledge_graph({ diff --git a/web/src/hooks/common-hooks.tsx b/web/src/hooks/common-hooks.tsx index 03a16509b..d683177ba 100644 --- a/web/src/hooks/common-hooks.tsx +++ b/web/src/hooks/common-hooks.tsx @@ -7,16 +7,16 @@ import { useTranslation } from 'react-i18next'; export const useSetModalState = () => { const [visible, setVisible] = useState(false); - const showModal = () => { + const showModal = useCallback(() => { setVisible(true); - }; - const hideModal = () => { + }, []); + const hideModal = useCallback(() => { setVisible(false); - }; + }, []); - const switchVisible = () => { + const switchVisible = useCallback(() => { setVisible(!visible); - }; + }, [visible]); return { visible, showModal, hideModal, switchVisible }; }; diff --git a/web/src/hooks/document-hooks.ts b/web/src/hooks/document-hooks.ts index aa34c8729..65b1bb9d2 100644 --- a/web/src/hooks/document-hooks.ts +++ b/web/src/hooks/document-hooks.ts @@ -1,7 +1,10 @@ +import { IDocumentInfo } from '@/interfaces/database/document'; import { IChunk, IKnowledgeFile } from '@/interfaces/database/knowledge'; import { IChangeParserConfigRequestBody } from '@/interfaces/request/document'; +import kbService from '@/services/knowledge-service'; import { api_host } from '@/utils/api'; import { buildChunkHighlights } from '@/utils/document-util'; +import { useQuery } from '@tanstack/react-query'; import { UploadFile } from 'antd'; import { useCallback, useMemo, useState } from 'react'; import { IHighlight } from 'react-pdf-highlighter'; @@ -253,3 +256,37 @@ export const useSelectRunDocumentLoading = () => { const loading = useOneNamespaceEffectsLoading('kFModel', ['document_run']); return loading; }; + +export const useFetchDocumentInfosByIds = () => { + const [ids, setDocumentIds] = useState([]); + const { data } = useQuery({ + queryKey: ['fetchDocumentInfos', ids], + enabled: ids.length > 0, + initialData: [], + queryFn: async () => { + const { data } = await kbService.document_infos({ doc_ids: ids }); + if (data.retcode === 0) { + return data.data; + } + + return []; + }, + }); + + return { data, setDocumentIds }; +}; + +export const useFetchDocumentThumbnailsByIds = () => { + const [ids, setDocumentIds] = useState([]); + const { data } = useQuery({ + queryKey: ['fetchDocumentThumbnails', ids], + initialData: [], + queryFn: async () => { + const { data } = await kbService.document_thumbnails({ doc_ids: ids }); + + return data; + }, + }); + + return { data, setDocumentIds }; +}; diff --git a/web/src/interfaces/database/chat.ts b/web/src/interfaces/database/chat.ts index c30fa3453..843caaf6a 100644 --- a/web/src/interfaces/database/chat.ts +++ b/web/src/interfaces/database/chat.ts @@ -66,6 +66,7 @@ export interface IConversation { export interface Message { content: string; role: MessageType; + doc_ids?: string[]; } export interface IReference { diff --git a/web/src/interfaces/database/document.ts b/web/src/interfaces/database/document.ts new file mode 100644 index 000000000..626aea4ec --- /dev/null +++ b/web/src/interfaces/database/document.ts @@ -0,0 +1,35 @@ +export interface IDocumentInfo { + chunk_num: number; + create_date: string; + create_time: number; + created_by: string; + id: string; + kb_id: string; + location: string; + name: string; + parser_config: Parserconfig; + parser_id: string; + process_begin_at: null; + process_duation: number; + progress: number; + progress_msg: string; + run: string; + size: number; + source_type: string; + status: string; + thumbnail: string; + token_num: number; + type: string; + update_date: string; + update_time: number; +} + +interface Parserconfig { + chunk_token_num: number; + layout_recognize: boolean; + raptor: Raptor; +} + +interface Raptor { + use_raptor: boolean; +} diff --git a/web/src/pages/add-knowledge/components/knowledge-chunk/components/knowledge-graph/modal.tsx b/web/src/pages/add-knowledge/components/knowledge-chunk/components/knowledge-graph/modal.tsx index d19e1fd1a..a9fcbe496 100644 --- a/web/src/pages/add-knowledge/components/knowledge-chunk/components/knowledge-graph/modal.tsx +++ b/web/src/pages/add-knowledge/components/knowledge-chunk/components/knowledge-graph/modal.tsx @@ -1,9 +1,10 @@ +import IndentedTree from '@/components/indented-tree/indented-tree'; import { useFetchKnowledgeGraph } from '@/hooks/chunk-hooks'; +import { useGetKnowledgeSearchParams } from '@/hooks/route-hook'; import { Flex, Modal, Segmented } from 'antd'; import React, { useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import ForceGraph from './force-graph'; -import IndentedTree from './indented-tree'; import styles from './index.less'; import { isDataExist } from './util'; @@ -14,7 +15,8 @@ enum SegmentedValue { const KnowledgeGraphModal: React.FC = () => { const [isModalOpen, setIsModalOpen] = useState(false); - const { data } = useFetchKnowledgeGraph(); + const { documentId } = useGetKnowledgeSearchParams(); + const { data } = useFetchKnowledgeGraph(documentId); const [value, setValue] = useState(SegmentedValue.Graph); const { t } = useTranslation(); diff --git a/web/src/pages/chat/chat-container/index.tsx b/web/src/pages/chat/chat-container/index.tsx index 72e410de6..7c26954f2 100644 --- a/web/src/pages/chat/chat-container/index.tsx +++ b/web/src/pages/chat/chat-container/index.tsx @@ -1,8 +1,7 @@ import MessageItem from '@/components/message-item'; import DocumentPreviewer from '@/components/pdf-previewer'; import { MessageType } from '@/constants/chat'; -import { useTranslate } from '@/hooks/common-hooks'; -import { Button, Drawer, Flex, Input, Spin } from 'antd'; +import { Drawer, Flex, Spin } from 'antd'; import { useClickDrawer, useFetchConversationOnMount, @@ -14,6 +13,7 @@ import { } from '../hooks'; import { buildMessageItemReference } from '../utils'; +import MessageInput from '@/components/message-input'; import { useFetchUserInfo } from '@/hooks/user-setting-hooks'; import styles from './index.less'; @@ -42,7 +42,6 @@ const ChatContainer = () => { const sendDisabled = useSendButtonDisabled(value); useGetFileIcon(); const loading = useSelectConversationLoading(); - const { t } = useTranslate('chat'); const { data: userInfo } = useFetchUserInfo(); return ( @@ -72,7 +71,16 @@ const ChatContainer = () => {
- + {/* { } onPressEnter={handlePressEnter} onChange={handleInputChange} - /> + /> */} { + async (message: string, documentIds: string[], id?: string) => { const res = await send({ conversation_id: id ?? conversationId, messages: [ @@ -555,6 +555,7 @@ export const useSendMessage = ( { role: MessageType.User, content: message, + doc_ids: documentIds, }, ], }); @@ -586,14 +587,14 @@ export const useSendMessage = ( ); const handleSendMessage = useCallback( - async (message: string) => { + async (message: string, documentIds: string[]) => { if (conversationId !== '') { - sendMessage(message); + return sendMessage(message, documentIds); } else { const data = await setConversation(message); if (data.retcode === 0) { const id = data.data.id; - sendMessage(message, id); + return sendMessage(message, documentIds, id); } } }, @@ -614,15 +615,19 @@ export const useSendMessage = ( } }, [setDone, conversationId]); - const handlePressEnter = useCallback(() => { - if (trim(value) === '') return; - - if (done) { - setValue(''); - handleSendMessage(value.trim()); - } - addNewestConversation(value); - }, [addNewestConversation, handleSendMessage, done, setValue, value]); + const handlePressEnter = useCallback( + async (documentIds: string[]) => { + if (trim(value) === '') return; + let ret; + if (done) { + setValue(''); + ret = await handleSendMessage(value.trim(), documentIds); + } + addNewestConversation(value); + return ret; + }, + [addNewestConversation, handleSendMessage, done, setValue, value], + ); return { handlePressEnter, diff --git a/web/src/pages/force-graph/index.tsx b/web/src/pages/force-graph/index.tsx index c131ca0b2..4c95d39f2 100644 --- a/web/src/pages/force-graph/index.tsx +++ b/web/src/pages/force-graph/index.tsx @@ -2,6 +2,7 @@ import { Graph } from '@antv/g6'; import { useSize } from 'ahooks'; import { useEffect, useRef } from 'react'; import { graphData } from './constant'; +import InputWithUpload from './input-upload'; import styles from './index.less'; import { Converter } from './util'; @@ -108,4 +109,4 @@ const ForceGraph = () => { return
; }; -export default ForceGraph; +export default InputWithUpload; diff --git a/web/src/pages/force-graph/input-upload.tsx b/web/src/pages/force-graph/input-upload.tsx new file mode 100644 index 000000000..a4b203d5a --- /dev/null +++ b/web/src/pages/force-graph/input-upload.tsx @@ -0,0 +1,118 @@ +import { Authorization } from '@/constants/authorization'; +import { getAuthorization } from '@/utils/authorization-util'; +import { PlusOutlined } from '@ant-design/icons'; +import type { GetProp, UploadFile, UploadProps } from 'antd'; +import { Image, Input, Upload } from 'antd'; +import { useState } from 'react'; +import { useGetChatSearchParams } from '../chat/hooks'; + +type FileType = Parameters>[0]; + +const getBase64 = (file: FileType): Promise => + new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => resolve(reader.result as string); + reader.onerror = (error) => reject(error); + }); + +const InputWithUpload = () => { + const [previewOpen, setPreviewOpen] = useState(false); + const [previewImage, setPreviewImage] = useState(''); + const { conversationId } = useGetChatSearchParams(); + const [fileList, setFileList] = useState([ + { + uid: '-1', + name: 'image.png', + status: 'done', + url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png', + }, + { + uid: '-2', + name: 'image.png', + status: 'done', + url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png', + }, + { + uid: '-3', + name: 'image.png', + status: 'done', + url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png', + }, + { + uid: '-4', + name: 'image.png', + status: 'done', + url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png', + }, + { + uid: '-xxx', + percent: 50, + name: 'image.png', + status: 'uploading', + url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png', + }, + { + uid: '-5', + name: 'image.png', + status: 'error', + }, + ]); + + const handlePreview = async (file: UploadFile) => { + if (!file.url && !file.preview) { + file.preview = await getBase64(file.originFileObj as FileType); + } + + setPreviewImage(file.url || (file.preview as string)); + setPreviewOpen(true); + }; + + const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) => + setFileList(newFileList); + + const uploadButton = ( + + ); + return ( + <> + + + {fileList.length >= 8 ? null : uploadButton} + + {previewImage && ( + setPreviewOpen(visible), + afterOpenChange: (visible) => !visible && setPreviewImage(''), + }} + src={previewImage} + /> + )} + + ); +}; + +export default () => { + return ( +
+
+ +
+ ); +}; diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.ts index c9b199d0c..6463a344f 100644 --- a/web/src/services/knowledge-service.ts +++ b/web/src/services/knowledge-service.ts @@ -28,6 +28,7 @@ const { document_upload, web_crawl, knowledge_graph, + document_infos, } = api; const methods = { @@ -93,6 +94,10 @@ const methods = { url: web_crawl, method: 'post', }, + document_infos: { + url: document_infos, + method: 'post', + }, // chunk管理 chunk_list: { url: chunk_list, diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts index 1051e704d..f8fd59e8a 100644 --- a/web/src/utils/api.ts +++ b/web/src/utils/api.ts @@ -38,7 +38,6 @@ export default { knowledge_graph: `${api_host}/chunk/knowledge_graph`, // document - upload: `${api_host}/document/upload`, get_document_list: `${api_host}/document/list`, document_change_status: `${api_host}/document/change_status`, document_rm: `${api_host}/document/rm`, @@ -50,6 +49,7 @@ export default { get_document_file: `${api_host}/document/get`, document_upload: `${api_host}/document/upload`, web_crawl: `${api_host}/document/web_crawl`, + document_infos: `${api_host}/document/infos`, // chat setDialog: `${api_host}/dialog/set`, diff --git a/web/src/utils/document-util.ts b/web/src/utils/document-util.ts index eeb581ed6..3b3f31000 100644 --- a/web/src/utils/document-util.ts +++ b/web/src/utils/document-util.ts @@ -1,4 +1,4 @@ -import { SupportedPreviewDocumentTypes } from '@/constants/common'; +import { Images, SupportedPreviewDocumentTypes } from '@/constants/common'; import { IChunk } from '@/interfaces/database/knowledge'; import { UploadFile } from 'antd'; import { v4 as uuid } from 'uuid'; @@ -51,3 +51,7 @@ export const getUnSupportedFilesCount = (message: string) => { export const isSupportedPreviewDocumentType = (fileExtension: string) => { return SupportedPreviewDocumentTypes.includes(fileExtension); }; + +export const isImage = (image: string) => { + return [...Images, 'svg'].some((x) => x === image); +};