feat: Supports chatting with files/images #1880 (#1943)

### What problem does this PR solve?

feat: Supports chatting with files/images #1880

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
balibabu 2024-08-14 17:26:47 +08:00 committed by GitHub
parent 78ed8fe9a5
commit a3a5a9966f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 487 additions and 37 deletions

View File

@ -0,0 +1,31 @@
import { useFetchKnowledgeGraph } from '@/hooks/chunk-hooks';
import { Modal } from 'antd';
import { useTranslation } from 'react-i18next';
import IndentedTree from './indented-tree';
import { IModalProps } from '@/interfaces/common';
const IndentedTreeModal = ({
documentId,
visible,
hideModal,
}: IModalProps<any> & { documentId: string }) => {
const { data } = useFetchKnowledgeGraph(documentId);
const { t } = useTranslation();
return (
<Modal
title={t('chunk.graph')}
open={visible}
onCancel={hideModal}
width={'90vw'}
footer={null}
>
<section>
<IndentedTree data={data?.data?.mind_map} show></IndentedTree>
</section>
</Modal>
);
};
export default IndentedTreeModal;

View File

@ -0,0 +1,129 @@
import { Authorization } from '@/constants/authorization';
import { useTranslate } from '@/hooks/common-hooks';
import { getAuthorization } from '@/utils/authorization-util';
import { PlusOutlined } from '@ant-design/icons';
import type { GetProp, UploadFile } from 'antd';
import { Button, Flex, Input, Upload, UploadProps } from 'antd';
import get from 'lodash/get';
import { ChangeEventHandler, useCallback, useState } from 'react';
type FileType = Parameters<GetProp<UploadProps, 'beforeUpload'>>[0];
interface IProps {
disabled: boolean;
value: string;
sendDisabled: boolean;
sendLoading: boolean;
onPressEnter(documentIds: string[]): Promise<any>;
onInputChange: ChangeEventHandler<HTMLInputElement>;
conversationId: string;
}
const getBase64 = (file: FileType): Promise<string> =>
new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file as any);
reader.onload = () => resolve(reader.result as string);
reader.onerror = (error) => reject(error);
});
const MessageInput = ({
disabled,
value,
onPressEnter,
sendDisabled,
sendLoading,
onInputChange,
conversationId,
}: IProps) => {
const { t } = useTranslate('chat');
const [fileList, setFileList] = useState<UploadFile[]>([
// {
// uid: '-1',
// name: 'image.png',
// status: 'done',
// url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
// },
// {
// uid: '-xxx',
// percent: 50,
// name: 'image.png',
// status: 'uploading',
// url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
// },
// {
// uid: '-5',
// name: 'image.png',
// status: 'error',
// },
]);
const handlePreview = async (file: UploadFile) => {
if (!file.url && !file.preview) {
file.preview = await getBase64(file.originFileObj as FileType);
}
// setPreviewImage(file.url || (file.preview as string));
// setPreviewOpen(true);
};
const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) => {
console.log('🚀 ~ newFileList:', newFileList);
setFileList(newFileList);
};
const handlePressEnter = useCallback(async () => {
const ids = fileList.reduce((pre, cur) => {
return pre.concat(get(cur, 'response.data', []));
}, []);
await onPressEnter(ids);
setFileList([]);
}, [fileList, onPressEnter]);
const uploadButton = (
<button style={{ border: 0, background: 'none' }} type="button">
<PlusOutlined />
<div style={{ marginTop: 8 }}>Upload</div>
</button>
);
return (
<Flex gap={10} vertical>
<Input
size="large"
placeholder={t('sendPlaceholder')}
value={value}
disabled={disabled}
suffix={
<Button
type="primary"
onClick={handlePressEnter}
loading={sendLoading}
disabled={sendDisabled}
>
{t('send')}
</Button>
}
onPressEnter={handlePressEnter}
onChange={onInputChange}
/>
<Upload
action="/v1/document/upload_and_parse"
listType="picture-card"
fileList={fileList}
onPreview={handlePreview}
onChange={handleChange}
multiple
headers={{ [Authorization]: getAuthorization() }}
data={{ conversation_id: conversationId }}
method="post"
>
{fileList.length >= 8 ? null : uploadButton}
</Upload>
</Flex>
);
};
export default MessageInput;

View File

@ -1,15 +1,17 @@
import { ReactComponent as AssistantIcon } from '@/assets/svg/assistant.svg';
import { MessageType } from '@/constants/chat';
import { useTranslate } from '@/hooks/common-hooks';
import { useSetModalState, useTranslate } from '@/hooks/common-hooks';
import { useSelectFileThumbnails } from '@/hooks/knowledge-hooks';
import { IReference, Message } from '@/interfaces/database/chat';
import { IChunk } from '@/interfaces/database/knowledge';
import classNames from 'classnames';
import { useMemo } from 'react';
import { memo, useCallback, useEffect, useMemo, useState } from 'react';
import { useFetchDocumentInfosByIds } from '@/hooks/document-hooks';
import MarkdownContent from '@/pages/chat/markdown-content';
import { getExtension } from '@/utils/document-util';
import { Avatar, Flex, List } from 'antd';
import { getExtension, isImage } from '@/utils/document-util';
import { Avatar, Button, Flex, List } from 'antd';
import IndentedTreeModal from '../indented-tree/modal';
import NewDocumentLink from '../new-document-link';
import SvgIcon from '../svg-icon';
import styles from './index.less';
@ -32,8 +34,13 @@ const MessageItem = ({
clickDocumentButton,
}: IProps) => {
const isAssistant = item.role === MessageType.Assistant;
const isUser = item.role === MessageType.User;
const { t } = useTranslate('chat');
const fileThumbnails = useSelectFileThumbnails();
const { data: documentList, setDocumentIds } = useFetchDocumentInfosByIds();
console.log('🚀 ~ documentList:', documentList);
const { visible, hideModal, showModal } = useSetModalState();
const [clickedDocumentId, setClickedDocumentId] = useState('');
const referenceDocumentList = useMemo(() => {
return reference?.doc_aggs ?? [];
@ -47,6 +54,21 @@ const MessageItem = ({
return loading ? text?.concat('~~2$$') : text;
}, [item.content, loading, t]);
const handleUserDocumentClick = useCallback(
(id: string) => () => {
setClickedDocumentId(id);
showModal();
},
[showModal],
);
useEffect(() => {
const ids = item?.doc_ids ?? [];
if (ids.length) {
setDocumentIds(ids);
}
}, [item.doc_ids, setDocumentIds]);
return (
<div
className={classNames(styles.messageItem, {
@ -124,11 +146,62 @@ const MessageItem = ({
}}
/>
)}
{isUser && documentList.length > 0 && (
<List
bordered
dataSource={documentList}
renderItem={(item) => {
const fileThumbnail = fileThumbnails[item.id];
const fileExtension = getExtension(item.name);
return (
<List.Item>
<Flex gap={'small'} align="center">
{fileThumbnail ? (
<img
src={fileThumbnail}
className={styles.thumbnailImg}
></img>
) : (
<SvgIcon
name={`file-icon/${fileExtension}`}
width={24}
></SvgIcon>
)}
{isImage(fileExtension) ? (
<NewDocumentLink
documentId={item.id}
documentName={item.name}
prefix="document"
>
{item.name}
</NewDocumentLink>
) : (
<Button
type={'text'}
onClick={handleUserDocumentClick(item.id)}
>
{item.name}
</Button>
)}
</Flex>
</List.Item>
);
}}
/>
)}
</Flex>
</div>
</section>
{visible && (
<IndentedTreeModal
visible={visible}
hideModal={hideModal}
documentId={clickedDocumentId}
></IndentedTreeModal>
)}
</div>
);
};
export default MessageItem;
export default memo(MessageItem);

View File

@ -207,12 +207,13 @@ export const useFetchChunk = (chunkId?: string): ResponseType<any> => {
return data;
};
export const useFetchKnowledgeGraph = (): ResponseType<any> => {
const { documentId } = useGetKnowledgeSearchParams();
export const useFetchKnowledgeGraph = (
documentId: string,
): ResponseType<any> => {
const { data } = useQuery({
queryKey: ['fetchKnowledgeGraph', documentId],
initialData: true,
enabled: !!documentId,
gcTime: 0,
queryFn: async () => {
const data = await kbService.knowledge_graph({

View File

@ -7,16 +7,16 @@ import { useTranslation } from 'react-i18next';
export const useSetModalState = () => {
const [visible, setVisible] = useState(false);
const showModal = () => {
const showModal = useCallback(() => {
setVisible(true);
};
const hideModal = () => {
}, []);
const hideModal = useCallback(() => {
setVisible(false);
};
}, []);
const switchVisible = () => {
const switchVisible = useCallback(() => {
setVisible(!visible);
};
}, [visible]);
return { visible, showModal, hideModal, switchVisible };
};

View File

@ -1,7 +1,10 @@
import { IDocumentInfo } from '@/interfaces/database/document';
import { IChunk, IKnowledgeFile } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
import kbService from '@/services/knowledge-service';
import { api_host } from '@/utils/api';
import { buildChunkHighlights } from '@/utils/document-util';
import { useQuery } from '@tanstack/react-query';
import { UploadFile } from 'antd';
import { useCallback, useMemo, useState } from 'react';
import { IHighlight } from 'react-pdf-highlighter';
@ -253,3 +256,37 @@ export const useSelectRunDocumentLoading = () => {
const loading = useOneNamespaceEffectsLoading('kFModel', ['document_run']);
return loading;
};
export const useFetchDocumentInfosByIds = () => {
const [ids, setDocumentIds] = useState<string[]>([]);
const { data } = useQuery<IDocumentInfo[]>({
queryKey: ['fetchDocumentInfos', ids],
enabled: ids.length > 0,
initialData: [],
queryFn: async () => {
const { data } = await kbService.document_infos({ doc_ids: ids });
if (data.retcode === 0) {
return data.data;
}
return [];
},
});
return { data, setDocumentIds };
};
export const useFetchDocumentThumbnailsByIds = () => {
const [ids, setDocumentIds] = useState<string[]>([]);
const { data } = useQuery({
queryKey: ['fetchDocumentThumbnails', ids],
initialData: [],
queryFn: async () => {
const { data } = await kbService.document_thumbnails({ doc_ids: ids });
return data;
},
});
return { data, setDocumentIds };
};

View File

@ -66,6 +66,7 @@ export interface IConversation {
export interface Message {
content: string;
role: MessageType;
doc_ids?: string[];
}
export interface IReference {

View File

@ -0,0 +1,35 @@
export interface IDocumentInfo {
chunk_num: number;
create_date: string;
create_time: number;
created_by: string;
id: string;
kb_id: string;
location: string;
name: string;
parser_config: Parserconfig;
parser_id: string;
process_begin_at: null;
process_duation: number;
progress: number;
progress_msg: string;
run: string;
size: number;
source_type: string;
status: string;
thumbnail: string;
token_num: number;
type: string;
update_date: string;
update_time: number;
}
interface Parserconfig {
chunk_token_num: number;
layout_recognize: boolean;
raptor: Raptor;
}
interface Raptor {
use_raptor: boolean;
}

View File

@ -1,9 +1,10 @@
import IndentedTree from '@/components/indented-tree/indented-tree';
import { useFetchKnowledgeGraph } from '@/hooks/chunk-hooks';
import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
import { Flex, Modal, Segmented } from 'antd';
import React, { useEffect, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import ForceGraph from './force-graph';
import IndentedTree from './indented-tree';
import styles from './index.less';
import { isDataExist } from './util';
@ -14,7 +15,8 @@ enum SegmentedValue {
const KnowledgeGraphModal: React.FC = () => {
const [isModalOpen, setIsModalOpen] = useState(false);
const { data } = useFetchKnowledgeGraph();
const { documentId } = useGetKnowledgeSearchParams();
const { data } = useFetchKnowledgeGraph(documentId);
const [value, setValue] = useState<SegmentedValue>(SegmentedValue.Graph);
const { t } = useTranslation();

View File

@ -1,8 +1,7 @@
import MessageItem from '@/components/message-item';
import DocumentPreviewer from '@/components/pdf-previewer';
import { MessageType } from '@/constants/chat';
import { useTranslate } from '@/hooks/common-hooks';
import { Button, Drawer, Flex, Input, Spin } from 'antd';
import { Drawer, Flex, Spin } from 'antd';
import {
useClickDrawer,
useFetchConversationOnMount,
@ -14,6 +13,7 @@ import {
} from '../hooks';
import { buildMessageItemReference } from '../utils';
import MessageInput from '@/components/message-input';
import { useFetchUserInfo } from '@/hooks/user-setting-hooks';
import styles from './index.less';
@ -42,7 +42,6 @@ const ChatContainer = () => {
const sendDisabled = useSendButtonDisabled(value);
useGetFileIcon();
const loading = useSelectConversationLoading();
const { t } = useTranslate('chat');
const { data: userInfo } = useFetchUserInfo();
return (
@ -72,7 +71,16 @@ const ChatContainer = () => {
</div>
<div ref={ref} />
</Flex>
<Input
<MessageInput
disabled={disabled}
sendDisabled={sendDisabled}
sendLoading={sendLoading}
value={value}
onInputChange={handleInputChange}
onPressEnter={handlePressEnter}
conversationId={conversation.id}
></MessageInput>
{/* <Input
size="large"
placeholder={t('sendPlaceholder')}
value={value}
@ -89,7 +97,7 @@ const ChatContainer = () => {
}
onPressEnter={handlePressEnter}
onChange={handleInputChange}
/>
/> */}
</Flex>
<Drawer
title="Document Previewer"

View File

@ -547,7 +547,7 @@ export const useSendMessage = (
const { send, answer, done, setDone } = useSendMessageWithSse();
const sendMessage = useCallback(
async (message: string, id?: string) => {
async (message: string, documentIds: string[], id?: string) => {
const res = await send({
conversation_id: id ?? conversationId,
messages: [
@ -555,6 +555,7 @@ export const useSendMessage = (
{
role: MessageType.User,
content: message,
doc_ids: documentIds,
},
],
});
@ -586,14 +587,14 @@ export const useSendMessage = (
);
const handleSendMessage = useCallback(
async (message: string) => {
async (message: string, documentIds: string[]) => {
if (conversationId !== '') {
sendMessage(message);
return sendMessage(message, documentIds);
} else {
const data = await setConversation(message);
if (data.retcode === 0) {
const id = data.data.id;
sendMessage(message, id);
return sendMessage(message, documentIds, id);
}
}
},
@ -614,15 +615,19 @@ export const useSendMessage = (
}
}, [setDone, conversationId]);
const handlePressEnter = useCallback(() => {
if (trim(value) === '') return;
if (done) {
setValue('');
handleSendMessage(value.trim());
}
addNewestConversation(value);
}, [addNewestConversation, handleSendMessage, done, setValue, value]);
const handlePressEnter = useCallback(
async (documentIds: string[]) => {
if (trim(value) === '') return;
let ret;
if (done) {
setValue('');
ret = await handleSendMessage(value.trim(), documentIds);
}
addNewestConversation(value);
return ret;
},
[addNewestConversation, handleSendMessage, done, setValue, value],
);
return {
handlePressEnter,

View File

@ -2,6 +2,7 @@ import { Graph } from '@antv/g6';
import { useSize } from 'ahooks';
import { useEffect, useRef } from 'react';
import { graphData } from './constant';
import InputWithUpload from './input-upload';
import styles from './index.less';
import { Converter } from './util';
@ -108,4 +109,4 @@ const ForceGraph = () => {
return <div ref={containerRef} className={styles.container} />;
};
export default ForceGraph;
export default InputWithUpload;

View File

@ -0,0 +1,118 @@
import { Authorization } from '@/constants/authorization';
import { getAuthorization } from '@/utils/authorization-util';
import { PlusOutlined } from '@ant-design/icons';
import type { GetProp, UploadFile, UploadProps } from 'antd';
import { Image, Input, Upload } from 'antd';
import { useState } from 'react';
import { useGetChatSearchParams } from '../chat/hooks';
type FileType = Parameters<GetProp<UploadProps, 'beforeUpload'>>[0];
const getBase64 = (file: FileType): Promise<string> =>
new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result as string);
reader.onerror = (error) => reject(error);
});
const InputWithUpload = () => {
const [previewOpen, setPreviewOpen] = useState(false);
const [previewImage, setPreviewImage] = useState('');
const { conversationId } = useGetChatSearchParams();
const [fileList, setFileList] = useState<UploadFile[]>([
{
uid: '-1',
name: 'image.png',
status: 'done',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-2',
name: 'image.png',
status: 'done',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-3',
name: 'image.png',
status: 'done',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-4',
name: 'image.png',
status: 'done',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-xxx',
percent: 50,
name: 'image.png',
status: 'uploading',
url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
},
{
uid: '-5',
name: 'image.png',
status: 'error',
},
]);
const handlePreview = async (file: UploadFile) => {
if (!file.url && !file.preview) {
file.preview = await getBase64(file.originFileObj as FileType);
}
setPreviewImage(file.url || (file.preview as string));
setPreviewOpen(true);
};
const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) =>
setFileList(newFileList);
const uploadButton = (
<button style={{ border: 0, background: 'none' }} type="button">
<PlusOutlined />
<div style={{ marginTop: 8 }}>Upload</div>
</button>
);
return (
<>
<Input placeholder="Basic usage"></Input>
<Upload
action="/v1/document/upload_and_parse"
listType="picture-card"
fileList={fileList}
onPreview={handlePreview}
onChange={handleChange}
multiple
headers={{ [Authorization]: getAuthorization() }}
data={{ conversation_id: '9e9f7d2453e511efb18efa163e197198' }}
method="post"
>
{fileList.length >= 8 ? null : uploadButton}
</Upload>
{previewImage && (
<Image
wrapperStyle={{ display: 'none' }}
preview={{
visible: previewOpen,
onVisibleChange: (visible) => setPreviewOpen(visible),
afterOpenChange: (visible) => !visible && setPreviewImage(''),
}}
src={previewImage}
/>
)}
</>
);
};
export default () => {
return (
<section style={{ height: 500, width: 400 }}>
<div style={{ height: 200 }}></div>
<InputWithUpload></InputWithUpload>
</section>
);
};

View File

@ -28,6 +28,7 @@ const {
document_upload,
web_crawl,
knowledge_graph,
document_infos,
} = api;
const methods = {
@ -93,6 +94,10 @@ const methods = {
url: web_crawl,
method: 'post',
},
document_infos: {
url: document_infos,
method: 'post',
},
// chunk管理
chunk_list: {
url: chunk_list,

View File

@ -38,7 +38,6 @@ export default {
knowledge_graph: `${api_host}/chunk/knowledge_graph`,
// document
upload: `${api_host}/document/upload`,
get_document_list: `${api_host}/document/list`,
document_change_status: `${api_host}/document/change_status`,
document_rm: `${api_host}/document/rm`,
@ -50,6 +49,7 @@ export default {
get_document_file: `${api_host}/document/get`,
document_upload: `${api_host}/document/upload`,
web_crawl: `${api_host}/document/web_crawl`,
document_infos: `${api_host}/document/infos`,
// chat
setDialog: `${api_host}/dialog/set`,

View File

@ -1,4 +1,4 @@
import { SupportedPreviewDocumentTypes } from '@/constants/common';
import { Images, SupportedPreviewDocumentTypes } from '@/constants/common';
import { IChunk } from '@/interfaces/database/knowledge';
import { UploadFile } from 'antd';
import { v4 as uuid } from 'uuid';
@ -51,3 +51,7 @@ export const getUnSupportedFilesCount = (message: string) => {
export const isSupportedPreviewDocumentType = (fileExtension: string) => {
return SupportedPreviewDocumentTypes.includes(fileExtension);
};
export const isImage = (image: string) => {
return [...Images, 'svg'].some((x) => x === image);
};