feat: select the corresponding parsing method according to the file type and after the document is successfully uploaded, use the ChunkMethodModal to select the parsing method. and remove ChunkMethodModal from knowledge-file (#158)

* feat: select the corresponding parsing method according to the file type

* feat: after the document is successfully uploaded, use the ChunkMethodModal  to select the parsing method.

* feat: add pdf types to ParserListMap

* feat: remove ChunkMethodModal from knowledge-file
This commit is contained in:
balibabu 2024-03-27 17:56:34 +08:00 committed by GitHub
parent bf2e3d7fc1
commit 44541a8c33
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 340 additions and 219 deletions

View File

@ -0,0 +1,82 @@
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import { useEffect, useMemo, useState } from 'react';
const ParserListMap = new Map([
[
['pdf'],
[
'naive',
'resume',
'manual',
'paper',
'book',
'laws',
'presentation',
'one',
],
],
[
['doc', 'docx'],
['naive', 'resume', 'book', 'laws', 'one'],
],
[
['xlsx', 'xls'],
['naive', 'qa', 'table', 'one'],
],
[['ppt', 'pptx'], ['presentation']],
[
['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tif', 'tiff', 'webp', 'svg', 'ico'],
['picture'],
],
[['txt'], ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table']],
[['csv'], ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table']],
]);
const getParserList = (
values: string[],
parserList: Array<{
value: string;
label: string;
}>,
) => {
return parserList.filter((x) => values?.some((y) => y === x.value));
};
export const useFetchParserListOnMount = (
parserId: string,
documentExtension: string,
) => {
const [selectedTag, setSelectedTag] = useState('');
const parserList = useSelectParserList();
const nextParserList = useMemo(() => {
const key = [...ParserListMap.keys()].find((x) =>
x.some((y) => y === documentExtension),
);
if (key) {
const values = ParserListMap.get(key);
return getParserList(values ?? [], parserList);
}
return getParserList(
['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table'],
parserList,
);
}, [parserList, documentExtension]);
useFetchTenantInfo();
useEffect(() => {
setSelectedTag(parserId);
}, [parserId]);
const handleChange = (tag: string, checked: boolean) => {
const nextSelectedTag = checked ? tag : selectedTag;
setSelectedTag(nextSelectedTag);
};
return { parserList: nextParserList, handleChange, selectedTag };
};

View File

@ -0,0 +1,10 @@
.pageInputNumber {
width: 220px;
}
.questionIcon {
margin-inline-start: 4px;
color: rgba(0, 0, 0, 0.45);
cursor: help;
writing-mode: horizontal-tb;
}

View File

@ -35,7 +35,7 @@ interface IProps extends Omit<IModalManagerChildrenProps, 'showModal'> {
showModal?(): void;
parserId: string;
parserConfig: IKnowledgeFileParserConfig;
documentType: string;
documentExtension: string;
}
const hidePagesChunkMethods = ['qa', 'table', 'picture', 'resume', 'one'];
@ -45,11 +45,13 @@ const ChunkMethodModal: React.FC<IProps> = ({
onOk,
hideModal,
visible,
documentType,
documentExtension,
parserConfig,
}) => {
const { parserList, handleChange, selectedTag } =
useFetchParserListOnMount(parserId);
const { parserList, handleChange, selectedTag } = useFetchParserListOnMount(
parserId,
documentExtension,
);
const [form] = Form.useForm();
const handleOk = async () => {
@ -62,11 +64,8 @@ const ChunkMethodModal: React.FC<IProps> = ({
};
const showPages = useMemo(() => {
return (
documentType === 'pdf' &&
hidePagesChunkMethods.every((x) => x !== selectedTag)
);
}, [documentType, selectedTag]);
return hidePagesChunkMethods.every((x) => x !== selectedTag);
}, [selectedTag]);
const showOne = useMemo(() => {
return showPages || selectedTag === 'one';
@ -114,7 +113,7 @@ const ChunkMethodModal: React.FC<IProps> = ({
</Space>
<Divider></Divider>
{
{documentExtension === 'pdf' && (
<Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
{showPages && (
<>
@ -271,7 +270,7 @@ const ChunkMethodModal: React.FC<IProps> = ({
{selectedTag === 'naive' && <MaxTokenNumber></MaxTokenNumber>}
</Form>
}
)}
</Modal>
);
};

View File

@ -2,6 +2,7 @@ import { IChunk, IKnowledgeFile } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
import { api_host } from '@/utils/api';
import { buildChunkHighlights } from '@/utils/documentUtils';
import { UploadFile } from 'antd';
import { useCallback, useMemo, useState } from 'react';
import { IHighlight } from 'react-pdf-highlighter';
import { useDispatch, useSelector } from 'umi';
@ -174,3 +175,47 @@ export const useRemoveDocument = (documentId: string) => {
return removeDocument;
};
export const useUploadDocument = () => {
const dispatch = useDispatch();
const { knowledgeId } = useGetKnowledgeSearchParams();
const uploadDocument = useCallback(
(file: UploadFile) => {
try {
return dispatch<any>({
type: 'kFModel/upload_document',
payload: {
file,
kb_id: knowledgeId,
},
});
} catch (errorInfo) {
console.log('Failed:', errorInfo);
}
},
[dispatch, knowledgeId],
);
return uploadDocument;
};
export const useRunDocument = () => {
const dispatch = useDispatch();
const runDocumentByIds = useCallback(
(ids: string[]) => {
try {
return dispatch<any>({
type: 'kFModel/document_run',
payload: { doc_ids: ids, run: 1 },
});
} catch (errorInfo) {
console.log('Failed:', errorInfo);
}
},
[dispatch],
);
return runDocumentByIds;
};

View File

@ -0,0 +1,47 @@
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
import { useCallback, useState } from 'react';
import { useSetModalState } from './commonHooks';
import { useSetDocumentParser } from './documentHooks';
import { useOneNamespaceEffectsLoading } from './storeHooks';
export const useChangeDocumentParser = (documentId: string) => {
const setDocumentParser = useSetDocumentParser();
const {
visible: changeParserVisible,
hideModal: hideChangeParserModal,
showModal: showChangeParserModal,
} = useSetModalState();
const loading = useOneNamespaceEffectsLoading('kFModel', [
'document_change_parser',
]);
const onChangeParserOk = useCallback(
async (parserId: string, parserConfig: IChangeParserConfigRequestBody) => {
const ret = await setDocumentParser(parserId, documentId, parserConfig);
if (ret === 0) {
hideChangeParserModal();
}
},
[hideChangeParserModal, setDocumentParser, documentId],
);
return {
changeParserLoading: loading,
onChangeParserOk,
changeParserVisible,
hideChangeParserModal,
showChangeParserModal,
};
};
export const useSetSelectedRecord = <T = IKnowledgeFile>() => {
const [currentRecord, setCurrentRecord] = useState<T>({} as T);
const setRecord = (record: T) => {
setCurrentRecord(record);
};
return { currentRecord, setRecord };
};

View File

@ -1,18 +1,24 @@
import { ReactComponent as SelectFilesEndIcon } from '@/assets/svg/select-files-end.svg';
import { ReactComponent as SelectFilesStartIcon } from '@/assets/svg/select-files-start.svg';
import ChunkMethodModal from '@/components/chunk-method-modal';
import { KnowledgeRouteKey } from '@/constants/knowledge';
import {
useRunDocument,
useSelectDocumentList,
useUploadDocument,
} from '@/hooks/documentHooks';
import {
useDeleteDocumentById,
useFetchKnowledgeDetail,
useGetDocumentDefaultParser,
useKnowledgeBaseId,
} from '@/hooks/knowledgeHook';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import uploadService from '@/services/uploadService';
import { isFileUploadDone } from '@/utils/documentUtils';
useChangeDocumentParser,
useSetSelectedRecord,
} from '@/hooks/logicHooks';
import { useFetchTenantInfo } from '@/hooks/userSettingHook';
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { getExtension, isFileUploadDone } from '@/utils/documentUtils';
import {
ArrowLeftOutlined,
CloudUploadOutlined,
@ -24,27 +30,16 @@ import {
Button,
Card,
Flex,
Popover,
Progress,
Radio,
RadioChangeEvent,
Space,
Upload,
UploadFile,
UploadProps,
} from 'antd';
import classNames from 'classnames';
import {
ReactElement,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react';
import { Link, useDispatch, useNavigate } from 'umi';
import { ReactElement, useCallback, useMemo, useRef, useState } from 'react';
import { Link, useNavigate } from 'umi';
import { useSetDocumentParser } from '@/hooks/documentHooks';
import styles from './index.less';
const { Dragger } = Upload;
@ -57,48 +52,21 @@ const UploaderItem = ({
file,
isUpload,
remove,
handleEdit,
}: {
isUpload: boolean;
originNode: ReactElement;
file: UploadFile;
fileList: object[];
showModal: () => void;
remove: (id: string) => void;
setRecord: (record: IKnowledgeFile) => void;
handleEdit: (id: string) => void;
}) => {
const { parserConfig, defaultParserId } = useGetDocumentDefaultParser();
const { removeDocument } = useDeleteDocumentById();
const [value, setValue] = useState(defaultParserId);
const setDocumentParser = useSetDocumentParser();
const documentId = file?.response?.id;
const parserList = useSelectParserList();
const saveParser = (parserId: string) => {
setDocumentParser(parserId, documentId, parserConfig as any);
};
const onChange = (e: RadioChangeEvent) => {
const val = e.target.value;
setValue(val);
saveParser(val);
};
const content = (
<Radio.Group onChange={onChange} value={value}>
<Space direction="vertical">
{parserList.map(
(
x, // value is lowercase, key is uppercase
) => (
<Radio value={x.value} key={x.value}>
{x.label}
</Radio>
),
)}
</Space>
</Radio.Group>
);
const handleRemove = async () => {
if (file.status === 'error') {
remove(documentId);
@ -110,9 +78,11 @@ const UploaderItem = ({
}
};
useEffect(() => {
setValue(defaultParserId);
}, [defaultParserId]);
const handleEditClick = () => {
if (file.status === 'done') {
handleEdit(documentId);
}
};
return (
<Card className={styles.uploaderItem}>
@ -130,9 +100,7 @@ const UploaderItem = ({
onClick={handleRemove}
/>
) : (
<Popover content={content} placement="bottom">
<EditOutlined />
</Popover>
<EditOutlined onClick={handleEditClick} />
)}
</Flex>
<Flex>
@ -153,10 +121,20 @@ const UploaderItem = ({
const KnowledgeUploadFile = () => {
const knowledgeBaseId = useKnowledgeBaseId();
const [isUpload, setIsUpload] = useState(true);
const dispatch = useDispatch();
const [uploadedFileIds, setUploadedFileIds] = useState<string[]>([]);
const fileListRef = useRef<UploadFile[]>([]);
const navigate = useNavigate();
const { currentRecord, setRecord } = useSetSelectedRecord();
const {
changeParserLoading,
onChangeParserOk,
changeParserVisible,
hideChangeParserModal,
showChangeParserModal,
} = useChangeDocumentParser(currentRecord.id);
const documentList = useSelectDocumentList();
const runDocumentByIds = useRunDocument();
const uploadDocument = useUploadDocument();
const enabled = useMemo(() => {
if (isUpload) {
@ -175,8 +153,7 @@ const KnowledgeUploadFile = () => {
onError,
// onProgress,
}) {
const ret = await uploadService.uploadFile(file, knowledgeBaseId);
const data = ret?.data;
const data = await uploadDocument(file as UploadFile);
if (data?.retcode === 0) {
setUploadedFileIds((pre) => {
return pre.concat(data.data.id);
@ -197,6 +174,17 @@ const KnowledgeUploadFile = () => {
});
}, []);
const handleItemEdit = useCallback(
(id: string) => {
const document = documentList.find((x) => x.id === id);
if (document) {
setRecord(document);
}
showChangeParserModal();
},
[documentList, showChangeParserModal, setRecord],
);
const props: UploadProps = {
name: 'file',
multiple: true,
@ -215,6 +203,9 @@ const KnowledgeUploadFile = () => {
fileList={fileList}
originNode={originNode}
remove={remove}
showModal={showChangeParserModal}
setRecord={setRecord}
handleEdit={handleItemEdit}
></UploaderItem>
);
},
@ -226,10 +217,7 @@ const KnowledgeUploadFile = () => {
const runSelectedDocument = () => {
const ids = fileListRef.current.map((x) => x.response.id);
dispatch({
type: 'kFModel/document_run',
payload: { doc_ids: ids, run: 1 },
});
runDocumentByIds(ids);
};
const handleNextClick = () => {
@ -245,67 +233,78 @@ const KnowledgeUploadFile = () => {
useFetchKnowledgeDetail();
return (
<div className={styles.uploadWrapper}>
<section>
<Space className={styles.backToList}>
<ArrowLeftOutlined />
<Link to={`/knowledge/dataset?id=${knowledgeBaseId}`}>
Back to select files
</Link>
</Space>
<div className={styles.progressWrapper}>
<Flex align="center" justify="center">
<SelectFilesStartIcon></SelectFilesStartIcon>
<Progress
percent={100}
showInfo={false}
className={styles.progress}
strokeColor="
rgba(127, 86, 217, 1)
"
/>
<SelectFilesEndIcon></SelectFilesEndIcon>
</Flex>
<Flex justify="space-around">
<p className={styles.selectFilesText}>
<b>Select files</b>
<>
<div className={styles.uploadWrapper}>
<section>
<Space className={styles.backToList}>
<ArrowLeftOutlined />
<Link to={`/knowledge/dataset?id=${knowledgeBaseId}`}>
Back to select files
</Link>
</Space>
<div className={styles.progressWrapper}>
<Flex align="center" justify="center">
<SelectFilesStartIcon></SelectFilesStartIcon>
<Progress
percent={100}
showInfo={false}
className={styles.progress}
strokeColor="
rgba(127, 86, 217, 1)
"
/>
<SelectFilesEndIcon></SelectFilesEndIcon>
</Flex>
<Flex justify="space-around">
<p className={styles.selectFilesText}>
<b>Select files</b>
</p>
<p className={styles.changeSpecificCategoryText}>
<b>Change specific category</b>
</p>
</Flex>
</div>
</section>
<section className={styles.uploadContent}>
<Dragger
{...props}
className={classNames(styles.uploader, {
[styles.hiddenUploader]: !isUpload,
})}
>
<Button className={styles.uploaderButton}>
<CloudUploadOutlined className={styles.uploaderIcon} />
</Button>
<p className="ant-upload-text">
Click or drag file to this area to upload
</p>
<p className={styles.changeSpecificCategoryText}>
<b>Change specific category</b>
<p className="ant-upload-hint">
Support for a single or bulk upload. Strictly prohibited from
uploading company data or other banned files.
</p>
</Flex>
</div>
</section>
<section className={styles.uploadContent}>
<Dragger
{...props}
className={classNames(styles.uploader, {
[styles.hiddenUploader]: !isUpload,
})}
>
<Button className={styles.uploaderButton}>
<CloudUploadOutlined className={styles.uploaderIcon} />
</Dragger>
</section>
<section className={styles.footer}>
<Button
type="primary"
// className={styles.nextButton}
onClick={handleNextClick}
disabled={!enabled}
>
Next
</Button>
<p className="ant-upload-text">
Click or drag file to this area to upload
</p>
<p className="ant-upload-hint">
Support for a single or bulk upload. Strictly prohibited from
uploading company data or other banned files.
</p>
</Dragger>
</section>
<section className={styles.footer}>
<Button
type="primary"
// className={styles.nextButton}
onClick={handleNextClick}
disabled={!enabled}
>
Next
</Button>
</section>
</div>
</section>
</div>
<ChunkMethodModal
parserId={currentRecord.parser_id}
parserConfig={currentRecord.parser_config}
documentExtension={getExtension(currentRecord.name)}
onOk={onChangeParserOk}
visible={changeParserVisible}
hideModal={hideChangeParserModal}
loading={changeParserLoading}
/>
</>
);
};

View File

@ -7,10 +7,7 @@ import {
} from '@/hooks/documentHooks';
import { useGetKnowledgeSearchParams } from '@/hooks/routeHook';
import { useOneNamespaceEffectsLoading } from '@/hooks/storeHooks';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import { useFetchTenantInfo } from '@/hooks/userSettingHook';
import { Pagination } from '@/interfaces/common';
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
@ -243,21 +240,3 @@ export const useChangeDocumentParser = (documentId: string) => {
showChangeParserModal,
};
};
export const useFetchParserListOnMount = (parserId: string) => {
const [selectedTag, setSelectedTag] = useState('');
const parserList = useSelectParserList();
useFetchTenantInfo();
useEffect(() => {
setSelectedTag(parserId);
}, [parserId]);
const handleChange = (tag: string, checked: boolean) => {
const nextSelectedTag = checked ? tag : selectedTag;
setSelectedTag(nextSelectedTag);
};
return { parserList, handleChange, selectedTag };
};

View File

@ -23,7 +23,6 @@ import {
} from 'antd';
import type { ColumnsType } from 'antd/es/table';
import { useMemo } from 'react';
import ChunkMethodModal from './chunk-method-modal';
import CreateFileModal from './create-file-modal';
import {
useChangeDocumentParser,
@ -39,6 +38,8 @@ import ParsingActionCell from './parsing-action-cell';
import ParsingStatusCell from './parsing-status-cell';
import RenameModal from './rename-modal';
import ChunkMethodModal from '@/components/chunk-method-modal';
import { getExtension } from '@/utils/documentUtils';
import styles from './index.less';
const KnowledgeFile = () => {
@ -227,7 +228,7 @@ const KnowledgeFile = () => {
<ChunkMethodModal
parserId={currentRecord.parser_id}
parserConfig={currentRecord.parser_config}
documentType={currentRecord.type}
documentExtension={getExtension(currentRecord.name)}
onOk={onChangeParserOk}
visible={changeParserVisible}
hideModal={hideChangeParserModal}

View File

@ -209,6 +209,19 @@ const model: DvaModel<KFModelState> = {
console.warn(error);
}
},
*upload_document({ payload = {} }, { call, put }) {
const formData = new FormData();
formData.append('file', payload.file);
formData.append('kb_id', payload.kb_id);
const { data } = yield call(kbService.document_upload, formData);
if (data.retcode === 0) {
yield put({
type: 'getKfList',
payload: { kb_id: payload.kb_id },
});
}
return data;
},
},
};
export default model;

View File

@ -1,42 +0,0 @@
import { useKnowledgeBaseId } from '@/hooks/knowledgeHook';
import uploadService from '@/services/uploadService';
import type { UploadProps } from 'antd';
import React from 'react';
import { Link } from 'umi';
interface PropsType {
kb_id: string;
getKfList: () => void;
}
type UploadRequestOption = Parameters<
NonNullable<UploadProps['customRequest']>
>[0];
const FileUpload: React.FC<PropsType> = ({ kb_id, getKfList }) => {
const knowledgeBaseId = useKnowledgeBaseId();
const createRequest: (props: UploadRequestOption) => void = async function ({
file,
onSuccess,
onError,
}) {
const { retcode, data } = await uploadService.uploadFile(file, kb_id);
if (retcode === 0) {
onSuccess && onSuccess(data, file);
} else {
onError && onError(data);
}
getKfList && getKfList();
};
const uploadProps: UploadProps = {
customRequest: createRequest,
showUploadList: false,
};
return (
// <Upload {...uploadProps}>
<Link to={`/knowledge/dataset/upload?id=${knowledgeBaseId}`}></Link>
// </Upload>
);
};
export default FileUpload;

View File

@ -25,6 +25,7 @@ const {
document_rename,
document_run,
get_document_file,
document_upload,
} = api;
const methods = {
@ -82,6 +83,10 @@ const methods = {
url: document_thumbnails,
method: 'get',
},
document_upload: {
url: document_upload,
method: 'post',
},
// chunk管理
chunk_list: {
url: chunk_list,

View File

@ -1,21 +0,0 @@
import api from '@/utils/api';
import request from '@/utils/request';
const { upload } = api;
const uploadService = {
uploadFile: function (file: any, kb_id: string) {
const formData = new FormData();
formData.append('file', file);
formData.append('kb_id', kb_id);
const options = {
method: 'post',
data: formData,
};
return request(upload, options);
},
};
export default uploadService;

View File

@ -45,6 +45,7 @@ export default {
document_change_parser: `${api_host}/document/change_parser`,
document_thumbnails: `${api_host}/document/thumbnails`,
get_document_file: `${api_host}/document/get`,
document_upload: `${api_host}/document/upload`,
setDialog: `${api_host}/dialog/set`,
getDialog: `${api_host}/dialog/get`,

View File

@ -35,3 +35,6 @@ export const buildChunkHighlights = (
};
export const isFileUploadDone = (file: UploadFile) => file.status === 'done';
export const getExtension = (name: string) =>
name?.slice(name.lastIndexOf('.') + 1).toLowerCase() ?? '';