feat: After selecting the parsing method as knowledge graph, the delimiter and chunk token number are displayed. #1594 (#1929)

### What problem does this PR solve?

feat: After selecting the parsing method as knowledge graph, the
delimiter and chunk token number are displayed. #1594

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
balibabu 2024-08-13 15:21:03 +08:00 committed by GitHub
parent 77f0fb03e3
commit 7a08e91909
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 67 additions and 11 deletions

View File

@ -1,4 +1,6 @@
import { useHandleChunkMethodSelectChange } from '@/hooks/logic-hooks';
import { useSelectParserList } from '@/hooks/user-setting-hooks'; import { useSelectParserList } from '@/hooks/user-setting-hooks';
import { FormInstance } from 'antd';
import { useEffect, useMemo, useState } from 'react'; import { useEffect, useMemo, useState } from 'react';
const ParserListMap = new Map([ const ParserListMap = new Map([
@ -84,9 +86,11 @@ export const useFetchParserListOnMount = (
documentId: string, documentId: string,
parserId: string, parserId: string,
documentExtension: string, documentExtension: string,
form: FormInstance,
) => { ) => {
const [selectedTag, setSelectedTag] = useState(''); const [selectedTag, setSelectedTag] = useState('');
const parserList = useSelectParserList(); const parserList = useSelectParserList();
const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form);
const nextParserList = useMemo(() => { const nextParserList = useMemo(() => {
const key = [...ParserListMap.keys()].find((x) => const key = [...ParserListMap.keys()].find((x) =>
@ -108,7 +112,7 @@ export const useFetchParserListOnMount = (
}, [parserId, documentId]); }, [parserId, documentId]);
const handleChange = (tag: string) => { const handleChange = (tag: string) => {
// const nextSelectedTag = checked ? tag : selectedTag; handleChunkMethodSelectChange(tag);
setSelectedTag(tag); setSelectedTag(tag);
}; };

View File

@ -62,12 +62,13 @@ const ChunkMethodModal: React.FC<IProps> = ({
parserConfig, parserConfig,
loading, loading,
}) => { }) => {
const [form] = Form.useForm();
const { parserList, handleChange, selectedTag } = useFetchParserListOnMount( const { parserList, handleChange, selectedTag } = useFetchParserListOnMount(
documentId, documentId,
parserId, parserId,
documentExtension, documentExtension,
form,
); );
const [form] = Form.useForm();
const { t } = useTranslate('knowledgeDetails'); const { t } = useTranslate('knowledgeDetails');
const handleOk = async () => { const handleOk = async () => {
@ -89,12 +90,13 @@ const ChunkMethodModal: React.FC<IProps> = ({
return ( return (
isPdf && isPdf &&
hidePagesChunkMethods hidePagesChunkMethods
.filter((x) => x !== 'one' && x !== 'knowledge_graph') .filter((x) => x !== 'one')
.every((x) => x !== selectedTag) .every((x) => x !== selectedTag)
); );
}, [selectedTag, isPdf]); }, [selectedTag, isPdf]);
const showMaxTokenNumber = selectedTag === 'naive'; const showMaxTokenNumber =
selectedTag === 'naive' || selectedTag === 'knowledge_graph';
const hideDivider = [showPages, showOne, showMaxTokenNumber].every( const hideDivider = [showPages, showOne, showMaxTokenNumber].every(
(x) => x === false, (x) => x === false,
@ -271,7 +273,9 @@ const ChunkMethodModal: React.FC<IProps> = ({
)} )}
{showMaxTokenNumber && ( {showMaxTokenNumber && (
<> <>
<MaxTokenNumber></MaxTokenNumber> <MaxTokenNumber
max={selectedTag === 'knowledge_graph' ? 8192 * 2 : 2048}
></MaxTokenNumber>
<Delimiter></Delimiter> <Delimiter></Delimiter>
</> </>
)} )}

View File

@ -1,3 +1,8 @@
.tweenGroup { .tweenGroup {
display: inline-block; display: inline-block;
:global(.ant-tag) {
margin-bottom: 8px;
font-size: 14px;
padding: 2px 8px;
}
} }

View File

@ -1,7 +1,12 @@
import { useTranslate } from '@/hooks/common-hooks'; import { useTranslate } from '@/hooks/common-hooks';
import { Flex, Form, InputNumber, Slider } from 'antd'; import { Flex, Form, InputNumber, Slider } from 'antd';
const MaxTokenNumber = () => { interface IProps {
initialValue?: number;
max?: number;
}
const MaxTokenNumber = ({ initialValue = 128, max = 2048 }: IProps) => {
const { t } = useTranslate('knowledgeConfiguration'); const { t } = useTranslate('knowledgeConfiguration');
return ( return (
@ -11,18 +16,19 @@ const MaxTokenNumber = () => {
<Form.Item <Form.Item
name={['parser_config', 'chunk_token_num']} name={['parser_config', 'chunk_token_num']}
noStyle noStyle
initialValue={128} initialValue={initialValue}
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]} rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
> >
<Slider max={2048} style={{ width: '100%' }} /> <Slider max={max} style={{ width: '100%' }} />
</Form.Item> </Form.Item>
</Flex> </Flex>
<Form.Item <Form.Item
name={['parser_config', 'chunk_token_num']} name={['parser_config', 'chunk_token_num']}
noStyle noStyle
initialValue={initialValue}
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]} rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
> >
<InputNumber max={2048} min={0} /> <InputNumber max={max} min={0} />
</Form.Item> </Form.Item>
</Flex> </Flex>
</Form.Item> </Form.Item>

View File

@ -8,6 +8,7 @@ import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
import api from '@/utils/api'; import api from '@/utils/api';
import { getAuthorization } from '@/utils/authorization-util'; import { getAuthorization } from '@/utils/authorization-util';
import { PaginationProps } from 'antd'; import { PaginationProps } from 'antd';
import { FormInstance } from 'antd/lib';
import axios from 'axios'; import axios from 'axios';
import { EventSourceParserStream } from 'eventsource-parser/stream'; import { EventSourceParserStream } from 'eventsource-parser/stream';
import { import {
@ -337,3 +338,25 @@ export const useFetchModelId = () => {
return tenantInfo?.llm_id ?? ''; return tenantInfo?.llm_id ?? '';
}; };
const ChunkTokenNumMap = {
naive: 128,
knowledge_graph: 8192,
};
export const useHandleChunkMethodSelectChange = (form: FormInstance) => {
// const form = Form.useFormInstance();
const handleChange = useCallback(
(value: string) => {
if (value in ChunkTokenNumMap) {
form.setFieldValue(
['parser_config', 'chunk_token_num'],
ChunkTokenNumMap[value as keyof typeof ChunkTokenNumMap],
);
}
},
[form],
);
return handleChange;
};

View File

@ -6,6 +6,7 @@ import ParseConfiguration, {
showRaptorParseConfiguration, showRaptorParseConfiguration,
} from '@/components/parse-configuration'; } from '@/components/parse-configuration';
import { useTranslate } from '@/hooks/common-hooks'; import { useTranslate } from '@/hooks/common-hooks';
import { useHandleChunkMethodSelectChange } from '@/hooks/logic-hooks';
import { normFile } from '@/utils/file-util'; import { normFile } from '@/utils/file-util';
import { PlusOutlined } from '@ant-design/icons'; import { PlusOutlined } from '@ant-design/icons';
import { Button, Form, Input, Radio, Select, Space, Upload } from 'antd'; import { Button, Form, Input, Radio, Select, Space, Upload } from 'antd';
@ -24,6 +25,7 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
const { parserList, embeddingModelOptions, disabled } = const { parserList, embeddingModelOptions, disabled } =
useFetchKnowledgeConfigurationOnMount(form); useFetchKnowledgeConfigurationOnMount(form);
const { t } = useTranslate('knowledgeConfiguration'); const { t } = useTranslate('knowledgeConfiguration');
const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form);
return ( return (
<Form form={form} name="validateOnly" layout="vertical" autoComplete="off"> <Form form={form} name="validateOnly" layout="vertical" autoComplete="off">
@ -91,7 +93,11 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
tooltip={t('chunkMethodTip')} tooltip={t('chunkMethodTip')}
rules={[{ required: true }]} rules={[{ required: true }]}
> >
<Select placeholder={t('chunkMethodPlaceholder')} disabled={disabled}> <Select
placeholder={t('chunkMethodPlaceholder')}
disabled={disabled}
onChange={handleChunkMethodSelectChange}
>
{parserList.map((x) => ( {parserList.map((x) => (
<Option value={x.value} key={x.value}> <Option value={x.value} key={x.value}>
{x.label} {x.label}
@ -107,7 +113,11 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
return ( return (
<> <>
{parserId === 'knowledge_graph' && ( {parserId === 'knowledge_graph' && (
<EntityTypesItem></EntityTypesItem> <>
<EntityTypesItem></EntityTypesItem>
<MaxTokenNumber max={8192 * 2}></MaxTokenNumber>
<Delimiter></Delimiter>
</>
)} )}
{parserId === 'naive' && ( {parserId === 'naive' && (
<> <>

View File

@ -80,5 +80,9 @@ export const useHandleChunkMethodChange = () => {
const [form] = Form.useForm(); const [form] = Form.useForm();
const chunkMethod = Form.useWatch('parser_id', form); const chunkMethod = Form.useWatch('parser_id', form);
useEffect(() => {
console.log('🚀 ~ useHandleChunkMethodChange ~ chunkMethod:', chunkMethod);
}, [chunkMethod]);
return { form, chunkMethod }; return { form, chunkMethod };
}; };