mirror of
https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git
synced 2025-08-18 16:05:56 +08:00
For any type of file, if the parsing method is general, the chunk token number needs to be displayed. #217 (#218)
…k token number needs to be displayed. #217 ### What problem does this PR solve? For any type of file, if the parsing method is general, the chunk token number needs to be displayed. #217 Issue link: #217 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
balibabu
GitHub
parent
963533bc27
commit
33d01150d5
@ -68,13 +68,26 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|||||||
onOk(selectedTag, parser_config);
|
onOk(selectedTag, parser_config);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const isPdf = documentExtension === 'pdf';
|
||||||
|
|
||||||
const showPages = useMemo(() => {
|
const showPages = useMemo(() => {
|
||||||
return hidePagesChunkMethods.every((x) => x !== selectedTag);
|
return isPdf && hidePagesChunkMethods.every((x) => x !== selectedTag);
|
||||||
}, [selectedTag]);
|
}, [selectedTag, isPdf]);
|
||||||
|
|
||||||
const showOne = useMemo(() => {
|
const showOne = useMemo(() => {
|
||||||
return showPages || selectedTag === 'one';
|
return (
|
||||||
}, [showPages, selectedTag]);
|
isPdf &&
|
||||||
|
hidePagesChunkMethods
|
||||||
|
.filter((x) => x !== 'one')
|
||||||
|
.every((x) => x !== selectedTag)
|
||||||
|
);
|
||||||
|
}, [selectedTag, isPdf]);
|
||||||
|
|
||||||
|
const showMaxTokenNumber = selectedTag === 'naive';
|
||||||
|
|
||||||
|
const hideDivider = [showPages, showOne, showMaxTokenNumber].every(
|
||||||
|
(x) => x === false,
|
||||||
|
);
|
||||||
|
|
||||||
const afterClose = () => {
|
const afterClose = () => {
|
||||||
form.resetFields();
|
form.resetFields();
|
||||||
@ -116,166 +129,161 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|||||||
})}
|
})}
|
||||||
</div>
|
</div>
|
||||||
</Space>
|
</Space>
|
||||||
<Divider></Divider>
|
{hideDivider || <Divider></Divider>}
|
||||||
|
<Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
|
||||||
{documentExtension === 'pdf' && (
|
{showPages && (
|
||||||
<Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
|
<>
|
||||||
{showPages && (
|
<Space>
|
||||||
<>
|
<p>Page Ranges:</p>
|
||||||
<Space>
|
<Tooltip
|
||||||
<p>Page Ranges:</p>
|
title={
|
||||||
<Tooltip
|
'page ranges: Define the page ranges that need to be parsed. The pages that not included in these ranges will be ignored.'
|
||||||
title={
|
}
|
||||||
'page ranges: Define the page ranges that need to be parsed. The pages that not included in these ranges will be ignored.'
|
>
|
||||||
}
|
<QuestionCircleOutlined
|
||||||
>
|
className={styles.questionIcon}
|
||||||
<QuestionCircleOutlined
|
></QuestionCircleOutlined>
|
||||||
className={styles.questionIcon}
|
</Tooltip>
|
||||||
></QuestionCircleOutlined>
|
</Space>
|
||||||
</Tooltip>
|
<Form.List name="pages">
|
||||||
</Space>
|
{(fields, { add, remove }) => (
|
||||||
<Form.List name="pages">
|
<>
|
||||||
{(fields, { add, remove }) => (
|
{fields.map(({ key, name, ...restField }) => (
|
||||||
<>
|
<Space
|
||||||
{fields.map(({ key, name, ...restField }) => (
|
key={key}
|
||||||
<Space
|
style={{
|
||||||
key={key}
|
display: 'flex',
|
||||||
style={{
|
}}
|
||||||
display: 'flex',
|
align="baseline"
|
||||||
}}
|
>
|
||||||
align="baseline"
|
<Form.Item
|
||||||
>
|
{...restField}
|
||||||
<Form.Item
|
name={[name, 'from']}
|
||||||
{...restField}
|
dependencies={name > 0 ? [name - 1, 'to'] : []}
|
||||||
name={[name, 'from']}
|
rules={[
|
||||||
dependencies={name > 0 ? [name - 1, 'to'] : []}
|
{
|
||||||
rules={[
|
required: true,
|
||||||
{
|
message: 'Missing start page number',
|
||||||
required: true,
|
},
|
||||||
message: 'Missing start page number',
|
({ getFieldValue }) => ({
|
||||||
|
validator(_, value) {
|
||||||
|
if (
|
||||||
|
name === 0 ||
|
||||||
|
!value ||
|
||||||
|
getFieldValue(['pages', name - 1, 'to']) < value
|
||||||
|
) {
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
return Promise.reject(
|
||||||
|
new Error(
|
||||||
|
'The current value must be greater than the previous to!',
|
||||||
|
),
|
||||||
|
);
|
||||||
},
|
},
|
||||||
({ getFieldValue }) => ({
|
}),
|
||||||
validator(_, value) {
|
]}
|
||||||
if (
|
|
||||||
name === 0 ||
|
|
||||||
!value ||
|
|
||||||
getFieldValue(['pages', name - 1, 'to']) <
|
|
||||||
value
|
|
||||||
) {
|
|
||||||
return Promise.resolve();
|
|
||||||
}
|
|
||||||
return Promise.reject(
|
|
||||||
new Error(
|
|
||||||
'The current value must be greater than the previous to!',
|
|
||||||
),
|
|
||||||
);
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
]}
|
|
||||||
>
|
|
||||||
<InputNumber
|
|
||||||
placeholder="from"
|
|
||||||
min={0}
|
|
||||||
precision={0}
|
|
||||||
className={styles.pageInputNumber}
|
|
||||||
/>
|
|
||||||
</Form.Item>
|
|
||||||
<Form.Item
|
|
||||||
{...restField}
|
|
||||||
name={[name, 'to']}
|
|
||||||
dependencies={[name, 'from']}
|
|
||||||
rules={[
|
|
||||||
{
|
|
||||||
required: true,
|
|
||||||
message: 'Missing end page number(excluded)',
|
|
||||||
},
|
|
||||||
({ getFieldValue }) => ({
|
|
||||||
validator(_, value) {
|
|
||||||
if (
|
|
||||||
!value ||
|
|
||||||
getFieldValue(['pages', name, 'from']) < value
|
|
||||||
) {
|
|
||||||
return Promise.resolve();
|
|
||||||
}
|
|
||||||
return Promise.reject(
|
|
||||||
new Error(
|
|
||||||
'The current value must be greater than to!',
|
|
||||||
),
|
|
||||||
);
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
]}
|
|
||||||
>
|
|
||||||
<InputNumber
|
|
||||||
placeholder="to"
|
|
||||||
min={0}
|
|
||||||
precision={0}
|
|
||||||
className={styles.pageInputNumber}
|
|
||||||
/>
|
|
||||||
</Form.Item>
|
|
||||||
{name > 0 && (
|
|
||||||
<MinusCircleOutlined onClick={() => remove(name)} />
|
|
||||||
)}
|
|
||||||
</Space>
|
|
||||||
))}
|
|
||||||
<Form.Item>
|
|
||||||
<Button
|
|
||||||
type="dashed"
|
|
||||||
onClick={() => add()}
|
|
||||||
block
|
|
||||||
icon={<PlusOutlined />}
|
|
||||||
>
|
>
|
||||||
Add page
|
<InputNumber
|
||||||
</Button>
|
placeholder="from"
|
||||||
</Form.Item>
|
min={0}
|
||||||
</>
|
precision={0}
|
||||||
)}
|
className={styles.pageInputNumber}
|
||||||
</Form.List>
|
/>
|
||||||
</>
|
</Form.Item>
|
||||||
)}
|
<Form.Item
|
||||||
{showOne && (
|
{...restField}
|
||||||
<Form.Item
|
name={[name, 'to']}
|
||||||
name={['parser_config', 'layout_recognize']}
|
dependencies={[name, 'from']}
|
||||||
label="Layout recognize"
|
rules={[
|
||||||
initialValue={true}
|
{
|
||||||
valuePropName="checked"
|
required: true,
|
||||||
tooltip={
|
message: 'Missing end page number(excluded)',
|
||||||
'Use visual models for layout analysis to better identify document structure, find where the titles, text blocks, images, and tables are. Without this feature, only the plain text of the PDF can be obtained.'
|
},
|
||||||
}
|
({ getFieldValue }) => ({
|
||||||
>
|
validator(_, value) {
|
||||||
<Switch />
|
if (
|
||||||
</Form.Item>
|
!value ||
|
||||||
)}
|
getFieldValue(['pages', name, 'from']) < value
|
||||||
{showPages && (
|
) {
|
||||||
<Form.Item
|
return Promise.resolve();
|
||||||
noStyle
|
}
|
||||||
dependencies={[['parser_config', 'layout_recognize']]}
|
return Promise.reject(
|
||||||
>
|
new Error(
|
||||||
{({ getFieldValue }) =>
|
'The current value must be greater than to!',
|
||||||
getFieldValue(['parser_config', 'layout_recognize']) && (
|
),
|
||||||
<Form.Item
|
);
|
||||||
name={['parser_config', 'task_page_size']}
|
},
|
||||||
label="Task page size"
|
}),
|
||||||
tooltip={`If using layout recognize, the PDF file will be split into groups of successive. Layout analysis will be performed parallelly between groups to increase the processing speed.
|
]}
|
||||||
The 'Task page size' determines the size of groups. The larger the page size is, the lower the chance of splitting continuous text between pages into different chunks.`}
|
>
|
||||||
initialValue={12}
|
<InputNumber
|
||||||
rules={[
|
placeholder="to"
|
||||||
{
|
min={0}
|
||||||
required: true,
|
precision={0}
|
||||||
message: 'Please input your task page size!',
|
className={styles.pageInputNumber}
|
||||||
},
|
/>
|
||||||
]}
|
</Form.Item>
|
||||||
>
|
{name > 0 && (
|
||||||
<InputNumber min={1} max={128} />
|
<MinusCircleOutlined onClick={() => remove(name)} />
|
||||||
|
)}
|
||||||
|
</Space>
|
||||||
|
))}
|
||||||
|
<Form.Item>
|
||||||
|
<Button
|
||||||
|
type="dashed"
|
||||||
|
onClick={() => add()}
|
||||||
|
block
|
||||||
|
icon={<PlusOutlined />}
|
||||||
|
>
|
||||||
|
Add page
|
||||||
|
</Button>
|
||||||
</Form.Item>
|
</Form.Item>
|
||||||
)
|
</>
|
||||||
}
|
)}
|
||||||
</Form.Item>
|
</Form.List>
|
||||||
)}
|
</>
|
||||||
|
)}
|
||||||
{selectedTag === 'naive' && <MaxTokenNumber></MaxTokenNumber>}
|
{showOne && (
|
||||||
</Form>
|
<Form.Item
|
||||||
)}
|
name={['parser_config', 'layout_recognize']}
|
||||||
|
label="Layout recognize"
|
||||||
|
initialValue={true}
|
||||||
|
valuePropName="checked"
|
||||||
|
tooltip={
|
||||||
|
'Use visual models for layout analysis to better identify document structure, find where the titles, text blocks, images, and tables are. Without this feature, only the plain text of the PDF can be obtained.'
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<Switch />
|
||||||
|
</Form.Item>
|
||||||
|
)}
|
||||||
|
{showPages && (
|
||||||
|
<Form.Item
|
||||||
|
noStyle
|
||||||
|
dependencies={[['parser_config', 'layout_recognize']]}
|
||||||
|
>
|
||||||
|
{({ getFieldValue }) =>
|
||||||
|
getFieldValue(['parser_config', 'layout_recognize']) && (
|
||||||
|
<Form.Item
|
||||||
|
name={['parser_config', 'task_page_size']}
|
||||||
|
label="Task page size"
|
||||||
|
tooltip={`If using layout recognize, the PDF file will be split into groups of successive. Layout analysis will be performed parallelly between groups to increase the processing speed.
|
||||||
|
The 'Task page size' determines the size of groups. The larger the page size is, the lower the chance of splitting continuous text between pages into different chunks.`}
|
||||||
|
initialValue={12}
|
||||||
|
rules={[
|
||||||
|
{
|
||||||
|
required: true,
|
||||||
|
message: 'Please input your task page size!',
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
>
|
||||||
|
<InputNumber min={1} max={128} />
|
||||||
|
</Form.Item>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
</Form.Item>
|
||||||
|
)}
|
||||||
|
{showMaxTokenNumber && <MaxTokenNumber></MaxTokenNumber>}
|
||||||
|
</Form>
|
||||||
</Modal>
|
</Modal>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user