feat: enhance document processing with embedding and rule detail components

This commit is contained in:
twwu 2025-06-03 17:42:40 +08:00
parent 5fb771218c
commit 87c15062e6
6 changed files with 422 additions and 12 deletions

View File

@ -23,7 +23,7 @@ import WebsitePreview from './preview/web-preview'
import ProcessDocuments from './process-documents'
import ChunkPreview from './preview/chunk-preview'
import Processing from './processing'
import type { PublishedPipelineRunPreviewResponse } from '@/models/pipeline'
import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
import { DatasourceType } from '@/models/pipeline'
import { TransferMethod } from '@/types/app'
import { useAddDocumentsSteps, useLocalFile, useNotionsPages, useWebsiteCrawl } from './hooks'
@ -38,6 +38,8 @@ const CreateFormPipeline = () => {
const retrievalMethod = useDatasetDetailContextWithSelector(s => s.dataset?.retrieval_model_dict.search_method)
const [datasource, setDatasource] = useState<Datasource>()
const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
const [batchId, setBatchId] = useState('')
const [documents, setDocuments] = useState<InitialDocumentDetail[]>([])
const isPreview = useRef(false)
const formRef = useRef<any>(null)
@ -189,7 +191,8 @@ const CreateFormPipeline = () => {
is_preview: false,
}, {
onSuccess: (res) => {
console.log('🚀 ~ handleProcess ~ res:', res)
setBatchId((res as PublishedPipelineRunResponse).batch || '')
setDocuments((res as PublishedPipelineRunResponse).documents || [])
handleNextStep()
},
})
@ -306,8 +309,8 @@ const CreateFormPipeline = () => {
currentStep === 3 && (
<Processing
datasetId={datasetId!}
batchId={''}
documents={[]}
batchId={batchId}
documents={documents}
indexingType={indexingType!}
retrievalMethod={retrievalMethod!}
/>

View File

@ -0,0 +1,247 @@
import React, { useEffect, useMemo, useRef, useState } from 'react'
import useSWR from 'swr'
import { useRouter } from 'next/navigation'
import { useTranslation } from 'react-i18next'
import { omit } from 'lodash-es'
import { ArrowRightIcon } from '@heroicons/react/24/solid'
import {
RiCheckboxCircleFill,
RiErrorWarningFill,
RiLoader2Fill,
RiTerminalBoxLine,
} from '@remixicon/react'
import cn from '@/utils/classnames'
import Button from '@/app/components/base/button'
import type { IndexingStatusResponse } from '@/models/datasets'
import { fetchIndexingStatusBatch as doFetchIndexingStatus, fetchProcessRule } from '@/service/datasets'
import NotionIcon from '@/app/components/base/notion-icon'
import PriorityLabel from '@/app/components/billing/priority-label'
import { Plan } from '@/app/components/billing/type'
import { ZapFast } from '@/app/components/base/icons/src/vender/solid/general'
import UpgradeBtn from '@/app/components/billing/upgrade-btn'
import { useProviderContext } from '@/context/provider-context'
import { sleep } from '@/utils'
import Tooltip from '@/app/components/base/tooltip'
import { useInvalidDocumentList } from '@/service/knowledge/use-document'
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
import RuleDetail from './rule-detail'
import type { IndexingType } from '@/app/components/datasets/create/step-two'
import type { RETRIEVE_METHOD } from '@/types/app'
import { DatasourceType, type InitialDocumentDetail } from '@/models/pipeline'
type EmbeddingProcessProps = {
datasetId: string
batchId: string
documents?: InitialDocumentDetail[]
indexingType?: IndexingType
retrievalMethod?: RETRIEVE_METHOD
}
const EmbeddingProcess = ({
datasetId,
batchId,
documents = [],
indexingType,
retrievalMethod,
}: EmbeddingProcessProps) => {
const { t } = useTranslation()
const { enableBilling, plan } = useProviderContext()
const firstDocument = documents[0]
const [indexingStatusBatchDetail, setIndexingStatusDetail] = useState<IndexingStatusResponse[]>([])
const fetchIndexingStatus = async () => {
const status = await doFetchIndexingStatus({ datasetId, batchId })
setIndexingStatusDetail(status.data)
return status.data
}
const [isStopQuery, setIsStopQuery] = useState(false)
const isStopQueryRef = useRef(isStopQuery)
useEffect(() => {
isStopQueryRef.current = isStopQuery
}, [isStopQuery])
const stopQueryStatus = () => {
setIsStopQuery(true)
}
const startQueryStatus = async () => {
if (isStopQueryRef.current)
return
try {
const indexingStatusBatchDetail = await fetchIndexingStatus()
const isCompleted = indexingStatusBatchDetail.every(indexingStatusDetail => ['completed', 'error', 'paused'].includes(indexingStatusDetail.indexing_status))
if (isCompleted) {
stopQueryStatus()
return
}
await sleep(2500)
await startQueryStatus()
}
catch {
await sleep(2500)
await startQueryStatus()
}
}
useEffect(() => {
setIsStopQuery(false)
startQueryStatus()
return () => {
stopQueryStatus()
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
// get rule
const { data: ruleDetail } = useSWR({
action: 'fetchProcessRule',
params: { documentId: firstDocument.id },
}, apiParams => fetchProcessRule(omit(apiParams, 'action')), {
revalidateOnFocus: false,
})
const router = useRouter()
const invalidDocumentList = useInvalidDocumentList()
const navToDocumentList = () => {
invalidDocumentList()
router.push(`/datasets/${datasetId}/documents`)
}
const navToApiDocs = () => {
router.push('/datasets?category=api')
}
const isEmbedding = useMemo(() => {
return indexingStatusBatchDetail.some(indexingStatusDetail => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || ''))
}, [indexingStatusBatchDetail])
const isEmbeddingCompleted = useMemo(() => {
return indexingStatusBatchDetail.every(indexingStatusDetail => ['completed', 'error', 'paused'].includes(indexingStatusDetail?.indexing_status || ''))
}, [indexingStatusBatchDetail])
const getSourceName = (id: string) => {
const doc = documents.find(document => document.id === id)
return doc?.name
}
const getFileType = (name?: string) => name?.split('.').pop() || 'txt'
const getSourcePercent = (detail: IndexingStatusResponse) => {
const completedCount = detail.completed_segments || 0
const totalCount = detail.total_segments || 0
if (totalCount === 0)
return 0
const percent = Math.round(completedCount * 100 / totalCount)
return percent > 100 ? 100 : percent
}
const getSourceType = (id: string) => {
const doc = documents.find(document => document.id === id)
return doc?.data_source_type
}
const getIcon = (id: string) => {
const doc = documents.find(document => document.id === id)
return doc?.data_source_info.notion_page_icon
}
const isSourceEmbedding = (detail: IndexingStatusResponse) =>
['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'].includes(detail.indexing_status || '')
return (
<>
<div className='mb-3 flex h-5 items-center'>
<div className='mr-2 flex items-center justify-between text-sm font-medium text-text-secondary'>
{isEmbedding && <div className='flex items-center'>
<RiLoader2Fill className='mr-1 size-4 animate-spin text-text-secondary' />
{t('datasetDocuments.embedding.processing')}
</div>}
{isEmbeddingCompleted && t('datasetDocuments.embedding.completed')}
</div>
</div>
{
enableBilling && plan.type !== Plan.team && (
<div className='mb-3 flex h-14 items-center rounded-xl border-[0.5px] border-black/5 bg-white p-3 shadow-md'>
<div className='flex h-8 w-8 shrink-0 items-center justify-center rounded-lg bg-[#FFF6ED]'>
<ZapFast className='h-4 w-4 text-[#FB6514]' />
</div>
<div className='mx-3 grow text-[13px] font-medium text-gray-700'>
{t('billing.plansCommon.documentProcessingPriorityUpgrade')}
</div>
<UpgradeBtn loc='knowledge-speed-up' />
</div>
)
}
<div className='flex flex-col gap-0.5 pb-2'>
{indexingStatusBatchDetail.map(indexingStatusDetail => (
<div key={indexingStatusDetail.id} className={cn(
'relative h-[26px] overflow-hidden rounded-md bg-components-progress-bar-bg',
indexingStatusDetail.indexing_status === 'error' && 'bg-state-destructive-hover-alt',
)}>
{isSourceEmbedding(indexingStatusDetail) && (
<div className='absolute left-0 top-0 h-full min-w-0.5 border-r-[2px] border-r-components-progress-bar-progress-highlight bg-components-progress-bar-progress' style={{ width: `${getSourcePercent(indexingStatusDetail)}%` }} />
)}
<div className='z-[1] flex h-full items-center gap-1 pl-[6px] pr-2'>
{getSourceType(indexingStatusDetail.id) === DatasourceType.localFile && (
<DocumentFileIcon
className='size-4 shrink-0'
name={getSourceName(indexingStatusDetail.id)}
extension={getFileType(getSourceName(indexingStatusDetail.id))}
/>
)}
{getSourceType(indexingStatusDetail.id) === DatasourceType.onlineDocument && (
<NotionIcon
className='shrink-0'
type='page'
src={getIcon(indexingStatusDetail.id)}
/>
)}
<div className='flex w-0 grow items-center gap-1' title={getSourceName(indexingStatusDetail.id)}>
<div className='system-xs-medium truncate text-text-secondary'>
{getSourceName(indexingStatusDetail.id)}
</div>
{
enableBilling && (
<PriorityLabel className='ml-0' />
)
}
</div>
{isSourceEmbedding(indexingStatusDetail) && (
<div className='shrink-0 text-xs text-text-secondary'>{`${getSourcePercent(indexingStatusDetail)}%`}</div>
)}
{indexingStatusDetail.indexing_status === 'error' && (
<Tooltip
popupClassName='px-4 py-[14px] max-w-60 text-sm leading-4 text-text-secondary border-[0.5px] border-components-panel-border rounded-xl'
offset={4}
popupContent={indexingStatusDetail.error}
>
<span>
<RiErrorWarningFill className='size-4 shrink-0 text-text-destructive' />
</span>
</Tooltip>
)}
{indexingStatusDetail.indexing_status === 'completed' && (
<RiCheckboxCircleFill className='size-4 shrink-0 text-text-success' />
)}
</div>
</div>
))}
</div>
<hr className='my-3 h-[1px] border-0 bg-divider-subtle' />
<RuleDetail
sourceData={ruleDetail}
indexingType={indexingType}
retrievalMethod={retrievalMethod}
/>
<div className='my-10 flex items-center gap-2'>
<Button className='w-fit' onClick={navToApiDocs}>
<RiTerminalBoxLine className='mr-2 size-4' />
<span>Access the API</span>
</Button>
<Button className='w-fit' variant='primary' onClick={navToDocumentList}>
<span>{t('datasetCreation.stepThree.navTo')}</span>
<ArrowRightIcon className='ml-2 size-4 stroke-current stroke-1' />
</Button>
</div>
</>
)
}
export default EmbeddingProcess

View File

@ -0,0 +1,128 @@
import React, { useCallback } from 'react'
import { IndexingType } from '@/app/components/datasets/create/step-two'
import { ProcessMode, type ProcessRuleResponse } from '@/models/datasets'
import { RETRIEVE_METHOD } from '@/types/app'
import { useTranslation } from 'react-i18next'
import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata'
import Image from 'next/image'
import { indexMethodIcon, retrievalIcon } from '@/app/components/datasets/create/icons'
type RuleDetailProps = {
sourceData?: ProcessRuleResponse
indexingType?: IndexingType
retrievalMethod?: RETRIEVE_METHOD
}
const RuleDetail = ({
sourceData,
indexingType,
retrievalMethod,
}: RuleDetailProps) => {
const { t } = useTranslation()
const segmentationRuleMap = {
mode: t('datasetDocuments.embedding.mode'),
segmentLength: t('datasetDocuments.embedding.segmentLength'),
textCleaning: t('datasetDocuments.embedding.textCleaning'),
}
const getRuleName = useCallback((key: string) => {
if (key === 'remove_extra_spaces')
return t('datasetCreation.stepTwo.removeExtraSpaces')
if (key === 'remove_urls_emails')
return t('datasetCreation.stepTwo.removeUrlEmails')
if (key === 'remove_stopwords')
return t('datasetCreation.stepTwo.removeStopwords')
}, [t])
const isNumber = useCallback((value: unknown) => {
return typeof value === 'number'
}, [])
const getValue = useCallback((field: string) => {
let value: string | number | undefined = '-'
const maxTokens = isNumber(sourceData?.rules?.segmentation?.max_tokens)
? sourceData.rules.segmentation.max_tokens
: value
const childMaxTokens = isNumber(sourceData?.rules?.subchunk_segmentation?.max_tokens)
? sourceData.rules.subchunk_segmentation.max_tokens
: value
switch (field) {
case 'mode':
value = !sourceData?.mode
? value
// eslint-disable-next-line sonarjs/no-nested-conditional
: sourceData.mode === ProcessMode.general
? (t('datasetDocuments.embedding.custom') as string)
// eslint-disable-next-line sonarjs/no-nested-conditional
: `${t('datasetDocuments.embedding.hierarchical')} · ${sourceData?.rules?.parent_mode === 'paragraph'
? t('dataset.parentMode.paragraph')
: t('dataset.parentMode.fullDoc')}`
break
case 'segmentLength':
value = !sourceData?.mode
? value
// eslint-disable-next-line sonarjs/no-nested-conditional
: sourceData.mode === ProcessMode.general
? maxTokens
: `${t('datasetDocuments.embedding.parentMaxTokens')} ${maxTokens}; ${t('datasetDocuments.embedding.childMaxTokens')} ${childMaxTokens}`
break
default:
value = !sourceData?.mode
? value
: sourceData?.rules?.pre_processing_rules?.filter(rule =>
rule.enabled).map(rule => getRuleName(rule.id)).join(',')
break
}
return value
}, [getRuleName, isNumber, sourceData, t])
return (
<div className='flex flex-col gap-1'>
{Object.keys(segmentationRuleMap).map((field) => {
return <FieldInfo
key={field}
label={segmentationRuleMap[field as keyof typeof segmentationRuleMap]}
displayedValue={String(getValue(field))}
/>
})}
<FieldInfo
label={t('datasetCreation.stepTwo.indexMode')}
displayedValue={t(`datasetCreation.stepTwo.${indexingType === IndexingType.ECONOMICAL ? 'economical' : 'qualified'}`) as string}
valueIcon={
<Image
className='size-4'
src={
indexingType === IndexingType.ECONOMICAL
? indexMethodIcon.economical
: indexMethodIcon.high_quality
}
alt=''
/>
}
/>
<FieldInfo
label={t('datasetSettings.form.retrievalSetting.title')}
displayedValue={t(`dataset.retrieval.${indexingType === IndexingType.ECONOMICAL ? 'invertedIndex' : retrievalMethod}.title`) as string}
valueIcon={
<Image
className='size-4'
src={
retrievalMethod === RETRIEVE_METHOD.fullText
? retrievalIcon.fullText
// eslint-disable-next-line sonarjs/no-nested-conditional
: retrievalMethod === RETRIEVE_METHOD.hybrid
? retrievalIcon.hybrid
: retrievalIcon.vector
}
alt=''
/>
}
/>
</div>
)
}
export default React.memo(RuleDetail)

View File

@ -2,14 +2,16 @@
import React from 'react'
import { useTranslation } from 'react-i18next'
import { RiBookOpenLine } from '@remixicon/react'
import type { FullDocumentDetail, InitialDocumentDetail } from '@/models/datasets'
import EmbeddingProcess from '../../../create/embedding-process'
import { useGetDocLanguage } from '@/context/i18n'
import EmbeddingProcess from './embedding-process'
import type { IndexingType } from '../../../create/step-two'
import type { RETRIEVE_METHOD } from '@/types/app'
import type { InitialDocumentDetail } from '@/models/pipeline'
type ProcessingProps = {
datasetId: string
indexingType: string
retrievalMethod: string
indexingType: IndexingType
retrievalMethod: RETRIEVE_METHOD
batchId: string
documents: InitialDocumentDetail[]
}
@ -30,8 +32,8 @@ const Processing = ({
<div className='max-w-[640px]'>
<EmbeddingProcess
datasetId={datasetId}
batchId={batchId || ''}
documents={documents as FullDocumentDetail[]}
batchId={batchId}
documents={documents}
indexingType={indexingType}
retrievalMethod={retrievalMethod}
/>

View File

@ -1,6 +1,6 @@
import type { Edge, EnvironmentVariable, Node, SupportUploadFileTypes } from '@/app/components/workflow/types'
import type { DSLImportMode, DSLImportStatus } from './app'
import type { ChunkingMode, DatasetPermission, FileIndexingEstimateResponse, IconInfo } from './datasets'
import type { ChunkingMode, DatasetPermission, DocumentIndexingStatus, FileIndexingEstimateResponse, IconInfo } from './datasets'
import type { Dependency } from '@/app/components/plugins/types'
import type { AppIconSelection } from '@/app/components/base/app-icon-picker'
import type { Viewport } from 'reactflow'
@ -187,10 +187,40 @@ export type PublishedPipelineRunRequest = {
}
export type PublishedPipelineRunPreviewResponse = {
task_iod: string
workflow_run_id: string
data: {
id: string
status: string
created_at: number
elapsed_time: number
error: string
finished_at: number
outputs: FileIndexingEstimateResponse
total_steps: number
total_tokens: number
workflow_id: string
}
}
export type PublishedPipelineRunResponse = {
batch: string
dataset: {
chunk_structure: ChunkingMode
description: string
id: string
name: string
}
documents: InitialDocumentDetail[]
}
export type InitialDocumentDetail = {
data_source_info: Record<string, any>
data_source_type: DatasourceType
enable: boolean
error: string
id: string
indexing_status: DocumentIndexingStatus
name: string
position: number
}

View File

@ -195,7 +195,7 @@ export const useRunPublishedPipeline = (
mutationKey: [NAME_SPACE, 'run-published-pipeline'],
mutationFn: (request: PublishedPipelineRunRequest) => {
const { pipeline_id: pipelineId, is_preview, ...rest } = request
return post<PublishedPipelineRunPreviewResponse | PublishedPipelineRunRequest>(`/rag/pipelines/${pipelineId}/workflows/published/run`, {
return post<PublishedPipelineRunPreviewResponse | PublishedPipelineRunResponse>(`/rag/pipelines/${pipelineId}/workflows/published/run`, {
body: {
...rest,
is_preview,