{!datasetId && (
<>
-
-
{t('datasetCreation.stepThree.creationTitle')}
-
{t('datasetCreation.stepThree.creationContent')}
-
{t('datasetCreation.stepThree.label')}
-
{datasetName || creationCache?.dataset?.name}
+
+
{t('datasetCreation.stepThree.creationTitle')}
+
{t('datasetCreation.stepThree.creationContent')}
+
+
+
+
{t('datasetCreation.stepThree.label')}
+
{datasetName || creationCache?.dataset?.name}
+
+
-
+
)}
{datasetId && (
-
-
{t('datasetCreation.stepThree.additionTitle')}
-
{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}
+
+
{t('datasetCreation.stepThree.additionTitle')}
+
{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}
)}
- {!isMobile &&
-
-
-
{t('datasetCreation.stepThree.sideTipTitle')}
-
{t('datasetCreation.stepThree.sideTipContent')}
+ {!isMobile && (
+
+
+
+
+
+
{t('datasetCreation.stepThree.sideTipTitle')}
+
{t('datasetCreation.stepThree.sideTipContent')}
+
-
}
+ )}
)
}
diff --git a/web/app/components/datasets/create/step-two/index.module.css b/web/app/components/datasets/create/step-two/index.module.css
index f89d6d67ea..178cbeba85 100644
--- a/web/app/components/datasets/create/step-two/index.module.css
+++ b/web/app/components/datasets/create/step-two/index.module.css
@@ -13,18 +13,6 @@
z-index: 10;
}
-.form {
- @apply px-16 pb-8;
-}
-
-.form .label {
- @apply pt-6 pb-2 flex items-center;
- font-weight: 500;
- font-size: 16px;
- line-height: 24px;
- color: #344054;
-}
-
.segmentationItem {
min-height: 68px;
}
@@ -75,6 +63,10 @@
cursor: pointer;
}
+.disabled {
+ cursor: not-allowed !important;
+}
+
.indexItem.disabled:hover {
background-color: #fcfcfd;
border-color: #f2f4f7;
@@ -87,8 +79,7 @@
}
.radioItem {
- @apply relative mb-2 rounded-xl border border-gray-100 cursor-pointer;
- background-color: #fcfcfd;
+ @apply relative mb-2 rounded-xl border border-components-option-card-option-border cursor-pointer bg-components-option-card-option-bg;
}
.radioItem.segmentationItem.custom {
@@ -146,7 +137,7 @@
}
.typeIcon.economical {
- background-image: url(../assets/piggy-bank-01.svg);
+ background-image: url(../assets/piggy-bank-mod.svg);
}
.radioItem .radio {
@@ -247,7 +238,7 @@
}
.ruleItem {
- @apply flex items-center;
+ @apply flex items-center py-1.5;
}
.formFooter {
@@ -394,19 +385,6 @@
max-width: 524px;
}
-.previewHeader {
- position: sticky;
- top: 0;
- left: 0;
- padding-top: 42px;
- background-color: #fff;
- font-weight: 600;
- font-size: 18px;
- line-height: 28px;
- color: #101828;
- z-index: 10;
-}
-
/*
* `fixed` must under `previewHeader` because of style override would not work
*/
@@ -432,4 +410,4 @@
font-size: 12px;
line-height: 18px;
}
-}
\ No newline at end of file
+}
diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx
index f915c68fef..0d7202967a 100644
--- a/web/app/components/datasets/create/step-two/index.tsx
+++ b/web/app/components/datasets/create/step-two/index.tsx
@@ -1,65 +1,80 @@
'use client'
-import React, { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
+import type { FC, PropsWithChildren } from 'react'
+import React, { useCallback, useEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
-import { useBoolean } from 'ahooks'
-import { XMarkIcon } from '@heroicons/react/20/solid'
-import { RocketLaunchIcon } from '@heroicons/react/24/outline'
import {
- RiCloseLine,
+ RiAlertFill,
+ RiArrowLeftLine,
+ RiSearchEyeLine,
} from '@remixicon/react'
import Link from 'next/link'
-import { groupBy } from 'lodash-es'
-import PreviewItem, { PreviewType } from './preview-item'
-import LanguageSelect from './language-select'
+import Image from 'next/image'
+import { useHover } from 'ahooks'
+import SettingCog from '../assets/setting-gear-mod.svg'
+import OrangeEffect from '../assets/option-card-effect-orange.svg'
+import FamilyMod from '../assets/family-mod.svg'
+import Note from '../assets/note-mod.svg'
+import FileList from '../assets/file-list-3-fill.svg'
+import { indexMethodIcon } from '../icons'
+import { PreviewContainer } from '../../preview/container'
+import { ChunkContainer, QAPreview } from '../../chunk'
+import { PreviewHeader } from '../../preview/header'
+import { FormattedText } from '../../formatted-text/formatted'
+import { PreviewSlice } from '../../formatted-text/flavours/preview-slice'
+import PreviewDocumentPicker from '../../common/document-picker/preview-document-picker'
import s from './index.module.css'
import unescape from './unescape'
import escape from './escape'
+import { OptionCard } from './option-card'
+import LanguageSelect from './language-select'
+import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs'
import cn from '@/utils/classnames'
-import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
-import {
- createDocument,
- createFirstDocument,
- fetchFileIndexingEstimate as didFetchFileIndexingEstimate,
- fetchDefaultProcessRule,
-} from '@/service/datasets'
+import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
+
import Button from '@/app/components/base/button'
-import Input from '@/app/components/base/input'
-import Loading from '@/app/components/base/loading'
import FloatRightContainer from '@/app/components/base/float-right-container'
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
import { type RetrievalConfig } from '@/types/app'
import { ensureRerankModelSelected, isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
import Toast from '@/app/components/base/toast'
-import { formatNumber } from '@/utils/format'
import type { NotionPage } from '@/models/common'
import { DataSourceProvider } from '@/models/common'
-import { DataSourceType, DocForm } from '@/models/datasets'
-import NotionIcon from '@/app/components/base/notion-icon'
-import Switch from '@/app/components/base/switch'
-import { MessageChatSquare } from '@/app/components/base/icons/src/public/common'
+import { ChunkingMode, DataSourceType, RerankingModeEnum } from '@/models/datasets'
import { useDatasetDetailContext } from '@/context/dataset-detail'
import I18n from '@/context/i18n'
-import { IS_CE_EDITION } from '@/config'
import { RETRIEVE_METHOD } from '@/types/app'
import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
-import Tooltip from '@/app/components/base/tooltip'
import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
import { LanguagesSupported } from '@/i18n/language'
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
-import { Globe01 } from '@/app/components/base/icons/src/vender/line/mapsAndTravel'
+import Checkbox from '@/app/components/base/checkbox'
+import RadioCard from '@/app/components/base/radio-card'
+import { IS_CE_EDITION } from '@/config'
+import Divider from '@/app/components/base/divider'
+import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument, useFetchDefaultProcessRule, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/knowledge/use-create-dataset'
+import Badge from '@/app/components/base/badge'
+import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton'
+import Tooltip from '@/app/components/base/tooltip'
+import CustomDialog from '@/app/components/base/dialog'
+import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem'
+import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
+
+const TextLabel: FC
= (props) => {
+ return
+}
-type ValueOf = T[keyof T]
type StepTwoProps = {
isSetting?: boolean
documentDetail?: FullDocumentDetail
isAPIKeySet: boolean
onSetting: () => void
datasetId?: string
- indexingType?: ValueOf
+ indexingType?: IndexingType
+ retrievalMethod?: string
dataSourceType: DataSourceType
files: CustomFile[]
notionPages?: NotionPage[]
@@ -69,21 +84,48 @@ type StepTwoProps = {
websiteCrawlJobId?: string
onStepChange?: (delta: number) => void
updateIndexingTypeCache?: (type: string) => void
+ updateRetrievalMethodCache?: (method: string) => void
updateResultCache?: (res: createDocumentResponse) => void
onSave?: () => void
onCancel?: () => void
}
-enum SegmentType {
+export enum SegmentType {
AUTO = 'automatic',
CUSTOM = 'custom',
}
-enum IndexingType {
+export enum IndexingType {
QUALIFIED = 'high_quality',
ECONOMICAL = 'economy',
}
const DEFAULT_SEGMENT_IDENTIFIER = '\\n\\n'
+const DEFAULT_MAXMIMUM_CHUNK_LENGTH = 500
+const DEFAULT_OVERLAP = 50
+
+type ParentChildConfig = {
+ chunkForContext: ParentMode
+ parent: {
+ delimiter: string
+ maxLength: number
+ }
+ child: {
+ delimiter: string
+ maxLength: number
+ }
+}
+
+const defaultParentChildConfig: ParentChildConfig = {
+ chunkForContext: 'paragraph',
+ parent: {
+ delimiter: '\\n\\n',
+ maxLength: 500,
+ },
+ child: {
+ delimiter: '\\n',
+ maxLength: 200,
+ },
+}
const StepTwo = ({
isSetting,
@@ -104,6 +146,7 @@ const StepTwo = ({
updateResultCache,
onSave,
onCancel,
+ updateRetrievalMethodCache,
}: StepTwoProps) => {
const { t } = useTranslation()
const { locale } = useContext(I18n)
@@ -111,66 +154,166 @@ const StepTwo = ({
const isMobile = media === MediaType.mobile
const { dataset: currentDataset, mutateDatasetRes } = useDatasetDetailContext()
+
+ const isInUpload = Boolean(currentDataset)
+ const isUploadInEmptyDataset = isInUpload && !currentDataset?.doc_form
+ const isNotUploadInEmptyDataset = !isUploadInEmptyDataset
+ const isInInit = !isInUpload && !isSetting
+
const isInCreatePage = !datasetId || (datasetId && !currentDataset?.data_source_type)
const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : currentDataset?.data_source_type
- const scrollRef = useRef(null)
- const [scrolled, setScrolled] = useState(false)
- const previewScrollRef = useRef(null)
- const [previewScrolled, setPreviewScrolled] = useState(false)
- const [segmentationType, setSegmentationType] = useState(SegmentType.AUTO)
+ const [segmentationType, setSegmentationType] = useState(SegmentType.CUSTOM)
const [segmentIdentifier, doSetSegmentIdentifier] = useState(DEFAULT_SEGMENT_IDENTIFIER)
- const setSegmentIdentifier = useCallback((value: string) => {
- doSetSegmentIdentifier(value ? escape(value) : DEFAULT_SEGMENT_IDENTIFIER)
+ const setSegmentIdentifier = useCallback((value: string, canEmpty?: boolean) => {
+ doSetSegmentIdentifier(value ? escape(value) : (canEmpty ? '' : DEFAULT_SEGMENT_IDENTIFIER))
}, [])
- const [maxChunkLength, setMaxChunkLength] = useState(4000) // default chunk length
+ const [maxChunkLength, setMaxChunkLength] = useState(DEFAULT_MAXMIMUM_CHUNK_LENGTH) // default chunk length
const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000)
- const [overlap, setOverlap] = useState(50)
+ const [overlap, setOverlap] = useState(DEFAULT_OVERLAP)
const [rules, setRules] = useState([])
const [defaultConfig, setDefaultConfig] = useState()
const hasSetIndexType = !!indexingType
- const [indexType, setIndexType] = useState>(
+ const [indexType, setIndexType] = useState(
(indexingType
|| isAPIKeySet)
? IndexingType.QUALIFIED
: IndexingType.ECONOMICAL,
)
- const [isLanguageSelectDisabled, setIsLanguageSelectDisabled] = useState(false)
- const [docForm, setDocForm] = useState(
- (datasetId && documentDetail) ? documentDetail.doc_form : DocForm.TEXT,
+
+ const [previewFile, setPreviewFile] = useState(
+ (datasetId && documentDetail)
+ ? documentDetail.file
+ : files[0],
)
+ const [previewNotionPage, setPreviewNotionPage] = useState(
+ (datasetId && documentDetail)
+ ? documentDetail.notion_page
+ : notionPages[0],
+ )
+
+ const [previewWebsitePage, setPreviewWebsitePage] = useState(
+ (datasetId && documentDetail)
+ ? documentDetail.website_page
+ : websitePages[0],
+ )
+
+ // QA Related
+ const [isLanguageSelectDisabled, _setIsLanguageSelectDisabled] = useState(false)
+ const [isQAConfirmDialogOpen, setIsQAConfirmDialogOpen] = useState(false)
+ const [docForm, setDocForm] = useState(
+ (datasetId && documentDetail) ? documentDetail.doc_form as ChunkingMode : ChunkingMode.text,
+ )
+ const handleChangeDocform = (value: ChunkingMode) => {
+ if (value === ChunkingMode.qa && indexType === IndexingType.ECONOMICAL) {
+ setIsQAConfirmDialogOpen(true)
+ return
+ }
+ if (value === ChunkingMode.parentChild && indexType === IndexingType.ECONOMICAL)
+ setIndexType(IndexingType.QUALIFIED)
+ setDocForm(value)
+ // eslint-disable-next-line @typescript-eslint/no-use-before-define
+ currentEstimateMutation.reset()
+ }
+
const [docLanguage, setDocLanguage] = useState(
(datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese'),
)
- const [QATipHide, setQATipHide] = useState(false)
- const [previewSwitched, setPreviewSwitched] = useState(false)
- const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
- const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState(null)
- const [automaticFileIndexingEstimate, setAutomaticFileIndexingEstimate] = useState(null)
- const fileIndexingEstimate = (() => {
- return segmentationType === SegmentType.AUTO ? automaticFileIndexingEstimate : customFileIndexingEstimate
- })()
- const [isCreating, setIsCreating] = useState(false)
+ const [parentChildConfig, setParentChildConfig] = useState(defaultParentChildConfig)
- const scrollHandle = (e: Event) => {
- if ((e.target as HTMLDivElement).scrollTop > 0)
- setScrolled(true)
+ const getIndexing_technique = () => indexingType || indexType
+ const currentDocForm = currentDataset?.doc_form || docForm
- else
- setScrolled(false)
+ const getProcessRule = (): ProcessRule => {
+ if (currentDocForm === ChunkingMode.parentChild) {
+ return {
+ rules: {
+ pre_processing_rules: rules,
+ segmentation: {
+ separator: unescape(
+ parentChildConfig.parent.delimiter,
+ ),
+ max_tokens: parentChildConfig.parent.maxLength,
+ },
+ parent_mode: parentChildConfig.chunkForContext,
+ subchunk_segmentation: {
+ separator: unescape(parentChildConfig.child.delimiter),
+ max_tokens: parentChildConfig.child.maxLength,
+ },
+ },
+ mode: 'hierarchical',
+ } as ProcessRule
+ }
+ return {
+ rules: {
+ pre_processing_rules: rules,
+ segmentation: {
+ separator: unescape(segmentIdentifier),
+ max_tokens: maxChunkLength,
+ chunk_overlap: overlap,
+ },
+ }, // api will check this. It will be removed after api refactored.
+ mode: segmentationType,
+ } as ProcessRule
}
- const previewScrollHandle = (e: Event) => {
- if ((e.target as HTMLDivElement).scrollTop > 0)
- setPreviewScrolled(true)
+ const fileIndexingEstimateQuery = useFetchFileIndexingEstimateForFile({
+ docForm: currentDocForm,
+ docLanguage,
+ dataSourceType: DataSourceType.FILE,
+ files: previewFile
+ ? [files.find(file => file.name === previewFile.name)!]
+ : files,
+ indexingTechnique: getIndexing_technique() as any,
+ processRule: getProcessRule(),
+ dataset_id: datasetId!,
+ })
+ const notionIndexingEstimateQuery = useFetchFileIndexingEstimateForNotion({
+ docForm: currentDocForm,
+ docLanguage,
+ dataSourceType: DataSourceType.NOTION,
+ notionPages: [previewNotionPage],
+ indexingTechnique: getIndexing_technique() as any,
+ processRule: getProcessRule(),
+ dataset_id: datasetId || '',
+ })
- else
- setPreviewScrolled(false)
- }
- const getFileName = (name: string) => {
- const arr = name.split('.')
- return arr.slice(0, -1).join('.')
- }
+ const websiteIndexingEstimateQuery = useFetchFileIndexingEstimateForWeb({
+ docForm: currentDocForm,
+ docLanguage,
+ dataSourceType: DataSourceType.WEB,
+ websitePages: [previewWebsitePage],
+ crawlOptions,
+ websiteCrawlProvider,
+ websiteCrawlJobId,
+ indexingTechnique: getIndexing_technique() as any,
+ processRule: getProcessRule(),
+ dataset_id: datasetId || '',
+ })
+
+ const currentEstimateMutation = dataSourceType === DataSourceType.FILE
+ ? fileIndexingEstimateQuery
+ : dataSourceType === DataSourceType.NOTION
+ ? notionIndexingEstimateQuery
+ : websiteIndexingEstimateQuery
+
+ const fetchEstimate = useCallback(() => {
+ if (dataSourceType === DataSourceType.FILE)
+ fileIndexingEstimateQuery.mutate()
+
+ if (dataSourceType === DataSourceType.NOTION)
+ notionIndexingEstimateQuery.mutate()
+
+ if (dataSourceType === DataSourceType.WEB)
+ websiteIndexingEstimateQuery.mutate()
+ }, [dataSourceType, fileIndexingEstimateQuery, notionIndexingEstimateQuery, websiteIndexingEstimateQuery])
+
+ const estimate
+ = dataSourceType === DataSourceType.FILE
+ ? fileIndexingEstimateQuery.data
+ : dataSourceType === DataSourceType.NOTION
+ ? notionIndexingEstimateQuery.data
+ : websiteIndexingEstimateQuery.data
const getRuleName = (key: string) => {
if (key === 'remove_extra_spaces')
@@ -198,128 +341,20 @@ const StepTwo = ({
if (defaultConfig) {
setSegmentIdentifier(defaultConfig.segmentation.separator)
setMaxChunkLength(defaultConfig.segmentation.max_tokens)
- setOverlap(defaultConfig.segmentation.chunk_overlap)
+ setOverlap(defaultConfig.segmentation.chunk_overlap!)
setRules(defaultConfig.pre_processing_rules)
}
+ setParentChildConfig(defaultParentChildConfig)
}
- const fetchFileIndexingEstimate = async (docForm = DocForm.TEXT, language?: string) => {
- // eslint-disable-next-line @typescript-eslint/no-use-before-define
- const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams(docForm, language)!)
- if (segmentationType === SegmentType.CUSTOM)
- setCustomFileIndexingEstimate(res)
- else
- setAutomaticFileIndexingEstimate(res)
- }
-
- const confirmChangeCustomConfig = () => {
- if (segmentationType === SegmentType.CUSTOM && maxChunkLength > limitMaxChunkLength) {
- Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: limitMaxChunkLength }) })
+ const updatePreview = () => {
+ if (segmentationType === SegmentType.CUSTOM && maxChunkLength > 4000) {
+ Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') })
return
}
- setCustomFileIndexingEstimate(null)
- setShowPreview()
- fetchFileIndexingEstimate()
- setPreviewSwitched(false)
+ fetchEstimate()
}
- const getIndexing_technique = () => indexingType || indexType
-
- const getProcessRule = () => {
- const processRule: ProcessRule = {
- rules: {} as any, // api will check this. It will be removed after api refactored.
- mode: segmentationType,
- }
- if (segmentationType === SegmentType.CUSTOM) {
- const ruleObj = {
- pre_processing_rules: rules,
- segmentation: {
- separator: unescape(segmentIdentifier),
- max_tokens: maxChunkLength,
- chunk_overlap: overlap,
- },
- }
- processRule.rules = ruleObj
- }
- return processRule
- }
-
- const getNotionInfo = () => {
- const workspacesMap = groupBy(notionPages, 'workspace_id')
- const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
- return {
- workspaceId,
- pages: workspacesMap[workspaceId],
- }
- })
- return workspaces.map((workspace) => {
- return {
- workspace_id: workspace.workspaceId,
- pages: workspace.pages.map((page) => {
- const { page_id, page_name, page_icon, type } = page
- return {
- page_id,
- page_name,
- page_icon,
- type,
- }
- }),
- }
- }) as NotionInfo[]
- }
-
- const getWebsiteInfo = () => {
- return {
- provider: websiteCrawlProvider,
- job_id: websiteCrawlJobId,
- urls: websitePages.map(page => page.source_url),
- only_main_content: crawlOptions?.only_main_content,
- }
- }
-
- const getFileIndexingEstimateParams = (docForm: DocForm, language?: string): IndexingEstimateParams | undefined => {
- if (dataSourceType === DataSourceType.FILE) {
- return {
- info_list: {
- data_source_type: dataSourceType,
- file_info_list: {
- file_ids: files.map(file => file.id) as string[],
- },
- },
- indexing_technique: getIndexing_technique() as string,
- process_rule: getProcessRule(),
- doc_form: docForm,
- doc_language: language || docLanguage,
- dataset_id: datasetId as string,
- }
- }
- if (dataSourceType === DataSourceType.NOTION) {
- return {
- info_list: {
- data_source_type: dataSourceType,
- notion_info_list: getNotionInfo(),
- },
- indexing_technique: getIndexing_technique() as string,
- process_rule: getProcessRule(),
- doc_form: docForm,
- doc_language: language || docLanguage,
- dataset_id: datasetId as string,
- }
- }
- if (dataSourceType === DataSourceType.WEB) {
- return {
- info_list: {
- data_source_type: dataSourceType,
- website_info_list: getWebsiteInfo(),
- },
- indexing_technique: getIndexing_technique() as string,
- process_rule: getProcessRule(),
- doc_form: docForm,
- doc_language: language || docLanguage,
- dataset_id: datasetId as string,
- }
- }
- }
const {
modelList: rerankModelList,
defaultModel: rerankDefaultModel,
@@ -351,13 +386,14 @@ const StepTwo = ({
if (isSetting) {
params = {
original_document_id: documentDetail?.id,
- doc_form: docForm,
+ doc_form: currentDocForm,
doc_language: docLanguage,
process_rule: getProcessRule(),
// eslint-disable-next-line @typescript-eslint/no-use-before-define
retrieval_model: retrievalConfig, // Readonly. If want to changed, just go to settings page.
embedding_model: embeddingModel.model, // Readonly
embedding_model_provider: embeddingModel.provider, // Readonly
+ indexing_technique: getIndexing_technique(),
} as CreateDocumentReq
}
else { // create
@@ -377,8 +413,12 @@ const StepTwo = ({
}
const postRetrievalConfig = ensureRerankModelSelected({
rerankDefaultModel: rerankDefaultModel!,
- // eslint-disable-next-line @typescript-eslint/no-use-before-define
- retrievalConfig,
+ retrievalConfig: {
+ // eslint-disable-next-line @typescript-eslint/no-use-before-define
+ ...retrievalConfig,
+ // eslint-disable-next-line @typescript-eslint/no-use-before-define
+ reranking_enable: retrievalConfig.reranking_mode === RerankingModeEnum.RerankingModel,
+ },
indexMethod: indexMethod as string,
})
params = {
@@ -390,7 +430,7 @@ const StepTwo = ({
},
indexing_technique: getIndexing_technique(),
process_rule: getProcessRule(),
- doc_form: docForm,
+ doc_form: currentDocForm,
doc_language: docLanguage,
retrieval_model: postRetrievalConfig,
@@ -403,29 +443,36 @@ const StepTwo = ({
}
}
if (dataSourceType === DataSourceType.NOTION)
- params.data_source.info_list.notion_info_list = getNotionInfo()
+ params.data_source.info_list.notion_info_list = getNotionInfo(notionPages)
- if (dataSourceType === DataSourceType.WEB)
- params.data_source.info_list.website_info_list = getWebsiteInfo()
+ if (dataSourceType === DataSourceType.WEB) {
+ params.data_source.info_list.website_info_list = getWebsiteInfo({
+ websiteCrawlProvider,
+ websiteCrawlJobId,
+ websitePages,
+ })
+ }
}
return params
}
- const getRules = async () => {
- try {
- const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' })
- const separator = res.rules.segmentation.separator
+ const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule({
+ onSuccess(data) {
+ const separator = data.rules.segmentation.separator
setSegmentIdentifier(separator)
- setMaxChunkLength(res.rules.segmentation.max_tokens)
- setLimitMaxChunkLength(res.limits.indexing_max_segmentation_tokens_length)
- setOverlap(res.rules.segmentation.chunk_overlap)
- setRules(res.rules.pre_processing_rules)
- setDefaultConfig(res.rules)
- }
- catch (err) {
- console.log(err)
- }
- }
+ setMaxChunkLength(data.rules.segmentation.max_tokens)
+ setOverlap(data.rules.segmentation.chunk_overlap!)
+ setRules(data.rules.pre_processing_rules)
+ setDefaultConfig(data.rules)
+ setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length)
+ },
+ onError(error) {
+ Toast.notify({
+ type: 'error',
+ message: `${error}`,
+ })
+ },
+ })
const getRulesFromDetail = () => {
if (documentDetail) {
@@ -435,7 +482,7 @@ const StepTwo = ({
const overlap = rules.segmentation.chunk_overlap
setSegmentIdentifier(separator)
setMaxChunkLength(max)
- setOverlap(overlap)
+ setOverlap(overlap!)
setRules(rules.pre_processing_rules)
setDefaultConfig(rules)
}
@@ -443,119 +490,81 @@ const StepTwo = ({
const getDefaultMode = () => {
if (documentDetail)
+ // @ts-expect-error fix after api refactored
setSegmentationType(documentDetail.dataset_process_rule.mode)
}
- const createHandle = async () => {
- if (isCreating)
- return
- setIsCreating(true)
- try {
- let res
- const params = getCreationParams()
- if (!params)
- return false
-
- setIsCreating(true)
- if (!datasetId) {
- res = await createFirstDocument({
- body: params as CreateDocumentReq,
- })
- updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
- updateResultCache && updateResultCache(res)
- }
- else {
- res = await createDocument({
- datasetId,
- body: params as CreateDocumentReq,
- })
- updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
- updateResultCache && updateResultCache(res)
- }
- if (mutateDatasetRes)
- mutateDatasetRes()
- onStepChange && onStepChange(+1)
- isSetting && onSave && onSave()
- }
- catch (err) {
+ const createFirstDocumentMutation = useCreateFirstDocument({
+ onError(error) {
Toast.notify({
type: 'error',
- message: `${err}`,
+ message: `${error}`,
+ })
+ },
+ })
+ const createDocumentMutation = useCreateDocument(datasetId!, {
+ onError(error) {
+ Toast.notify({
+ type: 'error',
+ message: `${error}`,
+ })
+ },
+ })
+
+ const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending
+
+ const createHandle = async () => {
+ const params = getCreationParams()
+ if (!params)
+ return false
+
+ if (!datasetId) {
+ await createFirstDocumentMutation.mutateAsync(
+ params,
+ {
+ onSuccess(data) {
+ updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
+ updateResultCache && updateResultCache(data)
+ // eslint-disable-next-line @typescript-eslint/no-use-before-define
+ updateRetrievalMethodCache && updateRetrievalMethodCache(retrievalConfig.search_method as string)
+ },
+ },
+ )
+ }
+ else {
+ await createDocumentMutation.mutateAsync(params, {
+ onSuccess(data) {
+ updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
+ updateResultCache && updateResultCache(data)
+ },
})
}
- finally {
- setIsCreating(false)
- }
- }
-
- const handleSwitch = (state: boolean) => {
- if (state)
- setDocForm(DocForm.QA)
- else
- setDocForm(DocForm.TEXT)
- }
-
- const previewSwitch = async (language?: string) => {
- setPreviewSwitched(true)
- setIsLanguageSelectDisabled(true)
- if (segmentationType === SegmentType.AUTO)
- setAutomaticFileIndexingEstimate(null)
- else
- setCustomFileIndexingEstimate(null)
- try {
- await fetchFileIndexingEstimate(DocForm.QA, language)
- }
- finally {
- setIsLanguageSelectDisabled(false)
- }
- }
-
- const handleSelect = (language: string) => {
- setDocLanguage(language)
- // Switch language, re-cutter
- if (docForm === DocForm.QA && previewSwitched)
- previewSwitch(language)
+ if (mutateDatasetRes)
+ mutateDatasetRes()
+ onStepChange && onStepChange(+1)
+ isSetting && onSave && onSave()
}
const changeToEconomicalType = () => {
- if (!hasSetIndexType) {
+ if (docForm !== ChunkingMode.text)
+ return
+
+ if (!hasSetIndexType)
setIndexType(IndexingType.ECONOMICAL)
- setDocForm(DocForm.TEXT)
- }
}
useEffect(() => {
// fetch rules
if (!isSetting) {
- getRules()
+ fetchDefaultProcessRuleMutation.mutate('/datasets/process-rule')
}
else {
getRulesFromDetail()
getDefaultMode()
}
+ // eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
- useEffect(() => {
- scrollRef.current?.addEventListener('scroll', scrollHandle)
- return () => {
- scrollRef.current?.removeEventListener('scroll', scrollHandle)
- }
- }, [])
-
- useLayoutEffect(() => {
- if (showPreview) {
- previewScrollRef.current?.addEventListener('scroll', previewScrollHandle)
- return () => {
- previewScrollRef.current?.removeEventListener('scroll', previewScrollHandle)
- }
- }
- }, [showPreview])
-
- useEffect(() => {
- if (indexingType === IndexingType.ECONOMICAL && docForm === DocForm.QA)
- setDocForm(DocForm.TEXT)
- }, [indexingType, docForm])
-
useEffect(() => {
// get indexing type by props
if (indexingType)
@@ -565,20 +574,6 @@ const StepTwo = ({
setIndexType(isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL)
}, [isAPIKeySet, indexingType, datasetId])
- useEffect(() => {
- if (segmentationType === SegmentType.AUTO) {
- setAutomaticFileIndexingEstimate(null)
- !isMobile && setShowPreview()
- fetchFileIndexingEstimate()
- setPreviewSwitched(false)
- }
- else {
- hidePreview()
- setCustomFileIndexingEstimate(null)
- setPreviewSwitched(false)
- }
- }, [segmentationType, indexType])
-
const [retrievalConfig, setRetrievalConfig] = useState(currentDataset?.retrieval_model_dict || {
search_method: RETRIEVE_METHOD.semantic,
reranking_enable: false,
@@ -591,433 +586,589 @@ const StepTwo = ({
score_threshold: 0.5,
} as RetrievalConfig)
+ const economyDomRef = useRef(null)
+ const isHoveringEconomy = useHover(economyDomRef)
+
return (
-
-
-
{t('datasetCreation.steps.two')}
- {(isMobile || !showPreview) && (
-
- )}
-
-
-
{t('datasetCreation.stepTwo.segmentation')}
-
-
setSegmentationType(SegmentType.AUTO)}
- >
-
-
-
-
{t('datasetCreation.stepTwo.auto')}
-
{t('datasetCreation.stepTwo.autoDescription')}
-
-
-
setSegmentationType(SegmentType.CUSTOM)}
- >
-
-
-
-
{t('datasetCreation.stepTwo.custom')}
-
{t('datasetCreation.stepTwo.customDescription')}
-
- {segmentationType === SegmentType.CUSTOM && (
-
-
-
-
- {t('datasetCreation.stepTwo.separator')}
-
- {t('datasetCreation.stepTwo.separatorTip')}
-
- }
- />
-
-
setSegmentIdentifier(e.target.value)}
- />
-
-
-
-
-
{t('datasetCreation.stepTwo.maxLength')}
-
setMaxChunkLength(parseInt(e.target.value.replace(/^0+/, ''), 10))}
- />
-
-
-
-
-
- {t('datasetCreation.stepTwo.overlap')}
-
- {t('datasetCreation.stepTwo.overlapTip')}
-
- }
- />
-
-
setOverlap(parseInt(e.target.value.replace(/^0+/, ''), 10))}
- />
-
-
-
-
-
{t('datasetCreation.stepTwo.rules')}
- {rules.map(rule => (
-
- ruleChangeHandle(rule.id)} className="w-4 h-4 rounded border-gray-300 text-blue-700 focus:ring-blue-700" />
-
-
- ))}
-
-
-
-
-
-
-
- )}
-
-
-
{t('datasetCreation.stepTwo.indexMode')}
-
-
- {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && (
-
{
- if (isAPIKeySet)
- setIndexType(IndexingType.QUALIFIED)
- }}
- >
-
- {!hasSetIndexType &&
}
-
-
- {t('datasetCreation.stepTwo.qualified')}
- {!hasSetIndexType && {t('datasetCreation.stepTwo.recommend')}}
-
-
{t('datasetCreation.stepTwo.qualifiedTip')}
-
- {!isAPIKeySet && (
-
- {t('datasetCreation.stepTwo.warning')}
- {t('datasetCreation.stepTwo.click')}
-
- )}
-
- )}
-
- {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
-
-
- {!hasSetIndexType &&
}
-
-
{t('datasetCreation.stepTwo.economical')}
-
{t('datasetCreation.stepTwo.economicalTip')}
-
-
- )}
-
- {hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
-
- {t('datasetCreation.stepTwo.indexSettingTip')}
- {t('datasetCreation.stepTwo.datasetSettingLink')}
-
- )}
- {IS_CE_EDITION && indexType === IndexingType.QUALIFIED && (
-
-
-
-
-
-
-
{t('datasetCreation.stepTwo.QATitle')}
-
- {t('datasetCreation.stepTwo.QALanguage')}
-
-
-
-
-
-
-
- {docForm === DocForm.QA && !QATipHide && (
-
- {t('datasetCreation.stepTwo.QATip')}
- setQATipHide(true)} />
-
- )}
-
- )}
- {/* Embedding model */}
- {indexType === IndexingType.QUALIFIED && (
-
-
{t('datasetSettings.form.embeddingModel')}
-
{
- setEmbeddingModel(model)
- }}
+
+
{t('datasetCreation.stepTwo.segmentation')}
+ {((isInUpload && [ChunkingMode.text, ChunkingMode.qa].includes(currentDataset!.doc_form))
+ || isUploadInEmptyDataset
+ || isInInit)
+ &&
}
+ activeHeaderClassName='bg-dataset-option-card-blue-gradient'
+ description={t('datasetCreation.stepTwo.generalTip')}
+ isActive={
+ [ChunkingMode.text, ChunkingMode.qa].includes(currentDocForm)
+ }
+ onSwitched={() =>
+ handleChangeDocform(ChunkingMode.text)
+ }
+ actions={
+ <>
+
+
+
+ }
+ noHighlight={isInUpload && isNotUploadInEmptyDataset}
+ >
+
+
+
setSegmentIdentifier(e.target.value, true)}
+ />
+
+
- {!!datasetId && (
-
- {t('datasetCreation.stepTwo.indexSettingTip')}
- {t('datasetCreation.stepTwo.datasetSettingLink')}
-
- )}
- )}
- {/* Retrieval Method Config */}
-
- {!datasetId
- ? (
-
-
{t('datasetSettings.form.retrievalSetting.title')}
-
-
{t('datasetSettings.form.retrievalSetting.learnMore')}
- {t('datasetSettings.form.retrievalSetting.longDescription')}
+
+
+
+ {t('datasetCreation.stepTwo.rules')}
+
+
+
+
+ {rules.map(rule => (
+
{
+ ruleChangeHandle(rule.id)
+ }}>
+
+
-
- )
- : (
-
-
{t('datasetSettings.form.retrievalSetting.title')}
-
- )}
-
-
- {
- getIndexing_technique() === IndexingType.QUALIFIED
- ? (
-
+
+
+
{
+ if (currentDataset?.doc_form)
+ return
+ if (docForm === ChunkingMode.qa)
+ handleChangeDocform(ChunkingMode.text)
+ else
+ handleChangeDocform(ChunkingMode.qa)
+ }}>
+
+
+
+
- )
- : (
-
- )
- }
+
+
+
+
+ {t('datasetCreation.stepTwo.QATip')}
+
+
+ )}
+ }
+
-
-
-
- {dataSourceType === DataSourceType.FILE && (
- <>
-
{t('datasetCreation.stepTwo.fileSource')}
-
-
- {getFileName(files[0].name || '')}
- {files.length > 1 && (
-
- {t('datasetCreation.stepTwo.other')}
- {files.length - 1}
- {t('datasetCreation.stepTwo.fileUnit')}
-
- )}
-
-
- )}
- {dataSourceType === DataSourceType.NOTION && (
- <>
-
{t('datasetCreation.stepTwo.notionSource')}
-
-
}
+ {
+ (
+ (isInUpload && currentDataset!.doc_form === ChunkingMode.parentChild)
+ || isUploadInEmptyDataset
+ || isInInit
+ )
+ && }
+ effectImg={OrangeEffect.src}
+ activeHeaderClassName='bg-dataset-option-card-orange-gradient'
+ description={t('datasetCreation.stepTwo.parentChildTip')}
+ isActive={currentDocForm === ChunkingMode.parentChild}
+ onSwitched={() => handleChangeDocform(ChunkingMode.parentChild)}
+ actions={
+ <>
+
+
+
+ }
+ noHighlight={isInUpload && isNotUploadInEmptyDataset}
+ >
+
+
+
+
+ {t('datasetCreation.stepTwo.parentChunkForContext')}
+
+
+
+
}
+ title={t('datasetCreation.stepTwo.paragraph')}
+ description={t('datasetCreation.stepTwo.paragraphTip')}
+ isChosen={parentChildConfig.chunkForContext === 'paragraph'}
+ onChosen={() => setParentChildConfig(
+ {
+ ...parentChildConfig,
+ chunkForContext: 'paragraph',
+ },
+ )}
+ chosenConfig={
+
+ setParentChildConfig({
+ ...parentChildConfig,
+ parent: {
+ ...parentChildConfig.parent,
+ delimiter: e.target.value ? escape(e.target.value) : '',
+ },
+ })}
+ />
+ setParentChildConfig({
+ ...parentChildConfig,
+ parent: {
+ ...parentChildConfig.parent,
+ maxLength: value,
+ },
+ })}
/>
- {notionPages[0]?.page_name}
- {notionPages.length > 1 && (
-
- {t('datasetCreation.stepTwo.other')}
- {notionPages.length - 1}
- {t('datasetCreation.stepTwo.notionUnit')}
-
- )}
-
- )}
- {dataSourceType === DataSourceType.WEB && (
- <>
-
{t('datasetCreation.stepTwo.websiteSource')}
-
-
- {websitePages[0].source_url}
- {websitePages.length > 1 && (
-
- {t('datasetCreation.stepTwo.other')}
- {websitePages.length - 1}
- {t('datasetCreation.stepTwo.webpageUnit')}
-
- )}
-
-
- )}
-
-
-
-
{t('datasetCreation.stepTwo.estimateSegment')}
-
- {
- fileIndexingEstimate
- ? (
-
{formatNumber(fileIndexingEstimate.total_segments)}
- )
- : (
-
{t('datasetCreation.stepTwo.calculating')}
- )
}
+ />
+
}
+ title={t('datasetCreation.stepTwo.fullDoc')}
+ description={t('datasetCreation.stepTwo.fullDocTip')}
+ onChosen={() => setParentChildConfig(
+ {
+ ...parentChildConfig,
+ chunkForContext: 'full-doc',
+ },
+ )}
+ isChosen={parentChildConfig.chunkForContext === 'full-doc'}
+ />
+
+
+
+
+
+ {t('datasetCreation.stepTwo.childChunkForRetrieval')}
+
+
+
+
+ setParentChildConfig({
+ ...parentChildConfig,
+ child: {
+ ...parentChildConfig.child,
+ delimiter: e.target.value ? escape(e.target.value) : '',
+ },
+ })}
+ />
+ setParentChildConfig({
+ ...parentChildConfig,
+ child: {
+ ...parentChildConfig.child,
+ maxLength: value,
+ },
+ })}
+ />
+
+
+
+
+
+ {t('datasetCreation.stepTwo.rules')}
+
+
+
+
+ {rules.map(rule => (
+
{
+ ruleChangeHandle(rule.id)
+ }}>
+
+
+
+ ))}
- {!isSetting
- ? (
-
-
-
-
+ }
+
+
{t('datasetCreation.stepTwo.indexMode')}
+
+ {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && (
+
+ {t('datasetCreation.stepTwo.qualified')}
+
+ {t('datasetCreation.stepTwo.recommend')}
+
+
+ {!hasSetIndexType && }
+
+
}
+ description={t('datasetCreation.stepTwo.qualifiedTip')}
+ icon={
}
+ isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED}
+ disabled={!isAPIKeySet || hasSetIndexType}
+ onSwitched={() => {
+ if (isAPIKeySet)
+ setIndexType(IndexingType.QUALIFIED)
+ }}
+ />
+ )}
+
+ {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
+ <>
+
setIsQAConfirmDialogOpen(false)} className='w-[432px]'>
+
+
+
+
- )
- : (
-
-
-
-
- )}
-
+
+
+
+ }
+ isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL}
+ disabled={!isAPIKeySet || hasSetIndexType || docForm !== ChunkingMode.text}
+ ref={economyDomRef}
+ onSwitched={() => {
+ if (isAPIKeySet && docForm === ChunkingMode.text)
+ setIndexType(IndexingType.ECONOMICAL)
+ }}
+ />
+
+
+
+ {
+ docForm === ChunkingMode.qa
+ ? t('datasetCreation.stepTwo.notAvailableForQA')
+ : t('datasetCreation.stepTwo.notAvailableForParentChild')
+ }
+
+
+
+ )}
-
- {showPreview &&
-
-
-
-
{t('datasetCreation.stepTwo.previewTitle')}
- {docForm === DocForm.QA && !previewSwitched && (
-
- )}
-
-
-
-
-
- {docForm === DocForm.QA && !previewSwitched && (
-
- {t('datasetCreation.stepTwo.previewSwitchTipStart')}
- {t('datasetCreation.stepTwo.previewSwitchTipEnd')}
-
- )}
-
-
- {previewSwitched && docForm === DocForm.QA && fileIndexingEstimate?.qa_preview && (
- <>
- {fileIndexingEstimate?.qa_preview.map((item, index) => (
-
- ))}
-
- )}
- {(docForm === DocForm.TEXT || !previewSwitched) && fileIndexingEstimate?.preview && (
- <>
- {fileIndexingEstimate?.preview.map((item, index) => (
-
- ))}
-
- )}
- {previewSwitched && docForm === DocForm.QA && !fileIndexingEstimate?.qa_preview && (
-
-
-
- )}
- {!previewSwitched && !fileIndexingEstimate?.preview && (
-
-
-
- )}
-
-
-
-
-
{t('datasetCreation.stepTwo.sideTipTitle')}
-
-
{t('datasetCreation.stepTwo.sideTipP1')}
-
{t('datasetCreation.stepTwo.sideTipP2')}
-
{t('datasetCreation.stepTwo.sideTipP3')}
-
{t('datasetCreation.stepTwo.sideTipP4')}
-
+ {!hasSetIndexType && indexType === IndexingType.QUALIFIED && (
+
+
+
+
{t('datasetCreation.stepTwo.highQualityTip')}
)}
+ {hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
+
+ {t('datasetCreation.stepTwo.indexSettingTip')}
+ {t('datasetCreation.stepTwo.datasetSettingLink')}
+
+ )}
+ {/* Embedding model */}
+ {indexType === IndexingType.QUALIFIED && (
+
+
{t('datasetSettings.form.embeddingModel')}
+
{
+ setEmbeddingModel(model)
+ }}
+ />
+ {!!datasetId && (
+
+ {t('datasetCreation.stepTwo.indexSettingTip')}
+ {t('datasetCreation.stepTwo.datasetSettingLink')}
+
+ )}
+
+ )}
+
+ {/* Retrieval Method Config */}
+
+ {!datasetId
+ ? (
+
+
{t('datasetSettings.form.retrievalSetting.title')}
+
+
+ )
+ : (
+
+
{t('datasetSettings.form.retrievalSetting.title')}
+
+ )}
+
+
+ {
+ getIndexing_technique() === IndexingType.QUALIFIED
+ ? (
+
+ )
+ : (
+
+ )
+ }
+
+
+
+ {!isSetting
+ ? (
+
+
+
+
+ )
+ : (
+
+
+
+
+ )}
+
+
{ }} footer={null}>
+
+
+ {dataSourceType === DataSourceType.FILE
+ &&
>}
+ onChange={(selected) => {
+ currentEstimateMutation.reset()
+ setPreviewFile(selected)
+ currentEstimateMutation.mutate()
+ }}
+ // when it is from setting, it just has one file
+ value={isSetting ? (files[0]! as Required) : previewFile}
+ />
+ }
+ {dataSourceType === DataSourceType.NOTION
+ && ({
+ id: page.page_id,
+ name: page.page_name,
+ extension: 'md',
+ }))
+ }
+ onChange={(selected) => {
+ currentEstimateMutation.reset()
+ const selectedPage = notionPages.find(page => page.page_id === selected.id)
+ setPreviewNotionPage(selectedPage!)
+ currentEstimateMutation.mutate()
+ }}
+ value={{
+ id: previewNotionPage?.page_id || '',
+ name: previewNotionPage?.page_name || '',
+ extension: 'md',
+ }}
+ />
+ }
+ {dataSourceType === DataSourceType.WEB
+ && ({
+ id: page.source_url,
+ name: page.title,
+ extension: 'md',
+ }))
+ }
+ onChange={(selected) => {
+ currentEstimateMutation.reset()
+ const selectedPage = websitePages.find(page => page.source_url === selected.id)
+ setPreviewWebsitePage(selectedPage!)
+ currentEstimateMutation.mutate()
+ }}
+ value={
+ {
+ id: previewWebsitePage?.source_url || '',
+ name: previewWebsitePage?.title || '',
+ extension: 'md',
+ }
+ }
+ />
+ }
+ {
+ currentDocForm !== ChunkingMode.qa
+ &&
+ }
+
+
+
+
+ {t('datasetCreation.stepTwo.previewChunkTip')}
+
+
+
+ {Array.from({ length: 10 }, (_, i) => (
+
+
+
+
+
+
+
+
+
+
+ ))}
+
+ )}
+
+ {props.label}
+ {props.children}
+
+}
+
+export const DelimiterInput: FC = (props) => {
+ const { t } = useTranslation()
+ return
+ {t('datasetCreation.stepTwo.separator')}
+
+ {props.tooltip || t('datasetCreation.stepTwo.separatorTip')}
+
+ }
+ />
+
}>
+
+
+}
+
+export const MaxLengthInput: FC
= (props) => {
+ const { t } = useTranslation()
+ return
+ {t('datasetCreation.stepTwo.maxLength')}
+ }>
+
+
+}
+
+export const OverlapInput: FC
= (props) => {
+ const { t } = useTranslation()
+ return
+ {t('datasetCreation.stepTwo.overlap')}
+
+ {t('datasetCreation.stepTwo.overlapTip')}
+
+ }
+ />
+
}>
+
+
+}
diff --git a/web/app/components/datasets/create/step-two/language-select/index.tsx b/web/app/components/datasets/create/step-two/language-select/index.tsx
index 41f3e0abb5..9cbf1a40d1 100644
--- a/web/app/components/datasets/create/step-two/language-select/index.tsx
+++ b/web/app/components/datasets/create/step-two/language-select/index.tsx
@@ -1,7 +1,7 @@
'use client'
import type { FC } from 'react'
import React from 'react'
-import { RiArrowDownSLine } from '@remixicon/react'
+import { RiArrowDownSLine, RiCheckLine } from '@remixicon/react'
import cn from '@/utils/classnames'
import Popover from '@/app/components/base/popover'
import { languages } from '@/i18n/language'
@@ -22,25 +22,40 @@ const LanguageSelect: FC
= ({
manualClose
trigger='click'
disabled={disabled}
+ popupClassName='z-20'
htmlContent={
-
+
{languages.filter(language => language.supported).map(({ prompt_name }) => (
onSelect(prompt_name)}>{prompt_name}
+ className='w-full py-2 px-3 inline-flex items-center justify-between hover:bg-state-base-hover rounded-lg cursor-pointer'
+ onClick={() => onSelect(prompt_name)}
+ >
+ {prompt_name}
+ {(currentLanguage === prompt_name) && }
))}
}
btnElement={
-
-
{currentLanguage}
-
+
+
+ {currentLanguage}
+
+
}
- btnClassName={open => cn('!border-0 !px-0 !py-0 !bg-inherit !hover:bg-inherit', open ? 'text-blue-600' : 'text-gray-500')}
- className='!w-[120px] h-fit !z-20 !translate-x-0 !left-[-16px]'
+ btnClassName={() => cn(
+ '!border-0 rounded-md !px-1.5 !py-1 !mx-1 !bg-components-button-tertiary-bg !hover:bg-components-button-tertiary-bg',
+ disabled ? 'bg-components-button-tertiary-bg-disabled' : '',
+ )}
+ className='!w-[140px] h-fit !z-20 !translate-x-0 !left-1'
/>
)
}
diff --git a/web/app/components/datasets/create/step-two/option-card.tsx b/web/app/components/datasets/create/step-two/option-card.tsx
new file mode 100644
index 0000000000..d0efdaabb1
--- /dev/null
+++ b/web/app/components/datasets/create/step-two/option-card.tsx
@@ -0,0 +1,98 @@
+import { type ComponentProps, type FC, type ReactNode, forwardRef } from 'react'
+import Image from 'next/image'
+import classNames from '@/utils/classnames'
+
+const TriangleArrow: FC
> = props => (
+
+)
+
+type OptionCardHeaderProps = {
+ icon: ReactNode
+ title: ReactNode
+ description: string
+ isActive?: boolean
+ activeClassName?: string
+ effectImg?: string
+}
+
+export const OptionCardHeader: FC = (props) => {
+ const { icon, title, description, isActive, activeClassName, effectImg } = props
+ return
+
+ {isActive && effectImg &&
}
+
+
+
+
+
{title}
+
{description}
+
+
{
+ if (!isActive && !disabled)
+ onSwitched?.()
+ }}
+ {...rest}
+ ref={ref}
+ >
+
+ {/** Body */}
+ {isActive && (children || actions) &&
+ {children}
+ {actions &&
+ {actions}
+
+ }
+
}
+
+ {steps.map((step, index) => {
+ const isLast = index === steps.length - 1
+ return (
+
+
+ {!isLast && }
+
+ )
+ })}
+
+
+
+
+
+
+ {t('datasetCreation.steps.header.creation')}
+
+
+
+ ({
+ name: t(STEP_T_MAP[i + 1]),
+ }))}
+ {...rest}
+ />
+
+
{errorMsg && (
diff --git a/web/app/components/datasets/create/website/jina-reader/index.tsx b/web/app/components/datasets/create/website/jina-reader/index.tsx
index 51d77d7121..1c133f935c 100644
--- a/web/app/components/datasets/create/website/jina-reader/index.tsx
+++ b/web/app/components/datasets/create/website/jina-reader/index.tsx
@@ -94,7 +94,6 @@ const JinaReader: FC
= ({
const waitForCrawlFinished = useCallback(async (jobId: string) => {
try {
const res = await checkJinaReaderTaskStatus(jobId) as any
- console.log('res', res)
if (res.status === 'completed') {
return {
isError: false,
diff --git a/web/app/components/datasets/create/website/preview.tsx b/web/app/components/datasets/create/website/preview.tsx
index 65abe83ed7..5180a83442 100644
--- a/web/app/components/datasets/create/website/preview.tsx
+++ b/web/app/components/datasets/create/website/preview.tsx
@@ -18,7 +18,7 @@ const WebsitePreview = ({
const { t } = useTranslation()
return (
-
+
{t('datasetCreation.stepOne.pagePreview')}
@@ -32,7 +32,7 @@ const WebsitePreview = ({
{payload.source_url}
-
{payload.markdown}
+
{payload.markdown}
)
diff --git a/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx b/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx
index 36216aa7c8..6602244a48 100644
--- a/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx
@@ -7,7 +7,7 @@ import {
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import { Download02 as DownloadIcon } from '@/app/components/base/icons/src/vender/solid/general'
-import { DocForm } from '@/models/datasets'
+import { ChunkingMode } from '@/models/datasets'
import I18n from '@/context/i18n'
import { LanguagesSupported } from '@/i18n/language'
@@ -32,18 +32,18 @@ const CSV_TEMPLATE_CN = [
['内容 2'],
]
-const CSVDownload: FC<{ docForm: DocForm }> = ({ docForm }) => {
+const CSVDownload: FC<{ docForm: ChunkingMode }> = ({ docForm }) => {
const { t } = useTranslation()
const { locale } = useContext(I18n)
const { CSVDownloader, Type } = useCSVDownloader()
const getTemplate = () => {
if (locale === LanguagesSupported[1]) {
- if (docForm === DocForm.QA)
+ if (docForm === ChunkingMode.qa)
return CSV_TEMPLATE_QA_CN
return CSV_TEMPLATE_CN
}
- if (docForm === DocForm.QA)
+ if (docForm === ChunkingMode.qa)
return CSV_TEMPLATE_QA_EN
return CSV_TEMPLATE_EN
}
@@ -52,7 +52,7 @@ const CSVDownload: FC<{ docForm: DocForm }> = ({ docForm }) => {
{t('share.generation.csvStructureTitle')}
- {docForm === DocForm.QA && (
+ {docForm === ChunkingMode.qa && (
@@ -72,7 +72,7 @@ const CSVDownload: FC<{ docForm: DocForm }> = ({ docForm }) => {
)}
- {docForm === DocForm.TEXT && (
+ {docForm === ChunkingMode.text && (
@@ -97,7 +97,7 @@ const CSVDownload: FC<{ docForm: DocForm }> = ({ docForm }) => {
bom={true}
data={getTemplate()}
>
-
+
{t('datasetDocuments.list.batchModal.template')}
diff --git a/web/app/components/datasets/documents/detail/batch-modal/index.tsx b/web/app/components/datasets/documents/detail/batch-modal/index.tsx
index 139a364cb4..c666ba6715 100644
--- a/web/app/components/datasets/documents/detail/batch-modal/index.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/index.tsx
@@ -7,11 +7,11 @@ import CSVUploader from './csv-uploader'
import CSVDownloader from './csv-downloader'
import Button from '@/app/components/base/button'
import Modal from '@/app/components/base/modal'
-import type { DocForm } from '@/models/datasets'
+import type { ChunkingMode } from '@/models/datasets'
export type IBatchModalProps = {
isShow: boolean
- docForm: DocForm
+ docForm: ChunkingMode
onCancel: () => void
onConfirm: (file: File) => void
}
diff --git a/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx b/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx
deleted file mode 100644
index 7b510bcf21..0000000000
--- a/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx
+++ /dev/null
@@ -1,98 +0,0 @@
-import type { CSSProperties, FC } from 'react'
-import React from 'react'
-import { FixedSizeList as List } from 'react-window'
-import InfiniteLoader from 'react-window-infinite-loader'
-import SegmentCard from './SegmentCard'
-import s from './style.module.css'
-import type { SegmentDetailModel } from '@/models/datasets'
-
-type IInfiniteVirtualListProps = {
- hasNextPage?: boolean // Are there more items to load? (This information comes from the most recent API request.)
- isNextPageLoading: boolean // Are we currently loading a page of items? (This may be an in-flight flag in your Redux store for example.)
- items: Array
// Array of items loaded so far.
- loadNextPage: () => Promise // Callback function responsible for loading the next page of items.
- onClick: (detail: SegmentDetailModel) => void
- onChangeSwitch: (segId: string, enabled: boolean) => Promise
- onDelete: (segId: string) => Promise
- archived?: boolean
- embeddingAvailable: boolean
-}
-
-const InfiniteVirtualList: FC = ({
- hasNextPage,
- isNextPageLoading,
- items,
- loadNextPage,
- onClick: onClickCard,
- onChangeSwitch,
- onDelete,
- archived,
- embeddingAvailable,
-}) => {
- // If there are more items to be loaded then add an extra row to hold a loading indicator.
- const itemCount = hasNextPage ? items.length + 1 : items.length
-
- // Only load 1 page of items at a time.
- // Pass an empty callback to InfiniteLoader in case it asks us to load more than once.
- const loadMoreItems = isNextPageLoading ? () => { } : loadNextPage
-
- // Every row is loaded except for our loading indicator row.
- const isItemLoaded = (index: number) => !hasNextPage || index < items.length
-
- // Render an item or a loading indicator.
- const Item = ({ index, style }: { index: number; style: CSSProperties }) => {
- let content
- if (!isItemLoaded(index)) {
- content = (
- <>
- {[1, 2, 3].map(v => (
-
- ))}
-
- )
- }
- else {
- content = items[index].map(segItem => (
- onClickCard(segItem)}
- onChangeSwitch={onChangeSwitch}
- onDelete={onDelete}
- loading={false}
- archived={archived}
- embeddingAvailable={embeddingAvailable}
- />
- ))
- }
-
- return (
-
- {content}
-
- )
- }
-
- return (
-
- {({ onItemsRendered, ref }) => (
-
- {Item}
-
- )}
-
- )
-}
-export default InfiniteVirtualList
diff --git a/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx b/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx
index 5b76acc936..264d62b68a 100644
--- a/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx
+++ b/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx
@@ -6,9 +6,9 @@ import {
RiDeleteBinLine,
} from '@remixicon/react'
import { StatusItem } from '../../list'
-import { DocumentTitle } from '../index'
+import style from '../../style.module.css'
import s from './style.module.css'
-import { SegmentIndexTag } from './index'
+import { SegmentIndexTag } from './common/segment-index-tag'
import cn from '@/utils/classnames'
import Confirm from '@/app/components/base/confirm'
import Switch from '@/app/components/base/switch'
@@ -31,6 +31,22 @@ const ProgressBar: FC<{ percent: number; loading: boolean }> = ({ percent, loadi
)
}
+type DocumentTitleProps = {
+ extension?: string
+ name?: string
+ iconCls?: string
+ textCls?: string
+ wrapperCls?: string
+}
+
+const DocumentTitle: FC = ({ extension, name, iconCls, textCls, wrapperCls }) => {
+ const localExtension = extension?.toLowerCase() || name?.split('.')?.pop()?.toLowerCase()
+ return
+}
+
export type UsageScene = 'doc' | 'hitTesting'
type ISegmentCardProps = {
diff --git a/web/app/components/datasets/documents/detail/completed/child-segment-detail.tsx b/web/app/components/datasets/documents/detail/completed/child-segment-detail.tsx
new file mode 100644
index 0000000000..085bfddc16
--- /dev/null
+++ b/web/app/components/datasets/documents/detail/completed/child-segment-detail.tsx
@@ -0,0 +1,134 @@
+import React, { type FC, useMemo, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+ RiCloseLine,
+ RiExpandDiagonalLine,
+} from '@remixicon/react'
+import ActionButtons from './common/action-buttons'
+import ChunkContent from './common/chunk-content'
+import Dot from './common/dot'
+import { SegmentIndexTag } from './common/segment-index-tag'
+import { useSegmentListContext } from './index'
+import type { ChildChunkDetail, ChunkingMode } from '@/models/datasets'
+import { useEventEmitterContextContext } from '@/context/event-emitter'
+import { formatNumber } from '@/utils/format'
+import classNames from '@/utils/classnames'
+import Divider from '@/app/components/base/divider'
+import { formatTime } from '@/utils/time'
+
+type IChildSegmentDetailProps = {
+ chunkId: string
+ childChunkInfo?: Partial & { id: string }
+ onUpdate: (segmentId: string, childChunkId: string, content: string) => void
+ onCancel: () => void
+ docForm: ChunkingMode
+}
+
+/**
+ * Show all the contents of the segment
+ */
+const ChildSegmentDetail: FC = ({
+ chunkId,
+ childChunkInfo,
+ onUpdate,
+ onCancel,
+ docForm,
+}) => {
+ const { t } = useTranslation()
+ const [content, setContent] = useState(childChunkInfo?.content || '')
+ const { eventEmitter } = useEventEmitterContextContext()
+ const [loading, setLoading] = useState(false)
+ const fullScreen = useSegmentListContext(s => s.fullScreen)
+ const toggleFullScreen = useSegmentListContext(s => s.toggleFullScreen)
+
+ eventEmitter?.useSubscription((v) => {
+ if (v === 'update-child-segment')
+ setLoading(true)
+ if (v === 'update-child-segment-done')
+ setLoading(false)
+ })
+
+ const handleCancel = () => {
+ onCancel()
+ setContent(childChunkInfo?.content || '')
+ }
+
+ const handleSave = () => {
+ onUpdate(chunkId, childChunkInfo?.id || '', content)
+ }
+
+ const wordCountText = useMemo(() => {
+ const count = content.length
+ return `${formatNumber(count)} ${t('datasetDocuments.segment.characters', { count })}`
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [content.length])
+
+ const EditTimeText = useMemo(() => {
+ const timeText = formatTime({
+ date: (childChunkInfo?.updated_at ?? 0) * 1000,
+ dateFormat: 'MM/DD/YYYY h:mm:ss',
+ })
+ return `${t('datasetDocuments.segment.editedAt')} ${timeText}`
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [childChunkInfo?.updated_at])
+
+ return (
+
+
+
+
{t('datasetDocuments.segment.editChildChunk')}
+
+
+
+ {wordCountText}
+
+
+ {EditTimeText}
+
+
+
+
+ {fullScreen && (
+ <>
+
+
+
+ )}
+
+
+
+
+
+
+
+
+
+
+ setContent(content)}
+ isEditMode={true}
+ />
+
+
+ {!fullScreen && (
+
+ )}
+
+ {isFullDocMode ?
: null}
+
+
{
+ event.stopPropagation()
+ toggleCollapse()
+ }}
+ >
+ {
+ isParagraphMode
+ ? collapsed
+ ? (
+
+ )
+ : ()
+ : null
+ }
+ {totalText}
+ ·
+
+
+ {isFullDocMode
+ ?
handleInputChange?.(e.target.value)}
+ onClear={() => handleInputChange?.('')}
+ />
+ : null}
+
+ {isLoading ?
: null}
+ {((isFullDocMode && !isLoading) || !collapsed)
+ ?
+ {isParagraphMode && (
+
+ )}
+ {childChunks.length > 0
+ ?
+ {childChunks.map((childChunk) => {
+ const edited = childChunk.updated_at !== childChunk.created_at
+ const focused = currChildChunk?.childChunkInfo?.id === childChunk.id
+ return onDelete?.(childChunk.segment_id, childChunk.id)}
+ labelClassName={focused ? 'bg-state-accent-solid text-text-primary-on-surface' : ''}
+ labelInnerClassName={'text-[10px] font-semibold align-bottom leading-6'}
+ contentClassName={classNames('!leading-6', focused ? 'bg-state-accent-hover-alt text-text-primary' : '')}
+ showDivider={false}
+ onClick={(e) => {
+ e.stopPropagation()
+ onClickSlice?.(childChunk)
+ }}
+ offsetOptions={({ rects }) => {
+ return {
+ mainAxis: isFullDocMode ? -rects.floating.width : 12 - rects.floating.width,
+ crossAxis: (20 - rects.floating.height) / 2,
+ }
+ }}
+ />
+ })}
+
+ : inputValue !== ''
+ ?
+
+
+ : null
+ }
+
+ : null}
+
+
+ {(isParentChildParagraphMode && actionType === 'edit' && !isChildChunk)
+ ?
+ : null
+ }
+
+
+
+ {t('datasetDocuments.segment.addAnother')}
+
+ )
+}
+
+export default React.memo(AddAnother)
diff --git a/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx b/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx
new file mode 100644
index 0000000000..3dd3689b64
--- /dev/null
+++ b/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx
@@ -0,0 +1,103 @@
+import React, { type FC } from 'react'
+import { RiArchive2Line, RiCheckboxCircleLine, RiCloseCircleLine, RiDeleteBinLine } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import { useBoolean } from 'ahooks'
+import Divider from '@/app/components/base/divider'
+import classNames from '@/utils/classnames'
+import Confirm from '@/app/components/base/confirm'
+
+const i18nPrefix = 'dataset.batchAction'
+type IBatchActionProps = {
+ className?: string
+ selectedIds: string[]
+ onBatchEnable: () => void
+ onBatchDisable: () => void
+ onBatchDelete: () => Promise
+ onArchive?: () => void
+ onCancel: () => void
+}
+
+const BatchAction: FC = ({
+ className,
+ selectedIds,
+ onBatchEnable,
+ onBatchDisable,
+ onArchive,
+ onBatchDelete,
+ onCancel,
+}) => {
+ const { t } = useTranslation()
+ const [isShowDeleteConfirm, {
+ setTrue: showDeleteConfirm,
+ setFalse: hideDeleteConfirm,
+ }] = useBoolean(false)
+ const [isDeleting, {
+ setTrue: setIsDeleting,
+ }] = useBoolean(false)
+
+ const handleBatchDelete = async () => {
+ setIsDeleting()
+ await onBatchDelete()
+ hideDeleteConfirm()
+ }
+ return (
+
+
+
+
+ {selectedIds.length}
+
+ {t(`${i18nPrefix}.selected`)}
+
+
+
+
+
+
+
+
+
+
+ {onArchive && (
+
+
+
+
+ )}
+
+
+
+
+
+
+
+
+ {
+ isShowDeleteConfirm && (
+
+ )
+ }
+
+
QUESTION
+
onQuestionChange(e.target.value)}
+ disabled={!isEditMode}
+ containerRef={containerRef}
+ labelRef={labelRef}
+ />
+ ANSWER
+ onAnswerChange?.(e.target.value)}
+ disabled={!isEditMode}
+ autoFocus
+ containerRef={containerRef}
+ labelRef={labelRef}
+ />
+