diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 6884e7fe80..6c73cb255a 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -148,14 +148,28 @@ class DatasetApi(Resource): dataset = DatasetService.get_dataset(dataset_id_str) if dataset is None: raise NotFound("Dataset not found.") - try: DatasetService.check_dataset_permission( dataset, current_user) except services.errors.account.NoPermissionError as e: raise Forbidden(str(e)) - - return marshal(dataset, dataset_detail_fields), 200 + data = marshal(dataset, dataset_detail_fields) + # check embedding setting + provider_service = ProviderService() + # get valid model list + valid_model_list = provider_service.get_valid_model_list(current_user.current_tenant_id, ModelType.EMBEDDINGS.value) + model_names = [] + for valid_model in valid_model_list: + model_names.append(f"{valid_model['model_name']}:{valid_model['model_provider']['provider_name']}") + if data['indexing_technique'] == 'high_quality': + item_model = f"{data['embedding_model']}:{data['embedding_model_provider']}" + if item_model in model_names: + data['embedding_available'] = True + else: + data['embedding_available'] = False + else: + data['embedding_available'] = True + return data, 200 @setup_required @login_required diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index d4bc8d833a..a40476a0e4 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -137,28 +137,31 @@ class DatasetService: @staticmethod def update_dataset(dataset_id, data, user): + filtered_data = {k: v for k, v in data.items() if v is not None or k == 'description'} dataset = DatasetService.get_dataset(dataset_id) DatasetService.check_dataset_permission(dataset, user) + action = None if dataset.indexing_technique != data['indexing_technique']: # if update indexing_technique if data['indexing_technique'] == 'economy': - deal_dataset_vector_index_task.delay(dataset_id, 'remove') + action = 'remove' + filtered_data['embedding_model'] = None + filtered_data['embedding_model_provider'] = None elif data['indexing_technique'] == 'high_quality': - # check embedding model setting + action = 'add' + # get embedding model setting try: - ModelFactory.get_embedding_model( - tenant_id=current_user.current_tenant_id, - model_provider_name=dataset.embedding_model_provider, - model_name=dataset.embedding_model + embedding_model = ModelFactory.get_embedding_model( + tenant_id=current_user.current_tenant_id ) + filtered_data['embedding_model'] = embedding_model.name + filtered_data['embedding_model_provider'] = embedding_model.model_provider.provider_name except LLMBadRequestError: raise ValueError( f"No Embedding Model available. Please configure a valid provider " f"in the Settings -> Model Provider.") except ProviderTokenNotInitError as ex: raise ValueError(ex.description) - deal_dataset_vector_index_task.delay(dataset_id, 'add') - filtered_data = {k: v for k, v in data.items() if v is not None or k == 'description'} filtered_data['updated_by'] = user.id filtered_data['updated_at'] = datetime.datetime.now() @@ -166,7 +169,8 @@ class DatasetService: dataset.query.filter_by(id=dataset_id).update(filtered_data) db.session.commit() - + if action: + deal_dataset_vector_index_task.delay(dataset_id, action) return dataset @staticmethod diff --git a/web/app/components/app/configuration/dataset-config/card-item/index.tsx b/web/app/components/app/configuration/dataset-config/card-item/index.tsx index 0f5721fd65..4d9ebc9f37 100644 --- a/web/app/components/app/configuration/dataset-config/card-item/index.tsx +++ b/web/app/components/app/configuration/dataset-config/card-item/index.tsx @@ -43,7 +43,7 @@ const CardItem: FC = ({ selector={`unavailable-tag-${config.id}`} htmlContent={t('dataset.unavailableTip')} > - {t('dataset.unavailable')} + {t('dataset.unavailable')} )} diff --git a/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx b/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx index 92dca3f966..7b510bcf21 100644 --- a/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx +++ b/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx @@ -15,7 +15,7 @@ type IInfiniteVirtualListProps = { onChangeSwitch: (segId: string, enabled: boolean) => Promise onDelete: (segId: string) => Promise archived?: boolean - + embeddingAvailable: boolean } const InfiniteVirtualList: FC = ({ @@ -27,6 +27,7 @@ const InfiniteVirtualList: FC = ({ onChangeSwitch, onDelete, archived, + embeddingAvailable, }) => { // If there are more items to be loaded then add an extra row to hold a loading indicator. const itemCount = hasNextPage ? items.length + 1 : items.length @@ -45,7 +46,7 @@ const InfiniteVirtualList: FC = ({ content = ( <> {[1, 2, 3].map(v => ( - + ))} ) @@ -60,6 +61,7 @@ const InfiniteVirtualList: FC = ({ onDelete={onDelete} loading={false} archived={archived} + embeddingAvailable={embeddingAvailable} /> )) } diff --git a/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx b/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx index a1fab8faad..0d8ca4acbb 100644 --- a/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx +++ b/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx @@ -43,6 +43,7 @@ type ISegmentCardProps = { scene?: UsageScene className?: string archived?: boolean + embeddingAvailable: boolean } const SegmentCard: FC = ({ @@ -55,6 +56,7 @@ const SegmentCard: FC = ({ scene = 'doc', className = '', archived, + embeddingAvailable, }) => { const { t } = useTranslation() const { @@ -115,24 +117,26 @@ const SegmentCard: FC = ({ : ( <> -
- -
) => - e.stopPropagation() - } - className="inline-flex items-center" - > - { - await onChangeSwitch?.(id, val) - }} - /> + {embeddingAvailable && ( +
+ +
) => + e.stopPropagation() + } + className="inline-flex items-center" + > + { + await onChangeSwitch?.(id, val) + }} + /> +
-
+ )} )}
@@ -173,7 +177,7 @@ const SegmentCard: FC = ({
{index_node_hash}
- {!archived && ( + {!archived && embeddingAvailable && (
{ e.stopPropagation() setShowModal(true) diff --git a/web/app/components/datasets/documents/detail/completed/index.tsx b/web/app/components/datasets/documents/detail/completed/index.tsx index fb629f38f5..fe1acbc864 100644 --- a/web/app/components/datasets/documents/detail/completed/index.tsx +++ b/web/app/components/datasets/documents/detail/completed/index.tsx @@ -46,6 +46,7 @@ export const SegmentIndexTag: FC<{ positionId: string | number; className?: stri } type ISegmentDetailProps = { + embeddingAvailable: boolean segInfo?: Partial & { id: string } onChangeSwitch?: (segId: string, enabled: boolean) => Promise onUpdate: (segmentId: string, q: string, a: string, k: string[]) => void @@ -56,6 +57,7 @@ type ISegmentDetailProps = { * Show all the contents of the segment */ const SegmentDetailComponent: FC = ({ + embeddingAvailable, segInfo, archived, onChangeSwitch, @@ -146,7 +148,7 @@ const SegmentDetailComponent: FC = ({ )} - {!isEditing && !archived && ( + {!isEditing && !archived && embeddingAvailable && ( <>
{t('common.operation.edit')}
@@ -183,15 +185,19 @@ const SegmentDetailComponent: FC = ({
- - { - await onChangeSwitch?.(segInfo?.id || '', val) - }} - disabled={archived} - /> + {embeddingAvailable && ( + <> + + { + await onChangeSwitch?.(segInfo?.id || '', val) + }} + disabled={archived} + /> + + )}
@@ -209,6 +215,7 @@ export const splitArray = (arr: any[], size = 3) => { } type ICompletedProps = { + embeddingAvailable: boolean showNewSegmentModal: boolean onNewSegmentModalChange: (state: boolean) => void importStatus: ProcessStatus | string | undefined @@ -220,6 +227,7 @@ type ICompletedProps = { * Support search and filter */ const Completed: FC = ({ + embeddingAvailable, showNewSegmentModal, onNewSegmentModalChange, importStatus, @@ -384,6 +392,7 @@ const Completed: FC = ({ = ({ /> {}} className='!max-w-[640px] !overflow-visible'> ({ docForm: '' }) @@ -50,6 +51,8 @@ const DocumentDetail: FC = ({ datasetId, documentId }) => { const router = useRouter() const { t } = useTranslation() const { notify } = useContext(ToastContext) + const { dataset } = useDatasetDetailContext() + const embeddingAvailable = !!dataset?.embedding_available const [showMetadata, setShowMetadata] = useState(true) const [newSegmentModalVisible, setNewSegmentModalVisible] = useState(false) const [batchModalVisible, setBatchModalVisible] = useState(false) @@ -128,7 +131,7 @@ const DocumentDetail: FC = ({ datasetId, documentId }) => { - {documentDetail && !documentDetail.archived && ( + {embeddingAvailable && documentDetail && !documentDetail.archived && ( = ({ datasetId, documentId }) => { )} = ({ datasetId, documentId }) => { {embedding ? : ) => { } -const EmptyElement: FC<{ onClick: () => void; type?: 'upload' | 'sync' }> = ({ onClick, type = 'upload' }) => { +const EmptyElement: FC<{ canAdd: boolean; onClick: () => void; type?: 'upload' | 'sync' }> = ({ canAdd = true, onClick, type = 'upload' }) => { const { t } = useTranslation() return
@@ -62,7 +62,7 @@ const EmptyElement: FC<{ onClick: () => void; type?: 'upload' | 'sync' }> = ({ o
{t(`datasetDocuments.list.empty.${type}.tip`)}
- {type === 'upload' && }
@@ -84,6 +84,7 @@ const Documents: FC = ({ datasetId }) => { const [notionPageSelectorModalVisible, setNotionPageSelectorModalVisible] = useState(false) const [timerCanRun, setTimerCanRun] = useState(true) const isDataSourceNotion = dataset?.data_source_type === DataSourceType.NOTION + const embeddingAvailable = !!dataset?.embedding_available const query = useMemo(() => { return { page: currPage + 1, limit, keyword: searchValue, fetch: isDataSourceNotion ? true : '' } @@ -205,20 +206,19 @@ const Documents: FC = ({ datasetId }) => { onChange={debounce(setSearchValue, 500)} value={searchValue} /> - + {embeddingAvailable && ( + + )}
{isLoading ? : total > 0 - ? - : + ? + : } {/* Show Pagination only if the total is more than the limit */} {(total && total > limit) diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index 8dbf84d1e6..748b17ac76 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -103,6 +103,7 @@ type OperationName = 'delete' | 'archive' | 'enable' | 'disable' | 'sync' | 'un_ // operation action for list and detail export const OperationAction: FC<{ + embeddingAvailable: boolean detail: { enabled: boolean archived: boolean @@ -114,7 +115,7 @@ export const OperationAction: FC<{ onUpdate: (operationName?: string) => void scene?: 'list' | 'detail' className?: string -}> = ({ datasetId, detail, onUpdate, scene = 'list', className = '' }) => { +}> = ({ embeddingAvailable, datasetId, detail, onUpdate, scene = 'list', className = '' }) => { const { id, enabled = false, archived = false, data_source_type } = detail || {} const [showModal, setShowModal] = useState(false) const { notify } = useContext(ToastContext) @@ -154,87 +155,94 @@ export const OperationAction: FC<{ } return
e.stopPropagation()}> - {isListScene && <> - {archived - ? -
- { }} disabled={true} size='md' /> -
- - : onOperate(v ? 'enable' : 'disable')} size='md' /> - } - - } - - {!isListScene && <> -
- - {!archived && enabled ? t('datasetDocuments.list.index.enable') : t('datasetDocuments.list.index.disable')} - - -
- !archived && onOperate(v ? 'enable' : 'disable')} - disabled={archived} - size='md' - /> -
- + {isListScene && !embeddingAvailable && ( + { }} disabled={true} size='md' /> + )} + {isListScene && embeddingAvailable && ( + <> + {archived + ? +
+ { }} disabled={true} size='md' />
-
- {!archived && enabled ? t('datasetDocuments.list.index.enableTip') : t('datasetDocuments.list.index.disableTip')} -
- - } - {!archived && ( - <> -
router.push(`/datasets/${datasetId}/documents/${detail.id}/settings`)}> - - {t('datasetDocuments.list.action.settings')} + + : onOperate(v ? 'enable' : 'disable')} size='md' /> + } + + + )} + {embeddingAvailable && ( + + {!isListScene && <> +
+ + {!archived && enabled ? t('datasetDocuments.list.index.enable') : t('datasetDocuments.list.index.disable')} + + +
+ !archived && onOperate(v ? 'enable' : 'disable')} + disabled={archived} + size='md' + /> +
+
- {data_source_type === 'notion_import' && ( -
onOperate('sync')}> - - {t('datasetDocuments.list.action.sync')} +
+ {!archived && enabled ? t('datasetDocuments.list.index.enableTip') : t('datasetDocuments.list.index.disableTip')} +
+ + } + {!archived && ( + <> +
router.push(`/datasets/${datasetId}/documents/${detail.id}/settings`)}> + + {t('datasetDocuments.list.action.settings')}
- )} - - - )} - {!archived &&
onOperate('archive')}> - - {t('datasetDocuments.list.action.archive')} -
} - {archived && ( -
onOperate('un_archive')}> + {data_source_type === 'notion_import' && ( +
onOperate('sync')}> + + {t('datasetDocuments.list.action.sync')} +
+ )} + + + )} + {!archived &&
onOperate('archive')}> - {t('datasetDocuments.list.action.unarchive')} + {t('datasetDocuments.list.action.archive')} +
} + {archived && ( +
onOperate('un_archive')}> + + {t('datasetDocuments.list.action.unarchive')} +
+ )} +
setShowModal(true)}> + + {t('datasetDocuments.list.action.delete')}
- )} -
setShowModal(true)}> - - {t('datasetDocuments.list.action.delete')}
-
- } - trigger='click' - position='br' - btnElement={ -
- -
- } - btnClassName={open => cn(isListScene ? s.actionIconWrapperList : s.actionIconWrapperDetail, open ? '!bg-gray-100 !shadow-none' : '!bg-transparent')} - className={`!w-[200px] h-fit !z-20 ${className}`} - /> + } + trigger='click' + position='br' + btnElement={ +
+ +
+ } + btnClassName={open => cn(isListScene ? s.actionIconWrapperList : s.actionIconWrapperDetail, open ? '!bg-gray-100 !shadow-none' : '!bg-transparent')} + className={`!w-[200px] h-fit !z-20 ${className}`} + /> + )} {showModal && setShowModal(false)} className={s.delModal} closable>
@@ -277,6 +285,7 @@ const renderCount = (count: number | undefined) => { type LocalDoc = SimpleDocumentDetail & { percent?: number } type IDocumentListProps = { + embeddingAvailable: boolean documents: LocalDoc[] datasetId: string onUpdate: () => void @@ -285,7 +294,7 @@ type IDocumentListProps = { /** * Document list component including basic information */ -const DocumentList: FC = ({ documents = [], datasetId, onUpdate }) => { +const DocumentList: FC = ({ embeddingAvailable, documents = [], datasetId, onUpdate }) => { const { t } = useTranslation() const router = useRouter() const [localDocs, setLocalDocs] = useState(documents) @@ -361,6 +370,7 @@ const DocumentList: FC = ({ documents = [], datasetId, onUpd { const HitTesting: FC = ({ datasetId }: Props) => { const { t } = useTranslation() - const [hitResult, setHitResult] = useState(); // 初始化记录为空数组 - const [submitLoading, setSubmitLoading] = useState(false); - const [currParagraph, setCurrParagraph] = useState<{ paraInfo?: HitTesting; showModal: boolean }>({ showModal: false }) - const [text, setText] = useState(''); + const [hitResult, setHitResult] = useState() // 初始化记录为空数组 + const [submitLoading, setSubmitLoading] = useState(false) + const [currParagraph, setCurrParagraph] = useState<{ paraInfo?: HitTestingType; showModal: boolean }>({ showModal: false }) + const [text, setText] = useState('') const [currPage, setCurrPage] = React.useState(0) const { data: recordsRes, error, mutate: recordsMutate } = useSWR({ action: 'fetchTestingRecords', datasetId, - params: { limit, page: currPage + 1, } + params: { limit, page: currPage + 1 }, }, apiParams => fetchTestingRecords(omit(apiParams, 'action'))) const total = recordsRes?.total || 0 - const points = useMemo(() => (hitResult?.records.map((v) => [v.tsne_position.x, v.tsne_position.y]) || []), [hitResult?.records]) + const points = useMemo(() => (hitResult?.records.map(v => [v.tsne_position.x, v.tsne_position.y]) || []), [hitResult?.records]) - const onClickCard = (detail: HitTesting) => { + const onClickCard = (detail: HitTestingType) => { setCurrParagraph({ paraInfo: detail, showModal: true }) } @@ -71,50 +72,56 @@ const HitTesting: FC = ({ datasetId }: Props) => { text={text} />
{t('datasetHitTesting.recents')}
- {!recordsRes && !error ? ( -
- ) : recordsRes?.data?.length ? ( - <> - - - - - - - - - - {recordsRes?.data?.map((record) => { - return setText(record.content)} - > - - - - - })} - -
{t('datasetHitTesting.table.header.source')}{t('datasetHitTesting.table.header.text')}{t('datasetHitTesting.table.header.time')}
-
-
- {record.source.replace('_', ' ')} -
-
{record.content} - {dayjs.unix(record.created_at).format(t('datasetHitTesting.dateTimeFormat') as string)} -
- {(total && total > limit) - ? - : null} - - ) : ( - - )} + {(!recordsRes && !error) + ? ( +
+ ) + : recordsRes?.data?.length + ? ( + <> +
+ + + + + + + + + + {recordsRes?.data?.map((record) => { + return setText(record.content)} + > + + + + + })} + +
{t('datasetHitTesting.table.header.source')}{t('datasetHitTesting.table.header.text')}{t('datasetHitTesting.table.header.time')}
+
+
+ {record.source.replace('_', ' ')} +
+
{record.content} + {dayjs.unix(record.created_at).format(t('datasetHitTesting.dateTimeFormat') as string)} +
+
+ {(total && total > limit) + ? + : null} + + ) + : ( + + )}
- {submitLoading ? -
+ {submitLoading + ?
= ({ datasetId }: Props) => { scene='hitTesting' className='h-[216px]' /> -
: !hitResult?.records.length ? ( -
-
-
- {t('datasetHitTesting.hit.emptyTip')} -
-
- ) : ( - <> -
{t('datasetHitTesting.hit.title')}
-
-
- {hitResult?.records.map((record, idx) => { - return onClickCard(record as any)} - /> - })} +
+ : !hitResult?.records.length + ? ( +
+
+
+ {t('datasetHitTesting.hit.emptyTip')}
- - ) + ) + : ( + <> +
{t('datasetHitTesting.hit.title')}
+
+
+ {hitResult?.records.map((record, idx) => { + return onClickCard(record as any)} + /> + })} +
+
+ + ) }
{t('datasetSettings.form.name')}
setName(e.target.value)} /> @@ -99,7 +101,8 @@ const Form = ({