diff --git a/web/app/components/app/configuration/dataset-config/card-item/item.tsx b/web/app/components/app/configuration/dataset-config/card-item/item.tsx index b9f1ced38e..184a598e10 100644 --- a/web/app/components/app/configuration/dataset-config/card-item/item.tsx +++ b/web/app/components/app/configuration/dataset-config/card-item/item.tsx @@ -1,18 +1,20 @@ 'use client' import type { FC } from 'react' import React, { useState } from 'react' -import { useTranslation } from 'react-i18next' -import { RiDeleteBinLine } from '@remixicon/react' +import { + RiDeleteBinLine, + RiEditLine, +} from '@remixicon/react' import SettingsModal from '../settings-modal' import type { DataSet } from '@/models/datasets' import { DataSourceType } from '@/models/datasets' -import { formatNumber } from '@/utils/format' import FileIcon from '@/app/components/base/file-icon' -import { Settings01 } from '@/app/components/base/icons/src/vender/line/general' import { Folder } from '@/app/components/base/icons/src/vender/solid/files' import { Globe06 } from '@/app/components/base/icons/src/vender/solid/mapsAndTravel' import Drawer from '@/app/components/base/drawer' import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' +import Badge from '@/app/components/base/badge' +import { useKnowledge } from '@/hooks/use-knowledge' type ItemProps = { className?: string @@ -27,12 +29,10 @@ const Item: FC = ({ onSave, onRemove, }) => { - const { t } = useTranslation() - const media = useBreakpoints() const isMobile = media === MediaType.mobile - const [showSettingsModal, setShowSettingsModal] = useState(false) + const { formatIndexingTechniqueAndMethod } = useKnowledge() const handleSave = (newDataset: DataSet) => { onSave(newDataset) @@ -65,22 +65,17 @@ const Item: FC = ({
{config.name}
-
- {formatNumber(config.word_count)} {t('appDebug.feature.dataSet.words')} · {formatNumber(config.document_count)} {t('appDebug.feature.dataSet.textBlocks')} -
+
- {/* { - config.description && ( -
{config.description}
- ) - } */}
setShowSettingsModal(true)} > - +
{ const handleSave = (newDataset: DataSet) => { const index = dataSet.findIndex(item => item.id === newDataset.id) - setDataSet([...dataSet.slice(0, index), newDataset, ...dataSet.slice(index + 1)]) + const newDatasets = [...dataSet.slice(0, index), newDataset, ...dataSet.slice(index + 1)] + setDataSet(newDatasets) formattingChangedDispatcher() } @@ -74,7 +75,7 @@ const DatasetConfig: FC = () => { title={t('appDebug.feature.dataSet.title')} headerRight={
- {!isAgent && } + {!isAgent && }
} diff --git a/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx b/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx index 640c78d202..d71d26bbed 100644 --- a/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx +++ b/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx @@ -1,10 +1,12 @@ 'use client' -import React from 'react' + +import { memo, useMemo } from 'react' import type { FC } from 'react' import { useTranslation } from 'react-i18next' import { RiQuestionLine, } from '@remixicon/react' +import WeightedScore from './weighted-score' import TopKItem from '@/app/components/base/param-item/top-k-item' import ScoreThresholdItem from '@/app/components/base/param-item/score-threshold-item' import RadioCard from '@/app/components/base/radio-card/simple' @@ -16,13 +18,20 @@ import { import type { DatasetConfigs, } from '@/models/debug' - import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector' import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import type { ModelConfig } from '@/app/components/workflow/types' import ModelParameterModal from '@/app/components/header/account-setting/model-provider-page/model-parameter-modal' import TooltipPlus from '@/app/components/base/tooltip-plus' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import type { + DataSet, + WeightedScoreEnum, +} from '@/models/datasets' +import { RerankingModeEnum } from '@/models/datasets' +import cn from '@/utils/classnames' +import { useSelectedDatasetsMode } from '@/app/components/workflow/nodes/knowledge-retrieval/hooks' +import Switch from '@/app/components/base/switch' type Props = { datasetConfigs: DatasetConfigs @@ -31,6 +40,7 @@ type Props = { singleRetrievalModelConfig?: ModelConfig onSingleRetrievalModelChange?: (config: ModelConfig) => void onSingleRetrievalModelParamsChange?: (config: ModelConfig) => void + selectedDatasets?: DataSet[] } const ConfigContent: FC = ({ @@ -40,8 +50,10 @@ const ConfigContent: FC = ({ singleRetrievalModelConfig: singleRetrievalConfig = {} as ModelConfig, onSingleRetrievalModelChange = () => { }, onSingleRetrievalModelParamsChange = () => { }, + selectedDatasets = [], }) => { const { t } = useTranslation() + const selectedDatasetsMode = useSelectedDatasetsMode(selectedDatasets) const type = datasetConfigs.retrieval_model const setType = (value: RETRIEVE_TYPE) => { onChange({ @@ -54,7 +66,7 @@ const ConfigContent: FC = ({ defaultModel: rerankDefaultModel, } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank) const rerankModel = (() => { - if (datasetConfigs.reranking_model) { + if (datasetConfigs.reranking_model?.reranking_provider_name) { return { provider_name: datasetConfigs.reranking_model.reranking_provider_name, model_name: datasetConfigs.reranking_model.reranking_model_name, @@ -93,14 +105,73 @@ const ConfigContent: FC = ({ }) } + const handleWeightedScoreChange = (value: { type: WeightedScoreEnum; value: number[] }) => { + const configs = { + ...datasetConfigs, + weights: { + ...datasetConfigs.weights!, + weight_type: value.type, + vector_setting: { + ...datasetConfigs.weights!.vector_setting!, + vector_weight: value.value[0], + }, + keyword_setting: { + keyword_weight: value.value[1], + }, + }, + } + onChange(configs) + } + + const handleRerankModeChange = (mode: RerankingModeEnum) => { + onChange({ + ...datasetConfigs, + reranking_mode: mode, + }) + } + const model = singleRetrievalConfig + const rerankingModeOptions = [ + { + value: RerankingModeEnum.WeightedScore, + label: t('dataset.weightedScore.title'), + tips: t('dataset.weightedScore.description'), + }, + { + value: RerankingModeEnum.RerankingModel, + label: t('common.modelProvider.rerankModel.key'), + tips: t('common.modelProvider.rerankModel.tip'), + }, + ] + + const showWeightedScore = selectedDatasetsMode.allHighQuality + && !selectedDatasetsMode.inconsistentEmbeddingModel + + const showWeightedScorePanel = showWeightedScore && datasetConfigs.reranking_mode === RerankingModeEnum.WeightedScore && datasetConfigs.weights + const selectedRerankMode = datasetConfigs.reranking_mode || RerankingModeEnum.RerankingModel + + const showRerankModel = useMemo(() => { + if (datasetConfigs.reranking_enable === false && selectedDatasetsMode.allEconomic) + return false + + return true + }, [datasetConfigs.reranking_enable, selectedDatasetsMode.allEconomic]) + return (
+
{t('dataset.retrievalSettings')}
} - title={t('appDebug.datasetConfig.retrieveOneWay.title')} + title={( +
+ {t('appDebug.datasetConfig.retrieveOneWay.title')} + {t('dataset.nTo1RetrievalLegacy')}
}> +
legacy
+ +
+ )} description={t('appDebug.datasetConfig.retrieveOneWay.description')} isChosen={type === RETRIEVE_TYPE.oneWay} onChosen={() => { setType(RETRIEVE_TYPE.oneWay) }} @@ -115,43 +186,152 @@ const ConfigContent: FC = ({
{type === RETRIEVE_TYPE.multiWay && ( <> -
-
{t('common.modelProvider.rerankModel.key')}
-
- { - onChange({ - ...datasetConfigs, - reranking_model: { - reranking_provider_name: v.provider, - reranking_model_name: v.model, - }, - }) - }} - modelList={rerankModelList} - /> -
-
-
- - +
+
+ {t('dataset.rerankSettings')}
+ { + selectedDatasetsMode.inconsistentEmbeddingModel + && ( +
+ {t('dataset.inconsistentEmbeddingModelTip')} +
+ ) + } + { + selectedDatasetsMode.mixtureHighQualityAndEconomic + && ( +
+ {t('dataset.mixtureHighQualityAndEconomicTip')} +
+ ) + } + { + showWeightedScore && ( +
+ { + rerankingModeOptions.map(option => ( +
handleRerankModeChange(option.value)} + > +
{option.label}
+ {option.tips}
} + hideArrow + > + + +
+ )) + } +
+ ) + } + { + !showWeightedScorePanel && ( +
+
+ { + selectedDatasetsMode.allEconomic && ( + { + onChange({ + ...datasetConfigs, + reranking_enable: v, + }) + }} + /> + ) + } +
{t('common.modelProvider.rerankModel.key')}
+ {t('common.modelProvider.rerankModel.tip')}
}> + + +
+
+ { + onChange({ + ...datasetConfigs, + reranking_model: { + reranking_provider_name: v.provider, + reranking_model_name: v.model, + }, + }) + }} + modelList={rerankModelList} + /> +
+
+ ) + } + { + showWeightedScorePanel + && ( +
+ + + +
+ ) + } + { + !showWeightedScorePanel + && ( +
+ + { + showRerankModel && ( + + ) + } +
+ ) + } )} {isInWorkflow && type === RETRIEVE_TYPE.oneWay && ( -
+
{t('common.modelProvider.systemReasoningModel.key')}
= ({
) } -export default React.memo(ConfigContent) +export default memo(ConfigContent) diff --git a/web/app/components/app/configuration/dataset-config/params-config/index.tsx b/web/app/components/app/configuration/dataset-config/params-config/index.tsx index 87d0d73b64..5cb76e32b2 100644 --- a/web/app/components/app/configuration/dataset-config/params-config/index.tsx +++ b/web/app/components/app/configuration/dataset-config/params-config/index.tsx @@ -1,29 +1,73 @@ 'use client' -import type { FC } from 'react' -import { memo, useState } from 'react' +import { memo, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' +import { RiEqualizer2Line } from '@remixicon/react' import ConfigContent from './config-content' import cn from '@/utils/classnames' -import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' import ConfigContext from '@/context/debug-configuration' import Modal from '@/app/components/base/modal' import Button from '@/app/components/base/button' import { RETRIEVE_TYPE } from '@/types/app' import Toast from '@/app/components/base/toast' -import { DATASET_DEFAULT } from '@/config' import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import type { DataSet } from '@/models/datasets' +import type { DatasetConfigs } from '@/models/debug' +import { + getMultipleRetrievalConfig, + getSelectedDatasetsMode, +} from '@/app/components/workflow/nodes/knowledge-retrieval/utils' -const ParamsConfig: FC = () => { +type ParamsConfigProps = { + disabled?: boolean + selectedDatasets: DataSet[] +} +const ParamsConfig = ({ + disabled, + selectedDatasets, +}: ParamsConfigProps) => { const { t } = useTranslation() - const [open, setOpen] = useState(false) const { datasetConfigs, setDatasetConfigs, + rerankSettingModalOpen, + setRerankSettingModalOpen, } = useContext(ConfigContext) const [tempDataSetConfigs, setTempDataSetConfigs] = useState(datasetConfigs) + useEffect(() => { + const { + allEconomic, + } = getSelectedDatasetsMode(selectedDatasets) + const { datasets, retrieval_model, score_threshold_enabled, ...restConfigs } = datasetConfigs + let rerankEnable = restConfigs.reranking_enable + + if (allEconomic && !restConfigs.reranking_model?.reranking_provider_name && rerankEnable === undefined) + rerankEnable = false + + setTempDataSetConfigs({ + ...getMultipleRetrievalConfig({ + top_k: restConfigs.top_k, + score_threshold: restConfigs.score_threshold, + reranking_model: restConfigs.reranking_model && { + provider: restConfigs.reranking_model.reranking_provider_name, + model: restConfigs.reranking_model.reranking_model_name, + }, + reranking_mode: restConfigs.reranking_mode, + weights: restConfigs.weights, + reranking_enable: rerankEnable, + }, selectedDatasets), + reranking_model: restConfigs.reranking_model && { + reranking_provider_name: restConfigs.reranking_model.reranking_provider_name, + reranking_model_name: restConfigs.reranking_model.reranking_model_name, + }, + retrieval_model, + score_threshold_enabled, + datasets, + }) + }, [selectedDatasets, datasetConfigs]) + const { defaultModel: rerankDefaultModel, currentModel: isRerankDefaultModelVaild, @@ -55,45 +99,68 @@ const ParamsConfig: FC = () => { } as any } setDatasetConfigs(config) - setOpen(false) + setRerankSettingModalOpen(false) + } + + const handleSetTempDataSetConfigs = (newDatasetConfigs: DatasetConfigs) => { + const { datasets, retrieval_model, score_threshold_enabled, ...restConfigs } = newDatasetConfigs + + const retrievalConfig = getMultipleRetrievalConfig({ + top_k: restConfigs.top_k, + score_threshold: restConfigs.score_threshold, + reranking_model: restConfigs.reranking_model && { + provider: restConfigs.reranking_model.reranking_provider_name, + model: restConfigs.reranking_model.reranking_model_name, + }, + reranking_mode: restConfigs.reranking_mode, + weights: restConfigs.weights, + reranking_enable: restConfigs.reranking_enable, + }, selectedDatasets) + + setTempDataSetConfigs({ + ...retrievalConfig, + reranking_model: restConfigs.reranking_model && { + reranking_provider_name: restConfigs.reranking_model.reranking_provider_name, + reranking_model_name: restConfigs.reranking_model.reranking_model_name, + }, + retrieval_model, + score_threshold_enabled, + datasets, + }) } return (
-
{ - setTempDataSetConfigs({ - ...datasetConfigs, - top_k: datasetConfigs.top_k || DATASET_DEFAULT.top_k, - score_threshold: datasetConfigs.score_threshold || DATASET_DEFAULT.score_threshold, - }) - setOpen(true) + setRerankSettingModalOpen(true) }} + disabled={disabled} > - -
- {t('appDebug.datasetConfig.params')} -
-
+ + {t('dataset.retrievalSettings')} + { - open && ( + rerankSettingModalOpen && ( { - setOpen(false) + setRerankSettingModalOpen(false) }} className='sm:min-w-[528px]' - title={t('appDebug.datasetConfig.settingTitle')} >
diff --git a/web/app/components/app/configuration/dataset-config/params-config/weighted-score.tsx b/web/app/components/app/configuration/dataset-config/params-config/weighted-score.tsx new file mode 100644 index 0000000000..d65dc38ce5 --- /dev/null +++ b/web/app/components/app/configuration/dataset-config/params-config/weighted-score.tsx @@ -0,0 +1,112 @@ +import { memo, useCallback } from 'react' +import { useTranslation } from 'react-i18next' +import { + DEFAULT_WEIGHTED_SCORE, + WeightedScoreEnum, +} from '@/models/datasets' +import Slider from '@/app/components/base/slider' +import cn from '@/utils/classnames' + +const formatNumber = (value: number) => { + if (value > 0 && value < 1) + return `0.${value * 10}` + else if (value === 1) + return '1.0' + + return value +} + +type Value = { + type: WeightedScoreEnum + value: number[] +} + +type WeightedScoreProps = { + value: Value + onChange: (value: Value) => void +} +const WeightedScore = ({ + value, + onChange = () => {}, +}: WeightedScoreProps) => { + const { t } = useTranslation() + const options = [ + { + value: WeightedScoreEnum.SemanticFirst, + label: t('dataset.weightedScore.semanticFirst'), + }, + { + value: WeightedScoreEnum.KeywordFirst, + label: t('dataset.weightedScore.keywordFirst'), + }, + { + value: WeightedScoreEnum.Customized, + label: t('dataset.weightedScore.customized'), + }, + ] + + const disabled = value.type !== WeightedScoreEnum.Customized + + const handleTypeChange = useCallback((type: WeightedScoreEnum) => { + const result = { ...value, type } + + if (type === WeightedScoreEnum.SemanticFirst) + result.value = [DEFAULT_WEIGHTED_SCORE.semanticFirst.semantic, DEFAULT_WEIGHTED_SCORE.semanticFirst.keyword] + + if (type === WeightedScoreEnum.KeywordFirst) + result.value = [DEFAULT_WEIGHTED_SCORE.keywordFirst.semantic, DEFAULT_WEIGHTED_SCORE.keywordFirst.keyword] + + onChange(result) + }, [value, onChange]) + + return ( +
+
+ { + options.map(option => ( +
handleTypeChange(option.value)} + > +
+
{option.label}
+
+ )) + } +
+
+
+
+ {t('dataset.weightedScore.semantic')} +
+ {formatNumber(value.value[0])} +
+ onChange({ type: value.type, value: [v, (10 - v * 10) / 10] })} + disabled={disabled} + thumbClassName={cn(disabled && '!cursor-not-allowed')} + trackClassName='!bg-transparent' + /> +
+ {formatNumber(value.value[1])} +
+ {t('dataset.weightedScore.keyword')} +
+
+
+
+ ) +} + +export default memo(WeightedScore) diff --git a/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx b/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx index 602525f579..f9117a51c3 100644 --- a/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx +++ b/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx @@ -13,7 +13,8 @@ import type { DataSet } from '@/models/datasets' import Button from '@/app/components/base/button' import { fetchDatasets } from '@/service/datasets' import Loading from '@/app/components/base/loading' -import { formatNumber } from '@/utils/format' +import Badge from '@/app/components/base/badge' +import { useKnowledge } from '@/hooks/use-knowledge' export type ISelectDataSetProps = { isShow: boolean @@ -38,6 +39,7 @@ const SelectDataSet: FC = ({ const listRef = useRef(null) const [page, setPage, getPage] = useGetState(1) const [isNoMore, setIsNoMore] = useState(false) + const { formatIndexingTechniqueAndMethod } = useKnowledge() useInfiniteScroll( async () => { @@ -45,7 +47,7 @@ const SelectDataSet: FC = ({ const { data, has_more } = await fetchDatasets({ url: '/datasets', params: { page } }) setPage(getPage() + 1) setIsNoMore(!has_more) - const newList = [...(datasets || []), ...data] + const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique)] setDataSets(newList) setLoaded(true) if (!selected.find(item => !item.name)) @@ -136,14 +138,13 @@ const SelectDataSet: FC = ({ {t('dataset.unavailable')} )}
- -
- {formatNumber(item.word_count)} - {t('appDebug.feature.dataSet.words')} - · - {formatNumber(item.document_count)} - {t('appDebug.feature.dataSet.textBlocks')} -
+ { + item.indexing_technique && ( + + ) + }
))}
diff --git a/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx b/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx index 2f53fb7738..eec5979dd5 100644 --- a/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx +++ b/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx @@ -259,7 +259,7 @@ const SettingsModal: FC = ({ {/* Retrieval Method Config */}
-
+
{t('datasetSettings.form.retrievalSetting.title')}
@@ -268,7 +268,7 @@ const SettingsModal: FC = ({
-
+
{indexMethod === 'high_quality' ? ( { }, []) const [datasetConfigs, setDatasetConfigs] = useState({ - retrieval_model: RETRIEVE_TYPE.oneWay, + retrieval_model: RETRIEVE_TYPE.multiWay, reranking_model: { reranking_provider_name: '', reranking_model_name: '', }, - top_k: 2, + top_k: DATASET_DEFAULT.top_k, score_threshold_enabled: false, - score_threshold: 0.7, + score_threshold: DATASET_DEFAULT.score_threshold, datasets: { datasets: [], }, @@ -202,6 +206,7 @@ const Configuration: FC = () => { const hasSetContextVar = !!contextVar const [isShowSelectDataSet, { setTrue: showSelectDataSet, setFalse: hideSelectDataSet }] = useBoolean(false) const selectedIds = dataSets.map(item => item.id) + const [rerankSettingModalOpen, setRerankSettingModalOpen] = useState(false) const handleSelect = (data: DataSet[]) => { if (isEqual(data.map(item => item.id), dataSets.map(item => item.id))) { hideSelectDataSet() @@ -209,6 +214,7 @@ const Configuration: FC = () => { } formattingChangedDispatcher() + let newDatasets = data if (data.find(item => !item.name)) { // has not loaded selected dataset const newSelected = produce(data, (draft: any) => { data.forEach((item, index) => { @@ -220,11 +226,45 @@ const Configuration: FC = () => { }) }) setDataSets(newSelected) + newDatasets = newSelected } else { setDataSets(data) } hideSelectDataSet() + const { + allEconomic, + mixtureHighQualityAndEconomic, + inconsistentEmbeddingModel, + } = getSelectedDatasetsMode(newDatasets) + + if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel) + setRerankSettingModalOpen(true) + + const { datasets, retrieval_model, score_threshold_enabled, ...restConfigs } = datasetConfigs + + const retrievalConfig = getMultipleRetrievalConfig({ + top_k: restConfigs.top_k, + score_threshold: restConfigs.score_threshold, + reranking_model: restConfigs.reranking_model && { + provider: restConfigs.reranking_model.reranking_provider_name, + model: restConfigs.reranking_model.reranking_model_name, + }, + reranking_mode: restConfigs.reranking_mode, + weights: restConfigs.weights, + reranking_enable: restConfigs.reranking_enable, + }, newDatasets) + + setDatasetConfigs({ + ...retrievalConfig, + reranking_model: restConfigs.reranking_model && { + reranking_provider_name: restConfigs.reranking_model.reranking_provider_name, + reranking_model_name: restConfigs.reranking_model.reranking_model_name, + }, + retrieval_model, + score_threshold_enabled, + datasets, + }) } const [isShowHistoryModal, { setTrue: showHistoryModal, setFalse: hideHistoryModal }] = useBoolean(false) @@ -509,7 +549,7 @@ const Configuration: FC = () => { syncToPublishedConfig(config) setPublishedConfig(config) setDatasetConfigs({ - retrieval_model: RETRIEVE_TYPE.oneWay, + retrieval_model: RETRIEVE_TYPE.multiWay, ...modelConfig.dataset_configs, }) setHasFetchedDetail(true) @@ -744,6 +784,8 @@ const Configuration: FC = () => { isShowVisionConfig, visionConfig, setVisionConfig: handleSetVisionConfig, + rerankSettingModalOpen, + setRerankSettingModalOpen, }} > <> diff --git a/web/app/components/base/badge.tsx b/web/app/components/base/badge.tsx new file mode 100644 index 0000000000..3e5414fa2c --- /dev/null +++ b/web/app/components/base/badge.tsx @@ -0,0 +1,25 @@ +import { memo } from 'react' +import cn from '@/utils/classnames' + +type BadgeProps = { + className?: string + text: string +} + +const Badge = ({ + className, + text, +}: BadgeProps) => { + return ( +
+ {text} +
+ ) +} + +export default memo(Badge) diff --git a/web/app/components/base/button/index.css b/web/app/components/base/button/index.css index 17932166ca..5656cb9fdb 100644 --- a/web/app/components/base/button/index.css +++ b/web/app/components/base/button/index.css @@ -2,46 +2,185 @@ @layer components { .btn { - @apply inline-flex justify-center items-center border-[0.5px] font-medium cursor-pointer whitespace-nowrap shadow; + @apply inline-flex justify-center items-center cursor-pointer whitespace-nowrap; } .btn-disabled { - @apply opacity-60 cursor-not-allowed; + @apply cursor-not-allowed; } .btn-small { - @apply px-2 h-6 rounded-md text-xs + @apply px-2 h-6 rounded-md text-xs font-medium; } .btn-medium { - @apply px-3.5 h-8 rounded-lg text-[13px] + @apply px-3.5 h-8 rounded-lg text-[13px] leading-4 font-medium; } .btn-large { - @apply px-4 h-9 rounded-[10px] text-sm font-semibold - } - - .btn-secondary { - @apply bg-white hover:bg-white/80 border-gray-200 hover:border-gray-300 text-gray-700; - } - - .btn-secondary-accent { - @apply bg-white hover:bg-white/80 border-gray-200 hover:border-gray-300 text-primary-600; + @apply px-4 h-9 rounded-[10px] text-sm font-semibold; } .btn-primary { - @apply bg-primary-600 hover:bg-primary-700 text-white; + @apply + shadow + bg-components-button-primary-bg + border-components-button-primary-border + hover:bg-components-button-primary-bg-hover + hover:border-components-button-primary-border-hover + text-components-button-primary-text; + } + + .btn-primary.btn-destructive { + @apply + bg-components-button-destructive-primary-bg + border-components-button-destructive-primary-border + hover:bg-components-button-destructive-primary-bg-hover + hover:border-components-button-destructive-primary-border-hover + text-components-button-destructive-primary-text; + } + + .btn-primary.btn-disabled { + @apply + shadow-none + bg-components-button-primary-bg-disabled + border-components-button-primary-border-disabled + text-components-button-primary-text-disabled; + } + + .btn-primary.btn-destructive.btn-disabled { + @apply + shadow-none + bg-components-button-destructive-primary-bg-disabled + border-components-button-destructive-primary-border-disabled + text-components-button-destructive-primary-text-disabled; + } + + .btn-secondary { + @apply + border-[0.5px] + shadow-xs + bg-components-button-secondary-bg + border-components-button-secondary-border + hover:bg-components-button-secondary-bg-hover + hover:border-components-button-secondary-border-hover + text-components-button-secondary-text; + } + + .btn-secondary.btn-disabled { + @apply + bg-components-button-secondary-bg-disabled + border-components-button-secondary-border-disabled + text-components-button-secondary-text-disabled; + } + + .btn-secondary.btn-destructive { + @apply + bg-components-button-destructive-secondary-bg + border-components-button-destructive-secondary-border + hover:bg-components-button-destructive-secondary-bg-hover + hover:border-components-button-destructive-secondary-border-hover + text-components-button-destructive-secondary-text; + } + + .btn-secondary.btn-destructive.btn-disabled { + @apply + bg-components-button-destructive-secondary-bg-disabled + border-components-button-destructive-secondary-border-disabled + text-components-button-destructive-secondary-text-disabled; + } + + + .btn-secondary-accent { + @apply + border-[0.5px] + shadow-xs + bg-components-button-secondary-bg + border-components-button-secondary-border + hover:bg-components-button-secondary-bg-hover + hover:border-components-button-secondary-border-hover + text-components-button-secondary-accent-text; + } + + .btn-secondary-accent.btn-disabled { + @apply + bg-components-button-secondary-bg-disabled + border-components-button-secondary-border-disabled + text-components-button-secondary-accent-text-disabled; } .btn-warning { - @apply bg-red-600 hover:bg-red-700 text-white; + @apply + bg-components-button-destructive-primary-bg + border-components-button-destructive-primary-border + hover:bg-components-button-destructive-primary-bg-hover + hover:border-components-button-destructive-primary-border-hover + text-components-button-destructive-primary-text; } - .btn-ghost { - @apply bg-transparent hover:bg-gray-200 border-transparent shadow-none text-gray-700; + .btn-warning.btn-disabled { + @apply + bg-components-button-destructive-primary-bg-disabled + border-components-button-destructive-primary-border-disabled + text-components-button-destructive-primary-text-disabled; } .btn-tertiary { - @apply bg-[#F2F4F7] hover:bg-[#E9EBF0] border-transparent shadow-none text-gray-700; + @apply + bg-components-button-tertiary-bg + hover:bg-components-button-tertiary-bg-hover + text-components-button-tertiary-text; + } + + .btn-tertiary.btn-disabled { + @apply + bg-components-button-tertiary-bg-disabled + text-components-button-tertiary-text-disabled; + } + + .btn-tertiary.btn-destructive { + @apply + bg-components-button-destructive-tertiary-bg + hover:bg-components-button-destructive-tertiary-bg-hover + text-components-button-destructive-tertiary-text; + } + + .btn-tertiary.btn-destructive.btn-disabled { + @apply + bg-components-button-destructive-tertiary-bg-disabled + text-components-button-destructive-tertiary-text-disabled; + } + + .btn-ghost { + @apply + hover:bg-components-button-ghost-bg-hover + text-components-button-ghost-text; + } + + .btn-ghost.btn-disabled { + @apply + text-components-button-ghost-text-disabled; + } + + .btn-ghost.btn-destructive { + @apply + hover:bg-components-button-destructive-ghost-bg-hover + text-components-button-destructive-ghost-text; + } + + .btn-ghost.btn-destructive.btn-disabled { + @apply + text-components-button-destructive-ghost-text-disabled; + } + + .btn-ghost-accent { + @apply + hover:bg-state-accent-hover + text-components-button-secondary-accent-text; + } + + .btn-ghost-accent.btn-disabled { + @apply + text-components-button-secondary-accent-text-disabled; } } \ No newline at end of file diff --git a/web/app/components/base/button/index.tsx b/web/app/components/base/button/index.tsx index 901c813fac..3bd4b21a5e 100644 --- a/web/app/components/base/button/index.tsx +++ b/web/app/components/base/button/index.tsx @@ -14,6 +14,7 @@ const buttonVariants = cva( 'secondary': 'btn-secondary', 'secondary-accent': 'btn-secondary-accent', 'ghost': 'btn-ghost', + 'ghost-accent': 'btn-ghost-accent', 'tertiary': 'btn-tertiary', }, size: { @@ -30,16 +31,20 @@ const buttonVariants = cva( ) export type ButtonProps = { + destructive?: boolean loading?: boolean styleCss?: CSSProperties } & React.ButtonHTMLAttributes & VariantProps const Button = React.forwardRef( - ({ className, variant, size, loading, styleCss, children, ...props }, ref) => { + ({ className, variant, size, destructive, loading, styleCss, children, ...props }, ref) => { return (
)} + {isShowSettingsModal && ( diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx index a739f7fb1c..ebe9d2151e 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx @@ -1,8 +1,8 @@ 'use client' import type { FC } from 'react' import React, { useCallback, useState } from 'react' +import { RiEqualizer2Line } from '@remixicon/react' import { useTranslation } from 'react-i18next' -import { RiArrowDownSLine } from '@remixicon/react' import type { MultipleRetrievalConfig, SingleRetrievalConfig } from '../types' import type { ModelConfig } from '../../../types' import cn from '@/utils/classnames' @@ -16,10 +16,9 @@ import { RETRIEVE_TYPE } from '@/types/app' import { DATASET_DEFAULT } from '@/config' import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' - -import type { - DatasetConfigs, -} from '@/models/debug' +import Button from '@/app/components/base/button' +import type { DatasetConfigs } from '@/models/debug' +import type { DataSet } from '@/models/datasets' type Props = { payload: { @@ -33,6 +32,9 @@ type Props = { onSingleRetrievalModelChange?: (config: ModelConfig) => void onSingleRetrievalModelParamsChange?: (config: ModelConfig) => void readonly?: boolean + openFromProps?: boolean + onOpenFromPropsChange?: (openFromProps: boolean) => void + selectedDatasets: DataSet[] } const RetrievalConfig: FC = ({ @@ -43,10 +45,18 @@ const RetrievalConfig: FC = ({ onSingleRetrievalModelChange, onSingleRetrievalModelParamsChange, readonly, + openFromProps, + onOpenFromPropsChange, + selectedDatasets, }) => { const { t } = useTranslation() - const [open, setOpen] = useState(false) + const mergedOpen = openFromProps !== undefined ? openFromProps : open + + const handleOpen = useCallback((newOpen: boolean) => { + setOpen(newOpen) + onOpenFromPropsChange?.(newOpen) + }, [onOpenFromPropsChange]) const { defaultModel: rerankDefaultModel, @@ -72,16 +82,18 @@ const RetrievalConfig: FC = ({ provider: configs.reranking_model?.reranking_provider_name, model: configs.reranking_model?.reranking_model_name, }), + reranking_mode: configs.reranking_mode, + weights: configs.weights as any, + reranking_enable: configs.reranking_enable, }) }, [onMultipleRetrievalConfigChange, payload.retrieval_mode, rerankDefaultModel?.provider?.provider, rerankDefaultModel?.model, onRetrievalModeChange]) return ( @@ -89,13 +101,18 @@ const RetrievalConfig: FC = ({ onClick={() => { if (readonly) return - setOpen(v => !v) + handleOpen(!mergedOpen) }} > -
-
{payload.retrieval_mode === RETRIEVE_TYPE.oneWay ? t('appDebug.datasetConfig.retrieveOneWay.title') : t('appDebug.datasetConfig.retrieveMultiWay.title')}
- {!readonly && } -
+
@@ -103,21 +120,24 @@ const RetrievalConfig: FC = ({ datasetConfigs={ { retrieval_model: payload.retrieval_mode, - reranking_model: !multiple_retrieval_config?.reranking_model?.provider + reranking_model: multiple_retrieval_config?.reranking_model?.provider ? { - reranking_provider_name: rerankDefaultModel?.provider?.provider || '', - reranking_model_name: rerankDefaultModel?.model || '', + reranking_provider_name: multiple_retrieval_config.reranking_model?.provider, + reranking_model_name: multiple_retrieval_config.reranking_model?.model, } : { - reranking_provider_name: multiple_retrieval_config?.reranking_model?.provider || '', - reranking_model_name: multiple_retrieval_config?.reranking_model?.model || '', + reranking_provider_name: '', + reranking_model_name: '', }, top_k: multiple_retrieval_config?.top_k || DATASET_DEFAULT.top_k, - score_threshold_enabled: !(multiple_retrieval_config?.score_threshold === undefined || multiple_retrieval_config?.score_threshold === null), + score_threshold_enabled: !(multiple_retrieval_config?.score_threshold === undefined || multiple_retrieval_config.score_threshold === null), score_threshold: multiple_retrieval_config?.score_threshold, datasets: { datasets: [], }, + reranking_mode: multiple_retrieval_config?.reranking_mode, + weights: multiple_retrieval_config?.weights, + reranking_enable: multiple_retrieval_config?.reranking_enable, } } onChange={handleChange} @@ -125,6 +145,7 @@ const RetrievalConfig: FC = ({ singleRetrievalModelConfig={singleRetrievalModelConfig} onSingleRetrievalModelChange={onSingleRetrievalModelChange} onSingleRetrievalModelParamsChange={onSingleRetrievalModelParamsChange} + selectedDatasets={selectedDatasets} />
diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/default.ts b/web/app/components/workflow/nodes/knowledge-retrieval/default.ts index 22edbc2360..d533cd5b3c 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/default.ts +++ b/web/app/components/workflow/nodes/knowledge-retrieval/default.ts @@ -2,7 +2,7 @@ import { BlockEnum } from '../../types' import type { NodeDefault } from '../../types' import type { KnowledgeRetrievalNodeType } from './types' import { ALL_CHAT_AVAILABLE_BLOCKS, ALL_COMPLETION_AVAILABLE_BLOCKS } from '@/app/components/workflow/constants' - +import { DATASET_DEFAULT } from '@/config' import { RETRIEVE_TYPE } from '@/types/app' const i18nPrefix = 'workflow' @@ -10,7 +10,12 @@ const nodeDefault: NodeDefault = { defaultValue: { query_variable_selector: [], dataset_ids: [], - retrieval_mode: RETRIEVE_TYPE.oneWay, + retrieval_mode: RETRIEVE_TYPE.multiWay, + multiple_retrieval_config: { + top_k: DATASET_DEFAULT.top_k, + score_threshold: undefined, + reranking_enable: false, + }, }, getAvailablePrevNodes(isChatMode: boolean) { const nodes = isChatMode diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/hooks.ts b/web/app/components/workflow/nodes/knowledge-retrieval/hooks.ts new file mode 100644 index 0000000000..139ac87382 --- /dev/null +++ b/web/app/components/workflow/nodes/knowledge-retrieval/hooks.ts @@ -0,0 +1,14 @@ +import { useMemo } from 'react' +import { getSelectedDatasetsMode } from './utils' +import type { + DataSet, + SelectedDatasetsMode, +} from '@/models/datasets' + +export const useSelectedDatasetsMode = (datasets: DataSet[]) => { + const selectedDatasetsMode: SelectedDatasetsMode = useMemo(() => { + return getSelectedDatasetsMode(datasets) + }, [datasets]) + + return selectedDatasetsMode +} diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx index 79bb04702a..3bfc7c56ed 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx @@ -1,5 +1,8 @@ import type { FC } from 'react' -import React from 'react' +import { + memo, + useCallback, +} from 'react' import { useTranslation } from 'react-i18next' import VarReferencePicker from '../_base/components/variable/var-reference-picker' import useConfig from './use-config' @@ -41,8 +44,14 @@ const Panel: FC> = ({ query, setQuery, runResult, + rerankModelOpen, + setRerankModelOpen, } = useConfig(id, data) + const handleOpenFromPropsChange = useCallback((openFromProps: boolean) => { + setRerankModelOpen(openFromProps) + }, [setRerankModelOpen]) + return (
@@ -75,7 +84,10 @@ const Panel: FC> = ({ singleRetrievalModelConfig={inputs.single_retrieval_config?.model} onSingleRetrievalModelChange={handleModelChanged as any} onSingleRetrievalModelParamsChange={handleCompletionParamsChange} - readonly={readOnly} + readonly={readOnly || !selectedDatasets.length} + openFromProps={rerankModelOpen} + onOpenFromPropsChange={handleOpenFromPropsChange} + selectedDatasets={selectedDatasets} /> {!readOnly && (
)} {!readOnly && ( @@ -162,4 +174,4 @@ const Panel: FC> = ({ ) } -export default React.memo(Panel) +export default memo(Panel) diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/types.ts b/web/app/components/workflow/nodes/knowledge-retrieval/types.ts index deefdbf8d2..b14a8bb15c 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/types.ts +++ b/web/app/components/workflow/nodes/knowledge-retrieval/types.ts @@ -1,5 +1,9 @@ import type { CommonNodeType, ModelConfig, ValueSelector } from '@/app/components/workflow/types' import type { RETRIEVE_TYPE } from '@/types/app' +import type { + RerankingModeEnum, + WeightedScoreEnum, +} from '@/models/datasets' export type MultipleRetrievalConfig = { top_k: number @@ -8,6 +12,19 @@ export type MultipleRetrievalConfig = { provider: string model: string } + reranking_mode?: RerankingModeEnum + weights?: { + weight_type: WeightedScoreEnum + vector_setting: { + vector_weight: number + embedding_provider_name: string + embedding_model_name: string + } + keyword_setting: { + keyword_weight: number + } + } + reranking_enable?: boolean } export type SingleRetrievalConfig = { diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts b/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts index 18a2e3f508..0a5631a08e 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts +++ b/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts @@ -1,4 +1,9 @@ -import { useCallback, useEffect, useRef, useState } from 'react' +import { + useCallback, + useEffect, + useRef, + useState, +} from 'react' import produce from 'immer' import { isEqual } from 'lodash-es' import type { ValueSelector, Var } from '../../types' @@ -8,6 +13,10 @@ import { useWorkflow, } from '../../hooks' import type { KnowledgeRetrievalNodeType, MultipleRetrievalConfig } from './types' +import { + getMultipleRetrievalConfig, + getSelectedDatasetsMode, +} from './utils' import { RETRIEVE_TYPE } from '@/types/app' import { DATASET_DEFAULT } from '@/config' import type { DataSet } from '@/models/datasets' @@ -126,34 +135,20 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { draft.multiple_retrieval_config = { top_k: multipleRetrievalConfig?.top_k || DATASET_DEFAULT.top_k, score_threshold: multipleRetrievalConfig?.score_threshold, - reranking_model: payload.retrieval_mode === RETRIEVE_TYPE.oneWay - ? undefined - : (!multipleRetrievalConfig?.reranking_model?.provider - ? { - provider: rerankDefaultModel?.provider?.provider || '', - model: rerankDefaultModel?.model || '', - } - : multipleRetrievalConfig?.reranking_model), + reranking_model: multipleRetrievalConfig?.reranking_model, } }) setInputs(newInput) // eslint-disable-next-line react-hooks/exhaustive-deps }, [currentProvider?.provider, currentModel, rerankDefaultModel]) - + const [selectedDatasets, setSelectedDatasets] = useState([]) + const [rerankModelOpen, setRerankModelOpen] = useState(false) const handleRetrievalModeChange = useCallback((newMode: RETRIEVE_TYPE) => { const newInputs = produce(inputs, (draft) => { draft.retrieval_mode = newMode if (newMode === RETRIEVE_TYPE.multiWay) { - draft.multiple_retrieval_config = { - top_k: draft.multiple_retrieval_config?.top_k || DATASET_DEFAULT.top_k, - score_threshold: draft.multiple_retrieval_config?.score_threshold, - reranking_model: !draft.multiple_retrieval_config?.reranking_model?.provider - ? { - provider: rerankDefaultModel?.provider?.provider || '', - model: rerankDefaultModel?.model || '', - } - : draft.multiple_retrieval_config?.reranking_model, - } + const multipleRetrievalConfig = draft.multiple_retrieval_config + draft.multiple_retrieval_config = getMultipleRetrievalConfig(multipleRetrievalConfig!, selectedDatasets) } else { const hasSetModel = draft.single_retrieval_config?.model?.provider @@ -170,17 +165,16 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { } }) setInputs(newInputs) - }, [currentModel?.model, currentModel?.model_properties?.mode, currentProvider?.provider, inputs, rerankDefaultModel?.model, rerankDefaultModel?.provider?.provider, setInputs]) + }, [currentModel?.model, currentModel?.model_properties?.mode, currentProvider?.provider, inputs, setInputs, selectedDatasets]) const handleMultipleRetrievalConfigChange = useCallback((newConfig: MultipleRetrievalConfig) => { const newInputs = produce(inputs, (draft) => { - draft.multiple_retrieval_config = newConfig + draft.multiple_retrieval_config = getMultipleRetrievalConfig(newConfig!, selectedDatasets) }) setInputs(newInputs) - }, [inputs, setInputs]) + }, [inputs, setInputs, selectedDatasets]) // datasets - const [selectedDatasets, setSelectedDatasets] = useState([]) useEffect(() => { (async () => { const inputs = inputRef.current @@ -210,12 +204,25 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { }, []) const handleOnDatasetsChange = useCallback((newDatasets: DataSet[]) => { + const { + allEconomic, + mixtureHighQualityAndEconomic, + inconsistentEmbeddingModel, + } = getSelectedDatasetsMode(newDatasets) const newInputs = produce(inputs, (draft) => { draft.dataset_ids = newDatasets.map(d => d.id) + + if (payload.retrieval_mode === RETRIEVE_TYPE.multiWay && newDatasets.length > 0) { + const multipleRetrievalConfig = draft.multiple_retrieval_config + draft.multiple_retrieval_config = getMultipleRetrievalConfig(multipleRetrievalConfig!, newDatasets) + } }) setInputs(newInputs) setSelectedDatasets(newDatasets) - }, [inputs, setInputs]) + + if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel) + setRerankModelOpen(true) + }, [inputs, setInputs, payload.retrieval_mode]) const filterVar = useCallback((varPayload: Var) => { return varPayload.type === VarType.string @@ -266,6 +273,8 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { query, setQuery, runResult, + rerankModelOpen, + setRerankModelOpen, } } diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/utils.ts b/web/app/components/workflow/nodes/knowledge-retrieval/utils.ts index dea91ed3aa..d4dfd3b914 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/utils.ts +++ b/web/app/components/workflow/nodes/knowledge-retrieval/utils.ts @@ -1,5 +1,124 @@ -import type { KnowledgeRetrievalNodeType } from './types' +import { uniq } from 'lodash-es' +import type { MultipleRetrievalConfig } from './types' +import type { + DataSet, + SelectedDatasetsMode, +} from '@/models/datasets' +import { + DEFAULT_WEIGHTED_SCORE, + RerankingModeEnum, + WeightedScoreEnum, +} from '@/models/datasets' +import { RETRIEVE_METHOD } from '@/types/app' +import { DATASET_DEFAULT } from '@/config' -export const checkNodeValid = (payload: KnowledgeRetrievalNodeType) => { +export const checkNodeValid = () => { return true } + +export const getSelectedDatasetsMode = (datasets: DataSet[]) => { + let allHighQuality = true + let allHighQualityVectorSearch = true + let allHighQualityFullTextSearch = true + let allEconomic = true + let mixtureHighQualityAndEconomic = true + let inconsistentEmbeddingModel = false + if (!datasets.length) { + allHighQuality = false + allHighQualityVectorSearch = false + allHighQualityFullTextSearch = false + allEconomic = false + mixtureHighQualityAndEconomic = false + inconsistentEmbeddingModel = false + } + datasets.forEach((dataset) => { + if (dataset.indexing_technique === 'economy') { + allHighQuality = false + allHighQualityVectorSearch = false + allHighQualityFullTextSearch = false + } + if (dataset.indexing_technique === 'high_quality') { + allEconomic = false + + if (dataset.retrieval_model_dict.search_method !== RETRIEVE_METHOD.semantic) + allHighQualityVectorSearch = false + + if (dataset.retrieval_model_dict.search_method !== RETRIEVE_METHOD.fullText) + allHighQualityFullTextSearch = false + } + }) + + if (allHighQuality || allEconomic) + mixtureHighQualityAndEconomic = false + + if (allHighQuality) + inconsistentEmbeddingModel = uniq(datasets.map(item => item.embedding_model)).length > 1 + + return { + allHighQuality, + allHighQualityVectorSearch, + allHighQualityFullTextSearch, + allEconomic, + mixtureHighQualityAndEconomic, + inconsistentEmbeddingModel, + } as SelectedDatasetsMode +} + +export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetrievalConfig, selectedDatasets: DataSet[]) => { + const { + allHighQuality, + allHighQualityVectorSearch, + allHighQualityFullTextSearch, + allEconomic, + mixtureHighQualityAndEconomic, + inconsistentEmbeddingModel, + } = getSelectedDatasetsMode(selectedDatasets) + + const { + top_k = DATASET_DEFAULT.top_k, + score_threshold, + reranking_mode, + reranking_model, + weights, + reranking_enable, + } = multipleRetrievalConfig || { top_k: DATASET_DEFAULT.top_k } + + const result = { + top_k, + score_threshold, + reranking_mode, + reranking_model, + weights, + reranking_enable, + } + + if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel) + result.reranking_mode = RerankingModeEnum.RerankingModel + + if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined) + result.reranking_mode = RerankingModeEnum.WeightedScore + + if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && !weights) { + result.weights = { + weight_type: WeightedScoreEnum.Customized, + vector_setting: { + vector_weight: allHighQualityVectorSearch + ? DEFAULT_WEIGHTED_SCORE.allHighQualityVectorSearch.semantic + : allHighQualityFullTextSearch + ? DEFAULT_WEIGHTED_SCORE.allHighQualityFullTextSearch.semantic + : DEFAULT_WEIGHTED_SCORE.other.semantic, + embedding_provider_name: selectedDatasets[0].embedding_model_provider, + embedding_model_name: selectedDatasets[0].embedding_model, + }, + keyword_setting: { + keyword_weight: allHighQualityVectorSearch + ? DEFAULT_WEIGHTED_SCORE.allHighQualityVectorSearch.keyword + : allHighQualityFullTextSearch + ? DEFAULT_WEIGHTED_SCORE.allHighQualityFullTextSearch.keyword + : DEFAULT_WEIGHTED_SCORE.other.keyword, + }, + } + } + + return result +} diff --git a/web/config/index.ts b/web/config/index.ts index 9bb4efcd5e..6b0e551e35 100644 --- a/web/config/index.ts +++ b/web/config/index.ts @@ -138,8 +138,8 @@ export const appDefaultIconBackground = '#D5F5F6' export const NEED_REFRESH_APP_LIST_KEY = 'needRefreshAppList' export const DATASET_DEFAULT = { - top_k: 2, - score_threshold: 0.5, + top_k: 4, + score_threshold: 0.8, } export const APP_PAGE_LIMIT = 10 diff --git a/web/context/debug-configuration.ts b/web/context/debug-configuration.ts index fc92cc7d10..ff65fc29e6 100644 --- a/web/context/debug-configuration.ts +++ b/web/context/debug-configuration.ts @@ -97,6 +97,8 @@ type IDebugConfiguration = { isShowVisionConfig: boolean visionConfig: VisionSettings setVisionConfig: (visionConfig: VisionSettings, noNotice?: boolean) => void + rerankSettingModalOpen: boolean + setRerankSettingModalOpen: (rerankSettingModalOpen: boolean) => void } const DebugConfigurationContext = createContext({ @@ -217,7 +219,7 @@ const DebugConfigurationContext = createContext({ showSelectDataSet: () => { }, setDataSets: () => { }, datasetConfigs: { - retrieval_model: RETRIEVE_TYPE.oneWay, + retrieval_model: RETRIEVE_TYPE.multiWay, reranking_model: { reranking_provider_name: '', reranking_model_name: '', @@ -239,6 +241,8 @@ const DebugConfigurationContext = createContext({ transfer_methods: [TransferMethod.remote_url], }, setVisionConfig: () => { }, + rerankSettingModalOpen: false, + setRerankSettingModalOpen: () => { }, }) export const useDebugConfigurationContext = () => useContext(DebugConfigurationContext) diff --git a/web/hooks/use-knowledge.ts b/web/hooks/use-knowledge.ts new file mode 100644 index 0000000000..657acb3f51 --- /dev/null +++ b/web/hooks/use-knowledge.ts @@ -0,0 +1,29 @@ +import { useCallback } from 'react' +import { useTranslation } from 'react-i18next' + +export const useKnowledge = () => { + const { t } = useTranslation() + + const formatIndexingTechnique = useCallback((indexingTechnique: string) => { + return t(`dataset.indexingTechnique.${indexingTechnique}`) + }, [t]) + + const formatIndexingMethod = useCallback((indexingMethod: string) => { + return t(`dataset.indexingMethod.${indexingMethod}`) + }, [t]) + + const formatIndexingTechniqueAndMethod = useCallback((indexingTechnique: string, indexingMethod: string) => { + let result = formatIndexingTechnique(indexingTechnique) + + if (indexingMethod) + result += ` · ${formatIndexingMethod(indexingMethod)}` + + return result + }, [formatIndexingTechnique, formatIndexingMethod]) + + return { + formatIndexingTechnique, + formatIndexingMethod, + formatIndexingTechniqueAndMethod, + } +} diff --git a/web/i18n/en-US/dataset.ts b/web/i18n/en-US/dataset.ts index 8251693154..d26c3fd061 100644 --- a/web/i18n/en-US/dataset.ts +++ b/web/i18n/en-US/dataset.ts @@ -45,6 +45,29 @@ const translation = { }, docsFailedNotice: 'documents failed to be indexed', retry: 'Retry', + indexingTechnique: { + high_quality: 'HQ', + economy: 'ECO', + }, + indexingMethod: { + semantic_search: 'VECTOR', + full_text_search: 'FULL TEXT', + hybrid_search: 'HYBRID', + }, + mixtureHighQualityAndEconomicTip: 'The Rerank model is required for mixture of high quality and economical knowledge bases.', + inconsistentEmbeddingModelTip: 'The Rerank model is required if the Embedding models of the selected knowledge bases are inconsistent.', + retrievalSettings: 'Retrieval Setting', + rerankSettings: 'Rerank Setting', + weightedScore: { + title: 'Weighted Score', + description: 'By adjusting the weights assigned， this rerank strategy determines whether to prioritize semantic or keyword matching.', + semanticFirst: 'Semantic first', + keywordFirst: 'Keyword first', + customized: 'Customized', + semantic: 'Semantic', + keyword: 'Keyword', + }, + nTo1RetrievalLegacy: 'According to product planning, N-to-1 retrieval will be officially deprecated in September. Until then you can still use it normally.', } export default translation diff --git a/web/i18n/zh-Hans/dataset.ts b/web/i18n/zh-Hans/dataset.ts index 20881ab57e..b0cf8cf057 100644 --- a/web/i18n/zh-Hans/dataset.ts +++ b/web/i18n/zh-Hans/dataset.ts @@ -45,6 +45,29 @@ const translation = { }, docsFailedNotice: '文档无法被索引', retry: '重试', + indexingTechnique: { + high_quality: '高质量', + economy: '经济', + }, + indexingMethod: { + semantic_search: '向量检索', + full_text_search: '全文检索', + hybrid_search: '混合检索', + }, + mixtureHighQualityAndEconomicTip: '混合使用高质量和经济型知识库需要配置 Rerank 模型。', + inconsistentEmbeddingModelTip: '当所选知识库配置的 Embedding 模型不一致时，需要配置 Rerank 模型。', + retrievalSettings: '召回设置', + rerankSettings: 'Rerank 设置', + weightedScore: { + title: '权重设置', + description: '通过调整分配的权重，重新排序策略确定是优先进行语义匹配还是关键字匹配。', + semanticFirst: '语义优先', + keywordFirst: '关键词优先', + customized: '自定义', + semantic: '语义', + keyword: '关键词', + }, + nTo1RetrievalLegacy: '根据产品规划，N 选 1 召回将于 9 月正式弃用。在那之前，您仍然可以正常使用它。', } export default translation diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 0d2a80ea75..3b9b4442c9 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -449,3 +449,46 @@ export type ErrorDocsResponse = { data: IndexingStatusResponse[] total: number } + +export type SelectedDatasetsMode = { + allHighQuality: boolean + allHighQualityVectorSearch: boolean + allHighQualityFullTextSearch: boolean + allEconomic: boolean + mixtureHighQualityAndEconomic: boolean + inconsistentEmbeddingModel: boolean +} + +export enum WeightedScoreEnum { + SemanticFirst = 'semantic_first', + KeywordFirst = 'keyword_first', + Customized = 'customized', +} + +export enum RerankingModeEnum { + RerankingModel = 'reranking_model', + WeightedScore = 'weighted_score', +} + +export const DEFAULT_WEIGHTED_SCORE = { + allHighQualityVectorSearch: { + semantic: 1.0, + keyword: 0, + }, + allHighQualityFullTextSearch: { + semantic: 0, + keyword: 1.0, + }, + semanticFirst: { + semantic: 0.7, + keyword: 0.3, + }, + keywordFirst: { + semantic: 0.3, + keyword: 0.7, + }, + other: { + semantic: 0.7, + keyword: 0.3, + }, +} diff --git a/web/models/debug.ts b/web/models/debug.ts index d610a9eba3..468d9fa0ba 100644 --- a/web/models/debug.ts +++ b/web/models/debug.ts @@ -1,4 +1,8 @@ import type { AgentStrategy, ModelModeType, RETRIEVE_TYPE, ToolItem, TtsAutoPlay } from '@/types/app' +import type { + RerankingModeEnum, + WeightedScoreEnum, +} from '@/models/datasets' export type Inputs = Record export enum PromptMode { @@ -144,13 +148,26 @@ export type DatasetConfigs = { } top_k: number score_threshold_enabled: boolean - score_threshold?: number | null + score_threshold: number | null | undefined datasets: { datasets: { enabled: boolean id: string }[] } + reranking_mode?: RerankingModeEnum + weights?: { + weight_type: WeightedScoreEnum + vector_setting: { + vector_weight: number + embedding_provider_name: string + embedding_model_name: string + } + keyword_setting: { + keyword_weight: number + } + } + reranking_enable?: boolean } export type DebugRequestBody = { diff --git a/web/types/app.ts b/web/types/app.ts index 3dcd262467..ebf20f2d68 100644 --- a/web/types/app.ts +++ b/web/types/app.ts @@ -2,6 +2,10 @@ import type { AnnotationReplyConfig, ChatPromptConfig, CompletionPromptConfig, D import type { CollectionType } from '@/app/components/tools/types' import type { LanguagesSupported } from '@/i18n/language' import type { Tag } from '@/app/components/base/tag-management/constant' +import type { + RerankingModeEnum, + WeightedScoreEnum, +} from '@/models/datasets' export enum Theme { light = 'light', @@ -403,4 +407,16 @@ export type RetrievalConfig = { top_k: number score_threshold_enabled: boolean score_threshold: number + reranking_mode?: RerankingModeEnum + weights?: { + weight_type: WeightedScoreEnum + vector_setting: { + vector_weight: number + embedding_provider_name: string + embedding_model_name: string + } + keyword_setting: { + keyword_weight: number + } + } }