From e2988acc2fd8d181f4febe923a0ee8634cb96dab Mon Sep 17 00:00:00 2001 From: KVOJJJin Date: Mon, 24 Mar 2025 12:51:21 +0800 Subject: [PATCH 01/84] Fix: web app sidebar cannot close when long title conversation existed (#16593) --- .../components/base/chat/chat-with-history/sidebar/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/base/chat/chat-with-history/sidebar/index.tsx b/web/app/components/base/chat/chat-with-history/sidebar/index.tsx index b2abaa64ce..65e8af828c 100644 --- a/web/app/components/base/chat/chat-with-history/sidebar/index.tsx +++ b/web/app/components/base/chat/chat-with-history/sidebar/index.tsx @@ -78,7 +78,7 @@ const Sidebar = ({ isPanel }: Props) => { return (
Date: Mon, 24 Mar 2025 14:27:31 +0800 Subject: [PATCH 02/84] feat: datasets openapi list segements support paged resp (#16603) --- .../service_api/dataset/segment.py | 24 +++++++++++++++++-- .../datasets/template/template.en.mdx | 12 +++++++++- .../datasets/template/template.zh.mdx | 12 +++++++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/api/controllers/service_api/dataset/segment.py b/api/controllers/service_api/dataset/segment.py index 25ae43f2ad..ee5178862a 100644 --- a/api/controllers/service_api/dataset/segment.py +++ b/api/controllers/service_api/dataset/segment.py @@ -1,3 +1,4 @@ +from flask import request from flask_login import current_user # type: ignore from flask_restful import marshal, reqparse # type: ignore from werkzeug.exceptions import NotFound @@ -74,6 +75,8 @@ class SegmentApi(DatasetApiResource): # check dataset dataset_id = str(dataset_id) tenant_id = str(tenant_id) + page = request.args.get("page", default=1, type=int) + limit = request.args.get("limit", default=20, type=int) dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first() if not dataset: raise NotFound("Dataset not found.") @@ -118,8 +121,25 @@ class SegmentApi(DatasetApiResource): query = query.where(DocumentSegment.content.ilike(f"%{keyword}%")) total = query.count() - segments = query.order_by(DocumentSegment.position).all() - return {"data": marshal(segments, segment_fields), "doc_form": document.doc_form, "total": total}, 200 + query = query.order_by(DocumentSegment.position) + paginated_segments = query.paginate( + page=page, + per_page=limit, + max_per_page=100, + error_out=False, + ) + segments = paginated_segments.items + + response = { + "data": marshal(segments, segment_fields), + "doc_form": document.doc_form, + "total": total, + "has_more": len(segments) == limit, + "limit": limit, + "page": page, + } + + return response, 200 class DatasetSegmentApi(DatasetApiResource): diff --git a/web/app/(commonLayout)/datasets/template/template.en.mdx b/web/app/(commonLayout)/datasets/template/template.en.mdx index a5f4c40ef6..d0e1d9fbc6 100644 --- a/web/app/(commonLayout)/datasets/template/template.en.mdx +++ b/web/app/(commonLayout)/datasets/template/template.en.mdx @@ -961,6 +961,12 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi Search status, completed + + Page number (optional) + + + Number of items returned, default 20, range 1-100 (optional) + @@ -1004,7 +1010,11 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi "error": null, "stopped_at": null }], - "doc_form": "text_model" + "doc_form": "text_model", + "has_more": false, + "limit": 20, + "total": 9, + "page": 1 } ``` diff --git a/web/app/(commonLayout)/datasets/template/template.zh.mdx b/web/app/(commonLayout)/datasets/template/template.zh.mdx index 282849f3db..d4ed85233b 100644 --- a/web/app/(commonLayout)/datasets/template/template.zh.mdx +++ b/web/app/(commonLayout)/datasets/template/template.zh.mdx @@ -961,6 +961,12 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi 搜索状态，completed + + 页码，可选 + + + 返回条数，可选，默认 20，范围 1-100 + @@ -1004,7 +1010,11 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi "error": null, "stopped_at": null }], - "doc_form": "text_model" + "doc_form": "text_model", + "has_more": false, + "limit": 20, + "total": 9, + "page": 1 } ``` From 9701b573e0c5f7855d74a6883dc725a13476aec1 Mon Sep 17 00:00:00 2001 From: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Date: Mon, 24 Mar 2025 14:30:26 +0800 Subject: [PATCH 03/84] =?UTF-8?q?feat:=20add=20datasets=20detail=20context?= =?UTF-8?q?=20and=20provider=20for=20improved=20data=20vali=E2=80=A6=20(#1?= =?UTF-8?q?6451)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasets-detail-store/provider.tsx | 53 +++++++++++++ .../workflow/datasets-detail-store/store.ts | 38 +++++++++ web/app/components/workflow/header/index.tsx | 2 +- .../workflow/hooks/use-checklist.ts | 77 +++++++++++++++++-- web/app/components/workflow/index.tsx | 13 ++-- .../nodes/knowledge-retrieval/node.tsx | 35 ++++----- .../nodes/knowledge-retrieval/use-config.ts | 9 +-- 7 files changed, 192 insertions(+), 35 deletions(-) create mode 100644 web/app/components/workflow/datasets-detail-store/provider.tsx create mode 100644 web/app/components/workflow/datasets-detail-store/store.ts diff --git a/web/app/components/workflow/datasets-detail-store/provider.tsx b/web/app/components/workflow/datasets-detail-store/provider.tsx new file mode 100644 index 0000000000..7d6ede823a --- /dev/null +++ b/web/app/components/workflow/datasets-detail-store/provider.tsx @@ -0,0 +1,53 @@ +import type { FC } from 'react' +import { createContext, useCallback, useEffect, useRef } from 'react' +import { createDatasetsDetailStore } from './store' +import type { CommonNodeType, Node } from '../types' +import { BlockEnum } from '../types' +import type { KnowledgeRetrievalNodeType } from '../nodes/knowledge-retrieval/types' +import { fetchDatasets } from '@/service/datasets' + +type DatasetsDetailStoreApi = ReturnType + +type DatasetsDetailContextType = DatasetsDetailStoreApi | undefined + +export const DatasetsDetailContext = createContext(undefined) + +type DatasetsDetailProviderProps = { + nodes: Node[] + children: React.ReactNode +} + +const DatasetsDetailProvider: FC = ({ + nodes, + children, +}) => { + const storeRef = useRef() + + if (!storeRef.current) + storeRef.current = createDatasetsDetailStore() + + const updateDatasetsDetail = useCallback(async (datasetIds: string[]) => { + const { data: datasetsDetail } = await fetchDatasets({ url: '/datasets', params: { page: 1, ids: datasetIds } }) + if (datasetsDetail && datasetsDetail.length > 0) + storeRef.current!.getState().updateDatasetsDetail(datasetsDetail) + }, []) + + useEffect(() => { + if (!storeRef.current) return + const knowledgeRetrievalNodes = nodes.filter(node => node.data.type === BlockEnum.KnowledgeRetrieval) + const allDatasetIds = knowledgeRetrievalNodes.reduce((acc, node) => { + return Array.from(new Set([...acc, ...(node.data as CommonNodeType).dataset_ids])) + }, []) + if (allDatasetIds.length === 0) return + updateDatasetsDetail(allDatasetIds) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + return ( + + {children} + + ) +} + +export default DatasetsDetailProvider diff --git a/web/app/components/workflow/datasets-detail-store/store.ts b/web/app/components/workflow/datasets-detail-store/store.ts new file mode 100644 index 0000000000..4bc8c335e5 --- /dev/null +++ b/web/app/components/workflow/datasets-detail-store/store.ts @@ -0,0 +1,38 @@ +import { useContext } from 'react' +import { createStore, useStore } from 'zustand' +import type { DataSet } from '@/models/datasets' +import { DatasetsDetailContext } from './provider' +import produce from 'immer' + +type DatasetsDetailStore = { + datasetsDetail: Record + updateDatasetsDetail: (datasetsDetail: DataSet[]) => void +} + +export const createDatasetsDetailStore = () => { + return createStore((set, get) => ({ + datasetsDetail: {}, + updateDatasetsDetail: (datasets: DataSet[]) => { + const oldDatasetsDetail = get().datasetsDetail + const datasetsDetail = datasets.reduce>((acc, dataset) => { + acc[dataset.id] = dataset + return acc + }, {}) + // Merge new datasets detail into old one + const newDatasetsDetail = produce(oldDatasetsDetail, (draft) => { + Object.entries(datasetsDetail).forEach(([key, value]) => { + draft[key] = value + }) + }) + set({ datasetsDetail: newDatasetsDetail }) + }, + })) +} + +export const useDatasetsDetailStore = (selector: (state: DatasetsDetailStore) => T): T => { + const store = useContext(DatasetsDetailContext) + if (!store) + throw new Error('Missing DatasetsDetailContext.Provider in the tree') + + return useStore(store, selector) +} diff --git a/web/app/components/workflow/header/index.tsx b/web/app/components/workflow/header/index.tsx index eb259dd01f..7e99f5dd6b 100644 --- a/web/app/components/workflow/header/index.tsx +++ b/web/app/components/workflow/header/index.tsx @@ -160,7 +160,7 @@ const Header: FC = () => { const { mutateAsync: publishWorkflow } = usePublishWorkflow(appID!) const onPublish = useCallback(async (params?: PublishWorkflowParams) => { - if (handleCheckBeforePublish()) { + if (await handleCheckBeforePublish()) { const res = await publishWorkflow({ title: params?.title || '', releaseNotes: params?.releaseNotes || '', diff --git a/web/app/components/workflow/hooks/use-checklist.ts b/web/app/components/workflow/hooks/use-checklist.ts index 7a3a99ab38..c1b0189b8b 100644 --- a/web/app/components/workflow/hooks/use-checklist.ts +++ b/web/app/components/workflow/hooks/use-checklist.ts @@ -1,10 +1,12 @@ import { useCallback, useMemo, + useRef, } from 'react' import { useTranslation } from 'react-i18next' import { useStoreApi } from 'reactflow' import type { + CommonNodeType, Edge, Node, } from '../types' @@ -27,6 +29,10 @@ import { useGetLanguage } from '@/context/i18n' import type { AgentNodeType } from '../nodes/agent/types' import { useStrategyProviders } from '@/service/use-strategy' import { canFindTool } from '@/utils' +import { useDatasetsDetailStore } from '../datasets-detail-store/store' +import type { KnowledgeRetrievalNodeType } from '../nodes/knowledge-retrieval/types' +import type { DataSet } from '@/models/datasets' +import { fetchDatasets } from '@/service/datasets' export const useChecklist = (nodes: Node[], edges: Edge[]) => { const { t } = useTranslation() @@ -37,6 +43,24 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => { const customTools = useStore(s => s.customTools) const workflowTools = useStore(s => s.workflowTools) const { data: strategyProviders } = useStrategyProviders() + const datasetsDetail = useDatasetsDetailStore(s => s.datasetsDetail) + + const getCheckData = useCallback((data: CommonNodeType<{}>) => { + let checkData = data + if (data.type === BlockEnum.KnowledgeRetrieval) { + const datasetIds = (data as CommonNodeType).dataset_ids + const _datasets = datasetIds.reduce((acc, id) => { + if (datasetsDetail[id]) + acc.push(datasetsDetail[id]) + return acc + }, []) + checkData = { + ...data, + _datasets, + } as CommonNodeType + } + return checkData + }, [datasetsDetail]) const needWarningNodes = useMemo(() => { const list = [] @@ -75,7 +99,8 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => { } if (node.type === CUSTOM_NODE) { - const { errorMessage } = nodesExtraData[node.data.type].checkValid(node.data, t, moreDataForCheckValid) + const checkData = getCheckData(node.data) + const { errorMessage } = nodesExtraData[node.data.type].checkValid(checkData, t, moreDataForCheckValid) if (errorMessage || !validNodes.find(n => n.id === node.id)) { list.push({ @@ -109,7 +134,7 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => { } return list - }, [nodes, edges, isChatMode, buildInTools, customTools, workflowTools, language, nodesExtraData, t, strategyProviders]) + }, [nodes, edges, isChatMode, buildInTools, customTools, workflowTools, language, nodesExtraData, t, strategyProviders, getCheckData]) return needWarningNodes } @@ -125,8 +150,31 @@ export const useChecklistBeforePublish = () => { const store = useStoreApi() const nodesExtraData = useNodesExtraData() const { data: strategyProviders } = useStrategyProviders() + const updateDatasetsDetail = useDatasetsDetailStore(s => s.updateDatasetsDetail) + const updateTime = useRef(0) - const handleCheckBeforePublish = useCallback(() => { + const getCheckData = useCallback((data: CommonNodeType<{}>, datasets: DataSet[]) => { + let checkData = data + if (data.type === BlockEnum.KnowledgeRetrieval) { + const datasetIds = (data as CommonNodeType).dataset_ids + const datasetsDetail = datasets.reduce>((acc, dataset) => { + acc[dataset.id] = dataset + return acc + }, {}) + const _datasets = datasetIds.reduce((acc, id) => { + if (datasetsDetail[id]) + acc.push(datasetsDetail[id]) + return acc + }, []) + checkData = { + ...data, + _datasets, + } as CommonNodeType + } + return checkData + }, []) + + const handleCheckBeforePublish = useCallback(async () => { const { getNodes, edges, @@ -141,6 +189,24 @@ export const useChecklistBeforePublish = () => { notify({ type: 'error', message: t('workflow.common.maxTreeDepth', { depth: MAX_TREE_DEPTH }) }) return false } + // Before publish, we need to fetch datasets detail, in case of the settings of datasets have been changed + const knowledgeRetrievalNodes = nodes.filter(node => node.data.type === BlockEnum.KnowledgeRetrieval) + const allDatasetIds = knowledgeRetrievalNodes.reduce((acc, node) => { + return Array.from(new Set([...acc, ...(node.data as CommonNodeType).dataset_ids])) + }, []) + let datasets: DataSet[] = [] + if (allDatasetIds.length > 0) { + updateTime.current = updateTime.current + 1 + const currUpdateTime = updateTime.current + const { data: datasetsDetail } = await fetchDatasets({ url: '/datasets', params: { page: 1, ids: allDatasetIds } }) + if (datasetsDetail && datasetsDetail.length > 0) { + // avoid old data to overwrite the new data + if (currUpdateTime < updateTime.current) + return false + datasets = datasetsDetail + updateDatasetsDetail(datasetsDetail) + } + } for (let i = 0; i < nodes.length; i++) { const node = nodes[i] @@ -161,7 +227,8 @@ export const useChecklistBeforePublish = () => { } } - const { errorMessage } = nodesExtraData[node.data.type as BlockEnum].checkValid(node.data, t, moreDataForCheckValid) + const checkData = getCheckData(node.data, datasets) + const { errorMessage } = nodesExtraData[node.data.type as BlockEnum].checkValid(checkData, t, moreDataForCheckValid) if (errorMessage) { notify({ type: 'error', message: `[${node.data.title}] ${errorMessage}` }) @@ -185,7 +252,7 @@ export const useChecklistBeforePublish = () => { } return true - }, [store, isChatMode, notify, t, buildInTools, customTools, workflowTools, language, nodesExtraData, strategyProviders]) + }, [store, isChatMode, notify, t, buildInTools, customTools, workflowTools, language, nodesExtraData, strategyProviders, updateDatasetsDetail, getCheckData]) return { handleCheckBeforePublish, diff --git a/web/app/components/workflow/index.tsx b/web/app/components/workflow/index.tsx index 3384e39707..0c4d5aa671 100644 --- a/web/app/components/workflow/index.tsx +++ b/web/app/components/workflow/index.tsx @@ -99,6 +99,7 @@ import { useEventEmitterContextContext } from '@/context/event-emitter' import Confirm from '@/app/components/base/confirm' import { FILE_EXTS } from '@/app/components/base/prompt-editor/constants' import { fetchFileUploadConfig } from '@/service/common' +import DatasetsDetailProvider from './datasets-detail-store/provider' const nodeTypes = { [CUSTOM_NODE]: CustomNode, @@ -448,11 +449,13 @@ const WorkflowWrap = memo(() => { nodes={nodesData} edges={edgesData} > - + + + diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/node.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/node.tsx index 8dcb0d5ce0..c24c3e089c 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/node.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/node.tsx @@ -1,33 +1,30 @@ -import { type FC, useEffect, useRef, useState } from 'react' +import { type FC, useEffect, useState } from 'react' import React from 'react' import type { KnowledgeRetrievalNodeType } from './types' import { Folder } from '@/app/components/base/icons/src/vender/solid/files' import type { NodeProps } from '@/app/components/workflow/types' -import { fetchDatasets } from '@/service/datasets' import type { DataSet } from '@/models/datasets' +import { useDatasetsDetailStore } from '../../datasets-detail-store/store' const Node: FC> = ({ data, }) => { const [selectedDatasets, setSelectedDatasets] = useState([]) - const updateTime = useRef(0) - useEffect(() => { - (async () => { - updateTime.current = updateTime.current + 1 - const currUpdateTime = updateTime.current + const datasetsDetail = useDatasetsDetailStore(s => s.datasetsDetail) - if (data.dataset_ids?.length > 0) { - const { data: dataSetsWithDetail } = await fetchDatasets({ url: '/datasets', params: { page: 1, ids: data.dataset_ids } }) - // avoid old data overwrite new data - if (currUpdateTime < updateTime.current) - return - setSelectedDatasets(dataSetsWithDetail) - } - else { - setSelectedDatasets([]) - } - })() - }, [data.dataset_ids]) + useEffect(() => { + if (data.dataset_ids?.length > 0) { + const dataSetsWithDetail = data.dataset_ids.reduce((acc, id) => { + if (datasetsDetail[id]) + acc.push(datasetsDetail[id]) + return acc + }, []) + setSelectedDatasets(dataSetsWithDetail) + } + else { + setSelectedDatasets([]) + } + }, [data.dataset_ids, datasetsDetail]) if (!selectedDatasets.length) return null diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts b/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts index 05c50f5ff7..42aa7def25 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts +++ b/web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts @@ -41,6 +41,7 @@ import useOneStepRun from '@/app/components/workflow/nodes/_base/hooks/use-one-s import { useCurrentProviderAndModel, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' import useAvailableVarList from '@/app/components/workflow/nodes/_base/hooks/use-available-var-list' +import { useDatasetsDetailStore } from '../../datasets-detail-store/store' const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { const { nodesReadOnly: readOnly } = useNodesReadOnly() @@ -49,6 +50,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { const startNode = getBeforeNodesInSameBranch(id).find(node => node.data.type === BlockEnum.Start) const startNodeId = startNode?.id const { inputs, setInputs: doSetInputs } = useNodeCrud(id, payload) + const updateDatasetsDetail = useDatasetsDetailStore(s => s.updateDatasetsDetail) const inputRef = useRef(inputs) @@ -218,15 +220,12 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { (async () => { const inputs = inputRef.current const datasetIds = inputs.dataset_ids - let _datasets = selectedDatasets if (datasetIds?.length > 0) { const { data: dataSetsWithDetail } = await fetchDatasets({ url: '/datasets', params: { page: 1, ids: datasetIds } as any }) - _datasets = dataSetsWithDetail setSelectedDatasets(dataSetsWithDetail) } const newInputs = produce(inputs, (draft) => { draft.dataset_ids = datasetIds - draft._datasets = _datasets }) setInputs(newInputs) setSelectedDatasetsLoaded(true) @@ -256,7 +255,6 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { } = getSelectedDatasetsMode(newDatasets) const newInputs = produce(inputs, (draft) => { draft.dataset_ids = newDatasets.map(d => d.id) - draft._datasets = newDatasets if (payload.retrieval_mode === RETRIEVE_TYPE.multiWay && newDatasets.length > 0) { const multipleRetrievalConfig = draft.multiple_retrieval_config @@ -266,6 +264,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { }) } }) + updateDatasetsDetail(newDatasets) setInputs(newInputs) setSelectedDatasets(newDatasets) @@ -275,7 +274,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => { || allExternal ) setRerankModelOpen(true) - }, [inputs, setInputs, payload.retrieval_mode, selectedDatasets, currentRerankModel, currentRerankProvider]) + }, [inputs, setInputs, payload.retrieval_mode, selectedDatasets, currentRerankModel, currentRerankProvider, updateDatasetsDetail]) const filterVar = useCallback((varPayload: Var) => { return varPayload.type === VarType.string From 16b6ffd9156f0da47f61ba97bb466e025fdf4b19 Mon Sep 17 00:00:00 2001 From: Joel Date: Mon, 24 Mar 2025 14:36:07 +0800 Subject: [PATCH 04/84] fix: sanitizer svg to avoid xss (#16606) --- web/app/components/base/markdown.tsx | 26 +++++++------------ web/app/components/base/svg-gallery/index.tsx | 3 ++- web/package.json | 5 ++-- web/pnpm-lock.yaml | 11 +++++--- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/web/app/components/base/markdown.tsx b/web/app/components/base/markdown.tsx index 72e7961cad..3b846c0e2f 100644 --- a/web/app/components/base/markdown.tsx +++ b/web/app/components/base/markdown.tsx @@ -27,6 +27,7 @@ import ThinkBlock from '@/app/components/base/markdown-blocks/think-block' import { Theme } from '@/types/app' import useTheme from '@/hooks/use-theme' import cn from '@/utils/classnames' +import SVGRenderer from './svg-gallery' // Available language https://github.com/react-syntax-highlighter/react-syntax-highlighter/blob/master/AVAILABLE_LANGUAGES_HLJS.MD const capitalizationLanguageNameMap: Record = { @@ -136,14 +137,13 @@ const CodeBlock: any = memo(({ inline, className, children, ...props }: any) =>
) } - // Attention: SVGRenderer has xss vulnerability - // else if (language === 'svg' && isSVG) { - // return ( - // - // - // - // ) - // } + else if (language === 'svg' && isSVG) { + return ( + + + + ) + } else { return ( { } } -function escapeSVGTags(htmlString: string): string { - return htmlString.replace(/()([\s\S]*?)(<\/svg>)/gi, (match: string, openTag: string, innerContent: string, closeTag: string): string => { - return openTag.replace(//g, '>') - + innerContent.replace(//g, '>') - + closeTag.replace(//g, '>') - }) -} - export function Markdown(props: { content: string; className?: string; customDisallowedElements?: string[] }) { const latexContent = flow([ preprocessThinkTag, preprocessLaTeX, - ])(escapeSVGTags(props.content)) + ])(props.content) return (
diff --git a/web/app/components/base/svg-gallery/index.tsx b/web/app/components/base/svg-gallery/index.tsx index 8b7bb03920..5fe8d7946a 100644 --- a/web/app/components/base/svg-gallery/index.tsx +++ b/web/app/components/base/svg-gallery/index.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from 'react' import { SVG } from '@svgdotjs/svg.js' import ImagePreview from '@/app/components/base/image-uploader/image-preview' +import DOMPurify from 'dompurify' export const SVGRenderer = ({ content }: { content: string }) => { const svgRef = useRef(null) @@ -44,7 +45,7 @@ export const SVGRenderer = ({ content }: { content: string }) => { svgRef.current.style.width = `${Math.min(originalWidth, 298)}px` - const rootElement = draw.svg(content) + const rootElement = draw.svg(DOMPurify.sanitize(content)) rootElement.click(() => { setImagePreview(svgToDataURL(svgElement as Element)) diff --git a/web/package.json b/web/package.json index 1a0b8119f3..47cf4f56fc 100644 --- a/web/package.json +++ b/web/package.json @@ -61,6 +61,7 @@ "crypto-js": "^4.2.0", "dayjs": "^1.11.13", "decimal.js": "^10.4.3", + "dompurify": "^3.2.4", "echarts": "^5.5.1", "echarts-for-react": "^3.0.2", "elkjs": "^0.9.3", @@ -132,10 +133,10 @@ }, "devDependencies": { "@antfu/eslint-config": "^4.1.1", - "@eslint/js": "^9.20.0", "@chromatic-com/storybook": "^3.1.0", "@eslint-react/eslint-plugin": "^1.15.0", "@eslint/eslintrc": "^3.1.0", + "@eslint/js": "^9.20.0", "@faker-js/faker": "^9.0.3", "@next/eslint-plugin-next": "^15.2.3", "@rgrove/parse-xml": "^4.1.0", @@ -174,11 +175,11 @@ "code-inspector-plugin": "^0.18.1", "cross-env": "^7.0.3", "eslint": "^9.20.1", + "eslint-config-next": "^15.0.0", "eslint-plugin-react-hooks": "^5.1.0", "eslint-plugin-react-refresh": "^0.4.19", "eslint-plugin-storybook": "^0.11.2", "eslint-plugin-tailwindcss": "^3.18.0", - "eslint-config-next": "^15.0.0", "husky": "^9.1.6", "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index eb31f688d8..a25cd4e6d2 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -121,6 +121,9 @@ importers: decimal.js: specifier: ^10.4.3 version: 10.4.3 + dompurify: + specifier: ^3.2.4 + version: 3.2.4 echarts: specifier: ^5.5.1 version: 5.5.1 @@ -4299,8 +4302,8 @@ packages: resolution: {integrity: sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==} engines: {node: '>= 4'} - dompurify@3.2.3: - resolution: {integrity: sha512-U1U5Hzc2MO0oW3DF+G9qYN0aT7atAou4AgI0XjWz061nyBPbdxkfdhfy5uMgGn6+oLFCfn44ZGbdDqCzVmlOWA==} + dompurify@3.2.4: + resolution: {integrity: sha512-ysFSFEDVduQpyhzAob/kkuJjf5zWkZD8/A9ywSp1byueyuCfHamrCBa14/Oc2iiB0e51B+NpxSl5gmzn+Ms/mg==} domutils@2.8.0: resolution: {integrity: sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==} @@ -13070,7 +13073,7 @@ snapshots: dependencies: domelementtype: 2.3.0 - dompurify@3.2.3: + dompurify@3.2.4: optionalDependencies: '@types/trusted-types': 2.0.7 @@ -15688,7 +15691,7 @@ snapshots: d3-sankey: 0.12.3 dagre-d3-es: 7.0.11 dayjs: 1.11.13 - dompurify: 3.2.3 + dompurify: 3.2.4 katex: 0.16.21 khroma: 2.1.0 lodash-es: 4.17.21 From 770c461a8f6892b3318177696b62a8e8427edf97 Mon Sep 17 00:00:00 2001 From: chenhuan0728 <54611342+chenhuan0728@users.noreply.github.com> Date: Mon, 24 Mar 2025 15:16:40 +0800 Subject: [PATCH 05/84] =?UTF-8?q?feat=EF=BC=9A=20add=20openGauss=20PQ=20ac?= =?UTF-8?q?celeration=20feature=20(#16432)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: chenhuan --- .../middleware/vdb/opengauss_config.py | 5 +++ .../rag/datasource/vdb/opengauss/opengauss.py | 31 +++++++++++++++++-- docker/.env.example | 1 + docker/docker-compose.yaml | 1 + 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/api/configs/middleware/vdb/opengauss_config.py b/api/configs/middleware/vdb/opengauss_config.py index 5ded627fb4..87ea292ab4 100644 --- a/api/configs/middleware/vdb/opengauss_config.py +++ b/api/configs/middleware/vdb/opengauss_config.py @@ -43,3 +43,8 @@ class OpenGaussConfig(BaseSettings): description="Max connection of the OpenGauss database", default=5, ) + + OPENGAUSS_ENABLE_PQ: bool = Field( + description="Enable openGauss PQ acceleration feature", + default=False, + ) diff --git a/api/core/rag/datasource/vdb/opengauss/opengauss.py b/api/core/rag/datasource/vdb/opengauss/opengauss.py index 2e5b7a31e4..4d57a651d5 100644 --- a/api/core/rag/datasource/vdb/opengauss/opengauss.py +++ b/api/core/rag/datasource/vdb/opengauss/opengauss.py @@ -25,6 +25,7 @@ class OpenGaussConfig(BaseModel): database: str min_connection: int max_connection: int + enable_pq: bool = False # Enable PQ acceleration @model_validator(mode="before") @classmethod @@ -57,6 +58,11 @@ CREATE TABLE IF NOT EXISTS {table_name} ( ); """ +SQL_CREATE_INDEX_PQ = """ +CREATE INDEX IF NOT EXISTS embedding_{table_name}_pq_idx ON {table_name} +USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64, enable_pq=on, pq_m={pq_m}); +""" + SQL_CREATE_INDEX = """ CREATE INDEX IF NOT EXISTS embedding_cosine_{table_name}_idx ON {table_name} USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64); @@ -68,6 +74,7 @@ class OpenGauss(BaseVector): super().__init__(collection_name) self.pool = self._create_connection_pool(config) self.table_name = f"embedding_{collection_name}" + self.pq_enabled = config.enable_pq def get_type(self) -> str: return VectorType.OPENGAUSS @@ -97,7 +104,26 @@ class OpenGauss(BaseVector): def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): dimension = len(embeddings[0]) self._create_collection(dimension) - return self.add_texts(texts, embeddings) + self.add_texts(texts, embeddings) + self._create_index(dimension) + + def _create_index(self, dimension: int): + index_cache_key = f"vector_index_{self._collection_name}" + lock_name = f"{index_cache_key}_lock" + with redis_client.lock(lock_name, timeout=60): + index_exist_cache_key = f"vector_index_{self._collection_name}" + if redis_client.get(index_exist_cache_key): + return + + with self._get_cursor() as cur: + if dimension <= 2000: + if self.pq_enabled: + cur.execute(SQL_CREATE_INDEX_PQ.format(table_name=self.table_name, pq_m=int(dimension / 4))) + cur.execute("SET hnsw_earlystop_threshold = 320") + + if not self.pq_enabled: + cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name)) + redis_client.set(index_exist_cache_key, 1, ex=3600) def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs): values = [] @@ -211,8 +237,6 @@ class OpenGauss(BaseVector): with self._get_cursor() as cur: cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name, dimension=dimension)) - if dimension <= 2000: - cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name)) redis_client.set(collection_exist_cache_key, 1, ex=3600) @@ -236,5 +260,6 @@ class OpenGaussFactory(AbstractVectorFactory): database=dify_config.OPENGAUSS_DATABASE or "dify", min_connection=dify_config.OPENGAUSS_MIN_CONNECTION, max_connection=dify_config.OPENGAUSS_MAX_CONNECTION, + enable_pq=dify_config.OPENGAUSS_ENABLE_PQ or False, ), ) diff --git a/docker/.env.example b/docker/.env.example index 6efad1bc9c..2e069db9e0 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -563,6 +563,7 @@ OPENGAUSS_PASSWORD=Dify@123 OPENGAUSS_DATABASE=dify OPENGAUSS_MIN_CONNECTION=1 OPENGAUSS_MAX_CONNECTION=5 +OPENGAUSS_ENABLE_PQ=false # Upstash Vector configuration, only available when VECTOR_STORE is `upstash` UPSTASH_VECTOR_URL=https://xxx-vector.upstash.io diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 138348659d..408b30a2fd 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -259,6 +259,7 @@ x-shared-env: &shared-api-worker-env OPENGAUSS_DATABASE: ${OPENGAUSS_DATABASE:-dify} OPENGAUSS_MIN_CONNECTION: ${OPENGAUSS_MIN_CONNECTION:-1} OPENGAUSS_MAX_CONNECTION: ${OPENGAUSS_MAX_CONNECTION:-5} + OPENGAUSS_ENABLE_PQ: ${OPENGAUSS_ENABLE_PQ:-false} UPSTASH_VECTOR_URL: ${UPSTASH_VECTOR_URL:-https://xxx-vector.upstash.io} UPSTASH_VECTOR_TOKEN: ${UPSTASH_VECTOR_TOKEN:-dify} UPLOAD_FILE_SIZE_LIMIT: ${UPLOAD_FILE_SIZE_LIMIT:-15} From 05eaef84bbbd1fb851880e3a8d8faeaf1e8960b1 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Mon, 24 Mar 2025 15:17:12 +0800 Subject: [PATCH 06/84] fix: cancel marketplace debounced search when clear search keywords (#16614) --- web/app/components/plugins/marketplace/context.tsx | 4 +++- web/app/components/plugins/marketplace/hooks.ts | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/web/app/components/plugins/marketplace/context.tsx b/web/app/components/plugins/marketplace/context.tsx index cba3835513..47da7ee160 100644 --- a/web/app/components/plugins/marketplace/context.tsx +++ b/web/app/components/plugins/marketplace/context.tsx @@ -143,6 +143,7 @@ export const MarketplaceContextProvider = ({ resetPlugins, queryPlugins, queryPluginsWithDebounced, + cancelQueryPluginsWithDebounced, isLoading: isPluginsLoading, } = useMarketplacePlugins() @@ -209,12 +210,13 @@ export const MarketplaceContextProvider = ({ const handleQuery = useCallback((debounced?: boolean) => { if (!searchPluginTextRef.current && !filterPluginTagsRef.current.length) { + cancelQueryPluginsWithDebounced() handleQueryMarketplaceCollectionsAndPlugins() return } handleQueryPlugins(debounced) - }, [handleQueryMarketplaceCollectionsAndPlugins, handleQueryPlugins]) + }, [handleQueryMarketplaceCollectionsAndPlugins, handleQueryPlugins, cancelQueryPluginsWithDebounced]) const handleSearchPluginTextChange = useCallback((text: string) => { setSearchPluginText(text) diff --git a/web/app/components/plugins/marketplace/hooks.ts b/web/app/components/plugins/marketplace/hooks.ts index 83a6709a47..c581ee0d91 100644 --- a/web/app/components/plugins/marketplace/hooks.ts +++ b/web/app/components/plugins/marketplace/hooks.ts @@ -89,7 +89,7 @@ export const useMarketplacePlugins = () => { handleUpdatePlugins(pluginsSearchParams) }, [handleUpdatePlugins]) - const { run: queryPluginsWithDebounced } = useDebounceFn((pluginsSearchParams: PluginsSearchParams) => { + const { run: queryPluginsWithDebounced, cancel: cancelQueryPluginsWithDebounced } = useDebounceFn((pluginsSearchParams: PluginsSearchParams) => { handleUpdatePlugins(pluginsSearchParams) }, { wait: 500, @@ -101,6 +101,7 @@ export const useMarketplacePlugins = () => { resetPlugins, queryPlugins, queryPluginsWithDebounced, + cancelQueryPluginsWithDebounced, isLoading: isPending, } } From 815d77856d1774b8fb975f697e0bcb9313a49e9b Mon Sep 17 00:00:00 2001 From: KVOJJJin Date: Mon, 24 Mar 2025 15:20:18 +0800 Subject: [PATCH 07/84] Fix: show feedback status in conversation (#16615) --- .../components/base/chat/chat/answer/operation.tsx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/web/app/components/base/chat/chat/answer/operation.tsx b/web/app/components/base/chat/chat/answer/operation.tsx index d03791ddf2..0c8d23de33 100644 --- a/web/app/components/base/chat/chat/answer/operation.tsx +++ b/web/app/components/base/chat/chat/answer/operation.tsx @@ -81,13 +81,13 @@ const Operation: FC = ({ const operationWidth = useMemo(() => { let width = 0 if (!isOpeningStatement) - width += 28 + width += 26 if (!isOpeningStatement && showPromptLog) - width += 102 + 8 + width += 28 + 8 if (!isOpeningStatement && config?.text_to_speech?.enabled) - width += 33 + width += 26 if (!isOpeningStatement && config?.supportAnnotation && config?.annotation_reply?.enabled) - width += 56 + 8 + width += 26 if (config?.supportFeedback && !localFeedback?.rating && onFeedback && !isOpeningStatement) width += 60 + 8 if (config?.supportFeedback && localFeedback?.rating && onFeedback && !isOpeningStatement) @@ -140,7 +140,7 @@ const Operation: FC = ({ )}
)} - {!isOpeningStatement && config?.supportFeedback && onFeedback && ( + {!isOpeningStatement && config?.supportFeedback && !localFeedback?.rating && onFeedback && (
{!localFeedback?.rating && ( <> @@ -152,6 +152,10 @@ const Operation: FC = ({ )} +
+ )} + {!isOpeningStatement && config?.supportFeedback && localFeedback?.rating && onFeedback && ( +
{localFeedback?.rating === 'like' && ( handleFeedback(null)}> From bc6f1223641ec6b6d333a24558cb6f0a71a032fb Mon Sep 17 00:00:00 2001 From: KVOJJJin Date: Mon, 24 Mar 2025 15:54:00 +0800 Subject: [PATCH 08/84] Fix: style issue of app detail panel in jp (#16620) --- web/app/components/app-sidebar/app-info.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/app-sidebar/app-info.tsx b/web/app/components/app-sidebar/app-info.tsx index edcd4c02e5..93f749a565 100644 --- a/web/app/components/app-sidebar/app-info.tsx +++ b/web/app/components/app-sidebar/app-info.tsx @@ -242,7 +242,7 @@ const AppInfo = ({ expand }: IAppInfoProps) => {
{appDetail.description}
)} {/* operations */} -
+
+ // need to wrapper the button with div when manualClose is true +
+ +
} btnElement={
From 2174225259d80655be08fa597bd8af202c5006f0 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Tue, 25 Mar 2025 16:22:43 +0800 Subject: [PATCH 26/84] fix milvus filter search (#16725) --- api/core/rag/datasource/vdb/milvus/milvus_vector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/core/rag/datasource/vdb/milvus/milvus_vector.py b/api/core/rag/datasource/vdb/milvus/milvus_vector.py index a1180a6505..7a3319f4a6 100644 --- a/api/core/rag/datasource/vdb/milvus/milvus_vector.py +++ b/api/core/rag/datasource/vdb/milvus/milvus_vector.py @@ -231,8 +231,8 @@ class MilvusVector(BaseVector): document_ids_filter = kwargs.get("document_ids_filter") filter = "" if document_ids_filter: - document_ids = ", ".join(f"'{id}'" for id in document_ids_filter) - filter = f'metadata["document_id"] in ({document_ids})' + document_ids = ", ".join(f'"{id}"' for id in document_ids_filter) + filter = f'metadata["document_id"] in [{document_ids}]' results = self._client.search( collection_name=self._collection_name, data=[query_vector], @@ -259,7 +259,7 @@ class MilvusVector(BaseVector): filter = "" if document_ids_filter: document_ids = ", ".join(f"'{id}'" for id in document_ids_filter) - filter = f'metadata["document_id"] in ({document_ids})' + filter = f'metadata["document_id"] in [{document_ids}]' results = self._client.search( collection_name=self._collection_name, From 58c62f0a34b540c621ad42971c574692aaccb3b0 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Tue, 25 Mar 2025 16:26:14 +0800 Subject: [PATCH 27/84] fix full-doc mode document doesn't reindex after enable or un_archive (#16737) --- api/core/indexing_runner.py | 2 +- api/models/dataset.py | 17 +++++++++++++++++ api/tasks/add_document_to_index_task.py | 2 +- api/tasks/deal_dataset_vector_index_task.py | 2 +- api/tasks/enable_segment_to_index_task.py | 2 +- api/tasks/enable_segments_to_index_task.py | 2 +- 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 8206a8d3ec..a75a4c22d1 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -187,7 +187,7 @@ class IndexingRunner: }, ) if dataset_document.doc_form == IndexType.PARENT_CHILD_INDEX: - child_chunks = document_segment.child_chunks + child_chunks = document_segment.get_child_chunks() if child_chunks: child_documents = [] for child_chunk in child_chunks: diff --git a/api/models/dataset.py b/api/models/dataset.py index f104c32b53..47f96c669e 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -720,6 +720,23 @@ class DocumentSegment(db.Model): # type: ignore[name-defined] else: return [] + def get_child_chunks(self): + process_rule = self.document.dataset_process_rule + if process_rule.mode == "hierarchical": + rules = Rule(**process_rule.rules_dict) + if rules.parent_mode: + child_chunks = ( + db.session.query(ChildChunk) + .filter(ChildChunk.segment_id == self.id) + .order_by(ChildChunk.position.asc()) + .all() + ) + return child_chunks or [] + else: + return [] + else: + return [] + @property def sign_content(self): return self.get_sign_content() diff --git a/api/tasks/add_document_to_index_task.py b/api/tasks/add_document_to_index_task.py index c5a5ddaadc..3b8b0466b1 100644 --- a/api/tasks/add_document_to_index_task.py +++ b/api/tasks/add_document_to_index_task.py @@ -59,7 +59,7 @@ def add_document_to_index_task(dataset_document_id: str): }, ) if dataset_document.doc_form == IndexType.PARENT_CHILD_INDEX: - child_chunks = segment.child_chunks + child_chunks = segment.get_child_chunks() if child_chunks: child_documents = [] for child_chunk in child_chunks: diff --git a/api/tasks/deal_dataset_vector_index_task.py b/api/tasks/deal_dataset_vector_index_task.py index a9b5ab91a8..5fd8647da8 100644 --- a/api/tasks/deal_dataset_vector_index_task.py +++ b/api/tasks/deal_dataset_vector_index_task.py @@ -130,7 +130,7 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str): }, ) if dataset_document.doc_form == IndexType.PARENT_CHILD_INDEX: - child_chunks = segment.child_chunks + child_chunks = segment.get_child_chunks() if child_chunks: child_documents = [] for child_chunk in child_chunks: diff --git a/api/tasks/enable_segment_to_index_task.py b/api/tasks/enable_segment_to_index_task.py index 76522f4720..0601e594fe 100644 --- a/api/tasks/enable_segment_to_index_task.py +++ b/api/tasks/enable_segment_to_index_task.py @@ -63,7 +63,7 @@ def enable_segment_to_index_task(segment_id: str): index_processor = IndexProcessorFactory(dataset_document.doc_form).init_index_processor() if dataset_document.doc_form == IndexType.PARENT_CHILD_INDEX: - child_chunks = segment.child_chunks + child_chunks = segment.get_child_chunks() if child_chunks: child_documents = [] for child_chunk in child_chunks: diff --git a/api/tasks/enable_segments_to_index_task.py b/api/tasks/enable_segments_to_index_task.py index 3942268afe..5129dbd24e 100644 --- a/api/tasks/enable_segments_to_index_task.py +++ b/api/tasks/enable_segments_to_index_task.py @@ -67,7 +67,7 @@ def enable_segments_to_index_task(segment_ids: list, dataset_id: str, document_i ) if dataset_document.doc_form == IndexType.PARENT_CHILD_INDEX: - child_chunks = segment.child_chunks + child_chunks = segment.get_child_chunks() if child_chunks: child_documents = [] for child_chunk in child_chunks: From 05c6d57f29a0724003e64dfe9c0c939a65ecd17c Mon Sep 17 00:00:00 2001 From: Novice <857526207@qq.com> Date: Tue, 25 Mar 2025 16:47:23 +0800 Subject: [PATCH 28/84] fix(slider): the slider's style is not as expected (#16711) --- web/app/components/base/slider/index.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/app/components/base/slider/index.tsx b/web/app/components/base/slider/index.tsx index 0bc89264e1..4e83ac5702 100644 --- a/web/app/components/base/slider/index.tsx +++ b/web/app/components/base/slider/index.tsx @@ -33,7 +33,8 @@ const Slider: React.FC = ({ step={step || 1} className={cn('slider relative', className)} thumbClassName={cn('absolute top-[-9px] h-5 w-2 rounded-[3px] border-[0.5px] border-components-slider-knob-border bg-components-slider-knob shadow-sm focus:outline-none', !disabled && 'cursor-pointer', thumbClassName)} - trackClassName={cn('slider-track h-0.5 rounded-full', trackClassName)} + // eslint-disable-next-line tailwindcss/classnames-order + trackClassName={cn('h-0.5 rounded-full slider-track', trackClassName)} onChange={onChange} /> } From 0811a23cd4ad7263c2ce51c348d2bc6e6098353d Mon Sep 17 00:00:00 2001 From: GuanMu Date: Tue, 25 Mar 2025 16:48:25 +0800 Subject: [PATCH 29/84] fix: trim whitespace from URL input in file uploader component #16700 (#16722) --- .../base/file-uploader/file-from-link-or-local/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/base/file-uploader/file-from-link-or-local/index.tsx b/web/app/components/base/file-uploader/file-from-link-or-local/index.tsx index 294044f764..4f11923fea 100644 --- a/web/app/components/base/file-uploader/file-from-link-or-local/index.tsx +++ b/web/app/components/base/file-uploader/file-from-link-or-local/index.tsx @@ -74,7 +74,7 @@ const FileFromLinkOrLocal = ({ value={url} onChange={(e) => { setShowError(false) - setUrl(e.target.value) + setUrl(e.target.value.trim()) }} disabled={disabled} /> From ac3577bc568ee49770892d53c3bbb76d720b8050 Mon Sep 17 00:00:00 2001 From: Novice <857526207@qq.com> Date: Tue, 25 Mar 2025 17:09:59 +0800 Subject: [PATCH 30/84] chore(slider): fix the slider lint error (#16746) --- web/app/components/base/slider/index.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/app/components/base/slider/index.tsx b/web/app/components/base/slider/index.tsx index 4e83ac5702..2cfbf80363 100644 --- a/web/app/components/base/slider/index.tsx +++ b/web/app/components/base/slider/index.tsx @@ -33,8 +33,7 @@ const Slider: React.FC = ({ step={step || 1} className={cn('slider relative', className)} thumbClassName={cn('absolute top-[-9px] h-5 w-2 rounded-[3px] border-[0.5px] border-components-slider-knob-border bg-components-slider-knob shadow-sm focus:outline-none', !disabled && 'cursor-pointer', thumbClassName)} - // eslint-disable-next-line tailwindcss/classnames-order - trackClassName={cn('h-0.5 rounded-full slider-track', trackClassName)} + trackClassName={cn('h-0.5 rounded-full', 'slider-track', trackClassName)} onChange={onChange} /> } From 0277a37fcad5ad86aeb239485c27fffd5cd90043 Mon Sep 17 00:00:00 2001 From: Yongtao Huang Date: Tue, 25 Mar 2025 18:53:09 +0800 Subject: [PATCH 31/84] Fix some typos in CONTRIBUTING.md (#16761) --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea66497017..5d4ba36485 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ Need to update an existing model runtime, tool, or squash some bugs? Head over t Join the fun, contribute, and let's build something awesome together! 💡✨ -Don't forget to link an existing issue or open an new issue in the PR's description. +Don't forget to link an existing issue or open a new issue in the PR's description. ### Bug reports @@ -68,7 +68,7 @@ How we prioritize: 4. Please add tests for your changes accordingly 5. Ensure your code passes the existing tests 6. Please link the issue in the PR description, `fixes #` -7. Get merrged! +7. Get merged! ### Setup the project #### Frontend @@ -90,4 +90,4 @@ We recommend reviewing this document carefully before proceeding with the setup, Feel free to reach out if you encounter any issues during the setup process. ## Getting Help -If you ever get stuck or got a burning question while contributing, simply shoot your queries our way via the related GitHub issue, or hop onto our [Discord](https://discord.gg/8Tpq4AcN9c) for a quick chat. +If you ever get stuck or get a burning question while contributing, simply shoot your queries our way via the related GitHub issue, or hop onto our [Discord](https://discord.gg/8Tpq4AcN9c) for a quick chat. From cb12b4436ff7fa57d697ba9659300ff345612e89 Mon Sep 17 00:00:00 2001 From: liuzhenghua <1090179900@qq.com> Date: Tue, 25 Mar 2025 18:56:53 +0800 Subject: [PATCH 32/84] fix: provider credentials load error (#16695) --- api/core/tools/builtin_tool/provider.py | 2 +- .../tools/builtin_tool/providers/webscraper/webscraper.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/tools/builtin_tool/provider.py b/api/core/tools/builtin_tool/provider.py index e776258527..037d86c89b 100644 --- a/api/core/tools/builtin_tool/provider.py +++ b/api/core/tools/builtin_tool/provider.py @@ -35,7 +35,7 @@ class BuiltinToolProviderController(ToolProviderController): provider_yaml["credentials_for_provider"][credential_name]["name"] = credential_name credentials_schema = [] - for credential in provider_yaml.get("credentials_for_provider", {}): + for credential in provider_yaml.get("credentials_for_provider", {}).values(): credentials_schema.append(credential) super().__init__( diff --git a/api/core/tools/builtin_tool/providers/webscraper/webscraper.yaml b/api/core/tools/builtin_tool/providers/webscraper/webscraper.yaml index d6d0a0d610..96edcf42fe 100644 --- a/api/core/tools/builtin_tool/providers/webscraper/webscraper.yaml +++ b/api/core/tools/builtin_tool/providers/webscraper/webscraper.yaml @@ -12,4 +12,4 @@ identity: icon: icon.svg tags: - productivity -credentials_for_provider: [] +credentials_for_provider: {} From bbf1639c63ad49f16a1414fcf8ff2d96234527ad Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Tue, 25 Mar 2025 20:31:01 +0800 Subject: [PATCH 33/84] feat: add jp_ja for knowledge api (#16766) --- web/app/(commonLayout)/datasets/Doc.tsx | 15 +- .../datasets/template/template.ja.mdx | 1977 +++++++++++++++++ 2 files changed, 1988 insertions(+), 4 deletions(-) create mode 100644 web/app/(commonLayout)/datasets/template/template.ja.mdx diff --git a/web/app/(commonLayout)/datasets/Doc.tsx b/web/app/(commonLayout)/datasets/Doc.tsx index 00ab967dc4..57d4b8dfef 100644 --- a/web/app/(commonLayout)/datasets/Doc.tsx +++ b/web/app/(commonLayout)/datasets/Doc.tsx @@ -6,6 +6,7 @@ import { useTranslation } from 'react-i18next' import { RiListUnordered } from '@remixicon/react' import TemplateEn from './template/template.en.mdx' import TemplateZh from './template/template.zh.mdx' +import TemplateJa from './template/template.ja.mdx' import I18n from '@/context/i18n' import { LanguagesSupported } from '@/i18n/language' @@ -106,10 +107,16 @@ const Doc = ({ apiBaseUrl }: DocProps) => { )}
- {locale !== LanguagesSupported[1] - ? - : - } + {(() => { + switch (locale) { + case LanguagesSupported[1]: + return + case LanguagesSupported[7]: + return + default: + return + } + })()}
) diff --git a/web/app/(commonLayout)/datasets/template/template.ja.mdx b/web/app/(commonLayout)/datasets/template/template.ja.mdx new file mode 100644 index 0000000000..4fe13b156f --- /dev/null +++ b/web/app/(commonLayout)/datasets/template/template.ja.mdx @@ -0,0 +1,1977 @@ +import { CodeGroup } from '@/app/components/develop/code.tsx' +import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstruction, Paragraph } from '@/app/components/develop/md.tsx' + +# ナレッジ API + +
+ ### 認証 + + Dify のサービス API は `API-Key` を使用して認証します。 + + 開発者は、`API-Key` をクライアント側で共有または保存するのではなく、バックエンドに保存することを推奨します。これにより、`API-Key` の漏洩による財産損失を防ぐことができます。 + + すべての API リクエストには、以下のように **`Authorization`** HTTP ヘッダーに `API-Key` を含める必要があります： + + + ```javascript + Authorization: Bearer {API_KEY} + + ``` + +
+ +
+ + + + + この API は既存のナレッジに基づいており、このナレッジを基にテキストを使用して新しいドキュメントを作成します。 + + ### パラメータ + + + ナレッジ ID + + + + ### リクエストボディ + + + ドキュメント名 + + + ドキュメント内容 + + + インデックスモード + - high_quality 高品質: 埋め込みモデルを使用してベクトルデータベースインデックスを構築 + - economy 経済: キーワードテーブルインデックスの反転インデックスを構築 + + + インデックス化された内容の形式 + - text_model テキストドキュメントは直接埋め込まれます; `economy` モードではこの形式がデフォルト + - hierarchical_model 親子モード + - qa_model Q&A モード: 分割されたドキュメントの質問と回答ペアを生成し、質問を埋め込みます + + + Q&A モードでは、ドキュメントの言語を指定します。例: English, Chinese + + + 処理ルール + - mode (string) クリーニング、セグメンテーションモード、自動 / カスタム + - rules (object) カスタムルール (自動モードでは、このフィールドは空) + - pre_processing_rules (array[object]) 前処理ルール + - id (string) 前処理ルールの一意識別子 + - 列挙 + - remove_extra_spaces 連続するスペース、改行、タブを置換 + - remove_urls_emails URL、メールアドレスを削除 + - enabled (bool) このルールを選択するかどうか。ドキュメント ID が渡されない場合、デフォルト値を表します。 + - segmentation (object) セグメンテーションルール + - separator カスタムセグメント識別子。現在は 1 つの区切り文字のみ設定可能。デフォルトは \n + - max_tokens 最大長 (トークン) デフォルトは 1000 + - parent_mode 親チャンクの検索モード: full-doc 全文検索 / paragraph 段落検索 + - subchunk_segmentation (object) 子チャンクルール + - separator セグメンテーション識別子。現在は 1 つの区切り文字のみ許可。デフォルトは *** + - max_tokens 最大長 (トークン) は親チャンクの長さより短いことを検証する必要があります + - chunk_overlap 隣接するチャンク間の重複を定義 (オプション) + + ナレッジベースにパラメータが設定されていない場合、最初のアップロードには以下のパラメータを提供する必要があります。提供されない場合、デフォルトパラメータが使用されます。 + + 検索モデル + - search_method (string) 検索方法 + - hybrid_search ハイブリッド検索 + - semantic_search セマンティック検索 + - full_text_search 全文検索 + - reranking_enable (bool) 再ランキングを有効にするかどうか + - reranking_mode (object) 再ランキングモデル構成 + - reranking_provider_name (string) 再ランキングモデルプロバイダー + - reranking_model_name (string) 再ランキングモデル名 + - top_k (int) 返される結果の数 + - score_threshold_enabled (bool) スコア閾値を有効にするかどうか + - score_threshold (float) スコア閾値 + + + 埋め込みモデル名 + + + 埋め込みモデルプロバイダー + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/create-by-text' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "name": "text", + "text": "text", + "indexing_technique": "high_quality", + "process_rule": { + "mode": "automatic" + } + }' + ``` + + + ```json {{ title: 'Response' }} + { + "document": { + "id": "", + "position": 1, + "data_source_type": "upload_file", + "data_source_info": { + "upload_file_id": "" + }, + "dataset_process_rule_id": "", + "name": "text.txt", + "created_from": "api", + "created_by": "", + "created_at": 1695690280, + "tokens": 0, + "indexing_status": "waiting", + "error": null, + "enabled": true, + "disabled_at": null, + "disabled_by": null, + "archived": false, + "display_status": "queuing", + "word_count": 0, + "hit_count": 0, + "doc_form": "text_model" + }, + "batch": "" + } + ``` + + + + +
+ + + + + この API は既存のナレッジに基づいており、このナレッジを基にファイルを使用して新しいドキュメントを作成します。 + + ### パラメータ + + + ナレッジ ID + + + + ### リクエストボディ + + + - original_document_id 元のドキュメント ID (オプション) + - ドキュメントを再アップロードまたはクリーニングとセグメンテーション構成を変更するために使用されます。欠落している情報は元のドキュメントからコピーされます。 + - 元のドキュメントはアーカイブされたドキュメントであってはなりません。 + - original_document_id が渡された場合、更新操作が実行されます。process_rule は入力可能な項目です。入力されない場合、元のドキュメントのセグメンテーション方法がデフォルトで使用されます。 + - original_document_id が渡されない場合、新しい操作が実行され、process_rule が必要です。 + + - indexing_technique インデックスモード + - high_quality 高品質: 埋め込みモデルを使用してベクトルデータベースインデックスを構築 + - economy 経済: キーワードテーブルインデックスの反転インデックスを構築 + + - doc_form インデックス化された内容の形式 + - text_model テキストドキュメントは直接埋め込まれます; `economy` モードではこの形式がデフォルト + - hierarchical_model 親子モード + - qa_model Q&A モード: 分割されたドキュメントの質問と回答ペアを生成し、質問を埋め込みます + + - doc_language Q&A モードでは、ドキュメントの言語を指定します。例: English, Chinese + + - process_rule 処理ルール + - mode (string) クリーニング、セグメンテーションモード、自動 / カスタム + - rules (object) カスタムルール (自動モードでは、このフィールドは空) + - pre_processing_rules (array[object]) 前処理ルール + - id (string) 前処理ルールの一意識別子 + - 列挙 + - remove_extra_spaces 連続するスペース、改行、タブを置換 + - remove_urls_emails URL、メールアドレスを削除 + - enabled (bool) このルールを選択するかどうか。ドキュメント ID が渡されない場合、デフォルト値を表します。 + - segmentation (object) セグメンテーションルール + - separator カスタムセグメント識別子。現在は 1 つの区切り文字のみ設定可能。デフォルトは \n + - max_tokens 最大長 (トークン) デフォルトは 1000 + - parent_mode 親チャンクの検索モード: full-doc 全文検索 / paragraph 段落検索 + - subchunk_segmentation (object) 子チャンクルール + - separator セグメンテーション識別子。現在は 1 つの区切り文字のみ許可。デフォルトは *** + - max_tokens 最大長 (トークン) は親チャンクの長さより短いことを検証する必要があります + - chunk_overlap 隣接するチャンク間の重複を定義 (オプション) + + + アップロードする必要があるファイル。 + + ナレッジベースにパラメータが設定されていない場合、最初のアップロードには以下のパラメータを提供する必要があります。提供されない場合、デフォルトパラメータが使用されます。 + + 検索モデル + - search_method (string) 検索方法 + - hybrid_search ハイブリッド検索 + - semantic_search セマンティック検索 + - full_text_search 全文検索 + - reranking_enable (bool) 再ランキングを有効にするかどうか + - reranking_mode (object) 再ランキングモデル構成 + - reranking_provider_name (string) 再ランキングモデルプロバイダー + - reranking_model_name (string) 再ランキングモデル名 + - top_k (int) 返される結果の数 + - score_threshold_enabled (bool) スコア閾値を有効にするかどうか + - score_threshold (float) スコア閾値 + + + 埋め込みモデル名 + + + 埋め込みモデルプロバイダー + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/create-by-file' \ + --header 'Authorization: Bearer {api_key}' \ + --form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \ + --form 'file=@"/path/to/file"' + ``` + + + ```json {{ title: 'Response' }} + { + "document": { + "id": "", + "position": 1, + "data_source_type": "upload_file", + "data_source_info": { + "upload_file_id": "" + }, + "dataset_process_rule_id": "", + "name": "Dify.txt", + "created_from": "api", + "created_by": "", + "created_at": 1695308667, + "tokens": 0, + "indexing_status": "waiting", + "error": null, + "enabled": true, + "disabled_at": null, + "disabled_by": null, + "archived": false, + "display_status": "queuing", + "word_count": 0, + "hit_count": 0, + "doc_form": "text_model" + }, + "batch": "" + } + ``` + + + + +
+ + + + + ### リクエストボディ + + + ナレッジ名 + + + ナレッジの説明 (オプション) + + + インデックス技術 (オプション) + - high_quality 高品質 + - economy 経済 + + + 権限 + - only_me 自分のみ + - all_team_members すべてのチームメンバー + - partial_members 一部のメンバー + + + プロバイダー (オプション、デフォルト: vendor) + - vendor ベンダー + - external 外部ナレッジ + + + 外部ナレッジ API ID (オプション) + + + 外部ナレッジ ID (オプション) + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request POST '${apiBaseUrl}/v1/datasets' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "name": "name", + "permission": "only_me" + }' + ``` + + + ```json {{ title: 'Response' }} + { + "id": "", + "name": "name", + "description": null, + "provider": "vendor", + "permission": "only_me", + "data_source_type": null, + "indexing_technique": null, + "app_count": 0, + "document_count": 0, + "word_count": 0, + "created_by": "", + "created_at": 1695636173, + "updated_by": "", + "updated_at": 1695636173, + "embedding_model": null, + "embedding_model_provider": null, + "embedding_available": null + } + ``` + + + + +
+ + + + + ### クエリ + + + ページ番号 + + + 返されるアイテム数、デフォルトは 20、範囲は 1-100 + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets?page=1&limit=20' \ + --header 'Authorization: Bearer {api_key}' + ``` + + + ```json {{ title: 'Response' }} + { + "data": [ + { + "id": "", + "name": "name", + "description": "desc", + "permission": "only_me", + "data_source_type": "upload_file", + "indexing_technique": "", + "app_count": 2, + "document_count": 10, + "word_count": 1200, + "created_by": "", + "created_at": "", + "updated_by": "", + "updated_at": "" + }, + ... + ], + "has_more": true, + "limit": 20, + "total": 50, + "page": 1 + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request DELETE '${props.apiBaseUrl}/datasets/{dataset_id}' \ + --header 'Authorization: Bearer {api_key}' + ``` + + + ```text {{ title: 'Response' }} + 204 No Content + ``` + + + + +
+ + + + + この API は既存のナレッジに基づいており、このナレッジを基にテキストを使用してドキュメントを更新します。 + + ### パラメータ + + + ナレッジ ID + + + ドキュメント ID + + + + ### リクエストボディ + + + ドキュメント名 (オプション) + + + ドキュメント内容 (オプション) + + + 処理ルール + - mode (string) クリーニング、セグメンテーションモード、自動 / カスタム + - rules (object) カスタムルール (自動モードでは、このフィールドは空) + - pre_processing_rules (array[object]) 前処理ルール + - id (string) 前処理ルールの一意識別子 + - 列挙 + - remove_extra_spaces 連続するスペース、改行、タブを置換 + - remove_urls_emails URL、メールアドレスを削除 + - enabled (bool) このルールを選択するかどうか。ドキュメント ID が渡されない場合、デフォルト値を表します。 + - segmentation (object) セグメンテーションルール + - separator カスタムセグメント識別子。現在は 1 つの区切り文字のみ設定可能。デフォルトは \n + - max_tokens 最大長 (トークン) デフォルトは 1000 + - parent_mode 親チャンクの検索モード: full-doc 全文検索 / paragraph 段落検索 + - subchunk_segmentation (object) 子チャンクルール + - separator セグメンテーション識別子。現在は 1 つの区切り文字のみ許可。デフォルトは *** + - max_tokens 最大長 (トークン) は親チャンクの長さより短いことを検証する必要があります + - chunk_overlap 隣接するチャンク間の重複を定義 (オプション) + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/update-by-text' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "name": "name", + "text": "text" + }' + ``` + + + ```json {{ title: 'Response' }} + { + "document": { + "id": "", + "position": 1, + "data_source_type": "upload_file", + "data_source_info": { + "upload_file_id": "" + }, + "dataset_process_rule_id": "", + "name": "name.txt", + "created_from": "api", + "created_by": "", + "created_at": 1695308667, + "tokens": 0, + "indexing_status": "waiting", + "error": null, + "enabled": true, + "disabled_at": null, + "disabled_by": null, + "archived": false, + "display_status": "queuing", + "word_count": 0, + "hit_count": 0, + "doc_form": "text_model" + }, + "batch": "" + } + ``` + + + + +
+ + + + + この API は既存のナレッジに基づいており、このナレッジを基にファイルを使用してドキュメントを更新します。 + + ### パラメータ + + + ナレッジ ID + + + ドキュメント ID + + + + ### リクエストボディ + + + ドキュメント名 (オプション) + + + アップロードするファイル + + + 処理ルール + - mode (string) クリーニング、セグメンテーションモード、自動 / カスタム + - rules (object) カスタムルール (自動モードでは、このフィールドは空) + - pre_processing_rules (array[object]) 前処理ルール + - id (string) 前処理ルールの一意識別子 + - 列挙 + - remove_extra_spaces 連続するスペース、改行、タブを置換 + - remove_urls_emails URL、メールアドレスを削除 + - enabled (bool) このルールを選択するかどうか。ドキュメント ID が渡されない場合、デフォルト値を表します。 + - segmentation (object) セグメンテーションルール + - separator カスタムセグメント識別子。現在は 1 つの区切り文字のみ設定可能。デフォルトは \n + - max_tokens 最大長 (トークン) デフォルトは 1000 + - parent_mode 親チャンクの検索モード: full-doc 全文検索 / paragraph 段落検索 + - subchunk_segmentation (object) 子チャンクルール + - separator セグメンテーション識別子。現在は 1 つの区切り文字のみ許可。デフォルトは *** + - max_tokens 最大長 (トークン) は親チャンクの長さより短いことを検証する必要があります + - chunk_overlap 隣接するチャンク間の重複を定義 (オプション) + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/update-by-file' \ + --header 'Authorization: Bearer {api_key}' \ + --form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \ + --form 'file=@"/path/to/file"' + ``` + + + ```json {{ title: 'Response' }} + { + "document": { + "id": "", + "position": 1, + "data_source_type": "upload_file", + "data_source_info": { + "upload_file_id": "" + }, + "dataset_process_rule_id": "", + "name": "Dify.txt", + "created_from": "api", + "created_by": "", + "created_at": 1695308667, + "tokens": 0, + "indexing_status": "waiting", + "error": null, + "enabled": true, + "disabled_at": null, + "disabled_by": null, + "archived": false, + "display_status": "queuing", + "word_count": 0, + "hit_count": 0, + "doc_form": "text_model" + }, + "batch": "20230921150427533684" + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + アップロードされたドキュメントのバッチ番号 + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{batch}/indexing-status' \ + --header 'Authorization: Bearer {api_key}' \ + ``` + + + ```json {{ title: 'Response' }} + { + "data":[{ + "id": "", + "indexing_status": "indexing", + "processing_started_at": 1681623462.0, + "parsing_completed_at": 1681623462.0, + "cleaning_completed_at": 1681623462.0, + "splitting_completed_at": 1681623462.0, + "completed_at": null, + "paused_at": null, + "error": null, + "stopped_at": null, + "completed_segments": 24, + "total_segments": 100 + }] + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + ドキュメント ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request DELETE '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}' \ + --header 'Authorization: Bearer {api_key}' \ + ``` + + + ```json {{ title: 'Response' }} + { + "result": "success" + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + + ### クエリ + + + 検索キーワード、現在はドキュメント名のみ検索 (オプション) + + + ページ番号 (オプション) + + + 返されるアイテム数、デフォルトは 20、範囲は 1-100 (オプション) + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents' \ + --header 'Authorization: Bearer {api_key}' \ + ``` + + + ```json {{ title: 'Response' }} + { + "data": [ + { + "id": "", + "position": 1, + "data_source_type": "file_upload", + "data_source_info": null, + "dataset_process_rule_id": null, + "name": "dify", + "created_from": "", + "created_by": "", + "created_at": 1681623639, + "tokens": 0, + "indexing_status": "waiting", + "error": null, + "enabled": true, + "disabled_at": null, + "disabled_by": null, + "archived": false + }, + ], + "has_more": false, + "limit": 20, + "total": 9, + "page": 1 + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + ドキュメント ID + + + + ### リクエストボディ + + + - content (text) テキスト内容 / 質問内容、必須 + - answer (text) 回答内容、ナレッジのモードが Q&A モードの場合に値を渡します (オプション) + - keywords (list) キーワード (オプション) + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/segments' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "segments": [ + { + "content": "1", + "answer": "1", + "keywords": ["a"] + } + ] + }' + ``` + + + ```json {{ title: 'Response' }} + { + "data": [{ + "id": "", + "position": 1, + "document_id": "", + "content": "1", + "answer": "1", + "word_count": 25, + "tokens": 0, + "keywords": [ + "a" + ], + "index_node_id": "", + "index_node_hash": "", + "hit_count": 0, + "enabled": true, + "disabled_at": null, + "disabled_by": null, + "status": "completed", + "created_by": "", + "created_at": 1695312007, + "indexing_at": 1695312007, + "completed_at": 1695312007, + "error": null, + "stopped_at": null + }], + "doc_form": "text_model" + } + ``` + + + + +
+ + + + + ### パス + + + ナレッジ ID + + + ドキュメント ID + + + + ### クエリ + + + キーワード (オプション) + + + 検索ステータス、completed + + + ページ番号 (オプション) + + + 返されるアイテム数、デフォルトは 20、範囲は 1-100 (オプション) + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/segments' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' + ``` + + + ```json {{ title: 'Response' }} + { + "data": [{ + "id": "", + "position": 1, + "document_id": "", + "content": "1", + "answer": "1", + "word_count": 25, + "tokens": 0, + "keywords": [ + "a" + ], + "index_node_id": "", + "index_node_hash": "", + "hit_count": 0, + "enabled": true, + "disabled_at": null, + "disabled_by": null, + "status": "completed", + "created_by": "", + "created_at": 1695312007, + "indexing_at": 1695312007, + "completed_at": 1695312007, + "error": null, + "stopped_at": null + }], + "doc_form": "text_model", + "has_more": false, + "limit": 20, + "total": 9, + "page": 1 + } + ``` + + + + +
+ + + + + ### パス + + + ナレッジ ID + + + ドキュメント ID + + + ドキュメントセグメント ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request DELETE '${props.apiBaseUrl}/datasets/{dataset_id}/segments/{segment_id}' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' + ``` + + + ```json {{ title: 'Response' }} + { + "result": "success" + } + ``` + + + + +
+ + + + + ### POST + + + ナレッジ ID + + + ドキュメント ID + + + ドキュメントセグメント ID + + + + ### リクエストボディ + + + - content (text) テキスト内容 / 質問内容、必須 + - answer (text) 回答内容、ナレッジが Q&A モードの場合に値を渡します (オプション) + - keywords (list) キーワード (オプション) + - enabled (bool) False / true (オプション) + - regenerate_child_chunks (bool) 子チャンクを再生成するかどうか (オプション) + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/segments/{segment_id}' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "segment": { + "content": "1", + "answer": "1", + "keywords": ["a"], + "enabled": false + } + }' + ``` + + + ```json {{ title: 'Response' }} + { + "data": { + "id": "", + "position": 1, + "document_id": "", + "content": "1", + "answer": "1", + "word_count": 25, + "tokens": 0, + "keywords": [ + "a" + ], + "index_node_id": "", + "index_node_hash": "", + "hit_count": 0, + "enabled": true, + "disabled_at": null, + "disabled_by": null, + "status": "completed", + "created_by": "", + "created_at": 1695312007, + "indexing_at": 1695312007, + "completed_at": 1695312007, + "error": null, + "stopped_at": null + }, + "doc_form": "text_model" + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + ドキュメント ID + + + セグメント ID + + + + ### リクエストボディ + + + 子チャンクの内容 + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/segments/{segment_id}/child_chunks' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "content": "Child chunk content" + }' + ``` + + + ```json {{ title: 'Response' }} + { + "data": { + "id": "", + "segment_id": "", + "content": "Child chunk content", + "word_count": 25, + "tokens": 0, + "index_node_id": "", + "index_node_hash": "", + "status": "completed", + "created_by": "", + "created_at": 1695312007, + "indexing_at": 1695312007, + "completed_at": 1695312007, + "error": null, + "stopped_at": null + } + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + ドキュメント ID + + + セグメント ID + + + + ### クエリ + + + 検索キーワード (オプション) + + + ページ番号 (オプション、デフォルト: 1) + + + ページあたりのアイテム数 (オプション、デフォルト: 20、最大: 100) + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/segments/{segment_id}/child_chunks?page=1&limit=20' \ + --header 'Authorization: Bearer {api_key}' + ``` + + + ```json {{ title: 'Response' }} + { + "data": [{ + "id": "", + "segment_id": "", + "content": "Child chunk content", + "word_count": 25, + "tokens": 0, + "index_node_id": "", + "index_node_hash": "", + "status": "completed", + "created_by": "", + "created_at": 1695312007, + "indexing_at": 1695312007, + "completed_at": 1695312007, + "error": null, + "stopped_at": null + }], + "total": 1, + "total_pages": 1, + "page": 1, + "limit": 20 + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + ドキュメント ID + + + セグメント ID + + + 子チャンク ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request DELETE '${props.apiBaseUrl}/datasets/{dataset_id}/segments/{segment_id}/child_chunks/{child_chunk_id}' \ + --header 'Authorization: Bearer {api_key}' + ``` + + + ```json {{ title: 'Response' }} + { + "result": "success" + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + ドキュメント ID + + + セグメント ID + + + 子チャンク ID + + + + ### リクエストボディ + + + 子チャンクの内容 + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request PATCH '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/segments/{segment_id}/child_chunks/{child_chunk_id}' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "content": "Updated child chunk content" + }' + ``` + + + ```json {{ title: 'Response' }} + { + "data": { + "id": "", + "segment_id": "", + "content": "Updated child chunk content", + "word_count": 25, + "tokens": 0, + "index_node_id": "", + "index_node_hash": "", + "status": "completed", + "created_by": "", + "created_at": 1695312007, + "indexing_at": 1695312007, + "completed_at": 1695312007, + "error": null, + "stopped_at": null + } + } + ``` + + + + +
+ + + + + ### パス + + + ナレッジ ID + + + ドキュメント ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/upload-file' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' + ``` + + + ```json {{ title: 'Response' }} + { + "id": "file_id", + "name": "file_name", + "size": 1024, + "extension": "txt", + "url": "preview_url", + "download_url": "download_url", + "mime_type": "text/plain", + "created_by": "user_id", + "created_at": 1728734540, + } + ``` + + + + +
+ + + + + ### パス + + + ナレッジ ID + + + + ### リクエストボディ + + + クエリキーワード + + + 検索モデル (オプション、入力されない場合はデフォルトの方法でリコールされます) + - search_method (text) 検索方法: 以下の 4 つのキーワードのいずれかが必要です + - keyword_search キーワード検索 + - semantic_search セマンティック検索 + - full_text_search 全文検索 + - hybrid_search ハイブリッド検索 + - reranking_enable (bool) 再ランキングを有効にするかどうか、検索モードが semantic_search または hybrid_search の場合に必須 (オプション) + - reranking_mode (object) 再ランキングモデル構成、再ランキングが有効な場合に必須 + - reranking_provider_name (string) 再ランキングモデルプロバイダー + - reranking_model_name (string) 再ランキングモデル名 + - weights (float) ハイブリッド検索モードでのセマンティック検索の重み設定 + - top_k (integer) 返される結果の数 (オプション) + - score_threshold_enabled (bool) スコア閾値を有効にするかどうか + - score_threshold (float) スコア閾値 + + + 未使用フィールド + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/retrieve' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "query": "test", + "retrieval_model": { + "search_method": "keyword_search", + "reranking_enable": false, + "reranking_mode": null, + "reranking_model": { + "reranking_provider_name": "", + "reranking_model_name": "" + }, + "weights": null, + "top_k": 2, + "score_threshold_enabled": false, + "score_threshold": null + } + }' + ``` + + + ```json {{ title: 'Response' }} + { + "query": { + "content": "test" + }, + "records": [ + { + "segment": { + "id": "7fa6f24f-8679-48b3-bc9d-bdf28d73f218", + "position": 1, + "document_id": "a8c6c36f-9f5d-4d7a-8472-f5d7b75d71d2", + "content": "Operation guide", + "answer": null, + "word_count": 847, + "tokens": 280, + "keywords": [ + "install", + "java", + "base", + "scripts", + "jdk", + "manual", + "internal", + "opens", + "add", + "vmoptions" + ], + "index_node_id": "39dd8443-d960-45a8-bb46-7275ad7fbc8e", + "index_node_hash": "0189157697b3c6a418ccf8264a09699f25858975578f3467c76d6bfc94df1d73", + "hit_count": 0, + "enabled": true, + "disabled_at": null, + "disabled_by": null, + "status": "completed", + "created_by": "dbcb1ab5-90c8-41a7-8b78-73b235eb6f6f", + "created_at": 1728734540, + "indexing_at": 1728734552, + "completed_at": 1728734584, + "error": null, + "stopped_at": null, + "document": { + "id": "a8c6c36f-9f5d-4d7a-8472-f5d7b75d71d2", + "data_source_type": "upload_file", + "name": "readme.txt", + } + }, + "score": 3.730463140527718e-05, + "tsne_position": null + } + ] + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + + ### リクエストボディ + + + - type (string) メタデータの種類、必須 + - name (string) メタデータの名前、必須 + + + + + + ```bash {{ title: 'cURL' }} + ``` + + + ```json {{ title: 'Response' }} + { + "id": "abc", + "type": "string", + "name": "test", + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + メタデータ ID + + + + ### リクエストボディ + + + - name (string) メタデータの名前、必須 + + + + + + ```bash {{ title: 'cURL' }} + ``` + + + ```json {{ title: 'Response' }} + { + "id": "abc", + "type": "string", + "name": "test", + } + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + メタデータ ID + + + + + + ```bash {{ title: 'cURL' }} + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + disable/enable + + + + + + ```bash {{ title: 'cURL' }} + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + + ### リクエストボディ + + + - document_id (string) ドキュメント ID + - metadata_list (list) メタデータリスト + - id (string) メタデータ ID + - value (string) メタデータの値 + - name (string) メタデータの名前 + + + + + + ```bash {{ title: 'cURL' }} + ``` + + + + +
+ + + + + ### パラメータ + + + ナレッジ ID + + + + + + ```bash {{ title: 'cURL' }} + ``` + + + ```json {{ title: 'Response' }} + { + "doc_metadata": [ + { + "id": "", + "name": "name", + "type": "string", + "use_count": 0, + }, + ... + ], + "built_in_field_enabled": true + } + ``` + + + + +
+ + + + ### エラーメッセージ + + + エラーコード + + + + + エラーステータス + + + + + エラーメッセージ + + + + + + ```json {{ title: 'Response' }} + { + "code": "no_file_uploaded", + "message": "Please upload your file.", + "status": 400 + } + ``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
codestatusmessage
no_file_uploaded400Please upload your file.
too_many_files400Only one file is allowed.
file_too_large413File size exceeded.
unsupported_file_type415File type not allowed.
high_quality_dataset_only400Current operation only supports 'high-quality' datasets.
dataset_not_initialized400The dataset is still being initialized or indexing. Please wait a moment.
archived_document_immutable403The archived document is not editable.
dataset_name_duplicate409The dataset name already exists. Please modify your dataset name.
invalid_action400Invalid action.
document_already_finished400The document has been processed. Please refresh the page or go to the document details.
document_indexing400The document is being processed and cannot be edited.
invalid_metadata400The metadata content is incorrect. Please check and verify.
+
From e0fc7f69dd4e18e824d9f4ffe55a70567af7ce9c Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Tue, 25 Mar 2025 21:23:44 +0800 Subject: [PATCH 34/84] fix the major and minor version of stub types declaration with build date stripped (#16709) --- api/poetry.lock | 96 +++++++++++++++++++++++++++++++++------------- api/pyproject.toml | 32 ++++++++-------- 2 files changed, 85 insertions(+), 43 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index 68516ca1f3..54809053b8 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -845,6 +845,10 @@ files = [ {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec"}, {file = "Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2"}, {file = "Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128"}, {file = "Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc"}, @@ -857,8 +861,14 @@ files = [ {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b"}, {file = "Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50"}, {file = "Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2"}, {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451"}, @@ -869,8 +879,24 @@ files = [ {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839"}, {file = "Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0"}, {file = "Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951"}, + {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5"}, + {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7"}, + {file = "Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0"}, + {file = "Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b"}, {file = "Brotli-1.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b"}, @@ -880,6 +906,10 @@ files = [ {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:aea440a510e14e818e67bfc4027880e2fb500c2ccb20ab21c7a7c8b5b4703d75"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:6974f52a02321b36847cd19d1b8e381bf39939c21efd6ee2fc13a28b0d99348c"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:a7e53012d2853a07a4a79c00643832161a910674a893d296c9f1259859a289d2"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:d7702622a8b40c49bffb46e1e3ba2e81268d5c04a34f460978c6b5517a34dd52"}, {file = "Brotli-1.1.0-cp36-cp36m-win32.whl", hash = "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460"}, {file = "Brotli-1.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579"}, {file = "Brotli-1.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c"}, @@ -891,6 +921,10 @@ files = [ {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:cb1dac1770878ade83f2ccdf7d25e494f05c9165f5246b46a621cc849341dc01"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:3ee8a80d67a4334482d9712b8e83ca6b1d9bc7e351931252ebef5d8f7335a547"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5e55da2c8724191e5b557f8e18943b1b4839b8efc3ef60d65985bcf6f587dd38"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:d342778ef319e1026af243ed0a07c97acf3bad33b9f29e7ae6a1f68fd083e90c"}, {file = "Brotli-1.1.0-cp37-cp37m-win32.whl", hash = "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95"}, {file = "Brotli-1.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68"}, {file = "Brotli-1.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3"}, @@ -903,6 +937,10 @@ files = [ {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d2b35ca2c7f81d173d2fadc2f4f31e88cc5f7a39ae5b6db5513cf3383b0e0ec7"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:af6fa6817889314555aede9a919612b23739395ce767fe7fcbea9a80bf140fe5"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:2feb1d960f760a575dbc5ab3b1c00504b24caaf6986e2dc2b01c09c87866a943"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4410f84b33374409552ac9b6903507cdb31cd30d2501fc5ca13d18f73548444a"}, {file = "Brotli-1.1.0-cp38-cp38-win32.whl", hash = "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b"}, {file = "Brotli-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0"}, {file = "Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a"}, @@ -915,6 +953,10 @@ files = [ {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb"}, {file = "Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64"}, {file = "Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467"}, {file = "Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724"}, @@ -8632,14 +8674,14 @@ files = [ [[package]] name = "types-flask-cors" -version = "5.0.0.20240902" +version = "4.0.0.20240828" description = "Typing stubs for Flask-Cors" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "types-Flask-Cors-5.0.0.20240902.tar.gz", hash = "sha256:8921b273bf7cd9636df136b66408efcfa6338a935e5c8f53f5eff1cee03f3394"}, - {file = "types_Flask_Cors-5.0.0.20240902-py3-none-any.whl", hash = "sha256:595e5f36056cd128ab905832e055f2e5d116fbdc685356eea4490bc77df82137"}, + {file = "types-Flask-Cors-4.0.0.20240828.tar.gz", hash = "sha256:f48ecf6366da923331311907cde3500e1435e07df01397ce0ef2306e263a5e85"}, + {file = "types_Flask_Cors-4.0.0.20240828-py3-none-any.whl", hash = "sha256:36b752e88d6517fb82973b4240fe9bde44d29485bbd92dfff762a7101bdac3a0"}, ] [package.dependencies] @@ -8687,26 +8729,26 @@ files = [ [[package]] name = "types-protobuf" -version = "5.29.1.20250315" +version = "4.25.0.20240417" description = "Typing stubs for protobuf" optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "types_protobuf-5.29.1.20250315-py3-none-any.whl", hash = "sha256:57efd51fd0979d1f5e1d94053d1e7cfff9c028e8d05b17e341b91a1c7fce37c4"}, - {file = "types_protobuf-5.29.1.20250315.tar.gz", hash = "sha256:0b05bc34621d046de54b94fddd5f4eb3bf849fe2e13a50f8fb8e89f35045ff49"}, -] - -[[package]] -name = "types-psutil" -version = "6.1.0.20241221" -description = "Typing stubs for psutil" -optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "types_psutil-6.1.0.20241221-py3-none-any.whl", hash = "sha256:8498dbe13285a9ba7d4b2fa934c569cc380efc74e3dacdb34ae16d2cdf389ec3"}, - {file = "types_psutil-6.1.0.20241221.tar.gz", hash = "sha256:600f5a36bd5e0eb8887f0e3f3ff2cf154d90690ad8123c8a707bba4ab94d3185"}, + {file = "types-protobuf-4.25.0.20240417.tar.gz", hash = "sha256:c34eff17b9b3a0adb6830622f0f302484e4c089f533a46e3f147568313544352"}, + {file = "types_protobuf-4.25.0.20240417-py3-none-any.whl", hash = "sha256:e9b613227c2127e3d4881d75d93c93b4d6fd97b5f6a099a0b654a05351c8685d"}, +] + +[[package]] +name = "types-psutil" +version = "7.0.0.20250218" +description = "Typing stubs for psutil" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_psutil-7.0.0.20250218-py3-none-any.whl", hash = "sha256:1447a30c282aafefcf8941ece854e1100eee7b0296a9d9be9977292f0269b121"}, + {file = "types_psutil-7.0.0.20250218.tar.gz", hash = "sha256:1e642cdafe837b240295b23b1cbd4691d80b08a07d29932143cbbae30eb0db9c"}, ] [[package]] @@ -8735,14 +8777,14 @@ files = [ [[package]] name = "types-pytz" -version = "2024.2.0.20241221" +version = "2025.1.0.20250318" description = "Typing stubs for pytz" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "types_pytz-2024.2.0.20241221-py3-none-any.whl", hash = "sha256:8fc03195329c43637ed4f593663df721fef919b60a969066e22606edf0b53ad5"}, - {file = "types_pytz-2024.2.0.20241221.tar.gz", hash = "sha256:06d7cde9613e9f7504766a0554a270c369434b50e00975b3a4a0f6eed0f2c1a9"}, + {file = "types_pytz-2025.1.0.20250318-py3-none-any.whl", hash = "sha256:04dba4907c5415777083f9548693c6d9f80ec53adcaff55a38526a3f8ddcae04"}, + {file = "types_pytz-2025.1.0.20250318.tar.gz", hash = "sha256:97e0e35184c6fe14e3a5014512057f2c57bb0c6582d63c1cfcc4809f82180449"}, ] [[package]] @@ -8771,14 +8813,14 @@ files = [ [[package]] name = "types-requests" -version = "2.32.0.20250306" +version = "2.31.0.20240406" description = "Typing stubs for requests" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "types_requests-2.32.0.20250306-py3-none-any.whl", hash = "sha256:25f2cbb5c8710b2022f8bbee7b2b66f319ef14aeea2f35d80f18c9dbf3b60a0b"}, - {file = "types_requests-2.32.0.20250306.tar.gz", hash = "sha256:0962352694ec5b2f95fda877ee60a159abdf84a0fc6fdace599f20acb41a03d1"}, + {file = "types-requests-2.31.0.20240406.tar.gz", hash = "sha256:4428df33c5503945c74b3f42e82b181e86ec7b724620419a2966e2de604ce1a1"}, + {file = "types_requests-2.31.0.20240406-py3-none-any.whl", hash = "sha256:6216cdac377c6b9a040ac1c0404f7284bd13199c0e1bb235f4324627e8898cf5"}, ] [package.dependencies] @@ -9905,4 +9947,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "adc577504435813e7e78b7433b9efb3dc6551f4eec2a65bc9aed762a8ef6540c" +content-hash = "bf69a3ba47aebb5ad9038c3c7abe8194c9557e4fdb622a2a19b9782c404c25ee" diff --git a/api/pyproject.toml b/api/pyproject.toml index 0e91e533b1..100b23dba3 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -150,22 +150,22 @@ pytest = "~8.3.2" pytest-benchmark = "~4.0.0" pytest-env = "~1.1.3" pytest-mock = "~3.14.0" -types-beautifulsoup4 = "~4.12.0.20241020" -types-deprecated = "~1.2.15.20250304" -types-flask-cors = "~5.0.0.20240902" -types-flask-migrate = "~4.1.0.20250112" -types-html5lib = "~1.1.11.20241018" -types-openpyxl = "~3.1.5.20241225" -types-protobuf = "~5.29.1.20241207" -types-psutil = "~6.1.0.20241221" -types-psycopg2 = "~2.9.21.20250121" -types-python-dateutil = "~2.9.0.20241206" -types-pytz = "~2024.2.0.20241221" -types-pyyaml = "~6.0.12.20241230" -types-regex = "~2024.11.6.20241221" -types-requests = "~2.32.0.20241016" -types-six = "~1.17.0.20241205" -types-tqdm = "~4.67.0.20241221" +types-beautifulsoup4 = "~4.12.0" +types-deprecated = "~1.2.15" +types-flask-cors = "~4.0.0" +types-flask-migrate = "~4.1.0" +types-html5lib = "~1.1.11" +types-openpyxl = "~3.1.5" +types-protobuf = "~4.25.0" +types-psutil = "~7.0.0" +types-psycopg2 = "~2.9.21" +types-python-dateutil = "~2.9.0" +types-pytz = "~2025.1" +types-pyyaml = "~6.0.2" +types-regex = "~2024.11.6" +types-requests = "~2.31.0" +types-six = "~1.17.0" +types-tqdm = "~4.67.0" ############################################################ # [ Lint ] dependency group From 0c2a459c304db6e4f3e15511a6275bbdd5a5a8d5 Mon Sep 17 00:00:00 2001 From: taokuizu Date: Wed, 26 Mar 2025 09:01:41 +0800 Subject: [PATCH 35/84] fix typo in _process_metadata_filter_func (#16780) --- api/core/rag/retrieval/dataset_retrieval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index d48da0f941..f88e8629f5 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -1003,7 +1003,7 @@ class DatasetRetrieval: filters.append(sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) < value) case "after" | ">": filters.append(sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) > value) - case "≤" | ">=": + case "≤" | "<=": filters.append(sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) <= value) case "≥" | ">=": filters.append(sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) >= value) From 37134c59877bd21b7db990bb39eb9e2d04ec4e71 Mon Sep 17 00:00:00 2001 From: Yongtao Huang Date: Wed, 26 Mar 2025 09:02:45 +0800 Subject: [PATCH 36/84] Remove the useless excluded item in mypy.ini (#16777) --- api/mypy.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/mypy.ini b/api/mypy.ini index 2c754f9fcd..2898b9b52d 100644 --- a/api/mypy.ini +++ b/api/mypy.ini @@ -3,8 +3,7 @@ warn_return_any = True warn_unused_configs = True check_untyped_defs = True exclude = (?x)( - core/tools/provider/builtin/ - | core/model_runtime/model_providers/ + core/model_runtime/model_providers/ | tests/ | migrations/ - ) \ No newline at end of file + ) From 71edaba9df3937ee759ef461c436d452627dce0a Mon Sep 17 00:00:00 2001 From: Good Wood Date: Wed, 26 Mar 2025 11:01:13 +0800 Subject: [PATCH 37/84] fix: fix ui layout when in mobile mode (#16793) --- .../base/chat/chat-with-history/chat-wrapper.tsx | 10 ++++++---- .../base/chat/chat/answer/suggested-questions.tsx | 7 ++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/web/app/components/base/chat/chat-with-history/chat-wrapper.tsx b/web/app/components/base/chat/chat-with-history/chat-wrapper.tsx index 3711da19f6..d6b4bdfafe 100644 --- a/web/app/components/base/chat/chat-with-history/chat-wrapper.tsx +++ b/web/app/components/base/chat/chat-with-history/chat-wrapper.tsx @@ -188,7 +188,7 @@ const ChatWrapper = () => { return null if (welcomeMessage.suggestedQuestions && welcomeMessage.suggestedQuestions?.length > 0) { return ( -
+
{ background={appData?.site.icon_background} imageUrl={appData?.site.icon_url} /> -
- - +
+
+ + +
diff --git a/web/app/components/base/chat/chat/answer/suggested-questions.tsx b/web/app/components/base/chat/chat/answer/suggested-questions.tsx index 11ef3e4d1c..7b8da0e9f0 100644 --- a/web/app/components/base/chat/chat/answer/suggested-questions.tsx +++ b/web/app/components/base/chat/chat/answer/suggested-questions.tsx @@ -3,6 +3,7 @@ import { memo } from 'react' import type { ChatItem } from '../../types' import { useChatContext } from '../context' import Button from '@/app/components/base/button' +import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' type SuggestedQuestionsProps = { item: ChatItem @@ -11,6 +12,10 @@ const SuggestedQuestions: FC = ({ item, }) => { const { onSend } = useChatContext() + const media = useBreakpoints() + const isMobile = media === MediaType.mobile + const klassName = `mr-1 mt-1 ${isMobile ? 'block overflow-hidden text-ellipsis' : ''} max-w-full shrink-0 last:mr-0` + const { isOpeningStatement, suggestedQuestions, @@ -25,7 +30,7 @@ const SuggestedQuestions: FC = ({