From 7b7ac7a495c952bf97331c8e4a506e50b0e8c0a7 Mon Sep 17 00:00:00 2001 From: Good Wood Date: Tue, 8 Apr 2025 09:24:42 +0800 Subject: [PATCH 01/53] fix: compatibility issues for currentStrategy.features is null (#17581) --- web/app/components/workflow/nodes/agent/panel.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/workflow/nodes/agent/panel.tsx b/web/app/components/workflow/nodes/agent/panel.tsx index 40c8338a7b..da87312a90 100644 --- a/web/app/components/workflow/nodes/agent/panel.tsx +++ b/web/app/components/workflow/nodes/agent/panel.tsx @@ -110,7 +110,7 @@ const AgentPanel: FC> = (props) => { />
- {isChatMode && currentStrategy?.features.includes(AgentFeature.HISTORY_MESSAGES) && ( + {isChatMode && currentStrategy?.features?.includes(AgentFeature.HISTORY_MESSAGES) && ( <> Date: Mon, 7 Apr 2025 23:15:07 -0400 Subject: [PATCH 02/53] feat: support select-type variables in Metadata Filtering (#17440 (#17445) --- .../components/app/configuration/dataset-config/index.tsx | 2 +- .../components/metadata/condition-list/condition-item.tsx | 8 ++++++-- .../components/metadata/condition-list/utils.ts | 1 + .../components/metadata/metadata-icon.tsx | 2 +- .../workflow/nodes/knowledge-retrieval/types.ts | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/web/app/components/app/configuration/dataset-config/index.tsx b/web/app/components/app/configuration/dataset-config/index.tsx index 01ba8c606d..6165cfdeec 100644 --- a/web/app/components/app/configuration/dataset-config/index.tsx +++ b/web/app/components/app/configuration/dataset-config/index.tsx @@ -270,7 +270,7 @@ const DatasetConfig: FC = () => { handleMetadataModelChange={handleMetadataModelChange} handleMetadataCompletionParamsChange={handleMetadataCompletionParamsChange} isCommonVariable - availableCommonStringVars={promptVariablesToSelect.filter(item => item.type === MetadataFilteringVariableType.string)} + availableCommonStringVars={promptVariablesToSelect.filter(item => item.type === MetadataFilteringVariableType.string || item.type === MetadataFilteringVariableType.select)} availableCommonNumberVars={promptVariablesToSelect.filter(item => item.type === MetadataFilteringVariableType.number)} />
diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-item.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-item.tsx index 910d753532..398a2294e2 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-item.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-item.tsx @@ -77,7 +77,9 @@ const ConditionItem = ({ const valueAndValueMethod = useMemo(() => { if ( - (currentMetadata?.type === MetadataFilteringVariableType.string || currentMetadata?.type === MetadataFilteringVariableType.number) + (currentMetadata?.type === MetadataFilteringVariableType.string + || currentMetadata?.type === MetadataFilteringVariableType.number + || currentMetadata?.type === MetadataFilteringVariableType.select) && typeof condition.value === 'string' ) { const regex = isCommonVariable ? COMMON_VARIABLE_REGEX : VARIABLE_REGEX @@ -140,7 +142,9 @@ const ConditionItem = ({
{ - !comparisonOperatorNotRequireValue(condition.comparison_operator) && currentMetadata?.type === MetadataFilteringVariableType.string && ( + !comparisonOperatorNotRequireValue(condition.comparison_operator) + && (currentMetadata?.type === MetadataFilteringVariableType.string + || currentMetadata?.type === MetadataFilteringVariableType.select) && ( { switch (type) { case MetadataFilteringVariableType.string: + case MetadataFilteringVariableType.select: return [ ComparisonOperator.is, ComparisonOperator.isNot, diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-icon.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-icon.tsx index 5830a1a54a..4a3f539ef4 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-icon.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-icon.tsx @@ -18,7 +18,7 @@ const MetadataIcon = ({ return ( <> { - type === MetadataFilteringVariableType.string && ( + (type === MetadataFilteringVariableType.string || type === MetadataFilteringVariableType.select) && ( ) } diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/types.ts b/web/app/components/workflow/nodes/knowledge-retrieval/types.ts index 5f8b39e02b..1cae4ecd3b 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/types.ts +++ b/web/app/components/workflow/nodes/knowledge-retrieval/types.ts @@ -80,6 +80,7 @@ export enum MetadataFilteringVariableType { string = 'string', number = 'number', time = 'time', + select = 'select', } export type MetadataFilteringCondition = { From 07ed72860506fe6fd040de94e5aac03dcdda6ce5 Mon Sep 17 00:00:00 2001 From: huangzhuo1949 <167434202+huangzhuo1949@users.noreply.github.com> Date: Tue, 8 Apr 2025 13:57:07 +0800 Subject: [PATCH 03/53] fix: segment keywords bug (#17599) Co-authored-by: huangzhuo --- api/services/dataset_service.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 61dc86a028..b019cf6b63 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -1786,12 +1786,8 @@ class SegmentService: ) elif document.doc_form in (IndexType.PARAGRAPH_INDEX, IndexType.QA_INDEX): if args.enabled or keyword_changed: - VectorService.create_segments_vector( - [args.keywords] if args.keywords else None, - [segment], - dataset, - document.doc_form, - ) + # update segment vector index + VectorService.update_segment_vector(args.keywords, segment, dataset) else: segment_hash = helper.generate_text_hash(content) tokens = 0 From abead647e226ca47aed2efa84c46bbffd1f6c987 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Tue, 8 Apr 2025 13:59:33 +0800 Subject: [PATCH 04/53] fix: Extract docx file fails when the file contains an invalid link (#17576) --- api/core/rag/extractor/word_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/rag/extractor/word_extractor.py b/api/core/rag/extractor/word_extractor.py index 0a6ffaa1dd..70c618a631 100644 --- a/api/core/rag/extractor/word_extractor.py +++ b/api/core/rag/extractor/word_extractor.py @@ -85,7 +85,7 @@ class WordExtractor(BaseExtractor): if "image" in rel.target_ref: image_count += 1 if rel.is_external: - url = rel.reltype + url = rel.target_ref response = ssrf_proxy.get(url) if response.status_code == 200: image_ext = mimetypes.guess_extension(response.headers["Content-Type"]) From 8d1a34bbb9b2708dff4952844373fab45461b7da Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 8 Apr 2025 14:33:55 +0800 Subject: [PATCH 05/53] fix: Sass @import warning (#17604) --- web/app/styles/markdown.scss | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/app/styles/markdown.scss b/web/app/styles/markdown.scss index 12ddeb1622..f1f2a7d670 100644 --- a/web/app/styles/markdown.scss +++ b/web/app/styles/markdown.scss @@ -1,7 +1,7 @@ -@import '../../themes/light'; -@import '../../themes/dark'; -@import '../../themes/markdown-light'; -@import '../../themes/markdown-dark'; +@use '../../themes/light'; +@use '../../themes/dark'; +@use '../../themes/markdown-light'; +@use '../../themes/markdown-dark'; .markdown-body { -ms-text-size-adjust: 100%; From 4124e804a0f9d0c32ca31873b9cf7696f2dbc046 Mon Sep 17 00:00:00 2001 From: IAOTW Date: Tue, 8 Apr 2025 16:04:50 +0800 Subject: [PATCH 06/53] fix(transport): add missing verify parameter to httpx.HTTPTransport (#17612) --- api/core/helper/ssrf_proxy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api/core/helper/ssrf_proxy.py b/api/core/helper/ssrf_proxy.py index 6367e45638..969cd112ee 100644 --- a/api/core/helper/ssrf_proxy.py +++ b/api/core/helper/ssrf_proxy.py @@ -56,8 +56,12 @@ def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): response = client.request(method=method, url=url, **kwargs) elif dify_config.SSRF_PROXY_HTTP_URL and dify_config.SSRF_PROXY_HTTPS_URL: proxy_mounts = { - "http://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTP_URL), - "https://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTPS_URL), + "http://": httpx.HTTPTransport( + proxy=dify_config.SSRF_PROXY_HTTP_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY + ), + "https://": httpx.HTTPTransport( + proxy=dify_config.SSRF_PROXY_HTTPS_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY + ), } with httpx.Client(mounts=proxy_mounts, verify=HTTP_REQUEST_NODE_SSL_VERIFY) as client: response = client.request(method=method, url=url, **kwargs) From 5a6219c726a6fec0c4ced717e35cee5ec7b4a7d7 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 8 Apr 2025 16:39:11 +0800 Subject: [PATCH 07/53] chore: add unit test to high frequency hooks (#17617) --- .../model-provider-page/hooks.spec.ts | 90 ++++++++++++++++++ web/hooks/use-breakpoints.spec.ts | 93 +++++++++++++++++++ web/jest.config.ts | 4 +- 3 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 web/app/components/header/account-setting/model-provider-page/hooks.spec.ts create mode 100644 web/hooks/use-breakpoints.spec.ts diff --git a/web/app/components/header/account-setting/model-provider-page/hooks.spec.ts b/web/app/components/header/account-setting/model-provider-page/hooks.spec.ts new file mode 100644 index 0000000000..4d6941ddc6 --- /dev/null +++ b/web/app/components/header/account-setting/model-provider-page/hooks.spec.ts @@ -0,0 +1,90 @@ +import { renderHook } from '@testing-library/react' +import { useLanguage } from './hooks' +import { useContext } from 'use-context-selector' +import { after } from 'node:test' + +jest.mock('swr', () => ({ + __esModule: true, + default: jest.fn(), // mock useSWR + useSWRConfig: jest.fn(), +})) + +// mock use-context-selector +jest.mock('use-context-selector', () => ({ + useContext: jest.fn(), +})) + +// mock service/common functions +jest.mock('@/service/common', () => ({ + fetchDefaultModal: jest.fn(), + fetchModelList: jest.fn(), + fetchModelProviderCredentials: jest.fn(), + fetchModelProviders: jest.fn(), + getPayUrl: jest.fn(), +})) + +// mock context hooks +jest.mock('@/context/i18n', () => ({ + __esModule: true, + default: jest.fn(), +})) + +jest.mock('@/context/provider-context', () => ({ + useProviderContext: jest.fn(), +})) + +jest.mock('@/context/modal-context', () => ({ + useModalContextSelector: jest.fn(), +})) + +jest.mock('@/context/event-emitter', () => ({ + useEventEmitterContextContext: jest.fn(), +})) + +// mock plugins +jest.mock('@/app/components/plugins/marketplace/hooks', () => ({ + useMarketplacePlugins: jest.fn(), +})) + +jest.mock('@/app/components/plugins/marketplace/utils', () => ({ + getMarketplacePluginsByCollectionId: jest.fn(), +})) + +jest.mock('./provider-added-card', () => { + // eslint-disable-next-line no-labels, ts/no-unused-expressions + UPDATE_MODEL_PROVIDER_CUSTOM_MODEL_LIST: [] +}) + +after(() => { + jest.resetModules() + jest.clearAllMocks() +}) + +describe('useLanguage', () => { + it('should replace hyphen with underscore in locale', () => { + (useContext as jest.Mock).mockReturnValue({ + locale: 'en-US', + }) + const { result } = renderHook(() => useLanguage()) + expect(result.current).toBe('en_US') + }) + + it('should return locale as is if no hyphen exists', () => { + (useContext as jest.Mock).mockReturnValue({ + locale: 'enUS', + }) + + const { result } = renderHook(() => useLanguage()) + expect(result.current).toBe('enUS') + }) + + it('should handle multiple hyphens', () => { + // Mock the I18n context return value + (useContext as jest.Mock).mockReturnValue({ + locale: 'zh-Hans-CN', + }) + + const { result } = renderHook(() => useLanguage()) + expect(result.current).toBe('zh_Hans-CN') + }) +}) diff --git a/web/hooks/use-breakpoints.spec.ts b/web/hooks/use-breakpoints.spec.ts new file mode 100644 index 0000000000..315e514f0f --- /dev/null +++ b/web/hooks/use-breakpoints.spec.ts @@ -0,0 +1,93 @@ +import { act, renderHook } from '@testing-library/react' +import useBreakpoints, { MediaType } from './use-breakpoints' + +describe('useBreakpoints', () => { + const originalInnerWidth = window.innerWidth + + // Mock the window resize event + const fireResize = (width: number) => { + window.innerWidth = width + act(() => { + window.dispatchEvent(new Event('resize')) + }) + } + + // Restore the original innerWidth after tests + afterAll(() => { + window.innerWidth = originalInnerWidth + }) + + it('should return mobile for width <= 640px', () => { + // Mock window.innerWidth for mobile + Object.defineProperty(window, 'innerWidth', { + writable: true, + configurable: true, + value: 640, + }) + + const { result } = renderHook(() => useBreakpoints()) + expect(result.current).toBe(MediaType.mobile) + }) + + it('should return tablet for width > 640px and <= 768px', () => { + // Mock window.innerWidth for tablet + Object.defineProperty(window, 'innerWidth', { + writable: true, + configurable: true, + value: 768, + }) + + const { result } = renderHook(() => useBreakpoints()) + expect(result.current).toBe(MediaType.tablet) + }) + + it('should return pc for width > 768px', () => { + // Mock window.innerWidth for pc + Object.defineProperty(window, 'innerWidth', { + writable: true, + configurable: true, + value: 1024, + }) + + const { result } = renderHook(() => useBreakpoints()) + expect(result.current).toBe(MediaType.pc) + }) + + it('should update media type when window resizes', () => { + // Start with desktop + Object.defineProperty(window, 'innerWidth', { + writable: true, + configurable: true, + value: 1024, + }) + + const { result } = renderHook(() => useBreakpoints()) + expect(result.current).toBe(MediaType.pc) + + // Resize to tablet + fireResize(768) + expect(result.current).toBe(MediaType.tablet) + + // Resize to mobile + fireResize(600) + expect(result.current).toBe(MediaType.mobile) + }) + + it('should clean up event listeners on unmount', () => { + // Spy on addEventListener and removeEventListener + const addEventListenerSpy = jest.spyOn(window, 'addEventListener') + const removeEventListenerSpy = jest.spyOn(window, 'removeEventListener') + + const { unmount } = renderHook(() => useBreakpoints()) + + // Unmount should trigger cleanup + unmount() + + expect(addEventListenerSpy).toHaveBeenCalledWith('resize', expect.any(Function)) + expect(removeEventListenerSpy).toHaveBeenCalledWith('resize', expect.any(Function)) + + // Clean up spies + addEventListenerSpy.mockRestore() + removeEventListenerSpy.mockRestore() + }) +}) diff --git a/web/jest.config.ts b/web/jest.config.ts index aa2f22bf82..9164734d64 100644 --- a/web/jest.config.ts +++ b/web/jest.config.ts @@ -98,7 +98,7 @@ const config: Config = { // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module moduleNameMapper: { - '^@/components/(.*)$': '/components/$1', + '^@/(.*)$': '/$1', '^lodash-es$': 'lodash', }, @@ -133,7 +133,7 @@ const config: Config = { // restoreMocks: false, // The root directory that Jest should scan for tests and modules within - // rootDir: undefined, + rootDir: './', // A list of paths to directories that Jest should use to search for files in // roots: [ From be3ebea45b0e86ed3d62a2869ecbb36b0ee1d388 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Tue, 8 Apr 2025 17:12:25 +0800 Subject: [PATCH 08/53] chore: bump pnpm to v10 in web dockerfile (#17611) --- .devcontainer/post_create_command.sh | 2 +- .github/workflows/web-tests.yml | 2 ++ web/Dockerfile | 2 +- web/README.md | 4 +++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.devcontainer/post_create_command.sh b/.devcontainer/post_create_command.sh index 5e76bdc2a3..c53c26bb9a 100755 --- a/.devcontainer/post_create_command.sh +++ b/.devcontainer/post_create_command.sh @@ -1,6 +1,6 @@ #!/bin/bash -npm add -g pnpm@9.12.2 +npm add -g pnpm@10.8.0 cd web && pnpm install pipx install poetry diff --git a/.github/workflows/web-tests.yml b/.github/workflows/web-tests.yml index acee26af2f..7fe3f45a8a 100644 --- a/.github/workflows/web-tests.yml +++ b/.github/workflows/web-tests.yml @@ -31,7 +31,9 @@ jobs: uses: tj-actions/changed-files@v45 with: files: web/** + - name: Install pnpm + if: steps.changed-files.outputs.any_changed == 'true' uses: pnpm/action-setup@v4 with: version: 10 diff --git a/web/Dockerfile b/web/Dockerfile index 8d50154873..80ec6d652c 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer="takatost@gmail.com" # RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories RUN apk add --no-cache tzdata -RUN npm install -g pnpm@9.12.2 +RUN npm install -g pnpm@10.8.0 ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" diff --git a/web/README.md b/web/README.md index 900924f348..3236347e80 100644 --- a/web/README.md +++ b/web/README.md @@ -6,7 +6,9 @@ This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next ### Run by source code -To start the web frontend service, you will need [Node.js v18.x (LTS)](https://nodejs.org/en) and [pnpm version 9.12.2](https://pnpm.io). +Before starting the web frontend service, please make sure the following environment is ready. +- [Node.js](https://nodejs.org) >= v18.x +- [pnpm](https://pnpm.io) v10.x First, install the dependencies: From cd7ac20d809a8cb3c5484624d7c8a105a7c9856d Mon Sep 17 00:00:00 2001 From: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Date: Tue, 8 Apr 2025 18:01:43 +0800 Subject: [PATCH 09/53] feat: enhance index type handling and add error notification for missing embedding model (#16836) --- .../datasets/create/step-two/index.tsx | 29 ++++++++++--------- web/i18n/en-US/app-debug.ts | 1 + web/i18n/zh-Hans/app-debug.ts | 1 + 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 9b485e1bde..6bef25ee9f 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -169,12 +169,11 @@ const StepTwo = ({ const [rules, setRules] = useState([]) const [defaultConfig, setDefaultConfig] = useState() const hasSetIndexType = !!indexingType - const [indexType, setIndexType] = useState( - (indexingType - || isAPIKeySet) - ? IndexingType.QUALIFIED - : IndexingType.ECONOMICAL, - ) + const [indexType, setIndexType] = useState(() => { + if (hasSetIndexType) + return indexingType + return isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL + }) const [previewFile, setPreviewFile] = useState( (datasetId && documentDetail) @@ -421,6 +420,13 @@ const StepTwo = ({ } else { // create const indexMethod = getIndexing_technique() + if (indexMethod === IndexingType.QUALIFIED && (!embeddingModel.model || !embeddingModel.provider)) { + Toast.notify({ + type: 'error', + message: t('appDebug.datasetConfig.embeddingModelRequired'), + }) + return + } if ( !isReRankModelSelected({ rerankModelList, @@ -568,7 +574,6 @@ const StepTwo = ({ // get indexing type by props if (indexingType) setIndexType(indexingType as IndexingType) - else setIndexType(isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL) }, [isAPIKeySet, indexingType, datasetId]) @@ -848,10 +853,9 @@ const StepTwo = ({ description={t('datasetCreation.stepTwo.qualifiedTip')} icon={} isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED} - disabled={!isAPIKeySet || hasSetIndexType} + disabled={hasSetIndexType} onSwitched={() => { - if (isAPIKeySet) - setIndexType(IndexingType.QUALIFIED) + setIndexType(IndexingType.QUALIFIED) }} /> )} @@ -894,11 +898,10 @@ const StepTwo = ({ description={t('datasetCreation.stepTwo.economicalTip')} icon={} isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL} - disabled={!isAPIKeySet || hasSetIndexType || docForm !== ChunkingMode.text} + disabled={hasSetIndexType || docForm !== ChunkingMode.text} ref={economyDomRef} onSwitched={() => { - if (isAPIKeySet && docForm === ChunkingMode.text) - setIndexType(IndexingType.ECONOMICAL) + setIndexType(IndexingType.ECONOMICAL) }} /> diff --git a/web/i18n/en-US/app-debug.ts b/web/i18n/en-US/app-debug.ts index 00f681f843..3ee5fd3e1d 100644 --- a/web/i18n/en-US/app-debug.ts +++ b/web/i18n/en-US/app-debug.ts @@ -483,6 +483,7 @@ const translation = { title: 'Multi-path retrieval', description: 'Based on user intent, queries across all Knowledge, retrieves relevant text from multi-sources, and selects the best results matching the user query after reranking.', }, + embeddingModelRequired: 'A configured Embedding Model is required', rerankModelRequired: 'A configured Rerank Model is required', params: 'Params', top_k: 'Top K', diff --git a/web/i18n/zh-Hans/app-debug.ts b/web/i18n/zh-Hans/app-debug.ts index 781ee39671..c2c659b41f 100644 --- a/web/i18n/zh-Hans/app-debug.ts +++ b/web/i18n/zh-Hans/app-debug.ts @@ -475,6 +475,7 @@ const translation = { title: '多路召回', description: '根据用户意图同时匹配所有知识库，从多路知识库查询相关文本片段，经过重排序步骤，从多路查询结果中选择匹配用户问题的最佳结果。', }, + embeddingModelRequired: '未配置 Embedding 模型', rerankModelRequired: '未配置 Rerank 模型', params: '参数设置', top_k: 'Top K', From 106604682ab062a120562051c953b8402a747f6a Mon Sep 17 00:00:00 2001 From: Lao Date: Tue, 8 Apr 2025 21:00:00 +0800 Subject: [PATCH 10/53] Fixed the model-modal titles not being clearly distinguished between "Add" and "Setup" (#17634) --- .../account-setting/model-provider-page/model-modal/index.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx b/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx index 4adab6d2e0..bd1bb6ced9 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx @@ -270,8 +270,7 @@ const ModelModal: FC = ({ } const renderTitlePrefix = () => { - const prefix = configurateMethod === ConfigurationMethodEnum.customizableModel ? t('common.operation.add') : t('common.operation.setup') - + const prefix = isEditMode ? t('common.operation.setup') : t('common.operation.add') return `${prefix} ${provider.label[language] || provider.label.en_US}` } From b73607da8061a2738fe64b1cc19186c1489d23a4 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Wed, 9 Apr 2025 09:40:11 +0800 Subject: [PATCH 11/53] chore: bump Nodejs in web image from 20 to 22 LTS (#13341) --- .github/workflows/style.yml | 3 ++- .github/workflows/tool-test-sdks.yaml | 2 +- .github/workflows/translate-i18n-base-on-english.yml | 2 +- .github/workflows/web-tests.yml | 2 +- web/Dockerfile | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index d73a782c93..625930b5f5 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -82,7 +82,7 @@ jobs: uses: actions/setup-node@v4 if: steps.changed-files.outputs.any_changed == 'true' with: - node-version: 20 + node-version: 22 cache: pnpm cache-dependency-path: ./web/package.json @@ -153,6 +153,7 @@ jobs: env: BASH_SEVERITY: warning DEFAULT_BRANCH: main + FILTER_REGEX_INCLUDE: pnpm-lock.yaml GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} IGNORE_GENERATED_FILES: true IGNORE_GITIGNORED_FILES: true diff --git a/.github/workflows/tool-test-sdks.yaml b/.github/workflows/tool-test-sdks.yaml index 93edb2737a..a6e48d1359 100644 --- a/.github/workflows/tool-test-sdks.yaml +++ b/.github/workflows/tool-test-sdks.yaml @@ -18,7 +18,7 @@ jobs: strategy: matrix: - node-version: [16, 18, 20] + node-version: [16, 18, 20, 22] defaults: run: diff --git a/.github/workflows/translate-i18n-base-on-english.yml b/.github/workflows/translate-i18n-base-on-english.yml index 80b78a1311..3f8082eb69 100644 --- a/.github/workflows/translate-i18n-base-on-english.yml +++ b/.github/workflows/translate-i18n-base-on-english.yml @@ -33,7 +33,7 @@ jobs: - name: Set up Node.js if: env.FILES_CHANGED == 'true' - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: node-version: 'lts/*' diff --git a/.github/workflows/web-tests.yml b/.github/workflows/web-tests.yml index 7fe3f45a8a..85e8b99473 100644 --- a/.github/workflows/web-tests.yml +++ b/.github/workflows/web-tests.yml @@ -43,7 +43,7 @@ jobs: uses: actions/setup-node@v4 if: steps.changed-files.outputs.any_changed == 'true' with: - node-version: 20 + node-version: 22 cache: pnpm cache-dependency-path: ./web/package.json diff --git a/web/Dockerfile b/web/Dockerfile index 80ec6d652c..dfc5ba8b46 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,5 +1,5 @@ # base image -FROM node:20-alpine3.20 AS base +FROM node:22-alpine3.21 AS base LABEL maintainer="takatost@gmail.com" # if you located in China, you can use aliyun mirror to speed up From b5498a373a4c05bb8ddd2f4617e3bb120fa87646 Mon Sep 17 00:00:00 2001 From: Han <109904848+wanghan5@users.noreply.github.com> Date: Wed, 9 Apr 2025 10:12:16 +0800 Subject: [PATCH 12/53] Accelerate migration (#17088) Co-authored-by: Wang Han --- api/services/plugin/data_migration.py | 52 ++++++++++++++++++--------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/api/services/plugin/data_migration.py b/api/services/plugin/data_migration.py index 7228a16632..597585588b 100644 --- a/api/services/plugin/data_migration.py +++ b/api/services/plugin/data_migration.py @@ -127,18 +127,32 @@ limit 1000""" processed_count = 0 failed_ids = [] + last_id = "00000000-0000-0000-0000-000000000000" + while True: - sql = f"""select id, {provider_column_name} as provider_name from {table_name} -where {provider_column_name} not like '%/%' and {provider_column_name} is not null and {provider_column_name} != '' -limit 1000""" + sql = f""" + SELECT id, {provider_column_name} AS provider_name + FROM {table_name} + WHERE {provider_column_name} NOT LIKE '%/%' + AND {provider_column_name} IS NOT NULL + AND {provider_column_name} != '' + AND id > :last_id + ORDER BY id ASC + LIMIT 5000 + """ + params = {"last_id": last_id or ""} + with db.engine.begin() as conn: - rs = conn.execute(db.text(sql)) + rs = conn.execute(db.text(sql), params) current_iter_count = 0 + batch_updates = [] + for i in rs: current_iter_count += 1 processed_count += 1 record_id = str(i.id) + last_id = record_id provider_name = str(i.provider_name) if record_id in failed_ids: @@ -152,19 +166,9 @@ limit 1000""" ) try: - # update provider name append with "langgenius/{provider_name}/{provider_name}" - sql = f"""update {table_name} - set {provider_column_name} = - concat('{DEFAULT_PLUGIN_ID}/', {provider_column_name}, '/', {provider_column_name}) - where id = :record_id""" - conn.execute(db.text(sql), {"record_id": record_id}) - click.echo( - click.style( - f"[{processed_count}] Migrated [{table_name}] {record_id} ({provider_name})", - fg="green", - ) - ) - except Exception: + updated_value = f"{DEFAULT_PLUGIN_ID}/{provider_name}/{provider_name}" + batch_updates.append((updated_value, record_id)) + except Exception as e: failed_ids.append(record_id) click.echo( click.style( @@ -177,6 +181,20 @@ limit 1000""" ) continue + if batch_updates: + update_sql = f""" + UPDATE {table_name} + SET {provider_column_name} = :updated_value + WHERE id = :record_id + """ + conn.execute(db.text(update_sql), [{"updated_value": u, "record_id": r} for u, r in batch_updates]) + click.echo( + click.style( + f"[{processed_count}] Batch migrated [{len(batch_updates)}] records from [{table_name}]", + fg="green", + ) + ) + if not current_iter_count: break From f1e4d5ed6c7f3f569e0ff0b191884dd50e63618b Mon Sep 17 00:00:00 2001 From: Han <109904848+wanghan5@users.noreply.github.com> Date: Wed, 9 Apr 2025 11:22:53 +0800 Subject: [PATCH 13/53] Fix Performance Issues: (#17083) Co-authored-by: Wang Han --- api/core/rag/datasource/retrieval_service.py | 189 ++++++++++-------- ..._change_documentsegment_and_childchunk_.py | 43 ++++ api/models/dataset.py | 4 +- api/services/hit_testing_service.py | 9 - 4 files changed, 151 insertions(+), 94 deletions(-) create mode 100644 api/migrations/versions/2025_03_29_2227-6a9f914f656c_change_documentsegment_and_childchunk_.py diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index fea4d0edf7..c4a1e9f059 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -1,4 +1,6 @@ import concurrent.futures +import logging +import time from concurrent.futures import ThreadPoolExecutor from typing import Optional @@ -46,7 +48,7 @@ class RetrievalService: if not query: return [] dataset = cls._get_dataset(dataset_id) - if not dataset or dataset.available_document_count == 0 or dataset.available_segment_count == 0: + if not dataset: return [] all_documents: list[Document] = [] @@ -178,6 +180,7 @@ class RetrievalService: if not dataset: raise ValueError("dataset not found") + start = time.time() vector = Vector(dataset=dataset) documents = vector.search_by_vector( query, @@ -187,6 +190,7 @@ class RetrievalService: filter={"group_id": [dataset.id]}, document_ids_filter=document_ids_filter, ) + logging.debug(f"embedding_search ends at {time.time() - start:.2f} seconds") if documents: if ( @@ -270,7 +274,8 @@ class RetrievalService: return [] try: - # Collect document IDs + start_time = time.time() + # Collect document IDs with existence check document_ids = {doc.metadata.get("document_id") for doc in documents if "document_id" in doc.metadata} if not document_ids: return [] @@ -288,43 +293,102 @@ class RetrievalService: include_segment_ids = set() segment_child_map = {} - # Process documents + # Precompute doc_forms to avoid redundant checks + doc_forms = {} + for doc in documents: + document_id = doc.metadata.get("document_id") + dataset_doc = dataset_documents.get(document_id) + if dataset_doc: + doc_forms[document_id] = dataset_doc.doc_form + + # Batch collect index node IDs with type safety + child_index_node_ids = [] + index_node_ids = [] + for doc in documents: + document_id = doc.metadata.get("document_id") + if doc_forms.get(document_id) == IndexType.PARENT_CHILD_INDEX: + child_index_node_ids.append(doc.metadata.get("doc_id")) + else: + index_node_ids.append(doc.metadata.get("doc_id")) + + # Batch query ChildChunk + child_chunks = db.session.query(ChildChunk).filter(ChildChunk.index_node_id.in_(child_index_node_ids)).all() + child_chunk_map = {chunk.index_node_id: chunk for chunk in child_chunks} + + # Batch query DocumentSegment with unified conditions + segment_map = { + segment.id: segment + for segment in db.session.query(DocumentSegment) + .filter( + ( + DocumentSegment.index_node_id.in_(index_node_ids) + | DocumentSegment.id.in_([chunk.segment_id for chunk in child_chunks]) + ), + DocumentSegment.enabled == True, + DocumentSegment.status == "completed", + ) + .options( + load_only( + DocumentSegment.id, + DocumentSegment.content, + DocumentSegment.answer, + ) + ) + .all() + } + for document in documents: document_id = document.metadata.get("document_id") - if document_id not in dataset_documents: - continue - - dataset_document = dataset_documents[document_id] + dataset_document = dataset_documents.get(document_id) if not dataset_document: continue - if dataset_document.doc_form == IndexType.PARENT_CHILD_INDEX: - # Handle parent-child documents + doc_form = doc_forms.get(document_id) + if doc_form == IndexType.PARENT_CHILD_INDEX: + # Handle parent-child documents using preloaded data child_index_node_id = document.metadata.get("doc_id") + if not child_index_node_id: + continue - child_chunk = ( - db.session.query(ChildChunk).filter(ChildChunk.index_node_id == child_index_node_id).first() - ) - + child_chunk = child_chunk_map.get(child_index_node_id) if not child_chunk: continue - segment = ( - db.session.query(DocumentSegment) - .filter( - DocumentSegment.dataset_id == dataset_document.dataset_id, - DocumentSegment.enabled == True, - DocumentSegment.status == "completed", - DocumentSegment.id == child_chunk.segment_id, - ) - .options( - load_only( - DocumentSegment.id, - DocumentSegment.content, - DocumentSegment.answer, - ) - ) - .first() + segment = segment_map.get(child_chunk.segment_id) + if not segment: + continue + + if segment.id not in include_segment_ids: + include_segment_ids.add(segment.id) + map_detail = {"max_score": document.metadata.get("score", 0.0), "child_chunks": []} + segment_child_map[segment.id] = map_detail + records.append({"segment": segment}) + + # Append child chunk details + child_chunk_detail = { + "id": child_chunk.id, + "content": child_chunk.content, + "position": child_chunk.position, + "score": document.metadata.get("score", 0.0), + } + segment_child_map[segment.id]["child_chunks"].append(child_chunk_detail) + segment_child_map[segment.id]["max_score"] = max( + segment_child_map[segment.id]["max_score"], document.metadata.get("score", 0.0) + ) + + else: + # Handle normal documents + index_node_id = document.metadata.get("doc_id") + if not index_node_id: + continue + + segment = next( + ( + s + for s in segment_map.values() + if s.index_node_id == index_node_id and s.dataset_id == dataset_document.dataset_id + ), + None, ) if not segment: @@ -332,66 +396,23 @@ class RetrievalService: if segment.id not in include_segment_ids: include_segment_ids.add(segment.id) - child_chunk_detail = { - "id": child_chunk.id, - "content": child_chunk.content, - "position": child_chunk.position, - "score": document.metadata.get("score", 0.0), - } - map_detail = { - "max_score": document.metadata.get("score", 0.0), - "child_chunks": [child_chunk_detail], - } - segment_child_map[segment.id] = map_detail - record = { - "segment": segment, - } - records.append(record) - else: - child_chunk_detail = { - "id": child_chunk.id, - "content": child_chunk.content, - "position": child_chunk.position, - "score": document.metadata.get("score", 0.0), - } - segment_child_map[segment.id]["child_chunks"].append(child_chunk_detail) - segment_child_map[segment.id]["max_score"] = max( - segment_child_map[segment.id]["max_score"], document.metadata.get("score", 0.0) + records.append( + { + "segment": segment, + "score": document.metadata.get("score", 0.0), + } ) - else: - # Handle normal documents - index_node_id = document.metadata.get("doc_id") - if not index_node_id: - continue - segment = ( - db.session.query(DocumentSegment) - .filter( - DocumentSegment.dataset_id == dataset_document.dataset_id, - DocumentSegment.enabled == True, - DocumentSegment.status == "completed", - DocumentSegment.index_node_id == index_node_id, - ) - .first() - ) - - if not segment: - continue - - include_segment_ids.add(segment.id) - record = { - "segment": segment, - "score": document.metadata.get("score"), # type: ignore - } - records.append(record) - - # Add child chunks information to records + # Merge child chunks information for record in records: - if record["segment"].id in segment_child_map: - record["child_chunks"] = segment_child_map[record["segment"].id].get("child_chunks") # type: ignore - record["score"] = segment_child_map[record["segment"].id]["max_score"] + segment_id = record["segment"].id + if segment_id in segment_child_map: + record["child_chunks"] = segment_child_map[segment_id]["child_chunks"] + record["score"] = segment_child_map[segment_id]["max_score"] + logging.debug(f"Formatting retrieval documents took {time.time() - start_time:.2f} seconds") return [RetrievalSegments(**record) for record in records] except Exception as e: + # Only rollback if there were write operations db.session.rollback() raise e diff --git a/api/migrations/versions/2025_03_29_2227-6a9f914f656c_change_documentsegment_and_childchunk_.py b/api/migrations/versions/2025_03_29_2227-6a9f914f656c_change_documentsegment_and_childchunk_.py new file mode 100644 index 0000000000..45904f0c80 --- /dev/null +++ b/api/migrations/versions/2025_03_29_2227-6a9f914f656c_change_documentsegment_and_childchunk_.py @@ -0,0 +1,43 @@ +"""change documentsegment and childchunk indexes + +Revision ID: 6a9f914f656c +Revises: d20049ed0af6 +Create Date: 2025-03-29 22:27:24.789481 + +""" +from alembic import op +import models as models +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '6a9f914f656c' +down_revision = 'd20049ed0af6' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('child_chunks', schema=None) as batch_op: + batch_op.create_index('child_chunks_node_idx', ['index_node_id', 'dataset_id'], unique=False) + batch_op.create_index('child_chunks_segment_idx', ['segment_id'], unique=False) + + with op.batch_alter_table('document_segments', schema=None) as batch_op: + batch_op.drop_index('document_segment_dataset_node_idx') + batch_op.create_index('document_segment_node_dataset_idx', ['index_node_id', 'dataset_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('document_segments', schema=None) as batch_op: + batch_op.drop_index('document_segment_node_dataset_idx') + batch_op.create_index('document_segment_dataset_node_idx', ['dataset_id', 'index_node_id'], unique=False) + + with op.batch_alter_table('child_chunks', schema=None) as batch_op: + batch_op.drop_index('child_chunks_segment_idx') + batch_op.drop_index('child_chunks_node_idx') + + # ### end Alembic commands ### diff --git a/api/models/dataset.py b/api/models/dataset.py index 47f96c669e..d6708ac88b 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -643,7 +643,7 @@ class DocumentSegment(db.Model): # type: ignore[name-defined] db.Index("document_segment_document_id_idx", "document_id"), db.Index("document_segment_tenant_dataset_idx", "dataset_id", "tenant_id"), db.Index("document_segment_tenant_document_idx", "document_id", "tenant_id"), - db.Index("document_segment_dataset_node_idx", "dataset_id", "index_node_id"), + db.Index("document_segment_node_dataset_idx", "index_node_id", "dataset_id"), db.Index("document_segment_tenant_idx", "tenant_id"), ) @@ -791,6 +791,8 @@ class ChildChunk(db.Model): # type: ignore[name-defined] __table_args__ = ( db.PrimaryKeyConstraint("id", name="child_chunk_pkey"), db.Index("child_chunk_dataset_id_idx", "tenant_id", "dataset_id", "document_id", "segment_id", "index_node_id"), + db.Index("child_chunks_node_idx", "index_node_id", "dataset_id"), + db.Index("child_chunks_segment_idx", "segment_id"), ) # initial fields diff --git a/api/services/hit_testing_service.py b/api/services/hit_testing_service.py index f8c1c1d297..0b98065f5d 100644 --- a/api/services/hit_testing_service.py +++ b/api/services/hit_testing_service.py @@ -29,15 +29,6 @@ class HitTestingService: external_retrieval_model: dict, limit: int = 10, ) -> dict: - if dataset.available_document_count == 0 or dataset.available_segment_count == 0: - return { - "query": { - "content": query, - "tsne_position": {"x": 0, "y": 0}, - }, - "records": [], - } - start = time.perf_counter() # get retrieval model , if the model is not setting , using default From f633d1ee9220bd80cca83674764151efcad7d101 Mon Sep 17 00:00:00 2001 From: yusheng chen Date: Wed, 9 Apr 2025 12:10:17 +0800 Subject: [PATCH 14/53] chore: add `'no-empty-function': 'error'` to `eslint.config.mjs` (#17656) --- .../dataset-config/params-config/config-content.tsx | 5 +++-- web/app/components/app/configuration/index.tsx | 3 --- .../components/base/audio-btn/audio.player.manager.ts | 5 +---- web/app/components/base/audio-btn/audio.ts | 10 +++------- web/app/components/base/chat/chat/hooks.ts | 3 ++- web/app/components/base/pagination/pagination.tsx | 3 ++- web/app/components/datasets/create/step-two/index.tsx | 3 ++- web/app/components/datasets/documents/list.tsx | 5 +++-- web/app/components/explore/create-app-modal/index.tsx | 3 ++- .../data-source-page/panel/config-item.tsx | 3 ++- .../block-selector/market-place-plugin/list.tsx | 3 ++- web/app/components/workflow/hooks/use-workflow-run.ts | 3 ++- .../workflow/nodes/_base/hooks/use-one-step-run.ts | 8 +++----- .../workflow/run/utils/format-log/agent/data.ts | 3 --- web/context/i18n.ts | 3 ++- web/eslint.config.mjs | 3 +++ 16 files changed, 32 insertions(+), 34 deletions(-) diff --git a/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx b/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx index 14f0c3d865..3b9078f1be 100644 --- a/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx +++ b/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx @@ -25,6 +25,7 @@ import { useSelectedDatasetsMode } from '@/app/components/workflow/nodes/knowled import Switch from '@/app/components/base/switch' import Toast from '@/app/components/base/toast' import Divider from '@/app/components/base/divider' +import { noop } from 'lodash-es' type Props = { datasetConfigs: DatasetConfigs @@ -41,8 +42,8 @@ const ConfigContent: FC = ({ onChange, isInWorkflow, singleRetrievalModelConfig: singleRetrievalConfig = {} as ModelConfig, - onSingleRetrievalModelChange = () => { }, - onSingleRetrievalModelParamsChange = () => { }, + onSingleRetrievalModelChange = noop, + onSingleRetrievalModelParamsChange = noop, selectedDatasets = [], }) => { const { t } = useTranslation() diff --git a/web/app/components/app/configuration/index.tsx b/web/app/components/app/configuration/index.tsx index cc6909d151..249624a294 100644 --- a/web/app/components/app/configuration/index.tsx +++ b/web/app/components/app/configuration/index.tsx @@ -197,9 +197,6 @@ const Configuration: FC = () => { const isOpenAI = modelConfig.provider === 'langgenius/openai/openai' const [collectionList, setCollectionList] = useState([]) - useEffect(() => { - - }, []) const [datasetConfigs, doSetDatasetConfigs] = useState({ retrieval_model: RETRIEVE_TYPE.multiWay, reranking_model: { diff --git a/web/app/components/base/audio-btn/audio.player.manager.ts b/web/app/components/base/audio-btn/audio.player.manager.ts index 848aef6cba..15be7a3d8c 100644 --- a/web/app/components/base/audio-btn/audio.player.manager.ts +++ b/web/app/components/base/audio-btn/audio.player.manager.ts @@ -12,9 +12,6 @@ export class AudioPlayerManager { private audioPlayers: AudioPlayer | null = null private msgId: string | undefined - private constructor() { - } - public static getInstance(): AudioPlayerManager { if (!AudioPlayerManager.instance) { AudioPlayerManager.instance = new AudioPlayerManager() @@ -24,7 +21,7 @@ export class AudioPlayerManager { return AudioPlayerManager.instance } - public getAudioPlayer(url: string, isPublic: boolean, id: string | undefined, msgContent: string | null | undefined, voice: string | undefined, callback: ((event: string) => {}) | null): AudioPlayer { + public getAudioPlayer(url: string, isPublic: boolean, id: string | undefined, msgContent: string | null | undefined, voice: string | undefined, callback: ((event: string) => void) | null): AudioPlayer { if (this.msgId && this.msgId === id && this.audioPlayers) { this.audioPlayers.setCallback(callback) return this.audioPlayers diff --git a/web/app/components/base/audio-btn/audio.ts b/web/app/components/base/audio-btn/audio.ts index d7fae02f82..cd40930f43 100644 --- a/web/app/components/base/audio-btn/audio.ts +++ b/web/app/components/base/audio-btn/audio.ts @@ -21,9 +21,9 @@ export default class AudioPlayer { isLoadData = false url: string isPublic: boolean - callback: ((event: string) => {}) | null + callback: ((event: string) => void) | null - constructor(streamUrl: string, isPublic: boolean, msgId: string | undefined, msgContent: string | null | undefined, voice: string | undefined, callback: ((event: string) => {}) | null) { + constructor(streamUrl: string, isPublic: boolean, msgId: string | undefined, msgContent: string | null | undefined, voice: string | undefined, callback: ((event: string) => void) | null) { this.audioContext = new AudioContext() this.msgId = msgId this.msgContent = msgContent @@ -68,7 +68,7 @@ export default class AudioPlayer { }) } - public setCallback(callback: ((event: string) => {}) | null) { + public setCallback(callback: ((event: string) => void) | null) { this.callback = callback if (callback) { this.audio.addEventListener('ended', () => { @@ -211,10 +211,6 @@ export default class AudioPlayer { this.audioContext.suspend() } - private cancer() { - - } - private receiveAudioData(unit8Array: Uint8Array) { if (!unit8Array) { this.finishStream() diff --git a/web/app/components/base/chat/chat/hooks.ts b/web/app/components/base/chat/chat/hooks.ts index eb48f9515b..aad17ccc52 100644 --- a/web/app/components/base/chat/chat/hooks.ts +++ b/web/app/components/base/chat/chat/hooks.ts @@ -34,6 +34,7 @@ import { getProcessedFiles, getProcessedFilesFromResponse, } from '@/app/components/base/file-uploader/utils' +import { noop } from 'lodash-es' type GetAbortController = (abortController: AbortController) => void type SendCallback = { @@ -308,7 +309,7 @@ export const useChat = ( else ttsUrl = `/apps/${params.appId}/text-to-audio` } - const player = AudioPlayerManager.getInstance().getAudioPlayer(ttsUrl, ttsIsPublic, uuidV4(), 'none', 'none', (_: any): any => { }) + const player = AudioPlayerManager.getInstance().getAudioPlayer(ttsUrl, ttsIsPublic, uuidV4(), 'none', 'none', noop) ssePost( url, { diff --git a/web/app/components/base/pagination/pagination.tsx b/web/app/components/base/pagination/pagination.tsx index 5898c4e924..ec8b0355f4 100644 --- a/web/app/components/base/pagination/pagination.tsx +++ b/web/app/components/base/pagination/pagination.tsx @@ -7,10 +7,11 @@ import type { IPaginationProps, PageButtonProps, } from './type' +import { noop } from 'lodash-es' const defaultState: IPagination = { currentPage: 0, - setCurrentPage: () => {}, + setCurrentPage: noop, truncableText: '...', truncableClassName: '', pages: [], diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 6bef25ee9f..12fd54d0fe 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -62,6 +62,7 @@ import Tooltip from '@/app/components/base/tooltip' import CustomDialog from '@/app/components/base/dialog' import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem' import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback' +import { noop } from 'lodash-es' const TextLabel: FC = (props) => { return @@ -1010,7 +1011,7 @@ const StepTwo = ({
)} - { }} footer={null}> + { const { t } = useTranslation() @@ -265,7 +266,7 @@ export const OperationAction: FC<{ return
e.stopPropagation()}> {isListScene && !embeddingAvailable && ( - { }} disabled={true} size='md' /> + )} {isListScene && embeddingAvailable && ( <> @@ -276,7 +277,7 @@ export const OperationAction: FC<{ needsDelay >
- { }} disabled={true} size='md' /> +
: handleSwitch(v ? 'enable' : 'disable')} size='md' /> diff --git a/web/app/components/explore/create-app-modal/index.tsx b/web/app/components/explore/create-app-modal/index.tsx index 585c52f828..62116192d7 100644 --- a/web/app/components/explore/create-app-modal/index.tsx +++ b/web/app/components/explore/create-app-modal/index.tsx @@ -13,6 +13,7 @@ import AppIcon from '@/app/components/base/app-icon' import { useProviderContext } from '@/context/provider-context' import AppsFull from '@/app/components/billing/apps-full-in-dialog' import type { AppIconType } from '@/types/app' +import { noop } from 'lodash-es' export type CreateAppModalProps = { show: boolean @@ -85,7 +86,7 @@ const CreateAppModal = ({ <> {}} + onClose={noop} className='relative !max-w-[480px] px-8' >
diff --git a/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx index 3dad51f566..6faf840529 100644 --- a/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx +++ b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx @@ -10,6 +10,7 @@ import Operate from '../data-source-notion/operate' import { DataSourceType } from './types' import s from './style.module.css' import cn from '@/utils/classnames' +import { noop } from 'lodash-es' export type ConfigItemType = { id: string @@ -41,7 +42,7 @@ const ConfigItem: FC = ({ const { t } = useTranslation() const isNotion = type === DataSourceType.notion const isWebsite = type === DataSourceType.website - const onChangeAuthorizedPage = notionActions?.onChangeAuthorizedPage || function () { } + const onChangeAuthorizedPage = notionActions?.onChangeAuthorizedPage || noop return (
diff --git a/web/app/components/workflow/block-selector/market-place-plugin/list.tsx b/web/app/components/workflow/block-selector/market-place-plugin/list.tsx index d74f170589..97110093b0 100644 --- a/web/app/components/workflow/block-selector/market-place-plugin/list.tsx +++ b/web/app/components/workflow/block-selector/market-place-plugin/list.tsx @@ -9,6 +9,7 @@ import Link from 'next/link' import { marketplaceUrlPrefix } from '@/config' import { RiArrowRightUpLine, RiSearchLine } from '@remixicon/react' // import { RiArrowRightUpLine } from '@remixicon/react' +import { noop } from 'lodash-es' type Props = { wrapElemRef: React.RefObject @@ -107,7 +108,7 @@ const List = ( { }} + onAction={noop} /> ))}
diff --git a/web/app/components/workflow/hooks/use-workflow-run.ts b/web/app/components/workflow/hooks/use-workflow-run.ts index 87ff2186fc..99d9a45702 100644 --- a/web/app/components/workflow/hooks/use-workflow-run.ts +++ b/web/app/components/workflow/hooks/use-workflow-run.ts @@ -18,6 +18,7 @@ import { stopWorkflowRun } from '@/service/workflow' import { useFeaturesStore } from '@/app/components/base/features/hooks' import { AudioPlayerManager } from '@/app/components/base/audio-btn/audio.player.manager' import type { VersionHistory } from '@/types/workflow' +import { noop } from 'lodash-es' export const useWorkflowRun = () => { const store = useStoreApi() @@ -168,7 +169,7 @@ export const useWorkflowRun = () => { else ttsUrl = `/apps/${params.appId}/text-to-audio` } - const player = AudioPlayerManager.getInstance().getAudioPlayer(ttsUrl, ttsIsPublic, uuidV4(), 'none', 'none', (_: any): any => { }) + const player = AudioPlayerManager.getInstance().getAudioPlayer(ttsUrl, ttsIsPublic, uuidV4(), 'none', 'none', noop) ssePost( url, diff --git a/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts b/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts index 7d8b7fe086..f23af5812c 100644 --- a/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts +++ b/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts @@ -30,7 +30,7 @@ import IterationDefault from '@/app/components/workflow/nodes/iteration/default' import DocumentExtractorDefault from '@/app/components/workflow/nodes/document-extractor/default' import LoopDefault from '@/app/components/workflow/nodes/loop/default' import { ssePost } from '@/service/base' - +import { noop } from 'lodash-es' import { getInputVars as doGetInputVars } from '@/app/components/base/prompt-editor/constants' import type { NodeTracing } from '@/types/workflow' const { checkValid: checkLLMValid } = LLMDefault @@ -233,8 +233,7 @@ const useOneStepRun = ({ getIterationSingleNodeRunUrl(isChatMode, appId!, id), { body: { inputs: submitData } }, { - onWorkflowStarted: () => { - }, + onWorkflowStarted: noop, onWorkflowFinished: (params) => { handleNodeDataUpdate({ id, @@ -331,8 +330,7 @@ const useOneStepRun = ({ getLoopSingleNodeRunUrl(isChatMode, appId!, id), { body: { inputs: submitData } }, { - onWorkflowStarted: () => { - }, + onWorkflowStarted: noop, onWorkflowFinished: (params) => { handleNodeDataUpdate({ id, diff --git a/web/app/components/workflow/run/utils/format-log/agent/data.ts b/web/app/components/workflow/run/utils/format-log/agent/data.ts index a1e06bf63b..d90933c293 100644 --- a/web/app/components/workflow/run/utils/format-log/agent/data.ts +++ b/web/app/components/workflow/run/utils/format-log/agent/data.ts @@ -177,6 +177,3 @@ export const multiStepsCircle = (() => { }], } })() - -export const CircleNestCircle = (() => { -})() diff --git a/web/context/i18n.ts b/web/context/i18n.ts index be41730b07..6db211dd5d 100644 --- a/web/context/i18n.ts +++ b/web/context/i18n.ts @@ -4,6 +4,7 @@ import { } from 'use-context-selector' import type { Locale } from '@/i18n' import { getLanguage } from '@/i18n/language' +import { noop } from 'lodash-es' type II18NContext = { locale: Locale @@ -14,7 +15,7 @@ type II18NContext = { const I18NContext = createContext({ locale: 'en-US', i18n: {}, - setLocaleOnClient: (_lang: Locale, _reloadPage?: boolean) => { }, + setLocaleOnClient: noop, }) export const useI18N = () => useContext(I18NContext) diff --git a/web/eslint.config.mjs b/web/eslint.config.mjs index 9ce151c751..204efc4715 100644 --- a/web/eslint.config.mjs +++ b/web/eslint.config.mjs @@ -117,6 +117,9 @@ export default combine( // antfu migrate to eslint-plugin-unused-imports 'unused-imports/no-unused-vars': 'warn', 'unused-imports/no-unused-imports': 'warn', + + // We use `import { noop } from 'lodash-es'` across `web` project + 'no-empty-function': 'error', }, languageOptions: { From eb0e51d44d9ef899d4fe8dd825c3bfc8d77ba9d2 Mon Sep 17 00:00:00 2001 From: FangHao Date: Wed, 9 Apr 2025 12:16:48 +0800 Subject: [PATCH 15/53] optimize: docker-compose.middleware.yaml update env_file dependence (#17646) Co-authored-by: fanghao --- docker/docker-compose.middleware.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index b4f772cc82..230b8a05be 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -29,6 +29,8 @@ services: redis: image: redis:6-alpine restart: always + env_file: + - ./middleware.env environment: REDISCLI_AUTH: ${REDIS_PASSWORD:-difyai123456} volumes: @@ -45,6 +47,8 @@ services: sandbox: image: langgenius/dify-sandbox:0.2.11 restart: always + env_file: + - ./middleware.env environment: # The DifySandbox configurations # Make sure you are changing this key for your deployment with a strong key. @@ -68,6 +72,8 @@ services: plugin_daemon: image: langgenius/dify-plugin-daemon:0.0.6-local restart: always + env_file: + - ./middleware.env environment: # Use the shared environment variables. DB_HOST: ${DB_HOST:-db} @@ -107,6 +113,8 @@ services: - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh entrypoint: [ "sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ] + env_file: + - ./middleware.env environment: # pls clearly modify the squid env vars to fit your network environment. HTTP_PORT: ${SSRF_HTTP_PORT:-3128} From 0b1259fc4a45294176ac2415ad2df06d899c6d85 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Wed, 9 Apr 2025 13:03:53 +0800 Subject: [PATCH 16/53] chore: add script for running mypy type checks and speed up mypy checks in CI jobs (#17489) --- .github/workflows/api-tests.yml | 9 +- api/poetry.lock | 454 ++++++++++++++++++++++++-------- api/pyproject.toml | 34 ++- dev/reformat | 3 + dev/run-mypy | 11 + 5 files changed, 391 insertions(+), 120 deletions(-) create mode 100755 dev/run-mypy diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index b9547b6452..dca8e640c7 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -53,9 +53,14 @@ jobs: - name: Run dify config tests run: poetry run -P api python dev/pytest/pytest_config_tests.py + - name: Cache MyPy + uses: actions/cache@v4 + with: + path: api/.mypy_cache + key: mypy-${{ matrix.python-version }}-${{ runner.os }}-${{ hashFiles('api/poetry.lock') }} + - name: Run mypy - run: | - poetry run -C api python -m mypy --install-types --non-interactive . + run: dev/run-mypy - name: Set up dotenvs run: | diff --git a/api/poetry.lock b/api/poetry.lock index 0cda9d322f..a91023707e 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -845,10 +845,6 @@ files = [ {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e"}, - {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c"}, - {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1"}, - {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2"}, - {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec"}, {file = "Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2"}, {file = "Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128"}, {file = "Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc"}, @@ -861,14 +857,8 @@ files = [ {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8"}, - {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f"}, - {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757"}, - {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0"}, - {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b"}, {file = "Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50"}, {file = "Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1"}, - {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28"}, - {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2"}, {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451"}, @@ -879,24 +869,8 @@ files = [ {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966"}, - {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9"}, - {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb"}, - {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111"}, - {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839"}, {file = "Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0"}, {file = "Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951"}, - {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5"}, - {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8"}, - {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f"}, - {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648"}, - {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0"}, - {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089"}, - {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368"}, - {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c"}, - {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284"}, - {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7"}, - {file = "Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0"}, - {file = "Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b"}, {file = "Brotli-1.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b"}, @@ -906,10 +880,6 @@ files = [ {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2"}, - {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:aea440a510e14e818e67bfc4027880e2fb500c2ccb20ab21c7a7c8b5b4703d75"}, - {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:6974f52a02321b36847cd19d1b8e381bf39939c21efd6ee2fc13a28b0d99348c"}, - {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:a7e53012d2853a07a4a79c00643832161a910674a893d296c9f1259859a289d2"}, - {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:d7702622a8b40c49bffb46e1e3ba2e81268d5c04a34f460978c6b5517a34dd52"}, {file = "Brotli-1.1.0-cp36-cp36m-win32.whl", hash = "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460"}, {file = "Brotli-1.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579"}, {file = "Brotli-1.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c"}, @@ -921,10 +891,6 @@ files = [ {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438"}, - {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:cb1dac1770878ade83f2ccdf7d25e494f05c9165f5246b46a621cc849341dc01"}, - {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:3ee8a80d67a4334482d9712b8e83ca6b1d9bc7e351931252ebef5d8f7335a547"}, - {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5e55da2c8724191e5b557f8e18943b1b4839b8efc3ef60d65985bcf6f587dd38"}, - {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:d342778ef319e1026af243ed0a07c97acf3bad33b9f29e7ae6a1f68fd083e90c"}, {file = "Brotli-1.1.0-cp37-cp37m-win32.whl", hash = "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95"}, {file = "Brotli-1.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68"}, {file = "Brotli-1.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3"}, @@ -937,10 +903,6 @@ files = [ {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596"}, - {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d2b35ca2c7f81d173d2fadc2f4f31e88cc5f7a39ae5b6db5513cf3383b0e0ec7"}, - {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:af6fa6817889314555aede9a919612b23739395ce767fe7fcbea9a80bf140fe5"}, - {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:2feb1d960f760a575dbc5ab3b1c00504b24caaf6986e2dc2b01c09c87866a943"}, - {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4410f84b33374409552ac9b6903507cdb31cd30d2501fc5ca13d18f73548444a"}, {file = "Brotli-1.1.0-cp38-cp38-win32.whl", hash = "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b"}, {file = "Brotli-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0"}, {file = "Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a"}, @@ -953,10 +915,6 @@ files = [ {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59"}, - {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419"}, - {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2"}, - {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f"}, - {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb"}, {file = "Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64"}, {file = "Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467"}, {file = "Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724"}, @@ -4405,6 +4363,21 @@ html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] source = ["Cython (>=3.0.11,<3.1.0)"] +[[package]] +name = "lxml-stubs" +version = "0.5.1" +description = "Type annotations for the lxml package" +optional = false +python-versions = "*" +groups = ["dev"] +files = [ + {file = "lxml-stubs-0.5.1.tar.gz", hash = "sha256:e0ec2aa1ce92d91278b719091ce4515c12adc1d564359dfaf81efa7d4feab79d"}, + {file = "lxml_stubs-0.5.1-py3-none-any.whl", hash = "sha256:1f689e5dbc4b9247cb09ae820c7d34daeb1fdbd1db06123814b856dae7787272"}, +] + +[package.extras] +test = ["coverage[toml] (>=7.2.5)", "mypy (>=1.2.0)", "pytest (>=7.3.0)", "pytest-mypy-plugins (>=1.10.1)"] + [[package]] name = "lz4" version = "4.4.3" @@ -4944,49 +4917,49 @@ files = [ [[package]] name = "mypy" -version = "1.13.0" +version = "1.15.0" description = "Optional static typing for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, - {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, - {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, - {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, - {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, - {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, - {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, - {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, - {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, - {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, - {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, - {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, - {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, - {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, - {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, - {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, - {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, - {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, - {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, - {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, - {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, - {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, + {file = "mypy-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:979e4e1a006511dacf628e36fadfecbcc0160a8af6ca7dad2f5025529e082c13"}, + {file = "mypy-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c4bb0e1bd29f7d34efcccd71cf733580191e9a264a2202b0239da95984c5b559"}, + {file = "mypy-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be68172e9fd9ad8fb876c6389f16d1c1b5f100ffa779f77b1fb2176fcc9ab95b"}, + {file = "mypy-1.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7be1e46525adfa0d97681432ee9fcd61a3964c2446795714699a998d193f1a3"}, + {file = "mypy-1.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2e2c2e6d3593f6451b18588848e66260ff62ccca522dd231cd4dd59b0160668b"}, + {file = "mypy-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:6983aae8b2f653e098edb77f893f7b6aca69f6cffb19b2cc7443f23cce5f4828"}, + {file = "mypy-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2922d42e16d6de288022e5ca321cd0618b238cfc5570e0263e5ba0a77dbef56f"}, + {file = "mypy-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ee2d57e01a7c35de00f4634ba1bbf015185b219e4dc5909e281016df43f5ee5"}, + {file = "mypy-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:973500e0774b85d9689715feeffcc980193086551110fd678ebe1f4342fb7c5e"}, + {file = "mypy-1.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a95fb17c13e29d2d5195869262f8125dfdb5c134dc8d9a9d0aecf7525b10c2c"}, + {file = "mypy-1.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1905f494bfd7d85a23a88c5d97840888a7bd516545fc5aaedff0267e0bb54e2f"}, + {file = "mypy-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:c9817fa23833ff189db061e6d2eff49b2f3b6ed9856b4a0a73046e41932d744f"}, + {file = "mypy-1.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:aea39e0583d05124836ea645f412e88a5c7d0fd77a6d694b60d9b6b2d9f184fd"}, + {file = "mypy-1.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f2147ab812b75e5b5499b01ade1f4a81489a147c01585cda36019102538615f"}, + {file = "mypy-1.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce436f4c6d218a070048ed6a44c0bbb10cd2cc5e272b29e7845f6a2f57ee4464"}, + {file = "mypy-1.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8023ff13985661b50a5928fc7a5ca15f3d1affb41e5f0a9952cb68ef090b31ee"}, + {file = "mypy-1.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1124a18bc11a6a62887e3e137f37f53fbae476dc36c185d549d4f837a2a6a14e"}, + {file = "mypy-1.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:171a9ca9a40cd1843abeca0e405bc1940cd9b305eaeea2dda769ba096932bb22"}, + {file = "mypy-1.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93faf3fdb04768d44bf28693293f3904bbb555d076b781ad2530214ee53e3445"}, + {file = "mypy-1.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:811aeccadfb730024c5d3e326b2fbe9249bb7413553f15499a4050f7c30e801d"}, + {file = "mypy-1.15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98b7b9b9aedb65fe628c62a6dc57f6d5088ef2dfca37903a7d9ee374d03acca5"}, + {file = "mypy-1.15.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c43a7682e24b4f576d93072216bf56eeff70d9140241f9edec0c104d0c515036"}, + {file = "mypy-1.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:baefc32840a9f00babd83251560e0ae1573e2f9d1b067719479bfb0e987c6357"}, + {file = "mypy-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:b9378e2c00146c44793c98b8d5a61039a048e31f429fb0eb546d93f4b000bedf"}, + {file = "mypy-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e601a7fa172c2131bff456bb3ee08a88360760d0d2f8cbd7a75a65497e2df078"}, + {file = "mypy-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:712e962a6357634fef20412699a3655c610110e01cdaa6180acec7fc9f8513ba"}, + {file = "mypy-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f95579473af29ab73a10bada2f9722856792a36ec5af5399b653aa28360290a5"}, + {file = "mypy-1.15.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f8722560a14cde92fdb1e31597760dc35f9f5524cce17836c0d22841830fd5b"}, + {file = "mypy-1.15.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fbb8da62dc352133d7d7ca90ed2fb0e9d42bb1a32724c287d3c76c58cbaa9c2"}, + {file = "mypy-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:d10d994b41fb3497719bbf866f227b3489048ea4bbbb5015357db306249f7980"}, + {file = "mypy-1.15.0-py3-none-any.whl", hash = "sha256:5469affef548bd1895d86d3bf10ce2b44e33d86923c29e4d675b3e323437ea3e"}, + {file = "mypy-1.15.0.tar.gz", hash = "sha256:404534629d51d3efea5c800ee7c42b72a6554d6c400e6a79eafe15d11341fd43"}, ] [package.dependencies] -mypy-extensions = ">=1.0.0" -typing-extensions = ">=4.6.0" +mypy_extensions = ">=1.0.0" +typing_extensions = ">=4.6.0" [package.extras] dmypy = ["psutil (>=4.0)"] @@ -5135,7 +5108,7 @@ version = "1.26.4" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" -groups = ["main", "indirect", "vdb"] +groups = ["main", "dev", "indirect", "vdb"] files = [ {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, @@ -8859,6 +8832,18 @@ rich = ">=10.11.0" shellingham = ">=1.3.0" typing-extensions = ">=3.7.4.3" +[[package]] +name = "types-aiofiles" +version = "24.1.0.20250326" +description = "Typing stubs for aiofiles" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_aiofiles-24.1.0.20250326-py3-none-any.whl", hash = "sha256:dfb58c9aa18bd449e80fb5d7f49dc3dd20d31de920a46223a61798ee4a521a70"}, + {file = "types_aiofiles-24.1.0.20250326.tar.gz", hash = "sha256:c4bbe432fd043911ba83fb635456f5cc54f6d05fda2aadf6bef12a84f07a6efe"}, +] + [[package]] name = "types-beautifulsoup4" version = "4.12.0.20250204" @@ -8874,6 +8859,42 @@ files = [ [package.dependencies] types-html5lib = "*" +[[package]] +name = "types-cachetools" +version = "5.5.0.20240820" +description = "Typing stubs for cachetools" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "types-cachetools-5.5.0.20240820.tar.gz", hash = "sha256:b888ab5c1a48116f7799cd5004b18474cd82b5463acb5ffb2db2fc9c7b053bc0"}, + {file = "types_cachetools-5.5.0.20240820-py3-none-any.whl", hash = "sha256:efb2ed8bf27a4b9d3ed70d33849f536362603a90b8090a328acf0cd42fda82e2"}, +] + +[[package]] +name = "types-colorama" +version = "0.4.15.20240311" +description = "Typing stubs for colorama" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "types-colorama-0.4.15.20240311.tar.gz", hash = "sha256:a28e7f98d17d2b14fb9565d32388e419f4108f557a7d939a66319969b2b99c7a"}, + {file = "types_colorama-0.4.15.20240311-py3-none-any.whl", hash = "sha256:6391de60ddc0db3f147e31ecb230006a6823e81e380862ffca1e4695c13a0b8e"}, +] + +[[package]] +name = "types-defusedxml" +version = "0.7.0.20240218" +description = "Typing stubs for defusedxml" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "types-defusedxml-0.7.0.20240218.tar.gz", hash = "sha256:05688a7724dc66ea74c4af5ca0efc554a150c329cb28c13a64902cab878d06ed"}, + {file = "types_defusedxml-0.7.0.20240218-py3-none-any.whl", hash = "sha256:2b7f3c5ca14fdbe728fab0b846f5f7eb98c4bd4fd2b83d25f79e923caa790ced"}, +] + [[package]] name = "types-deprecated" version = "1.2.15.20250304" @@ -8886,16 +8907,28 @@ files = [ {file = "types_deprecated-1.2.15.20250304.tar.gz", hash = "sha256:c329030553029de5cc6cb30f269c11f4e00e598c4241290179f63cda7d33f719"}, ] +[[package]] +name = "types-docutils" +version = "0.21.0.20241128" +description = "Typing stubs for docutils" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "types_docutils-0.21.0.20241128-py3-none-any.whl", hash = "sha256:e0409204009639e9b0bf4521eeabe58b5e574ce9c0db08421c2ac26c32be0039"}, + {file = "types_docutils-0.21.0.20241128.tar.gz", hash = "sha256:4dd059805b83ac6ec5a223699195c4e9eeb0446a4f7f2aeff1759a4a7cc17473"}, +] + [[package]] name = "types-flask-cors" -version = "4.0.0.20240828" +version = "5.0.0.20240902" description = "Typing stubs for Flask-Cors" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "types-Flask-Cors-4.0.0.20240828.tar.gz", hash = "sha256:f48ecf6366da923331311907cde3500e1435e07df01397ce0ef2306e263a5e85"}, - {file = "types_Flask_Cors-4.0.0.20240828-py3-none-any.whl", hash = "sha256:36b752e88d6517fb82973b4240fe9bde44d29485bbd92dfff762a7101bdac3a0"}, + {file = "types-Flask-Cors-5.0.0.20240902.tar.gz", hash = "sha256:8921b273bf7cd9636df136b66408efcfa6338a935e5c8f53f5eff1cee03f3394"}, + {file = "types_Flask_Cors-5.0.0.20240902-py3-none-any.whl", hash = "sha256:595e5f36056cd128ab905832e055f2e5d116fbdc685356eea4490bc77df82137"}, ] [package.dependencies] @@ -8917,6 +8950,34 @@ files = [ Flask = ">=2.0.0" Flask-SQLAlchemy = ">=3.0.1" +[[package]] +name = "types-gevent" +version = "24.11.0.20250401" +description = "Typing stubs for gevent" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_gevent-24.11.0.20250401-py3-none-any.whl", hash = "sha256:6764faf861ea99250c38179c58076392c44019ac3393029f71b06c4a15e8c1d1"}, + {file = "types_gevent-24.11.0.20250401.tar.gz", hash = "sha256:1443f796a442062698e67d818fca50aa88067dee4021d457a7c0c6bedd6f46ca"}, +] + +[package.dependencies] +types-greenlet = "*" +types-psutil = "*" + +[[package]] +name = "types-greenlet" +version = "3.1.0.20250401" +description = "Typing stubs for greenlet" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_greenlet-3.1.0.20250401-py3-none-any.whl", hash = "sha256:77987f3249b0f21415dc0254057e1ae4125a696a9bba28b0bcb67ee9e3dc14f6"}, + {file = "types_greenlet-3.1.0.20250401.tar.gz", hash = "sha256:949389b64c34ca9472f6335189e9fe0b2e9704436d4f0850e39e9b7145909082"}, +] + [[package]] name = "types-html5lib" version = "1.1.11.20241018" @@ -8929,6 +8990,54 @@ files = [ {file = "types_html5lib-1.1.11.20241018-py3-none-any.whl", hash = "sha256:3f1e064d9ed2c289001ae6392c84c93833abb0816165c6ff0abfc304a779f403"}, ] +[[package]] +name = "types-markdown" +version = "3.7.0.20250322" +description = "Typing stubs for Markdown" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_markdown-3.7.0.20250322-py3-none-any.whl", hash = "sha256:7e855503027b4290355a310fb834871940d9713da7c111f3e98a5e1cbc77acfb"}, + {file = "types_markdown-3.7.0.20250322.tar.gz", hash = "sha256:a48ed82dfcb6954592a10f104689d2d44df9125ce51b3cee20e0198a5216d55c"}, +] + +[[package]] +name = "types-oauthlib" +version = "3.2.0.20250403" +description = "Typing stubs for oauthlib" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_oauthlib-3.2.0.20250403-py3-none-any.whl", hash = "sha256:02466f91a01522adfa4aaf0d7e76274f00a102eed40034117c5ecae768a2571e"}, + {file = "types_oauthlib-3.2.0.20250403.tar.gz", hash = "sha256:40a4fcfb2e95235e399b5c0dd1cbe9d8c4b19415c09fb54c648d3397e02e0425"}, +] + +[[package]] +name = "types-objgraph" +version = "3.6.0.20240907" +description = "Typing stubs for objgraph" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "types-objgraph-3.6.0.20240907.tar.gz", hash = "sha256:2e3dee675843ae387889731550b0ddfed06e9420946cf78a4bca565b5fc53634"}, + {file = "types_objgraph-3.6.0.20240907-py3-none-any.whl", hash = "sha256:67207633a9b5789ee1911d740b269c3371081b79c0d8f68b00e7b8539f5c43f5"}, +] + +[[package]] +name = "types-olefile" +version = "0.47.0.20240806" +description = "Typing stubs for olefile" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "types-olefile-0.47.0.20240806.tar.gz", hash = "sha256:96490f208cbb449a52283855319d73688ba9167ae58858ef8c506bf7ca2c6b67"}, + {file = "types_olefile-0.47.0.20240806-py3-none-any.whl", hash = "sha256:c760a3deab7adb87a80d33b0e4edbbfbab865204a18d5121746022d7f8555118"}, +] + [[package]] name = "types-openpyxl" version = "3.1.5.20250306" @@ -8942,27 +9051,39 @@ files = [ ] [[package]] -name = "types-protobuf" -version = "4.25.0.20240417" -description = "Typing stubs for protobuf" +name = "types-pexpect" +version = "4.9.0.20241208" +description = "Typing stubs for pexpect" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "types-protobuf-4.25.0.20240417.tar.gz", hash = "sha256:c34eff17b9b3a0adb6830622f0f302484e4c089f533a46e3f147568313544352"}, - {file = "types_protobuf-4.25.0.20240417-py3-none-any.whl", hash = "sha256:e9b613227c2127e3d4881d75d93c93b4d6fd97b5f6a099a0b654a05351c8685d"}, + {file = "types_pexpect-4.9.0.20241208-py3-none-any.whl", hash = "sha256:1928f478528454f0fea3495c16cf1ee2e67fca5c9fe97d60b868ac48c1fd5633"}, + {file = "types_pexpect-4.9.0.20241208.tar.gz", hash = "sha256:bbca0d0819947a719989a5cfe83641d9212bef893e2f0a7a01e47926bc82401d"}, +] + +[[package]] +name = "types-protobuf" +version = "5.29.1.20250403" +description = "Typing stubs for protobuf" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_protobuf-5.29.1.20250403-py3-none-any.whl", hash = "sha256:c71de04106a2d54e5b2173d0a422058fae0ef2d058d70cf369fb797bf61ffa59"}, + {file = "types_protobuf-5.29.1.20250403.tar.gz", hash = "sha256:7ff44f15022119c9d7558ce16e78b2d485bf7040b4fadced4dd069bb5faf77a2"}, ] [[package]] name = "types-psutil" -version = "7.0.0.20250218" +version = "7.0.0.20250401" description = "Typing stubs for psutil" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_psutil-7.0.0.20250218-py3-none-any.whl", hash = "sha256:1447a30c282aafefcf8941ece854e1100eee7b0296a9d9be9977292f0269b121"}, - {file = "types_psutil-7.0.0.20250218.tar.gz", hash = "sha256:1e642cdafe837b240295b23b1cbd4691d80b08a07d29932143cbbae30eb0db9c"}, + {file = "types_psutil-7.0.0.20250401-py3-none-any.whl", hash = "sha256:ed23f7140368104afe4e05a6085a5fa56fbe8c880a0f4dfe8d63e041106071ed"}, + {file = "types_psutil-7.0.0.20250401.tar.gz", hash = "sha256:2a7d663c0888a079fc1643ebc109ad12e57a21c9552a9e2035da504191336dbf"}, ] [[package]] @@ -8977,6 +9098,33 @@ files = [ {file = "types_psycopg2-2.9.21.20250318.tar.gz", hash = "sha256:eb6eac5bfb16adfd5f16b818918b9e26a40ede147e0f2bbffdf53a6ef7025a87"}, ] +[[package]] +name = "types-pygments" +version = "2.19.0.20250305" +description = "Typing stubs for Pygments" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_pygments-2.19.0.20250305-py3-none-any.whl", hash = "sha256:ca88aae5ec426f9b107c0f7adc36dc096d2882d930a49f679eaf4b8b643db35d"}, + {file = "types_pygments-2.19.0.20250305.tar.gz", hash = "sha256:044c50e80ecd4128c00a7268f20355e16f5c55466d3d49dfda09be920af40b4b"}, +] + +[package.dependencies] +types-docutils = "*" + +[[package]] +name = "types-pymysql" +version = "1.1.0.20241103" +description = "Typing stubs for PyMySQL" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "types-PyMySQL-1.1.0.20241103.tar.gz", hash = "sha256:a7628542919a0ba87625fb79eefb2a2de45fb4ad32afe6e561e8f2f27fb58b8c"}, + {file = "types_PyMySQL-1.1.0.20241103-py3-none-any.whl", hash = "sha256:1a32efd8a74b5bf74c4de92a86c1cc6edaf3802dcfd5546635ab501eb5e3c096"}, +] + [[package]] name = "types-python-dateutil" version = "2.9.0.20241206" @@ -8995,78 +9143,162 @@ version = "2025.1.0.20250318" description = "Typing stubs for pytz" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "types_pytz-2025.1.0.20250318-py3-none-any.whl", hash = "sha256:04dba4907c5415777083f9548693c6d9f80ec53adcaff55a38526a3f8ddcae04"}, {file = "types_pytz-2025.1.0.20250318.tar.gz", hash = "sha256:97e0e35184c6fe14e3a5014512057f2c57bb0c6582d63c1cfcc4809f82180449"}, ] [[package]] -name = "types-pyyaml" -version = "6.0.12.20241230" -description = "Typing stubs for PyYAML" +name = "types-pywin32" +version = "310.0.0.20250319" +description = "Typing stubs for pywin32" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_PyYAML-6.0.12.20241230-py3-none-any.whl", hash = "sha256:fa4d32565219b68e6dee5f67534c722e53c00d1cfc09c435ef04d7353e1e96e6"}, - {file = "types_pyyaml-6.0.12.20241230.tar.gz", hash = "sha256:7f07622dbd34bb9c8b264fe860a17e0efcad00d50b5f27e93984909d9363498c"}, + {file = "types_pywin32-310.0.0.20250319-py3-none-any.whl", hash = "sha256:baeb558a82251f7d430d135036b054740893902fdee3f9fe568322730ff49779"}, + {file = "types_pywin32-310.0.0.20250319.tar.gz", hash = "sha256:4d28fb85b3f268a92905a7242df48c530c847cfe4cdb112386101ab6407660d8"}, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.20250402" +description = "Typing stubs for PyYAML" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_pyyaml-6.0.12.20250402-py3-none-any.whl", hash = "sha256:652348fa9e7a203d4b0d21066dfb00760d3cbd5a15ebb7cf8d33c88a49546681"}, + {file = "types_pyyaml-6.0.12.20250402.tar.gz", hash = "sha256:d7c13c3e6d335b6af4b0122a01ff1d270aba84ab96d1a1a1063ecba3e13ec075"}, ] [[package]] name = "types-regex" -version = "2024.11.6.20250318" +version = "2024.11.6.20250403" description = "Typing stubs for regex" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_regex-2024.11.6.20250318-py3-none-any.whl", hash = "sha256:9309fe5918ee7ffe859c04c18040697655fade366c4dc844bbebe86976a9980b"}, - {file = "types_regex-2024.11.6.20250318.tar.gz", hash = "sha256:6d472d0acf37b138cb32f67bd5ab1e7a200e94da8c1aa93ca3625a63e2efe1f3"}, + {file = "types_regex-2024.11.6.20250403-py3-none-any.whl", hash = "sha256:e22c0f67d73f4b4af6086a340f387b6f7d03bed8a0bb306224b75c51a29b0001"}, + {file = "types_regex-2024.11.6.20250403.tar.gz", hash = "sha256:3fdf2a70bbf830de4b3a28e9649a52d43dabb57cdb18fbfe2252eefb53666665"}, ] [[package]] name = "types-requests" -version = "2.31.0.20240406" +version = "2.32.0.20250328" description = "Typing stubs for requests" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types-requests-2.31.0.20240406.tar.gz", hash = "sha256:4428df33c5503945c74b3f42e82b181e86ec7b724620419a2966e2de604ce1a1"}, - {file = "types_requests-2.31.0.20240406-py3-none-any.whl", hash = "sha256:6216cdac377c6b9a040ac1c0404f7284bd13199c0e1bb235f4324627e8898cf5"}, + {file = "types_requests-2.32.0.20250328-py3-none-any.whl", hash = "sha256:72ff80f84b15eb3aa7a8e2625fffb6a93f2ad5a0c20215fc1dcfa61117bcb2a2"}, + {file = "types_requests-2.32.0.20250328.tar.gz", hash = "sha256:c9e67228ea103bd811c96984fac36ed2ae8da87a36a633964a21f199d60baf32"}, ] [package.dependencies] urllib3 = ">=2" +[[package]] +name = "types-requests-oauthlib" +version = "2.0.0.20250306" +description = "Typing stubs for requests-oauthlib" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_requests_oauthlib-2.0.0.20250306-py3-none-any.whl", hash = "sha256:37707de81d9ce54894afcccd70d4a845dbe4c59e747908faaeba59a96453d993"}, + {file = "types_requests_oauthlib-2.0.0.20250306.tar.gz", hash = "sha256:92e5f1ed35689b1804fdcd60b7ac39b0bd440a4b96693685879bc835b334797f"}, +] + +[package.dependencies] +types-oauthlib = "*" +types-requests = "*" + +[[package]] +name = "types-shapely" +version = "2.0.0.20250404" +description = "Typing stubs for shapely" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_shapely-2.0.0.20250404-py3-none-any.whl", hash = "sha256:170fb92f5c168a120db39b3287697fdec5c93ef3e1ad15e52552c36b25318821"}, + {file = "types_shapely-2.0.0.20250404.tar.gz", hash = "sha256:863f540b47fa626c33ae64eae06df171f9ab0347025d4458d2df496537296b4f"}, +] + +[package.dependencies] +numpy = ">=1.20" + +[[package]] +name = "types-simplejson" +version = "3.20.0.20250326" +description = "Typing stubs for simplejson" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_simplejson-3.20.0.20250326-py3-none-any.whl", hash = "sha256:db1ddea7b8f7623b27a137578f22fc6c618db8c83ccfb1828ca0d2f0ec11efa7"}, + {file = "types_simplejson-3.20.0.20250326.tar.gz", hash = "sha256:b2689bc91e0e672d7a5a947b4cb546b76ae7ddc2899c6678e72a10bf96cd97d2"}, +] + [[package]] name = "types-six" -version = "1.17.0.20250304" +version = "1.17.0.20250403" description = "Typing stubs for six" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_six-1.17.0.20250304-py3-none-any.whl", hash = "sha256:e482df1d439375f4b7c1f2540b1b8584aea82850164a296203ead4a7024fe14f"}, - {file = "types_six-1.17.0.20250304.tar.gz", hash = "sha256:eeb240f9faec63ddd0498d6c0b6abd0496b154a66f960c004d4d733cf31bb4bd"}, + {file = "types_six-1.17.0.20250403-py3-none-any.whl", hash = "sha256:0bbb20fc34a18163afe7cac70b85864bd6937e6d73413c5b8f424def28760ae8"}, + {file = "types_six-1.17.0.20250403.tar.gz", hash = "sha256:82076f86e6e672a95adbf8b52625b1b3c72a8b9a893180344c1a02a6daabead6"}, ] +[[package]] +name = "types-tensorflow" +version = "2.18.0.20250404" +description = "Typing stubs for tensorflow" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_tensorflow-2.18.0.20250404-py3-none-any.whl", hash = "sha256:4ad86534e6cfd6b36b2c97239ef9d122c44b167b25630b7c873a1483f9befd15"}, + {file = "types_tensorflow-2.18.0.20250404.tar.gz", hash = "sha256:b38a427bbec805e4879d248f070baea802673c04cc5ccbe5979d742faa160670"}, +] + +[package.dependencies] +numpy = ">=1.20" +types-protobuf = "*" +types-requests = "*" + [[package]] name = "types-tqdm" -version = "4.67.0.20250301" +version = "4.67.0.20250404" description = "Typing stubs for tqdm" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_tqdm-4.67.0.20250301-py3-none-any.whl", hash = "sha256:8af97deb8e6874af833555dc1fe0fcd456b1a789470bf6cd8813d4e7ee4f6c5b"}, - {file = "types_tqdm-4.67.0.20250301.tar.gz", hash = "sha256:5e89a38ad89b867823368eb97d9f90d2fc69806bb055dde62716a05da62b5e0d"}, + {file = "types_tqdm-4.67.0.20250404-py3-none-any.whl", hash = "sha256:4a9b897eb4036f757240f4cb4a794f296265c04de46fdd058e453891f0186eed"}, + {file = "types_tqdm-4.67.0.20250404.tar.gz", hash = "sha256:e9997c655ffbba3ab78f4418b5511c05a54e76824d073d212166dc73aa56c768"}, ] [package.dependencies] types-requests = "*" +[[package]] +name = "types-ujson" +version = "5.10.0.20250326" +description = "Typing stubs for ujson" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "types_ujson-5.10.0.20250326-py3-none-any.whl", hash = "sha256:acc0913f569def62ef6a892c8a47703f65d05669a3252391a97765cf207dca5b"}, + {file = "types_ujson-5.10.0.20250326.tar.gz", hash = "sha256:5469e05f2c31ecb3c4c0267cc8fe41bcd116826fbb4ded69801a645c687dd014"}, +] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -10148,4 +10380,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "a094082d2fd4d8ea480ac800e54029bdb604de70a7b4348778bdcddb39b06c7e" +content-hash = "7bdb4c26ad249bacd8149e8931f4cdc25d9d0cb319329b1e939e1b4f2c7f40b1" diff --git a/api/pyproject.toml b/api/pyproject.toml index 0cea0293a6..3879352293 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "dify-api" requires-python = ">=3.11,<3.13" -dynamic = [ "dependencies" ] +dynamic = ["dependencies"] [build-system] requires = ["poetry-core>=2.0.0"] @@ -147,27 +147,47 @@ optional = true [tool.poetry.group.dev.dependencies] coverage = "~7.2.4" faker = "~32.1.0" -mypy = "~1.13.0" +lxml-stubs = "~0.5.1" +mypy = "~1.15.0" pytest = "~8.3.2" pytest-benchmark = "~4.0.0" pytest-env = "~1.1.3" pytest-mock = "~3.14.0" +types-aiofiles = "~24.1.0" types-beautifulsoup4 = "~4.12.0" +types-cachetools = "~5.5.0" +types-colorama = "~0.4.15" +types-defusedxml = "~0.7.0" types-deprecated = "~1.2.15" -types-flask-cors = "~4.0.0" +types-docutils = "~0.21.0" +types-flask-cors = "~5.0.0" types-flask-migrate = "~4.1.0" +types-gevent = "~24.11.0" +types-greenlet = "~3.1.0" types-html5lib = "~1.1.11" +types-markdown = "~3.7.0" +types-oauthlib = "~3.2.0" +types-objgraph = "~3.6.0" +types-olefile = "~0.47.0" types-openpyxl = "~3.1.5" -types-protobuf = "~4.25.0" +types-pexpect = "~4.9.0" +types-protobuf = "~5.29.1" types-psutil = "~7.0.0" types-psycopg2 = "~2.9.21" +types-pygments = "~2.19.0" +types-pymysql = "~1.1.0" types-python-dateutil = "~2.9.0" -types-pytz = "~2025.1" -types-pyyaml = "~6.0.2" +types-pywin32 = "~310.0.0" +types-pyyaml = "~6.0.12" types-regex = "~2024.11.6" -types-requests = "~2.31.0" +types-requests = "~2.32.0" +types-requests-oauthlib = "~2.0.0" +types-shapely = "~2.0.0" +types-simplejson = "~3.20.0" types-six = "~1.17.0" +types-tensorflow = "~2.18.0" types-tqdm = "~4.67.0" +types-ujson = "~5.10.0" ############################################################ # [ Lint ] dependency group diff --git a/dev/reformat b/dev/reformat index 82f96b8e8f..daab538951 100755 --- a/dev/reformat +++ b/dev/reformat @@ -16,3 +16,6 @@ poetry run -C api ruff format ./ # run dotenv-linter linter poetry run -P api dotenv-linter ./api/.env.example ./web/.env.example + +# run mypy check +dev/run-mypy diff --git a/dev/run-mypy b/dev/run-mypy new file mode 100755 index 0000000000..cdbbef515d --- /dev/null +++ b/dev/run-mypy @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +if ! command -v mypy &> /dev/null; then + poetry install -C api --with dev +fi + +# run mypy checks +poetry run -C api \ + python -m mypy --install-types --non-interactive . From 9000f4ad050f967c5f4768c9f5cfad17998b498c Mon Sep 17 00:00:00 2001 From: quicksand Date: Wed, 9 Apr 2025 14:02:17 +0800 Subject: [PATCH 17/53] feat: add plugin daemon oss env config (#17663) --- docker/.env.example | 25 +++++++++++++++++++ docker/docker-compose-template.yaml | 17 +++++++++++++ docker/docker-compose.middleware.yaml | 17 +++++++++++++ docker/docker-compose.yaml | 35 +++++++++++++++++++++++++++ docker/middleware.env.example | 27 ++++++++++++++++++++- 5 files changed, 120 insertions(+), 1 deletion(-) diff --git a/docker/.env.example b/docker/.env.example index 4ab55a9623..29d33360ea 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1005,3 +1005,28 @@ PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120 PLUGIN_MAX_EXECUTION_TIMEOUT=600 # PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple PIP_MIRROR_URL= + +# https://github.com/langgenius/dify-plugin-daemon/blob/main/.env.example +# Plugin storage type, local aws_s3 tencent_cos azure_blob +PLUGIN_STORAGE_TYPE=local +PLUGIN_STORAGE_LOCAL_ROOT=/app/storage +PLUGIN_WORKING_PATH=/app/storage/cwd +PLUGIN_INSTALLED_PATH=plugin +PLUGIN_PACKAGE_CACHE_PATH=plugin_packages +PLUGIN_MEDIA_CACHE_PATH=assets +# Plugin oss bucket +PLUGIN_STORAGE_OSS_BUCKET= +# Plugin oss s3 credentials +PLUGIN_S3_USE_AWS_MANAGED_IAM= +PLUGIN_S3_ENDPOINT= +PLUGIN_S3_USE_PATH_STYLE= +PLUGIN_AWS_ACCESS_KEY= +PLUGIN_AWS_SECRET_KEY= +PLUGIN_AWS_REGION= +# Plugin oss azure blob +PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME= +PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING= +# Plugin oss tencent cos +PLUGIN_TENCENT_COS_SECRET_KEY= +PLUGIN_TENCENT_COS_SECRET_ID= +PLUGIN_TENCENT_COS_REGION= diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index e8ed382917..76f70c53b8 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -153,6 +153,23 @@ services: PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-} + S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-} + AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + AWS_REGION: ${PLUGIN_AWS_REGION:-} + AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} ports: - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" volumes: diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index 230b8a05be..0035183fca 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -97,6 +97,23 @@ services: PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-} + S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-} + AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + AWS_REGION: ${PLUGIN_AWS_REGION:-} + AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} ports: - "${EXPOSE_PLUGIN_DAEMON_PORT:-5002}:${PLUGIN_DAEMON_PORT:-5002}" - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 6a3e744cfd..3d84af07f4 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -436,6 +436,24 @@ x-shared-env: &shared-api-worker-env PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-} + PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-} + PLUGIN_AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + PLUGIN_AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + PLUGIN_AWS_REGION: ${PLUGIN_AWS_REGION:-} + PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + PLUGIN_TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + PLUGIN_TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + PLUGIN_TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} services: # API service @@ -591,6 +609,23 @@ services: PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-} + S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-} + AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + AWS_REGION: ${PLUGIN_AWS_REGION:-} + AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} ports: - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" volumes: diff --git a/docker/middleware.env.example b/docker/middleware.env.example index d01f9abe53..eb38526d57 100644 --- a/docker/middleware.env.example +++ b/docker/middleware.env.example @@ -119,4 +119,29 @@ FORCE_VERIFYING_SIGNATURE=true PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120 PLUGIN_MAX_EXECUTION_TIMEOUT=600 # PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple -PIP_MIRROR_URL= \ No newline at end of file +PIP_MIRROR_URL= + +# https://github.com/langgenius/dify-plugin-daemon/blob/main/.env.example +# Plugin storage type, local aws_s3 tencent_cos azure_blob +PLUGIN_STORAGE_TYPE=local +PLUGIN_STORAGE_LOCAL_ROOT=/app/storage +PLUGIN_WORKING_PATH=/app/storage/cwd +PLUGIN_INSTALLED_PATH=plugin +PLUGIN_PACKAGE_CACHE_PATH=plugin_packages +PLUGIN_MEDIA_CACHE_PATH=assets +# Plugin oss bucket +PLUGIN_STORAGE_OSS_BUCKET= +# Plugin oss s3 credentials +PLUGIN_S3_USE_AWS_MANAGED_IAM= +PLUGIN_S3_ENDPOINT= +PLUGIN_S3_USE_PATH_STYLE= +PLUGIN_AWS_ACCESS_KEY= +PLUGIN_AWS_SECRET_KEY= +PLUGIN_AWS_REGION= +# Plugin oss azure blob +PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME= +PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING= +# Plugin oss tencent cos +PLUGIN_TENCENT_COS_SECRET_KEY= +PLUGIN_TENCENT_COS_SECRET_ID= +PLUGIN_TENCENT_COS_REGION= \ No newline at end of file From eb8584613beda9621051936e391d31b63ff56efd Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Wed, 9 Apr 2025 14:07:32 +0800 Subject: [PATCH 18/53] fix: Account.query => db.session.query(Account) (#17667) --- api/controllers/service_api/wraps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index ff33b62eda..7facb03358 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -69,7 +69,7 @@ def validate_app_token(view: Optional[Callable] = None, *, fetch_user_arg: Optio ) # TODO: only owner information is required, so only one is returned. if tenant_account_join: tenant, ta = tenant_account_join - account = Account.query.filter_by(id=ta.account_id).first() + account = db.session.query(Account).filter(Account.id == ta.account_id).first() # Login admin if account: account.current_tenant = tenant From df03c89a48f6da90f28287eabd75276adc9a4596 Mon Sep 17 00:00:00 2001 From: KVOJJJin Date: Wed, 9 Apr 2025 15:10:08 +0800 Subject: [PATCH 19/53] Chore: remove beta tag of app type (#17676) --- web/app/components/app/create-app-modal/index.tsx | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/web/app/components/app/create-app-modal/index.tsx b/web/app/components/app/create-app-modal/index.tsx index c442b6e979..6abc871868 100644 --- a/web/app/components/app/create-app-modal/index.tsx +++ b/web/app/components/app/create-app-modal/index.tsx @@ -148,7 +148,6 @@ function CreateApp({ onClose, onSuccess, onCreateFromTemplate }: CreateAppProps)
void } -function AppTypeCard({ icon, title, beta = false, description, active, onClick }: AppTypeCardProps) { +function AppTypeCard({ icon, title, description, active, onClick }: AppTypeCardProps) { const { t } = useTranslation() return
- {beta &&
{t('common.menus.status')}
} {icon}
{title}
{description}
From c9f18aae0fca88ec7c440e60b6153333a64d14bb Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 9 Apr 2025 15:39:12 +0800 Subject: [PATCH 20/53] chore: find code with high complexity (#17679) --- web/.vscode/extensions.json | 5 +++-- web/eslint.config.mjs | 3 +-- web/package.json | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/web/.vscode/extensions.json b/web/.vscode/extensions.json index d7680d74a5..a9afbcc640 100644 --- a/web/.vscode/extensions.json +++ b/web/.vscode/extensions.json @@ -1,6 +1,7 @@ { "recommendations": [ "bradlc.vscode-tailwindcss", - "firsttris.vscode-jest-runner" + "firsttris.vscode-jest-runner", + "kisstkondoros.vscode-codemetrics" ] -} +} \ No newline at end of file diff --git a/web/eslint.config.mjs b/web/eslint.config.mjs index 204efc4715..750cee5545 100644 --- a/web/eslint.config.mjs +++ b/web/eslint.config.mjs @@ -65,8 +65,6 @@ export default combine( // use `ESLINT_CONFIG_INSPECTOR=true pnpx @eslint/config-inspector` to check the config // ...process.env.ESLINT_CONFIG_INSPECTOR // ? [] - // TODO: remove this when upgrade to nextjs 15 - // : fixupConfigRules(compat.extends('next')), { rules: { // performance issue, and not used. @@ -87,6 +85,7 @@ export default combine( { // orignal config rules: { + 'complexity': ['warn', { max: 10 }], // orignal ts/no-var-requires 'ts/no-require-imports': 'off', 'no-console': 'off', diff --git a/web/package.json b/web/package.json index 74eead4eba..f439c03767 100644 --- a/web/package.json +++ b/web/package.json @@ -13,6 +13,7 @@ "fix": "next lint --fix", "eslint-fix": "eslint --cache --cache-location node_modules/.cache/eslint/.eslint-cache --fix", "eslint-fix-only-show-error": "eslint --cache --cache-location node_modules/.cache/eslint/.eslint-cache --fix --quiet", + "eslint-complexity": "eslint --rule 'complexity: [error, {max: 15}]' --quiet", "prepare": "cd ../ && node -e \"if (process.env.NODE_ENV !== 'production'){process.exit(1)} \" || husky ./web/.husky", "gen-icons": "node ./app/components/base/icons/script.mjs", "uglify-embed": "node ./bin/uglify-embed", From ec29bcf013891c39e9fc02607d41788bde5185f5 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Wed, 9 Apr 2025 18:02:47 +0900 Subject: [PATCH 21/53] feat(graph_engine): yield control to other threads before node run. (#17689) Signed-off-by: -LAN- --- api/core/workflow/graph_engine/graph_engine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/core/workflow/graph_engine/graph_engine.py b/api/core/workflow/graph_engine/graph_engine.py index d0f3041d5d..36273d8ec1 100644 --- a/api/core/workflow/graph_engine/graph_engine.py +++ b/api/core/workflow/graph_engine/graph_engine.py @@ -641,6 +641,8 @@ class GraphEngine: try: # run node retry_start_at = datetime.now(UTC).replace(tzinfo=None) + # yield control to other threads + time.sleep(0.001) generator = node_instance.run() for item in generator: if isinstance(item, GraphEngineEvent): From 2c2efe2e1e46751206dfa5972790cef6ad31a93f Mon Sep 17 00:00:00 2001 From: -LAN- Date: Wed, 9 Apr 2025 18:12:40 +0900 Subject: [PATCH 22/53] chore(*): bump version to 1.2.0 (#17675) Signed-off-by: -LAN- --- .github/workflows/build-push.yml | 1 - api/configs/packaging/__init__.py | 2 +- docker/docker-compose-template.yaml | 8 ++++---- docker/docker-compose.middleware.yaml | 2 +- docker/docker-compose.yaml | 8 ++++---- web/package.json | 2 +- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 851621ee49..cc735ae67c 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -6,7 +6,6 @@ on: - "main" - "deploy/dev" - "deploy/enterprise" - - release/1.1.3-fix1 tags: - "*" diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py index 0ef5a724b3..c7aedc5b8a 100644 --- a/api/configs/packaging/__init__.py +++ b/api/configs/packaging/__init__.py @@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings): CURRENT_VERSION: str = Field( description="Dify version", - default="1.1.3", + default="1.2.0", ) COMMIT_SHA: str = Field( diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 76f70c53b8..d7bcff7ed1 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:1.1.3 + image: langgenius/dify-api:1.2.0 restart: always environment: # Use the shared environment variables. @@ -29,7 +29,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:1.1.3 + image: langgenius/dify-api:1.2.0 restart: always environment: # Use the shared environment variables. @@ -53,7 +53,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.1.3 + image: langgenius/dify-web:1.2.0 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -134,7 +134,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.0.6-local + image: langgenius/dify-plugin-daemon:0.0.7-local restart: always environment: # Use the shared environment variables. diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index 0035183fca..de238d9669 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -70,7 +70,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.0.6-local + image: langgenius/dify-plugin-daemon:0.0.7-local restart: always env_file: - ./middleware.env diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 3d84af07f4..f120e19bed 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -458,7 +458,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:1.1.3 + image: langgenius/dify-api:1.2.0 restart: always environment: # Use the shared environment variables. @@ -485,7 +485,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:1.1.3 + image: langgenius/dify-api:1.2.0 restart: always environment: # Use the shared environment variables. @@ -509,7 +509,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.1.3 + image: langgenius/dify-web:1.2.0 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -590,7 +590,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.0.6-local + image: langgenius/dify-plugin-daemon:0.0.7-local restart: always environment: # Use the shared environment variables. diff --git a/web/package.json b/web/package.json index f439c03767..b01466ded3 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "dify-web", - "version": "1.1.3", + "version": "1.2.0", "private": true, "engines": { "node": ">=18.18.0" From 33324ee23d5553215327b8a524e899988724f5bf Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 9 Apr 2025 18:49:27 +0900 Subject: [PATCH 23/53] refactor: add API endpoint to list latest plugin versions and query it in a asynchronous way (#17695) --- api/controllers/console/workspace/plugin.py | 18 ++++++++++++++ api/core/plugin/entities/plugin.py | 2 -- api/services/plugin/plugin_service.py | 23 ++++++------------ .../plugins/plugin-page/plugins-panel.tsx | 24 +++++++++++++++---- web/app/components/plugins/types.ts | 9 +++++++ web/service/use-plugins.ts | 14 +++++++++++ 6 files changed, 67 insertions(+), 23 deletions(-) diff --git a/api/controllers/console/workspace/plugin.py b/api/controllers/console/workspace/plugin.py index 3700f007f1..302bc30905 100644 --- a/api/controllers/console/workspace/plugin.py +++ b/api/controllers/console/workspace/plugin.py @@ -49,6 +49,23 @@ class PluginListApi(Resource): return jsonable_encoder({"plugins": plugins}) +class PluginListLatestVersionsApi(Resource): + @setup_required + @login_required + @account_initialization_required + def post(self): + req = reqparse.RequestParser() + req.add_argument("plugin_ids", type=list, required=True, location="json") + args = req.parse_args() + + try: + versions = PluginService.list_latest_versions(args["plugin_ids"]) + except PluginDaemonClientSideError as e: + raise ValueError(e) + + return jsonable_encoder({"versions": versions}) + + class PluginListInstallationsFromIdsApi(Resource): @setup_required @login_required @@ -453,6 +470,7 @@ class PluginFetchPermissionApi(Resource): api.add_resource(PluginDebuggingKeyApi, "/workspaces/current/plugin/debugging-key") api.add_resource(PluginListApi, "/workspaces/current/plugin/list") +api.add_resource(PluginListLatestVersionsApi, "/workspaces/current/plugin/list/latest-versions") api.add_resource(PluginListInstallationsFromIdsApi, "/workspaces/current/plugin/list/installations/ids") api.add_resource(PluginIconApi, "/workspaces/current/plugin/icon") api.add_resource(PluginUploadFromPkgApi, "/workspaces/current/plugin/upload/pkg") diff --git a/api/core/plugin/entities/plugin.py b/api/core/plugin/entities/plugin.py index 61f8a65918..421c16093d 100644 --- a/api/core/plugin/entities/plugin.py +++ b/api/core/plugin/entities/plugin.py @@ -120,8 +120,6 @@ class PluginEntity(PluginInstallation): name: str installation_id: str version: str - latest_version: Optional[str] = None - latest_unique_identifier: Optional[str] = None @model_validator(mode="after") def set_plugin_id(self): diff --git a/api/services/plugin/plugin_service.py b/api/services/plugin/plugin_service.py index 749bb1a5b4..25d192410f 100644 --- a/api/services/plugin/plugin_service.py +++ b/api/services/plugin/plugin_service.py @@ -94,6 +94,13 @@ class PluginService: manager = PluginDebuggingManager() return manager.get_debugging_key(tenant_id) + @staticmethod + def list_latest_versions(plugin_ids: Sequence[str]) -> Mapping[str, Optional[LatestPluginCache]]: + """ + List the latest versions of the plugins + """ + return PluginService.fetch_latest_plugin_version(plugin_ids) + @staticmethod def list(tenant_id: str) -> list[PluginEntity]: """ @@ -101,22 +108,6 @@ class PluginService: """ manager = PluginInstallationManager() plugins = manager.list_plugins(tenant_id) - plugin_ids = [plugin.plugin_id for plugin in plugins if plugin.source == PluginInstallationSource.Marketplace] - try: - manifests = PluginService.fetch_latest_plugin_version(plugin_ids) - except Exception: - manifests = {} - logger.exception("failed to fetch plugin manifests") - - for plugin in plugins: - if plugin.source == PluginInstallationSource.Marketplace: - if plugin.plugin_id in manifests: - latest_plugin_cache = manifests[plugin.plugin_id] - if latest_plugin_cache: - # set latest_version - plugin.latest_version = latest_plugin_cache.version - plugin.latest_unique_identifier = latest_plugin_cache.unique_identifier - return plugins @staticmethod diff --git a/web/app/components/plugins/plugin-page/plugins-panel.tsx b/web/app/components/plugins/plugin-page/plugins-panel.tsx index 063cec8721..125e6f0a70 100644 --- a/web/app/components/plugins/plugin-page/plugins-panel.tsx +++ b/web/app/components/plugins/plugin-page/plugins-panel.tsx @@ -3,17 +3,23 @@ import { useMemo } from 'react' import type { FilterState } from './filter-management' import FilterManagement from './filter-management' import List from './list' -import { useInstalledPluginList, useInvalidateInstalledPluginList } from '@/service/use-plugins' +import { useInstalledLatestVersion, useInstalledPluginList, useInvalidateInstalledPluginList } from '@/service/use-plugins' import PluginDetailPanel from '@/app/components/plugins/plugin-detail-panel' import { usePluginPageContext } from './context' import { useDebounceFn } from 'ahooks' import Empty from './empty' import Loading from '../../base/loading' +import { PluginSource } from '../types' const PluginsPanel = () => { const filters = usePluginPageContext(v => v.filters) as FilterState const setFilters = usePluginPageContext(v => v.setFilters) const { data: pluginList, isLoading: isPluginListLoading } = useInstalledPluginList() + const { data: installedLatestVersion } = useInstalledLatestVersion( + pluginList?.plugins + .filter(plugin => plugin.source === PluginSource.marketplace) + .map(plugin => plugin.plugin_id) ?? [], + ) const invalidateInstalledPluginList = useInvalidateInstalledPluginList() const currentPluginID = usePluginPageContext(v => v.currentPluginID) const setCurrentPluginID = usePluginPageContext(v => v.setCurrentPluginID) @@ -22,9 +28,17 @@ const PluginsPanel = () => { setFilters(filters) }, { wait: 500 }) + const pluginListWithLatestVersion = useMemo(() => { + return pluginList?.plugins.map(plugin => ({ + ...plugin, + latest_version: installedLatestVersion?.versions[plugin.plugin_id]?.version ?? '', + latest_unique_identifier: installedLatestVersion?.versions[plugin.plugin_id]?.unique_identifier ?? '', + })) || [] + }, [pluginList, installedLatestVersion]) + const filteredList = useMemo(() => { const { categories, searchQuery, tags } = filters - const filteredList = pluginList?.plugins.filter((plugin) => { + const filteredList = pluginListWithLatestVersion.filter((plugin) => { return ( (categories.length === 0 || categories.includes(plugin.declaration.category)) && (tags.length === 0 || tags.some(tag => plugin.declaration.tags.includes(tag))) @@ -32,12 +46,12 @@ const PluginsPanel = () => { ) }) return filteredList - }, [pluginList, filters]) + }, [pluginListWithLatestVersion, filters]) const currentPluginDetail = useMemo(() => { - const detail = pluginList?.plugins.find(plugin => plugin.plugin_id === currentPluginID) + const detail = pluginListWithLatestVersion.find(plugin => plugin.plugin_id === currentPluginID) return detail - }, [currentPluginID, pluginList?.plugins]) + }, [currentPluginID, pluginListWithLatestVersion]) const handleHide = () => setCurrentPluginID(undefined) diff --git a/web/app/components/plugins/types.ts b/web/app/components/plugins/types.ts index 1ed379511b..64f15a08a9 100644 --- a/web/app/components/plugins/types.ts +++ b/web/app/components/plugins/types.ts @@ -318,6 +318,15 @@ export type InstalledPluginListResponse = { plugins: PluginDetail[] } +export type InstalledLatestVersionResponse = { + versions: { + [plugin_id: string]: { + unique_identifier: string + version: string + } | null + } +} + export type UninstallPluginResponse = { success: boolean } diff --git a/web/service/use-plugins.ts b/web/service/use-plugins.ts index 9b5bab587e..c5f60b4a1e 100644 --- a/web/service/use-plugins.ts +++ b/web/service/use-plugins.ts @@ -9,6 +9,7 @@ import type { Dependency, GitHubItemAndMarketPlaceDependency, InstallPackageResponse, + InstalledLatestVersionResponse, InstalledPluginListResponse, PackageDependency, Permissions, @@ -72,6 +73,19 @@ export const useInstalledPluginList = (disable?: boolean) => { }) } +export const useInstalledLatestVersion = (pluginIds: string[]) => { + return useQuery({ + queryKey: [NAME_SPACE, 'installedLatestVersion', pluginIds], + queryFn: () => post('/workspaces/current/plugin/list/latest-versions', { + body: { + plugin_ids: pluginIds, + }, + }), + enabled: !!pluginIds.length, + initialData: pluginIds.length ? undefined : { versions: {} }, + }) +} + export const useInvalidateInstalledPluginList = () => { const queryClient = useQueryClient() const invalidateAllBuiltInTools = useInvalidateAllBuiltInTools() From 6cf1ada36ee68a884fbc575db4e81d125253b719 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 9 Apr 2025 18:31:31 +0800 Subject: [PATCH 24/53] chore: hide eslint complexity warning (#17698) --- web/eslint.config.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/web/eslint.config.mjs b/web/eslint.config.mjs index 750cee5545..554c34e169 100644 --- a/web/eslint.config.mjs +++ b/web/eslint.config.mjs @@ -85,7 +85,6 @@ export default combine( { // orignal config rules: { - 'complexity': ['warn', { max: 10 }], // orignal ts/no-var-requires 'ts/no-require-imports': 'off', 'no-console': 'off', From abfcd9d3b68852ff4c185f76db95853363c5e192 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Wed, 9 Apr 2025 19:09:08 +0800 Subject: [PATCH 25/53] fix segment query index not effect (#17704) --- api/core/rag/datasource/retrieval_service.py | 28 +++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index c4a1e9f059..0d400000db 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -5,7 +5,9 @@ from concurrent.futures import ThreadPoolExecutor from typing import Optional from flask import Flask, current_app +from sqlalchemy import and_, or_ from sqlalchemy.orm import load_only +from sqlalchemy.sql.expression import false from configs import dify_config from core.rag.data_post_processor.data_post_processor import DataPostProcessor @@ -315,17 +317,29 @@ class RetrievalService: child_chunks = db.session.query(ChildChunk).filter(ChildChunk.index_node_id.in_(child_index_node_ids)).all() child_chunk_map = {chunk.index_node_id: chunk for chunk in child_chunks} - # Batch query DocumentSegment with unified conditions + segment_ids_from_child = [chunk.segment_id for chunk in child_chunks] + segment_conditions = [] + + if index_node_ids: + segment_conditions.append(DocumentSegment.index_node_id.in_(index_node_ids)) + + if segment_ids_from_child: + segment_conditions.append(DocumentSegment.id.in_(segment_ids_from_child)) + + if segment_conditions: + filter_expr = or_(*segment_conditions) + else: + filter_expr = false() + segment_map = { segment.id: segment for segment in db.session.query(DocumentSegment) .filter( - ( - DocumentSegment.index_node_id.in_(index_node_ids) - | DocumentSegment.id.in_([chunk.segment_id for chunk in child_chunks]) - ), - DocumentSegment.enabled == True, - DocumentSegment.status == "completed", + and_( + filter_expr, + DocumentSegment.enabled == True, + DocumentSegment.status == "completed", + ) ) .options( load_only( From f148f1efa28ce286aca8621c1e11be29cb2168b4 Mon Sep 17 00:00:00 2001 From: wlleiiwang <1025164922@qq.com> Date: Wed, 9 Apr 2025 19:14:32 +0800 Subject: [PATCH 26/53] fix: Check collection exists before drop it. (#17692) Co-authored-by: wlleiiwang --- api/core/rag/datasource/vdb/tencent/tencent_vector.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/core/rag/datasource/vdb/tencent/tencent_vector.py b/api/core/rag/datasource/vdb/tencent/tencent_vector.py index 540d71bb88..5ca2b7e503 100644 --- a/api/core/rag/datasource/vdb/tencent/tencent_vector.py +++ b/api/core/rag/datasource/vdb/tencent/tencent_vector.py @@ -279,7 +279,10 @@ class TencentVector(BaseVector): return docs def delete(self) -> None: - self._client.drop_collection(database_name=self._client_config.database, collection_name=self.collection_name) + if self._has_collection(): + self._client.drop_collection( + database_name=self._client_config.database, collection_name=self.collection_name + ) class TencentVectorFactory(AbstractVectorFactory): From 1d5c07dedb249ddb32320c68cf8d268356df1bb2 Mon Sep 17 00:00:00 2001 From: quicksand Date: Wed, 9 Apr 2025 19:16:01 +0800 Subject: [PATCH 27/53] =?UTF-8?q?fix=20:=20PLUGIN=5FS3=5FUSE=5FAWS=5FMANAG?= =?UTF-8?q?ED=5FIAM=20AND=20PLUGIN=5FS3=5FUSE=5FPATH=5FSTYLE=20=E2=80=A6?= =?UTF-8?q?=20(#17702)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/.env.example | 4 ++-- docker/docker-compose-template.yaml | 4 ++-- docker/docker-compose.yaml | 8 ++++---- docker/middleware.env.example | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 29d33360ea..0dc25fb5fc 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1017,9 +1017,9 @@ PLUGIN_MEDIA_CACHE_PATH=assets # Plugin oss bucket PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials -PLUGIN_S3_USE_AWS_MANAGED_IAM= +PLUGIN_S3_USE_AWS_MANAGED_IAM=true PLUGIN_S3_ENDPOINT= -PLUGIN_S3_USE_PATH_STYLE= +PLUGIN_S3_USE_PATH_STYLE=true PLUGIN_AWS_ACCESS_KEY= PLUGIN_AWS_SECRET_KEY= PLUGIN_AWS_REGION= diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index d7bcff7ed1..236807812a 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -159,9 +159,9 @@ services: PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-true} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} - S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-true} AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} AWS_REGION: ${PLUGIN_AWS_REGION:-} diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index f120e19bed..e6fafd2dda 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -443,9 +443,9 @@ x-shared-env: &shared-api-worker-env PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-} + PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-true} PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} - PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-} + PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-true} PLUGIN_AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} PLUGIN_AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} PLUGIN_AWS_REGION: ${PLUGIN_AWS_REGION:-} @@ -615,9 +615,9 @@ services: PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-true} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} - S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-true} AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} AWS_REGION: ${PLUGIN_AWS_REGION:-} diff --git a/docker/middleware.env.example b/docker/middleware.env.example index eb38526d57..3fb9a51858 100644 --- a/docker/middleware.env.example +++ b/docker/middleware.env.example @@ -132,9 +132,9 @@ PLUGIN_MEDIA_CACHE_PATH=assets # Plugin oss bucket PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials -PLUGIN_S3_USE_AWS_MANAGED_IAM= +PLUGIN_S3_USE_AWS_MANAGED_IAM=true PLUGIN_S3_ENDPOINT= -PLUGIN_S3_USE_PATH_STYLE= +PLUGIN_S3_USE_PATH_STYLE=true PLUGIN_AWS_ACCESS_KEY= PLUGIN_AWS_SECRET_KEY= PLUGIN_AWS_REGION= From 8b3be4224d9d99204dc8fdc9ec5d6978d0ab10d6 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Wed, 9 Apr 2025 20:25:36 +0800 Subject: [PATCH 28/53] revert batch query (#17707) --- api/core/rag/datasource/retrieval_service.py | 183 ++++++++----------- 1 file changed, 74 insertions(+), 109 deletions(-) diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index 0d400000db..46a5330bdb 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -1,13 +1,9 @@ import concurrent.futures -import logging -import time from concurrent.futures import ThreadPoolExecutor from typing import Optional from flask import Flask, current_app -from sqlalchemy import and_, or_ from sqlalchemy.orm import load_only -from sqlalchemy.sql.expression import false from configs import dify_config from core.rag.data_post_processor.data_post_processor import DataPostProcessor @@ -182,7 +178,6 @@ class RetrievalService: if not dataset: raise ValueError("dataset not found") - start = time.time() vector = Vector(dataset=dataset) documents = vector.search_by_vector( query, @@ -192,7 +187,6 @@ class RetrievalService: filter={"group_id": [dataset.id]}, document_ids_filter=document_ids_filter, ) - logging.debug(f"embedding_search ends at {time.time() - start:.2f} seconds") if documents: if ( @@ -276,8 +270,7 @@ class RetrievalService: return [] try: - start_time = time.time() - # Collect document IDs with existence check + # Collect document IDs document_ids = {doc.metadata.get("document_id") for doc in documents if "document_id" in doc.metadata} if not document_ids: return [] @@ -295,138 +288,110 @@ class RetrievalService: include_segment_ids = set() segment_child_map = {} - # Precompute doc_forms to avoid redundant checks - doc_forms = {} - for doc in documents: - document_id = doc.metadata.get("document_id") - dataset_doc = dataset_documents.get(document_id) - if dataset_doc: - doc_forms[document_id] = dataset_doc.doc_form - - # Batch collect index node IDs with type safety - child_index_node_ids = [] - index_node_ids = [] - for doc in documents: - document_id = doc.metadata.get("document_id") - if doc_forms.get(document_id) == IndexType.PARENT_CHILD_INDEX: - child_index_node_ids.append(doc.metadata.get("doc_id")) - else: - index_node_ids.append(doc.metadata.get("doc_id")) - - # Batch query ChildChunk - child_chunks = db.session.query(ChildChunk).filter(ChildChunk.index_node_id.in_(child_index_node_ids)).all() - child_chunk_map = {chunk.index_node_id: chunk for chunk in child_chunks} - - segment_ids_from_child = [chunk.segment_id for chunk in child_chunks] - segment_conditions = [] - - if index_node_ids: - segment_conditions.append(DocumentSegment.index_node_id.in_(index_node_ids)) - - if segment_ids_from_child: - segment_conditions.append(DocumentSegment.id.in_(segment_ids_from_child)) - - if segment_conditions: - filter_expr = or_(*segment_conditions) - else: - filter_expr = false() - - segment_map = { - segment.id: segment - for segment in db.session.query(DocumentSegment) - .filter( - and_( - filter_expr, - DocumentSegment.enabled == True, - DocumentSegment.status == "completed", - ) - ) - .options( - load_only( - DocumentSegment.id, - DocumentSegment.content, - DocumentSegment.answer, - ) - ) - .all() - } - + # Process documents for document in documents: document_id = document.metadata.get("document_id") - dataset_document = dataset_documents.get(document_id) + if document_id not in dataset_documents: + continue + + dataset_document = dataset_documents[document_id] if not dataset_document: continue - doc_form = doc_forms.get(document_id) - if doc_form == IndexType.PARENT_CHILD_INDEX: - # Handle parent-child documents using preloaded data + if dataset_document.doc_form == IndexType.PARENT_CHILD_INDEX: + # Handle parent-child documents child_index_node_id = document.metadata.get("doc_id") - if not child_index_node_id: - continue - child_chunk = child_chunk_map.get(child_index_node_id) + child_chunk = ( + db.session.query(ChildChunk).filter(ChildChunk.index_node_id == child_index_node_id).first() + ) + if not child_chunk: continue - segment = segment_map.get(child_chunk.segment_id) + segment = ( + db.session.query(DocumentSegment) + .filter( + DocumentSegment.dataset_id == dataset_document.dataset_id, + DocumentSegment.enabled == True, + DocumentSegment.status == "completed", + DocumentSegment.id == child_chunk.segment_id, + ) + .options( + load_only( + DocumentSegment.id, + DocumentSegment.content, + DocumentSegment.answer, + ) + ) + .first() + ) + if not segment: continue if segment.id not in include_segment_ids: include_segment_ids.add(segment.id) - map_detail = {"max_score": document.metadata.get("score", 0.0), "child_chunks": []} + child_chunk_detail = { + "id": child_chunk.id, + "content": child_chunk.content, + "position": child_chunk.position, + "score": document.metadata.get("score", 0.0), + } + map_detail = { + "max_score": document.metadata.get("score", 0.0), + "child_chunks": [child_chunk_detail], + } segment_child_map[segment.id] = map_detail - records.append({"segment": segment}) - - # Append child chunk details - child_chunk_detail = { - "id": child_chunk.id, - "content": child_chunk.content, - "position": child_chunk.position, - "score": document.metadata.get("score", 0.0), - } - segment_child_map[segment.id]["child_chunks"].append(child_chunk_detail) - segment_child_map[segment.id]["max_score"] = max( - segment_child_map[segment.id]["max_score"], document.metadata.get("score", 0.0) - ) - + record = { + "segment": segment, + } + records.append(record) + else: + child_chunk_detail = { + "id": child_chunk.id, + "content": child_chunk.content, + "position": child_chunk.position, + "score": document.metadata.get("score", 0.0), + } + segment_child_map[segment.id]["child_chunks"].append(child_chunk_detail) + segment_child_map[segment.id]["max_score"] = max( + segment_child_map[segment.id]["max_score"], document.metadata.get("score", 0.0) + ) else: # Handle normal documents index_node_id = document.metadata.get("doc_id") if not index_node_id: continue - segment = next( - ( - s - for s in segment_map.values() - if s.index_node_id == index_node_id and s.dataset_id == dataset_document.dataset_id - ), - None, + segment = ( + db.session.query(DocumentSegment) + .filter( + DocumentSegment.dataset_id == dataset_document.dataset_id, + DocumentSegment.enabled == True, + DocumentSegment.status == "completed", + DocumentSegment.index_node_id == index_node_id, + ) + .first() ) if not segment: continue - if segment.id not in include_segment_ids: - include_segment_ids.add(segment.id) - records.append( - { - "segment": segment, - "score": document.metadata.get("score", 0.0), - } - ) + include_segment_ids.add(segment.id) + record = { + "segment": segment, + "score": document.metadata.get("score"), # type: ignore + } + records.append(record) - # Merge child chunks information + # Add child chunks information to records for record in records: - segment_id = record["segment"].id - if segment_id in segment_child_map: - record["child_chunks"] = segment_child_map[segment_id]["child_chunks"] - record["score"] = segment_child_map[segment_id]["max_score"] + if record["segment"].id in segment_child_map: + record["child_chunks"] = segment_child_map[record["segment"].id].get("child_chunks") # type: ignore + record["score"] = segment_child_map[record["segment"].id]["max_score"] - logging.debug(f"Formatting retrieval documents took {time.time() - start_time:.2f} seconds") return [RetrievalSegments(**record) for record in records] except Exception as e: - # Only rollback if there were write operations db.session.rollback() raise e From d3157b46eea7de9a4beef00bcd7dbbbb9dd8f544 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Wed, 9 Apr 2025 21:52:58 +0900 Subject: [PATCH 29/53] feat(large_language_model): Adds plugin-based token counting configuration option (#17706) Signed-off-by: -LAN- Co-authored-by: Yeuoly --- api/.env.example | 1 + api/configs/feature/__init__.py | 7 ++++- api/core/app/apps/agent_chat/app_runner.py | 14 ---------- api/core/app/apps/chat/app_runner.py | 14 ---------- api/core/app/apps/completion/app_runner.py | 14 ---------- .../en_US/customizable_model_scale_out.md | 2 +- .../zh_Hans/customizable_model_scale_out.md | 2 +- .../__base/large_language_model.py | 26 ++++++++++--------- docker/.env.example | 11 +++++--- docker/docker-compose.yaml | 1 + 10 files changed, 32 insertions(+), 60 deletions(-) diff --git a/api/.env.example b/api/.env.example index ba76274c34..3bbea44f2f 100644 --- a/api/.env.example +++ b/api/.env.example @@ -326,6 +326,7 @@ UPLOAD_AUDIO_FILE_SIZE_LIMIT=50 MULTIMODAL_SEND_FORMAT=base64 PROMPT_GENERATION_MAX_TOKENS=512 CODE_GENERATION_MAX_TOKENS=1024 +PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false # Mail configuration, support: resend, smtp MAIL_TYPE= diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index fa8e8c2bf6..d35a74e3ee 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -442,7 +442,7 @@ class LoggingConfig(BaseSettings): class ModelLoadBalanceConfig(BaseSettings): """ - Configuration for model load balancing + Configuration for model load balancing and token counting """ MODEL_LB_ENABLED: bool = Field( @@ -450,6 +450,11 @@ class ModelLoadBalanceConfig(BaseSettings): default=False, ) + PLUGIN_BASED_TOKEN_COUNTING_ENABLED: bool = Field( + description="Enable or disable plugin based token counting. If disabled, token counting will return 0.", + default=False, + ) + class BillingConfig(BaseSettings): """ diff --git a/api/core/app/apps/agent_chat/app_runner.py b/api/core/app/apps/agent_chat/app_runner.py index 72a1717112..71328f6d1b 100644 --- a/api/core/app/apps/agent_chat/app_runner.py +++ b/api/core/app/apps/agent_chat/app_runner.py @@ -53,20 +53,6 @@ class AgentChatAppRunner(AppRunner): query = application_generate_entity.query files = application_generate_entity.files - # Pre-calculate the number of tokens of the prompt messages, - # and return the rest number of tokens by model context token size limit and max token size limit. - # If the rest number of tokens is not enough, raise exception. - # Include: prompt template, inputs, query(optional), files(optional) - # Not Include: memory, external data, dataset context - self.get_pre_calculate_rest_tokens( - app_record=app_record, - model_config=application_generate_entity.model_conf, - prompt_template_entity=app_config.prompt_template, - inputs=dict(inputs), - files=list(files), - query=query, - ) - memory = None if application_generate_entity.conversation_id: # get memory of conversation (read-only) diff --git a/api/core/app/apps/chat/app_runner.py b/api/core/app/apps/chat/app_runner.py index 8641f188f7..39597fc036 100644 --- a/api/core/app/apps/chat/app_runner.py +++ b/api/core/app/apps/chat/app_runner.py @@ -61,20 +61,6 @@ class ChatAppRunner(AppRunner): ) image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW - # Pre-calculate the number of tokens of the prompt messages, - # and return the rest number of tokens by model context token size limit and max token size limit. - # If the rest number of tokens is not enough, raise exception. - # Include: prompt template, inputs, query(optional), files(optional) - # Not Include: memory, external data, dataset context - self.get_pre_calculate_rest_tokens( - app_record=app_record, - model_config=application_generate_entity.model_conf, - prompt_template_entity=app_config.prompt_template, - inputs=inputs, - files=files, - query=query, - ) - memory = None if application_generate_entity.conversation_id: # get memory of conversation (read-only) diff --git a/api/core/app/apps/completion/app_runner.py b/api/core/app/apps/completion/app_runner.py index 4f16247318..80fdd0b80e 100644 --- a/api/core/app/apps/completion/app_runner.py +++ b/api/core/app/apps/completion/app_runner.py @@ -54,20 +54,6 @@ class CompletionAppRunner(AppRunner): ) image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW - # Pre-calculate the number of tokens of the prompt messages, - # and return the rest number of tokens by model context token size limit and max token size limit. - # If the rest number of tokens is not enough, raise exception. - # Include: prompt template, inputs, query(optional), files(optional) - # Not Include: memory, external data, dataset context - self.get_pre_calculate_rest_tokens( - app_record=app_record, - model_config=application_generate_entity.model_conf, - prompt_template_entity=app_config.prompt_template, - inputs=inputs, - files=files, - query=query, - ) - # organize all inputs and template to prompt messages # Include: prompt template, inputs, query(optional), files(optional) prompt_messages, stop = self.organize_prompt_messages( diff --git a/api/core/model_runtime/docs/en_US/customizable_model_scale_out.md b/api/core/model_runtime/docs/en_US/customizable_model_scale_out.md index f050919d81..b5a714a172 100644 --- a/api/core/model_runtime/docs/en_US/customizable_model_scale_out.md +++ b/api/core/model_runtime/docs/en_US/customizable_model_scale_out.md @@ -192,7 +192,7 @@ def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[Pr ``` -Sometimes, you might not want to return 0 directly. In such cases, you can use `self._get_num_tokens_by_gpt2(text: str)` to get pre-computed tokens. This method is provided by the `AIModel` base class, and it uses GPT2's Tokenizer for calculation. However, it should be noted that this is only a substitute and may not be fully accurate. +Sometimes, you might not want to return 0 directly. In such cases, you can use `self._get_num_tokens_by_gpt2(text: str)` to get pre-computed tokens and ensure environment variable `PLUGIN_BASED_TOKEN_COUNTING_ENABLED` is set to `true`, This method is provided by the `AIModel` base class, and it uses GPT2's Tokenizer for calculation. However, it should be noted that this is only a substitute and may not be fully accurate. - Model Credentials Validation diff --git a/api/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md b/api/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md index 240f65802b..c36575b9af 100644 --- a/api/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md +++ b/api/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md @@ -179,7 +179,7 @@ provider_credential_schema: """ ``` - 有时候，也许你不需要直接返回0，所以你可以使用`self._get_num_tokens_by_gpt2(text: str)`来获取预计算的tokens，这个方法位于`AIModel`基类中，它会使用GPT2的Tokenizer进行计算，但是只能作为替代方法，并不完全准确。 + 有时候，也许你不需要直接返回0，所以你可以使用`self._get_num_tokens_by_gpt2(text: str)`来获取预计算的tokens，并确保环境变量`PLUGIN_BASED_TOKEN_COUNTING_ENABLED`设置为`true`，这个方法位于`AIModel`基类中，它会使用GPT2的Tokenizer进行计算，但是只能作为替代方法，并不完全准确。 - 模型凭据校验 diff --git a/api/core/model_runtime/model_providers/__base/large_language_model.py b/api/core/model_runtime/model_providers/__base/large_language_model.py index ed67fef768..b81ccafc1e 100644 --- a/api/core/model_runtime/model_providers/__base/large_language_model.py +++ b/api/core/model_runtime/model_providers/__base/large_language_model.py @@ -295,18 +295,20 @@ class LargeLanguageModel(AIModel): :param tools: tools for tool calling :return: """ - plugin_model_manager = PluginModelManager() - return plugin_model_manager.get_llm_num_tokens( - tenant_id=self.tenant_id, - user_id="unknown", - plugin_id=self.plugin_id, - provider=self.provider_name, - model_type=self.model_type.value, - model=model, - credentials=credentials, - prompt_messages=prompt_messages, - tools=tools, - ) + if dify_config.PLUGIN_BASED_TOKEN_COUNTING_ENABLED: + plugin_model_manager = PluginModelManager() + return plugin_model_manager.get_llm_num_tokens( + tenant_id=self.tenant_id, + user_id="unknown", + plugin_id=self.plugin_id, + provider=self.provider_name, + model_type=self.model_type.value, + model=model, + credentials=credentials, + prompt_messages=prompt_messages, + tools=tools, + ) + return 0 def _calc_response_usage( self, model: str, credentials: dict, prompt_tokens: int, completion_tokens: int diff --git a/docker/.env.example b/docker/.env.example index 0dc25fb5fc..6e2c4d3d9b 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -75,7 +75,7 @@ SECRET_KEY=sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U # Password for admin user initialization. # If left unset, admin user will not be prompted for a password -# when creating the initial admin account. +# when creating the initial admin account. # The length of the password cannot exceed 30 characters. INIT_PASSWORD= @@ -605,17 +605,22 @@ SCARF_NO_ANALYTICS=true # ------------------------------ # The maximum number of tokens allowed for prompt generation. -# This setting controls the upper limit of tokens that can be used by the LLM +# This setting controls the upper limit of tokens that can be used by the LLM # when generating a prompt in the prompt generation tool. # Default: 512 tokens. PROMPT_GENERATION_MAX_TOKENS=512 # The maximum number of tokens allowed for code generation. -# This setting controls the upper limit of tokens that can be used by the LLM +# This setting controls the upper limit of tokens that can be used by the LLM # when generating code in the code generation tool. # Default: 1024 tokens. CODE_GENERATION_MAX_TOKENS=1024 +# Enable or disable plugin based token counting. If disabled, token counting will return 0. +# This can improve performance by skipping token counting operations. +# Default: false (disabled). +PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false + # ------------------------------ # Multi-modal Configuration # ------------------------------ diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index e6fafd2dda..b317fffc00 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -276,6 +276,7 @@ x-shared-env: &shared-api-worker-env SCARF_NO_ANALYTICS: ${SCARF_NO_ANALYTICS:-true} PROMPT_GENERATION_MAX_TOKENS: ${PROMPT_GENERATION_MAX_TOKENS:-512} CODE_GENERATION_MAX_TOKENS: ${CODE_GENERATION_MAX_TOKENS:-1024} + PLUGIN_BASED_TOKEN_COUNTING_ENABLED: ${PLUGIN_BASED_TOKEN_COUNTING_ENABLED:-false} MULTIMODAL_SEND_FORMAT: ${MULTIMODAL_SEND_FORMAT:-base64} UPLOAD_IMAGE_FILE_SIZE_LIMIT: ${UPLOAD_IMAGE_FILE_SIZE_LIMIT:-10} UPLOAD_VIDEO_FILE_SIZE_LIMIT: ${UPLOAD_VIDEO_FILE_SIZE_LIMIT:-100} From 0b1f938389c3c35f7d0f28a5352cda982ab1a41d Mon Sep 17 00:00:00 2001 From: quicksand Date: Thu, 10 Apr 2025 09:57:50 +0800 Subject: [PATCH 30/53] fix: docker compose plugin s3 config default value (#17722) --- docker/.env.example | 4 ++-- docker/docker-compose-template.yaml | 4 ++-- docker/docker-compose.middleware.yaml | 4 ++-- docker/docker-compose.yaml | 8 ++++---- docker/middleware.env.example | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 6e2c4d3d9b..84da4f3df5 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1022,9 +1022,9 @@ PLUGIN_MEDIA_CACHE_PATH=assets # Plugin oss bucket PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials -PLUGIN_S3_USE_AWS_MANAGED_IAM=true +PLUGIN_S3_USE_AWS_MANAGED_IAM=false PLUGIN_S3_ENDPOINT= -PLUGIN_S3_USE_PATH_STYLE=true +PLUGIN_S3_USE_PATH_STYLE=false PLUGIN_AWS_ACCESS_KEY= PLUGIN_AWS_SECRET_KEY= PLUGIN_AWS_REGION= diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 236807812a..ef58bf99f3 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -159,9 +159,9 @@ services: PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-true} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} - S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-true} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} AWS_REGION: ${PLUGIN_AWS_REGION:-} diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index de238d9669..bfd2354314 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -103,9 +103,9 @@ services: PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} - S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} AWS_REGION: ${PLUGIN_AWS_REGION:-} diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index b317fffc00..0068dc0c58 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -444,9 +444,9 @@ x-shared-env: &shared-api-worker-env PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-true} + PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} - PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-true} + PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} PLUGIN_AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} PLUGIN_AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} PLUGIN_AWS_REGION: ${PLUGIN_AWS_REGION:-} @@ -616,9 +616,9 @@ services: PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-true} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} - S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-true} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} AWS_REGION: ${PLUGIN_AWS_REGION:-} diff --git a/docker/middleware.env.example b/docker/middleware.env.example index 3fb9a51858..1a4484a9b5 100644 --- a/docker/middleware.env.example +++ b/docker/middleware.env.example @@ -132,9 +132,9 @@ PLUGIN_MEDIA_CACHE_PATH=assets # Plugin oss bucket PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials -PLUGIN_S3_USE_AWS_MANAGED_IAM=true +PLUGIN_S3_USE_AWS_MANAGED_IAM=false PLUGIN_S3_ENDPOINT= -PLUGIN_S3_USE_PATH_STYLE=true +PLUGIN_S3_USE_PATH_STYLE=false PLUGIN_AWS_ACCESS_KEY= PLUGIN_AWS_SECRET_KEY= PLUGIN_AWS_REGION= From 9d5a0fdd8a2373d0fadb70c611975cab94df6af6 Mon Sep 17 00:00:00 2001 From: GuanMu Date: Thu, 10 Apr 2025 10:01:44 +0800 Subject: [PATCH 31/53] Fix create blank app (#17724) --- web/app/(commonLayout)/apps/Apps.tsx | 5 +++-- web/app/components/app/create-app-modal/index.tsx | 13 +++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/web/app/(commonLayout)/apps/Apps.tsx b/web/app/(commonLayout)/apps/Apps.tsx index d98851c4e9..e72860861c 100644 --- a/web/app/(commonLayout)/apps/Apps.tsx +++ b/web/app/(commonLayout)/apps/Apps.tsx @@ -66,6 +66,7 @@ const Apps = () => { const [isCreatedByMe, setIsCreatedByMe] = useState(queryIsCreatedByMe) const [tagFilterValue, setTagFilterValue] = useState(tagIDs) const [searchKeywords, setSearchKeywords] = useState(keywords) + const newAppCardRef = useRef(null) const setKeywords = useCallback((keywords: string) => { setQuery(prev => ({ ...prev, keywords })) }, [setQuery]) @@ -166,14 +167,14 @@ const Apps = () => { {(data && data[0].total > 0) ?
{isCurrentWorkspaceEditor - && } + && } {data.map(({ data: apps }) => apps.map(app => ( )))}
:
{isCurrentWorkspaceEditor - && } + && }
} diff --git a/web/app/components/app/create-app-modal/index.tsx b/web/app/components/app/create-app-modal/index.tsx index 6abc871868..51a57cba93 100644 --- a/web/app/components/app/create-app-modal/index.tsx +++ b/web/app/components/app/create-app-modal/index.tsx @@ -1,9 +1,9 @@ 'use client' -import { useCallback, useRef, useState } from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' -import { useRouter } from 'next/navigation' +import { useRouter, useSearchParams } from 'next/navigation' import { useContext, useContextSelector } from 'use-context-selector' import { RiArrowRightLine, RiCommandLine, RiCornerDownLeftLine, RiExchange2Fill } from '@remixicon/react' import Link from 'next/link' @@ -18,6 +18,7 @@ import AppsContext, { useAppContext } from '@/context/app-context' import { useProviderContext } from '@/context/provider-context' import { ToastContext } from '@/app/components/base/toast' import type { AppMode } from '@/types/app' +import { AppModes } from '@/types/app' import { createApp } from '@/service/apps' import Input from '@/app/components/base/input' import Textarea from '@/app/components/base/textarea' @@ -53,6 +54,14 @@ function CreateApp({ onClose, onSuccess, onCreateFromTemplate }: CreateAppProps) const isCreatingRef = useRef(false) + const searchParams = useSearchParams() + + useEffect(() => { + const category = searchParams.get('category') + if (category && AppModes.includes(category as AppMode)) + setAppMode(category as AppMode) + }, [searchParams]) + const onCreate = useCallback(async () => { if (!appMode) { notify({ type: 'error', message: t('app.newApp.appTypeRequired') }) From 30f7118c7aafea450fc1d2a38b7c6e8cd7762921 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Thu, 10 Apr 2025 10:03:19 +0800 Subject: [PATCH 32/53] Chore/slice workflow utils (#17730) --- .../model-provider-page/model-icon/index.tsx | 2 +- .../provider-icon/index.tsx | 2 +- web/app/components/plugins/card/index.tsx | 2 +- .../workflow/nodes/agent/default.ts | 2 +- web/app/components/workflow/utils.ts | 1060 ----------------- web/app/components/workflow/utils/common.ts | 35 + web/app/components/workflow/utils/edge.ts | 23 + web/app/components/workflow/utils/index.ts | 8 + web/app/components/workflow/utils/layout.ts | 178 +++ web/app/components/workflow/utils/node.ts | 145 +++ web/app/components/workflow/utils/tool.ts | 43 + web/app/components/workflow/utils/variable.ts | 21 + .../workflow/utils/workflow-init.spec.ts | 69 ++ .../workflow/utils/workflow-init.ts | 338 ++++++ web/app/components/workflow/utils/workflow.ts | 329 +++++ web/hooks/use-i18n.ts | 8 +- web/i18n/index.ts | 7 + web/package.json | 1 + web/pnpm-lock.yaml | 23 +- 19 files changed, 1215 insertions(+), 1081 deletions(-) delete mode 100644 web/app/components/workflow/utils.ts create mode 100644 web/app/components/workflow/utils/common.ts create mode 100644 web/app/components/workflow/utils/edge.ts create mode 100644 web/app/components/workflow/utils/index.ts create mode 100644 web/app/components/workflow/utils/layout.ts create mode 100644 web/app/components/workflow/utils/node.ts create mode 100644 web/app/components/workflow/utils/tool.ts create mode 100644 web/app/components/workflow/utils/variable.ts create mode 100644 web/app/components/workflow/utils/workflow-init.spec.ts create mode 100644 web/app/components/workflow/utils/workflow-init.ts create mode 100644 web/app/components/workflow/utils/workflow.ts diff --git a/web/app/components/header/account-setting/model-provider-page/model-icon/index.tsx b/web/app/components/header/account-setting/model-provider-page/model-icon/index.tsx index c2fbe7930e..9d1846cdf0 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-icon/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-icon/index.tsx @@ -7,7 +7,7 @@ import { useLanguage } from '../hooks' import { Group } from '@/app/components/base/icons/src/vender/other' import { OpenaiBlue, OpenaiViolet } from '@/app/components/base/icons/src/public/llm' import cn from '@/utils/classnames' -import { renderI18nObject } from '@/hooks/use-i18n' +import { renderI18nObject } from '@/i18n' type ModelIconProps = { provider?: Model | ModelProvider diff --git a/web/app/components/header/account-setting/model-provider-page/provider-icon/index.tsx b/web/app/components/header/account-setting/model-provider-page/provider-icon/index.tsx index 1eb579a7a0..253269d920 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-icon/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-icon/index.tsx @@ -3,7 +3,7 @@ import type { ModelProvider } from '../declarations' import { useLanguage } from '../hooks' import { Openai } from '@/app/components/base/icons/src/vender/other' import { AnthropicDark, AnthropicLight } from '@/app/components/base/icons/src/public/llm' -import { renderI18nObject } from '@/hooks/use-i18n' +import { renderI18nObject } from '@/i18n' import { Theme } from '@/types/app' import cn from '@/utils/classnames' import useTheme from '@/hooks/use-theme' diff --git a/web/app/components/plugins/card/index.tsx b/web/app/components/plugins/card/index.tsx index f4878a433c..1cc18ac24f 100644 --- a/web/app/components/plugins/card/index.tsx +++ b/web/app/components/plugins/card/index.tsx @@ -11,7 +11,7 @@ import cn from '@/utils/classnames' import { useGetLanguage } from '@/context/i18n' import { getLanguage } from '@/i18n/language' import { useSingleCategories } from '../hooks' -import { renderI18nObject } from '@/hooks/use-i18n' +import { renderI18nObject } from '@/i18n' import { useMixedTranslation } from '@/app/components/plugins/marketplace/hooks' import Partner from '../base/badges/partner' import Verified from '../base/badges/verified' diff --git a/web/app/components/workflow/nodes/agent/default.ts b/web/app/components/workflow/nodes/agent/default.ts index 6069f90991..a0abfb411e 100644 --- a/web/app/components/workflow/nodes/agent/default.ts +++ b/web/app/components/workflow/nodes/agent/default.ts @@ -3,7 +3,7 @@ import { ALL_CHAT_AVAILABLE_BLOCKS, ALL_COMPLETION_AVAILABLE_BLOCKS } from '@/ap import type { NodeDefault } from '../../types' import type { AgentNodeType } from './types' import { FormTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' -import { renderI18nObject } from '@/hooks/use-i18n' +import { renderI18nObject } from '@/i18n' const nodeDefault: NodeDefault = { defaultValue: { diff --git a/web/app/components/workflow/utils.ts b/web/app/components/workflow/utils.ts deleted file mode 100644 index a99e8aa5d2..0000000000 --- a/web/app/components/workflow/utils.ts +++ /dev/null @@ -1,1060 +0,0 @@ -import { - Position, - getConnectedEdges, - getIncomers, - getOutgoers, -} from 'reactflow' -import dagre from '@dagrejs/dagre' -import { v4 as uuid4 } from 'uuid' -import { - cloneDeep, - groupBy, - isEqual, - uniqBy, -} from 'lodash-es' -import type { - Edge, - InputVar, - Node, - ToolWithProvider, - ValueSelector, -} from './types' -import { - BlockEnum, - ErrorHandleMode, - NodeRunningStatus, -} from './types' -import { - CUSTOM_NODE, - DEFAULT_RETRY_INTERVAL, - DEFAULT_RETRY_MAX, - ITERATION_CHILDREN_Z_INDEX, - ITERATION_NODE_Z_INDEX, - LOOP_CHILDREN_Z_INDEX, - LOOP_NODE_Z_INDEX, - NODE_LAYOUT_HORIZONTAL_PADDING, - NODE_LAYOUT_MIN_DISTANCE, - NODE_LAYOUT_VERTICAL_PADDING, - NODE_WIDTH_X_OFFSET, - START_INITIAL_POSITION, -} from './constants' -import { CUSTOM_ITERATION_START_NODE } from './nodes/iteration-start/constants' -import { CUSTOM_LOOP_START_NODE } from './nodes/loop-start/constants' -import type { QuestionClassifierNodeType } from './nodes/question-classifier/types' -import type { IfElseNodeType } from './nodes/if-else/types' -import { branchNameCorrect } from './nodes/if-else/utils' -import type { ToolNodeType } from './nodes/tool/types' -import type { IterationNodeType } from './nodes/iteration/types' -import type { LoopNodeType } from './nodes/loop/types' -import { CollectionType } from '@/app/components/tools/types' -import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema' -import { canFindTool, correctModelProvider } from '@/utils' -import { CUSTOM_SIMPLE_NODE } from '@/app/components/workflow/simple-node/constants' - -const WHITE = 'WHITE' -const GRAY = 'GRAY' -const BLACK = 'BLACK' - -const isCyclicUtil = (nodeId: string, color: Record, adjList: Record, stack: string[]) => { - color[nodeId] = GRAY - stack.push(nodeId) - - for (let i = 0; i < adjList[nodeId].length; ++i) { - const childId = adjList[nodeId][i] - - if (color[childId] === GRAY) { - stack.push(childId) - return true - } - if (color[childId] === WHITE && isCyclicUtil(childId, color, adjList, stack)) - return true - } - color[nodeId] = BLACK - if (stack.length > 0 && stack[stack.length - 1] === nodeId) - stack.pop() - return false -} - -const getCycleEdges = (nodes: Node[], edges: Edge[]) => { - const adjList: Record = {} - const color: Record = {} - const stack: string[] = [] - - for (const node of nodes) { - color[node.id] = WHITE - adjList[node.id] = [] - } - - for (const edge of edges) - adjList[edge.source]?.push(edge.target) - - for (let i = 0; i < nodes.length; i++) { - if (color[nodes[i].id] === WHITE) - isCyclicUtil(nodes[i].id, color, adjList, stack) - } - - const cycleEdges = [] - if (stack.length > 0) { - const cycleNodes = new Set(stack) - for (const edge of edges) { - if (cycleNodes.has(edge.source) && cycleNodes.has(edge.target)) - cycleEdges.push(edge) - } - } - - return cycleEdges -} - -export function getIterationStartNode(iterationId: string): Node { - return generateNewNode({ - id: `${iterationId}start`, - type: CUSTOM_ITERATION_START_NODE, - data: { - title: '', - desc: '', - type: BlockEnum.IterationStart, - isInIteration: true, - }, - position: { - x: 24, - y: 68, - }, - zIndex: ITERATION_CHILDREN_Z_INDEX, - parentId: iterationId, - selectable: false, - draggable: false, - }).newNode -} - -export function getLoopStartNode(loopId: string): Node { - return generateNewNode({ - id: `${loopId}start`, - type: CUSTOM_LOOP_START_NODE, - data: { - title: '', - desc: '', - type: BlockEnum.LoopStart, - isInLoop: true, - }, - position: { - x: 24, - y: 68, - }, - zIndex: LOOP_CHILDREN_Z_INDEX, - parentId: loopId, - selectable: false, - draggable: false, - }).newNode -} - -export function generateNewNode({ data, position, id, zIndex, type, ...rest }: Omit & { id?: string }): { - newNode: Node - newIterationStartNode?: Node - newLoopStartNode?: Node -} { - const newNode = { - id: id || `${Date.now()}`, - type: type || CUSTOM_NODE, - data, - position, - targetPosition: Position.Left, - sourcePosition: Position.Right, - zIndex: data.type === BlockEnum.Iteration ? ITERATION_NODE_Z_INDEX : (data.type === BlockEnum.Loop ? LOOP_NODE_Z_INDEX : zIndex), - ...rest, - } as Node - - if (data.type === BlockEnum.Iteration) { - const newIterationStartNode = getIterationStartNode(newNode.id); - (newNode.data as IterationNodeType).start_node_id = newIterationStartNode.id; - (newNode.data as IterationNodeType)._children = [{ nodeId: newIterationStartNode.id, nodeType: BlockEnum.IterationStart }] - return { - newNode, - newIterationStartNode, - } - } - - if (data.type === BlockEnum.Loop) { - const newLoopStartNode = getLoopStartNode(newNode.id); - (newNode.data as LoopNodeType).start_node_id = newLoopStartNode.id; - (newNode.data as LoopNodeType)._children = [{ nodeId: newLoopStartNode.id, nodeType: BlockEnum.LoopStart }] - return { - newNode, - newLoopStartNode, - } - } - - return { - newNode, - } -} - -export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { - const hasIterationNode = nodes.some(node => node.data.type === BlockEnum.Iteration) - const hasLoopNode = nodes.some(node => node.data.type === BlockEnum.Loop) - - if (!hasIterationNode) { - return { - nodes, - edges, - } - } - - if (!hasLoopNode) { - return { - nodes, - edges, - } - } - - const nodesMap = nodes.reduce((prev, next) => { - prev[next.id] = next - return prev - }, {} as Record) - - const iterationNodesWithStartNode = [] - const iterationNodesWithoutStartNode = [] - const loopNodesWithStartNode = [] - const loopNodesWithoutStartNode = [] - - for (let i = 0; i < nodes.length; i++) { - const currentNode = nodes[i] as Node - - if (currentNode.data.type === BlockEnum.Iteration) { - if (currentNode.data.start_node_id) { - if (nodesMap[currentNode.data.start_node_id]?.type !== CUSTOM_ITERATION_START_NODE) - iterationNodesWithStartNode.push(currentNode) - } - else { - iterationNodesWithoutStartNode.push(currentNode) - } - } - - if (currentNode.data.type === BlockEnum.Loop) { - if (currentNode.data.start_node_id) { - if (nodesMap[currentNode.data.start_node_id]?.type !== CUSTOM_LOOP_START_NODE) - loopNodesWithStartNode.push(currentNode) - } - else { - loopNodesWithoutStartNode.push(currentNode) - } - } - } - - const newIterationStartNodesMap = {} as Record - const newIterationStartNodes = [...iterationNodesWithStartNode, ...iterationNodesWithoutStartNode].map((iterationNode, index) => { - const newNode = getIterationStartNode(iterationNode.id) - newNode.id = newNode.id + index - newIterationStartNodesMap[iterationNode.id] = newNode - return newNode - }) - - const newLoopStartNodesMap = {} as Record - const newLoopStartNodes = [...loopNodesWithStartNode, ...loopNodesWithoutStartNode].map((loopNode, index) => { - const newNode = getLoopStartNode(loopNode.id) - newNode.id = newNode.id + index - newLoopStartNodesMap[loopNode.id] = newNode - return newNode - }) - - const newEdges = [...iterationNodesWithStartNode, ...loopNodesWithStartNode].map((nodeItem) => { - const isIteration = nodeItem.data.type === BlockEnum.Iteration - const newNode = (isIteration ? newIterationStartNodesMap : newLoopStartNodesMap)[nodeItem.id] - const startNode = nodesMap[nodeItem.data.start_node_id] - const source = newNode.id - const sourceHandle = 'source' - const target = startNode.id - const targetHandle = 'target' - - const parentNode = nodes.find(node => node.id === startNode.parentId) || null - const isInIteration = !!parentNode && parentNode.data.type === BlockEnum.Iteration - const isInLoop = !!parentNode && parentNode.data.type === BlockEnum.Loop - - return { - id: `${source}-${sourceHandle}-${target}-${targetHandle}`, - type: 'custom', - source, - sourceHandle, - target, - targetHandle, - data: { - sourceType: newNode.data.type, - targetType: startNode.data.type, - isInIteration, - iteration_id: isInIteration ? startNode.parentId : undefined, - isInLoop, - loop_id: isInLoop ? startNode.parentId : undefined, - _connectedNodeIsSelected: true, - }, - zIndex: isIteration ? ITERATION_CHILDREN_Z_INDEX : LOOP_CHILDREN_Z_INDEX, - } - }) - nodes.forEach((node) => { - if (node.data.type === BlockEnum.Iteration && newIterationStartNodesMap[node.id]) - (node.data as IterationNodeType).start_node_id = newIterationStartNodesMap[node.id].id - - if (node.data.type === BlockEnum.Loop && newLoopStartNodesMap[node.id]) - (node.data as LoopNodeType).start_node_id = newLoopStartNodesMap[node.id].id - }) - - return { - nodes: [...nodes, ...newIterationStartNodes, ...newLoopStartNodes], - edges: [...edges, ...newEdges], - } -} - -export const initialNodes = (originNodes: Node[], originEdges: Edge[]) => { - const { nodes, edges } = preprocessNodesAndEdges(cloneDeep(originNodes), cloneDeep(originEdges)) - const firstNode = nodes[0] - - if (!firstNode?.position) { - nodes.forEach((node, index) => { - node.position = { - x: START_INITIAL_POSITION.x + index * NODE_WIDTH_X_OFFSET, - y: START_INITIAL_POSITION.y, - } - }) - } - - const iterationOrLoopNodeMap = nodes.reduce((acc, node) => { - if (node.parentId) { - if (acc[node.parentId]) - acc[node.parentId].push({ nodeId: node.id, nodeType: node.data.type }) - else - acc[node.parentId] = [{ nodeId: node.id, nodeType: node.data.type }] - } - return acc - }, {} as Record) - - return nodes.map((node) => { - if (!node.type) - node.type = CUSTOM_NODE - - const connectedEdges = getConnectedEdges([node], edges) - node.data._connectedSourceHandleIds = connectedEdges.filter(edge => edge.source === node.id).map(edge => edge.sourceHandle || 'source') - node.data._connectedTargetHandleIds = connectedEdges.filter(edge => edge.target === node.id).map(edge => edge.targetHandle || 'target') - - if (node.data.type === BlockEnum.IfElse) { - const nodeData = node.data as IfElseNodeType - - if (!nodeData.cases && nodeData.logical_operator && nodeData.conditions) { - (node.data as IfElseNodeType).cases = [ - { - case_id: 'true', - logical_operator: nodeData.logical_operator, - conditions: nodeData.conditions, - }, - ] - } - node.data._targetBranches = branchNameCorrect([ - ...(node.data as IfElseNodeType).cases.map(item => ({ id: item.case_id, name: '' })), - { id: 'false', name: '' }, - ]) - } - - if (node.data.type === BlockEnum.QuestionClassifier) { - node.data._targetBranches = (node.data as QuestionClassifierNodeType).classes.map((topic) => { - return topic - }) - } - - if (node.data.type === BlockEnum.Iteration) { - const iterationNodeData = node.data as IterationNodeType - iterationNodeData._children = iterationOrLoopNodeMap[node.id] || [] - iterationNodeData.is_parallel = iterationNodeData.is_parallel || false - iterationNodeData.parallel_nums = iterationNodeData.parallel_nums || 10 - iterationNodeData.error_handle_mode = iterationNodeData.error_handle_mode || ErrorHandleMode.Terminated - } - - // TODO: loop error handle mode - if (node.data.type === BlockEnum.Loop) { - const loopNodeData = node.data as LoopNodeType - loopNodeData._children = iterationOrLoopNodeMap[node.id] || [] - loopNodeData.error_handle_mode = loopNodeData.error_handle_mode || ErrorHandleMode.Terminated - } - - // legacy provider handle - if (node.data.type === BlockEnum.LLM) - (node as any).data.model.provider = correctModelProvider((node as any).data.model.provider) - - if (node.data.type === BlockEnum.KnowledgeRetrieval && (node as any).data.multiple_retrieval_config?.reranking_model) - (node as any).data.multiple_retrieval_config.reranking_model.provider = correctModelProvider((node as any).data.multiple_retrieval_config?.reranking_model.provider) - - if (node.data.type === BlockEnum.QuestionClassifier) - (node as any).data.model.provider = correctModelProvider((node as any).data.model.provider) - - if (node.data.type === BlockEnum.ParameterExtractor) - (node as any).data.model.provider = correctModelProvider((node as any).data.model.provider) - if (node.data.type === BlockEnum.HttpRequest && !node.data.retry_config) { - node.data.retry_config = { - retry_enabled: true, - max_retries: DEFAULT_RETRY_MAX, - retry_interval: DEFAULT_RETRY_INTERVAL, - } - } - - return node - }) -} - -export const initialEdges = (originEdges: Edge[], originNodes: Node[]) => { - const { nodes, edges } = preprocessNodesAndEdges(cloneDeep(originNodes), cloneDeep(originEdges)) - let selectedNode: Node | null = null - const nodesMap = nodes.reduce((acc, node) => { - acc[node.id] = node - - if (node.data?.selected) - selectedNode = node - - return acc - }, {} as Record) - - const cycleEdges = getCycleEdges(nodes, edges) - return edges.filter((edge) => { - return !cycleEdges.find(cycEdge => cycEdge.source === edge.source && cycEdge.target === edge.target) - }).map((edge) => { - edge.type = 'custom' - - if (!edge.sourceHandle) - edge.sourceHandle = 'source' - - if (!edge.targetHandle) - edge.targetHandle = 'target' - - if (!edge.data?.sourceType && edge.source && nodesMap[edge.source]) { - edge.data = { - ...edge.data, - sourceType: nodesMap[edge.source].data.type!, - } as any - } - - if (!edge.data?.targetType && edge.target && nodesMap[edge.target]) { - edge.data = { - ...edge.data, - targetType: nodesMap[edge.target].data.type!, - } as any - } - - if (selectedNode) { - edge.data = { - ...edge.data, - _connectedNodeIsSelected: edge.source === selectedNode.id || edge.target === selectedNode.id, - } as any - } - - return edge - }) -} - -export const getLayoutByDagre = (originNodes: Node[], originEdges: Edge[]) => { - const dagreGraph = new dagre.graphlib.Graph() - dagreGraph.setDefaultEdgeLabel(() => ({})) - const nodes = cloneDeep(originNodes).filter(node => !node.parentId && node.type === CUSTOM_NODE) - const edges = cloneDeep(originEdges).filter(edge => (!edge.data?.isInIteration && !edge.data?.isInLoop)) - dagreGraph.setGraph({ - rankdir: 'LR', - align: 'UL', - nodesep: 40, - ranksep: 60, - ranker: 'tight-tree', - marginx: 30, - marginy: 200, - }) - nodes.forEach((node) => { - dagreGraph.setNode(node.id, { - width: node.width!, - height: node.height!, - }) - }) - edges.forEach((edge) => { - dagreGraph.setEdge(edge.source, edge.target) - }) - dagre.layout(dagreGraph) - return dagreGraph -} - -export const getLayoutForChildNodes = (parentNodeId: string, originNodes: Node[], originEdges: Edge[]) => { - const dagreGraph = new dagre.graphlib.Graph() - dagreGraph.setDefaultEdgeLabel(() => ({})) - - const nodes = cloneDeep(originNodes).filter(node => node.parentId === parentNodeId) - const edges = cloneDeep(originEdges).filter(edge => - (edge.data?.isInIteration && edge.data?.iteration_id === parentNodeId) - || (edge.data?.isInLoop && edge.data?.loop_id === parentNodeId), - ) - - const startNode = nodes.find(node => - node.type === CUSTOM_ITERATION_START_NODE - || node.type === CUSTOM_LOOP_START_NODE - || node.data?.type === BlockEnum.LoopStart - || node.data?.type === BlockEnum.IterationStart, - ) - - if (!startNode) { - dagreGraph.setGraph({ - rankdir: 'LR', - align: 'UL', - nodesep: 40, - ranksep: 60, - marginx: NODE_LAYOUT_HORIZONTAL_PADDING, - marginy: NODE_LAYOUT_VERTICAL_PADDING, - }) - - nodes.forEach((node) => { - dagreGraph.setNode(node.id, { - width: node.width || 244, - height: node.height || 100, - }) - }) - - edges.forEach((edge) => { - dagreGraph.setEdge(edge.source, edge.target) - }) - - dagre.layout(dagreGraph) - return dagreGraph - } - - const startNodeOutEdges = edges.filter(edge => edge.source === startNode.id) - const firstConnectedNodes = startNodeOutEdges.map(edge => - nodes.find(node => node.id === edge.target), - ).filter(Boolean) as Node[] - - const nonStartNodes = nodes.filter(node => node.id !== startNode.id) - const nonStartEdges = edges.filter(edge => edge.source !== startNode.id && edge.target !== startNode.id) - - dagreGraph.setGraph({ - rankdir: 'LR', - align: 'UL', - nodesep: 40, - ranksep: 60, - marginx: NODE_LAYOUT_HORIZONTAL_PADDING / 2, - marginy: NODE_LAYOUT_VERTICAL_PADDING / 2, - }) - - nonStartNodes.forEach((node) => { - dagreGraph.setNode(node.id, { - width: node.width || 244, - height: node.height || 100, - }) - }) - - nonStartEdges.forEach((edge) => { - dagreGraph.setEdge(edge.source, edge.target) - }) - - dagre.layout(dagreGraph) - - const startNodeSize = { - width: startNode.width || 44, - height: startNode.height || 48, - } - - const startNodeX = NODE_LAYOUT_HORIZONTAL_PADDING / 1.5 - let startNodeY = 100 - - let minFirstLayerX = Infinity - let avgFirstLayerY = 0 - let firstLayerCount = 0 - - if (firstConnectedNodes.length > 0) { - firstConnectedNodes.forEach((node) => { - if (dagreGraph.node(node.id)) { - const nodePos = dagreGraph.node(node.id) - avgFirstLayerY += nodePos.y - firstLayerCount++ - minFirstLayerX = Math.min(minFirstLayerX, nodePos.x - nodePos.width / 2) - } - }) - - if (firstLayerCount > 0) { - avgFirstLayerY /= firstLayerCount - startNodeY = avgFirstLayerY - } - - const minRequiredX = startNodeX + startNodeSize.width + NODE_LAYOUT_MIN_DISTANCE - - if (minFirstLayerX < minRequiredX) { - const shiftX = minRequiredX - minFirstLayerX - - nonStartNodes.forEach((node) => { - if (dagreGraph.node(node.id)) { - const nodePos = dagreGraph.node(node.id) - dagreGraph.setNode(node.id, { - x: nodePos.x + shiftX, - y: nodePos.y, - width: nodePos.width, - height: nodePos.height, - }) - } - }) - } - } - - dagreGraph.setNode(startNode.id, { - x: startNodeX + startNodeSize.width / 2, - y: startNodeY, - width: startNodeSize.width, - height: startNodeSize.height, - }) - - startNodeOutEdges.forEach((edge) => { - dagreGraph.setEdge(edge.source, edge.target) - }) - - return dagreGraph -} - -export const canRunBySingle = (nodeType: BlockEnum) => { - return nodeType === BlockEnum.LLM - || nodeType === BlockEnum.KnowledgeRetrieval - || nodeType === BlockEnum.Code - || nodeType === BlockEnum.TemplateTransform - || nodeType === BlockEnum.QuestionClassifier - || nodeType === BlockEnum.HttpRequest - || nodeType === BlockEnum.Tool - || nodeType === BlockEnum.ParameterExtractor - || nodeType === BlockEnum.Iteration - || nodeType === BlockEnum.Agent - || nodeType === BlockEnum.DocExtractor - || nodeType === BlockEnum.Loop -} - -type ConnectedSourceOrTargetNodesChange = { - type: string - edge: Edge -}[] -export const getNodesConnectedSourceOrTargetHandleIdsMap = (changes: ConnectedSourceOrTargetNodesChange, nodes: Node[]) => { - const nodesConnectedSourceOrTargetHandleIdsMap = {} as Record - - changes.forEach((change) => { - const { - edge, - type, - } = change - const sourceNode = nodes.find(node => node.id === edge.source)! - if (sourceNode) { - nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id] = nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id] || { - _connectedSourceHandleIds: [...(sourceNode?.data._connectedSourceHandleIds || [])], - _connectedTargetHandleIds: [...(sourceNode?.data._connectedTargetHandleIds || [])], - } - } - - const targetNode = nodes.find(node => node.id === edge.target)! - if (targetNode) { - nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id] = nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id] || { - _connectedSourceHandleIds: [...(targetNode?.data._connectedSourceHandleIds || [])], - _connectedTargetHandleIds: [...(targetNode?.data._connectedTargetHandleIds || [])], - } - } - - if (sourceNode) { - if (type === 'remove') { - const index = nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id]._connectedSourceHandleIds.findIndex((handleId: string) => handleId === edge.sourceHandle) - nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id]._connectedSourceHandleIds.splice(index, 1) - } - - if (type === 'add') - nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id]._connectedSourceHandleIds.push(edge.sourceHandle || 'source') - } - - if (targetNode) { - if (type === 'remove') { - const index = nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id]._connectedTargetHandleIds.findIndex((handleId: string) => handleId === edge.targetHandle) - nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id]._connectedTargetHandleIds.splice(index, 1) - } - - if (type === 'add') - nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id]._connectedTargetHandleIds.push(edge.targetHandle || 'target') - } - }) - - return nodesConnectedSourceOrTargetHandleIdsMap -} - -export const genNewNodeTitleFromOld = (oldTitle: string) => { - const regex = /^(.+?)\s*\((\d+)\)\s*$/ - const match = oldTitle.match(regex) - - if (match) { - const title = match[1] - const num = Number.parseInt(match[2], 10) - return `${title} (${num + 1})` - } - else { - return `${oldTitle} (1)` - } -} - -export const getValidTreeNodes = (nodes: Node[], edges: Edge[]) => { - const startNode = nodes.find(node => node.data.type === BlockEnum.Start) - - if (!startNode) { - return { - validNodes: [], - maxDepth: 0, - } - } - - const list: Node[] = [startNode] - let maxDepth = 1 - - const traverse = (root: Node, depth: number) => { - if (depth > maxDepth) - maxDepth = depth - - const outgoers = getOutgoers(root, nodes, edges) - - if (outgoers.length) { - outgoers.forEach((outgoer) => { - list.push(outgoer) - - if (outgoer.data.type === BlockEnum.Iteration) - list.push(...nodes.filter(node => node.parentId === outgoer.id)) - if (outgoer.data.type === BlockEnum.Loop) - list.push(...nodes.filter(node => node.parentId === outgoer.id)) - - traverse(outgoer, depth + 1) - }) - } - else { - list.push(root) - - if (root.data.type === BlockEnum.Iteration) - list.push(...nodes.filter(node => node.parentId === root.id)) - if (root.data.type === BlockEnum.Loop) - list.push(...nodes.filter(node => node.parentId === root.id)) - } - } - - traverse(startNode, maxDepth) - - return { - validNodes: uniqBy(list, 'id'), - maxDepth, - } -} - -export const getToolCheckParams = ( - toolData: ToolNodeType, - buildInTools: ToolWithProvider[], - customTools: ToolWithProvider[], - workflowTools: ToolWithProvider[], - language: string, -) => { - const { provider_id, provider_type, tool_name } = toolData - const isBuiltIn = provider_type === CollectionType.builtIn - const currentTools = provider_type === CollectionType.builtIn ? buildInTools : provider_type === CollectionType.custom ? customTools : workflowTools - const currCollection = currentTools.find(item => canFindTool(item.id, provider_id)) - const currTool = currCollection?.tools.find(tool => tool.name === tool_name) - const formSchemas = currTool ? toolParametersToFormSchemas(currTool.parameters) : [] - const toolInputVarSchema = formSchemas.filter((item: any) => item.form === 'llm') - const toolSettingSchema = formSchemas.filter((item: any) => item.form !== 'llm') - - return { - toolInputsSchema: (() => { - const formInputs: InputVar[] = [] - toolInputVarSchema.forEach((item: any) => { - formInputs.push({ - label: item.label[language] || item.label.en_US, - variable: item.variable, - type: item.type, - required: item.required, - }) - }) - return formInputs - })(), - notAuthed: isBuiltIn && !!currCollection?.allow_delete && !currCollection?.is_team_authorization, - toolSettingSchema, - language, - } -} - -export const changeNodesAndEdgesId = (nodes: Node[], edges: Edge[]) => { - const idMap = nodes.reduce((acc, node) => { - acc[node.id] = uuid4() - - return acc - }, {} as Record) - - const newNodes = nodes.map((node) => { - return { - ...node, - id: idMap[node.id], - } - }) - - const newEdges = edges.map((edge) => { - return { - ...edge, - source: idMap[edge.source], - target: idMap[edge.target], - } - }) - - return [newNodes, newEdges] as [Node[], Edge[]] -} - -export const isMac = () => { - return navigator.userAgent.toUpperCase().includes('MAC') -} - -const specialKeysNameMap: Record = { - ctrl: '⌘', - alt: '⌥', - shift: '⇧', -} - -export const getKeyboardKeyNameBySystem = (key: string) => { - if (isMac()) - return specialKeysNameMap[key] || key - - return key -} - -const specialKeysCodeMap: Record = { - ctrl: 'meta', -} - -export const getKeyboardKeyCodeBySystem = (key: string) => { - if (isMac()) - return specialKeysCodeMap[key] || key - - return key -} - -export const getTopLeftNodePosition = (nodes: Node[]) => { - let minX = Infinity - let minY = Infinity - - nodes.forEach((node) => { - if (node.position.x < minX) - minX = node.position.x - - if (node.position.y < minY) - minY = node.position.y - }) - - return { - x: minX, - y: minY, - } -} - -export const isEventTargetInputArea = (target: HTMLElement) => { - if (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA') - return true - - if (target.contentEditable === 'true') - return true -} - -export const variableTransformer = (v: ValueSelector | string) => { - if (typeof v === 'string') - return v.replace(/^{{#|#}}$/g, '').split('.') - - return `{{#${v.join('.')}#}}` -} - -type ParallelInfoItem = { - parallelNodeId: string - depth: number - isBranch?: boolean -} -type NodeParallelInfo = { - parallelNodeId: string - edgeHandleId: string - depth: number -} -type NodeHandle = { - node: Node - handle: string -} -type NodeStreamInfo = { - upstreamNodes: Set - downstreamEdges: Set -} -export const getParallelInfo = (nodes: Node[], edges: Edge[], parentNodeId?: string) => { - let startNode - - if (parentNodeId) { - const parentNode = nodes.find(node => node.id === parentNodeId) - if (!parentNode) - throw new Error('Parent node not found') - - startNode = nodes.find(node => node.id === (parentNode.data as (IterationNodeType | LoopNodeType)).start_node_id) - } - else { - startNode = nodes.find(node => node.data.type === BlockEnum.Start) - } - if (!startNode) - throw new Error('Start node not found') - - const parallelList = [] as ParallelInfoItem[] - const nextNodeHandles = [{ node: startNode, handle: 'source' }] - let hasAbnormalEdges = false - - const traverse = (firstNodeHandle: NodeHandle) => { - const nodeEdgesSet = {} as Record> - const totalEdgesSet = new Set() - const nextHandles = [firstNodeHandle] - const streamInfo = {} as Record - const parallelListItem = { - parallelNodeId: '', - depth: 0, - } as ParallelInfoItem - const nodeParallelInfoMap = {} as Record - nodeParallelInfoMap[firstNodeHandle.node.id] = { - parallelNodeId: '', - edgeHandleId: '', - depth: 0, - } - - while (nextHandles.length) { - const currentNodeHandle = nextHandles.shift()! - const { node: currentNode, handle: currentHandle = 'source' } = currentNodeHandle - const currentNodeHandleKey = currentNode.id - const connectedEdges = edges.filter(edge => edge.source === currentNode.id && edge.sourceHandle === currentHandle) - const connectedEdgesLength = connectedEdges.length - const outgoers = nodes.filter(node => connectedEdges.some(edge => edge.target === node.id)) - const incomers = getIncomers(currentNode, nodes, edges) - - if (!streamInfo[currentNodeHandleKey]) { - streamInfo[currentNodeHandleKey] = { - upstreamNodes: new Set(), - downstreamEdges: new Set(), - } - } - - if (nodeEdgesSet[currentNodeHandleKey]?.size > 0 && incomers.length > 1) { - const newSet = new Set() - for (const item of totalEdgesSet) { - if (!streamInfo[currentNodeHandleKey].downstreamEdges.has(item)) - newSet.add(item) - } - if (isEqual(nodeEdgesSet[currentNodeHandleKey], newSet)) { - parallelListItem.depth = nodeParallelInfoMap[currentNode.id].depth - nextNodeHandles.push({ node: currentNode, handle: currentHandle }) - break - } - } - - if (nodeParallelInfoMap[currentNode.id].depth > parallelListItem.depth) - parallelListItem.depth = nodeParallelInfoMap[currentNode.id].depth - - outgoers.forEach((outgoer) => { - const outgoerConnectedEdges = getConnectedEdges([outgoer], edges).filter(edge => edge.source === outgoer.id) - const sourceEdgesGroup = groupBy(outgoerConnectedEdges, 'sourceHandle') - const incomers = getIncomers(outgoer, nodes, edges) - - if (outgoers.length > 1 && incomers.length > 1) - hasAbnormalEdges = true - - Object.keys(sourceEdgesGroup).forEach((sourceHandle) => { - nextHandles.push({ node: outgoer, handle: sourceHandle }) - }) - if (!outgoerConnectedEdges.length) - nextHandles.push({ node: outgoer, handle: 'source' }) - - const outgoerKey = outgoer.id - if (!nodeEdgesSet[outgoerKey]) - nodeEdgesSet[outgoerKey] = new Set() - - if (nodeEdgesSet[currentNodeHandleKey]) { - for (const item of nodeEdgesSet[currentNodeHandleKey]) - nodeEdgesSet[outgoerKey].add(item) - } - - if (!streamInfo[outgoerKey]) { - streamInfo[outgoerKey] = { - upstreamNodes: new Set(), - downstreamEdges: new Set(), - } - } - - if (!nodeParallelInfoMap[outgoer.id]) { - nodeParallelInfoMap[outgoer.id] = { - ...nodeParallelInfoMap[currentNode.id], - } - } - - if (connectedEdgesLength > 1) { - const edge = connectedEdges.find(edge => edge.target === outgoer.id)! - nodeEdgesSet[outgoerKey].add(edge.id) - totalEdgesSet.add(edge.id) - - streamInfo[currentNodeHandleKey].downstreamEdges.add(edge.id) - streamInfo[outgoerKey].upstreamNodes.add(currentNodeHandleKey) - - for (const item of streamInfo[currentNodeHandleKey].upstreamNodes) - streamInfo[item].downstreamEdges.add(edge.id) - - if (!parallelListItem.parallelNodeId) - parallelListItem.parallelNodeId = currentNode.id - - const prevDepth = nodeParallelInfoMap[currentNode.id].depth + 1 - const currentDepth = nodeParallelInfoMap[outgoer.id].depth - - nodeParallelInfoMap[outgoer.id].depth = Math.max(prevDepth, currentDepth) - } - else { - for (const item of streamInfo[currentNodeHandleKey].upstreamNodes) - streamInfo[outgoerKey].upstreamNodes.add(item) - - nodeParallelInfoMap[outgoer.id].depth = nodeParallelInfoMap[currentNode.id].depth - } - }) - } - - parallelList.push(parallelListItem) - } - - while (nextNodeHandles.length) { - const nodeHandle = nextNodeHandles.shift()! - traverse(nodeHandle) - } - - return { - parallelList, - hasAbnormalEdges, - } -} - -export const hasErrorHandleNode = (nodeType?: BlockEnum) => { - return nodeType === BlockEnum.LLM || nodeType === BlockEnum.Tool || nodeType === BlockEnum.HttpRequest || nodeType === BlockEnum.Code -} - -export const getEdgeColor = (nodeRunningStatus?: NodeRunningStatus, isFailBranch?: boolean) => { - if (nodeRunningStatus === NodeRunningStatus.Succeeded) - return 'var(--color-workflow-link-line-success-handle)' - - if (nodeRunningStatus === NodeRunningStatus.Failed) - return 'var(--color-workflow-link-line-error-handle)' - - if (nodeRunningStatus === NodeRunningStatus.Exception) - return 'var(--color-workflow-link-line-failure-handle)' - - if (nodeRunningStatus === NodeRunningStatus.Running) { - if (isFailBranch) - return 'var(--color-workflow-link-line-failure-handle)' - - return 'var(--color-workflow-link-line-handle)' - } - - return 'var(--color-workflow-link-line-normal)' -} - -export const isExceptionVariable = (variable: string, nodeType?: BlockEnum) => { - if ((variable === 'error_message' || variable === 'error_type') && hasErrorHandleNode(nodeType)) - return true - - return false -} - -export const hasRetryNode = (nodeType?: BlockEnum) => { - return nodeType === BlockEnum.LLM || nodeType === BlockEnum.Tool || nodeType === BlockEnum.HttpRequest || nodeType === BlockEnum.Code -} - -export const getNodeCustomTypeByNodeDataType = (nodeType: BlockEnum) => { - if (nodeType === BlockEnum.LoopEnd) - return CUSTOM_SIMPLE_NODE -} diff --git a/web/app/components/workflow/utils/common.ts b/web/app/components/workflow/utils/common.ts new file mode 100644 index 0000000000..8a8afbb264 --- /dev/null +++ b/web/app/components/workflow/utils/common.ts @@ -0,0 +1,35 @@ +export const isMac = () => { + return navigator.userAgent.toUpperCase().includes('MAC') +} + +const specialKeysNameMap: Record = { + ctrl: '⌘', + alt: '⌥', + shift: '⇧', +} + +export const getKeyboardKeyNameBySystem = (key: string) => { + if (isMac()) + return specialKeysNameMap[key] || key + + return key +} + +const specialKeysCodeMap: Record = { + ctrl: 'meta', +} + +export const getKeyboardKeyCodeBySystem = (key: string) => { + if (isMac()) + return specialKeysCodeMap[key] || key + + return key +} + +export const isEventTargetInputArea = (target: HTMLElement) => { + if (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA') + return true + + if (target.contentEditable === 'true') + return true +} diff --git a/web/app/components/workflow/utils/edge.ts b/web/app/components/workflow/utils/edge.ts new file mode 100644 index 0000000000..b539c218d7 --- /dev/null +++ b/web/app/components/workflow/utils/edge.ts @@ -0,0 +1,23 @@ +import { + NodeRunningStatus, +} from '../types' + +export const getEdgeColor = (nodeRunningStatus?: NodeRunningStatus, isFailBranch?: boolean) => { + if (nodeRunningStatus === NodeRunningStatus.Succeeded) + return 'var(--color-workflow-link-line-success-handle)' + + if (nodeRunningStatus === NodeRunningStatus.Failed) + return 'var(--color-workflow-link-line-error-handle)' + + if (nodeRunningStatus === NodeRunningStatus.Exception) + return 'var(--color-workflow-link-line-failure-handle)' + + if (nodeRunningStatus === NodeRunningStatus.Running) { + if (isFailBranch) + return 'var(--color-workflow-link-line-failure-handle)' + + return 'var(--color-workflow-link-line-handle)' + } + + return 'var(--color-workflow-link-line-normal)' +} diff --git a/web/app/components/workflow/utils/index.ts b/web/app/components/workflow/utils/index.ts new file mode 100644 index 0000000000..4a1da760d4 --- /dev/null +++ b/web/app/components/workflow/utils/index.ts @@ -0,0 +1,8 @@ +export * from './node' +export * from './edge' +export * from './workflow-init' +export * from './layout' +export * from './common' +export * from './tool' +export * from './workflow' +export * from './variable' diff --git a/web/app/components/workflow/utils/layout.ts b/web/app/components/workflow/utils/layout.ts new file mode 100644 index 0000000000..3c4189b5bc --- /dev/null +++ b/web/app/components/workflow/utils/layout.ts @@ -0,0 +1,178 @@ +import dagre from '@dagrejs/dagre' +import { + cloneDeep, +} from 'lodash-es' +import type { + Edge, + Node, +} from '../types' +import { + BlockEnum, +} from '../types' +import { + CUSTOM_NODE, + NODE_LAYOUT_HORIZONTAL_PADDING, + NODE_LAYOUT_MIN_DISTANCE, + NODE_LAYOUT_VERTICAL_PADDING, +} from '../constants' +import { CUSTOM_ITERATION_START_NODE } from '../nodes/iteration-start/constants' +import { CUSTOM_LOOP_START_NODE } from '../nodes/loop-start/constants' + +export const getLayoutByDagre = (originNodes: Node[], originEdges: Edge[]) => { + const dagreGraph = new dagre.graphlib.Graph() + dagreGraph.setDefaultEdgeLabel(() => ({})) + const nodes = cloneDeep(originNodes).filter(node => !node.parentId && node.type === CUSTOM_NODE) + const edges = cloneDeep(originEdges).filter(edge => (!edge.data?.isInIteration && !edge.data?.isInLoop)) + dagreGraph.setGraph({ + rankdir: 'LR', + align: 'UL', + nodesep: 40, + ranksep: 60, + ranker: 'tight-tree', + marginx: 30, + marginy: 200, + }) + nodes.forEach((node) => { + dagreGraph.setNode(node.id, { + width: node.width!, + height: node.height!, + }) + }) + edges.forEach((edge) => { + dagreGraph.setEdge(edge.source, edge.target) + }) + dagre.layout(dagreGraph) + return dagreGraph +} + +export const getLayoutForChildNodes = (parentNodeId: string, originNodes: Node[], originEdges: Edge[]) => { + const dagreGraph = new dagre.graphlib.Graph() + dagreGraph.setDefaultEdgeLabel(() => ({})) + + const nodes = cloneDeep(originNodes).filter(node => node.parentId === parentNodeId) + const edges = cloneDeep(originEdges).filter(edge => + (edge.data?.isInIteration && edge.data?.iteration_id === parentNodeId) + || (edge.data?.isInLoop && edge.data?.loop_id === parentNodeId), + ) + + const startNode = nodes.find(node => + node.type === CUSTOM_ITERATION_START_NODE + || node.type === CUSTOM_LOOP_START_NODE + || node.data?.type === BlockEnum.LoopStart + || node.data?.type === BlockEnum.IterationStart, + ) + + if (!startNode) { + dagreGraph.setGraph({ + rankdir: 'LR', + align: 'UL', + nodesep: 40, + ranksep: 60, + marginx: NODE_LAYOUT_HORIZONTAL_PADDING, + marginy: NODE_LAYOUT_VERTICAL_PADDING, + }) + + nodes.forEach((node) => { + dagreGraph.setNode(node.id, { + width: node.width || 244, + height: node.height || 100, + }) + }) + + edges.forEach((edge) => { + dagreGraph.setEdge(edge.source, edge.target) + }) + + dagre.layout(dagreGraph) + return dagreGraph + } + + const startNodeOutEdges = edges.filter(edge => edge.source === startNode.id) + const firstConnectedNodes = startNodeOutEdges.map(edge => + nodes.find(node => node.id === edge.target), + ).filter(Boolean) as Node[] + + const nonStartNodes = nodes.filter(node => node.id !== startNode.id) + const nonStartEdges = edges.filter(edge => edge.source !== startNode.id && edge.target !== startNode.id) + + dagreGraph.setGraph({ + rankdir: 'LR', + align: 'UL', + nodesep: 40, + ranksep: 60, + marginx: NODE_LAYOUT_HORIZONTAL_PADDING / 2, + marginy: NODE_LAYOUT_VERTICAL_PADDING / 2, + }) + + nonStartNodes.forEach((node) => { + dagreGraph.setNode(node.id, { + width: node.width || 244, + height: node.height || 100, + }) + }) + + nonStartEdges.forEach((edge) => { + dagreGraph.setEdge(edge.source, edge.target) + }) + + dagre.layout(dagreGraph) + + const startNodeSize = { + width: startNode.width || 44, + height: startNode.height || 48, + } + + const startNodeX = NODE_LAYOUT_HORIZONTAL_PADDING / 1.5 + let startNodeY = 100 + + let minFirstLayerX = Infinity + let avgFirstLayerY = 0 + let firstLayerCount = 0 + + if (firstConnectedNodes.length > 0) { + firstConnectedNodes.forEach((node) => { + if (dagreGraph.node(node.id)) { + const nodePos = dagreGraph.node(node.id) + avgFirstLayerY += nodePos.y + firstLayerCount++ + minFirstLayerX = Math.min(minFirstLayerX, nodePos.x - nodePos.width / 2) + } + }) + + if (firstLayerCount > 0) { + avgFirstLayerY /= firstLayerCount + startNodeY = avgFirstLayerY + } + + const minRequiredX = startNodeX + startNodeSize.width + NODE_LAYOUT_MIN_DISTANCE + + if (minFirstLayerX < minRequiredX) { + const shiftX = minRequiredX - minFirstLayerX + + nonStartNodes.forEach((node) => { + if (dagreGraph.node(node.id)) { + const nodePos = dagreGraph.node(node.id) + dagreGraph.setNode(node.id, { + x: nodePos.x + shiftX, + y: nodePos.y, + width: nodePos.width, + height: nodePos.height, + }) + } + }) + } + } + + dagreGraph.setNode(startNode.id, { + x: startNodeX + startNodeSize.width / 2, + y: startNodeY, + width: startNodeSize.width, + height: startNodeSize.height, + }) + + startNodeOutEdges.forEach((edge) => { + dagreGraph.setEdge(edge.source, edge.target) + }) + + return dagreGraph +} diff --git a/web/app/components/workflow/utils/node.ts b/web/app/components/workflow/utils/node.ts new file mode 100644 index 0000000000..7a9e33b2f6 --- /dev/null +++ b/web/app/components/workflow/utils/node.ts @@ -0,0 +1,145 @@ +import { + Position, +} from 'reactflow' +import type { + Node, +} from '../types' +import { + BlockEnum, +} from '../types' +import { + CUSTOM_NODE, + ITERATION_CHILDREN_Z_INDEX, + ITERATION_NODE_Z_INDEX, + LOOP_CHILDREN_Z_INDEX, + LOOP_NODE_Z_INDEX, +} from '../constants' +import { CUSTOM_ITERATION_START_NODE } from '../nodes/iteration-start/constants' +import { CUSTOM_LOOP_START_NODE } from '../nodes/loop-start/constants' +import type { IterationNodeType } from '../nodes/iteration/types' +import type { LoopNodeType } from '../nodes/loop/types' +import { CUSTOM_SIMPLE_NODE } from '@/app/components/workflow/simple-node/constants' + +export function generateNewNode({ data, position, id, zIndex, type, ...rest }: Omit & { id?: string }): { + newNode: Node + newIterationStartNode?: Node + newLoopStartNode?: Node +} { + const newNode = { + id: id || `${Date.now()}`, + type: type || CUSTOM_NODE, + data, + position, + targetPosition: Position.Left, + sourcePosition: Position.Right, + zIndex: data.type === BlockEnum.Iteration ? ITERATION_NODE_Z_INDEX : (data.type === BlockEnum.Loop ? LOOP_NODE_Z_INDEX : zIndex), + ...rest, + } as Node + + if (data.type === BlockEnum.Iteration) { + const newIterationStartNode = getIterationStartNode(newNode.id); + (newNode.data as IterationNodeType).start_node_id = newIterationStartNode.id; + (newNode.data as IterationNodeType)._children = [{ nodeId: newIterationStartNode.id, nodeType: BlockEnum.IterationStart }] + return { + newNode, + newIterationStartNode, + } + } + + if (data.type === BlockEnum.Loop) { + const newLoopStartNode = getLoopStartNode(newNode.id); + (newNode.data as LoopNodeType).start_node_id = newLoopStartNode.id; + (newNode.data as LoopNodeType)._children = [{ nodeId: newLoopStartNode.id, nodeType: BlockEnum.LoopStart }] + return { + newNode, + newLoopStartNode, + } + } + + return { + newNode, + } +} + +export function getIterationStartNode(iterationId: string): Node { + return generateNewNode({ + id: `${iterationId}start`, + type: CUSTOM_ITERATION_START_NODE, + data: { + title: '', + desc: '', + type: BlockEnum.IterationStart, + isInIteration: true, + }, + position: { + x: 24, + y: 68, + }, + zIndex: ITERATION_CHILDREN_Z_INDEX, + parentId: iterationId, + selectable: false, + draggable: false, + }).newNode +} + +export function getLoopStartNode(loopId: string): Node { + return generateNewNode({ + id: `${loopId}start`, + type: CUSTOM_LOOP_START_NODE, + data: { + title: '', + desc: '', + type: BlockEnum.LoopStart, + isInLoop: true, + }, + position: { + x: 24, + y: 68, + }, + zIndex: LOOP_CHILDREN_Z_INDEX, + parentId: loopId, + selectable: false, + draggable: false, + }).newNode +} + +export const genNewNodeTitleFromOld = (oldTitle: string) => { + const regex = /^(.+?)\s*\((\d+)\)\s*$/ + const match = oldTitle.match(regex) + + if (match) { + const title = match[1] + const num = Number.parseInt(match[2], 10) + return `${title} (${num + 1})` + } + else { + return `${oldTitle} (1)` + } +} + +export const getTopLeftNodePosition = (nodes: Node[]) => { + let minX = Infinity + let minY = Infinity + + nodes.forEach((node) => { + if (node.position.x < minX) + minX = node.position.x + + if (node.position.y < minY) + minY = node.position.y + }) + + return { + x: minX, + y: minY, + } +} + +export const hasRetryNode = (nodeType?: BlockEnum) => { + return nodeType === BlockEnum.LLM || nodeType === BlockEnum.Tool || nodeType === BlockEnum.HttpRequest || nodeType === BlockEnum.Code +} + +export const getNodeCustomTypeByNodeDataType = (nodeType: BlockEnum) => { + if (nodeType === BlockEnum.LoopEnd) + return CUSTOM_SIMPLE_NODE +} diff --git a/web/app/components/workflow/utils/tool.ts b/web/app/components/workflow/utils/tool.ts new file mode 100644 index 0000000000..9fb5d5cc07 --- /dev/null +++ b/web/app/components/workflow/utils/tool.ts @@ -0,0 +1,43 @@ +import type { + InputVar, + ToolWithProvider, +} from '../types' +import type { ToolNodeType } from '../nodes/tool/types' +import { CollectionType } from '@/app/components/tools/types' +import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema' +import { canFindTool } from '@/utils' + +export const getToolCheckParams = ( + toolData: ToolNodeType, + buildInTools: ToolWithProvider[], + customTools: ToolWithProvider[], + workflowTools: ToolWithProvider[], + language: string, +) => { + const { provider_id, provider_type, tool_name } = toolData + const isBuiltIn = provider_type === CollectionType.builtIn + const currentTools = provider_type === CollectionType.builtIn ? buildInTools : provider_type === CollectionType.custom ? customTools : workflowTools + const currCollection = currentTools.find(item => canFindTool(item.id, provider_id)) + const currTool = currCollection?.tools.find(tool => tool.name === tool_name) + const formSchemas = currTool ? toolParametersToFormSchemas(currTool.parameters) : [] + const toolInputVarSchema = formSchemas.filter((item: any) => item.form === 'llm') + const toolSettingSchema = formSchemas.filter((item: any) => item.form !== 'llm') + + return { + toolInputsSchema: (() => { + const formInputs: InputVar[] = [] + toolInputVarSchema.forEach((item: any) => { + formInputs.push({ + label: item.label[language] || item.label.en_US, + variable: item.variable, + type: item.type, + required: item.required, + }) + }) + return formInputs + })(), + notAuthed: isBuiltIn && !!currCollection?.allow_delete && !currCollection?.is_team_authorization, + toolSettingSchema, + language, + } +} diff --git a/web/app/components/workflow/utils/variable.ts b/web/app/components/workflow/utils/variable.ts new file mode 100644 index 0000000000..cbefe563e5 --- /dev/null +++ b/web/app/components/workflow/utils/variable.ts @@ -0,0 +1,21 @@ +import type { + ValueSelector, +} from '../types' +import type { + BlockEnum, +} from '../types' +import { hasErrorHandleNode } from '.' + +export const variableTransformer = (v: ValueSelector | string) => { + if (typeof v === 'string') + return v.replace(/^{{#|#}}$/g, '').split('.') + + return `{{#${v.join('.')}#}}` +} + +export const isExceptionVariable = (variable: string, nodeType?: BlockEnum) => { + if ((variable === 'error_message' || variable === 'error_type') && hasErrorHandleNode(nodeType)) + return true + + return false +} diff --git a/web/app/components/workflow/utils/workflow-init.spec.ts b/web/app/components/workflow/utils/workflow-init.spec.ts new file mode 100644 index 0000000000..8b7bdfaa92 --- /dev/null +++ b/web/app/components/workflow/utils/workflow-init.spec.ts @@ -0,0 +1,69 @@ +import { preprocessNodesAndEdges } from './workflow-init' +import { BlockEnum } from '@/app/components/workflow/types' +import type { + Node, +} from '@/app/components/workflow/types' +import { CUSTOM_ITERATION_START_NODE } from '@/app/components/workflow/nodes/iteration-start/constants' + +describe('preprocessNodesAndEdges', () => { + it('process nodes without iteration node or loop node should return origin nodes and edges.', () => { + const nodes = [ + { + data: { + type: BlockEnum.Code, + }, + }, + ] + + const result = preprocessNodesAndEdges(nodes as Node[], []) + expect(result).toEqual({ + nodes, + edges: [], + }) + }) + + it('process nodes with iteration node should return nodes with iteration start node', () => { + const nodes = [ + { + id: 'iteration', + data: { + type: BlockEnum.Iteration, + }, + }, + ] + + const result = preprocessNodesAndEdges(nodes as Node[], []) + expect(result.nodes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + type: BlockEnum.IterationStart, + }), + }), + ]), + ) + }) + + it('process nodes with iteration node start should return origin', () => { + const nodes = [ + { + data: { + type: BlockEnum.Iteration, + start_node_id: 'iterationStart', + }, + }, + { + id: 'iterationStart', + type: CUSTOM_ITERATION_START_NODE, + data: { + type: BlockEnum.IterationStart, + }, + }, + ] + const result = preprocessNodesAndEdges(nodes as Node[], []) + expect(result).toEqual({ + nodes, + edges: [], + }) + }) +}) diff --git a/web/app/components/workflow/utils/workflow-init.ts b/web/app/components/workflow/utils/workflow-init.ts new file mode 100644 index 0000000000..93a61230ba --- /dev/null +++ b/web/app/components/workflow/utils/workflow-init.ts @@ -0,0 +1,338 @@ +import { + getConnectedEdges, +} from 'reactflow' +import { + cloneDeep, +} from 'lodash-es' +import type { + Edge, + Node, +} from '../types' +import { + BlockEnum, + ErrorHandleMode, +} from '../types' +import { + CUSTOM_NODE, + DEFAULT_RETRY_INTERVAL, + DEFAULT_RETRY_MAX, + ITERATION_CHILDREN_Z_INDEX, + LOOP_CHILDREN_Z_INDEX, + NODE_WIDTH_X_OFFSET, + START_INITIAL_POSITION, +} from '../constants' +import { CUSTOM_ITERATION_START_NODE } from '../nodes/iteration-start/constants' +import { CUSTOM_LOOP_START_NODE } from '../nodes/loop-start/constants' +import type { QuestionClassifierNodeType } from '../nodes/question-classifier/types' +import type { IfElseNodeType } from '../nodes/if-else/types' +import { branchNameCorrect } from '../nodes/if-else/utils' +import type { IterationNodeType } from '../nodes/iteration/types' +import type { LoopNodeType } from '../nodes/loop/types' +import { + getIterationStartNode, + getLoopStartNode, +} from '.' +import { correctModelProvider } from '@/utils' + +const WHITE = 'WHITE' +const GRAY = 'GRAY' +const BLACK = 'BLACK' +const isCyclicUtil = (nodeId: string, color: Record, adjList: Record, stack: string[]) => { + color[nodeId] = GRAY + stack.push(nodeId) + + for (let i = 0; i < adjList[nodeId].length; ++i) { + const childId = adjList[nodeId][i] + + if (color[childId] === GRAY) { + stack.push(childId) + return true + } + if (color[childId] === WHITE && isCyclicUtil(childId, color, adjList, stack)) + return true + } + color[nodeId] = BLACK + if (stack.length > 0 && stack[stack.length - 1] === nodeId) + stack.pop() + return false +} + +const getCycleEdges = (nodes: Node[], edges: Edge[]) => { + const adjList: Record = {} + const color: Record = {} + const stack: string[] = [] + + for (const node of nodes) { + color[node.id] = WHITE + adjList[node.id] = [] + } + + for (const edge of edges) + adjList[edge.source]?.push(edge.target) + + for (let i = 0; i < nodes.length; i++) { + if (color[nodes[i].id] === WHITE) + isCyclicUtil(nodes[i].id, color, adjList, stack) + } + + const cycleEdges = [] + if (stack.length > 0) { + const cycleNodes = new Set(stack) + for (const edge of edges) { + if (cycleNodes.has(edge.source) && cycleNodes.has(edge.target)) + cycleEdges.push(edge) + } + } + + return cycleEdges +} + +export const preprocessNodesAndEdges = (nodes: Node[], edges: Edge[]) => { + const hasIterationNode = nodes.some(node => node.data.type === BlockEnum.Iteration) + const hasLoopNode = nodes.some(node => node.data.type === BlockEnum.Loop) + + if (!hasIterationNode && !hasLoopNode) { + return { + nodes, + edges, + } + } + + const nodesMap = nodes.reduce((prev, next) => { + prev[next.id] = next + return prev + }, {} as Record) + + const iterationNodesWithStartNode = [] + const iterationNodesWithoutStartNode = [] + const loopNodesWithStartNode = [] + const loopNodesWithoutStartNode = [] + + for (let i = 0; i < nodes.length; i++) { + const currentNode = nodes[i] as Node + + if (currentNode.data.type === BlockEnum.Iteration) { + if (currentNode.data.start_node_id) { + if (nodesMap[currentNode.data.start_node_id]?.type !== CUSTOM_ITERATION_START_NODE) + iterationNodesWithStartNode.push(currentNode) + } + else { + iterationNodesWithoutStartNode.push(currentNode) + } + } + + if (currentNode.data.type === BlockEnum.Loop) { + if (currentNode.data.start_node_id) { + if (nodesMap[currentNode.data.start_node_id]?.type !== CUSTOM_LOOP_START_NODE) + loopNodesWithStartNode.push(currentNode) + } + else { + loopNodesWithoutStartNode.push(currentNode) + } + } + } + + const newIterationStartNodesMap = {} as Record + const newIterationStartNodes = [...iterationNodesWithStartNode, ...iterationNodesWithoutStartNode].map((iterationNode, index) => { + const newNode = getIterationStartNode(iterationNode.id) + newNode.id = newNode.id + index + newIterationStartNodesMap[iterationNode.id] = newNode + return newNode + }) + + const newLoopStartNodesMap = {} as Record + const newLoopStartNodes = [...loopNodesWithStartNode, ...loopNodesWithoutStartNode].map((loopNode, index) => { + const newNode = getLoopStartNode(loopNode.id) + newNode.id = newNode.id + index + newLoopStartNodesMap[loopNode.id] = newNode + return newNode + }) + + const newEdges = [...iterationNodesWithStartNode, ...loopNodesWithStartNode].map((nodeItem) => { + const isIteration = nodeItem.data.type === BlockEnum.Iteration + const newNode = (isIteration ? newIterationStartNodesMap : newLoopStartNodesMap)[nodeItem.id] + const startNode = nodesMap[nodeItem.data.start_node_id] + const source = newNode.id + const sourceHandle = 'source' + const target = startNode.id + const targetHandle = 'target' + + const parentNode = nodes.find(node => node.id === startNode.parentId) || null + const isInIteration = !!parentNode && parentNode.data.type === BlockEnum.Iteration + const isInLoop = !!parentNode && parentNode.data.type === BlockEnum.Loop + + return { + id: `${source}-${sourceHandle}-${target}-${targetHandle}`, + type: 'custom', + source, + sourceHandle, + target, + targetHandle, + data: { + sourceType: newNode.data.type, + targetType: startNode.data.type, + isInIteration, + iteration_id: isInIteration ? startNode.parentId : undefined, + isInLoop, + loop_id: isInLoop ? startNode.parentId : undefined, + _connectedNodeIsSelected: true, + }, + zIndex: isIteration ? ITERATION_CHILDREN_Z_INDEX : LOOP_CHILDREN_Z_INDEX, + } + }) + nodes.forEach((node) => { + if (node.data.type === BlockEnum.Iteration && newIterationStartNodesMap[node.id]) + (node.data as IterationNodeType).start_node_id = newIterationStartNodesMap[node.id].id + + if (node.data.type === BlockEnum.Loop && newLoopStartNodesMap[node.id]) + (node.data as LoopNodeType).start_node_id = newLoopStartNodesMap[node.id].id + }) + + return { + nodes: [...nodes, ...newIterationStartNodes, ...newLoopStartNodes], + edges: [...edges, ...newEdges], + } +} + +export const initialNodes = (originNodes: Node[], originEdges: Edge[]) => { + const { nodes, edges } = preprocessNodesAndEdges(cloneDeep(originNodes), cloneDeep(originEdges)) + const firstNode = nodes[0] + + if (!firstNode?.position) { + nodes.forEach((node, index) => { + node.position = { + x: START_INITIAL_POSITION.x + index * NODE_WIDTH_X_OFFSET, + y: START_INITIAL_POSITION.y, + } + }) + } + + const iterationOrLoopNodeMap = nodes.reduce((acc, node) => { + if (node.parentId) { + if (acc[node.parentId]) + acc[node.parentId].push({ nodeId: node.id, nodeType: node.data.type }) + else + acc[node.parentId] = [{ nodeId: node.id, nodeType: node.data.type }] + } + return acc + }, {} as Record) + + return nodes.map((node) => { + if (!node.type) + node.type = CUSTOM_NODE + + const connectedEdges = getConnectedEdges([node], edges) + node.data._connectedSourceHandleIds = connectedEdges.filter(edge => edge.source === node.id).map(edge => edge.sourceHandle || 'source') + node.data._connectedTargetHandleIds = connectedEdges.filter(edge => edge.target === node.id).map(edge => edge.targetHandle || 'target') + + if (node.data.type === BlockEnum.IfElse) { + const nodeData = node.data as IfElseNodeType + + if (!nodeData.cases && nodeData.logical_operator && nodeData.conditions) { + (node.data as IfElseNodeType).cases = [ + { + case_id: 'true', + logical_operator: nodeData.logical_operator, + conditions: nodeData.conditions, + }, + ] + } + node.data._targetBranches = branchNameCorrect([ + ...(node.data as IfElseNodeType).cases.map(item => ({ id: item.case_id, name: '' })), + { id: 'false', name: '' }, + ]) + } + + if (node.data.type === BlockEnum.QuestionClassifier) { + node.data._targetBranches = (node.data as QuestionClassifierNodeType).classes.map((topic) => { + return topic + }) + } + + if (node.data.type === BlockEnum.Iteration) { + const iterationNodeData = node.data as IterationNodeType + iterationNodeData._children = iterationOrLoopNodeMap[node.id] || [] + iterationNodeData.is_parallel = iterationNodeData.is_parallel || false + iterationNodeData.parallel_nums = iterationNodeData.parallel_nums || 10 + iterationNodeData.error_handle_mode = iterationNodeData.error_handle_mode || ErrorHandleMode.Terminated + } + + // TODO: loop error handle mode + if (node.data.type === BlockEnum.Loop) { + const loopNodeData = node.data as LoopNodeType + loopNodeData._children = iterationOrLoopNodeMap[node.id] || [] + loopNodeData.error_handle_mode = loopNodeData.error_handle_mode || ErrorHandleMode.Terminated + } + + // legacy provider handle + if (node.data.type === BlockEnum.LLM) + (node as any).data.model.provider = correctModelProvider((node as any).data.model.provider) + + if (node.data.type === BlockEnum.KnowledgeRetrieval && (node as any).data.multiple_retrieval_config?.reranking_model) + (node as any).data.multiple_retrieval_config.reranking_model.provider = correctModelProvider((node as any).data.multiple_retrieval_config?.reranking_model.provider) + + if (node.data.type === BlockEnum.QuestionClassifier) + (node as any).data.model.provider = correctModelProvider((node as any).data.model.provider) + + if (node.data.type === BlockEnum.ParameterExtractor) + (node as any).data.model.provider = correctModelProvider((node as any).data.model.provider) + if (node.data.type === BlockEnum.HttpRequest && !node.data.retry_config) { + node.data.retry_config = { + retry_enabled: true, + max_retries: DEFAULT_RETRY_MAX, + retry_interval: DEFAULT_RETRY_INTERVAL, + } + } + + return node + }) +} + +export const initialEdges = (originEdges: Edge[], originNodes: Node[]) => { + const { nodes, edges } = preprocessNodesAndEdges(cloneDeep(originNodes), cloneDeep(originEdges)) + let selectedNode: Node | null = null + const nodesMap = nodes.reduce((acc, node) => { + acc[node.id] = node + + if (node.data?.selected) + selectedNode = node + + return acc + }, {} as Record) + + const cycleEdges = getCycleEdges(nodes, edges) + return edges.filter((edge) => { + return !cycleEdges.find(cycEdge => cycEdge.source === edge.source && cycEdge.target === edge.target) + }).map((edge) => { + edge.type = 'custom' + + if (!edge.sourceHandle) + edge.sourceHandle = 'source' + + if (!edge.targetHandle) + edge.targetHandle = 'target' + + if (!edge.data?.sourceType && edge.source && nodesMap[edge.source]) { + edge.data = { + ...edge.data, + sourceType: nodesMap[edge.source].data.type!, + } as any + } + + if (!edge.data?.targetType && edge.target && nodesMap[edge.target]) { + edge.data = { + ...edge.data, + targetType: nodesMap[edge.target].data.type!, + } as any + } + + if (selectedNode) { + edge.data = { + ...edge.data, + _connectedNodeIsSelected: edge.source === selectedNode.id || edge.target === selectedNode.id, + } as any + } + + return edge + }) +} diff --git a/web/app/components/workflow/utils/workflow.ts b/web/app/components/workflow/utils/workflow.ts new file mode 100644 index 0000000000..88c31f09b5 --- /dev/null +++ b/web/app/components/workflow/utils/workflow.ts @@ -0,0 +1,329 @@ +import { + getConnectedEdges, + getIncomers, + getOutgoers, +} from 'reactflow' +import { v4 as uuid4 } from 'uuid' +import { + groupBy, + isEqual, + uniqBy, +} from 'lodash-es' +import type { + Edge, + Node, +} from '../types' +import { + BlockEnum, +} from '../types' +import type { IterationNodeType } from '../nodes/iteration/types' +import type { LoopNodeType } from '../nodes/loop/types' + +export const canRunBySingle = (nodeType: BlockEnum) => { + return nodeType === BlockEnum.LLM + || nodeType === BlockEnum.KnowledgeRetrieval + || nodeType === BlockEnum.Code + || nodeType === BlockEnum.TemplateTransform + || nodeType === BlockEnum.QuestionClassifier + || nodeType === BlockEnum.HttpRequest + || nodeType === BlockEnum.Tool + || nodeType === BlockEnum.ParameterExtractor + || nodeType === BlockEnum.Iteration + || nodeType === BlockEnum.Agent + || nodeType === BlockEnum.DocExtractor + || nodeType === BlockEnum.Loop +} + +type ConnectedSourceOrTargetNodesChange = { + type: string + edge: Edge +}[] +export const getNodesConnectedSourceOrTargetHandleIdsMap = (changes: ConnectedSourceOrTargetNodesChange, nodes: Node[]) => { + const nodesConnectedSourceOrTargetHandleIdsMap = {} as Record + + changes.forEach((change) => { + const { + edge, + type, + } = change + const sourceNode = nodes.find(node => node.id === edge.source)! + if (sourceNode) { + nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id] = nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id] || { + _connectedSourceHandleIds: [...(sourceNode?.data._connectedSourceHandleIds || [])], + _connectedTargetHandleIds: [...(sourceNode?.data._connectedTargetHandleIds || [])], + } + } + + const targetNode = nodes.find(node => node.id === edge.target)! + if (targetNode) { + nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id] = nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id] || { + _connectedSourceHandleIds: [...(targetNode?.data._connectedSourceHandleIds || [])], + _connectedTargetHandleIds: [...(targetNode?.data._connectedTargetHandleIds || [])], + } + } + + if (sourceNode) { + if (type === 'remove') { + const index = nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id]._connectedSourceHandleIds.findIndex((handleId: string) => handleId === edge.sourceHandle) + nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id]._connectedSourceHandleIds.splice(index, 1) + } + + if (type === 'add') + nodesConnectedSourceOrTargetHandleIdsMap[sourceNode.id]._connectedSourceHandleIds.push(edge.sourceHandle || 'source') + } + + if (targetNode) { + if (type === 'remove') { + const index = nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id]._connectedTargetHandleIds.findIndex((handleId: string) => handleId === edge.targetHandle) + nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id]._connectedTargetHandleIds.splice(index, 1) + } + + if (type === 'add') + nodesConnectedSourceOrTargetHandleIdsMap[targetNode.id]._connectedTargetHandleIds.push(edge.targetHandle || 'target') + } + }) + + return nodesConnectedSourceOrTargetHandleIdsMap +} + +export const getValidTreeNodes = (nodes: Node[], edges: Edge[]) => { + const startNode = nodes.find(node => node.data.type === BlockEnum.Start) + + if (!startNode) { + return { + validNodes: [], + maxDepth: 0, + } + } + + const list: Node[] = [startNode] + let maxDepth = 1 + + const traverse = (root: Node, depth: number) => { + if (depth > maxDepth) + maxDepth = depth + + const outgoers = getOutgoers(root, nodes, edges) + + if (outgoers.length) { + outgoers.forEach((outgoer) => { + list.push(outgoer) + + if (outgoer.data.type === BlockEnum.Iteration) + list.push(...nodes.filter(node => node.parentId === outgoer.id)) + if (outgoer.data.type === BlockEnum.Loop) + list.push(...nodes.filter(node => node.parentId === outgoer.id)) + + traverse(outgoer, depth + 1) + }) + } + else { + list.push(root) + + if (root.data.type === BlockEnum.Iteration) + list.push(...nodes.filter(node => node.parentId === root.id)) + if (root.data.type === BlockEnum.Loop) + list.push(...nodes.filter(node => node.parentId === root.id)) + } + } + + traverse(startNode, maxDepth) + + return { + validNodes: uniqBy(list, 'id'), + maxDepth, + } +} + +export const changeNodesAndEdgesId = (nodes: Node[], edges: Edge[]) => { + const idMap = nodes.reduce((acc, node) => { + acc[node.id] = uuid4() + + return acc + }, {} as Record) + + const newNodes = nodes.map((node) => { + return { + ...node, + id: idMap[node.id], + } + }) + + const newEdges = edges.map((edge) => { + return { + ...edge, + source: idMap[edge.source], + target: idMap[edge.target], + } + }) + + return [newNodes, newEdges] as [Node[], Edge[]] +} + +type ParallelInfoItem = { + parallelNodeId: string + depth: number + isBranch?: boolean +} +type NodeParallelInfo = { + parallelNodeId: string + edgeHandleId: string + depth: number +} +type NodeHandle = { + node: Node + handle: string +} +type NodeStreamInfo = { + upstreamNodes: Set + downstreamEdges: Set +} +export const getParallelInfo = (nodes: Node[], edges: Edge[], parentNodeId?: string) => { + let startNode + + if (parentNodeId) { + const parentNode = nodes.find(node => node.id === parentNodeId) + if (!parentNode) + throw new Error('Parent node not found') + + startNode = nodes.find(node => node.id === (parentNode.data as (IterationNodeType | LoopNodeType)).start_node_id) + } + else { + startNode = nodes.find(node => node.data.type === BlockEnum.Start) + } + if (!startNode) + throw new Error('Start node not found') + + const parallelList = [] as ParallelInfoItem[] + const nextNodeHandles = [{ node: startNode, handle: 'source' }] + let hasAbnormalEdges = false + + const traverse = (firstNodeHandle: NodeHandle) => { + const nodeEdgesSet = {} as Record> + const totalEdgesSet = new Set() + const nextHandles = [firstNodeHandle] + const streamInfo = {} as Record + const parallelListItem = { + parallelNodeId: '', + depth: 0, + } as ParallelInfoItem + const nodeParallelInfoMap = {} as Record + nodeParallelInfoMap[firstNodeHandle.node.id] = { + parallelNodeId: '', + edgeHandleId: '', + depth: 0, + } + + while (nextHandles.length) { + const currentNodeHandle = nextHandles.shift()! + const { node: currentNode, handle: currentHandle = 'source' } = currentNodeHandle + const currentNodeHandleKey = currentNode.id + const connectedEdges = edges.filter(edge => edge.source === currentNode.id && edge.sourceHandle === currentHandle) + const connectedEdgesLength = connectedEdges.length + const outgoers = nodes.filter(node => connectedEdges.some(edge => edge.target === node.id)) + const incomers = getIncomers(currentNode, nodes, edges) + + if (!streamInfo[currentNodeHandleKey]) { + streamInfo[currentNodeHandleKey] = { + upstreamNodes: new Set(), + downstreamEdges: new Set(), + } + } + + if (nodeEdgesSet[currentNodeHandleKey]?.size > 0 && incomers.length > 1) { + const newSet = new Set() + for (const item of totalEdgesSet) { + if (!streamInfo[currentNodeHandleKey].downstreamEdges.has(item)) + newSet.add(item) + } + if (isEqual(nodeEdgesSet[currentNodeHandleKey], newSet)) { + parallelListItem.depth = nodeParallelInfoMap[currentNode.id].depth + nextNodeHandles.push({ node: currentNode, handle: currentHandle }) + break + } + } + + if (nodeParallelInfoMap[currentNode.id].depth > parallelListItem.depth) + parallelListItem.depth = nodeParallelInfoMap[currentNode.id].depth + + outgoers.forEach((outgoer) => { + const outgoerConnectedEdges = getConnectedEdges([outgoer], edges).filter(edge => edge.source === outgoer.id) + const sourceEdgesGroup = groupBy(outgoerConnectedEdges, 'sourceHandle') + const incomers = getIncomers(outgoer, nodes, edges) + + if (outgoers.length > 1 && incomers.length > 1) + hasAbnormalEdges = true + + Object.keys(sourceEdgesGroup).forEach((sourceHandle) => { + nextHandles.push({ node: outgoer, handle: sourceHandle }) + }) + if (!outgoerConnectedEdges.length) + nextHandles.push({ node: outgoer, handle: 'source' }) + + const outgoerKey = outgoer.id + if (!nodeEdgesSet[outgoerKey]) + nodeEdgesSet[outgoerKey] = new Set() + + if (nodeEdgesSet[currentNodeHandleKey]) { + for (const item of nodeEdgesSet[currentNodeHandleKey]) + nodeEdgesSet[outgoerKey].add(item) + } + + if (!streamInfo[outgoerKey]) { + streamInfo[outgoerKey] = { + upstreamNodes: new Set(), + downstreamEdges: new Set(), + } + } + + if (!nodeParallelInfoMap[outgoer.id]) { + nodeParallelInfoMap[outgoer.id] = { + ...nodeParallelInfoMap[currentNode.id], + } + } + + if (connectedEdgesLength > 1) { + const edge = connectedEdges.find(edge => edge.target === outgoer.id)! + nodeEdgesSet[outgoerKey].add(edge.id) + totalEdgesSet.add(edge.id) + + streamInfo[currentNodeHandleKey].downstreamEdges.add(edge.id) + streamInfo[outgoerKey].upstreamNodes.add(currentNodeHandleKey) + + for (const item of streamInfo[currentNodeHandleKey].upstreamNodes) + streamInfo[item].downstreamEdges.add(edge.id) + + if (!parallelListItem.parallelNodeId) + parallelListItem.parallelNodeId = currentNode.id + + const prevDepth = nodeParallelInfoMap[currentNode.id].depth + 1 + const currentDepth = nodeParallelInfoMap[outgoer.id].depth + + nodeParallelInfoMap[outgoer.id].depth = Math.max(prevDepth, currentDepth) + } + else { + for (const item of streamInfo[currentNodeHandleKey].upstreamNodes) + streamInfo[outgoerKey].upstreamNodes.add(item) + + nodeParallelInfoMap[outgoer.id].depth = nodeParallelInfoMap[currentNode.id].depth + } + }) + } + + parallelList.push(parallelListItem) + } + + while (nextNodeHandles.length) { + const nodeHandle = nextNodeHandles.shift()! + traverse(nodeHandle) + } + + return { + parallelList, + hasAbnormalEdges, + } +} + +export const hasErrorHandleNode = (nodeType?: BlockEnum) => { + return nodeType === BlockEnum.LLM || nodeType === BlockEnum.Tool || nodeType === BlockEnum.HttpRequest || nodeType === BlockEnum.Code +} diff --git a/web/hooks/use-i18n.ts b/web/hooks/use-i18n.ts index d95ef0d114..c2356b12a8 100644 --- a/web/hooks/use-i18n.ts +++ b/web/hooks/use-i18n.ts @@ -1,11 +1,5 @@ import { useLanguage } from '@/app/components/header/account-setting/model-provider-page/hooks' - -export const renderI18nObject = (obj: Record, language: string) => { - if (!obj) return '' - if (obj?.[language]) return obj[language] - if (obj?.en_US) return obj.en_US - return Object.values(obj)[0] -} +import { renderI18nObject } from '@/i18n' export const useRenderI18nObject = () => { const language = useLanguage() diff --git a/web/i18n/index.ts b/web/i18n/index.ts index 1eec0f3589..6a0d82ea36 100644 --- a/web/i18n/index.ts +++ b/web/i18n/index.ts @@ -20,3 +20,10 @@ export const setLocaleOnClient = (locale: Locale, reloadPage = true) => { export const getLocaleOnClient = (): Locale => { return Cookies.get(LOCALE_COOKIE_NAME) as Locale || i18n.defaultLocale } + +export const renderI18nObject = (obj: Record, language: string) => { + if (!obj) return '' + if (obj?.[language]) return obj[language] + if (obj?.en_US) return obj.en_US + return Object.values(obj)[0] +} diff --git a/web/package.json b/web/package.json index b01466ded3..42240fc936 100644 --- a/web/package.json +++ b/web/package.json @@ -185,6 +185,7 @@ "husky": "^9.1.6", "jest": "^29.7.0", "lint-staged": "^15.2.10", + "lodash": "^4.17.21", "magicast": "^0.3.4", "postcss": "^8.4.47", "sass": "^1.80.3", diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index f02da2209d..828b34e521 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -63,7 +63,7 @@ importers: version: 0.18.0 '@mdx-js/loader': specifier: ^3.1.0 - version: 3.1.0(acorn@8.13.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3)) + version: 3.1.0(acorn@8.14.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3)) '@mdx-js/react': specifier: ^3.1.0 version: 3.1.0(@types/react@18.2.79)(react@19.0.0) @@ -72,7 +72,7 @@ importers: version: 4.6.0(monaco-editor@0.52.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) '@next/mdx': specifier: 15.2.3 - version: 15.2.3(@mdx-js/loader@3.1.0(acorn@8.13.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3)))(@mdx-js/react@3.1.0(@types/react@18.2.79)(react@19.0.0)) + version: 15.2.3(@mdx-js/loader@3.1.0(acorn@8.14.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3)))(@mdx-js/react@3.1.0(@types/react@18.2.79)(react@19.0.0)) '@octokit/core': specifier: ^6.1.2 version: 6.1.2 @@ -485,6 +485,9 @@ importers: lint-staged: specifier: ^15.2.10 version: 15.2.10 + lodash: + specifier: ^4.17.21 + version: 4.17.21 magicast: specifier: ^0.3.4 version: 0.3.5 @@ -10130,9 +10133,9 @@ snapshots: - supports-color optional: true - '@mdx-js/loader@3.1.0(acorn@8.13.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3))': + '@mdx-js/loader@3.1.0(acorn@8.14.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3))': dependencies: - '@mdx-js/mdx': 3.1.0(acorn@8.13.0) + '@mdx-js/mdx': 3.1.0(acorn@8.14.0) source-map: 0.7.4 optionalDependencies: webpack: 5.95.0(esbuild@0.23.1)(uglify-js@3.19.3) @@ -10140,7 +10143,7 @@ snapshots: - acorn - supports-color - '@mdx-js/mdx@3.1.0(acorn@8.13.0)': + '@mdx-js/mdx@3.1.0(acorn@8.14.0)': dependencies: '@types/estree': 1.0.6 '@types/estree-jsx': 1.0.5 @@ -10154,7 +10157,7 @@ snapshots: hast-util-to-jsx-runtime: 2.3.2 markdown-extensions: 2.0.0 recma-build-jsx: 1.0.0 - recma-jsx: 1.0.0(acorn@8.13.0) + recma-jsx: 1.0.0(acorn@8.14.0) recma-stringify: 1.0.0 rehype-recma: 1.0.0 remark-mdx: 3.1.0 @@ -10211,11 +10214,11 @@ snapshots: dependencies: fast-glob: 3.3.1 - '@next/mdx@15.2.3(@mdx-js/loader@3.1.0(acorn@8.13.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3)))(@mdx-js/react@3.1.0(@types/react@18.2.79)(react@19.0.0))': + '@next/mdx@15.2.3(@mdx-js/loader@3.1.0(acorn@8.14.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3)))(@mdx-js/react@3.1.0(@types/react@18.2.79)(react@19.0.0))': dependencies: source-map: 0.7.4 optionalDependencies: - '@mdx-js/loader': 3.1.0(acorn@8.13.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3)) + '@mdx-js/loader': 3.1.0(acorn@8.14.0)(webpack@5.95.0(esbuild@0.23.1)(uglify-js@3.19.3)) '@mdx-js/react': 3.1.0(@types/react@18.2.79)(react@19.0.0) '@next/swc-darwin-arm64@15.2.3': @@ -16765,9 +16768,9 @@ snapshots: estree-util-build-jsx: 3.0.1 vfile: 6.0.3 - recma-jsx@1.0.0(acorn@8.13.0): + recma-jsx@1.0.0(acorn@8.14.0): dependencies: - acorn-jsx: 5.3.2(acorn@8.13.0) + acorn-jsx: 5.3.2(acorn@8.14.0) estree-util-to-js: 2.0.0 recma-parse: 1.0.0 recma-stringify: 1.0.0 From 63ba6077387ce5b0c1d4dd58079f9e8596240816 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Thu, 10 Apr 2025 11:09:38 +0800 Subject: [PATCH 33/53] fix: 17712-get-messages-api-encountered-internal-server-error (#17716) --- api/controllers/service_api/app/message.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/api/controllers/service_api/app/message.py b/api/controllers/service_api/app/message.py index 38917bf345..95e538f4c7 100644 --- a/api/controllers/service_api/app/message.py +++ b/api/controllers/service_api/app/message.py @@ -20,14 +20,6 @@ from services.message_service import MessageService class MessageListApi(Resource): - def get_retriever_resources(self): - try: - if self.message_metadata: - return json.loads(self.message_metadata).get("retriever_resources", []) - return [] - except (json.JSONDecodeError, TypeError): - return [] - message_fields = { "id": fields.String, "conversation_id": fields.String, @@ -37,7 +29,11 @@ class MessageListApi(Resource): "answer": fields.String(attribute="re_sign_file_url_answer"), "message_files": fields.List(fields.Nested(message_file_fields)), "feedback": fields.Nested(feedback_fields, attribute="user_feedback", allow_null=True), - "retriever_resources": get_retriever_resources, + "retriever_resources": fields.Raw( + attribute=lambda obj: json.loads(obj.message_metadata).get("retriever_resources", []) + if obj.message_metadata + else [] + ), "created_at": TimestampField, "agent_thoughts": fields.List(fields.Nested(agent_thought_fields)), "status": fields.String, From 6df02152460e6ab4aface58007a0d4581df2986f Mon Sep 17 00:00:00 2001 From: GuanMu Date: Thu, 10 Apr 2025 11:12:34 +0800 Subject: [PATCH 34/53] fix: Enhance error handling and retry logic in Apps component (#17733) --- web/app/(commonLayout)/apps/Apps.tsx | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/web/app/(commonLayout)/apps/Apps.tsx b/web/app/(commonLayout)/apps/Apps.tsx index e72860861c..1375f4dfd6 100644 --- a/web/app/(commonLayout)/apps/Apps.tsx +++ b/web/app/(commonLayout)/apps/Apps.tsx @@ -74,10 +74,15 @@ const Apps = () => { setQuery(prev => ({ ...prev, tagIDs })) }, [setQuery]) - const { data, isLoading, setSize, mutate } = useSWRInfinite( + const { data, isLoading, error, setSize, mutate } = useSWRInfinite( (pageIndex: number, previousPageData: AppListResponse) => getKey(pageIndex, previousPageData, activeTab, isCreatedByMe, tagIDs, searchKeywords), fetchAppList, - { revalidateFirstPage: true }, + { + revalidateFirstPage: true, + shouldRetryOnError: false, + dedupingInterval: 500, + errorRetryCount: 3, + }, ) const anchorRef = useRef(null) @@ -106,15 +111,22 @@ const Apps = () => { useEffect(() => { const hasMore = data?.at(-1)?.has_more ?? true let observer: IntersectionObserver | undefined + + if (error) { + if (observer) + observer.disconnect() + return + } + if (anchorRef.current) { observer = new IntersectionObserver((entries) => { - if (entries[0].isIntersecting && !isLoading && hasMore) + if (entries[0].isIntersecting && !isLoading && !error && hasMore) setSize((size: number) => size + 1) }, { rootMargin: '100px' }) observer.observe(anchorRef.current) } return () => observer?.disconnect() - }, [isLoading, setSize, anchorRef, mutate, data]) + }, [isLoading, setSize, anchorRef, mutate, data, error]) const { run: handleSearch } = useDebounceFn(() => { setSearchKeywords(keywords) From 0e136b42a2211ffdfdaa8e3f959dc3e1ff42bf85 Mon Sep 17 00:00:00 2001 From: Qun <51054082+QunBB@users.noreply.github.com> Date: Thu, 10 Apr 2025 11:14:20 +0800 Subject: [PATCH 35/53] enhance guessing mimetype of tool file (#17640) --- api/core/tools/tool_file_manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py index 1573e0c219..7e8d4280d4 100644 --- a/api/core/tools/tool_file_manager.py +++ b/api/core/tools/tool_file_manager.py @@ -108,7 +108,11 @@ class ToolFileManager: except httpx.TimeoutException: raise ValueError(f"timeout when downloading file from {file_url}") - mimetype = guess_type(file_url)[0] or "application/octet-stream" + mimetype = ( + guess_type(file_url)[0] + or response.headers.get("Content-Type", "").split(";")[0].strip() + or "application/octet-stream" + ) extension = guess_extension(mimetype) or ".bin" unique_name = uuid4().hex filename = f"{unique_name}{extension}" From 63aab5cdd639202ca19d218b8c5e8e6f899c4e60 Mon Sep 17 00:00:00 2001 From: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Date: Thu, 10 Apr 2025 11:18:43 +0800 Subject: [PATCH 36/53] feat: add search params to url (#17684) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- web/app/(commonLayout)/plugins/page.tsx | 2 +- web/app/components/plugins/hooks.ts | 14 ++++++ .../plugins/marketplace/context.tsx | 43 ++++++++++++++++--- .../components/plugins/marketplace/index.tsx | 4 ++ .../marketplace/plugin-type-switch.tsx | 20 +++++++++ .../components/plugins/marketplace/utils.ts | 20 +++++++++ .../plugins/plugin-page/context.tsx | 15 ++----- .../components/plugins/plugin-page/index.tsx | 23 ++++++---- 8 files changed, 114 insertions(+), 27 deletions(-) diff --git a/web/app/(commonLayout)/plugins/page.tsx b/web/app/(commonLayout)/plugins/page.tsx index cc525992fa..47f2791075 100644 --- a/web/app/(commonLayout)/plugins/page.tsx +++ b/web/app/(commonLayout)/plugins/page.tsx @@ -8,7 +8,7 @@ const PluginList = async () => { return ( } - marketplace={} + marketplace={} /> ) } diff --git a/web/app/components/plugins/hooks.ts b/web/app/components/plugins/hooks.ts index f4b81d98c1..0349c46f9e 100644 --- a/web/app/components/plugins/hooks.ts +++ b/web/app/components/plugins/hooks.ts @@ -92,3 +92,17 @@ export const useSingleCategories = (translateFromOut?: TFunction) => { categoriesMap, } } + +export const PLUGIN_PAGE_TABS_MAP = { + plugins: 'plugins', + marketplace: 'discover', +} + +export const usePluginPageTabs = () => { + const { t } = useTranslation() + const tabs = [ + { value: PLUGIN_PAGE_TABS_MAP.plugins, text: t('common.menus.plugins') }, + { value: PLUGIN_PAGE_TABS_MAP.marketplace, text: t('common.menus.exploreMarketplace') }, + ] + return tabs +} diff --git a/web/app/components/plugins/marketplace/context.tsx b/web/app/components/plugins/marketplace/context.tsx index 53f57c0252..91621afaf8 100644 --- a/web/app/components/plugins/marketplace/context.tsx +++ b/web/app/components/plugins/marketplace/context.tsx @@ -35,9 +35,10 @@ import { import { getMarketplaceListCondition, getMarketplaceListFilterType, + updateSearchParams, } from './utils' import { useInstalledPluginList } from '@/service/use-plugins' -import { noop } from 'lodash-es' +import { debounce, noop } from 'lodash-es' export type MarketplaceContextValue = { intersected: boolean @@ -96,6 +97,7 @@ type MarketplaceContextProviderProps = { searchParams?: SearchParams shouldExclude?: boolean scrollContainerId?: string + showSearchParams?: boolean } export function useMarketplaceContext(selector: (value: MarketplaceContextValue) => any) { @@ -107,6 +109,7 @@ export const MarketplaceContextProvider = ({ searchParams, shouldExclude, scrollContainerId, + showSearchParams, }: MarketplaceContextProviderProps) => { const { data, isSuccess } = useInstalledPluginList(!shouldExclude) const exclude = useMemo(() => { @@ -159,7 +162,10 @@ export const MarketplaceContextProvider = ({ type: getMarketplaceListFilterType(activePluginTypeRef.current), page: pageRef.current, }) - history.pushState({}, '', `/${searchParams?.language ? `?language=${searchParams?.language}` : ''}`) + const url = new URL(window.location.href) + if (searchParams?.language) + url.searchParams.set('language', searchParams?.language) + history.replaceState({}, '', url) } else { if (shouldExclude && isSuccess) { @@ -182,7 +188,31 @@ export const MarketplaceContextProvider = ({ resetPlugins() }, [exclude, queryMarketplaceCollectionsAndPlugins, resetPlugins]) + const debouncedUpdateSearchParams = useMemo(() => debounce(() => { + updateSearchParams({ + query: searchPluginTextRef.current, + category: activePluginTypeRef.current, + tags: filterPluginTagsRef.current, + }) + }, 500), []) + + const handleUpdateSearchParams = useCallback((debounced?: boolean) => { + if (!showSearchParams) + return + if (debounced) { + debouncedUpdateSearchParams() + } + else { + updateSearchParams({ + query: searchPluginTextRef.current, + category: activePluginTypeRef.current, + tags: filterPluginTagsRef.current, + }) + } + }, [debouncedUpdateSearchParams, showSearchParams]) + const handleQueryPlugins = useCallback((debounced?: boolean) => { + handleUpdateSearchParams(debounced) if (debounced) { queryPluginsWithDebounced({ query: searchPluginTextRef.current, @@ -207,17 +237,18 @@ export const MarketplaceContextProvider = ({ page: pageRef.current, }) } - }, [exclude, queryPluginsWithDebounced, queryPlugins]) + }, [exclude, queryPluginsWithDebounced, queryPlugins, handleUpdateSearchParams]) const handleQuery = useCallback((debounced?: boolean) => { if (!searchPluginTextRef.current && !filterPluginTagsRef.current.length) { + handleUpdateSearchParams(debounced) cancelQueryPluginsWithDebounced() handleQueryMarketplaceCollectionsAndPlugins() return } handleQueryPlugins(debounced) - }, [handleQueryMarketplaceCollectionsAndPlugins, handleQueryPlugins, cancelQueryPluginsWithDebounced]) + }, [handleQueryMarketplaceCollectionsAndPlugins, handleQueryPlugins, cancelQueryPluginsWithDebounced, handleUpdateSearchParams]) const handleSearchPluginTextChange = useCallback((text: string) => { setSearchPluginText(text) @@ -242,11 +273,9 @@ export const MarketplaceContextProvider = ({ activePluginTypeRef.current = type setPage(1) pageRef.current = 1 - }, []) - useEffect(() => { handleQuery() - }, [activePluginType, handleQuery]) + }, [handleQuery]) const handleSortChange = useCallback((sort: PluginsSort) => { setSort(sort) diff --git a/web/app/components/plugins/marketplace/index.tsx b/web/app/components/plugins/marketplace/index.tsx index 5e6fbeec97..7a29556bda 100644 --- a/web/app/components/plugins/marketplace/index.tsx +++ b/web/app/components/plugins/marketplace/index.tsx @@ -17,6 +17,7 @@ type MarketplaceProps = { pluginTypeSwitchClassName?: string intersectionContainerId?: string scrollContainerId?: string + showSearchParams?: boolean } const Marketplace = async ({ locale, @@ -27,6 +28,7 @@ const Marketplace = async ({ pluginTypeSwitchClassName, intersectionContainerId, scrollContainerId, + showSearchParams = true, }: MarketplaceProps) => { let marketplaceCollections: any = [] let marketplaceCollectionPluginsMap = {} @@ -42,6 +44,7 @@ const Marketplace = async ({ searchParams={searchParams} shouldExclude={shouldExclude} scrollContainerId={scrollContainerId} + showSearchParams={showSearchParams} > @@ -53,6 +56,7 @@ const Marketplace = async ({ locale={locale} className={pluginTypeSwitchClassName} searchBoxAutoAnimate={searchBoxAutoAnimate} + showSearchParams={showSearchParams} /> { const { t } = useMixedTranslation(locale) const activePluginType = useMarketplaceContext(s => s.activePluginType) @@ -70,6 +73,23 @@ const PluginTypeSwitch = ({ }, ] + const handlePopState = useCallback(() => { + if (!showSearchParams) + return + const url = new URL(window.location.href) + const category = url.searchParams.get('category') || PLUGIN_TYPE_SEARCH_MAP.all + handleActivePluginTypeChange(category) + }, [showSearchParams, handleActivePluginTypeChange]) + + useEffect(() => { + window.addEventListener('popstate', () => { + handlePopState() + }) + return () => { + window.removeEventListener('popstate', handlePopState) + } + }, [handlePopState]) + return (
{ return 'plugin' } + +export const updateSearchParams = (pluginsSearchParams: PluginsSearchParams) => { + const { query, category, tags } = pluginsSearchParams + const url = new URL(window.location.href) + const categoryChanged = url.searchParams.get('category') !== category + if (query) + url.searchParams.set('q', query) + else + url.searchParams.delete('q') + if (category) + url.searchParams.set('category', category) + else + url.searchParams.delete('category') + if (tags && tags.length) + url.searchParams.set('tags', tags.join(',')) + else + url.searchParams.delete('tags') + history[`${categoryChanged ? 'pushState' : 'replaceState'}`]({}, '', url) +} diff --git a/web/app/components/plugins/plugin-page/context.tsx b/web/app/components/plugins/plugin-page/context.tsx index cf26cd4e08..ae1ad7d053 100644 --- a/web/app/components/plugins/plugin-page/context.tsx +++ b/web/app/components/plugins/plugin-page/context.tsx @@ -12,9 +12,9 @@ import { } from 'use-context-selector' import { useSelector as useAppContextSelector } from '@/context/app-context' import type { FilterState } from './filter-management' -import { useTranslation } from 'react-i18next' import { useTabSearchParams } from '@/hooks/use-tab-searchparams' import { noop } from 'lodash-es' +import { PLUGIN_PAGE_TABS_MAP, usePluginPageTabs } from '../hooks' export type PluginPageContextValue = { containerRef: React.RefObject @@ -53,7 +53,6 @@ export function usePluginPageContext(selector: (value: PluginPageContextValue) = export const PluginPageContextProvider = ({ children, }: PluginPageContextProviderProps) => { - const { t } = useTranslation() const containerRef = useRef(null) const [filters, setFilters] = useState({ categories: [], @@ -63,16 +62,10 @@ export const PluginPageContextProvider = ({ const [currentPluginID, setCurrentPluginID] = useState() const { enable_marketplace } = useAppContextSelector(s => s.systemFeatures) + const tabs = usePluginPageTabs() const options = useMemo(() => { - return [ - { value: 'plugins', text: t('common.menus.plugins') }, - ...( - enable_marketplace - ? [{ value: 'discover', text: t('common.menus.exploreMarketplace') }] - : [] - ), - ] - }, [t, enable_marketplace]) + return enable_marketplace ? tabs : tabs.filter(tab => tab.value !== PLUGIN_PAGE_TABS_MAP.marketplace) + }, [tabs, enable_marketplace]) const [activeTab, setActiveTab] = useTabSearchParams({ defaultTab: options[0].value, }) diff --git a/web/app/components/plugins/plugin-page/index.tsx b/web/app/components/plugins/plugin-page/index.tsx index 801eaf6607..072b8ee22f 100644 --- a/web/app/components/plugins/plugin-page/index.tsx +++ b/web/app/components/plugins/plugin-page/index.tsx @@ -40,6 +40,8 @@ import { SUPPORT_INSTALL_LOCAL_FILE_EXTENSIONS } from '@/config' import { LanguagesSupported } from '@/i18n/language' import I18n from '@/context/i18n' import { noop } from 'lodash-es' +import { PLUGIN_TYPE_SEARCH_MAP } from '../marketplace/plugin-type-switch' +import { PLUGIN_PAGE_TABS_MAP } from '../hooks' const PACKAGE_IDS_KEY = 'package-ids' const BUNDLE_INFO_KEY = 'bundle-info' @@ -136,40 +138,45 @@ const PluginPage = ({ const setActiveTab = usePluginPageContext(v => v.setActiveTab) const { enable_marketplace } = useAppContextSelector(s => s.systemFeatures) + const isPluginsTab = useMemo(() => activeTab === PLUGIN_PAGE_TABS_MAP.plugins, [activeTab]) + const isExploringMarketplace = useMemo(() => { + const values = Object.values(PLUGIN_TYPE_SEARCH_MAP) + return activeTab === PLUGIN_PAGE_TABS_MAP.marketplace || values.includes(activeTab) + }, [activeTab]) + const uploaderProps = useUploader({ onFileChange: setCurrentFile, containerRef, - enabled: activeTab === 'plugins', + enabled: isPluginsTab, }) const { dragging, fileUploader, fileChangeHandle, removeFile } = uploaderProps - return (
{ - activeTab === 'discover' && ( + isExploringMarketplace && ( <>
- {activeTab === 'plugins' && ( + {isPluginsTab && ( <> {plugins} {dragging && ( @@ -246,7 +253,7 @@ const PluginPage = ({ )} { - activeTab === 'discover' && enable_marketplace && marketplace + isExploringMarketplace && enable_marketplace && marketplace } {showPluginSettingModal && ( From ef27942b8a50d7453e4361d83f7a75ac04128487 Mon Sep 17 00:00:00 2001 From: Hanqing Zhao Date: Thu, 10 Apr 2025 13:56:36 +0800 Subject: [PATCH 37/53] Add and modify jp translation (#17748) --- web/i18n/ja-JP/workflow.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/web/i18n/ja-JP/workflow.ts b/web/i18n/ja-JP/workflow.ts index f27a2f3c74..c4a36ccdc9 100644 --- a/web/i18n/ja-JP/workflow.ts +++ b/web/i18n/ja-JP/workflow.ts @@ -249,6 +249,7 @@ const translation = { 'agent': 'エージェント', 'loop-start': 'ループ開始', 'loop': 'ループ', + 'loop-end': 'ループ完了', }, blocksAbout: { 'start': 'ワークフロー開始時の初期パラメータを定義します。', @@ -266,6 +267,7 @@ const translation = { 'variable-aggregator': '複数分岐の変数を集約し、下流ノードの設定を統一します。', 'iteration': 'リスト要素に対して反復処理を実行し全結果を出力します。', 'loop': '終了条件達成まで、または最大反復回数までロジックを繰り返します。', + 'loop-end': '「break」相当の機能です。このノードに設定項目はなく、ループ処理中にこのノードに到達すると即時終了します。', 'parameter-extractor': '自然言語から構造化パラメータを抽出し、後続処理で利用します。', 'document-extractor': 'アップロード文書をLLM処理用に最適化されたテキストに変換します。', 'list-operator': '配列のフィルタリングやソート処理を行います。', @@ -713,6 +715,16 @@ const translation = { continueOnError: 'エラーを無視して継続', removeAbnormalOutput: '異常出力を除外', }, + loopVariables: 'ループ変数', + initialLoopVariables: '初期ループ変数', + finalLoopVariables: '最終ループ変数', + setLoopVariables: 'ループスコープ内で変数を設定', + variableName: '変数名', + inputMode: '入力モード', + exitConditionTip: 'ループノードには少なくとも1つの終了条件が必要です', + loopNode: 'ループノード', + currentLoopCount: '現在のループ回数: {{count}}', + totalLoopCount: '総ループ回数: {{count}}', }, note: { addNote: 'コメントを追加', From 88cb81d3d6bb22a96b47f5c8ddf86562c48874ac Mon Sep 17 00:00:00 2001 From: Panpan Date: Thu, 10 Apr 2025 13:58:35 +0800 Subject: [PATCH 38/53] fix: fix inputs lost (#17747) --- web/app/components/base/chat/chat-with-history/hooks.tsx | 2 +- web/app/components/base/chat/embedded-chatbot/hooks.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/components/base/chat/chat-with-history/hooks.tsx b/web/app/components/base/chat/chat-with-history/hooks.tsx index 88f6c8f616..0a4cbae964 100644 --- a/web/app/components/base/chat/chat-with-history/hooks.tsx +++ b/web/app/components/base/chat/chat-with-history/hooks.tsx @@ -266,7 +266,7 @@ export const useChatWithHistory = (installedAppInfo?: InstalledApp) => { const currentConversationLatestInputs = useMemo(() => { if (!currentConversationId || !appChatListData?.data.length) - return {} + return newConversationInputsRef.current || {} return appChatListData.data.slice().pop().inputs || {} }, [appChatListData, currentConversationId]) const [currentConversationInputs, setCurrentConversationInputs] = useState>(currentConversationLatestInputs || {}) diff --git a/web/app/components/base/chat/embedded-chatbot/hooks.tsx b/web/app/components/base/chat/embedded-chatbot/hooks.tsx index 197aa7649c..a5665ab346 100644 --- a/web/app/components/base/chat/embedded-chatbot/hooks.tsx +++ b/web/app/components/base/chat/embedded-chatbot/hooks.tsx @@ -242,7 +242,7 @@ export const useEmbeddedChatbot = () => { const currentConversationLatestInputs = useMemo(() => { if (!currentConversationId || !appChatListData?.data.length) - return {} + return newConversationInputsRef.current || {} return appChatListData.data.slice().pop().inputs || {} }, [appChatListData, currentConversationId]) const [currentConversationInputs, setCurrentConversationInputs] = useState>(currentConversationLatestInputs || {}) From d0d02be7119da789c4005b1225cfb7369f5c3460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AF=97=E6=B5=93?= <844670992@qq.com> Date: Thu, 10 Apr 2025 14:58:39 +0800 Subject: [PATCH 39/53] feat: add consistent keyboard shortcut support and visual indicators across all app creation dialogs (#17138) --- .../app/create-app-dialog/index.tsx | 9 +++++ .../app/create-from-dsl-modal/index.tsx | 28 +++++++++++++-- .../explore/create-app-modal/index.tsx | 34 ++++++++++++++++--- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/web/app/components/app/create-app-dialog/index.tsx b/web/app/components/app/create-app-dialog/index.tsx index acc3650211..794bbbf9e8 100644 --- a/web/app/components/app/create-app-dialog/index.tsx +++ b/web/app/components/app/create-app-dialog/index.tsx @@ -1,4 +1,6 @@ 'use client' +import { useCallback } from 'react' +import { useKeyPress } from 'ahooks' import AppList from './app-list' import FullScreenModal from '@/app/components/base/fullscreen-modal' @@ -10,6 +12,13 @@ type CreateAppDialogProps = { } const CreateAppTemplateDialog = ({ show, onSuccess, onClose, onCreateFromBlank }: CreateAppDialogProps) => { + const handleEscKeyPress = useCallback(() => { + if (show) + onClose() + }, [show, onClose]) + + useKeyPress('esc', handleEscKeyPress) + return ( { + if (show && !isAppsFull && ((currentTab === CreateFromDSLModalTab.FROM_FILE && currentFile) || (currentTab === CreateFromDSLModalTab.FROM_URL && dslUrlValue))) + handleCreateApp() + }) + + useKeyPress('esc', () => { + if (show && !showErrorModal) + onClose() + }) + const onDSLConfirm: MouseEventHandler = async () => { try { if (!importId) @@ -266,7 +279,18 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS )}
- +
= plan.total.buildApps) - const submit = () => { + const submit = useCallback(() => { if (!name.trim()) { Toast.notify({ type: 'error', message: t('explore.appCustomize.nameRequired') }) return @@ -80,7 +81,19 @@ const CreateAppModal = ({ use_icon_as_answer_icon: useIconAsAnswerIcon, }) onHide() - } + }, [name, appIcon, description, useIconAsAnswerIcon, onConfirm, onHide, t]) + + const { run: handleSubmit } = useDebounceFn(submit, { wait: 300 }) + + useKeyPress(['meta.enter', 'ctrl.enter'], () => { + if (show && !(!isEditModal && isAppsFull) && name.trim()) + handleSubmit() + }) + + useKeyPress('esc', () => { + if (show) + onHide() + }) return ( <> @@ -146,7 +159,18 @@ const CreateAppModal = ({ {!isEditModal && isAppsFull && }
- +
From 29720b7360b82c2ff549bd0369be2f9c857e0142 Mon Sep 17 00:00:00 2001 From: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:53:50 +0800 Subject: [PATCH 40/53] fix: adjust spacing in ViewHistory and Panel components (#17766) --- web/app/components/workflow/header/view-history.tsx | 2 +- web/app/components/workflow/nodes/variable-assigner/panel.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/components/workflow/header/view-history.tsx b/web/app/components/workflow/header/view-history.tsx index 48053c8336..e56731952c 100644 --- a/web/app/components/workflow/header/view-history.tsx +++ b/web/app/components/workflow/header/view-history.tsx @@ -199,7 +199,7 @@ const ViewHistory = ({ item.id === historyWorkflowData?.id && 'text-primary-600', )} > - {`Test ${isChatMode ? 'Chat' : 'Run'}#${item.sequence_number}`} + {`Test ${isChatMode ? 'Chat' : 'Run'} #${item.sequence_number}`}
{item.created_by_account?.name} · {formatTimeFromNow((item.finished_at || item.created_at) * 1000)} diff --git a/web/app/components/workflow/nodes/variable-assigner/panel.tsx b/web/app/components/workflow/nodes/variable-assigner/panel.tsx index 57ecfbb1ce..1655c21b1f 100644 --- a/web/app/components/workflow/nodes/variable-assigner/panel.tsx +++ b/web/app/components/workflow/nodes/variable-assigner/panel.tsx @@ -99,7 +99,7 @@ const Panel: FC> = ({ {isEnableGroup && ( <> -
+
<> {inputs.advanced_settings?.groups.map((item, index) => ( From 636a0ba37f703b8c9acc5b39e1e0941a3aa60d3e Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Thu, 10 Apr 2025 17:12:48 +0800 Subject: [PATCH 41/53] chore: skip document segments fetching with non-existed dataset of DatasetDocument in add_document_to_index_task task (#17784) --- api/tasks/add_document_to_index_task.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/api/tasks/add_document_to_index_task.py b/api/tasks/add_document_to_index_task.py index 0b7d2ad31f..be88881efc 100644 --- a/api/tasks/add_document_to_index_task.py +++ b/api/tasks/add_document_to_index_task.py @@ -37,6 +37,10 @@ def add_document_to_index_task(dataset_document_id: str): indexing_cache_key = "document_{}_indexing".format(dataset_document.id) try: + dataset = dataset_document.dataset + if not dataset: + raise Exception(f"Document {dataset_document.id} dataset {dataset_document.dataset_id} doesn't exist.") + segments = ( db.session.query(DocumentSegment) .filter( @@ -77,11 +81,6 @@ def add_document_to_index_task(dataset_document_id: str): document.children = child_documents documents.append(document) - dataset = dataset_document.dataset - - if not dataset: - raise Exception("Document has no dataset") - index_type = dataset.doc_form index_processor = IndexProcessorFactory(index_type).init_index_processor() index_processor.load(dataset, documents) From 17a26da1e60221dfaef5b934b0f0e8baf879cdd7 Mon Sep 17 00:00:00 2001 From: KVOJJJin Date: Thu, 10 Apr 2025 17:15:48 +0800 Subject: [PATCH 42/53] Feat: workflow dark mode (#17785) --- .../config-var/config-select/index.tsx | 45 ++++++++++-------- .../config-var/config-select/style.module.css | 21 --------- web/app/components/base/tag-input/index.tsx | 2 +- .../model-parameter-modal/trigger.tsx | 2 +- .../workflow/block-selector/index.tsx | 2 +- .../components/workflow/header/checklist.tsx | 26 +++++------ .../workflow/header/view-history.tsx | 26 +++++------ .../_base/components/add-variable-popup.tsx | 4 +- .../components/before-run-form/index.tsx | 16 +++---- .../components/editor/code-editor/index.tsx | 46 ++++--------------- .../components/editor/code-editor/style.css | 11 +++++ .../error-handle/fail-branch-card.tsx | 2 +- .../components/input-support-select-var.tsx | 2 +- .../nodes/_base/components/memory-config.tsx | 8 ++-- .../nodes/_base/components/output-vars.tsx | 2 +- .../panel-operator/change-block.tsx | 2 +- .../panel-operator/panel-operator-popup.tsx | 28 +++++------ .../nodes/_base/components/prompt/editor.tsx | 17 ++++--- .../readonly-input-with-select-var.tsx | 10 ++-- .../nodes/_base/components/remove-button.tsx | 12 ++--- .../_base/components/retry/retry-on-panel.tsx | 4 +- .../nodes/_base/components/selector.tsx | 10 ++-- .../components/title-description-input.tsx | 6 +-- .../nodes/_base/components/variable-tag.tsx | 4 +- .../_base/components/variable/var-list.tsx | 5 +- .../variable/var-reference-picker.tsx | 16 +++---- .../variable/var-reference-popup.tsx | 2 +- .../components/variable/var-type-picker.tsx | 10 ++-- .../nodes/_base/hooks/use-toggle-expend.ts | 4 +- .../components/workflow/nodes/_base/panel.tsx | 10 ++-- .../components/workflow/nodes/end/node.tsx | 16 +++---- .../nodes/http/components/api-input.tsx | 8 ++-- .../http/components/authorization/index.tsx | 6 +-- .../components/authorization/radio-group.tsx | 7 +-- .../nodes/http/components/curl-panel.tsx | 5 +- .../nodes/http/components/edit-body/index.tsx | 2 +- .../key-value/key-value-edit/input-item.tsx | 8 ++-- .../key-value/key-value-edit/item.tsx | 6 +-- .../nodes/http/components/timeout/index.tsx | 4 +- .../components/workflow/nodes/http/node.tsx | 5 +- .../components/workflow/nodes/http/panel.tsx | 14 +++--- .../workflow/nodes/iteration-start/index.tsx | 2 +- .../components/dataset-item.tsx | 2 +- .../components/dataset-list.tsx | 2 +- .../components/retrieval-config.tsx | 2 +- .../nodes/knowledge-retrieval/panel.tsx | 2 +- .../components/extract-input.tsx | 2 +- .../llm/components/config-prompt-item.tsx | 6 +-- .../nodes/llm/components/config-prompt.tsx | 2 +- .../llm/components/resolution-picker.tsx | 2 +- .../components/workflow/nodes/llm/panel.tsx | 8 ++-- .../workflow/nodes/loop-start/index.tsx | 4 +- .../workflow/nodes/loop/add-block.tsx | 6 +-- .../components/workflow/nodes/loop/node.tsx | 4 +- .../extract-parameter/import-from-tool.tsx | 4 +- .../components/extract-parameter/item.tsx | 25 +++++----- .../components/extract-parameter/update.tsx | 4 +- .../nodes/parameter-extractor/panel.tsx | 2 +- .../nodes/start/components/var-item.tsx | 22 ++++----- .../nodes/start/components/var-list.tsx | 2 +- .../components/workflow/nodes/start/node.tsx | 10 ++-- .../components/workflow/nodes/start/panel.tsx | 16 +++---- .../nodes/template-transform/panel.tsx | 4 +- .../components/workflow/nodes/tool/node.tsx | 10 ++-- .../components/workflow/nodes/tool/panel.tsx | 4 +- .../components/add-variable/index.tsx | 6 +-- .../components/node-group-item.tsx | 12 ++--- .../components/node-variable-item.tsx | 8 ++-- .../components/var-group-item.tsx | 2 +- .../components/var-list/index.tsx | 1 - .../nodes/variable-assigner/panel.tsx | 30 ++++++------ .../workflow/panel/chat-record/index.tsx | 14 +++--- .../workflow/panel/chat-record/user-input.tsx | 8 ++-- .../panel/debug-and-preview/index.tsx | 4 +- .../workflow/panel/workflow-preview.tsx | 24 +++++----- 75 files changed, 324 insertions(+), 368 deletions(-) delete mode 100644 web/app/components/app/configuration/config-var/config-select/style.module.css diff --git a/web/app/components/app/configuration/config-var/config-select/index.tsx b/web/app/components/app/configuration/config-var/config-select/index.tsx index 0e4256c691..d2dc1662c1 100644 --- a/web/app/components/app/configuration/config-var/config-select/index.tsx +++ b/web/app/components/app/configuration/config-var/config-select/index.tsx @@ -1,12 +1,10 @@ 'use client' import type { FC } from 'react' -import React from 'react' +import React, { useState } from 'react' +import { RiAddLine, RiDeleteBinLine, RiDraggable } from '@remixicon/react' import { useTranslation } from 'react-i18next' -import { PlusIcon } from '@heroicons/react/24/outline' import { ReactSortable } from 'react-sortablejs' -import RemoveIcon from '../../base/icons/remove-icon' - -import s from './style.module.css' +import cn from '@/utils/classnames' export type Options = string[] export type IConfigSelectProps = { @@ -19,6 +17,8 @@ const ConfigSelect: FC = ({ onChange, }) => { const { t } = useTranslation() + const [focusID, setFocusID] = useState(null) + const [deletingID, setDeletingID] = useState(null) const optionList = options.map((content, index) => { return ({ @@ -40,12 +40,15 @@ const ConfigSelect: FC = ({ animation={150} > {options.map((o, index) => ( -
-
- - - -
+
+ = ({ return item })) }} - className={'h-9 w-full grow cursor-pointer border-0 bg-transparent pl-1.5 pr-8 text-sm leading-9 text-gray-900 focus:outline-none'} + className={'h-9 w-full grow cursor-pointer overflow-x-auto rounded-lg border-0 bg-transparent pl-1.5 pr-8 text-sm leading-9 text-text-secondary focus:outline-none'} + onFocus={() => setFocusID(index)} + onBlur={() => setFocusID(null)} /> - { onChange(options.filter((_, i) => index !== i)) }} - /> + onMouseEnter={() => setDeletingID(index)} + onMouseLeave={() => setDeletingID(null)} + > + +
))} @@ -75,9 +84,9 @@ const ConfigSelect: FC = ({
{ onChange([...options, '']) }} - className='flex h-9 cursor-pointer items-center gap-2 rounded-lg bg-gray-100 px-3 text-gray-400'> - -
{t('appDebug.variableConfig.addOption')}
+ className='mt-1 flex h-9 cursor-pointer items-center gap-2 rounded-lg bg-components-button-tertiary-bg px-3 text-components-button-tertiary-text hover:bg-components-button-tertiary-bg-hover'> + +
{t('appDebug.variableConfig.addOption')}
) diff --git a/web/app/components/app/configuration/config-var/config-select/style.module.css b/web/app/components/app/configuration/config-var/config-select/style.module.css deleted file mode 100644 index a09d19537d..0000000000 --- a/web/app/components/app/configuration/config-var/config-select/style.module.css +++ /dev/null @@ -1,21 +0,0 @@ -.inputWrap { - display: flex; - align-items: center; - border-radius: 8px; - border: 1px solid #EAECF0; - padding-left: 10px; - cursor: pointer; -} - -.deleteBtn { - display: none; - display: flex; -} - -.inputWrap:hover { - box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05); -} - -.inputWrap:hover .deleteBtn { - display: flex; -} \ No newline at end of file diff --git a/web/app/components/base/tag-input/index.tsx b/web/app/components/base/tag-input/index.tsx index f3df585d48..2be9c5ffc7 100644 --- a/web/app/components/base/tag-input/index.tsx +++ b/web/app/components/base/tag-input/index.tsx @@ -70,7 +70,7 @@ const TagInput: FC = ({ } return ( -
+
{ (items || []).map((item, index) => (
= ({ 'relative flex h-8 cursor-pointer items-center rounded-lg px-2', !isInWorkflow && 'border ring-inset hover:ring-[0.5px]', !isInWorkflow && (disabled ? 'border-text-warning bg-state-warning-hover ring-text-warning' : 'border-util-colors-indigo-indigo-600 bg-state-accent-hover ring-util-colors-indigo-indigo-600'), - isInWorkflow && 'border border-workflow-block-parma-bg bg-workflow-block-parma-bg pr-[30px] hover:border-gray-200', + isInWorkflow && 'border border-workflow-block-parma-bg bg-workflow-block-parma-bg pr-[30px] hover:border-components-input-border-active', )} > { diff --git a/web/app/components/workflow/block-selector/index.tsx b/web/app/components/workflow/block-selector/index.tsx index 263ae0c227..f0f57adefe 100644 --- a/web/app/components/workflow/block-selector/index.tsx +++ b/web/app/components/workflow/block-selector/index.tsx @@ -128,7 +128,7 @@ const NodeSelector: FC = ({ } -
+
e.stopPropagation()}> {activeTab === TabsEnum.Blocks && (
-
+
{t('workflow.panel.checklist')}{needWarningNodes.length ? `(${needWarningNodes.length})` : ''}
setOpen(false)} > - +
{ !!needWarningNodes.length && ( <> -
{t('workflow.panel.checklistTip')}
+
{t('workflow.panel.checklistTip')}
{ needWarningNodes.map(node => (
{ handleNodeSelect(node.id) setOpen(false) }} > -
+
-
+
{ node.unConnected && ( -
-
+
+
{t('workflow.common.needConnectTip')}
@@ -132,8 +132,8 @@ const WorkflowChecklist = ({ } { node.errorMessage && ( -
-
+
+
{node.errorMessage}
@@ -150,8 +150,8 @@ const WorkflowChecklist = ({ } { !needWarningNodes.length && ( -
- +
+ {t('workflow.panel.checklistResolved')}
) diff --git a/web/app/components/workflow/header/view-history.tsx b/web/app/components/workflow/header/view-history.tsx index e56731952c..1298c0e42d 100644 --- a/web/app/components/workflow/header/view-history.tsx +++ b/web/app/components/workflow/header/view-history.tsx @@ -87,9 +87,9 @@ const ViewHistory = ({ { withText && (
-
+
{t('workflow.common.runHistory')}
- +
{ @@ -149,8 +149,8 @@ const ViewHistory = ({ { !data?.data.length && (
- -
+ +
{t('workflow.common.notRunning')}
@@ -161,8 +161,8 @@ const ViewHistory = ({
{ workflowStore.setState({ @@ -195,13 +195,13 @@ const ViewHistory = ({
{`Test ${isChatMode ? 'Chat' : 'Run'} #${item.sequence_number}`}
-
+
{item.created_by_account?.name} · {formatTimeFromNow((item.finished_at || item.created_at) * 1000)}
diff --git a/web/app/components/workflow/nodes/_base/components/add-variable-popup.tsx b/web/app/components/workflow/nodes/_base/components/add-variable-popup.tsx index b442cf64a4..a7a3a89676 100644 --- a/web/app/components/workflow/nodes/_base/components/add-variable-popup.tsx +++ b/web/app/components/workflow/nodes/_base/components/add-variable-popup.tsx @@ -18,8 +18,8 @@ export const AddVariablePopup = ({ const { t } = useTranslation() return ( -
-
+
+
{t('workflow.nodes.variableAssigner.setAssignVariable')}
diff --git a/web/app/components/workflow/nodes/_base/components/before-run-form/index.tsx b/web/app/components/workflow/nodes/_base/components/before-run-form/index.tsx index 6fa5264277..ef4ba15c5c 100644 --- a/web/app/components/workflow/nodes/_base/components/before-run-form/index.tsx +++ b/web/app/components/workflow/nodes/_base/components/before-run-form/index.tsx @@ -139,18 +139,16 @@ const BeforeRunForm: FC = ({ onRun(submitData) }, [forms, onRun, t]) return ( -
-
+
+
-
+
{t(`${i18nPrefix}.testRun`)} {nodeName}
{ onHide() }}> - +
{ @@ -178,14 +176,14 @@ const BeforeRunForm: FC = ({
{isRunning && (
- +
)}
diff --git a/web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx b/web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx index 8aacfba64b..5a4a35ff8a 100644 --- a/web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx +++ b/web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx @@ -8,6 +8,8 @@ import { CodeLanguage } from '@/app/components/workflow/nodes/code/types' import { getFilesInLogs, } from '@/app/components/base/file-uploader/utils' +import { Theme } from '@/types/app' +import useTheme from '@/hooks/use-theme' import './style.css' import { noop } from 'lodash-es' @@ -43,15 +45,6 @@ export const languageMap = { [CodeLanguage.json]: 'json', } -const DEFAULT_THEME = { - base: 'vs', - inherit: true, - rules: [], - colors: { - 'editor.background': '#F2F4F7', // #00000000 transparent. But it will has a blue border - }, -} - const CodeEditor: FC = ({ value = '', placeholder = '', @@ -76,7 +69,7 @@ const CodeEditor: FC = ({ const [isMounted, setIsMounted] = React.useState(false) const minHeight = height || 200 const [editorContentHeight, setEditorContentHeight] = useState(56) - + const { theme: appTheme } = useTheme() const valueRef = useRef(value) useEffect(() => { valueRef.current = value @@ -114,27 +107,7 @@ const CodeEditor: FC = ({ setIsFocus(false) }) - monaco.editor.defineTheme('default-theme', DEFAULT_THEME) - - monaco.editor.defineTheme('blur-theme', { - base: 'vs', - inherit: true, - rules: [], - colors: { - 'editor.background': '#F2F4F7', - }, - }) - - monaco.editor.defineTheme('focus-theme', { - base: 'vs', - inherit: true, - rules: [], - colors: { - 'editor.background': '#ffffff', - }, - }) - - monaco.editor.setTheme('default-theme') // Fix: sometimes not load the default theme + monaco.editor.setTheme(appTheme === Theme.light ? 'light' : 'vs-dark') // Fix: sometimes not load the default theme onMount?.(editor, monaco) setIsMounted(true) @@ -151,12 +124,11 @@ const CodeEditor: FC = ({ } })() - const theme = (() => { - if (noWrapper) - return 'default-theme' - - return isFocus ? 'focus-theme' : 'blur-theme' - })() + const theme = useMemo(() => { + if (appTheme === Theme.light) + return 'light' + return 'vs-dark' + }, [appTheme]) const main = ( <> diff --git a/web/app/components/workflow/nodes/_base/components/editor/code-editor/style.css b/web/app/components/workflow/nodes/_base/components/editor/code-editor/style.css index 3a6624267a..296ea0ab14 100644 --- a/web/app/components/workflow/nodes/_base/components/editor/code-editor/style.css +++ b/web/app/components/workflow/nodes/_base/components/editor/code-editor/style.css @@ -6,6 +6,17 @@ padding-left: 0; } +.monaco-editor { + background-color: transparent !important; + outline: none !important; +} +.monaco-editor .monaco-editor-background { + background-color: transparent !important; +} +.monaco-editor .margin { + background-color: transparent !important; +} + /* hide readonly tooltip */ .monaco-editor-overlaymessage { display: none !important; diff --git a/web/app/components/workflow/nodes/_base/components/error-handle/fail-branch-card.tsx b/web/app/components/workflow/nodes/_base/components/error-handle/fail-branch-card.tsx index ec5c9e67ca..05a6cb96af 100644 --- a/web/app/components/workflow/nodes/_base/components/error-handle/fail-branch-card.tsx +++ b/web/app/components/workflow/nodes/_base/components/error-handle/fail-branch-card.tsx @@ -7,7 +7,7 @@ const FailBranchCard = () => { return (
-
+
diff --git a/web/app/components/workflow/nodes/_base/components/input-support-select-var.tsx b/web/app/components/workflow/nodes/_base/components/input-support-select-var.tsx index 33e13637d4..aab14bb6f9 100644 --- a/web/app/components/workflow/nodes/_base/components/input-support-select-var.tsx +++ b/web/app/components/workflow/nodes/_base/components/input-support-select-var.tsx @@ -113,7 +113,7 @@ const Editor: FC = ({ -
+
diff --git a/web/app/components/workflow/nodes/_base/components/memory-config.tsx b/web/app/components/workflow/nodes/_base/components/memory-config.tsx index 0ae6f2c92c..446fcfa8ae 100644 --- a/web/app/components/workflow/nodes/_base/components/memory-config.tsx +++ b/web/app/components/workflow/nodes/_base/components/memory-config.tsx @@ -32,12 +32,12 @@ const RoleItem: FC = ({ }, [onChange]) return (
-
{title}
- {title}
+
) @@ -180,7 +180,7 @@ const MemoryConfig: FC = ({
{canSetRoleName && (
-
{t(`${i18nPrefix}.conversationRoleName`)}
+
{t(`${i18nPrefix}.conversationRoleName`)}
= ({
{description} {subItems && ( -
+
{subItems.map((item, index) => ( { return ( -
+
{t('workflow.panel.changeBlock')}
) diff --git a/web/app/components/workflow/nodes/_base/components/panel-operator/panel-operator-popup.tsx b/web/app/components/workflow/nodes/_base/components/panel-operator/panel-operator-popup.tsx index 23150b3896..28d7358ddb 100644 --- a/web/app/components/workflow/nodes/_base/components/panel-operator/panel-operator-popup.tsx +++ b/web/app/components/workflow/nodes/_base/components/panel-operator/panel-operator-popup.tsx @@ -84,7 +84,7 @@ const PanelOperatorPopup = ({ const link = useNodeHelpLink(data.type) return ( -
+
{ (showChangeBlock || canRunBySingle(data.type)) && ( <> @@ -93,8 +93,8 @@ const PanelOperatorPopup = ({ canRunBySingle(data.type) && (
{ handleNodeSelect(id) @@ -117,7 +117,7 @@ const PanelOperatorPopup = ({ ) }
-
+
) } @@ -126,7 +126,7 @@ const PanelOperatorPopup = ({ <>
{ onClosePopup() handleNodesCopy(id) @@ -136,7 +136,7 @@ const PanelOperatorPopup = ({
{ onClosePopup() handleNodesDuplicate(id) @@ -146,12 +146,12 @@ const PanelOperatorPopup = ({
-
+
handleNodeDelete(id)} > @@ -159,7 +159,7 @@ const PanelOperatorPopup = ({
-
+
) } @@ -170,21 +170,21 @@ const PanelOperatorPopup = ({ {t('workflow.panel.helpLink')}
-
+
) }
-
+
{t('workflow.panel.about').toLocaleUpperCase()}
-
{about}
+
{about}
{t('workflow.panel.createdBy')} {author}
diff --git a/web/app/components/workflow/nodes/_base/components/prompt/editor.tsx b/web/app/components/workflow/nodes/_base/components/prompt/editor.tsx index cc8799bbed..dd4d837d12 100644 --- a/web/app/components/workflow/nodes/_base/components/prompt/editor.tsx +++ b/web/app/components/workflow/nodes/_base/components/prompt/editor.tsx @@ -26,7 +26,6 @@ import { Clipboard, ClipboardCheck, } from '@/app/components/base/icons/src/vender/line/files' -import s from '@/app/components/app/configuration/config-prompt/style.module.css' import { useEventEmitterContextContext } from '@/context/event-emitter' import { PROMPT_EDITOR_INSERT_QUICKLY } from '@/app/components/base/prompt-editor/plugins/update-block' import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' @@ -147,20 +146,20 @@ const Editor: FC = ({ return ( -
-
+
+
-
{title} {required && *}
+
{title} {required && *}
{titleTooltip && }
-
{value?.length || 0}
+
{value?.length || 0}
{isSupportPromptGenerator && ( )} -
+
{/* Operations */}
{isSupportJinja && ( @@ -168,13 +167,13 @@ const Editor: FC = ({ popupContent={ } needsDelay > -
- +
+ = ({ return ( {str} -
+
{!isEnv && !isChatVar && (
-
{node?.title}
+
{node?.title}
)} -
+
{!isEnv && !isChatVar && } {isEnv && } {isChatVar && } -
{varName}
+
{varName}
) diff --git a/web/app/components/workflow/nodes/_base/components/remove-button.tsx b/web/app/components/workflow/nodes/_base/components/remove-button.tsx index 9fc9c166bb..62381f8c2a 100644 --- a/web/app/components/workflow/nodes/_base/components/remove-button.tsx +++ b/web/app/components/workflow/nodes/_base/components/remove-button.tsx @@ -2,7 +2,7 @@ import type { FC } from 'react' import React from 'react' import { RiDeleteBinLine } from '@remixicon/react' -import cn from '@/utils/classnames' +import ActionButton from '@/app/components/base/action-button' type Props = { className?: string @@ -10,16 +10,12 @@ type Props = { } const Remove: FC = ({ - className, onClick, }) => { return ( -
- -
+ + + ) } export default React.memo(Remove) diff --git a/web/app/components/workflow/nodes/_base/components/retry/retry-on-panel.tsx b/web/app/components/workflow/nodes/_base/components/retry/retry-on-panel.tsx index b07538a014..0e5b807ff4 100644 --- a/web/app/components/workflow/nodes/_base/components/retry/retry-on-panel.tsx +++ b/web/app/components/workflow/nodes/_base/components/retry/retry-on-panel.tsx @@ -66,7 +66,7 @@ const RetryOnPanel = ({ retry_config?.retry_enabled && (
-
{t('workflow.nodes.common.retry.maxRetries')}
+
{t('workflow.nodes.common.retry.maxRetries')}
-
{t('workflow.nodes.common.retry.retryInterval')}
+
{t('workflow.nodes.common.retry.retryInterval')}
= ({ : (
-
{!noValue ? item?.label : placeholder}
+ className={cn(showOption && 'bg-state-base-hover', 'flex h-5 cursor-pointer items-center rounded-md pl-1 pr-0.5 text-xs font-semibold text-text-secondary hover:bg-state-base-hover')}> +
{!noValue ? item?.label : placeholder}
{!readonly && }
)} {(showOption && !readonly) && ( -
+
{list.map(item => (
= ({ setHide() onChange(item.value) }} - className={cn(itemClassName, uppercase && 'uppercase', 'flex h-[30px] min-w-[44px] cursor-pointer items-center justify-between rounded-lg px-3 text-[13px] font-medium text-gray-700 hover:bg-gray-50')} + className={cn(itemClassName, uppercase && 'uppercase', 'flex h-[30px] min-w-[44px] cursor-pointer items-center justify-between rounded-lg px-3 text-[13px] font-medium text-text-secondary hover:bg-state-base-hover')} >
{item.label}
- {showChecked && item.value === value && } + {showChecked && item.value === value && }
)) } diff --git a/web/app/components/workflow/nodes/_base/components/title-description-input.tsx b/web/app/components/workflow/nodes/_base/components/title-description-input.tsx index ec0f6fbcda..062190aee9 100644 --- a/web/app/components/workflow/nodes/_base/components/title-description-input.tsx +++ b/web/app/components/workflow/nodes/_base/components/title-description-input.tsx @@ -33,7 +33,7 @@ export const TitleInput = memo(({ value={localValue} onChange={e => setLocalValue(e.target.value)} className={` - system-xl-semibold mr-2 h-7 min-w-0 grow appearance-none rounded-md border border-transparent px-1 text-text-primary + system-xl-semibold mr-2 h-7 min-w-0 grow appearance-none rounded-md border border-transparent bg-transparent px-1 text-text-primary outline-none focus:shadow-xs `} placeholder={t('workflow.common.addTitle') || ''} @@ -76,8 +76,8 @@ export const DescriptionInput = memo(({ onBlur={handleBlur} className={` w-full resize-none appearance-none bg-transparent text-xs - leading-[18px] text-gray-900 caret-[#295EFF] - outline-none placeholder:text-gray-400 + leading-[18px] text-text-primary caret-[#295EFF] + outline-none placeholder:text-text-quaternary `} placeholder={t('workflow.common.addDescription') || ''} /> diff --git a/web/app/components/workflow/nodes/_base/components/variable-tag.tsx b/web/app/components/workflow/nodes/_base/components/variable-tag.tsx index 5a5553b18c..83b07715fe 100644 --- a/web/app/components/workflow/nodes/_base/components/variable-tag.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable-tag.tsx @@ -51,7 +51,7 @@ const VariableTag = ({ const { t } = useTranslation() return ( -
{(!isEnv && !isChatVar && <> @@ -59,7 +59,7 @@ const VariableTag = ({ <>
= ({ isSupportFileVar={isSupportFileVar} /> {!readonly && ( - + )}
))} diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx index a9b51d5a1e..568dd7150a 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx @@ -333,9 +333,9 @@ const VarReferencePicker: FC = ({ }} className='h-full grow' > -
+
-
+
{hasValue ? ( <> @@ -343,17 +343,17 @@ const VarReferencePicker: FC = ({
{outputVarNode?.type && }
-
{outputVarNode?.title}
)} -
+
{!hasValue && } {isEnv && } {isChatVar && } @@ -364,7 +364,7 @@ const VarReferencePicker: FC = ({
{type}
- {!isValidVar && } + {!isValidVar && } ) :
{placeholder ?? t('workflow.common.setVarValuePlaceholder')}
} @@ -375,10 +375,10 @@ const VarReferencePicker: FC = ({ )} {(hasValue && !readonly && !isInTable) && (
- +
)} {!hasValue && valueTypePlaceHolder && ( = ({ const { locale } = useContext(I18n) // max-h-[300px] overflow-y-auto todo: use portal to handle long list return ( -
{((!vars || vars.length === 0) && popupFor) diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-type-picker.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-type-picker.tsx index dc48f9f795..926d7ac705 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-type-picker.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-type-picker.tsx @@ -43,23 +43,23 @@ const VarReferencePicker: FC = ({ offset={4} > setOpen(!open)} className='w-[120px] cursor-pointer'> -
+
{value}
- +
-
+
{TYPES.map(type => (
{type}
- {type === value && } + {type === value && }
))}
diff --git a/web/app/components/workflow/nodes/_base/hooks/use-toggle-expend.ts b/web/app/components/workflow/nodes/_base/hooks/use-toggle-expend.ts index 4c980ee7d1..e90f079761 100644 --- a/web/app/components/workflow/nodes/_base/hooks/use-toggle-expend.ts +++ b/web/app/components/workflow/nodes/_base/hooks/use-toggle-expend.ts @@ -20,9 +20,9 @@ const useToggleExpend = ({ ref, hasFooter = true, isInNode }: Params) => { return '' if (isInNode) - return 'fixed z-10 right-[9px] top-[166px] bottom-[8px] p-4 bg-white rounded-xl' + return 'fixed z-10 right-[9px] top-[166px] bottom-[8px] p-4 bg-components-panel-bg rounded-xl' - return 'absolute z-10 left-4 right-6 top-[52px] bottom-0 pb-4 bg-white' + return 'absolute z-10 left-4 right-6 top-[52px] bottom-0 pb-4 bg-components-panel-bg' })() const wrapStyle = isExpand ? { diff --git a/web/app/components/workflow/nodes/_base/panel.tsx b/web/app/components/workflow/nodes/_base/panel.tsx index dee94cf291..2ee39a3b06 100644 --- a/web/app/components/workflow/nodes/_base/panel.tsx +++ b/web/app/components/workflow/nodes/_base/panel.tsx @@ -119,7 +119,7 @@ const BasePanel: FC = ({ width: `${panelWidth}px`, }} > -
+
= ({ value={data.title || ''} onBlur={handleTitleBlur} /> -
+
{ canRunBySingle(data.type) && !nodesReadOnly && ( = ({ popupClassName='mr-1' >
{ handleNodeDataUpdate({ id, data: { _isSingleRun: true } }) handleSyncWorkflowDraft(true) @@ -169,7 +169,7 @@ const BasePanel: FC = ({
- {cloneElement(children, { id, data })} + {cloneElement(children as any, { id, data })}
{ @@ -190,7 +190,7 @@ const BasePanel: FC = ({ } { !!availableNextBlocks.length && ( -
+
{t('workflow.panel.nextStep').toLocaleUpperCase()}
diff --git a/web/app/components/workflow/nodes/end/node.tsx b/web/app/components/workflow/nodes/end/node.tsx index dae8a6de3c..e0c5604391 100644 --- a/web/app/components/workflow/nodes/end/node.tsx +++ b/web/app/components/workflow/nodes/end/node.tsx @@ -52,13 +52,13 @@ const Node: FC> = ({ isChatMode, }) return ( -
-
+
+
{!isEnv && !isChatVar && ( <>
@@ -66,16 +66,16 @@ const Node: FC> = ({ )} -
- {!isEnv && !isChatVar && } +
+ {!isEnv && !isChatVar && } {isEnv && } {isChatVar && } -
{varName}
+
{varName}
-
-
{varType}
+
+
{varType}
) diff --git a/web/app/components/workflow/nodes/http/components/api-input.tsx b/web/app/components/workflow/nodes/http/components/api-input.tsx index 164926b9c2..000011e4cd 100644 --- a/web/app/components/workflow/nodes/http/components/api-input.tsx +++ b/web/app/components/workflow/nodes/http/components/api-input.tsx @@ -53,9 +53,9 @@ const ApiInput: FC = ({ onChange={onMethodChange} options={MethodOptions} trigger={ -
-
{method}
- {!readonly && } +
+
{method}
+ {!readonly && }
} popupClassName='top-[34px] w-[108px]' @@ -65,7 +65,7 @@ const ApiInput: FC = ({ { return (
-
+
{title} - {isRequired && *} + {isRequired && *}
{children}
@@ -158,7 +158,7 @@ const Authorization: FC = ({
= ({ return (
{title} diff --git a/web/app/components/workflow/nodes/http/components/curl-panel.tsx b/web/app/components/workflow/nodes/http/components/curl-panel.tsx index f2e1a44488..52e28d7336 100644 --- a/web/app/components/workflow/nodes/http/components/curl-panel.tsx +++ b/web/app/components/workflow/nodes/http/components/curl-panel.tsx @@ -5,6 +5,7 @@ import { useTranslation } from 'react-i18next' import { BodyType, type HttpNodeType, Method } from '../types' import Modal from '@/app/components/base/modal' import Button from '@/app/components/base/button' +import Textarea from '@/app/components/base/textarea' import Toast from '@/app/components/base/toast' import { useNodesInteractions } from '@/app/components/workflow/hooks' @@ -141,9 +142,9 @@ const CurlPanel: FC = ({ nodeId, isShow, onHide, handleCurlImport }) => { className='!w-[400px] !max-w-[400px] !p-4' >
-