diff --git a/web/app/components/workflow/nodes/_base/components/before-run-form/form-item.tsx b/web/app/components/workflow/nodes/_base/components/before-run-form/form-item.tsx index a4885759c4..6c5a8e37ff 100644 --- a/web/app/components/workflow/nodes/_base/components/before-run-form/form-item.tsx +++ b/web/app/components/workflow/nodes/_base/components/before-run-form/form-item.tsx @@ -1,6 +1,6 @@ 'use client' import type { FC } from 'react' -import React, { useCallback } from 'react' +import React, { useCallback, useMemo } from 'react' import { useTranslation } from 'react-i18next' import produce from 'immer' import { @@ -24,8 +24,9 @@ import { Variable02 } from '@/app/components/base/icons/src/vender/solid/develop import { BubbleX } from '@/app/components/base/icons/src/vender/line/others' import { FILE_EXTS } from '@/app/components/base/prompt-editor/constants' import cn from '@/utils/classnames' +import type { FileEntity } from '@/app/components/base/file-uploader/types' -interface Props { +type Props = { payload: InputVar value: any onChange: (value: any) => void @@ -94,6 +95,21 @@ const FormItem: FC = ({ const isArrayLikeType = [InputVarType.contexts, InputVarType.iterator].includes(type) const isContext = type === InputVarType.contexts const isIterator = type === InputVarType.iterator + const singleFileValue = useMemo(() => { + if (payload.variable === '#files#') + return value?.[0] || [] + + return value ? [value] : [] + }, [payload.variable, value]) + const handleSingleFileChange = useCallback((files: FileEntity[]) => { + if (payload.variable === '#files#') + onChange(files) + else if (files.length) + onChange(files[0]) + else + onChange(null) + }, [onChange, payload.variable]) + return (
{!isArrayLikeType && ( @@ -161,13 +177,8 @@ const FormItem: FC = ({ } {(type === InputVarType.singleFile) && ( { - if (files.length) - onChange(files[0]) - else - onChange(null) - }} + value={singleFileValue} + onChange={handleSingleFileChange} fileConfig={{ allowed_file_types: inStepRun ? [ diff --git a/web/app/components/workflow/nodes/_base/components/before-run-form/index.tsx b/web/app/components/workflow/nodes/_base/components/before-run-form/index.tsx index 2fb873e604..bd1b053395 100644 --- a/web/app/components/workflow/nodes/_base/components/before-run-form/index.tsx +++ b/web/app/components/workflow/nodes/_base/components/before-run-form/index.tsx @@ -50,8 +50,11 @@ function formatValue(value: string | any, type: InputVarType) { if (type === InputVarType.multiFiles) return getProcessedFiles(value) - if (type === InputVarType.singleFile) + if (type === InputVarType.singleFile) { + if (Array.isArray(value)) + return getProcessedFiles(value) return getProcessedFiles([value])[0] + } return value } diff --git a/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts b/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts index 443ba1bbd2..c816ee28dd 100644 --- a/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts +++ b/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { unionBy } from 'lodash-es' import produce from 'immer' @@ -139,6 +139,11 @@ const useOneStepRun = ({ const checkValid = checkValidFns[data.type] const appId = useAppStore.getState().appDetail?.id const [runInputData, setRunInputData] = useState>(defaultRunInputData || {}) + const runInputDataRef = useRef(runInputData) + const handleSetRunInputData = useCallback((data: Record) => { + runInputDataRef.current = data + setRunInputData(data) + }, []) const iterationTimes = iteratorInputKey ? runInputData[iteratorInputKey].length : 0 const [runResult, setRunResult] = useState(null) @@ -421,7 +426,8 @@ const useOneStepRun = ({ handleRun, handleStop, runInputData, - setRunInputData, + runInputDataRef, + setRunInputData: handleSetRunInputData, runResult, iterationRunResult, } diff --git a/web/app/components/workflow/nodes/llm/panel.tsx b/web/app/components/workflow/nodes/llm/panel.tsx index cc0f1c18f4..6c3831f5bc 100644 --- a/web/app/components/workflow/nodes/llm/panel.tsx +++ b/web/app/components/workflow/nodes/llm/panel.tsx @@ -5,6 +5,7 @@ import MemoryConfig from '../_base/components/memory-config' import VarReferencePicker from '../_base/components/variable/var-reference-picker' import ConfigVision from '../_base/components/config-vision' import useConfig from './use-config' +import { findVariableWhenOnLLMVision } from '../utils' import type { LLMNodeType } from './types' import ConfigPrompt from './components/config-prompt' import VarList from '@/app/components/workflow/nodes/_base/components/variable/var-list' @@ -102,15 +103,16 @@ const Panel: FC> = ({ ) } - if (isVisionModel) { - const variableName = data.vision.configs?.variable_selector?.[1] || t(`${i18nPrefix}.files`)! + if (isVisionModel && data.vision.enabled && data.vision.configs?.variable_selector) { + const currentVariable = findVariableWhenOnLLMVision(data.vision.configs.variable_selector, availableVars) + forms.push( { label: t(`${i18nPrefix}.vision`)!, inputs: [{ - label: variableName!, + label: currentVariable?.variable as any, variable: '#files#', - type: InputVarType.files, + type: currentVariable?.formType as any, required: false, }], values: { '#files#': visionFiles }, diff --git a/web/app/components/workflow/nodes/llm/use-config.ts b/web/app/components/workflow/nodes/llm/use-config.ts index ee9f2ca915..6b2d27e70f 100644 --- a/web/app/components/workflow/nodes/llm/use-config.ts +++ b/web/app/components/workflow/nodes/llm/use-config.ts @@ -306,6 +306,7 @@ const useConfig = (id: string, payload: LLMNodeType) => { handleRun, handleStop, runInputData, + runInputDataRef, setRunInputData, runResult, toVarInputs, @@ -331,27 +332,27 @@ const useConfig = (id: string, payload: LLMNodeType) => { const setInputVarValues = useCallback((newPayload: Record) => { const newVars = { ...newPayload, - '#context#': runInputData['#context#'], - '#files#': runInputData['#files#'], + '#context#': runInputDataRef.current['#context#'], + '#files#': runInputDataRef.current['#files#'], } setRunInputData(newVars) - }, [runInputData, setRunInputData]) + }, [runInputDataRef, setRunInputData]) const contexts = runInputData['#context#'] const setContexts = useCallback((newContexts: string[]) => { setRunInputData({ - ...runInputData, + ...runInputDataRef.current, '#context#': newContexts, }) - }, [runInputData, setRunInputData]) + }, [runInputDataRef, setRunInputData]) const visionFiles = runInputData['#files#'] const setVisionFiles = useCallback((newFiles: any[]) => { setRunInputData({ - ...runInputData, + ...runInputDataRef.current, '#files#': newFiles, }) - }, [runInputData, setRunInputData]) + }, [runInputDataRef, setRunInputData]) const allVarStrArr = (() => { const arr = isChatModel ? (inputs.prompt_template as PromptItem[]).filter(item => item.edition_type !== EditionType.jinja2).map(item => item.text) : [(inputs.prompt_template as PromptItem).text] diff --git a/web/app/components/workflow/nodes/parameter-extractor/panel.tsx b/web/app/components/workflow/nodes/parameter-extractor/panel.tsx index 183c940c2d..84bce38fa9 100644 --- a/web/app/components/workflow/nodes/parameter-extractor/panel.tsx +++ b/web/app/components/workflow/nodes/parameter-extractor/panel.tsx @@ -6,6 +6,7 @@ import VarReferencePicker from '../_base/components/variable/var-reference-picke import Editor from '../_base/components/prompt/editor' import ResultPanel from '../../run/result-panel' import ConfigVision from '../_base/components/config-vision' +import { findVariableWhenOnLLMVision } from '../utils' import useConfig from './use-config' import type { ParameterExtractorNodeType } from './types' import ExtractParameter from './components/extract-parameter/list' @@ -21,6 +22,7 @@ import Tooltip from '@/app/components/base/tooltip' import BeforeRunForm from '@/app/components/workflow/nodes/_base/components/before-run-form' import { VarType } from '@/app/components/workflow/types' import { FieldCollapse } from '@/app/components/workflow/nodes/_base/components/collapse' +import type { Props as FormProps } from '@/app/components/workflow/nodes/_base/components/before-run-form/form' const i18nPrefix = 'workflow.nodes.parameterExtractor' const i18nCommonPrefix = 'workflow.common' @@ -51,6 +53,7 @@ const Panel: FC> = ({ handleReasoningModeChange, availableVars, availableNodesWithParent, + availableVisionVars, inputVarValues, varInputs, isVisionModel, @@ -63,10 +66,50 @@ const Panel: FC> = ({ handleStop, runResult, setInputVarValues, + visionFiles, + setVisionFiles, } = useConfig(id, data) const model = inputs.model + const singleRunForms = (() => { + const forms: FormProps[] = [] + + forms.push( + { + label: t('workflow.nodes.llm.singleRun.variable')!, + inputs: [{ + label: t(`${i18nPrefix}.inputVar`)!, + variable: 'query', + type: InputVarType.paragraph, + required: true, + }, ...varInputs], + values: inputVarValues, + onChange: setInputVarValues, + }, + ) + + if (isVisionModel && data.vision.enabled && data.vision.configs?.variable_selector) { + const currentVariable = findVariableWhenOnLLMVision(data.vision.configs.variable_selector, availableVisionVars) + + forms.push( + { + label: t('workflow.nodes.llm.vision')!, + inputs: [{ + label: currentVariable?.variable as any, + variable: '#files#', + type: currentVariable?.formType as any, + required: false, + }], + values: { '#files#': visionFiles }, + onChange: keyValue => setVisionFiles((keyValue as any)['#files#']), + }, + ) + } + + return forms + })() + return (
@@ -213,18 +256,7 @@ const Panel: FC> = ({ { return [VarType.number, VarType.string].includes(varPayload.type) }, []) + const filterVisionInputVar = useCallback((varPayload: Var) => { + return [VarType.file, VarType.arrayFile].includes(varPayload.type) + }, []) + const { availableVars, availableNodesWithParent, @@ -173,6 +177,13 @@ const useConfig = (id: string, payload: ParameterExtractorNodeType) => { filterVar: filterInputVar, }) + const { + availableVars: availableVisionVars, + } = useAvailableVarList(id, { + onlyLeafNodeVar: false, + filterVar: filterVisionInputVar, + }) + const handleCompletionParamsChange = useCallback((newParams: Record) => { const newInputs = produce(inputs, (draft) => { draft.model.completion_params = newParams @@ -223,13 +234,15 @@ const useConfig = (id: string, payload: ParameterExtractorNodeType) => { handleRun, handleStop, runInputData, + runInputDataRef, setRunInputData, runResult, } = useOneStepRun({ id, data: inputs, defaultRunInputData: { - query: '', + 'query': '', + '#files#': [], }, }) @@ -247,6 +260,14 @@ const useConfig = (id: string, payload: ParameterExtractorNodeType) => { setRunInputData(newPayload) }, [setRunInputData]) + const visionFiles = runInputData['#files#'] + const setVisionFiles = useCallback((newFiles: any[]) => { + setRunInputData({ + ...runInputDataRef.current, + '#files#': newFiles, + }) + }, [runInputDataRef, setRunInputData]) + return { readOnly, handleInputVarChange, @@ -264,6 +285,7 @@ const useConfig = (id: string, payload: ParameterExtractorNodeType) => { hasSetBlockStatus, availableVars, availableNodesWithParent, + availableVisionVars, isSupportFunctionCall, handleReasoningModeChange, handleMemoryChange, @@ -279,6 +301,8 @@ const useConfig = (id: string, payload: ParameterExtractorNodeType) => { handleStop, runResult, setInputVarValues, + visionFiles, + setVisionFiles, } } diff --git a/web/app/components/workflow/nodes/question-classifier/panel.tsx b/web/app/components/workflow/nodes/question-classifier/panel.tsx index 7d27a89d29..a5b362f4e8 100644 --- a/web/app/components/workflow/nodes/question-classifier/panel.tsx +++ b/web/app/components/workflow/nodes/question-classifier/panel.tsx @@ -3,6 +3,7 @@ import React from 'react' import { useTranslation } from 'react-i18next' import VarReferencePicker from '../_base/components/variable/var-reference-picker' import ConfigVision from '../_base/components/config-vision' +import { findVariableWhenOnLLMVision } from '../utils' import useConfig from './use-config' import ClassList from './components/class-list' import AdvancedSetting from './components/advanced-setting' @@ -15,6 +16,7 @@ import ResultPanel from '@/app/components/workflow/run/result-panel' import Split from '@/app/components/workflow/nodes/_base/components/split' import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars' import { FieldCollapse } from '@/app/components/workflow/nodes/_base/components/collapse' +import type { Props as FormProps } from '@/app/components/workflow/nodes/_base/components/before-run-form/form' const i18nPrefix = 'workflow.nodes.questionClassifiers' @@ -36,6 +38,7 @@ const Panel: FC> = ({ hasSetBlockStatus, availableVars, availableNodesWithParent, + availableVisionVars, handleInstructionChange, inputVarValues, varInputs, @@ -51,10 +54,50 @@ const Panel: FC> = ({ handleStop, runResult, filterVar, + visionFiles, + setVisionFiles, } = useConfig(id, data) const model = inputs.model + const singleRunForms = (() => { + const forms: FormProps[] = [] + + forms.push( + { + label: t('workflow.nodes.llm.singleRun.variable')!, + inputs: [{ + label: t(`${i18nPrefix}.inputVars`)!, + variable: 'query', + type: InputVarType.paragraph, + required: true, + }, ...varInputs], + values: inputVarValues, + onChange: setInputVarValues, + }, + ) + + if (isVisionModel && data.vision.enabled && data.vision.configs?.variable_selector) { + const currentVariable = findVariableWhenOnLLMVision(data.vision.configs.variable_selector, availableVisionVars) + + forms.push( + { + label: t('workflow.nodes.llm.vision')!, + inputs: [{ + label: currentVariable?.variable as any, + variable: '#files#', + type: currentVariable?.formType as any, + required: false, + }], + values: { '#files#': visionFiles }, + onChange: keyValue => setVisionFiles((keyValue as any)['#files#']), + }, + ) + } + + return forms + })() + return (
@@ -143,18 +186,7 @@ const Panel: FC> = ({ { return [VarType.number, VarType.string].includes(varPayload.type) }, []) + const filterVisionInputVar = useCallback((varPayload: Var) => { + return [VarType.file, VarType.arrayFile].includes(varPayload.type) + }, []) + const { availableVars, availableNodesWithParent, @@ -132,6 +136,13 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { filterVar: filterInputVar, }) + const { + availableVars: availableVisionVars, + } = useAvailableVarList(id, { + onlyLeafNodeVar: false, + filterVar: filterVisionInputVar, + }) + const hasSetBlockStatus = { history: false, query: isChatMode ? checkHasQueryBlock(inputs.instruction) : false, @@ -161,13 +172,15 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { handleRun, handleStop, runInputData, + runInputDataRef, setRunInputData, runResult, } = useOneStepRun({ id, data: inputs, defaultRunInputData: { - query: '', + 'query': '', + '#files#': [], }, }) @@ -195,6 +208,14 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { setRunInputData(newPayload) }, [setRunInputData]) + const visionFiles = runInputData['#files#'] + const setVisionFiles = useCallback((newFiles: any[]) => { + setRunInputData({ + ...runInputDataRef.current, + '#files#': newFiles, + }) + }, [runInputDataRef, setRunInputData]) + const filterVar = useCallback((varPayload: Var) => { return varPayload.type === VarType.string }, []) @@ -212,6 +233,7 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { hasSetBlockStatus, availableVars, availableNodesWithParent, + availableVisionVars, handleInstructionChange, varInputs, inputVarValues, @@ -228,6 +250,8 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { query, setQuery, runResult, + visionFiles, + setVisionFiles, } } diff --git a/web/app/components/workflow/nodes/utils.ts b/web/app/components/workflow/nodes/utils.ts new file mode 100644 index 0000000000..262dde62e7 --- /dev/null +++ b/web/app/components/workflow/nodes/utils.ts @@ -0,0 +1,30 @@ +import type { + NodeOutPutVar, + ValueSelector, +} from '@/app/components/workflow/types' +import { InputVarType } from '@/app/components/workflow/types' + +export const findVariableWhenOnLLMVision = (valueSelector: ValueSelector, availableVars: NodeOutPutVar[]) => { + const currentVariableNode = availableVars.find((availableVar) => { + if (valueSelector[0] === 'sys' && availableVar.isStartNode) + return true + + return valueSelector[0] === availableVar.nodeId + }) + const currentVariable = currentVariableNode?.vars.find((variable) => { + if (valueSelector[0] === 'sys' && variable.variable === `sys.${valueSelector[1]}`) + return true + return variable.variable === valueSelector[1] + }) + + let formType = '' + if (currentVariable?.type === 'array[file]') + formType = InputVarType.multiFiles + if (currentVariable?.type === 'file') + formType = InputVarType.singleFile + + return currentVariable && { + ...currentVariable, + formType, + } +}