From 6fafd410d264988684070459ff829503a0bc0632 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 26 Jul 2024 11:21:17 +0800 Subject: [PATCH] feat: doc extract struct --- .../workflow/block-selector/constants.tsx | 5 +++ web/app/components/workflow/constants.ts | 17 +++++++ .../nodes/_base/components/variable/utils.ts | 10 +++++ .../components/workflow/nodes/constants.ts | 4 ++ .../workflow/nodes/doc-extractor/default.ts | 35 +++++++++++++++ .../workflow/nodes/doc-extractor/node.tsx | 40 +++++++++++++++++ .../workflow/nodes/doc-extractor/panel.tsx | 45 +++++++++++++++++++ .../workflow/nodes/doc-extractor/types.ts | 5 +++ .../nodes/doc-extractor/use-config.ts | 35 +++++++++++++++ .../components/node-variable-item.tsx | 4 +- web/app/components/workflow/types.ts | 3 ++ web/i18n/en-US/workflow.ts | 4 ++ 12 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 web/app/components/workflow/nodes/doc-extractor/default.ts create mode 100644 web/app/components/workflow/nodes/doc-extractor/node.tsx create mode 100644 web/app/components/workflow/nodes/doc-extractor/panel.tsx create mode 100644 web/app/components/workflow/nodes/doc-extractor/types.ts create mode 100644 web/app/components/workflow/nodes/doc-extractor/use-config.ts diff --git a/web/app/components/workflow/block-selector/constants.tsx b/web/app/components/workflow/block-selector/constants.tsx index 517d9356f2..3b227a9853 100644 --- a/web/app/components/workflow/block-selector/constants.tsx +++ b/web/app/components/workflow/block-selector/constants.tsx @@ -69,6 +69,11 @@ export const BLOCKS: Block[] = [ type: BlockEnum.HttpRequest, title: 'HTTP Request', }, + { + classification: BlockClassificationEnum.Utilities, + type: BlockEnum.DocExtractor, + title: 'Doc Extractor', + }, ] export const BLOCK_CLASSIFICATIONS: string[] = [ diff --git a/web/app/components/workflow/constants.ts b/web/app/components/workflow/constants.ts index 46d12764f1..eb615a6651 100644 --- a/web/app/components/workflow/constants.ts +++ b/web/app/components/workflow/constants.ts @@ -14,6 +14,7 @@ import ToolDefault from './nodes/tool/default' import VariableAssignerDefault from './nodes/variable-assigner/default' import EndNodeDefault from './nodes/end/default' import IterationDefault from './nodes/iteration/default' +import DocExtractorDefault from './nodes/doc-extractor/default' type NodesExtraData = { author: string @@ -160,6 +161,16 @@ export const NODES_EXTRA_DATA: Record = { getAvailableNextNodes: ToolDefault.getAvailableNextNodes, checkValid: ToolDefault.checkValid, }, + [BlockEnum.DocExtractor]: { + author: 'Dify', + about: '', + availablePrevNodes: [], + availableNextNodes: [], + getAvailablePrevNodes: DocExtractorDefault.getAvailablePrevNodes, + getAvailableNextNodes: DocExtractorDefault.getAvailableNextNodes, + checkValid: DocExtractorDefault.checkValid, + }, + } export const ALL_CHAT_AVAILABLE_BLOCKS = Object.keys(NODES_EXTRA_DATA).filter(key => key !== BlockEnum.End && key !== BlockEnum.Start) as BlockEnum[] @@ -274,6 +285,12 @@ export const NODES_INITIAL_DATA = { desc: '', ...ToolDefault.defaultValue, }, + [BlockEnum.DocExtractor]: { + type: BlockEnum.DocExtractor, + title: '', + desc: '', + ...DocExtractorDefault.defaultValue, + }, } export const NODE_WIDTH = 240 diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.ts index 07c9a13dac..9e99792085 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts +++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts @@ -235,6 +235,16 @@ const formatItem = ( break } + case BlockEnum.DocExtractor: { + res.vars = [ + { + variable: 'text', + type: VarType.string, + }, + ] + break + } + case 'env': { res.vars = data.envList.map((env: EnvironmentVariable) => { return { diff --git a/web/app/components/workflow/nodes/constants.ts b/web/app/components/workflow/nodes/constants.ts index a97aa086ed..bed2d6069b 100644 --- a/web/app/components/workflow/nodes/constants.ts +++ b/web/app/components/workflow/nodes/constants.ts @@ -28,6 +28,8 @@ import ParameterExtractorNode from './parameter-extractor/node' import ParameterExtractorPanel from './parameter-extractor/panel' import IterationNode from './iteration/node' import IterationPanel from './iteration/panel' +import DocExtractorNode from './doc-extractor/node' +import DocExtractorPanel from './doc-extractor/panel' export const NodeComponentMap: Record> = { [BlockEnum.Start]: StartNode, @@ -45,6 +47,7 @@ export const NodeComponentMap: Record> = { [BlockEnum.VariableAggregator]: VariableAssignerNode, [BlockEnum.ParameterExtractor]: ParameterExtractorNode, [BlockEnum.Iteration]: IterationNode, + [BlockEnum.DocExtractor]: DocExtractorNode, } export const PanelComponentMap: Record> = { @@ -63,6 +66,7 @@ export const PanelComponentMap: Record> = { [BlockEnum.VariableAggregator]: VariableAssignerPanel, [BlockEnum.ParameterExtractor]: ParameterExtractorPanel, [BlockEnum.Iteration]: IterationPanel, + [BlockEnum.DocExtractor]: DocExtractorPanel, } export const CUSTOM_NODE_TYPE = 'custom' diff --git a/web/app/components/workflow/nodes/doc-extractor/default.ts b/web/app/components/workflow/nodes/doc-extractor/default.ts new file mode 100644 index 0000000000..204b1bcb5a --- /dev/null +++ b/web/app/components/workflow/nodes/doc-extractor/default.ts @@ -0,0 +1,35 @@ +import { BlockEnum } from '../../types' +import type { NodeDefault } from '../../types' +import { type DocExtractorNodeType } from './types' +import { ALL_CHAT_AVAILABLE_BLOCKS, ALL_COMPLETION_AVAILABLE_BLOCKS } from '@/app/components/workflow/constants' +const i18nPrefix = 'workflow.errorMsg' + +const nodeDefault: NodeDefault = { + defaultValue: { + variable: [], + }, + getAvailablePrevNodes(isChatMode: boolean) { + const nodes = isChatMode + ? ALL_CHAT_AVAILABLE_BLOCKS + : ALL_COMPLETION_AVAILABLE_BLOCKS.filter(type => type !== BlockEnum.End) + return nodes + }, + getAvailableNextNodes(isChatMode: boolean) { + const nodes = isChatMode ? ALL_CHAT_AVAILABLE_BLOCKS : ALL_COMPLETION_AVAILABLE_BLOCKS + return nodes + }, + checkValid(payload: DocExtractorNodeType, t: any) { + let errorMessages = '' + const { variable } = payload + + if (!errorMessages && !variable?.length) + errorMessages = t(`${i18nPrefix}.fieldRequired`, { field: t('workflow.nodes.assigner.assignedVariable') }) + + return { + isValid: !errorMessages, + errorMessage: errorMessages, + } + }, +} + +export default nodeDefault diff --git a/web/app/components/workflow/nodes/doc-extractor/node.tsx b/web/app/components/workflow/nodes/doc-extractor/node.tsx new file mode 100644 index 0000000000..8558ca81eb --- /dev/null +++ b/web/app/components/workflow/nodes/doc-extractor/node.tsx @@ -0,0 +1,40 @@ +import type { FC } from 'react' +import React from 'react' +import { useNodes } from 'reactflow' +import { useTranslation } from 'react-i18next' +import NodeVariableItem from '../variable-assigner/components/node-variable-item' +import { type DocExtractorNodeType } from './types' +import { isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import { BlockEnum, type Node, type NodeProps } from '@/app/components/workflow/types' + +const i18nPrefix = 'workflow.nodes.assigner' + +const NodeComponent: FC> = ({ + data, +}) => { + const { t } = useTranslation() + + const nodes: Node[] = useNodes() + const { variable } = data + + if (!variable || variable.length === 0) + return null + + const isSystem = isSystemVar(variable) + const isEnv = isENV(variable) + const node = isSystem ? nodes.find(node => node.data.type === BlockEnum.Start) : nodes.find(node => node.id === variable[0]) + const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.') + return ( +
+
{t(`${i18nPrefix}.assignedVariable`)}
+ +
+ ) +} + +export default React.memo(NodeComponent) diff --git a/web/app/components/workflow/nodes/doc-extractor/panel.tsx b/web/app/components/workflow/nodes/doc-extractor/panel.tsx new file mode 100644 index 0000000000..612220b811 --- /dev/null +++ b/web/app/components/workflow/nodes/doc-extractor/panel.tsx @@ -0,0 +1,45 @@ +import type { FC } from 'react' +import React from 'react' +import { useTranslation } from 'react-i18next' +import VarReferencePicker from '../_base/components/variable/var-reference-picker' +import useConfig from './use-config' +import type { DocExtractorNodeType } from './types' +import Field from '@/app/components/workflow/nodes/_base/components/field' +import { type NodePanelProps } from '@/app/components/workflow/types' + +const i18nPrefix = 'workflow.nodes.docExtractor' + +const Panel: FC> = ({ + id, + data, +}) => { + const { t } = useTranslation() + + const { + readOnly, + inputs, + handleVarChanges, + filterVar, + } = useConfig(id, data) + + return ( +
+
+ + + +
+
+ ) +} + +export default React.memo(Panel) diff --git a/web/app/components/workflow/nodes/doc-extractor/types.ts b/web/app/components/workflow/nodes/doc-extractor/types.ts new file mode 100644 index 0000000000..054e4b6cf2 --- /dev/null +++ b/web/app/components/workflow/nodes/doc-extractor/types.ts @@ -0,0 +1,5 @@ +import type { CommonNodeType, ValueSelector } from '@/app/components/workflow/types' + +export type DocExtractorNodeType = CommonNodeType & { + variable: ValueSelector +} diff --git a/web/app/components/workflow/nodes/doc-extractor/use-config.ts b/web/app/components/workflow/nodes/doc-extractor/use-config.ts new file mode 100644 index 0000000000..45ad397003 --- /dev/null +++ b/web/app/components/workflow/nodes/doc-extractor/use-config.ts @@ -0,0 +1,35 @@ +import { useCallback } from 'react' +import produce from 'immer' +import type { ValueSelector, Var } from '../../types' +import { VarType } from '../../types' +import { type DocExtractorNodeType } from './types' +import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud' +import { + useNodesReadOnly, +} from '@/app/components/workflow/hooks' + +const useConfig = (id: string, payload: DocExtractorNodeType) => { + const { nodesReadOnly: readOnly } = useNodesReadOnly() + + const { inputs, setInputs } = useNodeCrud(id, payload) + + const handleVarChanges = useCallback((variable: ValueSelector | string) => { + const newInputs = produce(inputs, (draft) => { + draft.variable = variable as ValueSelector + }) + setInputs(newInputs) + }, [inputs, setInputs]) + + const filterVar = useCallback((varPayload: Var) => { + return varPayload.type !== VarType.file + }, []) + + return { + readOnly, + inputs, + filterVar, + handleVarChanges, + } +} + +export default useConfig diff --git a/web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx b/web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx index 6b32ebaf02..62911e521c 100644 --- a/web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx +++ b/web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx @@ -8,12 +8,14 @@ import type { Node } from '@/app/components/workflow/types' import { BlockEnum } from '@/app/components/workflow/types' type NodeVariableItemProps = { + className?: string isEnv: boolean node: Node varName: string showBorder?: boolean } const NodeVariableItem = ({ + className, isEnv, node, varName, @@ -22,7 +24,7 @@ const NodeVariableItem = ({ return (
{!isEnv && (
diff --git a/web/app/components/workflow/types.ts b/web/app/components/workflow/types.ts index 8b0e3113bc..e455687ce3 100644 --- a/web/app/components/workflow/types.ts +++ b/web/app/components/workflow/types.ts @@ -25,6 +25,8 @@ export enum BlockEnum { Tool = 'tool', ParameterExtractor = 'parameter-extractor', Iteration = 'iteration', + DocExtractor = 'doc-extractor', + ListFilter = 'list-filter', } export type Branch = { @@ -193,6 +195,7 @@ export enum VarType { secret = 'secret', boolean = 'boolean', object = 'object', + file = 'file', array = 'array', arrayString = 'array[string]', arrayNumber = 'array[number]', diff --git a/web/i18n/en-US/workflow.ts b/web/i18n/en-US/workflow.ts index 233ba40450..c416ca296d 100644 --- a/web/i18n/en-US/workflow.ts +++ b/web/i18n/en-US/workflow.ts @@ -176,6 +176,7 @@ const translation = { 'iteration-start': 'Iteration Start', 'iteration': 'Iteration', 'parameter-extractor': 'Parameter Extractor', + 'doc-extractor': 'Doc Extractor', }, blocksAbout: { 'start': 'Define the initial parameters for launching a workflow', @@ -192,6 +193,7 @@ const translation = { 'variable-aggregator': 'Aggregate multi-branch variables into a single variable for unified configuration of downstream nodes.', 'iteration': 'Perform multiple steps on a list object until all results are outputted.', 'parameter-extractor': 'Use LLM to extract structured parameters from natural language for tool invocations or HTTP requests.', + 'doc-extractor': 'TODO', }, operator: { zoomIn: 'Zoom In', @@ -489,6 +491,8 @@ const translation = { showAuthor: 'Show Author', }, }, + docExtractor: { + }, }, tracing: { stopBy: 'Stop by {{user}}',