AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-18 06:55:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(api/core/app/segments): Update segment types and variables (#6734)

Browse Source 
Signed-off-by: -LAN- <laipz8200@outlook.com>
This commit is contained in:
-LAN- 2024-07-27 14:43:51 +08:00 committed by GitHub
parent b6c3010f02
commit 6a3bef8378
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 410 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
api/core/app/segments/__init__.py
View File

@ -1,6 +1,6 @@
from .segment_group import SegmentGroup
from .segments import (
ArraySegment,
ArrayAnySegment,
FileSegment,
FloatSegment,
IntegerSegment,
@ -11,7 +11,11 @@ from .segments import (
)
from .types import SegmentType
from .variables import (
ArrayVariable,
ArrayAnyVariable,
ArrayFileVariable,
ArrayNumberVariable,
ArrayObjectVariable,
ArrayStringVariable,
FileVariable,
FloatVariable,
IntegerVariable,
@ -29,7 +33,7 @@ __all__ = [
'SecretVariable',
'FileVariable',
'StringVariable',
'ArrayVariable',
'ArrayAnyVariable',
'Variable',
'SegmentType',
'SegmentGroup',
@ -39,7 +43,11 @@ __all__ = [
'IntegerSegment',
'FloatSegment',
'ObjectSegment',
'ArraySegment',
'ArrayAnySegment',
'FileSegment',
'StringSegment',
'ArrayStringVariable',
'ArrayNumberVariable',
'ArrayObjectVariable',
'ArrayFileVariable',
]

28
api/core/app/segments/factory.py
View File

@ -4,7 +4,7 @@ from typing import Any
from core.file.file_obj import FileVar
from .segments import (
ArraySegment,
ArrayAnySegment,
FileSegment,
FloatSegment,
IntegerSegment,
@ -15,8 +15,14 @@ from .segments import (
)
from .types import SegmentType
from .variables import (
ArrayFileVariable,
ArrayNumberVariable,
ArrayObjectVariable,
ArrayStringVariable,
FileVariable,
FloatVariable,
IntegerVariable,
ObjectVariable,
SecretVariable,
StringVariable,
Variable,
@ -33,14 +39,28 @@ def build_variable_from_mapping(m: Mapping[str, Any], /) -> Variable:
match value_type:
case SegmentType.STRING:
return StringVariable.model_validate(m)
case SegmentType.SECRET:
return SecretVariable.model_validate(m)
case SegmentType.NUMBER if isinstance(value, int):
return IntegerVariable.model_validate(m)
case SegmentType.NUMBER if isinstance(value, float):
return FloatVariable.model_validate(m)
case SegmentType.SECRET:
return SecretVariable.model_validate(m)
case SegmentType.NUMBER if not isinstance(value, float | int):
raise ValueError(f'invalid number value {value}')
case SegmentType.FILE:
return FileVariable.model_validate(m)
case SegmentType.OBJECT if isinstance(value, dict):
return ObjectVariable.model_validate(
{**m, 'value': {k: build_variable_from_mapping(v) for k, v in value.items()}}
)
case SegmentType.ARRAY_STRING if isinstance(value, list):
return ArrayStringVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]})
case SegmentType.ARRAY_NUMBER if isinstance(value, list):
return ArrayNumberVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]})
case SegmentType.ARRAY_OBJECT if isinstance(value, list):
return ArrayObjectVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]})
case SegmentType.ARRAY_FILE if isinstance(value, list):
return ArrayFileVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]})
raise ValueError(f'not supported value type {value_type}')
@ -60,7 +80,7 @@ def build_segment(value: Any, /) -> Segment:
if isinstance(value, list):
# TODO: Limit the depth of the array
elements = [build_segment(v) for v in value]
return ArraySegment(value=elements)
return ArrayAnySegment(value=elements)
if isinstance(value, FileVar):
return FileSegment(value=value)
raise ValueError(f'not supported value {value}')

43
api/core/app/segments/segments.py
View File

@ -62,6 +62,7 @@ class StringSegment(Segment):
value_type: SegmentType = SegmentType.STRING
value: str
class FloatSegment(Segment):
value_type: SegmentType = SegmentType.NUMBER
value: float
@ -72,6 +73,16 @@ class IntegerSegment(Segment):
value: int
class FileSegment(Segment):
value_type: SegmentType = SegmentType.FILE
# TODO: embed FileVar in this model.
value: FileVar
@property
def markdown(self) -> str:
return self.value.to_markdown()
class ObjectSegment(Segment):
value_type: SegmentType = SegmentType.OBJECT
value: Mapping[str, Segment]
@ -96,9 +107,6 @@ class ObjectSegment(Segment):
class ArraySegment(Segment):
value_type: SegmentType = SegmentType.ARRAY
value: Sequence[Segment]
@property
def markdown(self) -> str:
return '\n'.join(['- ' + item.markdown for item in self.value])
@ -107,11 +115,26 @@ class ArraySegment(Segment):
return [v.to_object() for v in self.value]
class FileSegment(Segment):
value_type: SegmentType = SegmentType.FILE
# TODO: embed FileVar in this model.
value: FileVar
class ArrayAnySegment(ArraySegment):
value_type: SegmentType = SegmentType.ARRAY_ANY
value: Sequence[Segment]
@property
def markdown(self) -> str:
return self.value.to_markdown()
class ArrayStringSegment(ArraySegment):
value_type: SegmentType = SegmentType.ARRAY_STRING
value: Sequence[StringSegment]
class ArrayNumberSegment(ArraySegment):
value_type: SegmentType = SegmentType.ARRAY_NUMBER
value: Sequence[FloatSegment | IntegerSegment]
class ArrayObjectSegment(ArraySegment):
value_type: SegmentType = SegmentType.ARRAY_OBJECT
value: Sequence[ObjectSegment]
class ArrayFileSegment(ArraySegment):
value_type: SegmentType = SegmentType.ARRAY_FILE
value: Sequence[FileSegment]

6
api/core/app/segments/types.py
View File

@ -6,7 +6,11 @@ class SegmentType(str, Enum):
NUMBER = 'number'
STRING = 'string'
SECRET = 'secret'
ARRAY = 'array'
ARRAY_ANY = 'array[any]'
ARRAY_STRING = 'array[string]'
ARRAY_NUMBER = 'array[number]'
ARRAY_OBJECT = 'array[object]'
ARRAY_FILE = 'array[file]'
OBJECT = 'object'
FILE = 'file'

27
api/core/app/segments/variables.py
View File

@ -1,10 +1,13 @@
from pydantic import Field
from core.helper import encrypter
from .segments import (
ArraySegment,
ArrayAnySegment,
ArrayFileSegment,
ArrayNumberSegment,
ArrayObjectSegment,
ArrayStringSegment,
FileSegment,
FloatSegment,
IntegerSegment,
@ -41,15 +44,31 @@ class IntegerVariable(IntegerSegment, Variable):
pass
class FileVariable(FileSegment, Variable):
pass
class ObjectVariable(ObjectSegment, Variable):
pass
class ArrayVariable(ArraySegment, Variable):
class ArrayAnyVariable(ArrayAnySegment, Variable):
pass
class FileVariable(FileSegment, Variable):
class ArrayStringVariable(ArrayStringSegment, Variable):
pass
class ArrayNumberVariable(ArrayNumberSegment, Variable):
pass
class ArrayObjectVariable(ArrayObjectSegment, Variable):
pass
class ArrayFileVariable(ArrayFileSegment, Variable):
pass

307
api/tests/unit_tests/core/app/segments/test_factory.py Normal file
View File

@ -0,0 +1,307 @@
from uuid import uuid4
import pytest
from core.app.segments import (
ArrayFileVariable,
ArrayNumberVariable,
ArrayObjectVariable,
ArrayStringVariable,
FileVariable,
FloatVariable,
IntegerVariable,
NoneSegment,
ObjectSegment,
SecretVariable,
StringVariable,
factory,
)
def test_string_variable():
test_data = {'value_type': 'string', 'name': 'test_text', 'value': 'Hello, World!'}
result = factory.build_variable_from_mapping(test_data)
assert isinstance(result, StringVariable)
def test_integer_variable():
test_data = {'value_type': 'number', 'name': 'test_int', 'value': 42}
result = factory.build_variable_from_mapping(test_data)
assert isinstance(result, IntegerVariable)
def test_float_variable():
test_data = {'value_type': 'number', 'name': 'test_float', 'value': 3.14}
result = factory.build_variable_from_mapping(test_data)
assert isinstance(result, FloatVariable)
def test_secret_variable():
test_data = {'value_type': 'secret', 'name': 'test_secret', 'value': 'secret_value'}
result = factory.build_variable_from_mapping(test_data)
assert isinstance(result, SecretVariable)
def test_invalid_value_type():
test_data = {'value_type': 'unknown', 'name': 'test_invalid', 'value': 'value'}
with pytest.raises(ValueError):
factory.build_variable_from_mapping(test_data)
def test_build_a_blank_string():
result = factory.build_variable_from_mapping(
{
'value_type': 'string',
'name': 'blank',
'value': '',
}
)
assert isinstance(result, StringVariable)
assert result.value == ''
def test_build_a_object_variable_with_none_value():
var = factory.build_segment(
{
'key1': None,
}
)
assert isinstance(var, ObjectSegment)
assert isinstance(var.value['key1'], NoneSegment)
def test_object_variable():
mapping = {
'id': str(uuid4()),
'value_type': 'object',
'name': 'test_object',
'description': 'Description of the variable.',
'value': {
'key1': {
'id': str(uuid4()),
'value_type': 'string',
'name': 'text',
'value': 'text',
'description': 'Description of the variable.',
},
'key2': {
'id': str(uuid4()),
'value_type': 'number',
'name': 'number',
'value': 1,
'description': 'Description of the variable.',
},
},
}
variable = factory.build_variable_from_mapping(mapping)
assert isinstance(variable, ObjectSegment)
assert isinstance(variable.value['key1'], StringVariable)
assert isinstance(variable.value['key2'], IntegerVariable)
def test_array_string_variable():
mapping = {
'id': str(uuid4()),
'value_type': 'array[string]',
'name': 'test_array',
'description': 'Description of the variable.',
'value': [
{
'id': str(uuid4()),
'value_type': 'string',
'name': 'text',
'value': 'text',
'description': 'Description of the variable.',
},
{
'id': str(uuid4()),
'value_type': 'string',
'name': 'text',
'value': 'text',
'description': 'Description of the variable.',
},
],
}
variable = factory.build_variable_from_mapping(mapping)
assert isinstance(variable, ArrayStringVariable)
assert isinstance(variable.value[0], StringVariable)
assert isinstance(variable.value[1], StringVariable)
def test_array_number_variable():
mapping = {
'id': str(uuid4()),
'value_type': 'array[number]',
'name': 'test_array',
'description': 'Description of the variable.',
'value': [
{
'id': str(uuid4()),
'value_type': 'number',
'name': 'number',
'value': 1,
'description': 'Description of the variable.',
},
{
'id': str(uuid4()),
'value_type': 'number',
'name': 'number',
'value': 2.0,
'description': 'Description of the variable.',
},
],
}
variable = factory.build_variable_from_mapping(mapping)
assert isinstance(variable, ArrayNumberVariable)
assert isinstance(variable.value[0], IntegerVariable)
assert isinstance(variable.value[1], FloatVariable)
def test_array_object_variable():
mapping = {
'id': str(uuid4()),
'value_type': 'array[object]',
'name': 'test_array',
'description': 'Description of the variable.',
'value': [
{
'id': str(uuid4()),
'value_type': 'object',
'name': 'object',
'description': 'Description of the variable.',
'value': {
'key1': {
'id': str(uuid4()),
'value_type': 'string',
'name': 'text',
'value': 'text',
'description': 'Description of the variable.',
},
'key2': {
'id': str(uuid4()),
'value_type': 'number',
'name': 'number',
'value': 1,
'description': 'Description of the variable.',
},
},
},
{
'id': str(uuid4()),
'value_type': 'object',
'name': 'object',
'description': 'Description of the variable.',
'value': {
'key1': {
'id': str(uuid4()),
'value_type': 'string',
'name': 'text',
'value': 'text',
'description': 'Description of the variable.',
},
'key2': {
'id': str(uuid4()),
'value_type': 'number',
'name': 'number',
'value': 1,
'description': 'Description of the variable.',
},
},
},
],
}
variable = factory.build_variable_from_mapping(mapping)
assert isinstance(variable, ArrayObjectVariable)
assert isinstance(variable.value[0], ObjectSegment)
assert isinstance(variable.value[1], ObjectSegment)
assert isinstance(variable.value[0].value['key1'], StringVariable)
assert isinstance(variable.value[0].value['key2'], IntegerVariable)
assert isinstance(variable.value[1].value['key1'], StringVariable)
assert isinstance(variable.value[1].value['key2'], IntegerVariable)
def test_file_variable():
mapping = {
'id': str(uuid4()),
'value_type': 'file',
'name': 'test_file',
'description': 'Description of the variable.',
'value': {
'id': str(uuid4()),
'tenant_id': 'tenant_id',
'type': 'image',
'transfer_method': 'local_file',
'url': 'url',
'related_id': 'related_id',
'extra_config': {
'image_config': {
'width': 100,
'height': 100,
},
},
'filename': 'filename',
'extension': 'extension',
'mime_type': 'mime_type',
},
}
variable = factory.build_variable_from_mapping(mapping)
assert isinstance(variable, FileVariable)
def test_array_file_variable():
mapping = {
'id': str(uuid4()),
'value_type': 'array[file]',
'name': 'test_array_file',
'description': 'Description of the variable.',
'value': [
{
'id': str(uuid4()),
'name': 'file',
'value_type': 'file',
'value': {
'id': str(uuid4()),
'tenant_id': 'tenant_id',
'type': 'image',
'transfer_method': 'local_file',
'url': 'url',
'related_id': 'related_id',
'extra_config': {
'image_config': {
'width': 100,
'height': 100,
},
},
'filename': 'filename',
'extension': 'extension',
'mime_type': 'mime_type',
},
},
{
'id': str(uuid4()),
'name': 'file',
'value_type': 'file',
'value': {
'id': str(uuid4()),
'tenant_id': 'tenant_id',
'type': 'image',
'transfer_method': 'local_file',
'url': 'url',
'related_id': 'related_id',
'extra_config': {
'image_config': {
'width': 100,
'height': 100,
},
},
'filename': 'filename',
'extension': 'extension',
'mime_type': 'mime_type',
},
},
],
}
variable = factory.build_variable_from_mapping(mapping)
assert isinstance(variable, ArrayFileVariable)
assert isinstance(variable.value[0], FileVariable)
assert isinstance(variable.value[1], FileVariable)

0
api/tests/unit_tests/core/app/test_segment.py → api/tests/unit_tests/core/app/segments/test_segment.py
View File

67
api/tests/unit_tests/core/app/test_variables.py → api/tests/unit_tests/core/app/segments/test_variables.py
View File

@ -2,49 +2,16 @@ import pytest
from pydantic import ValidationError
from core.app.segments import (
ArrayVariable,
ArrayAnyVariable,
FloatVariable,
IntegerVariable,
NoneSegment,
ObjectSegment,
ObjectVariable,
SecretVariable,
SegmentType,
StringVariable,
factory,
)
def test_string_variable():
test_data = {'value_type': 'string', 'name': 'test_text', 'value': 'Hello, World!'}
result = factory.build_variable_from_mapping(test_data)
assert isinstance(result, StringVariable)
def test_integer_variable():
test_data = {'value_type': 'number', 'name': 'test_int', 'value': 42}
result = factory.build_variable_from_mapping(test_data)
assert isinstance(result, IntegerVariable)
def test_float_variable():
test_data = {'value_type': 'number', 'name': 'test_float', 'value': 3.14}
result = factory.build_variable_from_mapping(test_data)
assert isinstance(result, FloatVariable)
def test_secret_variable():
test_data = {'value_type': 'secret', 'name': 'test_secret', 'value': 'secret_value'}
result = factory.build_variable_from_mapping(test_data)
assert isinstance(result, SecretVariable)
def test_invalid_value_type():
test_data = {'value_type': 'unknown', 'name': 'test_invalid', 'value': 'value'}
with pytest.raises(ValueError):
factory.build_variable_from_mapping(test_data)
def test_frozen_variables():
var = StringVariable(name='text', value='text')
with pytest.raises(ValidationError):
@ -65,34 +32,22 @@ def test_frozen_variables():
def test_variable_value_type_immutable():
with pytest.raises(ValidationError):
StringVariable(value_type=SegmentType.ARRAY, name='text', value='text')
StringVariable(value_type=SegmentType.ARRAY_ANY, name='text', value='text')
with pytest.raises(ValidationError):
StringVariable.model_validate({'value_type': 'not text', 'name': 'text', 'value': 'text'})
var = IntegerVariable(name='integer', value=42)
with pytest.raises(ValidationError):
IntegerVariable(value_type=SegmentType.ARRAY, name=var.name, value=var.value)
IntegerVariable(value_type=SegmentType.ARRAY_ANY, name=var.name, value=var.value)
var = FloatVariable(name='float', value=3.14)
with pytest.raises(ValidationError):
FloatVariable(value_type=SegmentType.ARRAY, name=var.name, value=var.value)
FloatVariable(value_type=SegmentType.ARRAY_ANY, name=var.name, value=var.value)
var = SecretVariable(name='secret', value='secret_value')
with pytest.raises(ValidationError):
SecretVariable(value_type=SegmentType.ARRAY, name=var.name, value=var.value)
def test_build_a_blank_string():
result = factory.build_variable_from_mapping(
{
'value_type': 'string',
'name': 'blank',
'value': '',
}
)
assert isinstance(result, StringVariable)
assert result.value == ''
SecretVariable(value_type=SegmentType.ARRAY_ANY, name=var.name, value=var.value)
def test_object_variable_to_object():
@ -105,7 +60,7 @@ def test_object_variable_to_object():
'key2': StringVariable(name='key2', value='value2'),
},
),
'key2': ArrayVariable(
'key2': ArrayAnyVariable(
name='array',
value=[
StringVariable(name='key5_1', value='value5_1'),
@ -137,13 +92,3 @@ def test_variable_to_object():
assert var.to_object() == 3.14
var = SecretVariable(name='secret', value='secret_value')
assert var.to_object() == 'secret_value'
def test_build_a_object_variable_with_none_value():
var = factory.build_segment(
{
'key1': None,
}
)
assert isinstance(var, ObjectSegment)
assert isinstance(var.value['key1'], NoneSegment)