From 6a3bef83788a73572de04d9001c532d74f3ae626 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Sat, 27 Jul 2024 14:43:51 +0800 Subject: [PATCH] feat(api/core/app/segments): Update segment types and variables (#6734) Signed-off-by: -LAN- --- api/core/app/segments/__init__.py | 16 +- api/core/app/segments/factory.py | 28 +- api/core/app/segments/segments.py | 43 ++- api/core/app/segments/types.py | 6 +- api/core/app/segments/variables.py | 27 +- .../core/app/segments/test_factory.py | 307 ++++++++++++++++++ .../core/app/{ => segments}/test_segment.py | 0 .../core/app/{ => segments}/test_variables.py | 67 +--- 8 files changed, 410 insertions(+), 84 deletions(-) create mode 100644 api/tests/unit_tests/core/app/segments/test_factory.py rename api/tests/unit_tests/core/app/{ => segments}/test_segment.py (100%) rename api/tests/unit_tests/core/app/{ => segments}/test_variables.py (54%) diff --git a/api/core/app/segments/__init__.py b/api/core/app/segments/__init__.py index b5d36bff3b..d5cd0a589c 100644 --- a/api/core/app/segments/__init__.py +++ b/api/core/app/segments/__init__.py @@ -1,6 +1,6 @@ from .segment_group import SegmentGroup from .segments import ( - ArraySegment, + ArrayAnySegment, FileSegment, FloatSegment, IntegerSegment, @@ -11,7 +11,11 @@ from .segments import ( ) from .types import SegmentType from .variables import ( - ArrayVariable, + ArrayAnyVariable, + ArrayFileVariable, + ArrayNumberVariable, + ArrayObjectVariable, + ArrayStringVariable, FileVariable, FloatVariable, IntegerVariable, @@ -29,7 +33,7 @@ __all__ = [ 'SecretVariable', 'FileVariable', 'StringVariable', - 'ArrayVariable', + 'ArrayAnyVariable', 'Variable', 'SegmentType', 'SegmentGroup', @@ -39,7 +43,11 @@ __all__ = [ 'IntegerSegment', 'FloatSegment', 'ObjectSegment', - 'ArraySegment', + 'ArrayAnySegment', 'FileSegment', 'StringSegment', + 'ArrayStringVariable', + 'ArrayNumberVariable', + 'ArrayObjectVariable', + 'ArrayFileVariable', ] diff --git a/api/core/app/segments/factory.py b/api/core/app/segments/factory.py index 8e77b43bb7..f62e44bf07 100644 --- a/api/core/app/segments/factory.py +++ b/api/core/app/segments/factory.py @@ -4,7 +4,7 @@ from typing import Any from core.file.file_obj import FileVar from .segments import ( - ArraySegment, + ArrayAnySegment, FileSegment, FloatSegment, IntegerSegment, @@ -15,8 +15,14 @@ from .segments import ( ) from .types import SegmentType from .variables import ( + ArrayFileVariable, + ArrayNumberVariable, + ArrayObjectVariable, + ArrayStringVariable, + FileVariable, FloatVariable, IntegerVariable, + ObjectVariable, SecretVariable, StringVariable, Variable, @@ -33,14 +39,28 @@ def build_variable_from_mapping(m: Mapping[str, Any], /) -> Variable: match value_type: case SegmentType.STRING: return StringVariable.model_validate(m) + case SegmentType.SECRET: + return SecretVariable.model_validate(m) case SegmentType.NUMBER if isinstance(value, int): return IntegerVariable.model_validate(m) case SegmentType.NUMBER if isinstance(value, float): return FloatVariable.model_validate(m) - case SegmentType.SECRET: - return SecretVariable.model_validate(m) case SegmentType.NUMBER if not isinstance(value, float | int): raise ValueError(f'invalid number value {value}') + case SegmentType.FILE: + return FileVariable.model_validate(m) + case SegmentType.OBJECT if isinstance(value, dict): + return ObjectVariable.model_validate( + {**m, 'value': {k: build_variable_from_mapping(v) for k, v in value.items()}} + ) + case SegmentType.ARRAY_STRING if isinstance(value, list): + return ArrayStringVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]}) + case SegmentType.ARRAY_NUMBER if isinstance(value, list): + return ArrayNumberVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]}) + case SegmentType.ARRAY_OBJECT if isinstance(value, list): + return ArrayObjectVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]}) + case SegmentType.ARRAY_FILE if isinstance(value, list): + return ArrayFileVariable.model_validate({**m, 'value': [build_variable_from_mapping(v) for v in value]}) raise ValueError(f'not supported value type {value_type}') @@ -60,7 +80,7 @@ def build_segment(value: Any, /) -> Segment: if isinstance(value, list): # TODO: Limit the depth of the array elements = [build_segment(v) for v in value] - return ArraySegment(value=elements) + return ArrayAnySegment(value=elements) if isinstance(value, FileVar): return FileSegment(value=value) raise ValueError(f'not supported value {value}') diff --git a/api/core/app/segments/segments.py b/api/core/app/segments/segments.py index f317054bc7..4227f154e6 100644 --- a/api/core/app/segments/segments.py +++ b/api/core/app/segments/segments.py @@ -62,6 +62,7 @@ class StringSegment(Segment): value_type: SegmentType = SegmentType.STRING value: str + class FloatSegment(Segment): value_type: SegmentType = SegmentType.NUMBER value: float @@ -72,6 +73,16 @@ class IntegerSegment(Segment): value: int +class FileSegment(Segment): + value_type: SegmentType = SegmentType.FILE + # TODO: embed FileVar in this model. + value: FileVar + + @property + def markdown(self) -> str: + return self.value.to_markdown() + + class ObjectSegment(Segment): value_type: SegmentType = SegmentType.OBJECT value: Mapping[str, Segment] @@ -96,9 +107,6 @@ class ObjectSegment(Segment): class ArraySegment(Segment): - value_type: SegmentType = SegmentType.ARRAY - value: Sequence[Segment] - @property def markdown(self) -> str: return '\n'.join(['- ' + item.markdown for item in self.value]) @@ -107,11 +115,26 @@ class ArraySegment(Segment): return [v.to_object() for v in self.value] -class FileSegment(Segment): - value_type: SegmentType = SegmentType.FILE - # TODO: embed FileVar in this model. - value: FileVar +class ArrayAnySegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_ANY + value: Sequence[Segment] - @property - def markdown(self) -> str: - return self.value.to_markdown() + +class ArrayStringSegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_STRING + value: Sequence[StringSegment] + + +class ArrayNumberSegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_NUMBER + value: Sequence[FloatSegment | IntegerSegment] + + +class ArrayObjectSegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_OBJECT + value: Sequence[ObjectSegment] + + +class ArrayFileSegment(ArraySegment): + value_type: SegmentType = SegmentType.ARRAY_FILE + value: Sequence[FileSegment] diff --git a/api/core/app/segments/types.py b/api/core/app/segments/types.py index 133755bbc6..a371058ef5 100644 --- a/api/core/app/segments/types.py +++ b/api/core/app/segments/types.py @@ -6,7 +6,11 @@ class SegmentType(str, Enum): NUMBER = 'number' STRING = 'string' SECRET = 'secret' - ARRAY = 'array' + ARRAY_ANY = 'array[any]' + ARRAY_STRING = 'array[string]' + ARRAY_NUMBER = 'array[number]' + ARRAY_OBJECT = 'array[object]' + ARRAY_FILE = 'array[file]' OBJECT = 'object' FILE = 'file' diff --git a/api/core/app/segments/variables.py b/api/core/app/segments/variables.py index ba55022726..ac26e16542 100644 --- a/api/core/app/segments/variables.py +++ b/api/core/app/segments/variables.py @@ -1,10 +1,13 @@ - from pydantic import Field from core.helper import encrypter from .segments import ( - ArraySegment, + ArrayAnySegment, + ArrayFileSegment, + ArrayNumberSegment, + ArrayObjectSegment, + ArrayStringSegment, FileSegment, FloatSegment, IntegerSegment, @@ -41,15 +44,31 @@ class IntegerVariable(IntegerSegment, Variable): pass +class FileVariable(FileSegment, Variable): + pass + + class ObjectVariable(ObjectSegment, Variable): pass -class ArrayVariable(ArraySegment, Variable): +class ArrayAnyVariable(ArrayAnySegment, Variable): pass -class FileVariable(FileSegment, Variable): +class ArrayStringVariable(ArrayStringSegment, Variable): + pass + + +class ArrayNumberVariable(ArrayNumberSegment, Variable): + pass + + +class ArrayObjectVariable(ArrayObjectSegment, Variable): + pass + + +class ArrayFileVariable(ArrayFileSegment, Variable): pass diff --git a/api/tests/unit_tests/core/app/segments/test_factory.py b/api/tests/unit_tests/core/app/segments/test_factory.py new file mode 100644 index 0000000000..85321ee374 --- /dev/null +++ b/api/tests/unit_tests/core/app/segments/test_factory.py @@ -0,0 +1,307 @@ +from uuid import uuid4 + +import pytest + +from core.app.segments import ( + ArrayFileVariable, + ArrayNumberVariable, + ArrayObjectVariable, + ArrayStringVariable, + FileVariable, + FloatVariable, + IntegerVariable, + NoneSegment, + ObjectSegment, + SecretVariable, + StringVariable, + factory, +) + + +def test_string_variable(): + test_data = {'value_type': 'string', 'name': 'test_text', 'value': 'Hello, World!'} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, StringVariable) + + +def test_integer_variable(): + test_data = {'value_type': 'number', 'name': 'test_int', 'value': 42} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, IntegerVariable) + + +def test_float_variable(): + test_data = {'value_type': 'number', 'name': 'test_float', 'value': 3.14} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, FloatVariable) + + +def test_secret_variable(): + test_data = {'value_type': 'secret', 'name': 'test_secret', 'value': 'secret_value'} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, SecretVariable) + + +def test_invalid_value_type(): + test_data = {'value_type': 'unknown', 'name': 'test_invalid', 'value': 'value'} + with pytest.raises(ValueError): + factory.build_variable_from_mapping(test_data) + + +def test_build_a_blank_string(): + result = factory.build_variable_from_mapping( + { + 'value_type': 'string', + 'name': 'blank', + 'value': '', + } + ) + assert isinstance(result, StringVariable) + assert result.value == '' + + +def test_build_a_object_variable_with_none_value(): + var = factory.build_segment( + { + 'key1': None, + } + ) + assert isinstance(var, ObjectSegment) + assert isinstance(var.value['key1'], NoneSegment) + + +def test_object_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'object', + 'name': 'test_object', + 'description': 'Description of the variable.', + 'value': { + 'key1': { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + 'key2': { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 1, + 'description': 'Description of the variable.', + }, + }, + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ObjectSegment) + assert isinstance(variable.value['key1'], StringVariable) + assert isinstance(variable.value['key2'], IntegerVariable) + + +def test_array_string_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'array[string]', + 'name': 'test_array', + 'description': 'Description of the variable.', + 'value': [ + { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + ], + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ArrayStringVariable) + assert isinstance(variable.value[0], StringVariable) + assert isinstance(variable.value[1], StringVariable) + + +def test_array_number_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'array[number]', + 'name': 'test_array', + 'description': 'Description of the variable.', + 'value': [ + { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 1, + 'description': 'Description of the variable.', + }, + { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 2.0, + 'description': 'Description of the variable.', + }, + ], + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ArrayNumberVariable) + assert isinstance(variable.value[0], IntegerVariable) + assert isinstance(variable.value[1], FloatVariable) + + +def test_array_object_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'array[object]', + 'name': 'test_array', + 'description': 'Description of the variable.', + 'value': [ + { + 'id': str(uuid4()), + 'value_type': 'object', + 'name': 'object', + 'description': 'Description of the variable.', + 'value': { + 'key1': { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + 'key2': { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 1, + 'description': 'Description of the variable.', + }, + }, + }, + { + 'id': str(uuid4()), + 'value_type': 'object', + 'name': 'object', + 'description': 'Description of the variable.', + 'value': { + 'key1': { + 'id': str(uuid4()), + 'value_type': 'string', + 'name': 'text', + 'value': 'text', + 'description': 'Description of the variable.', + }, + 'key2': { + 'id': str(uuid4()), + 'value_type': 'number', + 'name': 'number', + 'value': 1, + 'description': 'Description of the variable.', + }, + }, + }, + ], + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ArrayObjectVariable) + assert isinstance(variable.value[0], ObjectSegment) + assert isinstance(variable.value[1], ObjectSegment) + assert isinstance(variable.value[0].value['key1'], StringVariable) + assert isinstance(variable.value[0].value['key2'], IntegerVariable) + assert isinstance(variable.value[1].value['key1'], StringVariable) + assert isinstance(variable.value[1].value['key2'], IntegerVariable) + + +def test_file_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'file', + 'name': 'test_file', + 'description': 'Description of the variable.', + 'value': { + 'id': str(uuid4()), + 'tenant_id': 'tenant_id', + 'type': 'image', + 'transfer_method': 'local_file', + 'url': 'url', + 'related_id': 'related_id', + 'extra_config': { + 'image_config': { + 'width': 100, + 'height': 100, + }, + }, + 'filename': 'filename', + 'extension': 'extension', + 'mime_type': 'mime_type', + }, + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, FileVariable) + + +def test_array_file_variable(): + mapping = { + 'id': str(uuid4()), + 'value_type': 'array[file]', + 'name': 'test_array_file', + 'description': 'Description of the variable.', + 'value': [ + { + 'id': str(uuid4()), + 'name': 'file', + 'value_type': 'file', + 'value': { + 'id': str(uuid4()), + 'tenant_id': 'tenant_id', + 'type': 'image', + 'transfer_method': 'local_file', + 'url': 'url', + 'related_id': 'related_id', + 'extra_config': { + 'image_config': { + 'width': 100, + 'height': 100, + }, + }, + 'filename': 'filename', + 'extension': 'extension', + 'mime_type': 'mime_type', + }, + }, + { + 'id': str(uuid4()), + 'name': 'file', + 'value_type': 'file', + 'value': { + 'id': str(uuid4()), + 'tenant_id': 'tenant_id', + 'type': 'image', + 'transfer_method': 'local_file', + 'url': 'url', + 'related_id': 'related_id', + 'extra_config': { + 'image_config': { + 'width': 100, + 'height': 100, + }, + }, + 'filename': 'filename', + 'extension': 'extension', + 'mime_type': 'mime_type', + }, + }, + ], + } + variable = factory.build_variable_from_mapping(mapping) + assert isinstance(variable, ArrayFileVariable) + assert isinstance(variable.value[0], FileVariable) + assert isinstance(variable.value[1], FileVariable) diff --git a/api/tests/unit_tests/core/app/test_segment.py b/api/tests/unit_tests/core/app/segments/test_segment.py similarity index 100% rename from api/tests/unit_tests/core/app/test_segment.py rename to api/tests/unit_tests/core/app/segments/test_segment.py diff --git a/api/tests/unit_tests/core/app/test_variables.py b/api/tests/unit_tests/core/app/segments/test_variables.py similarity index 54% rename from api/tests/unit_tests/core/app/test_variables.py rename to api/tests/unit_tests/core/app/segments/test_variables.py index afed29e3cb..e3f513971a 100644 --- a/api/tests/unit_tests/core/app/test_variables.py +++ b/api/tests/unit_tests/core/app/segments/test_variables.py @@ -2,49 +2,16 @@ import pytest from pydantic import ValidationError from core.app.segments import ( - ArrayVariable, + ArrayAnyVariable, FloatVariable, IntegerVariable, - NoneSegment, - ObjectSegment, ObjectVariable, SecretVariable, SegmentType, StringVariable, - factory, ) -def test_string_variable(): - test_data = {'value_type': 'string', 'name': 'test_text', 'value': 'Hello, World!'} - result = factory.build_variable_from_mapping(test_data) - assert isinstance(result, StringVariable) - - -def test_integer_variable(): - test_data = {'value_type': 'number', 'name': 'test_int', 'value': 42} - result = factory.build_variable_from_mapping(test_data) - assert isinstance(result, IntegerVariable) - - -def test_float_variable(): - test_data = {'value_type': 'number', 'name': 'test_float', 'value': 3.14} - result = factory.build_variable_from_mapping(test_data) - assert isinstance(result, FloatVariable) - - -def test_secret_variable(): - test_data = {'value_type': 'secret', 'name': 'test_secret', 'value': 'secret_value'} - result = factory.build_variable_from_mapping(test_data) - assert isinstance(result, SecretVariable) - - -def test_invalid_value_type(): - test_data = {'value_type': 'unknown', 'name': 'test_invalid', 'value': 'value'} - with pytest.raises(ValueError): - factory.build_variable_from_mapping(test_data) - - def test_frozen_variables(): var = StringVariable(name='text', value='text') with pytest.raises(ValidationError): @@ -65,34 +32,22 @@ def test_frozen_variables(): def test_variable_value_type_immutable(): with pytest.raises(ValidationError): - StringVariable(value_type=SegmentType.ARRAY, name='text', value='text') + StringVariable(value_type=SegmentType.ARRAY_ANY, name='text', value='text') with pytest.raises(ValidationError): StringVariable.model_validate({'value_type': 'not text', 'name': 'text', 'value': 'text'}) var = IntegerVariable(name='integer', value=42) with pytest.raises(ValidationError): - IntegerVariable(value_type=SegmentType.ARRAY, name=var.name, value=var.value) + IntegerVariable(value_type=SegmentType.ARRAY_ANY, name=var.name, value=var.value) var = FloatVariable(name='float', value=3.14) with pytest.raises(ValidationError): - FloatVariable(value_type=SegmentType.ARRAY, name=var.name, value=var.value) + FloatVariable(value_type=SegmentType.ARRAY_ANY, name=var.name, value=var.value) var = SecretVariable(name='secret', value='secret_value') with pytest.raises(ValidationError): - SecretVariable(value_type=SegmentType.ARRAY, name=var.name, value=var.value) - - -def test_build_a_blank_string(): - result = factory.build_variable_from_mapping( - { - 'value_type': 'string', - 'name': 'blank', - 'value': '', - } - ) - assert isinstance(result, StringVariable) - assert result.value == '' + SecretVariable(value_type=SegmentType.ARRAY_ANY, name=var.name, value=var.value) def test_object_variable_to_object(): @@ -105,7 +60,7 @@ def test_object_variable_to_object(): 'key2': StringVariable(name='key2', value='value2'), }, ), - 'key2': ArrayVariable( + 'key2': ArrayAnyVariable( name='array', value=[ StringVariable(name='key5_1', value='value5_1'), @@ -137,13 +92,3 @@ def test_variable_to_object(): assert var.to_object() == 3.14 var = SecretVariable(name='secret', value='secret_value') assert var.to_object() == 'secret_value' - - -def test_build_a_object_variable_with_none_value(): - var = factory.build_segment( - { - 'key1': None, - } - ) - assert isinstance(var, ObjectSegment) - assert isinstance(var.value['key1'], NoneSegment)