This commit is contained in:
jyong 2025-04-22 16:08:58 +08:00
parent 5c4bf2a9e4
commit c7f4b41920
7 changed files with 321 additions and 149 deletions

View File

@ -1,21 +1,22 @@
from typing import Any
from core.datasource.datasource_tool.tool import DatasourceTool
from core.datasource.entities.datasource_entities import DatasourceProviderEntityWithPlugin, DatasourceProviderType
from core.plugin.manager.tool import PluginToolManager
from core.tools.__base.tool_runtime import ToolRuntime
from core.tools.builtin_tool.provider import BuiltinToolProviderController
from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin, ToolProviderType
from core.tools.errors import ToolProviderCredentialValidationError
from core.tools.plugin_tool.tool import PluginTool
class PluginToolProviderController(BuiltinToolProviderController):
entity: ToolProviderEntityWithPlugin
class DatasourceToolProviderController(BuiltinToolProviderController):
entity: DatasourceProviderEntityWithPlugin
tenant_id: str
plugin_id: str
plugin_unique_identifier: str
def __init__(
self, entity: ToolProviderEntityWithPlugin, plugin_id: str, plugin_unique_identifier: str, tenant_id: str
self, entity: DatasourceProviderEntityWithPlugin, plugin_id: str, plugin_unique_identifier: str, tenant_id: str
) -> None:
self.entity = entity
self.tenant_id = tenant_id
@ -23,13 +24,13 @@ class PluginToolProviderController(BuiltinToolProviderController):
self.plugin_unique_identifier = plugin_unique_identifier
@property
def provider_type(self) -> ToolProviderType:
def provider_type(self) -> DatasourceProviderType:
"""
returns the type of the provider
:return: type of the provider
"""
return ToolProviderType.PLUGIN
return DatasourceProviderType.RAG_PIPELINE
def _validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
"""
@ -44,36 +45,36 @@ class PluginToolProviderController(BuiltinToolProviderController):
):
raise ToolProviderCredentialValidationError("Invalid credentials")
def get_tool(self, tool_name: str) -> PluginTool: # type: ignore
def get_datasource(self, datasource_name: str) -> DatasourceTool: # type: ignore
"""
return tool with given name
return datasource with given name
"""
tool_entity = next(
(tool_entity for tool_entity in self.entity.tools if tool_entity.identity.name == tool_name), None
datasource_entity = next(
(datasource_entity for datasource_entity in self.entity.datasources if datasource_entity.identity.name == datasource_name), None
)
if not tool_entity:
raise ValueError(f"Tool with name {tool_name} not found")
if not datasource_entity:
raise ValueError(f"Datasource with name {datasource_name} not found")
return PluginTool(
entity=tool_entity,
return DatasourceTool(
entity=datasource_entity,
runtime=ToolRuntime(tenant_id=self.tenant_id),
tenant_id=self.tenant_id,
icon=self.entity.identity.icon,
plugin_unique_identifier=self.plugin_unique_identifier,
)
def get_tools(self) -> list[PluginTool]: # type: ignore
def get_datasources(self) -> list[DatasourceTool]: # type: ignore
"""
get all tools
get all datasources
"""
return [
PluginTool(
entity=tool_entity,
DatasourceTool(
entity=datasource_entity,
runtime=ToolRuntime(tenant_id=self.tenant_id),
tenant_id=self.tenant_id,
icon=self.entity.identity.icon,
plugin_unique_identifier=self.plugin_unique_identifier,
)
for tool_entity in self.entity.tools
for datasource_entity in self.entity.datasources
]

View File

@ -1,6 +1,7 @@
from collections.abc import Generator
from typing import Any, Optional
from core.datasource.entities.datasource_entities import DatasourceEntity, DatasourceParameter, DatasourceProviderType
from core.plugin.manager.tool import PluginToolManager
from core.plugin.utils.converter import convert_parameters_to_plugin_format
from core.tools.__base.tool import Tool
@ -8,14 +9,14 @@ from core.tools.__base.tool_runtime import ToolRuntime
from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, ToolParameter, ToolProviderType
class PluginTool(Tool):
class DatasourceTool(Tool):
tenant_id: str
icon: str
plugin_unique_identifier: str
runtime_parameters: Optional[list[ToolParameter]]
runtime_parameters: Optional[list[DatasourceParameter]]
def __init__(
self, entity: ToolEntity, runtime: ToolRuntime, tenant_id: str, icon: str, plugin_unique_identifier: str
self, entity: DatasourceEntity, runtime: ToolRuntime, tenant_id: str, icon: str, plugin_unique_identifier: str
) -> None:
super().__init__(entity, runtime)
self.tenant_id = tenant_id
@ -23,20 +24,44 @@ class PluginTool(Tool):
self.plugin_unique_identifier = plugin_unique_identifier
self.runtime_parameters = None
def tool_provider_type(self) -> ToolProviderType:
return ToolProviderType.PLUGIN
def datasource_provider_type(self) -> DatasourceProviderType:
return DatasourceProviderType.RAG_PIPELINE
def _invoke(
def _invoke_first_step(
self,
user_id: str,
tool_parameters: dict[str, Any],
datasource_parameters: dict[str, Any],
conversation_id: Optional[str] = None,
app_id: Optional[str] = None,
rag_pipeline_id: Optional[str] = None,
message_id: Optional[str] = None,
) -> Generator[ToolInvokeMessage, None, None]:
manager = PluginToolManager()
tool_parameters = convert_parameters_to_plugin_format(tool_parameters)
datasource_parameters = convert_parameters_to_plugin_format(datasource_parameters)
yield from manager.invoke_first_step(
tenant_id=self.tenant_id,
user_id=user_id,
tool_provider=self.entity.identity.provider,
tool_name=self.entity.identity.name,
credentials=self.runtime.credentials,
tool_parameters=tool_parameters,
conversation_id=conversation_id,
app_id=app_id,
message_id=message_id,
)
def _invoke_second_step(
self,
user_id: str,
datasource_parameters: dict[str, Any],
conversation_id: Optional[str] = None,
rag_pipeline_id: Optional[str] = None,
message_id: Optional[str] = None,
) -> Generator[ToolInvokeMessage, None, None]:
manager = PluginToolManager()
datasource_parameters = convert_parameters_to_plugin_format(datasource_parameters)
yield from manager.invoke(
tenant_id=self.tenant_id,
@ -50,8 +75,9 @@ class PluginTool(Tool):
message_id=message_id,
)
def fork_tool_runtime(self, runtime: ToolRuntime) -> "PluginTool":
return PluginTool(
def fork_tool_runtime(self, runtime: ToolRuntime) -> "DatasourceTool":
return DatasourceTool(
entity=self.entity,
runtime=runtime,
tenant_id=self.tenant_id,
@ -64,7 +90,7 @@ class PluginTool(Tool):
conversation_id: Optional[str] = None,
app_id: Optional[str] = None,
message_id: Optional[str] = None,
) -> list[ToolParameter]:
) -> list[DatasourceParameter]:
"""
get the runtime parameters
"""

View File

@ -1,72 +0,0 @@
from typing import Literal, Optional
from pydantic import BaseModel, Field, field_validator
from core.model_runtime.utils.encoders import jsonable_encoder
from core.tools.__base.tool import ToolParameter
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_entities import ToolProviderType
class ToolApiEntity(BaseModel):
author: str
name: str # identifier
label: I18nObject # label
description: I18nObject
parameters: Optional[list[ToolParameter]] = None
labels: list[str] = Field(default_factory=list)
output_schema: Optional[dict] = None
ToolProviderTypeApiLiteral = Optional[Literal["builtin", "api", "workflow"]]
class ToolProviderApiEntity(BaseModel):
id: str
author: str
name: str # identifier
description: I18nObject
icon: str | dict
label: I18nObject # label
type: ToolProviderType
masked_credentials: Optional[dict] = None
original_credentials: Optional[dict] = None
is_team_authorization: bool = False
allow_delete: bool = True
plugin_id: Optional[str] = Field(default="", description="The plugin id of the tool")
plugin_unique_identifier: Optional[str] = Field(default="", description="The unique identifier of the tool")
tools: list[ToolApiEntity] = Field(default_factory=list)
labels: list[str] = Field(default_factory=list)
@field_validator("tools", mode="before")
@classmethod
def convert_none_to_empty_list(cls, v):
return v if v is not None else []
def to_dict(self) -> dict:
# -------------
# overwrite tool parameter types for temp fix
tools = jsonable_encoder(self.tools)
for tool in tools:
if tool.get("parameters"):
for parameter in tool.get("parameters"):
if parameter.get("type") == ToolParameter.ToolParameterType.SYSTEM_FILES.value:
parameter["type"] = "files"
# -------------
return {
"id": self.id,
"author": self.author,
"name": self.name,
"plugin_id": self.plugin_id,
"plugin_unique_identifier": self.plugin_unique_identifier,
"description": self.description.to_dict(),
"icon": self.icon,
"label": self.label.to_dict(),
"type": self.type.value,
"team_credentials": self.masked_credentials,
"is_team_authorization": self.is_team_authorization,
"allow_delete": self.allow_delete,
"tools": tools,
"labels": self.labels,
}

View File

@ -38,20 +38,15 @@ class ToolLabelEnum(Enum):
OTHER = "other"
class ToolProviderType(enum.StrEnum):
class DatasourceProviderType(enum.StrEnum):
"""
Enum class for tool provider
Enum class for datasource provider
"""
PLUGIN = "plugin"
BUILT_IN = "builtin"
WORKFLOW = "workflow"
API = "api"
APP = "app"
DATASET_RETRIEVAL = "dataset-retrieval"
RAG_PIPELINE = "rag_pipeline"
@classmethod
def value_of(cls, value: str) -> "ToolProviderType":
def value_of(cls, value: str) -> "DatasourceProviderType":
"""
Get value of given mode.
@ -211,12 +206,12 @@ class ToolInvokeMessageBinary(BaseModel):
file_var: Optional[dict[str, Any]] = None
class ToolParameter(PluginParameter):
class DatasourceParameter(PluginParameter):
"""
Overrides type
"""
class ToolParameterType(enum.StrEnum):
class DatasourceParameterType(enum.StrEnum):
"""
removes TOOLS_SELECTOR from PluginParameterType
"""
@ -240,14 +235,14 @@ class ToolParameter(PluginParameter):
def cast_value(self, value: Any):
return cast_parameter_value(self, value)
class ToolParameterForm(Enum):
class DatasourceParameterForm(Enum):
SCHEMA = "schema" # should be set while adding tool
FORM = "form" # should be set before invoking tool
LLM = "llm" # will be set by LLM
type: ToolParameterType = Field(..., description="The type of the parameter")
type: DatasourceParameterType = Field(..., description="The type of the parameter")
human_description: Optional[I18nObject] = Field(default=None, description="The description presented to the user")
form: ToolParameterForm = Field(..., description="The form of the parameter, schema/form/llm")
form: DatasourceParameterForm = Field(..., description="The form of the parameter, schema/form/llm")
llm_description: Optional[str] = None
@classmethod
@ -255,12 +250,12 @@ class ToolParameter(PluginParameter):
cls,
name: str,
llm_description: str,
typ: ToolParameterType,
typ: DatasourceParameterType,
required: bool,
options: Optional[list[str]] = None,
) -> "ToolParameter":
) -> "DatasourceParameter":
"""
get a simple tool parameter
get a simple datasource parameter
:param name: the name of the parameter
:param llm_description: the description presented to the LLM
@ -306,7 +301,7 @@ class ToolProviderIdentity(BaseModel):
)
class ToolIdentity(BaseModel):
class DatasourceIdentity(BaseModel):
author: str = Field(..., description="The author of the tool")
name: str = Field(..., description="The name of the tool")
label: I18nObject = Field(..., description="The label of the tool")
@ -314,15 +309,15 @@ class ToolIdentity(BaseModel):
icon: Optional[str] = None
class ToolDescription(BaseModel):
class DatasourceDescription(BaseModel):
human: I18nObject = Field(..., description="The description presented to the user")
llm: str = Field(..., description="The description presented to the LLM")
class ToolEntity(BaseModel):
identity: ToolIdentity
parameters: list[ToolParameter] = Field(default_factory=list)
description: Optional[ToolDescription] = None
class DatasourceEntity(BaseModel):
identity: DatasourceIdentity
parameters: list[DatasourceParameter] = Field(default_factory=list)
description: Optional[DatasourceDescription] = None
output_schema: Optional[dict] = None
has_runtime_parameters: bool = Field(default=False, description="Whether the tool has runtime parameters")
@ -331,7 +326,7 @@ class ToolEntity(BaseModel):
@field_validator("parameters", mode="before")
@classmethod
def set_parameters(cls, v, validation_info: ValidationInfo) -> list[ToolParameter]:
def set_parameters(cls, v, validation_info: ValidationInfo) -> list[DatasourceParameter]:
return v or []
@ -341,8 +336,8 @@ class ToolProviderEntity(BaseModel):
credentials_schema: list[ProviderConfig] = Field(default_factory=list)
class ToolProviderEntityWithPlugin(ToolProviderEntity):
tools: list[ToolEntity] = Field(default_factory=list)
class DatasourceProviderEntityWithPlugin(ToolProviderEntity):
datasources: list[DatasourceEntity] = Field(default_factory=list)
class WorkflowToolParameterConfiguration(BaseModel):
@ -352,12 +347,12 @@ class WorkflowToolParameterConfiguration(BaseModel):
name: str = Field(..., description="The name of the parameter")
description: str = Field(..., description="The description of the parameter")
form: ToolParameter.ToolParameterForm = Field(..., description="The form of the parameter")
form: DatasourceParameter.DatasourceParameterForm = Field(..., description="The form of the parameter")
class ToolInvokeMeta(BaseModel):
class DatasourceInvokeMeta(BaseModel):
"""
Tool invoke meta
Datasource invoke meta
"""
time_cost: float = Field(..., description="The time cost of the tool invoke")
@ -365,16 +360,16 @@ class ToolInvokeMeta(BaseModel):
tool_config: Optional[dict] = None
@classmethod
def empty(cls) -> "ToolInvokeMeta":
def empty(cls) -> "DatasourceInvokeMeta":
"""
Get an empty instance of ToolInvokeMeta
Get an empty instance of DatasourceInvokeMeta
"""
return cls(time_cost=0.0, error=None, tool_config={})
@classmethod
def error_instance(cls, error: str) -> "ToolInvokeMeta":
def error_instance(cls, error: str) -> "DatasourceInvokeMeta":
"""
Get an instance of ToolInvokeMeta with error
Get an instance of DatasourceInvokeMeta with error
"""
return cls(time_cost=0.0, error=error, tool_config={})
@ -386,9 +381,9 @@ class ToolInvokeMeta(BaseModel):
}
class ToolLabel(BaseModel):
class DatasourceLabel(BaseModel):
"""
Tool label
Datasource label
"""
name: str = Field(..., description="The name of the tool")
@ -396,32 +391,30 @@ class ToolLabel(BaseModel):
icon: str = Field(..., description="The icon of the tool")
class ToolInvokeFrom(Enum):
class DatasourceInvokeFrom(Enum):
"""
Enum class for tool invoke
Enum class for datasource invoke
"""
WORKFLOW = "workflow"
AGENT = "agent"
PLUGIN = "plugin"
RAG_PIPELINE = "rag_pipeline"
class ToolSelector(BaseModel):
class DatasourceSelector(BaseModel):
dify_model_identity: str = TOOL_SELECTOR_MODEL_IDENTITY
class Parameter(BaseModel):
name: str = Field(..., description="The name of the parameter")
type: ToolParameter.ToolParameterType = Field(..., description="The type of the parameter")
type: DatasourceParameter.DatasourceParameterType = Field(..., description="The type of the parameter")
required: bool = Field(..., description="Whether the parameter is required")
description: str = Field(..., description="The description of the parameter")
default: Optional[Union[int, float, str]] = None
options: Optional[list[PluginParameterOption]] = None
provider_id: str = Field(..., description="The id of the provider")
tool_name: str = Field(..., description="The name of the tool")
tool_description: str = Field(..., description="The description of the tool")
tool_configuration: Mapping[str, Any] = Field(..., description="Configuration, type form")
tool_parameters: Mapping[str, Parameter] = Field(..., description="Parameters, type llm")
datasource_name: str = Field(..., description="The name of the datasource")
datasource_description: str = Field(..., description="The description of the datasource")
datasource_configuration: Mapping[str, Any] = Field(..., description="Configuration, type form")
datasource_parameters: Mapping[str, Parameter] = Field(..., description="Parameters, type llm")
def to_plugin_parameter(self) -> dict[str, Any]:
return self.model_dump()

View File

@ -0,0 +1,217 @@
from collections.abc import Generator
from typing import Any, Optional
from pydantic import BaseModel
from core.plugin.entities.plugin import GenericProviderID, ToolProviderID
from core.plugin.entities.plugin_daemon import PluginBasicBooleanResponse, PluginToolProviderEntity
from core.plugin.manager.base import BasePluginManager
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
class PluginDatasourceManager(BasePluginManager):
def fetch_datasource_providers(self, tenant_id: str) -> list[PluginToolProviderEntity]:
"""
Fetch datasource providers for the given tenant.
"""
def transformer(json_response: dict[str, Any]) -> dict:
for provider in json_response.get("data", []):
declaration = provider.get("declaration", {}) or {}
provider_name = declaration.get("identity", {}).get("name")
for tool in declaration.get("tools", []):
tool["identity"]["provider"] = provider_name
return json_response
response = self._request_with_plugin_daemon_response(
"GET",
f"plugin/{tenant_id}/management/datasources",
list[PluginToolProviderEntity],
params={"page": 1, "page_size": 256},
transformer=transformer,
)
for provider in response:
provider.declaration.identity.name = f"{provider.plugin_id}/{provider.declaration.identity.name}"
# override the provider name for each tool to plugin_id/provider_name
for tool in provider.declaration.tools:
tool.identity.provider = provider.declaration.identity.name
return response
def fetch_datasource_provider(self, tenant_id: str, provider: str) -> PluginToolProviderEntity:
"""
Fetch datasource provider for the given tenant and plugin.
"""
tool_provider_id = ToolProviderID(provider)
def transformer(json_response: dict[str, Any]) -> dict:
data = json_response.get("data")
if data:
for datasource in data.get("declaration", {}).get("datasources", []):
datasource["identity"]["provider"] = tool_provider_id.provider_name
return json_response
response = self._request_with_plugin_daemon_response(
"GET",
f"plugin/{tenant_id}/management/datasources",
PluginToolProviderEntity,
params={"provider": tool_provider_id.provider_name, "plugin_id": tool_provider_id.plugin_id},
transformer=transformer,
)
response.declaration.identity.name = f"{response.plugin_id}/{response.declaration.identity.name}"
# override the provider name for each tool to plugin_id/provider_name
for tool in response.declaration.tools:
tool.identity.provider = response.declaration.identity.name
return response
def invoke_first_step(
self,
tenant_id: str,
user_id: str,
datasource_provider: str,
datasource_name: str,
credentials: dict[str, Any],
datasource_parameters: dict[str, Any],
) -> Generator[ToolInvokeMessage, None, None]:
"""
Invoke the datasource with the given tenant, user, plugin, provider, name, credentials and parameters.
"""
datasource_provider_id = GenericProviderID(datasource_provider)
response = self._request_with_plugin_daemon_response_stream(
"POST",
f"plugin/{tenant_id}/dispatch/datasource/invoke_first_step",
ToolInvokeMessage,
data={
"user_id": user_id,
"data": {
"provider": datasource_provider_id.provider_name,
"datasource": datasource_name,
"credentials": credentials,
"datasource_parameters": datasource_parameters,
},
},
headers={
"X-Plugin-ID": datasource_provider_id.plugin_id,
"Content-Type": "application/json",
},
)
return response
def invoke_second_step(
self,
tenant_id: str,
user_id: str,
datasource_provider: str,
datasource_name: str,
credentials: dict[str, Any],
datasource_parameters: dict[str, Any],
) -> Generator[ToolInvokeMessage, None, None]:
"""
Invoke the datasource with the given tenant, user, plugin, provider, name, credentials and parameters.
"""
datasource_provider_id = GenericProviderID(datasource_provider)
response = self._request_with_plugin_daemon_response_stream(
"POST",
f"plugin/{tenant_id}/dispatch/datasource/invoke_second_step",
ToolInvokeMessage,
data={
"user_id": user_id,
"data": {
"provider": datasource_provider_id.provider_name,
"datasource": datasource_name,
"credentials": credentials,
"datasource_parameters": datasource_parameters,
},
},
headers={
"X-Plugin-ID": datasource_provider_id.plugin_id,
"Content-Type": "application/json",
},
)
return response
def validate_provider_credentials(
self, tenant_id: str, user_id: str, provider: str, credentials: dict[str, Any]
) -> bool:
"""
validate the credentials of the provider
"""
tool_provider_id = GenericProviderID(provider)
response = self._request_with_plugin_daemon_response_stream(
"POST",
f"plugin/{tenant_id}/dispatch/tool/validate_credentials",
PluginBasicBooleanResponse,
data={
"user_id": user_id,
"data": {
"provider": tool_provider_id.provider_name,
"credentials": credentials,
},
},
headers={
"X-Plugin-ID": tool_provider_id.plugin_id,
"Content-Type": "application/json",
},
)
for resp in response:
return resp.result
return False
def get_runtime_parameters(
self,
tenant_id: str,
user_id: str,
provider: str,
credentials: dict[str, Any],
datasource: str,
conversation_id: Optional[str] = None,
app_id: Optional[str] = None,
message_id: Optional[str] = None,
) -> list[ToolParameter]:
"""
get the runtime parameters of the datasource
"""
datasource_provider_id = GenericProviderID(provider)
class RuntimeParametersResponse(BaseModel):
parameters: list[ToolParameter]
response = self._request_with_plugin_daemon_response_stream(
"POST",
f"plugin/{tenant_id}/dispatch/datasource/get_runtime_parameters",
RuntimeParametersResponse,
data={
"user_id": user_id,
"conversation_id": conversation_id,
"app_id": app_id,
"message_id": message_id,
"data": {
"provider": datasource_provider_id.provider_name,
"datasource": datasource,
"credentials": credentials,
},
},
headers={
"X-Plugin-ID": datasource_provider_id.plugin_id,
"Content-Type": "application/json",
},
)
for resp in response:
return resp.parameters
return []

View File

@ -1,5 +1,6 @@
from typing import Any
from core.datasource.entities.datasource_entities import DatasourceSelector
from core.file.models import File
from core.tools.entities.tool_entities import ToolSelector
@ -18,4 +19,10 @@ def convert_parameters_to_plugin_format(parameters: dict[str, Any]) -> dict[str,
parameters[parameter_name] = []
for p in parameter:
parameters[parameter_name].append(p.to_plugin_parameter())
elif isinstance(parameter, DatasourceSelector):
parameters[parameter_name] = parameter.to_plugin_parameter()
elif isinstance(parameter, list) and all(isinstance(p, DatasourceSelector) for p in parameter):
parameters[parameter_name] = []
for p in parameter:
parameters[parameter_name].append(p.to_plugin_parameter())
return parameters