mirror of
https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git
synced 2025-08-20 06:39:16 +08:00
feat: export dsl with dependencies
This commit is contained in:
parent
21fd58caf9
commit
f6136427a4
@ -11,6 +11,7 @@ from controllers.console.wraps import (
|
|||||||
cloud_edition_billing_resource_check,
|
cloud_edition_billing_resource_check,
|
||||||
setup_required,
|
setup_required,
|
||||||
)
|
)
|
||||||
|
from core.model_runtime.utils.encoders import jsonable_encoder
|
||||||
from core.ops.ops_trace_manager import OpsTraceManager
|
from core.ops.ops_trace_manager import OpsTraceManager
|
||||||
from fields.app_fields import (
|
from fields.app_fields import (
|
||||||
app_detail_fields,
|
app_detail_fields,
|
||||||
@ -90,6 +91,28 @@ class AppListApi(Resource):
|
|||||||
return app, 201
|
return app, 201
|
||||||
|
|
||||||
|
|
||||||
|
class AppImportDependenciesCheckApi(Resource):
|
||||||
|
@setup_required
|
||||||
|
@login_required
|
||||||
|
@account_initialization_required
|
||||||
|
@cloud_edition_billing_resource_check("apps")
|
||||||
|
def post(self):
|
||||||
|
"""Check dependencies"""
|
||||||
|
# The role of the current user in the ta table must be admin, owner, or editor
|
||||||
|
if not current_user.is_editor:
|
||||||
|
raise Forbidden()
|
||||||
|
|
||||||
|
parser = reqparse.RequestParser()
|
||||||
|
parser.add_argument("data", type=str, required=True, nullable=False, location="json")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
leaked_dependencies = AppDslService.check_dependencies(
|
||||||
|
tenant_id=current_user.current_tenant_id, data=args["data"], account=current_user
|
||||||
|
)
|
||||||
|
|
||||||
|
return jsonable_encoder({"leaked": leaked_dependencies}), 200
|
||||||
|
|
||||||
|
|
||||||
class AppImportApi(Resource):
|
class AppImportApi(Resource):
|
||||||
@setup_required
|
@setup_required
|
||||||
@login_required
|
@login_required
|
||||||
@ -365,6 +388,7 @@ class AppTraceApi(Resource):
|
|||||||
|
|
||||||
|
|
||||||
api.add_resource(AppListApi, "/apps")
|
api.add_resource(AppListApi, "/apps")
|
||||||
|
api.add_resource(AppImportDependenciesCheckApi, "/apps/import/dependencies/check")
|
||||||
api.add_resource(AppImportApi, "/apps/import")
|
api.add_resource(AppImportApi, "/apps/import")
|
||||||
api.add_resource(AppImportFromUrlApi, "/apps/import/url")
|
api.add_resource(AppImportFromUrlApi, "/apps/import/url")
|
||||||
api.add_resource(AppApi, "/apps/<uuid:app_id>")
|
api.add_resource(AppApi, "/apps/<uuid:app_id>")
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
import datetime
|
import datetime
|
||||||
|
import re
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
@ -95,21 +96,24 @@ class PluginDeclaration(BaseModel):
|
|||||||
return values
|
return values
|
||||||
|
|
||||||
|
|
||||||
class PluginEntity(BasePluginEntity):
|
class PluginInstallation(BasePluginEntity):
|
||||||
name: str
|
|
||||||
plugin_id: str
|
|
||||||
plugin_unique_identifier: str
|
|
||||||
declaration: PluginDeclaration
|
|
||||||
installation_id: str
|
|
||||||
tenant_id: str
|
tenant_id: str
|
||||||
endpoints_setups: int
|
endpoints_setups: int
|
||||||
endpoints_active: int
|
endpoints_active: int
|
||||||
runtime_type: str
|
runtime_type: str
|
||||||
|
source: PluginInstallationSource
|
||||||
|
meta: Mapping[str, Any]
|
||||||
|
plugin_id: str
|
||||||
|
plugin_unique_identifier: str
|
||||||
|
|
||||||
|
|
||||||
|
class PluginEntity(PluginInstallation):
|
||||||
|
name: str
|
||||||
|
declaration: PluginDeclaration
|
||||||
|
installation_id: str
|
||||||
version: str
|
version: str
|
||||||
latest_version: Optional[str] = None
|
latest_version: Optional[str] = None
|
||||||
latest_unique_identifier: Optional[str] = None
|
latest_unique_identifier: Optional[str] = None
|
||||||
source: PluginInstallationSource
|
|
||||||
meta: Mapping[str, Any]
|
|
||||||
|
|
||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def set_plugin_id(self):
|
def set_plugin_id(self):
|
||||||
@ -127,3 +131,60 @@ class GithubPackage(BaseModel):
|
|||||||
class GithubVersion(BaseModel):
|
class GithubVersion(BaseModel):
|
||||||
repo: str
|
repo: str
|
||||||
version: str
|
version: str
|
||||||
|
|
||||||
|
|
||||||
|
class GenericProviderID:
|
||||||
|
organization: str
|
||||||
|
plugin_name: str
|
||||||
|
provider_name: str
|
||||||
|
|
||||||
|
def to_string(self) -> str:
|
||||||
|
return str(self)
|
||||||
|
|
||||||
|
def __str__(self) -> str:
|
||||||
|
return f"{self.organization}/{self.plugin_name}/{self.provider_name}"
|
||||||
|
|
||||||
|
def __init__(self, value: str) -> None:
|
||||||
|
# check if the value is a valid plugin id with format: $organization/$plugin_name/$provider_name
|
||||||
|
if not re.match(r"^[a-z0-9_-]+\/[a-z0-9_-]+\/[a-z0-9_-]+$", value):
|
||||||
|
# check if matches [a-z0-9_-]+, if yes, append with langgenius/$value/$value
|
||||||
|
if re.match(r"^[a-z0-9_-]+$", value):
|
||||||
|
value = f"langgenius/{value}/{value}"
|
||||||
|
else:
|
||||||
|
raise ValueError("Invalid plugin id")
|
||||||
|
|
||||||
|
self.organization, self.plugin_name, self.provider_name = value.split("/")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def plugin_id(self) -> str:
|
||||||
|
return f"{self.organization}/{self.plugin_name}"
|
||||||
|
|
||||||
|
|
||||||
|
class PluginDependency(BaseModel):
|
||||||
|
class Type(str, Enum):
|
||||||
|
Github = "github"
|
||||||
|
Marketplace = "marketplace"
|
||||||
|
Package = "package"
|
||||||
|
|
||||||
|
class Github(BaseModel):
|
||||||
|
repo: str
|
||||||
|
version: str
|
||||||
|
package: str
|
||||||
|
github_plugin_unique_identifier: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def plugin_unique_identifier(self) -> str:
|
||||||
|
return self.github_plugin_unique_identifier
|
||||||
|
|
||||||
|
class Marketplace(BaseModel):
|
||||||
|
marketplace_plugin_unique_identifier: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def plugin_unique_identifier(self) -> str:
|
||||||
|
return self.marketplace_plugin_unique_identifier
|
||||||
|
|
||||||
|
class Package(BaseModel):
|
||||||
|
plugin_unique_identifier: str
|
||||||
|
|
||||||
|
type: Type
|
||||||
|
value: Github | Marketplace | Package
|
||||||
|
@ -2,7 +2,12 @@ from collections.abc import Sequence
|
|||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from core.plugin.entities.plugin import PluginDeclaration, PluginEntity, PluginInstallationSource
|
from core.plugin.entities.plugin import (
|
||||||
|
PluginDeclaration,
|
||||||
|
PluginEntity,
|
||||||
|
PluginInstallation,
|
||||||
|
PluginInstallationSource,
|
||||||
|
)
|
||||||
from core.plugin.entities.plugin_daemon import PluginInstallTask, PluginInstallTaskStartResponse, PluginUploadResponse
|
from core.plugin.entities.plugin_daemon import PluginInstallTask, PluginInstallTaskStartResponse, PluginUploadResponse
|
||||||
from core.plugin.manager.base import BasePluginManager
|
from core.plugin.manager.base import BasePluginManager
|
||||||
|
|
||||||
@ -128,6 +133,30 @@ class PluginInstallationManager(BasePluginManager):
|
|||||||
params={"plugin_unique_identifier": plugin_unique_identifier},
|
params={"plugin_unique_identifier": plugin_unique_identifier},
|
||||||
).declaration
|
).declaration
|
||||||
|
|
||||||
|
def fetch_plugin_installation_by_ids(self, tenant_id: str, plugin_ids: Sequence[str]) -> list[PluginInstallation]:
|
||||||
|
"""
|
||||||
|
Fetch plugin installations by ids.
|
||||||
|
"""
|
||||||
|
return self._request_with_plugin_daemon_response(
|
||||||
|
"POST",
|
||||||
|
f"plugin/{tenant_id}/management/installation/fetch/batch",
|
||||||
|
list[PluginInstallation],
|
||||||
|
data={"plugin_ids": plugin_ids},
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
|
||||||
|
def fetch_missing_dependencies(self, tenant_id: str, plugin_unique_identifiers: list[str]) -> list[str]:
|
||||||
|
"""
|
||||||
|
Fetch missing dependencies
|
||||||
|
"""
|
||||||
|
return self._request_with_plugin_daemon_response(
|
||||||
|
"POST",
|
||||||
|
f"plugin/{tenant_id}/management/installation/missing",
|
||||||
|
list[str],
|
||||||
|
data={"plugin_unique_identifiers": plugin_unique_identifiers},
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
|
||||||
def uninstall(self, tenant_id: str, plugin_installation_id: str) -> bool:
|
def uninstall(self, tenant_id: str, plugin_installation_id: str) -> bool:
|
||||||
"""
|
"""
|
||||||
Uninstall a plugin.
|
Uninstall a plugin.
|
||||||
|
@ -3,26 +3,13 @@ from typing import Any, Optional
|
|||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from core.plugin.entities.plugin import GenericProviderID
|
||||||
from core.plugin.entities.plugin_daemon import PluginBasicBooleanResponse, PluginToolProviderEntity
|
from core.plugin.entities.plugin_daemon import PluginBasicBooleanResponse, PluginToolProviderEntity
|
||||||
from core.plugin.manager.base import BasePluginManager
|
from core.plugin.manager.base import BasePluginManager
|
||||||
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
|
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
|
||||||
|
|
||||||
|
|
||||||
class PluginToolManager(BasePluginManager):
|
class PluginToolManager(BasePluginManager):
|
||||||
def _split_provider(self, provider: str) -> tuple[str, str]:
|
|
||||||
"""
|
|
||||||
split the provider to plugin_id and provider_name
|
|
||||||
|
|
||||||
provider follows format: plugin_id/provider_name
|
|
||||||
"""
|
|
||||||
if "/" in provider:
|
|
||||||
parts = provider.split("/", -1)
|
|
||||||
if len(parts) >= 2:
|
|
||||||
return "/".join(parts[:-1]), parts[-1]
|
|
||||||
raise ValueError(f"invalid provider format: {provider}")
|
|
||||||
|
|
||||||
raise ValueError(f"invalid provider format: {provider}")
|
|
||||||
|
|
||||||
def fetch_tool_providers(self, tenant_id: str) -> list[PluginToolProviderEntity]:
|
def fetch_tool_providers(self, tenant_id: str) -> list[PluginToolProviderEntity]:
|
||||||
"""
|
"""
|
||||||
Fetch tool providers for the given tenant.
|
Fetch tool providers for the given tenant.
|
||||||
@ -58,11 +45,11 @@ class PluginToolManager(BasePluginManager):
|
|||||||
"""
|
"""
|
||||||
Fetch tool provider for the given tenant and plugin.
|
Fetch tool provider for the given tenant and plugin.
|
||||||
"""
|
"""
|
||||||
plugin_id, provider_name = self._split_provider(provider)
|
tool_provider_id = GenericProviderID(provider)
|
||||||
|
|
||||||
def transformer(json_response: dict[str, Any]) -> dict:
|
def transformer(json_response: dict[str, Any]) -> dict:
|
||||||
for tool in json_response.get("data", {}).get("declaration", {}).get("tools", []):
|
for tool in json_response.get("data", {}).get("declaration", {}).get("tools", []):
|
||||||
tool["identity"]["provider"] = provider_name
|
tool["identity"]["provider"] = tool_provider_id.provider_name
|
||||||
|
|
||||||
return json_response
|
return json_response
|
||||||
|
|
||||||
@ -70,7 +57,7 @@ class PluginToolManager(BasePluginManager):
|
|||||||
"GET",
|
"GET",
|
||||||
f"plugin/{tenant_id}/management/tool",
|
f"plugin/{tenant_id}/management/tool",
|
||||||
PluginToolProviderEntity,
|
PluginToolProviderEntity,
|
||||||
params={"provider": provider_name, "plugin_id": plugin_id},
|
params={"provider": tool_provider_id.provider_name, "plugin_id": tool_provider_id.plugin_id},
|
||||||
transformer=transformer,
|
transformer=transformer,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -98,7 +85,7 @@ class PluginToolManager(BasePluginManager):
|
|||||||
Invoke the tool with the given tenant, user, plugin, provider, name, credentials and parameters.
|
Invoke the tool with the given tenant, user, plugin, provider, name, credentials and parameters.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
plugin_id, provider_name = self._split_provider(tool_provider)
|
tool_provider_id = GenericProviderID(tool_provider)
|
||||||
|
|
||||||
response = self._request_with_plugin_daemon_response_stream(
|
response = self._request_with_plugin_daemon_response_stream(
|
||||||
"POST",
|
"POST",
|
||||||
@ -110,14 +97,14 @@ class PluginToolManager(BasePluginManager):
|
|||||||
"app_id": app_id,
|
"app_id": app_id,
|
||||||
"message_id": message_id,
|
"message_id": message_id,
|
||||||
"data": {
|
"data": {
|
||||||
"provider": provider_name,
|
"provider": tool_provider_id.provider_name,
|
||||||
"tool": tool_name,
|
"tool": tool_name,
|
||||||
"credentials": credentials,
|
"credentials": credentials,
|
||||||
"tool_parameters": tool_parameters,
|
"tool_parameters": tool_parameters,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
headers={
|
headers={
|
||||||
"X-Plugin-ID": plugin_id,
|
"X-Plugin-ID": tool_provider_id.plugin_id,
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -129,7 +116,7 @@ class PluginToolManager(BasePluginManager):
|
|||||||
"""
|
"""
|
||||||
validate the credentials of the provider
|
validate the credentials of the provider
|
||||||
"""
|
"""
|
||||||
plugin_id, provider_name = self._split_provider(provider)
|
tool_provider_id = GenericProviderID(provider)
|
||||||
|
|
||||||
response = self._request_with_plugin_daemon_response_stream(
|
response = self._request_with_plugin_daemon_response_stream(
|
||||||
"POST",
|
"POST",
|
||||||
@ -138,12 +125,12 @@ class PluginToolManager(BasePluginManager):
|
|||||||
data={
|
data={
|
||||||
"user_id": user_id,
|
"user_id": user_id,
|
||||||
"data": {
|
"data": {
|
||||||
"provider": provider_name,
|
"provider": tool_provider_id.provider_name,
|
||||||
"credentials": credentials,
|
"credentials": credentials,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
headers={
|
headers={
|
||||||
"X-Plugin-ID": plugin_id,
|
"X-Plugin-ID": tool_provider_id.plugin_id,
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -167,7 +154,7 @@ class PluginToolManager(BasePluginManager):
|
|||||||
"""
|
"""
|
||||||
get the runtime parameters of the tool
|
get the runtime parameters of the tool
|
||||||
"""
|
"""
|
||||||
plugin_id, provider_name = self._split_provider(provider)
|
tool_provider_id = GenericProviderID(provider)
|
||||||
|
|
||||||
class RuntimeParametersResponse(BaseModel):
|
class RuntimeParametersResponse(BaseModel):
|
||||||
parameters: list[ToolParameter]
|
parameters: list[ToolParameter]
|
||||||
@ -182,13 +169,13 @@ class PluginToolManager(BasePluginManager):
|
|||||||
"app_id": app_id,
|
"app_id": app_id,
|
||||||
"message_id": message_id,
|
"message_id": message_id,
|
||||||
"data": {
|
"data": {
|
||||||
"provider": provider_name,
|
"provider": tool_provider_id.provider_name,
|
||||||
"tool": tool,
|
"tool": tool,
|
||||||
"credentials": credentials,
|
"credentials": credentials,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
headers={
|
headers={
|
||||||
"X-Plugin-ID": plugin_id,
|
"X-Plugin-ID": tool_provider_id.plugin_id,
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
@ -1,5 +1,4 @@
|
|||||||
import base64
|
import base64
|
||||||
import re
|
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Any, Optional, Union
|
from typing import Any, Optional, Union
|
||||||
|
|
||||||
@ -467,19 +466,3 @@ class ToolInvokeFrom(Enum):
|
|||||||
WORKFLOW = "workflow"
|
WORKFLOW = "workflow"
|
||||||
AGENT = "agent"
|
AGENT = "agent"
|
||||||
PLUGIN = "plugin"
|
PLUGIN = "plugin"
|
||||||
|
|
||||||
|
|
||||||
class ToolProviderID:
|
|
||||||
organization: str
|
|
||||||
plugin_name: str
|
|
||||||
provider_name: str
|
|
||||||
|
|
||||||
def __str__(self) -> str:
|
|
||||||
return f"{self.organization}/{self.plugin_name}/{self.provider_name}"
|
|
||||||
|
|
||||||
def __init__(self, value: str) -> None:
|
|
||||||
# check if the value is a valid plugin id with format: $organization/$plugin_name/$provider_name
|
|
||||||
if not re.match(r"^[a-z0-9_-]+\/[a-z0-9_-]+\/[a-z0-9_-]+$", value):
|
|
||||||
raise ValueError("Invalid plugin id")
|
|
||||||
|
|
||||||
self.organization, self.plugin_name, self.provider_name = value.split("/")
|
|
||||||
|
@ -6,6 +6,7 @@ from os import listdir, path
|
|||||||
from threading import Lock
|
from threading import Lock
|
||||||
from typing import TYPE_CHECKING, Any, Union, cast
|
from typing import TYPE_CHECKING, Any, Union, cast
|
||||||
|
|
||||||
|
from core.plugin.entities.plugin import GenericProviderID
|
||||||
from core.plugin.manager.tool import PluginToolManager
|
from core.plugin.manager.tool import PluginToolManager
|
||||||
from core.tools.__base.tool_runtime import ToolRuntime
|
from core.tools.__base.tool_runtime import ToolRuntime
|
||||||
from core.tools.plugin_tool.provider import PluginToolProviderController
|
from core.tools.plugin_tool.provider import PluginToolProviderController
|
||||||
@ -33,7 +34,6 @@ from core.tools.entities.tool_entities import (
|
|||||||
ApiProviderAuthType,
|
ApiProviderAuthType,
|
||||||
ToolInvokeFrom,
|
ToolInvokeFrom,
|
||||||
ToolParameter,
|
ToolParameter,
|
||||||
ToolProviderID,
|
|
||||||
ToolProviderType,
|
ToolProviderType,
|
||||||
)
|
)
|
||||||
from core.tools.errors import ToolProviderNotFoundError
|
from core.tools.errors import ToolProviderNotFoundError
|
||||||
@ -164,7 +164,7 @@ class ToolManager:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if isinstance(provider_controller, PluginToolProviderController):
|
if isinstance(provider_controller, PluginToolProviderController):
|
||||||
provider_id_entity = ToolProviderID(provider_id)
|
provider_id_entity = GenericProviderID(provider_id)
|
||||||
# get credentials
|
# get credentials
|
||||||
builtin_provider: BuiltinToolProvider | None = (
|
builtin_provider: BuiltinToolProvider | None = (
|
||||||
db.session.query(BuiltinToolProvider)
|
db.session.query(BuiltinToolProvider)
|
||||||
@ -582,10 +582,8 @@ class ToolManager:
|
|||||||
|
|
||||||
# rewrite db_builtin_providers
|
# rewrite db_builtin_providers
|
||||||
for db_provider in db_builtin_providers:
|
for db_provider in db_builtin_providers:
|
||||||
try:
|
tool_provider_id = GenericProviderID(db_provider.provider)
|
||||||
ToolProviderID(db_provider.provider)
|
db_provider.provider = tool_provider_id.to_string()
|
||||||
except Exception:
|
|
||||||
db_provider.provider = f"langgenius/{db_provider.provider}/{db_provider.provider}"
|
|
||||||
|
|
||||||
find_db_builtin_provider = lambda provider: next(
|
find_db_builtin_provider = lambda provider: next(
|
||||||
(x for x in db_builtin_providers if x.provider == provider), None
|
(x for x in db_builtin_providers if x.provider == provider), None
|
||||||
|
@ -6,12 +6,21 @@ import yaml
|
|||||||
from packaging import version
|
from packaging import version
|
||||||
|
|
||||||
from core.helper import ssrf_proxy
|
from core.helper import ssrf_proxy
|
||||||
|
from core.model_runtime.utils.encoders import jsonable_encoder
|
||||||
|
from core.plugin.entities.plugin import PluginDependency
|
||||||
|
from core.workflow.nodes.enums import NodeType
|
||||||
|
from core.workflow.nodes.knowledge_retrieval.entities import KnowledgeRetrievalNodeData
|
||||||
|
from core.workflow.nodes.llm.entities import LLMNodeData
|
||||||
|
from core.workflow.nodes.parameter_extractor.entities import ParameterExtractorNodeData
|
||||||
|
from core.workflow.nodes.question_classifier.entities import QuestionClassifierNodeData
|
||||||
|
from core.workflow.nodes.tool.entities import ToolNodeData
|
||||||
from events.app_event import app_model_config_was_updated, app_was_created
|
from events.app_event import app_model_config_was_updated, app_was_created
|
||||||
from extensions.ext_database import db
|
from extensions.ext_database import db
|
||||||
from factories import variable_factory
|
from factories import variable_factory
|
||||||
from models.account import Account
|
from models.account import Account
|
||||||
from models.model import App, AppMode, AppModelConfig
|
from models.model import App, AppMode, AppModelConfig
|
||||||
from models.workflow import Workflow
|
from models.workflow import Workflow
|
||||||
|
from services.plugin.dependencies_analysis import DependenciesAnalysisService
|
||||||
from services.workflow_service import WorkflowService
|
from services.workflow_service import WorkflowService
|
||||||
|
|
||||||
from .exc import (
|
from .exc import (
|
||||||
@ -58,6 +67,22 @@ class AppDslService:
|
|||||||
|
|
||||||
return cls.import_and_create_new_app(tenant_id, data, args, account)
|
return cls.import_and_create_new_app(tenant_id, data, args, account)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def check_dependencies(cls, tenant_id: str, data: str, account: Account) -> list[PluginDependency]:
|
||||||
|
"""
|
||||||
|
Returns the leaked dependencies in current workspace
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import_data = yaml.safe_load(data) or {}
|
||||||
|
except yaml.YAMLError:
|
||||||
|
raise InvalidYAMLFormatError("Invalid YAML format in data argument.")
|
||||||
|
|
||||||
|
dependencies = [PluginDependency(**dep) for dep in import_data.get("dependencies", [])]
|
||||||
|
if not dependencies:
|
||||||
|
return []
|
||||||
|
|
||||||
|
return DependenciesAnalysisService.check_dependencies(tenant_id=tenant_id, dependencies=dependencies)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def import_and_create_new_app(cls, tenant_id: str, data: str, args: dict, account: Account) -> App:
|
def import_and_create_new_app(cls, tenant_id: str, data: str, args: dict, account: Account) -> App:
|
||||||
"""
|
"""
|
||||||
@ -436,6 +461,13 @@ class AppDslService:
|
|||||||
raise ValueError("Missing draft workflow configuration, please check.")
|
raise ValueError("Missing draft workflow configuration, please check.")
|
||||||
|
|
||||||
export_data["workflow"] = workflow.to_dict(include_secret=include_secret)
|
export_data["workflow"] = workflow.to_dict(include_secret=include_secret)
|
||||||
|
dependencies = cls._extract_dependencies_from_workflow(workflow)
|
||||||
|
export_data["dependencies"] = [
|
||||||
|
jsonable_encoder(d.model_dump())
|
||||||
|
for d in DependenciesAnalysisService.generate_dependencies(
|
||||||
|
tenant_id=app_model.tenant_id, dependencies=dependencies
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _append_model_config_export_data(cls, export_data: dict, app_model: App) -> None:
|
def _append_model_config_export_data(cls, export_data: dict, app_model: App) -> None:
|
||||||
@ -449,6 +481,137 @@ class AppDslService:
|
|||||||
raise ValueError("Missing app configuration, please check.")
|
raise ValueError("Missing app configuration, please check.")
|
||||||
|
|
||||||
export_data["model_config"] = app_model_config.to_dict()
|
export_data["model_config"] = app_model_config.to_dict()
|
||||||
|
dependencies = cls._extract_dependencies_from_model_config(app_model_config)
|
||||||
|
export_data["dependencies"] = [
|
||||||
|
jsonable_encoder(d.model_dump())
|
||||||
|
for d in DependenciesAnalysisService.generate_dependencies(
|
||||||
|
tenant_id=app_model.tenant_id, dependencies=dependencies
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _extract_dependencies_from_workflow(cls, workflow: Workflow) -> list[str]:
|
||||||
|
"""
|
||||||
|
Extract dependencies from workflow
|
||||||
|
:param workflow: Workflow instance
|
||||||
|
:return: dependencies list format like ["langgenius/google"]
|
||||||
|
"""
|
||||||
|
graph = workflow.graph_dict
|
||||||
|
dependencies = []
|
||||||
|
for node in graph.get("nodes", []):
|
||||||
|
try:
|
||||||
|
typ = node.get("data", {}).get("type")
|
||||||
|
match typ:
|
||||||
|
case NodeType.TOOL.value:
|
||||||
|
tool_entity = ToolNodeData(**node["data"])
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_tool_dependency(tool_entity.provider_id),
|
||||||
|
)
|
||||||
|
case NodeType.LLM.value:
|
||||||
|
llm_entity = LLMNodeData(**node["data"])
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_model_provider_dependency(llm_entity.model.provider),
|
||||||
|
)
|
||||||
|
case NodeType.QUESTION_CLASSIFIER.value:
|
||||||
|
question_classifier_entity = QuestionClassifierNodeData(**node["data"])
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_model_provider_dependency(
|
||||||
|
question_classifier_entity.model.provider
|
||||||
|
),
|
||||||
|
)
|
||||||
|
case NodeType.PARAMETER_EXTRACTOR.value:
|
||||||
|
parameter_extractor_entity = ParameterExtractorNodeData(**node["data"])
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_model_provider_dependency(
|
||||||
|
parameter_extractor_entity.model.provider
|
||||||
|
),
|
||||||
|
)
|
||||||
|
case NodeType.KNOWLEDGE_RETRIEVAL.value:
|
||||||
|
knowledge_retrieval_entity = KnowledgeRetrievalNodeData(**node["data"])
|
||||||
|
if knowledge_retrieval_entity.retrieval_mode == "multiple":
|
||||||
|
if knowledge_retrieval_entity.multiple_retrieval_config:
|
||||||
|
if (
|
||||||
|
knowledge_retrieval_entity.multiple_retrieval_config.reranking_mode
|
||||||
|
== "reranking_model"
|
||||||
|
):
|
||||||
|
if knowledge_retrieval_entity.multiple_retrieval_config.reranking_model:
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_model_provider_dependency(
|
||||||
|
knowledge_retrieval_entity.multiple_retrieval_config.reranking_model.provider
|
||||||
|
),
|
||||||
|
)
|
||||||
|
elif (
|
||||||
|
knowledge_retrieval_entity.multiple_retrieval_config.reranking_mode
|
||||||
|
== "weighted_score"
|
||||||
|
):
|
||||||
|
if knowledge_retrieval_entity.multiple_retrieval_config.weights:
|
||||||
|
vector_setting = (
|
||||||
|
knowledge_retrieval_entity.multiple_retrieval_config.weights.vector_setting
|
||||||
|
)
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_model_provider_dependency(
|
||||||
|
vector_setting.embedding_provider_name
|
||||||
|
),
|
||||||
|
)
|
||||||
|
elif knowledge_retrieval_entity.retrieval_mode == "single":
|
||||||
|
model_config = knowledge_retrieval_entity.single_retrieval_config
|
||||||
|
if model_config:
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_model_provider_dependency(
|
||||||
|
model_config.model.provider
|
||||||
|
),
|
||||||
|
)
|
||||||
|
case _:
|
||||||
|
# Handle default case or unknown node types
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("Error extracting node dependency", exc_info=e)
|
||||||
|
|
||||||
|
return dependencies
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _extract_dependencies_from_model_config(cls, model_config: AppModelConfig) -> list[str]:
|
||||||
|
"""
|
||||||
|
Extract dependencies from model config
|
||||||
|
:param model_config: AppModelConfig instance
|
||||||
|
:return: dependencies list format like ["langgenius/google:1.0.0@abcdef1234567890"]
|
||||||
|
"""
|
||||||
|
dependencies = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
# completion model
|
||||||
|
model_dict = model_config.model_dict
|
||||||
|
if model_dict:
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_model_provider_dependency(model_dict.get("provider"))
|
||||||
|
)
|
||||||
|
|
||||||
|
# reranking model
|
||||||
|
dataset_configs = model_config.dataset_configs_dict
|
||||||
|
if dataset_configs:
|
||||||
|
for dataset_config in dataset_configs:
|
||||||
|
if dataset_config.get("reranking_model"):
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_model_provider_dependency(
|
||||||
|
dataset_config.get("reranking_model", {})
|
||||||
|
.get("reranking_provider_name", {})
|
||||||
|
.get("provider")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# tools
|
||||||
|
agent_configs = model_config.agent_mode_dict
|
||||||
|
if agent_configs:
|
||||||
|
for agent_config in agent_configs:
|
||||||
|
if agent_config.get("tools"):
|
||||||
|
for tool in agent_config.get("tools", []):
|
||||||
|
dependencies.append(
|
||||||
|
DependenciesAnalysisService.analyze_tool_dependency(tool.get("provider_id"))
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("Error extracting model config dependency", exc_info=e)
|
||||||
|
|
||||||
|
return dependencies
|
||||||
|
|
||||||
|
|
||||||
def _check_or_fix_dsl(import_data: dict[str, Any]) -> Mapping[str, Any]:
|
def _check_or_fix_dsl(import_data: dict[str, Any]) -> Mapping[str, Any]:
|
||||||
|
100
api/services/plugin/dependencies_analysis.py
Normal file
100
api/services/plugin/dependencies_analysis.py
Normal file
@ -0,0 +1,100 @@
|
|||||||
|
from core.plugin.entities.plugin import GenericProviderID, PluginDependency, PluginInstallationSource
|
||||||
|
from core.plugin.manager.plugin import PluginInstallationManager
|
||||||
|
|
||||||
|
|
||||||
|
class DependenciesAnalysisService:
|
||||||
|
@classmethod
|
||||||
|
def analyze_tool_dependency(cls, tool_id: str) -> str:
|
||||||
|
"""
|
||||||
|
Analyze the dependency of a tool.
|
||||||
|
|
||||||
|
Convert the tool id to the plugin_id
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
tool_provider_id = GenericProviderID(tool_id)
|
||||||
|
return tool_provider_id.plugin_id
|
||||||
|
except Exception as e:
|
||||||
|
raise e
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def analyze_model_provider_dependency(cls, model_provider_id: str) -> str:
|
||||||
|
"""
|
||||||
|
Analyze the dependency of a model provider.
|
||||||
|
|
||||||
|
Convert the model provider id to the plugin_id
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
generic_provider_id = GenericProviderID(model_provider_id)
|
||||||
|
return generic_provider_id.plugin_id
|
||||||
|
except Exception as e:
|
||||||
|
raise e
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def check_dependencies(cls, tenant_id: str, dependencies: list[PluginDependency]) -> list[PluginDependency]:
|
||||||
|
"""
|
||||||
|
Check dependencies, returns the leaked dependencies in current workspace
|
||||||
|
"""
|
||||||
|
required_plugin_unique_identifiers = []
|
||||||
|
for dependency in dependencies:
|
||||||
|
required_plugin_unique_identifiers.append(dependency.value.plugin_unique_identifier)
|
||||||
|
|
||||||
|
manager = PluginInstallationManager()
|
||||||
|
missing_plugin_unique_identifiers = manager.fetch_missing_dependencies(
|
||||||
|
tenant_id, required_plugin_unique_identifiers
|
||||||
|
)
|
||||||
|
|
||||||
|
leaked_dependencies = []
|
||||||
|
for dependency in dependencies:
|
||||||
|
unique_identifier = dependency.value.plugin_unique_identifier
|
||||||
|
if unique_identifier in missing_plugin_unique_identifiers:
|
||||||
|
leaked_dependencies.append(dependency)
|
||||||
|
|
||||||
|
return leaked_dependencies
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def generate_dependencies(cls, tenant_id: str, dependencies: list[str]) -> list[PluginDependency]:
|
||||||
|
"""
|
||||||
|
Generate dependencies through the list of plugin ids
|
||||||
|
"""
|
||||||
|
dependencies = list(set(dependencies))
|
||||||
|
manager = PluginInstallationManager()
|
||||||
|
plugins = manager.fetch_plugin_installation_by_ids(tenant_id, dependencies)
|
||||||
|
result = []
|
||||||
|
for plugin in plugins:
|
||||||
|
if plugin.source == PluginInstallationSource.Github:
|
||||||
|
result.append(
|
||||||
|
PluginDependency(
|
||||||
|
type=PluginDependency.Type.Github,
|
||||||
|
value=PluginDependency.Github(
|
||||||
|
repo=plugin.meta["repo"],
|
||||||
|
version=plugin.meta["version"],
|
||||||
|
package=plugin.meta["package"],
|
||||||
|
github_plugin_unique_identifier=plugin.plugin_unique_identifier,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif plugin.source == PluginInstallationSource.Marketplace:
|
||||||
|
result.append(
|
||||||
|
PluginDependency(
|
||||||
|
type=PluginDependency.Type.Marketplace,
|
||||||
|
value=PluginDependency.Marketplace(
|
||||||
|
marketplace_plugin_unique_identifier=plugin.plugin_unique_identifier
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif plugin.source == PluginInstallationSource.Package:
|
||||||
|
result.append(
|
||||||
|
PluginDependency(
|
||||||
|
type=PluginDependency.Type.Package,
|
||||||
|
value=PluginDependency.Package(plugin_unique_identifier=plugin.plugin_unique_identifier),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif plugin.source == PluginInstallationSource.Remote:
|
||||||
|
raise ValueError(
|
||||||
|
f"You used a remote plugin: {plugin.plugin_unique_identifier} in the app, please remove it first"
|
||||||
|
" if you want to export the DSL."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unknown plugin source: {plugin.source}")
|
||||||
|
|
||||||
|
return result
|
@ -5,9 +5,9 @@ from pathlib import Path
|
|||||||
from configs import dify_config
|
from configs import dify_config
|
||||||
from core.helper.position_helper import is_filtered
|
from core.helper.position_helper import is_filtered
|
||||||
from core.model_runtime.utils.encoders import jsonable_encoder
|
from core.model_runtime.utils.encoders import jsonable_encoder
|
||||||
|
from core.plugin.entities.plugin import GenericProviderID
|
||||||
from core.tools.builtin_tool.providers._positions import BuiltinToolProviderSort
|
from core.tools.builtin_tool.providers._positions import BuiltinToolProviderSort
|
||||||
from core.tools.entities.api_entities import ToolApiEntity, ToolProviderApiEntity
|
from core.tools.entities.api_entities import ToolApiEntity, ToolProviderApiEntity
|
||||||
from core.tools.entities.tool_entities import ToolProviderID
|
|
||||||
from core.tools.errors import ToolNotFoundError, ToolProviderCredentialValidationError, ToolProviderNotFoundError
|
from core.tools.errors import ToolNotFoundError, ToolProviderCredentialValidationError, ToolProviderNotFoundError
|
||||||
from core.tools.tool_label_manager import ToolLabelManager
|
from core.tools.tool_label_manager import ToolLabelManager
|
||||||
from core.tools.tool_manager import ToolManager
|
from core.tools.tool_manager import ToolManager
|
||||||
@ -234,7 +234,7 @@ class BuiltinToolManageService:
|
|||||||
# rewrite db_providers
|
# rewrite db_providers
|
||||||
for db_provider in db_providers:
|
for db_provider in db_providers:
|
||||||
try:
|
try:
|
||||||
ToolProviderID(db_provider.provider)
|
GenericProviderID(db_provider.provider)
|
||||||
except Exception:
|
except Exception:
|
||||||
db_provider.provider = f"langgenius/{db_provider.provider}/{db_provider.provider}"
|
db_provider.provider = f"langgenius/{db_provider.provider}/{db_provider.provider}"
|
||||||
|
|
||||||
@ -287,7 +287,7 @@ class BuiltinToolManageService:
|
|||||||
def _fetch_builtin_provider(provider_name: str, tenant_id: str) -> BuiltinToolProvider | None:
|
def _fetch_builtin_provider(provider_name: str, tenant_id: str) -> BuiltinToolProvider | None:
|
||||||
try:
|
try:
|
||||||
full_provider_name = provider_name
|
full_provider_name = provider_name
|
||||||
provider_id_entity = ToolProviderID(provider_name)
|
provider_id_entity = GenericProviderID(provider_name)
|
||||||
provider_name = provider_id_entity.provider_name
|
provider_name = provider_id_entity.provider_name
|
||||||
if provider_id_entity.organization != "langgenius":
|
if provider_id_entity.organization != "langgenius":
|
||||||
return None
|
return None
|
||||||
@ -305,11 +305,7 @@ class BuiltinToolManageService:
|
|||||||
if provider_obj is None:
|
if provider_obj is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
try:
|
provider_obj.provider = GenericProviderID(provider_obj.provider).to_string()
|
||||||
ToolProviderID(provider_obj.provider)
|
|
||||||
except Exception:
|
|
||||||
provider_obj.provider = f"langgenius/{provider_obj.provider}/{provider_obj.provider}"
|
|
||||||
|
|
||||||
return provider_obj
|
return provider_obj
|
||||||
except Exception:
|
except Exception:
|
||||||
# it's an old provider without organization
|
# it's an old provider without organization
|
||||||
|
Loading…
x
Reference in New Issue
Block a user