AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-14 06:15:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(backend): support import DSL from URL (#6287)

Browse Source
This commit is contained in:
takatost 2024-07-15 16:23:40 +08:00 committed by GitHub
parent ec181649ae
commit 46a5294d94
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 459 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
api/controllers/console/app/app.py
View File

@ -15,6 +15,7 @@ from fields.app_fields import (
app_pagination_fields,
)
from libs.login import login_required
from services.app_dsl_service import AppDslService
from services.app_service import AppService
ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion']
@ -97,8 +98,42 @@ class AppImportApi(Resource):
parser.add_argument('icon_background', type=str, location='json')
args = parser.parse_args()
app_service = AppService()
app = app_service.import_app(current_user.current_tenant_id, args['data'], args, current_user)
app = AppDslService.import_and_create_new_app(
tenant_id=current_user.current_tenant_id,
data=args['data'],
args=args,
account=current_user
)
return app, 201
class AppImportFromUrlApi(Resource):
@setup_required
@login_required
@account_initialization_required
@marshal_with(app_detail_fields_with_site)
@cloud_edition_billing_resource_check('apps')
def post(self):
"""Import app from url"""
# The role of the current user in the ta table must be admin, owner, or editor
if not current_user.is_editor:
raise Forbidden()
parser = reqparse.RequestParser()
parser.add_argument('url', type=str, required=True, nullable=False, location='json')
parser.add_argument('name', type=str, location='json')
parser.add_argument('description', type=str, location='json')
parser.add_argument('icon', type=str, location='json')
parser.add_argument('icon_background', type=str, location='json')
args = parser.parse_args()
app = AppDslService.import_and_create_new_app_from_url(
tenant_id=current_user.current_tenant_id,
url=args['url'],
args=args,
account=current_user
)
return app, 201
@ -177,9 +212,13 @@ class AppCopyApi(Resource):
parser.add_argument('icon_background', type=str, location='json')
args = parser.parse_args()
app_service = AppService()
data = app_service.export_app(app_model)
app = app_service.import_app(current_user.current_tenant_id, data, args, current_user)
data = AppDslService.export_dsl(app_model=app_model)
app = AppDslService.import_and_create_new_app(
tenant_id=current_user.current_tenant_id,
data=data,
args=args,
account=current_user
)
return app, 201
@ -195,10 +234,8 @@ class AppExportApi(Resource):
if not current_user.is_editor:
raise Forbidden()
app_service = AppService()
return {
"data": app_service.export_app(app_model)
"data": AppDslService.export_dsl(app_model=app_model)
}
@ -322,6 +359,7 @@ class AppTraceApi(Resource):
api.add_resource(AppListApi, '/apps')
api.add_resource(AppImportApi, '/apps/import')
api.add_resource(AppImportFromUrlApi, '/apps/import/url')
api.add_resource(AppApi, '/apps/<uuid:app_id>')
api.add_resource(AppCopyApi, '/apps/<uuid:app_id>/copy')
api.add_resource(AppExportApi, '/apps/<uuid:app_id>/export')

4
api/controllers/console/app/workflow.py
View File

@ -20,6 +20,7 @@ from libs import helper
from libs.helper import TimestampField, uuid_value
from libs.login import current_user, login_required
from models.model import App, AppMode
from services.app_dsl_service import AppDslService
from services.app_generate_service import AppGenerateService
from services.errors.app import WorkflowHashNotEqualError
from services.workflow_service import WorkflowService
@ -128,8 +129,7 @@ class DraftWorkflowImportApi(Resource):
parser.add_argument('data', type=str, required=True, nullable=False, location='json')
args = parser.parse_args()
workflow_service = WorkflowService()
workflow = workflow_service.import_draft_workflow(
workflow = AppDslService.import_and_overwrite_workflow(
app_model=app_model,
data=args['data'],
account=current_user

407
api/services/app_dsl_service.py Normal file
View File

@ -0,0 +1,407 @@
import logging
import httpx
import yaml # type: ignore
from events.app_event import app_model_config_was_updated, app_was_created
from extensions.ext_database import db
from models.account import Account
from models.model import App, AppMode, AppModelConfig
from models.workflow import Workflow
from services.workflow_service import WorkflowService
logger = logging.getLogger(__name__)
current_dsl_version = "0.1.0"
dsl_to_dify_version_mapping: dict[str, str] = {
"0.1.0": "0.6.0", # dsl version -> from dify version
}
class AppDslService:
@classmethod
def import_and_create_new_app_from_url(cls, tenant_id: str, url: str, args: dict, account: Account) -> App:
"""
Import app dsl from url and create new app
:param tenant_id: tenant id
:param url: import url
:param args: request args
:param account: Account instance
"""
try:
max_size = 10 * 1024 * 1024 # 10MB
timeout = httpx.Timeout(10.0)
with httpx.stream("GET", url.strip(), follow_redirects=True, timeout=timeout) as response:
response.raise_for_status()
total_size = 0
content = b""
for chunk in response.iter_bytes():
total_size += len(chunk)
if total_size > max_size:
raise ValueError("File size exceeds the limit of 10MB")
content += chunk
except httpx.HTTPStatusError as http_err:
raise ValueError(f"HTTP error occurred: {http_err}")
except httpx.RequestError as req_err:
raise ValueError(f"Request error occurred: {req_err}")
except Exception as e:
raise ValueError(f"Failed to fetch DSL from URL: {e}")
if not content:
raise ValueError("Empty content from url")
try:
data = content.decode("utf-8")
except UnicodeDecodeError as e:
raise ValueError(f"Error decoding content: {e}")
return cls.import_and_create_new_app(tenant_id, data, args, account)
@classmethod
def import_and_create_new_app(cls, tenant_id: str, data: str, args: dict, account: Account) -> App:
"""
Import app dsl and create new app
:param tenant_id: tenant id
:param data: import data
:param args: request args
:param account: Account instance
"""
try:
import_data = yaml.safe_load(data)
except yaml.YAMLError:
raise ValueError("Invalid YAML format in data argument.")
# check or repair dsl version
import_data = cls._check_or_fix_dsl(import_data)
app_data = import_data.get('app')
if not app_data:
raise ValueError("Missing app in data argument")
# get app basic info
name = args.get("name") if args.get("name") else app_data.get('name')
description = args.get("description") if args.get("description") else app_data.get('description', '')
icon = args.get("icon") if args.get("icon") else app_data.get('icon')
icon_background = args.get("icon_background") if args.get("icon_background") \
else app_data.get('icon_background')
# import dsl and create app
app_mode = AppMode.value_of(app_data.get('mode'))
if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
app = cls._import_and_create_new_workflow_based_app(
tenant_id=tenant_id,
app_mode=app_mode,
workflow_data=import_data.get('workflow'),
account=account,
name=name,
description=description,
icon=icon,
icon_background=icon_background
)
elif app_mode in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION]:
app = cls._import_and_create_new_model_config_based_app(
tenant_id=tenant_id,
app_mode=app_mode,
model_config_data=import_data.get('model_config'),
account=account,
name=name,
description=description,
icon=icon,
icon_background=icon_background
)
else:
raise ValueError("Invalid app mode")
return app
@classmethod
def import_and_overwrite_workflow(cls, app_model: App, data: str, account: Account) -> Workflow:
"""
Import app dsl and overwrite workflow
:param app_model: App instance
:param data: import data
:param account: Account instance
"""
try:
import_data = yaml.safe_load(data)
except yaml.YAMLError:
raise ValueError("Invalid YAML format in data argument.")
# check or repair dsl version
import_data = cls._check_or_fix_dsl(import_data)
app_data = import_data.get('app')
if not app_data:
raise ValueError("Missing app in data argument")
# import dsl and overwrite app
app_mode = AppMode.value_of(app_data.get('mode'))
if app_mode not in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
raise ValueError("Only support import workflow in advanced-chat or workflow app.")
if app_data.get('mode') != app_model.mode:
raise ValueError(
f"App mode {app_data.get('mode')} is not matched with current app mode {app_mode.value}")
return cls._import_and_overwrite_workflow_based_app(
app_model=app_model,
workflow_data=import_data.get('workflow'),
account=account,
)
@classmethod
def export_dsl(cls, app_model: App) -> str:
"""
Export app
:param app_model: App instance
:return:
"""
app_mode = AppMode.value_of(app_model.mode)
export_data = {
"version": current_dsl_version,
"kind": "app",
"app": {
"name": app_model.name,
"mode": app_model.mode,
"icon": app_model.icon,
"icon_background": app_model.icon_background,
"description": app_model.description
}
}
if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
cls._append_workflow_export_data(export_data, app_model)
else:
cls._append_model_config_export_data(export_data, app_model)
return yaml.dump(export_data)
@classmethod
def _check_or_fix_dsl(cls, import_data: dict) -> dict:
"""
Check or fix dsl
:param import_data: import data
"""
if not import_data.get('version'):
import_data['version'] = "0.1.0"
if not import_data.get('kind') or import_data.get('kind') != "app":
import_data['kind'] = "app"
if import_data.get('version') != current_dsl_version:
# Currently only one DSL version, so no difference checks or compatibility fixes will be performed.
logger.warning(f"DSL version {import_data.get('version')} is not compatible "
f"with current version {current_dsl_version}, related to "
f"Dify version {dsl_to_dify_version_mapping.get(current_dsl_version)}.")
return import_data
@classmethod
def _import_and_create_new_workflow_based_app(cls,
tenant_id: str,
app_mode: AppMode,
workflow_data: dict,
account: Account,
name: str,
description: str,
icon: str,
icon_background: str) -> App:
"""
Import app dsl and create new workflow based app
:param tenant_id: tenant id
:param app_mode: app mode
:param workflow_data: workflow data
:param account: Account instance
:param name: app name
:param description: app description
:param icon: app icon
:param icon_background: app icon background
"""
if not workflow_data:
raise ValueError("Missing workflow in data argument "
"when app mode is advanced-chat or workflow")
app = cls._create_app(
tenant_id=tenant_id,
app_mode=app_mode,
account=account,
name=name,
description=description,
icon=icon,
icon_background=icon_background
)
# init draft workflow
workflow_service = WorkflowService()
draft_workflow = workflow_service.sync_draft_workflow(
app_model=app,
graph=workflow_data.get('graph', {}),
features=workflow_data.get('../core/app/features', {}),
unique_hash=None,
account=account
)
workflow_service.publish_workflow(
app_model=app,
account=account,
draft_workflow=draft_workflow
)
return app
@classmethod
def _import_and_overwrite_workflow_based_app(cls,
app_model: App,
workflow_data: dict,
account: Account) -> Workflow:
"""
Import app dsl and overwrite workflow based app
:param app_model: App instance
:param workflow_data: workflow data
:param account: Account instance
"""
if not workflow_data:
raise ValueError("Missing workflow in data argument "
"when app mode is advanced-chat or workflow")
# fetch draft workflow by app_model
workflow_service = WorkflowService()
current_draft_workflow = workflow_service.get_draft_workflow(app_model=app_model)
if current_draft_workflow:
unique_hash = current_draft_workflow.unique_hash
else:
unique_hash = None
# sync draft workflow
draft_workflow = workflow_service.sync_draft_workflow(
app_model=app_model,
graph=workflow_data.get('graph', {}),
features=workflow_data.get('features', {}),
unique_hash=unique_hash,
account=account
)
return draft_workflow
@classmethod
def _import_and_create_new_model_config_based_app(cls,
tenant_id: str,
app_mode: AppMode,
model_config_data: dict,
account: Account,
name: str,
description: str,
icon: str,
icon_background: str) -> App:
"""
Import app dsl and create new model config based app
:param tenant_id: tenant id
:param app_mode: app mode
:param model_config_data: model config data
:param account: Account instance
:param name: app name
:param description: app description
:param icon: app icon
:param icon_background: app icon background
"""
if not model_config_data:
raise ValueError("Missing model_config in data argument "
"when app mode is chat, agent-chat or completion")
app = cls._create_app(
tenant_id=tenant_id,
app_mode=app_mode,
account=account,
name=name,
description=description,
icon=icon,
icon_background=icon_background
)
app_model_config = AppModelConfig()
app_model_config = app_model_config.from_model_config_dict(model_config_data)
app_model_config.app_id = app.id
db.session.add(app_model_config)
db.session.commit()
app.app_model_config_id = app_model_config.id
app_model_config_was_updated.send(
app,
app_model_config=app_model_config
)
return app
@classmethod
def _create_app(cls,
tenant_id: str,
app_mode: AppMode,
account: Account,
name: str,
description: str,
icon: str,
icon_background: str) -> App:
"""
Create new app
:param tenant_id: tenant id
:param app_mode: app mode
:param account: Account instance
:param name: app name
:param description: app description
:param icon: app icon
:param icon_background: app icon background
"""
app = App(
tenant_id=tenant_id,
mode=app_mode.value,
name=name,
description=description,
icon=icon,
icon_background=icon_background,
enable_site=True,
enable_api=True
)
db.session.add(app)
db.session.commit()
app_was_created.send(app, account=account)
return app
@classmethod
def _append_workflow_export_data(cls, export_data: dict, app_model: App) -> None:
"""
Append workflow export data
:param export_data: export data
:param app_model: App instance
"""
workflow_service = WorkflowService()
workflow = workflow_service.get_draft_workflow(app_model)
if not workflow:
raise ValueError("Missing draft workflow configuration, please check.")
export_data['workflow'] = {
"graph": workflow.graph_dict,
"features": workflow.features_dict
}
@classmethod
def _append_model_config_export_data(cls, export_data: dict, app_model: App) -> None:
"""
Append model config export data
:param export_data: export data
:param app_model: App instance
"""
app_model_config = app_model.app_model_config
if not app_model_config:
raise ValueError("Missing app configuration, please check.")
export_data['model_config'] = app_model_config.to_dict()

118
api/services/app_service.py
View File

@ -3,7 +3,6 @@ import logging
from datetime import datetime, timezone
from typing import cast
import yaml
from flask_login import current_user
from flask_sqlalchemy.pagination import Pagination
@ -17,13 +16,12 @@ from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelTy
from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
from core.tools.tool_manager import ToolManager
from core.tools.utils.configuration import ToolParameterConfigurationManager
from events.app_event import app_model_config_was_updated, app_was_created
from events.app_event import app_was_created
from extensions.ext_database import db
from models.account import Account
from models.model import App, AppMode, AppModelConfig
from models.tools import ApiToolProvider
from services.tag_service import TagService
from services.workflow_service import WorkflowService
from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task
@ -144,120 +142,6 @@ class AppService:
return app
def import_app(self, tenant_id: str, data: str, args: dict, account: Account) -> App:
"""
Import app
:param tenant_id: tenant id
:param data: import data
:param args: request args
:param account: Account instance
"""
try:
import_data = yaml.safe_load(data)
except yaml.YAMLError as e:
raise ValueError("Invalid YAML format in data argument.")
app_data = import_data.get('app')
model_config_data = import_data.get('model_config')
workflow = import_data.get('workflow')
if not app_data:
raise ValueError("Missing app in data argument")
app_mode = AppMode.value_of(app_data.get('mode'))
if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
if not workflow:
raise ValueError("Missing workflow in data argument "
"when app mode is advanced-chat or workflow")
elif app_mode in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION]:
if not model_config_data:
raise ValueError("Missing model_config in data argument "
"when app mode is chat, agent-chat or completion")
else:
raise ValueError("Invalid app mode")
app = App(
tenant_id=tenant_id,
mode=app_data.get('mode'),
name=args.get("name") if args.get("name") else app_data.get('name'),
description=args.get("description") if args.get("description") else app_data.get('description', ''),
icon=args.get("icon") if args.get("icon") else app_data.get('icon'),
icon_background=args.get("icon_background") if args.get("icon_background") \
else app_data.get('icon_background'),
enable_site=True,
enable_api=True
)
db.session.add(app)
db.session.commit()
app_was_created.send(app, account=account)
if workflow:
# init draft workflow
workflow_service = WorkflowService()
draft_workflow = workflow_service.sync_draft_workflow(
app_model=app,
graph=workflow.get('graph'),
features=workflow.get('features'),
unique_hash=None,
account=account
)
workflow_service.publish_workflow(
app_model=app,
account=account,
draft_workflow=draft_workflow
)
if model_config_data:
app_model_config = AppModelConfig()
app_model_config = app_model_config.from_model_config_dict(model_config_data)
app_model_config.app_id = app.id
db.session.add(app_model_config)
db.session.commit()
app.app_model_config_id = app_model_config.id
app_model_config_was_updated.send(
app,
app_model_config=app_model_config
)
return app
def export_app(self, app: App) -> str:
"""
Export app
:param app: App instance
:return:
"""
app_mode = AppMode.value_of(app.mode)
export_data = {
"app": {
"name": app.name,
"mode": app.mode,
"icon": app.icon,
"icon_background": app.icon_background,
"description": app.description
}
}
if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
workflow_service = WorkflowService()
workflow = workflow_service.get_draft_workflow(app)
export_data['workflow'] = {
"graph": workflow.graph_dict,
"features": workflow.features_dict
}
else:
app_model_config = app.app_model_config
export_data['model_config'] = app_model_config.to_dict()
return yaml.dump(export_data)
def get_app(self, app: App) -> App:
"""
Get App

8
api/services/recommended_app_service.py
View File

@ -4,12 +4,13 @@ from os import path
from typing import Optional
import requests
from flask import current_app
from configs import dify_config
from constants.languages import languages
from extensions.ext_database import db
from models.model import App, RecommendedApp
from services.app_service import AppService
from services.app_dsl_service import AppDslService
logger = logging.getLogger(__name__)
@ -186,16 +187,13 @@ class RecommendedAppService:
if not app_model or not app_model.is_public:
return None
app_service = AppService()
export_str = app_service.export_app(app_model)
return {
'id': app_model.id,
'name': app_model.name,
'icon': app_model.icon,
'icon_background': app_model.icon_background,
'mode': app_model.mode,
'export_data': export_str
'export_data': AppDslService.export_dsl(app_model=app_model)
}
@classmethod

52
api/services/workflow_service.py
View File

@ -3,8 +3,6 @@ import time
from datetime import datetime, timezone
from typing import Optional
import yaml
from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager
from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager
from core.model_runtime.utils.encoders import jsonable_encoder
@ -114,56 +112,6 @@ class WorkflowService:
# return draft workflow
return workflow
def import_draft_workflow(self, app_model: App,
data: str,
account: Account) -> Workflow:
"""
Import draft workflow
:param app_model: App instance
:param data: import data
:param account: Account instance
:return:
"""
try:
import_data = yaml.safe_load(data)
except yaml.YAMLError as e:
raise ValueError("Invalid YAML format in data argument.")
app_data = import_data.get('app')
workflow = import_data.get('workflow')
if not app_data:
raise ValueError("Missing app in data argument")
app_mode = AppMode.value_of(app_data.get('mode'))
if app_mode not in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
raise ValueError("Only support import workflow in advanced-chat or workflow app.")
if app_data.get('mode') != app_model.mode:
raise ValueError(f"App mode {app_data.get('mode')} is not matched with current app mode {app_model.mode}")
if not workflow:
raise ValueError("Missing workflow in data argument "
"when app mode is advanced-chat or workflow")
# fetch draft workflow by app_model
current_draft_workflow = self.get_draft_workflow(app_model=app_model)
if current_draft_workflow:
unique_hash = current_draft_workflow.unique_hash
else:
unique_hash = None
# sync draft workflow
draft_workflow = self.sync_draft_workflow(
app_model=app_model,
graph=workflow.get('graph'),
features=workflow.get('features'),
unique_hash=unique_hash,
account=account
)
return draft_workflow
def publish_workflow(self, app_model: App,
account: Account,
draft_workflow: Optional[Workflow] = None) -> Workflow: