diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 6952940649..1c42a57d43 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -15,6 +15,7 @@ from fields.app_fields import ( app_pagination_fields, ) from libs.login import login_required +from services.app_dsl_service import AppDslService from services.app_service import AppService ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion'] @@ -97,8 +98,42 @@ class AppImportApi(Resource): parser.add_argument('icon_background', type=str, location='json') args = parser.parse_args() - app_service = AppService() - app = app_service.import_app(current_user.current_tenant_id, args['data'], args, current_user) + app = AppDslService.import_and_create_new_app( + tenant_id=current_user.current_tenant_id, + data=args['data'], + args=args, + account=current_user + ) + + return app, 201 + + +class AppImportFromUrlApi(Resource): + @setup_required + @login_required + @account_initialization_required + @marshal_with(app_detail_fields_with_site) + @cloud_edition_billing_resource_check('apps') + def post(self): + """Import app from url""" + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: + raise Forbidden() + + parser = reqparse.RequestParser() + parser.add_argument('url', type=str, required=True, nullable=False, location='json') + parser.add_argument('name', type=str, location='json') + parser.add_argument('description', type=str, location='json') + parser.add_argument('icon', type=str, location='json') + parser.add_argument('icon_background', type=str, location='json') + args = parser.parse_args() + + app = AppDslService.import_and_create_new_app_from_url( + tenant_id=current_user.current_tenant_id, + url=args['url'], + args=args, + account=current_user + ) return app, 201 @@ -177,9 +212,13 @@ class AppCopyApi(Resource): parser.add_argument('icon_background', type=str, location='json') args = parser.parse_args() - app_service = AppService() - data = app_service.export_app(app_model) - app = app_service.import_app(current_user.current_tenant_id, data, args, current_user) + data = AppDslService.export_dsl(app_model=app_model) + app = AppDslService.import_and_create_new_app( + tenant_id=current_user.current_tenant_id, + data=data, + args=args, + account=current_user + ) return app, 201 @@ -195,10 +234,8 @@ class AppExportApi(Resource): if not current_user.is_editor: raise Forbidden() - app_service = AppService() - return { - "data": app_service.export_app(app_model) + "data": AppDslService.export_dsl(app_model=app_model) } @@ -322,6 +359,7 @@ class AppTraceApi(Resource): api.add_resource(AppListApi, '/apps') api.add_resource(AppImportApi, '/apps/import') +api.add_resource(AppImportFromUrlApi, '/apps/import/url') api.add_resource(AppApi, '/apps/') api.add_resource(AppCopyApi, '/apps//copy') api.add_resource(AppExportApi, '/apps//export') diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.py index cadb75c547..9f745ca120 100644 --- a/api/controllers/console/app/workflow.py +++ b/api/controllers/console/app/workflow.py @@ -20,6 +20,7 @@ from libs import helper from libs.helper import TimestampField, uuid_value from libs.login import current_user, login_required from models.model import App, AppMode +from services.app_dsl_service import AppDslService from services.app_generate_service import AppGenerateService from services.errors.app import WorkflowHashNotEqualError from services.workflow_service import WorkflowService @@ -128,8 +129,7 @@ class DraftWorkflowImportApi(Resource): parser.add_argument('data', type=str, required=True, nullable=False, location='json') args = parser.parse_args() - workflow_service = WorkflowService() - workflow = workflow_service.import_draft_workflow( + workflow = AppDslService.import_and_overwrite_workflow( app_model=app_model, data=args['data'], account=current_user diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py new file mode 100644 index 0000000000..050295002e --- /dev/null +++ b/api/services/app_dsl_service.py @@ -0,0 +1,407 @@ +import logging + +import httpx +import yaml # type: ignore + +from events.app_event import app_model_config_was_updated, app_was_created +from extensions.ext_database import db +from models.account import Account +from models.model import App, AppMode, AppModelConfig +from models.workflow import Workflow +from services.workflow_service import WorkflowService + +logger = logging.getLogger(__name__) + +current_dsl_version = "0.1.0" +dsl_to_dify_version_mapping: dict[str, str] = { + "0.1.0": "0.6.0", # dsl version -> from dify version +} + + +class AppDslService: + @classmethod + def import_and_create_new_app_from_url(cls, tenant_id: str, url: str, args: dict, account: Account) -> App: + """ + Import app dsl from url and create new app + :param tenant_id: tenant id + :param url: import url + :param args: request args + :param account: Account instance + """ + try: + max_size = 10 * 1024 * 1024 # 10MB + timeout = httpx.Timeout(10.0) + with httpx.stream("GET", url.strip(), follow_redirects=True, timeout=timeout) as response: + response.raise_for_status() + total_size = 0 + content = b"" + for chunk in response.iter_bytes(): + total_size += len(chunk) + if total_size > max_size: + raise ValueError("File size exceeds the limit of 10MB") + content += chunk + except httpx.HTTPStatusError as http_err: + raise ValueError(f"HTTP error occurred: {http_err}") + except httpx.RequestError as req_err: + raise ValueError(f"Request error occurred: {req_err}") + except Exception as e: + raise ValueError(f"Failed to fetch DSL from URL: {e}") + + if not content: + raise ValueError("Empty content from url") + + try: + data = content.decode("utf-8") + except UnicodeDecodeError as e: + raise ValueError(f"Error decoding content: {e}") + + return cls.import_and_create_new_app(tenant_id, data, args, account) + + @classmethod + def import_and_create_new_app(cls, tenant_id: str, data: str, args: dict, account: Account) -> App: + """ + Import app dsl and create new app + :param tenant_id: tenant id + :param data: import data + :param args: request args + :param account: Account instance + """ + try: + import_data = yaml.safe_load(data) + except yaml.YAMLError: + raise ValueError("Invalid YAML format in data argument.") + + # check or repair dsl version + import_data = cls._check_or_fix_dsl(import_data) + + app_data = import_data.get('app') + if not app_data: + raise ValueError("Missing app in data argument") + + # get app basic info + name = args.get("name") if args.get("name") else app_data.get('name') + description = args.get("description") if args.get("description") else app_data.get('description', '') + icon = args.get("icon") if args.get("icon") else app_data.get('icon') + icon_background = args.get("icon_background") if args.get("icon_background") \ + else app_data.get('icon_background') + + # import dsl and create app + app_mode = AppMode.value_of(app_data.get('mode')) + if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: + app = cls._import_and_create_new_workflow_based_app( + tenant_id=tenant_id, + app_mode=app_mode, + workflow_data=import_data.get('workflow'), + account=account, + name=name, + description=description, + icon=icon, + icon_background=icon_background + ) + elif app_mode in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION]: + app = cls._import_and_create_new_model_config_based_app( + tenant_id=tenant_id, + app_mode=app_mode, + model_config_data=import_data.get('model_config'), + account=account, + name=name, + description=description, + icon=icon, + icon_background=icon_background + ) + else: + raise ValueError("Invalid app mode") + + return app + + @classmethod + def import_and_overwrite_workflow(cls, app_model: App, data: str, account: Account) -> Workflow: + """ + Import app dsl and overwrite workflow + :param app_model: App instance + :param data: import data + :param account: Account instance + """ + try: + import_data = yaml.safe_load(data) + except yaml.YAMLError: + raise ValueError("Invalid YAML format in data argument.") + + # check or repair dsl version + import_data = cls._check_or_fix_dsl(import_data) + + app_data = import_data.get('app') + if not app_data: + raise ValueError("Missing app in data argument") + + # import dsl and overwrite app + app_mode = AppMode.value_of(app_data.get('mode')) + if app_mode not in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: + raise ValueError("Only support import workflow in advanced-chat or workflow app.") + + if app_data.get('mode') != app_model.mode: + raise ValueError( + f"App mode {app_data.get('mode')} is not matched with current app mode {app_mode.value}") + + return cls._import_and_overwrite_workflow_based_app( + app_model=app_model, + workflow_data=import_data.get('workflow'), + account=account, + ) + + @classmethod + def export_dsl(cls, app_model: App) -> str: + """ + Export app + :param app_model: App instance + :return: + """ + app_mode = AppMode.value_of(app_model.mode) + + export_data = { + "version": current_dsl_version, + "kind": "app", + "app": { + "name": app_model.name, + "mode": app_model.mode, + "icon": app_model.icon, + "icon_background": app_model.icon_background, + "description": app_model.description + } + } + + if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: + cls._append_workflow_export_data(export_data, app_model) + else: + cls._append_model_config_export_data(export_data, app_model) + + return yaml.dump(export_data) + + @classmethod + def _check_or_fix_dsl(cls, import_data: dict) -> dict: + """ + Check or fix dsl + + :param import_data: import data + """ + if not import_data.get('version'): + import_data['version'] = "0.1.0" + + if not import_data.get('kind') or import_data.get('kind') != "app": + import_data['kind'] = "app" + + if import_data.get('version') != current_dsl_version: + # Currently only one DSL version, so no difference checks or compatibility fixes will be performed. + logger.warning(f"DSL version {import_data.get('version')} is not compatible " + f"with current version {current_dsl_version}, related to " + f"Dify version {dsl_to_dify_version_mapping.get(current_dsl_version)}.") + + return import_data + + @classmethod + def _import_and_create_new_workflow_based_app(cls, + tenant_id: str, + app_mode: AppMode, + workflow_data: dict, + account: Account, + name: str, + description: str, + icon: str, + icon_background: str) -> App: + """ + Import app dsl and create new workflow based app + + :param tenant_id: tenant id + :param app_mode: app mode + :param workflow_data: workflow data + :param account: Account instance + :param name: app name + :param description: app description + :param icon: app icon + :param icon_background: app icon background + """ + if not workflow_data: + raise ValueError("Missing workflow in data argument " + "when app mode is advanced-chat or workflow") + + app = cls._create_app( + tenant_id=tenant_id, + app_mode=app_mode, + account=account, + name=name, + description=description, + icon=icon, + icon_background=icon_background + ) + + # init draft workflow + workflow_service = WorkflowService() + draft_workflow = workflow_service.sync_draft_workflow( + app_model=app, + graph=workflow_data.get('graph', {}), + features=workflow_data.get('../core/app/features', {}), + unique_hash=None, + account=account + ) + workflow_service.publish_workflow( + app_model=app, + account=account, + draft_workflow=draft_workflow + ) + + return app + + @classmethod + def _import_and_overwrite_workflow_based_app(cls, + app_model: App, + workflow_data: dict, + account: Account) -> Workflow: + """ + Import app dsl and overwrite workflow based app + + :param app_model: App instance + :param workflow_data: workflow data + :param account: Account instance + """ + if not workflow_data: + raise ValueError("Missing workflow in data argument " + "when app mode is advanced-chat or workflow") + + # fetch draft workflow by app_model + workflow_service = WorkflowService() + current_draft_workflow = workflow_service.get_draft_workflow(app_model=app_model) + if current_draft_workflow: + unique_hash = current_draft_workflow.unique_hash + else: + unique_hash = None + + # sync draft workflow + draft_workflow = workflow_service.sync_draft_workflow( + app_model=app_model, + graph=workflow_data.get('graph', {}), + features=workflow_data.get('features', {}), + unique_hash=unique_hash, + account=account + ) + + return draft_workflow + + @classmethod + def _import_and_create_new_model_config_based_app(cls, + tenant_id: str, + app_mode: AppMode, + model_config_data: dict, + account: Account, + name: str, + description: str, + icon: str, + icon_background: str) -> App: + """ + Import app dsl and create new model config based app + + :param tenant_id: tenant id + :param app_mode: app mode + :param model_config_data: model config data + :param account: Account instance + :param name: app name + :param description: app description + :param icon: app icon + :param icon_background: app icon background + """ + if not model_config_data: + raise ValueError("Missing model_config in data argument " + "when app mode is chat, agent-chat or completion") + + app = cls._create_app( + tenant_id=tenant_id, + app_mode=app_mode, + account=account, + name=name, + description=description, + icon=icon, + icon_background=icon_background + ) + + app_model_config = AppModelConfig() + app_model_config = app_model_config.from_model_config_dict(model_config_data) + app_model_config.app_id = app.id + + db.session.add(app_model_config) + db.session.commit() + + app.app_model_config_id = app_model_config.id + + app_model_config_was_updated.send( + app, + app_model_config=app_model_config + ) + + return app + + @classmethod + def _create_app(cls, + tenant_id: str, + app_mode: AppMode, + account: Account, + name: str, + description: str, + icon: str, + icon_background: str) -> App: + """ + Create new app + + :param tenant_id: tenant id + :param app_mode: app mode + :param account: Account instance + :param name: app name + :param description: app description + :param icon: app icon + :param icon_background: app icon background + """ + app = App( + tenant_id=tenant_id, + mode=app_mode.value, + name=name, + description=description, + icon=icon, + icon_background=icon_background, + enable_site=True, + enable_api=True + ) + + db.session.add(app) + db.session.commit() + + app_was_created.send(app, account=account) + + return app + + @classmethod + def _append_workflow_export_data(cls, export_data: dict, app_model: App) -> None: + """ + Append workflow export data + :param export_data: export data + :param app_model: App instance + """ + workflow_service = WorkflowService() + workflow = workflow_service.get_draft_workflow(app_model) + if not workflow: + raise ValueError("Missing draft workflow configuration, please check.") + + export_data['workflow'] = { + "graph": workflow.graph_dict, + "features": workflow.features_dict + } + + @classmethod + def _append_model_config_export_data(cls, export_data: dict, app_model: App) -> None: + """ + Append model config export data + :param export_data: export data + :param app_model: App instance + """ + app_model_config = app_model.app_model_config + if not app_model_config: + raise ValueError("Missing app configuration, please check.") + + export_data['model_config'] = app_model_config.to_dict() diff --git a/api/services/app_service.py b/api/services/app_service.py index ca3c8d4fdc..36efde7825 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -3,7 +3,6 @@ import logging from datetime import datetime, timezone from typing import cast -import yaml from flask_login import current_user from flask_sqlalchemy.pagination import Pagination @@ -17,13 +16,12 @@ from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelTy from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.tools.tool_manager import ToolManager from core.tools.utils.configuration import ToolParameterConfigurationManager -from events.app_event import app_model_config_was_updated, app_was_created +from events.app_event import app_was_created from extensions.ext_database import db from models.account import Account from models.model import App, AppMode, AppModelConfig from models.tools import ApiToolProvider from services.tag_service import TagService -from services.workflow_service import WorkflowService from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task @@ -144,120 +142,6 @@ class AppService: return app - def import_app(self, tenant_id: str, data: str, args: dict, account: Account) -> App: - """ - Import app - :param tenant_id: tenant id - :param data: import data - :param args: request args - :param account: Account instance - """ - try: - import_data = yaml.safe_load(data) - except yaml.YAMLError as e: - raise ValueError("Invalid YAML format in data argument.") - - app_data = import_data.get('app') - model_config_data = import_data.get('model_config') - workflow = import_data.get('workflow') - - if not app_data: - raise ValueError("Missing app in data argument") - - app_mode = AppMode.value_of(app_data.get('mode')) - if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: - if not workflow: - raise ValueError("Missing workflow in data argument " - "when app mode is advanced-chat or workflow") - elif app_mode in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION]: - if not model_config_data: - raise ValueError("Missing model_config in data argument " - "when app mode is chat, agent-chat or completion") - else: - raise ValueError("Invalid app mode") - - app = App( - tenant_id=tenant_id, - mode=app_data.get('mode'), - name=args.get("name") if args.get("name") else app_data.get('name'), - description=args.get("description") if args.get("description") else app_data.get('description', ''), - icon=args.get("icon") if args.get("icon") else app_data.get('icon'), - icon_background=args.get("icon_background") if args.get("icon_background") \ - else app_data.get('icon_background'), - enable_site=True, - enable_api=True - ) - - db.session.add(app) - db.session.commit() - - app_was_created.send(app, account=account) - - if workflow: - # init draft workflow - workflow_service = WorkflowService() - draft_workflow = workflow_service.sync_draft_workflow( - app_model=app, - graph=workflow.get('graph'), - features=workflow.get('features'), - unique_hash=None, - account=account - ) - workflow_service.publish_workflow( - app_model=app, - account=account, - draft_workflow=draft_workflow - ) - - if model_config_data: - app_model_config = AppModelConfig() - app_model_config = app_model_config.from_model_config_dict(model_config_data) - app_model_config.app_id = app.id - - db.session.add(app_model_config) - db.session.commit() - - app.app_model_config_id = app_model_config.id - - app_model_config_was_updated.send( - app, - app_model_config=app_model_config - ) - - return app - - def export_app(self, app: App) -> str: - """ - Export app - :param app: App instance - :return: - """ - app_mode = AppMode.value_of(app.mode) - - export_data = { - "app": { - "name": app.name, - "mode": app.mode, - "icon": app.icon, - "icon_background": app.icon_background, - "description": app.description - } - } - - if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: - workflow_service = WorkflowService() - workflow = workflow_service.get_draft_workflow(app) - export_data['workflow'] = { - "graph": workflow.graph_dict, - "features": workflow.features_dict - } - else: - app_model_config = app.app_model_config - - export_data['model_config'] = app_model_config.to_dict() - - return yaml.dump(export_data) - def get_app(self, app: App) -> App: """ Get App diff --git a/api/services/recommended_app_service.py b/api/services/recommended_app_service.py index c4733b6d3f..1c1c5be17c 100644 --- a/api/services/recommended_app_service.py +++ b/api/services/recommended_app_service.py @@ -4,12 +4,13 @@ from os import path from typing import Optional import requests +from flask import current_app from configs import dify_config from constants.languages import languages from extensions.ext_database import db from models.model import App, RecommendedApp -from services.app_service import AppService +from services.app_dsl_service import AppDslService logger = logging.getLogger(__name__) @@ -186,16 +187,13 @@ class RecommendedAppService: if not app_model or not app_model.is_public: return None - app_service = AppService() - export_str = app_service.export_app(app_model) - return { 'id': app_model.id, 'name': app_model.name, 'icon': app_model.icon, 'icon_background': app_model.icon_background, 'mode': app_model.mode, - 'export_data': export_str + 'export_data': AppDslService.export_dsl(app_model=app_model) } @classmethod diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 025c1090b4..6235ecf0a3 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -3,8 +3,6 @@ import time from datetime import datetime, timezone from typing import Optional -import yaml - from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager from core.model_runtime.utils.encoders import jsonable_encoder @@ -114,56 +112,6 @@ class WorkflowService: # return draft workflow return workflow - def import_draft_workflow(self, app_model: App, - data: str, - account: Account) -> Workflow: - """ - Import draft workflow - :param app_model: App instance - :param data: import data - :param account: Account instance - :return: - """ - try: - import_data = yaml.safe_load(data) - except yaml.YAMLError as e: - raise ValueError("Invalid YAML format in data argument.") - - app_data = import_data.get('app') - workflow = import_data.get('workflow') - - if not app_data: - raise ValueError("Missing app in data argument") - - app_mode = AppMode.value_of(app_data.get('mode')) - if app_mode not in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: - raise ValueError("Only support import workflow in advanced-chat or workflow app.") - - if app_data.get('mode') != app_model.mode: - raise ValueError(f"App mode {app_data.get('mode')} is not matched with current app mode {app_model.mode}") - - if not workflow: - raise ValueError("Missing workflow in data argument " - "when app mode is advanced-chat or workflow") - - # fetch draft workflow by app_model - current_draft_workflow = self.get_draft_workflow(app_model=app_model) - if current_draft_workflow: - unique_hash = current_draft_workflow.unique_hash - else: - unique_hash = None - - # sync draft workflow - draft_workflow = self.sync_draft_workflow( - app_model=app_model, - graph=workflow.get('graph'), - features=workflow.get('features'), - unique_hash=unique_hash, - account=account - ) - - return draft_workflow - def publish_workflow(self, app_model: App, account: Account, draft_workflow: Optional[Workflow] = None) -> Workflow: