Merge branch 'main' into feat/r2

.editorconfig

@@ -5,18 +5,35 @@ root = true
 
 # Unix-style newlines with a newline ending every file
 [*]
+charset = utf-8
 end_of_line = lf
 insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.py]
+indent_size = 4
+indent_style = space
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2
+
+[*.toml]
+indent_size = 4
+indent_style = space
+
+# Markdown and MDX are whitespace sensitive languages.
+# Do not remove trailing spaces.
+[*.{md,mdx}]
+trim_trailing_whitespace = false
 
 # Matches multiple files with brace expansion notation
 # Set default charset
 [*.{js,tsx}]
-charset = utf-8
 indent_style = space
 indent_size = 2
 
-
+# Matches the exact files package.json
-# Matches the exact files either package.json or .travis.yml
+[package.json]
-[{package.json,.travis.yml}]
 indent_style = space
 indent_size = 2

.github/linters/editorconfig-checker.json

@@ -0,0 +1,22 @@
+{
+    "Verbose": false,
+    "Debug": false,
+    "IgnoreDefaults": false,
+    "SpacesAfterTabs": false,
+    "NoColor": false,
+    "Exclude": [
+        "^web/public/vs/",
+        "^web/public/pdf.worker.min.mjs$",
+        "web/app/components/base/icons/src/vender/"
+    ],
+    "AllowedContentTypes": [],
+    "PassedFiles": [],
+    "Disable": {
+        "EndOfLine": false,
+        "Indentation": false,
+        "IndentSize": true,
+        "InsertFinalNewline": false,
+        "TrimTrailingWhitespace": false,
+        "MaxLineLength": false
+    }
+}

.github/workflows/api-tests.yml

@@ -88,3 +88,6 @@ jobs:
 
       - name: Run Workflow
         run: uv run --project api bash dev/pytest/pytest_workflow.sh
+
+      - name: Run Tool
+        run: uv run --project api bash dev/pytest/pytest_tools.sh

.github/workflows/style.yml

@@ -9,6 +9,12 @@ concurrency:
   group: style-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+permissions:
+  checks: write
+  statuses: write
+  contents: read
+
+
 jobs:
   python-style:
     name: Python Style
@@ -43,8 +49,8 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
           uv run --directory api ruff --version
-          uv run --directory api ruff check ./
+          uv run --directory api ruff check --diff ./
-          uv run --directory api ruff format --check ./
+          uv run --directory api ruff format --check --diff ./
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -163,3 +169,14 @@ jobs:
           VALIDATE_DOCKERFILE_HADOLINT: true
           VALIDATE_XML: true
           VALIDATE_YAML: true
+
+      - name: EditorConfig checks
+        uses: super-linter/super-linter/slim@v7
+        env:
+          DEFAULT_BRANCH: main
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          IGNORE_GENERATED_FILES: true
+          IGNORE_GITIGNORED_FILES: true
+          # EditorConfig validation
+          VALIDATE_EDITORCONFIG: true
+          EDITORCONFIG_FILE_NAME: editorconfig-checker.json

api/.env.example

@@ -476,6 +476,7 @@ LOGIN_LOCKOUT_DURATION=86400
 ENABLE_OTEL=false
 OTLP_BASE_ENDPOINT=http://localhost:4318
 OTLP_API_KEY=
+OTEL_EXPORTER_OTLP_PROTOCOL=
 OTEL_EXPORTER_TYPE=otlp
 OTEL_SAMPLING_RATE=0.1
 OTEL_BATCH_EXPORT_SCHEDULE_DELAY=5000

api/README.md

@@ -90,3 +90,4 @@
    ```bash
    uv run -P api bash dev/pytest/pytest_all_tests.sh
    ```
+

api/app.py

@@ -18,7 +18,7 @@ else:
     # so we need to disable gevent in debug mode.
     # If you are using debugpy and set GEVENT_SUPPORT=True, you can debug with gevent.
     if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}:
-        from gevent import monkey  # type: ignore
+        from gevent import monkey
 
         # gevent
         monkey.patch_all()

api/app_factory.py

@@ -52,10 +52,8 @@ def initialize_extensions(app: DifyApp):
         ext_mail,
         ext_migrate,
         ext_otel,
-        ext_otel_patch,
         ext_proxy_fix,
         ext_redis,
-        ext_repositories,
         ext_sentry,
         ext_set_secretkey,
         ext_storage,
@@ -76,7 +74,6 @@ def initialize_extensions(app: DifyApp):
         ext_migrate,
         ext_redis,
         ext_storage,
-        ext_repositories,
         ext_celery,
         ext_login,
         ext_mail,
@@ -85,7 +82,6 @@ def initialize_extensions(app: DifyApp):
         ext_proxy_fix,
         ext_blueprints,
         ext_commands,
-        ext_otel_patch,  # Apply patch before initializing OpenTelemetry
         ext_otel,
     ]
     for ext in extensions:

api/commands.py

@@ -6,6 +6,7 @@ from typing import Optional
 
 import click
 from flask import current_app
+from sqlalchemy import select
 from werkzeug.exceptions import NotFound
 
 from configs import dify_config
@@ -297,11 +298,11 @@ def migrate_knowledge_vector_database():
     page = 1
     while True:
         try:
-            datasets = (
+            stmt = (
-                Dataset.query.filter(Dataset.indexing_technique == "high_quality")
+                select(Dataset).filter(Dataset.indexing_technique == "high_quality").order_by(Dataset.created_at.desc())
-                .order_by(Dataset.created_at.desc())
-                .paginate(page=page, per_page=50)
             )
+
+            datasets = db.paginate(select=stmt, page=page, per_page=50, max_per_page=50, error_out=False)
         except NotFound:
             break
 
@@ -551,11 +552,12 @@ def old_metadata_migration():
     page = 1
     while True:
         try:
-            documents = (
+            stmt = (
-                DatasetDocument.query.filter(DatasetDocument.doc_metadata is not None)
+                select(DatasetDocument)
+                .filter(DatasetDocument.doc_metadata.is_not(None))
                 .order_by(DatasetDocument.created_at.desc())
-                .paginate(page=page, per_page=50)
             )
+            documents = db.paginate(select=stmt, page=page, per_page=50, max_per_page=50, error_out=False)
         except NotFound:
             break
         if not documents:
@@ -592,11 +594,15 @@ def old_metadata_migration():
                             )
                             db.session.add(dataset_metadata_binding)
                         else:
-                            dataset_metadata_binding = DatasetMetadataBinding.query.filter(
+                            dataset_metadata_binding = (
+                                db.session.query(DatasetMetadataBinding)  # type: ignore
+                                .filter(
                                     DatasetMetadataBinding.dataset_id == document.dataset_id,
                                     DatasetMetadataBinding.document_id == document.id,
                                     DatasetMetadataBinding.metadata_id == dataset_metadata.id,
-                            ).first()
+                                )
+                                .first()
+                            )
                             if not dataset_metadata_binding:
                                 dataset_metadata_binding = DatasetMetadataBinding(
                                     tenant_id=document.tenant_id,
@@ -668,7 +674,7 @@ def upgrade_db():
             click.echo(click.style("Starting database migration.", fg="green"))
 
             # run db migration
-            import flask_migrate  # type: ignore
+            import flask_migrate
 
             flask_migrate.upgrade()
 
@@ -818,8 +824,9 @@ def clear_free_plan_tenant_expired_logs(days: int, batch: int, tenant_ids: list[
     click.echo(click.style("Clear free plan tenant expired logs completed.", fg="green"))
 
 
+@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.")
 @click.command("clear-orphaned-file-records", help="Clear orphaned file records.")
-def clear_orphaned_file_records():
+def clear_orphaned_file_records(force: bool):
     """
     Clear orphaned file records in the database.
     """
@@ -845,7 +852,15 @@ def clear_orphaned_file_records():
 
     # notify user and ask for confirmation
     click.echo(
-        click.style("This command will find and delete orphaned file records in the following tables:", fg="yellow")
+        click.style(
+            "This command will first find and delete orphaned file records from the message_files table,", fg="yellow"
+        )
+    )
+    click.echo(
+        click.style(
+            "and then it will find and delete orphaned file records in the following tables:",
+            fg="yellow",
+        )
     )
     for files_table in files_tables:
         click.echo(click.style(f"- {files_table['table']}", fg="yellow"))
@@ -878,11 +893,55 @@ def clear_orphaned_file_records():
             fg="yellow",
         )
     )
+    if not force:
         click.confirm("Do you want to proceed?", abort=True)
 
     # start the cleanup process
     click.echo(click.style("Starting orphaned file records cleanup.", fg="white"))
 
+    # clean up the orphaned records in the message_files table where message_id doesn't exist in messages table
+    try:
+        click.echo(
+            click.style("- Listing message_files records where message_id doesn't exist in messages table", fg="white")
+        )
+        query = (
+            "SELECT mf.id, mf.message_id "
+            "FROM message_files mf LEFT JOIN messages m ON mf.message_id = m.id "
+            "WHERE m.id IS NULL"
+        )
+        orphaned_message_files = []
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(query))
+            for i in rs:
+                orphaned_message_files.append({"id": str(i[0]), "message_id": str(i[1])})
+
+        if orphaned_message_files:
+            click.echo(click.style(f"Found {len(orphaned_message_files)} orphaned message_files records:", fg="white"))
+            for record in orphaned_message_files:
+                click.echo(click.style(f"  - id: {record['id']}, message_id: {record['message_id']}", fg="black"))
+
+            if not force:
+                click.confirm(
+                    (
+                        f"Do you want to proceed "
+                        f"to delete all {len(orphaned_message_files)} orphaned message_files records?"
+                    ),
+                    abort=True,
+                )
+
+            click.echo(click.style("- Deleting orphaned message_files records", fg="white"))
+            query = "DELETE FROM message_files WHERE id IN :ids"
+            with db.engine.begin() as conn:
+                conn.execute(db.text(query), {"ids": tuple([record["id"] for record in orphaned_message_files])})
+            click.echo(
+                click.style(f"Removed {len(orphaned_message_files)} orphaned message_files records.", fg="green")
+            )
+        else:
+            click.echo(click.style("No orphaned message_files records found. There is nothing to delete.", fg="green"))
+    except Exception as e:
+        click.echo(click.style(f"Error deleting orphaned message_files records: {str(e)}", fg="red"))
+
+    # clean up the orphaned records in the rest of the *_files tables
     try:
         # fetch file id and keys from each table
         all_files_in_tables = []
@@ -964,6 +1023,7 @@ def clear_orphaned_file_records():
     click.echo(click.style(f"Found {len(orphaned_files)} orphaned file records.", fg="white"))
     for file in orphaned_files:
         click.echo(click.style(f"- orphaned file id: {file}", fg="black"))
+    if not force:
         click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True)
 
     # delete orphaned records for each file
@@ -979,8 +1039,9 @@ def clear_orphaned_file_records():
     click.echo(click.style(f"Removed {len(orphaned_files)} orphaned file records.", fg="green"))
 
 
+@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.")
 @click.command("remove-orphaned-files-on-storage", help="Remove orphaned files on the storage.")
-def remove_orphaned_files_on_storage():
+def remove_orphaned_files_on_storage(force: bool):
     """
     Remove orphaned files on the storage.
     """
@@ -1028,6 +1089,7 @@ def remove_orphaned_files_on_storage():
             fg="yellow",
         )
     )
+    if not force:
         click.confirm("Do you want to proceed?", abort=True)
 
     # start the cleanup process
@@ -1069,6 +1131,7 @@ def remove_orphaned_files_on_storage():
     click.echo(click.style(f"Found {len(orphaned_files)} orphaned files.", fg="white"))
     for file in orphaned_files:
         click.echo(click.style(f"- orphaned file: {file}", fg="black"))
+    if not force:
         click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True)
 
     # delete orphaned files

api/configs/feature/__init__.py

@@ -74,7 +74,7 @@ class CodeExecutionSandboxConfig(BaseSettings):
 
     CODE_EXECUTION_ENDPOINT: HttpUrl = Field(
         description="URL endpoint for the code execution service",
-        default="http://sandbox:8194",
+        default=HttpUrl("http://sandbox:8194"),
     )
 
     CODE_EXECUTION_API_KEY: str = Field(
@@ -145,7 +145,7 @@ class PluginConfig(BaseSettings):
 
     PLUGIN_DAEMON_URL: HttpUrl = Field(
         description="Plugin API URL",
-        default="http://localhost:5002",
+        default=HttpUrl("http://localhost:5002"),
     )
 
     PLUGIN_DAEMON_KEY: str = Field(
@@ -188,7 +188,7 @@ class MarketplaceConfig(BaseSettings):
 
     MARKETPLACE_API_URL: HttpUrl = Field(
         description="Marketplace API URL",
-        default="https://marketplace.dify.ai",
+        default=HttpUrl("https://marketplace.dify.ai"),
     )
 
 
@@ -398,6 +398,11 @@ class InnerAPIConfig(BaseSettings):
         default=False,
     )
 
+    INNER_API_KEY: Optional[str] = Field(
+        description="API key for accessing the internal API",
+        default=None,
+    )
+
 
 class LoggingConfig(BaseSettings):
     """

api/configs/middleware/__init__.py

@@ -1,6 +1,6 @@
 import os
 from typing import Any, Literal, Optional
-from urllib.parse import quote_plus
+from urllib.parse import parse_qsl, quote_plus
 
 from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt, computed_field
 from pydantic_settings import BaseSettings
@@ -173,17 +173,31 @@ class DatabaseConfig(BaseSettings):
 
     RETRIEVAL_SERVICE_EXECUTORS: NonNegativeInt = Field(
         description="Number of processes for the retrieval service, default to CPU cores.",
-        default=os.cpu_count(),
+        default=os.cpu_count() or 1,
     )
 
-    @computed_field
+    @computed_field  # type: ignore[misc]
+    @property
     def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]:
+        # Parse DB_EXTRAS for 'options'
+        db_extras_dict = dict(parse_qsl(self.DB_EXTRAS))
+        options = db_extras_dict.get("options", "")
+        # Always include timezone
+        timezone_opt = "-c timezone=UTC"
+        if options:
+            # Merge user options and timezone
+            merged_options = f"{options} {timezone_opt}"
+        else:
+            merged_options = timezone_opt
+
+        connect_args = {"options": merged_options}
+
         return {
             "pool_size": self.SQLALCHEMY_POOL_SIZE,
             "max_overflow": self.SQLALCHEMY_MAX_OVERFLOW,
             "pool_recycle": self.SQLALCHEMY_POOL_RECYCLE,
             "pool_pre_ping": self.SQLALCHEMY_POOL_PRE_PING,
-            "connect_args": {"options": "-c timezone=UTC"},
+            "connect_args": connect_args,
         }
 
 

api/configs/middleware/cache/redis_config.py

@@ -83,3 +83,13 @@ class RedisConfig(BaseSettings):
         description="Password for Redis Clusters authentication (if required)",
         default=None,
     )
+
+    REDIS_SERIALIZATION_PROTOCOL: int = Field(
+        description="Redis serialization protocol (RESP) version",
+        default=3,
+    )
+
+    REDIS_ENABLE_CLIENT_SIDE_CACHE: bool = Field(
+        description="Enable client side cache in redis",
+        default=False,
+    )

api/configs/middleware/vdb/opensearch_config.py

@@ -1,4 +1,5 @@
-from typing import Optional
+import enum
+from typing import Literal, Optional
 
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
@@ -9,6 +10,14 @@ class OpenSearchConfig(BaseSettings):
     Configuration settings for OpenSearch
     """
 
+    class AuthMethod(enum.StrEnum):
+        """
+        Authentication method for OpenSearch
+        """
+
+        BASIC = "basic"
+        AWS_MANAGED_IAM = "aws_managed_iam"
+
     OPENSEARCH_HOST: Optional[str] = Field(
         description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
         default=None,
@@ -19,6 +28,16 @@ class OpenSearchConfig(BaseSettings):
         default=9200,
     )
 
+    OPENSEARCH_SECURE: bool = Field(
+        description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
+        default=False,
+    )
+
+    OPENSEARCH_AUTH_METHOD: AuthMethod = Field(
+        description="Authentication method for OpenSearch connection (default is 'basic')",
+        default=AuthMethod.BASIC,
+    )
+
     OPENSEARCH_USER: Optional[str] = Field(
         description="Username for authenticating with OpenSearch",
         default=None,
@@ -29,7 +48,11 @@ class OpenSearchConfig(BaseSettings):
         default=None,
     )
 
-    OPENSEARCH_SECURE: bool = Field(
+    OPENSEARCH_AWS_REGION: Optional[str] = Field(
-        description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
+        description="AWS region for OpenSearch (e.g. 'us-west-2')",
-        default=False,
+        default=None,
+    )
+
+    OPENSEARCH_AWS_SERVICE: Optional[Literal["es", "aoss"]] = Field(
+        description="AWS service for OpenSearch (e.g. 'aoss' for OpenSearch Serverless)", default=None
     )

api/configs/observability/otel/otel_config.py

@@ -27,6 +27,11 @@ class OTelConfig(BaseSettings):
         default="otlp",
     )
 
+    OTEL_EXPORTER_OTLP_PROTOCOL: str = Field(
+        description="OTLP exporter protocol ('grpc' or 'http')",
+        default="http",
+    )
+
     OTEL_SAMPLING_RATE: float = Field(default=0.1, description="Sampling rate for traces (0.0 to 1.0)")
 
     OTEL_BATCH_EXPORT_SCHEDULE_DELAY: int = Field(

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="1.3.0",
+        default="1.3.1",
     )
 
     COMMIT_SHA: str = Field(

api/constants/__init__.py

@@ -16,11 +16,25 @@ AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
 
 
 if dify_config.ETL_TYPE == "Unstructured":
-    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls"]
+    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "vtt", "properties"]
     DOCUMENT_EXTENSIONS.extend(("doc", "docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
     if dify_config.UNSTRUCTURED_API_URL:
         DOCUMENT_EXTENSIONS.append("ppt")
     DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
 else:
-    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
+    DOCUMENT_EXTENSIONS = [
+        "txt",
+        "markdown",
+        "md",
+        "mdx",
+        "pdf",
+        "html",
+        "htm",
+        "xlsx",
+        "xls",
+        "docx",
+        "csv",
+        "vtt",
+        "properties",
+    ]
     DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])

api/constants/mimetypes.py

@@ -0,0 +1,7 @@
+# The two constants below should keep in sync.
+# Default content type for files which have no explicit content type.
+
+DEFAULT_MIME_TYPE = "application/octet-stream"
+# Default file extension for files which have no explicit content type, should
+# correspond to the `DEFAULT_MIME_TYPE` above.
+DEFAULT_EXTENSION = ".bin"

api/controllers/common/fields.py

@@ -1,4 +1,6 @@
-from flask_restful import fields  # type: ignore
+from flask_restful import fields
+
+from libs.helper import AppIconUrlField
 
 parameters__system_parameters = {
     "image_file_size_limit": fields.Integer,
@@ -22,3 +24,20 @@ parameters_fields = {
     "file_upload": fields.Raw,
     "system_parameters": fields.Nested(parameters__system_parameters),
 }
+
+site_fields = {
+    "title": fields.String,
+    "chat_color_theme": fields.String,
+    "chat_color_theme_inverted": fields.Boolean,
+    "icon_type": fields.String,
+    "icon": fields.String,
+    "icon_background": fields.String,
+    "icon_url": AppIconUrlField,
+    "description": fields.String,
+    "copyright": fields.String,
+    "privacy_policy": fields.String,
+    "custom_disclaimer": fields.String,
+    "default_language": fields.String,
+    "show_workflow_steps": fields.Boolean,
+    "use_icon_as_answer_icon": fields.Boolean,
+}

api/controllers/console/admin.py

@@ -1,7 +1,7 @@
 from functools import wraps
 
 from flask import request
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound, Unauthorized

api/controllers/console/apikey.py

@@ -1,7 +1,7 @@
 from typing import Any
 
-import flask_restful  # type: ignore
+import flask_restful
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with
 from sqlalchemy import select
 from sqlalchemy.orm import Session

api/controllers/console/app/advanced_prompt_template.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required, setup_required

api/controllers/console/app/agent.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model

api/controllers/console/app/annotation.py

@@ -1,6 +1,6 @@
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api

api/controllers/console/app/app.py

@@ -1,8 +1,8 @@
 import uuid
 from typing import cast
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, inputs, marshal, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, inputs, marshal, marshal_with, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import BadRequest, Forbidden, abort

api/controllers/console/app/app_import.py

@@ -1,7 +1,7 @@
 from typing import cast
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
 

api/controllers/console/app/audio.py

@@ -1,7 +1,7 @@
 import logging
 
 from flask import request
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import InternalServerError
 
 import services

api/controllers/console/app/completion.py

@@ -1,7 +1,7 @@
 import logging
 
-import flask_login  # type: ignore
+import flask_login
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import InternalServerError, NotFound
 
 import services

api/controllers/console/app/conversation.py

@@ -1,9 +1,9 @@
 from datetime import UTC, datetime
 
 import pytz  # pip install pytz
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range  # type: ignore
+from flask_restful.inputs import int_range
 from sqlalchemy import func, or_
 from sqlalchemy.orm import joinedload
 from werkzeug.exceptions import Forbidden, NotFound

api/controllers/console/app/conversation_variables.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 

api/controllers/console/app/generator.py

@@ -1,7 +1,7 @@
 import os
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.error import (

api/controllers/console/app/message.py

@@ -1,8 +1,8 @@
 import logging
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, fields, marshal_with, reqparse
-from flask_restful.inputs import int_range  # type: ignore
+from flask_restful.inputs import int_range
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 from controllers.console import api

api/controllers/console/app/model_config.py

@@ -2,8 +2,8 @@ import json
 from typing import cast
 
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource  # type: ignore
+from flask_restful import Resource
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model

api/controllers/console/app/ops_trace.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import BadRequest
 
 from controllers.console import api

api/controllers/console/app/site.py

@@ -1,7 +1,7 @@
 from datetime import UTC, datetime
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden, NotFound
 
 from constants.languages import supported_language

api/controllers/console/app/statistic.py

@@ -3,8 +3,8 @@ from decimal import Decimal
 
 import pytz
 from flask import jsonify
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model

api/controllers/console/app/workflow.py

@@ -3,7 +3,7 @@ import logging
 from typing import cast
 
 from flask import abort, request
-from flask_restful import Resource, inputs, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, inputs, marshal_with, reqparse
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 

api/controllers/console/app/workflow_app_log.py

@@ -1,6 +1,6 @@
 from dateutil.parser import isoparse
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range  # type: ignore
+from flask_restful.inputs import int_range
 from sqlalchemy.orm import Session
 
 from controllers.console import api

api/controllers/console/app/workflow_run.py

@@ -1,5 +1,5 @@
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range  # type: ignore
+from flask_restful.inputs import int_range
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model

api/controllers/console/app/workflow_statistic.py

@@ -3,8 +3,8 @@ from decimal import Decimal
 
 import pytz
 from flask import jsonify
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model

api/controllers/console/auth/activate.py

@@ -1,7 +1,7 @@
 import datetime
 
 from flask import request
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from constants.languages import supported_language
 from controllers.console import api

api/controllers/console/auth/data_source_bearer_auth.py

@@ -1,5 +1,5 @@
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api

api/controllers/console/auth/data_source_oauth.py

@@ -2,8 +2,8 @@ import logging
 
 import requests
 from flask import current_app, redirect, request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource  # type: ignore
+from flask_restful import Resource
 from werkzeug.exceptions import Forbidden
 
 from configs import dify_config

api/controllers/console/auth/forgot_password.py

@@ -2,7 +2,7 @@ import base64
 import secrets
 
 from flask import request
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 

api/controllers/console/auth/login.py

@@ -1,8 +1,8 @@
 from typing import cast
 
-import flask_login  # type: ignore
+import flask_login
 from flask import request
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 import services
 from configs import dify_config

api/controllers/console/auth/oauth.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 import requests
 from flask import current_app, redirect, request
-from flask_restful import Resource  # type: ignore
+from flask_restful import Resource
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Unauthorized

api/controllers/console/billing/billing.py

@@ -1,5 +1,5 @@
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required, only_edition_cloud, setup_required

api/controllers/console/billing/compliance.py

@@ -1,6 +1,6 @@
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from libs.helper import extract_remote_ip
 from libs.login import login_required

api/controllers/console/datasets/data_source.py

@@ -2,8 +2,8 @@ import datetime
 import json
 
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound

api/controllers/console/datasets/datasets.py

@@ -1,7 +1,7 @@
-import flask_restful  # type: ignore
+import flask_restful
 from flask import request
-from flask_login import current_user  # type: ignore  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
@@ -526,14 +526,20 @@ class DatasetIndexingStatusApi(Resource):
         )
         documents_status = []
         for document in documents:
-            completed_segments = DocumentSegment.query.filter(
+            completed_segments = (
+                db.session.query(DocumentSegment)
+                .filter(
                     DocumentSegment.completed_at.isnot(None),
                     DocumentSegment.document_id == str(document.id),
                     DocumentSegment.status != "re_segment",
-            ).count()
+                )
-            total_segments = DocumentSegment.query.filter(
+                .count()
-                DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment"
+            )
-            ).count()
+            total_segments = (
+                db.session.query(DocumentSegment)
+                .filter(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
+                .count()
+            )
             document.completed_segments = completed_segments
             document.total_segments = total_segments
             documents_status.append(marshal(document, document_status_fields))

api/controllers/console/datasets/datasets_document.py

@@ -4,9 +4,9 @@ from datetime import UTC, datetime
 from typing import cast
 
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, fields, marshal, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, fields, marshal, marshal_with, reqparse
-from sqlalchemy import asc, desc
+from sqlalchemy import asc, desc, select
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
@@ -112,7 +112,7 @@ class GetProcessRuleApi(Resource):
         limits = DocumentService.DEFAULT_RULES["limits"]
         if document_id:
             # get the latest process rule
-            document = Document.query.get_or_404(document_id)
+            document = db.get_or_404(Document, document_id)
 
             dataset = DatasetService.get_dataset(document.dataset_id)
 
@@ -175,7 +175,7 @@ class DatasetDocumentListApi(Resource):
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
-        query = Document.query.filter_by(dataset_id=str(dataset_id), tenant_id=current_user.current_tenant_id)
+        query = select(Document).filter_by(dataset_id=str(dataset_id), tenant_id=current_user.current_tenant_id)
 
         if search:
             search = f"%{search}%"
@@ -209,18 +209,24 @@ class DatasetDocumentListApi(Resource):
                 desc(Document.position),
             )
 
-        paginated_documents = query.paginate(page=page, per_page=limit, max_per_page=100, error_out=False)
+        paginated_documents = db.paginate(select=query, page=page, per_page=limit, max_per_page=100, error_out=False)
         documents = paginated_documents.items
         if fetch:
             for document in documents:
-                completed_segments = DocumentSegment.query.filter(
+                completed_segments = (
+                    db.session.query(DocumentSegment)
+                    .filter(
                         DocumentSegment.completed_at.isnot(None),
                         DocumentSegment.document_id == str(document.id),
                         DocumentSegment.status != "re_segment",
-                ).count()
+                    )
-                total_segments = DocumentSegment.query.filter(
+                    .count()
-                    DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment"
+                )
-                ).count()
+                total_segments = (
+                    db.session.query(DocumentSegment)
+                    .filter(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
+                    .count()
+                )
                 document.completed_segments = completed_segments
                 document.total_segments = total_segments
             data = marshal(documents, document_with_segments_fields)
@@ -563,14 +569,20 @@ class DocumentBatchIndexingStatusApi(DocumentResource):
         documents = self.get_batch_documents(dataset_id, batch)
         documents_status = []
         for document in documents:
-            completed_segments = DocumentSegment.query.filter(
+            completed_segments = (
+                db.session.query(DocumentSegment)
+                .filter(
                     DocumentSegment.completed_at.isnot(None),
                     DocumentSegment.document_id == str(document.id),
                     DocumentSegment.status != "re_segment",
-            ).count()
+                )
-            total_segments = DocumentSegment.query.filter(
+                .count()
-                DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment"
+            )
-            ).count()
+            total_segments = (
+                db.session.query(DocumentSegment)
+                .filter(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
+                .count()
+            )
             document.completed_segments = completed_segments
             document.total_segments = total_segments
             if document.is_paused:
@@ -589,14 +601,20 @@ class DocumentIndexingStatusApi(DocumentResource):
         document_id = str(document_id)
         document = self.get_document(dataset_id, document_id)
 
-        completed_segments = DocumentSegment.query.filter(
+        completed_segments = (
+            db.session.query(DocumentSegment)
+            .filter(
                 DocumentSegment.completed_at.isnot(None),
                 DocumentSegment.document_id == str(document_id),
                 DocumentSegment.status != "re_segment",
-        ).count()
+            )
-        total_segments = DocumentSegment.query.filter(
+            .count()
-            DocumentSegment.document_id == str(document_id), DocumentSegment.status != "re_segment"
+        )
-        ).count()
+        total_segments = (
+            db.session.query(DocumentSegment)
+            .filter(DocumentSegment.document_id == str(document_id), DocumentSegment.status != "re_segment")
+            .count()
+        )
 
         document.completed_segments = completed_segments
         document.total_segments = total_segments

api/controllers/console/datasets/datasets_segments.py

@@ -2,8 +2,9 @@ import uuid
 
 import pandas as pd
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse  # type: ignore
+from flask_restful import Resource, marshal, reqparse
+from sqlalchemy import select
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
@@ -26,6 +27,7 @@ from controllers.console.wraps import (
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
+from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from fields.segment_fields import child_chunk_fields, segment_fields
 from libs.login import login_required
@@ -74,9 +76,14 @@ class DatasetDocumentSegmentListApi(Resource):
         hit_count_gte = args["hit_count_gte"]
         keyword = args["keyword"]
 
-        query = DocumentSegment.query.filter(
+        query = (
-            DocumentSegment.document_id == str(document_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+            select(DocumentSegment)
-        ).order_by(DocumentSegment.position.asc())
+            .filter(
+                DocumentSegment.document_id == str(document_id),
+                DocumentSegment.tenant_id == current_user.current_tenant_id,
+            )
+            .order_by(DocumentSegment.position.asc())
+        )
 
         if status_list:
             query = query.filter(DocumentSegment.status.in_(status_list))
@@ -93,7 +100,7 @@ class DatasetDocumentSegmentListApi(Resource):
             elif args["enabled"].lower() == "false":
                 query = query.filter(DocumentSegment.enabled == False)
 
-        segments = query.paginate(page=page, per_page=limit, max_per_page=100, error_out=False)
+        segments = db.paginate(select=query, page=page, per_page=limit, max_per_page=100, error_out=False)
 
         response = {
             "data": marshal(segments.items, segment_fields),
@@ -276,9 +283,11 @@ class DatasetDocumentSegmentUpdateApi(Resource):
                 raise ProviderNotInitializeError(ex.description)
             # check segment
         segment_id = str(segment_id)
-        segment = DocumentSegment.query.filter(
+        segment = (
-            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+            db.session.query(DocumentSegment)
-        ).first()
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .first()
+        )
         if not segment:
             raise NotFound("Segment not found.")
         # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
@@ -320,9 +329,11 @@ class DatasetDocumentSegmentUpdateApi(Resource):
             raise NotFound("Document not found.")
         # check segment
         segment_id = str(segment_id)
-        segment = DocumentSegment.query.filter(
+        segment = (
-            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+            db.session.query(DocumentSegment)
-        ).first()
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .first()
+        )
         if not segment:
             raise NotFound("Segment not found.")
         # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
@@ -423,9 +434,11 @@ class ChildChunkAddApi(Resource):
             raise NotFound("Document not found.")
         # check segment
         segment_id = str(segment_id)
-        segment = DocumentSegment.query.filter(
+        segment = (
-            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+            db.session.query(DocumentSegment)
-        ).first()
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .first()
+        )
         if not segment:
             raise NotFound("Segment not found.")
         if not current_user.is_dataset_editor:
@@ -478,9 +491,11 @@ class ChildChunkAddApi(Resource):
             raise NotFound("Document not found.")
         # check segment
         segment_id = str(segment_id)
-        segment = DocumentSegment.query.filter(
+        segment = (
-            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+            db.session.query(DocumentSegment)
-        ).first()
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .first()
+        )
         if not segment:
             raise NotFound("Segment not found.")
         parser = reqparse.RequestParser()
@@ -523,9 +538,11 @@ class ChildChunkAddApi(Resource):
             raise NotFound("Document not found.")
             # check segment
         segment_id = str(segment_id)
-        segment = DocumentSegment.query.filter(
+        segment = (
-            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+            db.session.query(DocumentSegment)
-        ).first()
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .first()
+        )
         if not segment:
             raise NotFound("Segment not found.")
         # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
@@ -567,16 +584,20 @@ class ChildChunkUpdateApi(Resource):
             raise NotFound("Document not found.")
         # check segment
         segment_id = str(segment_id)
-        segment = DocumentSegment.query.filter(
+        segment = (
-            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+            db.session.query(DocumentSegment)
-        ).first()
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .first()
+        )
         if not segment:
             raise NotFound("Segment not found.")
         # check child chunk
         child_chunk_id = str(child_chunk_id)
-        child_chunk = ChildChunk.query.filter(
+        child_chunk = (
-            ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id
+            db.session.query(ChildChunk)
-        ).first()
+            .filter(ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id)
+            .first()
+        )
         if not child_chunk:
             raise NotFound("Child chunk not found.")
         # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
@@ -612,16 +633,20 @@ class ChildChunkUpdateApi(Resource):
             raise NotFound("Document not found.")
             # check segment
         segment_id = str(segment_id)
-        segment = DocumentSegment.query.filter(
+        segment = (
-            DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id
+            db.session.query(DocumentSegment)
-        ).first()
+            .filter(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_user.current_tenant_id)
+            .first()
+        )
         if not segment:
             raise NotFound("Segment not found.")
         # check child chunk
         child_chunk_id = str(child_chunk_id)
-        child_chunk = ChildChunk.query.filter(
+        child_chunk = (
-            ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id
+            db.session.query(ChildChunk)
-        ).first()
+            .filter(ChildChunk.id == str(child_chunk_id), ChildChunk.tenant_id == current_user.current_tenant_id)
+            .first()
+        )
         if not child_chunk:
             raise NotFound("Child chunk not found.")
         # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor

api/controllers/console/datasets/external.py

@@ -1,6 +1,6 @@
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse  # type: ignore
+from flask_restful import Resource, marshal, reqparse
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
@@ -209,6 +209,7 @@ class ExternalKnowledgeHitTestingApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument("query", type=str, location="json")
         parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
+        parser.add_argument("metadata_filtering_conditions", type=dict, required=False, location="json")
         args = parser.parse_args()
 
         HitTestingService.hit_testing_args_check(args)
@@ -219,6 +220,7 @@ class ExternalKnowledgeHitTestingApi(Resource):
                 query=args["query"],
                 account=current_user,
                 external_retrieval_model=args["external_retrieval_model"],
+                metadata_filtering_conditions=args["metadata_filtering_conditions"],
             )
 
             return response

api/controllers/console/datasets/hit_testing.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource  # type: ignore
+from flask_restful import Resource
 
 from controllers.console import api
 from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase

api/controllers/console/datasets/hit_testing_base.py

@@ -1,7 +1,7 @@
 import logging
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import marshal, reqparse  # type: ignore
+from flask_restful import marshal, reqparse
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services.dataset_service

api/controllers/console/datasets/metadata.py

@@ -1,5 +1,5 @@
-from flask_login import current_user  # type: ignore  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import NotFound
 
 from controllers.console import api

api/controllers/console/datasets/website.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.datasets.error import WebsiteCrawlError

api/controllers/console/explore/audio.py

@@ -66,7 +66,7 @@ class ChatAudioApi(InstalledAppResource):
 
 class ChatTextApi(InstalledAppResource):
     def post(self, installed_app):
-        from flask_restful import reqparse  # type: ignore
+        from flask_restful import reqparse
 
         app_model = installed_app.app
         try:

api/controllers/console/explore/completion.py

@@ -1,8 +1,8 @@
 import logging
 from datetime import UTC, datetime
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import reqparse  # type: ignore
+from flask_restful import reqparse
 from werkzeug.exceptions import InternalServerError, NotFound
 
 import services

api/controllers/console/explore/conversation.py

@@ -1,6 +1,6 @@
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import marshal_with, reqparse  # type: ignore
+from flask_restful import marshal_with, reqparse
-from flask_restful.inputs import int_range  # type: ignore
+from flask_restful.inputs import int_range
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound
 

api/controllers/console/explore/installed_app.py

@@ -2,8 +2,8 @@ from datetime import UTC, datetime
 from typing import Any
 
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, inputs, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, inputs, marshal_with, reqparse
 from sqlalchemy import and_
 from werkzeug.exceptions import BadRequest, Forbidden, NotFound
 
@@ -66,7 +66,7 @@ class InstalledAppsListApi(Resource):
         parser.add_argument("app_id", type=str, required=True, help="Invalid app_id")
         args = parser.parse_args()
 
-        recommended_app = RecommendedApp.query.filter(RecommendedApp.app_id == args["app_id"]).first()
+        recommended_app = db.session.query(RecommendedApp).filter(RecommendedApp.app_id == args["app_id"]).first()
         if recommended_app is None:
             raise NotFound("App not found")
 
@@ -79,9 +79,11 @@ class InstalledAppsListApi(Resource):
         if not app.is_public:
             raise Forbidden("You can't install a non-public app")
 
-        installed_app = InstalledApp.query.filter(
+        installed_app = (
-            and_(InstalledApp.app_id == args["app_id"], InstalledApp.tenant_id == current_tenant_id)
+            db.session.query(InstalledApp)
-        ).first()
+            .filter(and_(InstalledApp.app_id == args["app_id"], InstalledApp.tenant_id == current_tenant_id))
+            .first()
+        )
 
         if installed_app is None:
             # todo: position

api/controllers/console/explore/message.py

@@ -1,8 +1,8 @@
 import logging
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import marshal_with, reqparse  # type: ignore
+from flask_restful import marshal_with, reqparse
-from flask_restful.inputs import int_range  # type: ignore
+from flask_restful.inputs import int_range
 from werkzeug.exceptions import InternalServerError, NotFound
 
 import services

api/controllers/console/explore/parameter.py

@@ -1,4 +1,4 @@
-from flask_restful import marshal_with  # type: ignore
+from flask_restful import marshal_with
 
 from controllers.common import fields
 from controllers.console import api

api/controllers/console/explore/recommended_app.py

@@ -1,5 +1,5 @@
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, fields, marshal_with, reqparse
 
 from constants.languages import languages
 from controllers.console import api

api/controllers/console/explore/saved_message.py

@@ -1,6 +1,6 @@
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse  # type: ignore
+from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range  # type: ignore
+from flask_restful.inputs import int_range
 from werkzeug.exceptions import NotFound
 
 from controllers.console import api

api/controllers/console/explore/workflow.py

@@ -1,6 +1,6 @@
 import logging
 
-from flask_restful import reqparse  # type: ignore
+from flask_restful import reqparse
 from werkzeug.exceptions import InternalServerError
 
 from controllers.console.app.error import (

api/controllers/console/explore/wraps.py

@@ -1,7 +1,7 @@
 from functools import wraps
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource  # type: ignore
+from flask_restful import Resource
 from werkzeug.exceptions import NotFound
 
 from controllers.console.wraps import account_initialization_required

api/controllers/console/extension.py

@@ -1,5 +1,5 @@
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
 
 from constants import HIDDEN_VALUE
 from controllers.console import api

api/controllers/console/feature.py

@@ -1,5 +1,5 @@
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource  # type: ignore
+from flask_restful import Resource
 
 from libs.login import login_required
 from services.feature_service import FeatureService

api/controllers/console/files.py

@@ -1,8 +1,8 @@
 from typing import Literal
 
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal_with  # type: ignore
+from flask_restful import Resource, marshal_with
 from werkzeug.exceptions import Forbidden
 
 import services

api/controllers/console/init_validate.py

@@ -1,7 +1,7 @@
 import os
 
 from flask import session
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 

api/controllers/console/ping.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource  # type: ignore
+from flask_restful import Resource
 
 from controllers.console import api
 

api/controllers/console/remote_files.py

@@ -2,8 +2,8 @@ import urllib.parse
 from typing import cast
 
 import httpx
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
 
 import services
 from controllers.common import helpers

api/controllers/console/setup.py

@@ -1,5 +1,5 @@
 from flask import request
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from configs import dify_config
 from libs.helper import StrLen, email, extract_remote_ip

api/controllers/console/tag/tags.py

@@ -1,6 +1,6 @@
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api

api/controllers/console/version.py

@@ -2,7 +2,7 @@ import json
 import logging
 
 import requests
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from packaging import version
 
 from configs import dify_config

api/controllers/console/workspace/__init__.py

@@ -1,6 +1,6 @@
 from functools import wraps
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
 

api/controllers/console/workspace/account.py

@@ -2,8 +2,8 @@ import datetime
 
 import pytz
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, fields, marshal_with, reqparse
 
 from configs import dify_config
 from constants.languages import supported_language

api/controllers/console/workspace/agent_providers.py

@@ -1,5 +1,5 @@
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource  # type: ignore
+from flask_restful import Resource
 
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required, setup_required

api/controllers/console/workspace/endpoint.py

@@ -1,5 +1,5 @@
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api

api/controllers/console/workspace/load_balancing_config.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api

api/controllers/console/workspace/members.py

@@ -1,7 +1,7 @@
 from urllib import parse
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, abort, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, abort, marshal_with, reqparse
 
 import services
 from configs import dify_config
@@ -71,7 +71,6 @@ class MemberInviteEmailApi(Resource):
                 invitation_results.append(
                     {"status": "success", "email": invitee_email, "url": f"{console_web_url}/signin"}
                 )
-                break
             except Exception as e:
                 invitation_results.append({"status": "failed", "email": invitee_email, "message": str(e)})
 

api/controllers/console/workspace/model_providers.py

@@ -1,8 +1,8 @@
 import io
 
 from flask import send_file
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api

api/controllers/console/workspace/models.py

@@ -1,7 +1,7 @@
 import logging
 
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api

api/controllers/console/workspace/plugin.py

@@ -1,8 +1,8 @@
 import io
 
 from flask import request, send_file
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from configs import dify_config

api/controllers/console/workspace/tool_providers.py

@@ -1,8 +1,8 @@
 import io
 
 from flask import send_file
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
 

api/controllers/console/workspace/workspace.py

@@ -1,8 +1,9 @@
 import logging
 
 from flask import request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
-from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse
+from sqlalchemy import select
 from werkzeug.exceptions import Unauthorized
 
 import services
@@ -88,9 +89,8 @@ class WorkspaceListApi(Resource):
         parser.add_argument("limit", type=inputs.int_range(1, 100), required=False, default=20, location="args")
         args = parser.parse_args()
 
-        tenants = Tenant.query.order_by(Tenant.created_at.desc()).paginate(
+        stmt = select(Tenant).order_by(Tenant.created_at.desc())
-            page=args["page"], per_page=args["limit"], error_out=False
+        tenants = db.paginate(select=stmt, page=args["page"], per_page=args["limit"], error_out=False)
-        )
         has_more = False
 
         if tenants.has_next:
@@ -162,7 +162,7 @@ class CustomConfigWorkspaceApi(Resource):
         parser.add_argument("replace_webapp_logo", type=str, location="json")
         args = parser.parse_args()
 
-        tenant = Tenant.query.filter(Tenant.id == current_user.current_tenant_id).one_or_404()
+        tenant = db.get_or_404(Tenant, current_user.current_tenant_id)
 
         custom_config_dict = {
             "remove_webapp_brand": args["remove_webapp_brand"],
@@ -226,7 +226,7 @@ class WorkspaceInfoApi(Resource):
         parser.add_argument("name", type=str, required=True, location="json")
         args = parser.parse_args()
 
-        tenant = Tenant.query.filter(Tenant.id == current_user.current_tenant_id).one_or_404()
+        tenant = db.get_or_404(Tenant, current_user.current_tenant_id)
         tenant.name = args["name"]
         db.session.commit()
 

api/controllers/console/wraps.py

@@ -4,7 +4,7 @@ import time
 from functools import wraps
 
 from flask import abort, request
-from flask_login import current_user  # type: ignore
+from flask_login import current_user
 
 from configs import dify_config
 from controllers.console.workspace.error import AccountNotInitializedError

api/controllers/files/image_preview.py

@@ -1,7 +1,7 @@
 from urllib.parse import quote
 
 from flask import Response, request
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import NotFound
 
 import services
@@ -70,6 +70,20 @@ class FilePreviewApi(Resource):
             direct_passthrough=True,
             headers={},
         )
+        # add Accept-Ranges header for audio/video files
+        if upload_file.mime_type in [
+            "audio/mpeg",
+            "audio/wav",
+            "audio/mp4",
+            "audio/ogg",
+            "audio/flac",
+            "audio/aac",
+            "video/mp4",
+            "video/webm",
+            "video/quicktime",
+            "audio/x-m4a",
+        ]:
+            response.headers["Accept-Ranges"] = "bytes"
         if upload_file.size > 0:
             response.headers["Content-Length"] = str(upload_file.size)
         if args["as_attachment"]:

api/controllers/files/tool_files.py

@@ -1,10 +1,14 @@
+from urllib.parse import quote
+
 from flask import Response
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden, NotFound
 
 from controllers.files import api
 from controllers.files.error import UnsupportedFileTypeError
+from core.tools.signature import verify_tool_file_signature
 from core.tools.tool_file_manager import ToolFileManager
+from models import db as global_db
 
 
 class ToolFilePreviewApi(Resource):
@@ -19,17 +23,14 @@ class ToolFilePreviewApi(Resource):
         parser.add_argument("as_attachment", type=bool, required=False, default=False, location="args")
 
         args = parser.parse_args()
-
+        if not verify_tool_file_signature(
-        if not ToolFileManager.verify_file(
+            file_id=file_id, timestamp=args["timestamp"], nonce=args["nonce"], sign=args["sign"]
-            file_id=file_id,
-            timestamp=args["timestamp"],
-            nonce=args["nonce"],
-            sign=args["sign"],
         ):
             raise Forbidden("Invalid request.")
 
         try:
-            stream, tool_file = ToolFileManager.get_file_generator_by_tool_file_id(
+            tool_file_manager = ToolFileManager(engine=global_db.engine)
+            stream, tool_file = tool_file_manager.get_file_generator_by_tool_file_id(
                 file_id,
             )
 
@@ -47,7 +48,8 @@ class ToolFilePreviewApi(Resource):
         if tool_file.size > 0:
             response.headers["Content-Length"] = str(tool_file.size)
         if args["as_attachment"]:
-            response.headers["Content-Disposition"] = f"attachment; filename={tool_file.name}"
+            encoded_filename = quote(tool_file.name)
+            response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}"
 
         return response
 

api/controllers/files/upload.py

@@ -1,7 +1,7 @@
 from mimetypes import guess_extension
 
 from flask import request
-from flask_restful import Resource, marshal_with  # type: ignore
+from flask_restful import Resource, marshal_with
 from werkzeug.exceptions import Forbidden
 
 import services
@@ -53,7 +53,7 @@ class PluginUploadFileApi(Resource):
             raise Forbidden("Invalid request.")
 
         try:
-            tool_file = ToolFileManager.create_file_by_raw(
+            tool_file = ToolFileManager().create_file_by_raw(
                 user_id=user.id,
                 tenant_id=tenant_id,
                 file_binary=file.read(),

api/controllers/inner_api/plugin/plugin.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource  # type: ignore
+from flask_restful import Resource
 
 from controllers.console.wraps import setup_required
 from controllers.inner_api import api

api/controllers/inner_api/plugin/wraps.py

@@ -3,7 +3,7 @@ from functools import wraps
 from typing import Optional
 
 from flask import request
-from flask_restful import reqparse  # type: ignore
+from flask_restful import reqparse
 from pydantic import BaseModel
 from sqlalchemy.orm import Session
 

api/controllers/inner_api/workspace/workspace.py

@@ -1,6 +1,6 @@
 import json
 
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 
 from controllers.console.wraps import setup_required
 from controllers.inner_api import api

api/controllers/inner_api/wraps.py

@@ -18,7 +18,7 @@ def enterprise_inner_api_only(view):
 
         # get header 'X-Inner-Api-Key'
         inner_api_key = request.headers.get("X-Inner-Api-Key")
-        if not inner_api_key or inner_api_key != dify_config.INNER_API_KEY_FOR_PLUGIN:
+        if not inner_api_key or inner_api_key != dify_config.INNER_API_KEY:
             abort(401)
 
         return view(*args, **kwargs)

api/controllers/service_api/__init__.py

@@ -6,6 +6,6 @@ bp = Blueprint("service_api", __name__, url_prefix="/v1")
 api = ExternalApi(bp)
 
 from . import index
-from .app import annotation, app, audio, completion, conversation, file, message, workflow
+from .app import annotation, app, audio, completion, conversation, file, message, site, workflow
 from .dataset import dataset, document, hit_testing, metadata, segment, upload_file
 from .workspace import models

api/controllers/service_api/app/annotation.py

@@ -1,5 +1,5 @@
 from flask import request
-from flask_restful import Resource, marshal, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.service_api import api
@@ -79,7 +79,7 @@ class AnnotationListApi(Resource):
 class AnnotationUpdateDeleteApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON))
     @marshal_with(annotation_fields)
-    def post(self, app_model: App, end_user: EndUser, annotation_id):
+    def put(self, app_model: App, end_user: EndUser, annotation_id):
         if not current_user.is_editor:
             raise Forbidden()
 

api/controllers/service_api/app/app.py

@@ -1,4 +1,4 @@
-from flask_restful import Resource, marshal_with  # type: ignore
+from flask_restful import Resource, marshal_with
 
 from controllers.common import fields
 from controllers.service_api import api
@@ -47,7 +47,7 @@ class AppInfoApi(Resource):
     def get(self, app_model: App):
         """Get app information"""
         tags = [tag.name for tag in app_model.tags]
-        return {"name": app_model.name, "description": app_model.description, "tags": tags}
+        return {"name": app_model.name, "description": app_model.description, "tags": tags, "mode": app_model.mode}
 
 
 api.add_resource(AppParameterApi, "/parameters")

api/controllers/service_api/app/audio.py

@@ -1,7 +1,7 @@
 import logging
 
 from flask import request
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import InternalServerError
 
 import services

api/controllers/service_api/app/completion.py

@@ -1,6 +1,6 @@
 import logging
 
-from flask_restful import Resource, reqparse  # type: ignore
+from flask_restful import Resource, reqparse
 from werkzeug.exceptions import InternalServerError, NotFound
 
 import services

api/controllers/service_api/app/conversation.py

@@ -1,5 +1,5 @@
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range  # type: ignore
+from flask_restful.inputs import int_range
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound