Merge branch 'main' into feat/structured-output

.devcontainer/post_create_command.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-npm add -g pnpm@9.12.2
+npm add -g pnpm@10.8.0
 cd web && pnpm install
 pipx install poetry
 

.github/workflows/api-tests.yml

@@ -53,9 +53,14 @@ jobs:
       - name: Run dify config tests
         run: poetry run -P api python dev/pytest/pytest_config_tests.py
 
+      - name: Cache MyPy
+        uses: actions/cache@v4
+        with:
+          path: api/.mypy_cache
+          key: mypy-${{ matrix.python-version }}-${{ runner.os }}-${{ hashFiles('api/poetry.lock') }}
+
       - name: Run mypy
-        run: |
+        run: dev/run-mypy
-          poetry run -C api python -m mypy --install-types --non-interactive .
 
       - name: Set up dotenvs
         run: |

.github/workflows/build-push.yml

@@ -6,7 +6,6 @@ on:
       - "main"
       - "deploy/dev"
       - "deploy/enterprise"
-      - release/1.1.3-fix1
     tags:
       - "*"
 

.github/workflows/style.yml

@@ -82,7 +82,7 @@ jobs:
         uses: actions/setup-node@v4
         if: steps.changed-files.outputs.any_changed == 'true'
         with:
-          node-version: 20
+          node-version: 22
           cache: pnpm
           cache-dependency-path: ./web/package.json
 
@@ -153,6 +153,7 @@ jobs:
         env:
           BASH_SEVERITY: warning
           DEFAULT_BRANCH: main
+          FILTER_REGEX_INCLUDE: pnpm-lock.yaml
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           IGNORE_GENERATED_FILES: true
           IGNORE_GITIGNORED_FILES: true

.github/workflows/tool-test-sdks.yaml

@@ -18,7 +18,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16, 18, 20]
+        node-version: [16, 18, 20, 22]
 
     defaults:
       run:

.github/workflows/translate-i18n-base-on-english.yml

@@ -33,7 +33,7 @@ jobs:
 
       - name: Set up Node.js
         if: env.FILES_CHANGED == 'true'
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
           node-version: 'lts/*'
 

.github/workflows/web-tests.yml

@@ -31,7 +31,9 @@ jobs:
         uses: tj-actions/changed-files@v45
         with:
           files: web/**
+
       - name: Install pnpm
+        if: steps.changed-files.outputs.any_changed == 'true'
         uses: pnpm/action-setup@v4
         with:
           version: 10
@@ -41,7 +43,7 @@ jobs:
         uses: actions/setup-node@v4
         if: steps.changed-files.outputs.any_changed == 'true'
         with:
-          node-version: 20
+          node-version: 22
           cache: pnpm
           cache-dependency-path: ./web/package.json
 

api/.env.example

@@ -327,6 +327,7 @@ MULTIMODAL_SEND_FORMAT=base64
 PROMPT_GENERATION_MAX_TOKENS=512
 CODE_GENERATION_MAX_TOKENS=1024
 STRUCTURED_OUTPUT_MAX_TOKENS=1024
+PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false
 
 # Mail configuration, support: resend, smtp
 MAIL_TYPE=

api/configs/feature/__init__.py

@@ -442,7 +442,7 @@ class LoggingConfig(BaseSettings):
 
 class ModelLoadBalanceConfig(BaseSettings):
     """
-    Configuration for model load balancing
+    Configuration for model load balancing and token counting
     """
 
     MODEL_LB_ENABLED: bool = Field(
@@ -450,6 +450,11 @@ class ModelLoadBalanceConfig(BaseSettings):
         default=False,
     )
 
+    PLUGIN_BASED_TOKEN_COUNTING_ENABLED: bool = Field(
+        description="Enable or disable plugin based token counting. If disabled, token counting will return 0.",
+        default=False,
+    )
+
 
 class BillingConfig(BaseSettings):
     """

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="1.1.3",
+        default="1.2.0",
     )
 
     COMMIT_SHA: str = Field(

api/controllers/console/workspace/plugin.py

@@ -49,6 +49,23 @@ class PluginListApi(Resource):
         return jsonable_encoder({"plugins": plugins})
 
 
+class PluginListLatestVersionsApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        req = reqparse.RequestParser()
+        req.add_argument("plugin_ids", type=list, required=True, location="json")
+        args = req.parse_args()
+
+        try:
+            versions = PluginService.list_latest_versions(args["plugin_ids"])
+        except PluginDaemonClientSideError as e:
+            raise ValueError(e)
+
+        return jsonable_encoder({"versions": versions})
+
+
 class PluginListInstallationsFromIdsApi(Resource):
     @setup_required
     @login_required
@@ -453,6 +470,7 @@ class PluginFetchPermissionApi(Resource):
 
 api.add_resource(PluginDebuggingKeyApi, "/workspaces/current/plugin/debugging-key")
 api.add_resource(PluginListApi, "/workspaces/current/plugin/list")
+api.add_resource(PluginListLatestVersionsApi, "/workspaces/current/plugin/list/latest-versions")
 api.add_resource(PluginListInstallationsFromIdsApi, "/workspaces/current/plugin/list/installations/ids")
 api.add_resource(PluginIconApi, "/workspaces/current/plugin/icon")
 api.add_resource(PluginUploadFromPkgApi, "/workspaces/current/plugin/upload/pkg")

api/controllers/service_api/app/message.py

@@ -20,14 +20,6 @@ from services.message_service import MessageService
 
 
 class MessageListApi(Resource):
-    def get_retriever_resources(self):
-        try:
-            if self.message_metadata:
-                return json.loads(self.message_metadata).get("retriever_resources", [])
-            return []
-        except (json.JSONDecodeError, TypeError):
-            return []
-
     message_fields = {
         "id": fields.String,
         "conversation_id": fields.String,
@@ -37,7 +29,11 @@ class MessageListApi(Resource):
         "answer": fields.String(attribute="re_sign_file_url_answer"),
         "message_files": fields.List(fields.Nested(message_file_fields)),
         "feedback": fields.Nested(feedback_fields, attribute="user_feedback", allow_null=True),
-        "retriever_resources": get_retriever_resources,
+        "retriever_resources": fields.Raw(
+            attribute=lambda obj: json.loads(obj.message_metadata).get("retriever_resources", [])
+            if obj.message_metadata
+            else []
+        ),
         "created_at": TimestampField,
         "agent_thoughts": fields.List(fields.Nested(agent_thought_fields)),
         "status": fields.String,

api/controllers/service_api/wraps.py

@@ -69,7 +69,7 @@ def validate_app_token(view: Optional[Callable] = None, *, fetch_user_arg: Optio
             )  # TODO: only owner information is required, so only one is returned.
             if tenant_account_join:
                 tenant, ta = tenant_account_join
-                account = Account.query.filter_by(id=ta.account_id).first()
+                account = db.session.query(Account).filter(Account.id == ta.account_id).first()
                 # Login admin
                 if account:
                     account.current_tenant = tenant

api/core/app/apps/agent_chat/app_runner.py

@@ -53,20 +53,6 @@ class AgentChatAppRunner(AppRunner):
         query = application_generate_entity.query
         files = application_generate_entity.files
 
-        # Pre-calculate the number of tokens of the prompt messages,
-        # and return the rest number of tokens by model context token size limit and max token size limit.
-        # If the rest number of tokens is not enough, raise exception.
-        # Include: prompt template, inputs, query(optional), files(optional)
-        # Not Include: memory, external data, dataset context
-        self.get_pre_calculate_rest_tokens(
-            app_record=app_record,
-            model_config=application_generate_entity.model_conf,
-            prompt_template_entity=app_config.prompt_template,
-            inputs=dict(inputs),
-            files=list(files),
-            query=query,
-        )
-
         memory = None
         if application_generate_entity.conversation_id:
             # get memory of conversation (read-only)

api/core/app/apps/chat/app_runner.py

@@ -61,20 +61,6 @@ class ChatAppRunner(AppRunner):
         )
         image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
 
-        # Pre-calculate the number of tokens of the prompt messages,
-        # and return the rest number of tokens by model context token size limit and max token size limit.
-        # If the rest number of tokens is not enough, raise exception.
-        # Include: prompt template, inputs, query(optional), files(optional)
-        # Not Include: memory, external data, dataset context
-        self.get_pre_calculate_rest_tokens(
-            app_record=app_record,
-            model_config=application_generate_entity.model_conf,
-            prompt_template_entity=app_config.prompt_template,
-            inputs=inputs,
-            files=files,
-            query=query,
-        )
-
         memory = None
         if application_generate_entity.conversation_id:
             # get memory of conversation (read-only)

api/core/app/apps/completion/app_runner.py

@@ -54,20 +54,6 @@ class CompletionAppRunner(AppRunner):
         )
         image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
 
-        # Pre-calculate the number of tokens of the prompt messages,
-        # and return the rest number of tokens by model context token size limit and max token size limit.
-        # If the rest number of tokens is not enough, raise exception.
-        # Include: prompt template, inputs, query(optional), files(optional)
-        # Not Include: memory, external data, dataset context
-        self.get_pre_calculate_rest_tokens(
-            app_record=app_record,
-            model_config=application_generate_entity.model_conf,
-            prompt_template_entity=app_config.prompt_template,
-            inputs=inputs,
-            files=files,
-            query=query,
-        )
-
         # organize all inputs and template to prompt messages
         # Include: prompt template, inputs, query(optional), files(optional)
         prompt_messages, stop = self.organize_prompt_messages(

api/core/helper/ssrf_proxy.py

@@ -56,8 +56,12 @@ def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs):
                     response = client.request(method=method, url=url, **kwargs)
             elif dify_config.SSRF_PROXY_HTTP_URL and dify_config.SSRF_PROXY_HTTPS_URL:
                 proxy_mounts = {
-                    "http://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTP_URL),
+                    "http://": httpx.HTTPTransport(
-                    "https://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTPS_URL),
+                        proxy=dify_config.SSRF_PROXY_HTTP_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY
+                    ),
+                    "https://": httpx.HTTPTransport(
+                        proxy=dify_config.SSRF_PROXY_HTTPS_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY
+                    ),
                 }
                 with httpx.Client(mounts=proxy_mounts, verify=HTTP_REQUEST_NODE_SSL_VERIFY) as client:
                     response = client.request(method=method, url=url, **kwargs)

api/core/model_runtime/docs/en_US/customizable_model_scale_out.md

@@ -192,7 +192,7 @@ def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[Pr
 ```
 
 
-Sometimes, you might not want to return 0 directly. In such cases, you can use `self._get_num_tokens_by_gpt2(text: str)` to get pre-computed tokens. This method is provided by the `AIModel` base class, and it uses GPT2's Tokenizer for calculation. However, it should be noted that this is only a substitute and may not be fully accurate.
+Sometimes, you might not want to return 0 directly. In such cases, you can use `self._get_num_tokens_by_gpt2(text: str)` to get pre-computed tokens and ensure environment variable `PLUGIN_BASED_TOKEN_COUNTING_ENABLED` is set to `true`, This method is provided by the `AIModel` base class, and it uses GPT2's Tokenizer for calculation. However, it should be noted that this is only a substitute and may not be fully accurate.
 
 - Model Credentials Validation
 

api/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md

@@ -179,7 +179,7 @@ provider_credential_schema:
     """
   ```
 
-  有时候，也许你不需要直接返回0，所以你可以使用`self._get_num_tokens_by_gpt2(text: str)`来获取预计算的tokens，这个方法位于`AIModel`基类中，它会使用GPT2的Tokenizer进行计算，但是只能作为替代方法，并不完全准确。
+  有时候，也许你不需要直接返回0，所以你可以使用`self._get_num_tokens_by_gpt2(text: str)`来获取预计算的tokens，并确保环境变量`PLUGIN_BASED_TOKEN_COUNTING_ENABLED`设置为`true`，这个方法位于`AIModel`基类中，它会使用GPT2的Tokenizer进行计算，但是只能作为替代方法，并不完全准确。
 
 - 模型凭据校验
 

api/core/model_runtime/model_providers/__base/large_language_model.py

@@ -295,6 +295,7 @@ class LargeLanguageModel(AIModel):
         :param tools: tools for tool calling
         :return:
         """
+        if dify_config.PLUGIN_BASED_TOKEN_COUNTING_ENABLED:
             plugin_model_manager = PluginModelManager()
             return plugin_model_manager.get_llm_num_tokens(
                 tenant_id=self.tenant_id,
@@ -307,6 +308,7 @@ class LargeLanguageModel(AIModel):
                 prompt_messages=prompt_messages,
                 tools=tools,
             )
+        return 0
 
     def _calc_response_usage(
         self, model: str, credentials: dict, prompt_tokens: int, completion_tokens: int

api/core/plugin/entities/plugin.py

@@ -120,8 +120,6 @@ class PluginEntity(PluginInstallation):
     name: str
     installation_id: str
     version: str
-    latest_version: Optional[str] = None
-    latest_unique_identifier: Optional[str] = None
 
     @model_validator(mode="after")
     def set_plugin_id(self):

api/core/rag/datasource/retrieval_service.py

@@ -46,7 +46,7 @@ class RetrievalService:
         if not query:
             return []
         dataset = cls._get_dataset(dataset_id)
-        if not dataset or dataset.available_document_count == 0 or dataset.available_segment_count == 0:
+        if not dataset:
             return []
 
         all_documents: list[Document] = []

api/core/rag/datasource/vdb/tencent/tencent_vector.py

@@ -51,6 +51,7 @@ class TencentVector(BaseVector):
         self._client = RPCVectorDBClient(**self._client_config.to_tencent_params())
         self._enable_hybrid_search = False
         self._dimension = 1024
+        self._init_database()
         self._load_collection()
         self._bm25 = BM25Encoder.default("zh")
 
@@ -279,7 +280,10 @@ class TencentVector(BaseVector):
         return docs
 
     def delete(self) -> None:
-        self._client.drop_collection(database_name=self._client_config.database, collection_name=self.collection_name)
+        if self._has_collection():
+            self._client.drop_collection(
+                database_name=self._client_config.database, collection_name=self.collection_name
+            )
 
 
 class TencentVectorFactory(AbstractVectorFactory):

api/core/rag/extractor/word_extractor.py

@@ -85,7 +85,7 @@ class WordExtractor(BaseExtractor):
             if "image" in rel.target_ref:
                 image_count += 1
                 if rel.is_external:
-                    url = rel.reltype
+                    url = rel.target_ref
                     response = ssrf_proxy.get(url)
                     if response.status_code == 200:
                         image_ext = mimetypes.guess_extension(response.headers["Content-Type"])

api/core/tools/tool_file_manager.py

@@ -108,7 +108,11 @@ class ToolFileManager:
         except httpx.TimeoutException:
             raise ValueError(f"timeout when downloading file from {file_url}")
 
-        mimetype = guess_type(file_url)[0] or "application/octet-stream"
+        mimetype = (
+            guess_type(file_url)[0]
+            or response.headers.get("Content-Type", "").split(";")[0].strip()
+            or "application/octet-stream"
+        )
         extension = guess_extension(mimetype) or ".bin"
         unique_name = uuid4().hex
         filename = f"{unique_name}{extension}"

api/core/workflow/graph_engine/graph_engine.py

@@ -641,6 +641,8 @@ class GraphEngine:
             try:
                 # run node
                 retry_start_at = datetime.now(UTC).replace(tzinfo=None)
+                # yield control to other threads
+                time.sleep(0.001)
                 generator = node_instance.run()
                 for item in generator:
                     if isinstance(item, GraphEngineEvent):

api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import uuid
 from collections.abc import Mapping, Sequence
 from typing import Any, Optional, cast
@@ -58,6 +59,30 @@ from .prompts import (
     FUNCTION_CALLING_EXTRACTOR_USER_TEMPLATE,
 )
 
+logger = logging.getLogger(__name__)
+
+
+def extract_json(text):
+    """
+    From a given JSON started from '{' or '[' extract the complete JSON object.
+    """
+    stack = []
+    for i, c in enumerate(text):
+        if c in {"{", "["}:
+            stack.append(c)
+        elif c in {"}", "]"}:
+            # check if stack is empty
+            if not stack:
+                return text[:i]
+            # check if the last element in stack is matching
+            if (c == "}" and stack[-1] == "{") or (c == "]" and stack[-1] == "["):
+                stack.pop()
+                if not stack:
+                    return text[: i + 1]
+            else:
+                return text[:i]
+    return None
+
 
 class ParameterExtractorNode(LLMNode):
     """
@@ -594,27 +619,6 @@ class ParameterExtractorNode(LLMNode):
         Extract complete json response.
         """
 
-        def extract_json(text):
-            """
-            From a given JSON started from '{' or '[' extract the complete JSON object.
-            """
-            stack = []
-            for i, c in enumerate(text):
-                if c in {"{", "["}:
-                    stack.append(c)
-                elif c in {"}", "]"}:
-                    # check if stack is empty
-                    if not stack:
-                        return text[:i]
-                    # check if the last element in stack is matching
-                    if (c == "}" and stack[-1] == "{") or (c == "]" and stack[-1] == "["):
-                        stack.pop()
-                        if not stack:
-                            return text[: i + 1]
-                    else:
-                        return text[:i]
-            return None
-
         # extract json from the text
         for idx in range(len(result)):
             if result[idx] == "{" or result[idx] == "[":
@@ -624,6 +628,7 @@ class ParameterExtractorNode(LLMNode):
                         return cast(dict, json.loads(json_str))
                     except Exception:
                         pass
+        logger.info(f"extra error: {result}")
         return None
 
     def _extract_json_from_tool_call(self, tool_call: AssistantPromptMessage.ToolCall) -> Optional[dict]:
@@ -633,7 +638,18 @@ class ParameterExtractorNode(LLMNode):
         if not tool_call or not tool_call.function.arguments:
             return None
 
-        return cast(dict, json.loads(tool_call.function.arguments))
+        result = tool_call.function.arguments
+        # extract json from the arguments
+        for idx in range(len(result)):
+            if result[idx] == "{" or result[idx] == "[":
+                json_str = extract_json(result[idx:])
+                if json_str:
+                    try:
+                        return cast(dict, json.loads(json_str))
+                    except Exception:
+                        pass
+        logger.info(f"extra error: {result}")
+        return None
 
     def _generate_default_result(self, data: ParameterExtractorNodeData) -> dict:
         """

api/core/workflow/nodes/question_classifier/template_prompts.py

@@ -1,21 +1,21 @@
 QUESTION_CLASSIFIER_SYSTEM_PROMPT = """
-    ### Job Description',
+### Job Description',
-    You are a text classification engine that analyzes text data and assigns categories based on user input or automatically determined categories.
+You are a text classification engine that analyzes text data and assigns categories based on user input or automatically determined categories.
-    ### Task
+### Task
-    Your task is to assign one categories ONLY to the input text and only one category may be assigned returned in the output. Additionally, you need to extract the key words from the text that are related to the classification.
+Your task is to assign one categories ONLY to the input text and only one category may be assigned returned in the output. Additionally, you need to extract the key words from the text that are related to the classification.
-    ### Format
+### Format
-    The input text is in the variable input_text. Categories are specified as a category list with two filed category_id and category_name in the variable categories. Classification instructions may be included to improve the classification accuracy.
+The input text is in the variable input_text. Categories are specified as a category list with two filed category_id and category_name in the variable categories. Classification instructions may be included to improve the classification accuracy.
-    ### Constraint
+### Constraint
-    DO NOT include anything other than the JSON array in your response.
+DO NOT include anything other than the JSON array in your response.
-    ### Memory
+### Memory
-    Here are the chat histories between human and assistant, inside <histories></histories> XML tags.
+Here are the chat histories between human and assistant, inside <histories></histories> XML tags.
-    <histories>
+<histories>
-    {histories}
+{histories}
-    </histories>
+</histories>
 """  # noqa: E501
 
 QUESTION_CLASSIFIER_USER_PROMPT_1 = """
-    { "input_text": ["I recently had a great experience with your company. The service was prompt and the staff was very friendly."],
+    {"input_text": ["I recently had a great experience with your company. The service was prompt and the staff was very friendly."],
     "categories": [{"category_id":"f5660049-284f-41a7-b301-fd24176a711c","category_name":"Customer Service"},{"category_id":"8d007d06-f2c9-4be5-8ff6-cd4381c13c60","category_name":"Satisfaction"},{"category_id":"5fbbbb18-9843-466d-9b8e-b9bfbb9482c8","category_name":"Sales"},{"category_id":"23623c75-7184-4a2e-8226-466c2e4631e4","category_name":"Product"}],
     "classification_instructions": ["classify the text based on the feedback provided by customer"]}
 """  # noqa: E501
@@ -43,9 +43,9 @@ QUESTION_CLASSIFIER_ASSISTANT_PROMPT_2 = """
 """
 
 QUESTION_CLASSIFIER_USER_PROMPT_3 = """
-    '{{"input_text": ["{input_text}"],',
+    {{"input_text": ["{input_text}"],
-    '"categories": {categories}, ',
+    "categories": {categories},
-    '"classification_instructions": ["{classification_instructions}"]}}'
+    "classification_instructions": ["{classification_instructions}"]}}
 """
 
 QUESTION_CLASSIFIER_COMPLETION_PROMPT = """

api/migrations/versions/2025_03_29_2227-6a9f914f656c_change_documentsegment_and_childchunk_.py

@@ -0,0 +1,43 @@
+"""change documentsegment and childchunk indexes
+
+Revision ID: 6a9f914f656c
+Revises: d20049ed0af6
+Create Date: 2025-03-29 22:27:24.789481
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6a9f914f656c'
+down_revision = 'd20049ed0af6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('child_chunks', schema=None) as batch_op:
+        batch_op.create_index('child_chunks_node_idx', ['index_node_id', 'dataset_id'], unique=False)
+        batch_op.create_index('child_chunks_segment_idx', ['segment_id'], unique=False)
+
+    with op.batch_alter_table('document_segments', schema=None) as batch_op:
+        batch_op.drop_index('document_segment_dataset_node_idx')
+        batch_op.create_index('document_segment_node_dataset_idx', ['index_node_id', 'dataset_id'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('document_segments', schema=None) as batch_op:
+        batch_op.drop_index('document_segment_node_dataset_idx')
+        batch_op.create_index('document_segment_dataset_node_idx', ['dataset_id', 'index_node_id'], unique=False)
+
+    with op.batch_alter_table('child_chunks', schema=None) as batch_op:
+        batch_op.drop_index('child_chunks_segment_idx')
+        batch_op.drop_index('child_chunks_node_idx')
+
+    # ### end Alembic commands ###

api/models/dataset.py

@@ -643,7 +643,7 @@ class DocumentSegment(db.Model):  # type: ignore[name-defined]
         db.Index("document_segment_document_id_idx", "document_id"),
         db.Index("document_segment_tenant_dataset_idx", "dataset_id", "tenant_id"),
         db.Index("document_segment_tenant_document_idx", "document_id", "tenant_id"),
-        db.Index("document_segment_dataset_node_idx", "dataset_id", "index_node_id"),
+        db.Index("document_segment_node_dataset_idx", "index_node_id", "dataset_id"),
         db.Index("document_segment_tenant_idx", "tenant_id"),
     )
 
@@ -791,6 +791,8 @@ class ChildChunk(db.Model):  # type: ignore[name-defined]
     __table_args__ = (
         db.PrimaryKeyConstraint("id", name="child_chunk_pkey"),
         db.Index("child_chunk_dataset_id_idx", "tenant_id", "dataset_id", "document_id", "segment_id", "index_node_id"),
+        db.Index("child_chunks_node_idx", "index_node_id", "dataset_id"),
+        db.Index("child_chunks_segment_idx", "segment_id"),
     )
 
     # initial fields

api/poetry.lock

@@ -845,10 +845,6 @@ files = [
     {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d"},
     {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0"},
     {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e"},
-    {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c"},
-    {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1"},
-    {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2"},
-    {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec"},
     {file = "Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2"},
     {file = "Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128"},
     {file = "Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc"},
@@ -861,14 +857,8 @@ files = [
     {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9"},
     {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265"},
     {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8"},
-    {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f"},
-    {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757"},
-    {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0"},
-    {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b"},
     {file = "Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50"},
     {file = "Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1"},
-    {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28"},
-    {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f"},
     {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409"},
     {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2"},
     {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451"},
@@ -879,24 +869,8 @@ files = [
     {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180"},
     {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248"},
     {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966"},
-    {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9"},
-    {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb"},
-    {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111"},
-    {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839"},
     {file = "Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0"},
     {file = "Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951"},
-    {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5"},
-    {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8"},
-    {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f"},
-    {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648"},
-    {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0"},
-    {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089"},
-    {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368"},
-    {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c"},
-    {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284"},
-    {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7"},
-    {file = "Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0"},
-    {file = "Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b"},
     {file = "Brotli-1.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1"},
     {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d"},
     {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b"},
@@ -906,10 +880,6 @@ files = [
     {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2"},
     {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354"},
     {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2"},
-    {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:aea440a510e14e818e67bfc4027880e2fb500c2ccb20ab21c7a7c8b5b4703d75"},
-    {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:6974f52a02321b36847cd19d1b8e381bf39939c21efd6ee2fc13a28b0d99348c"},
-    {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:a7e53012d2853a07a4a79c00643832161a910674a893d296c9f1259859a289d2"},
-    {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:d7702622a8b40c49bffb46e1e3ba2e81268d5c04a34f460978c6b5517a34dd52"},
     {file = "Brotli-1.1.0-cp36-cp36m-win32.whl", hash = "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460"},
     {file = "Brotli-1.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579"},
     {file = "Brotli-1.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c"},
@@ -921,10 +891,6 @@ files = [
     {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74"},
     {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b"},
     {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438"},
-    {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:cb1dac1770878ade83f2ccdf7d25e494f05c9165f5246b46a621cc849341dc01"},
-    {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:3ee8a80d67a4334482d9712b8e83ca6b1d9bc7e351931252ebef5d8f7335a547"},
-    {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5e55da2c8724191e5b557f8e18943b1b4839b8efc3ef60d65985bcf6f587dd38"},
-    {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:d342778ef319e1026af243ed0a07c97acf3bad33b9f29e7ae6a1f68fd083e90c"},
     {file = "Brotli-1.1.0-cp37-cp37m-win32.whl", hash = "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95"},
     {file = "Brotli-1.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68"},
     {file = "Brotli-1.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3"},
@@ -937,10 +903,6 @@ files = [
     {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a"},
     {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088"},
     {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596"},
-    {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d2b35ca2c7f81d173d2fadc2f4f31e88cc5f7a39ae5b6db5513cf3383b0e0ec7"},
-    {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:af6fa6817889314555aede9a919612b23739395ce767fe7fcbea9a80bf140fe5"},
-    {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:2feb1d960f760a575dbc5ab3b1c00504b24caaf6986e2dc2b01c09c87866a943"},
-    {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4410f84b33374409552ac9b6903507cdb31cd30d2501fc5ca13d18f73548444a"},
     {file = "Brotli-1.1.0-cp38-cp38-win32.whl", hash = "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b"},
     {file = "Brotli-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0"},
     {file = "Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a"},
@@ -953,10 +915,6 @@ files = [
     {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c"},
     {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d"},
     {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59"},
-    {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419"},
-    {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2"},
-    {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f"},
-    {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb"},
     {file = "Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64"},
     {file = "Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467"},
     {file = "Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724"},
@@ -4417,6 +4375,21 @@ html5 = ["html5lib"]
 htmlsoup = ["BeautifulSoup4"]
 source = ["Cython (>=3.0.11,<3.1.0)"]
 
+[[package]]
+name = "lxml-stubs"
+version = "0.5.1"
+description = "Type annotations for the lxml package"
+optional = false
+python-versions = "*"
+groups = ["dev"]
+files = [
+    {file = "lxml-stubs-0.5.1.tar.gz", hash = "sha256:e0ec2aa1ce92d91278b719091ce4515c12adc1d564359dfaf81efa7d4feab79d"},
+    {file = "lxml_stubs-0.5.1-py3-none-any.whl", hash = "sha256:1f689e5dbc4b9247cb09ae820c7d34daeb1fdbd1db06123814b856dae7787272"},
+]
+
+[package.extras]
+test = ["coverage[toml] (>=7.2.5)", "mypy (>=1.2.0)", "pytest (>=7.3.0)", "pytest-mypy-plugins (>=1.10.1)"]
+
 [[package]]
 name = "lz4"
 version = "4.4.3"
@@ -4956,49 +4929,49 @@ files = [
 
 [[package]]
 name = "mypy"
-version = "1.13.0"
+version = "1.15.0"
 description = "Optional static typing for Python"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
 groups = ["dev"]
 files = [
-    {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"},
+    {file = "mypy-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:979e4e1a006511dacf628e36fadfecbcc0160a8af6ca7dad2f5025529e082c13"},
-    {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"},
+    {file = "mypy-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c4bb0e1bd29f7d34efcccd71cf733580191e9a264a2202b0239da95984c5b559"},
-    {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"},
+    {file = "mypy-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be68172e9fd9ad8fb876c6389f16d1c1b5f100ffa779f77b1fb2176fcc9ab95b"},
-    {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"},
+    {file = "mypy-1.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7be1e46525adfa0d97681432ee9fcd61a3964c2446795714699a998d193f1a3"},
-    {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"},
+    {file = "mypy-1.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2e2c2e6d3593f6451b18588848e66260ff62ccca522dd231cd4dd59b0160668b"},
-    {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"},
+    {file = "mypy-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:6983aae8b2f653e098edb77f893f7b6aca69f6cffb19b2cc7443f23cce5f4828"},
-    {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"},
+    {file = "mypy-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2922d42e16d6de288022e5ca321cd0618b238cfc5570e0263e5ba0a77dbef56f"},
-    {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"},
+    {file = "mypy-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ee2d57e01a7c35de00f4634ba1bbf015185b219e4dc5909e281016df43f5ee5"},
-    {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"},
+    {file = "mypy-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:973500e0774b85d9689715feeffcc980193086551110fd678ebe1f4342fb7c5e"},
-    {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"},
+    {file = "mypy-1.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a95fb17c13e29d2d5195869262f8125dfdb5c134dc8d9a9d0aecf7525b10c2c"},
-    {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"},
+    {file = "mypy-1.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1905f494bfd7d85a23a88c5d97840888a7bd516545fc5aaedff0267e0bb54e2f"},
-    {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"},
+    {file = "mypy-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:c9817fa23833ff189db061e6d2eff49b2f3b6ed9856b4a0a73046e41932d744f"},
-    {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"},
+    {file = "mypy-1.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:aea39e0583d05124836ea645f412e88a5c7d0fd77a6d694b60d9b6b2d9f184fd"},
-    {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"},
+    {file = "mypy-1.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f2147ab812b75e5b5499b01ade1f4a81489a147c01585cda36019102538615f"},
-    {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"},
+    {file = "mypy-1.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce436f4c6d218a070048ed6a44c0bbb10cd2cc5e272b29e7845f6a2f57ee4464"},
-    {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"},
+    {file = "mypy-1.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8023ff13985661b50a5928fc7a5ca15f3d1affb41e5f0a9952cb68ef090b31ee"},
-    {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"},
+    {file = "mypy-1.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1124a18bc11a6a62887e3e137f37f53fbae476dc36c185d549d4f837a2a6a14e"},
-    {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"},
+    {file = "mypy-1.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:171a9ca9a40cd1843abeca0e405bc1940cd9b305eaeea2dda769ba096932bb22"},
-    {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"},
+    {file = "mypy-1.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93faf3fdb04768d44bf28693293f3904bbb555d076b781ad2530214ee53e3445"},
-    {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"},
+    {file = "mypy-1.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:811aeccadfb730024c5d3e326b2fbe9249bb7413553f15499a4050f7c30e801d"},
-    {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"},
+    {file = "mypy-1.15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98b7b9b9aedb65fe628c62a6dc57f6d5088ef2dfca37903a7d9ee374d03acca5"},
-    {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"},
+    {file = "mypy-1.15.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c43a7682e24b4f576d93072216bf56eeff70d9140241f9edec0c104d0c515036"},
-    {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"},
+    {file = "mypy-1.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:baefc32840a9f00babd83251560e0ae1573e2f9d1b067719479bfb0e987c6357"},
-    {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"},
+    {file = "mypy-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:b9378e2c00146c44793c98b8d5a61039a048e31f429fb0eb546d93f4b000bedf"},
-    {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"},
+    {file = "mypy-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e601a7fa172c2131bff456bb3ee08a88360760d0d2f8cbd7a75a65497e2df078"},
-    {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"},
+    {file = "mypy-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:712e962a6357634fef20412699a3655c610110e01cdaa6180acec7fc9f8513ba"},
-    {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"},
+    {file = "mypy-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f95579473af29ab73a10bada2f9722856792a36ec5af5399b653aa28360290a5"},
-    {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"},
+    {file = "mypy-1.15.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f8722560a14cde92fdb1e31597760dc35f9f5524cce17836c0d22841830fd5b"},
-    {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"},
+    {file = "mypy-1.15.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1fbb8da62dc352133d7d7ca90ed2fb0e9d42bb1a32724c287d3c76c58cbaa9c2"},
-    {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"},
+    {file = "mypy-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:d10d994b41fb3497719bbf866f227b3489048ea4bbbb5015357db306249f7980"},
-    {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"},
+    {file = "mypy-1.15.0-py3-none-any.whl", hash = "sha256:5469affef548bd1895d86d3bf10ce2b44e33d86923c29e4d675b3e323437ea3e"},
-    {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"},
+    {file = "mypy-1.15.0.tar.gz", hash = "sha256:404534629d51d3efea5c800ee7c42b72a6554d6c400e6a79eafe15d11341fd43"},
 ]
 
 [package.dependencies]
-mypy-extensions = ">=1.0.0"
+mypy_extensions = ">=1.0.0"
-typing-extensions = ">=4.6.0"
+typing_extensions = ">=4.6.0"
 
 [package.extras]
 dmypy = ["psutil (>=4.0)"]
@@ -5147,7 +5120,7 @@ version = "1.26.4"
 description = "Fundamental package for array computing in Python"
 optional = false
 python-versions = ">=3.9"
-groups = ["main", "indirect", "vdb"]
+groups = ["main", "dev", "indirect", "vdb"]
 files = [
     {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"},
     {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"},
@@ -8871,6 +8844,18 @@ rich = ">=10.11.0"
 shellingham = ">=1.3.0"
 typing-extensions = ">=3.7.4.3"
 
+[[package]]
+name = "types-aiofiles"
+version = "24.1.0.20250326"
+description = "Typing stubs for aiofiles"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_aiofiles-24.1.0.20250326-py3-none-any.whl", hash = "sha256:dfb58c9aa18bd449e80fb5d7f49dc3dd20d31de920a46223a61798ee4a521a70"},
+    {file = "types_aiofiles-24.1.0.20250326.tar.gz", hash = "sha256:c4bbe432fd043911ba83fb635456f5cc54f6d05fda2aadf6bef12a84f07a6efe"},
+]
+
 [[package]]
 name = "types-beautifulsoup4"
 version = "4.12.0.20250204"
@@ -8886,6 +8871,42 @@ files = [
 [package.dependencies]
 types-html5lib = "*"
 
+[[package]]
+name = "types-cachetools"
+version = "5.5.0.20240820"
+description = "Typing stubs for cachetools"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "types-cachetools-5.5.0.20240820.tar.gz", hash = "sha256:b888ab5c1a48116f7799cd5004b18474cd82b5463acb5ffb2db2fc9c7b053bc0"},
+    {file = "types_cachetools-5.5.0.20240820-py3-none-any.whl", hash = "sha256:efb2ed8bf27a4b9d3ed70d33849f536362603a90b8090a328acf0cd42fda82e2"},
+]
+
+[[package]]
+name = "types-colorama"
+version = "0.4.15.20240311"
+description = "Typing stubs for colorama"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "types-colorama-0.4.15.20240311.tar.gz", hash = "sha256:a28e7f98d17d2b14fb9565d32388e419f4108f557a7d939a66319969b2b99c7a"},
+    {file = "types_colorama-0.4.15.20240311-py3-none-any.whl", hash = "sha256:6391de60ddc0db3f147e31ecb230006a6823e81e380862ffca1e4695c13a0b8e"},
+]
+
+[[package]]
+name = "types-defusedxml"
+version = "0.7.0.20240218"
+description = "Typing stubs for defusedxml"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "types-defusedxml-0.7.0.20240218.tar.gz", hash = "sha256:05688a7724dc66ea74c4af5ca0efc554a150c329cb28c13a64902cab878d06ed"},
+    {file = "types_defusedxml-0.7.0.20240218-py3-none-any.whl", hash = "sha256:2b7f3c5ca14fdbe728fab0b846f5f7eb98c4bd4fd2b83d25f79e923caa790ced"},
+]
+
 [[package]]
 name = "types-deprecated"
 version = "1.2.15.20250304"
@@ -8898,16 +8919,28 @@ files = [
     {file = "types_deprecated-1.2.15.20250304.tar.gz", hash = "sha256:c329030553029de5cc6cb30f269c11f4e00e598c4241290179f63cda7d33f719"},
 ]
 
+[[package]]
+name = "types-docutils"
+version = "0.21.0.20241128"
+description = "Typing stubs for docutils"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "types_docutils-0.21.0.20241128-py3-none-any.whl", hash = "sha256:e0409204009639e9b0bf4521eeabe58b5e574ce9c0db08421c2ac26c32be0039"},
+    {file = "types_docutils-0.21.0.20241128.tar.gz", hash = "sha256:4dd059805b83ac6ec5a223699195c4e9eeb0446a4f7f2aeff1759a4a7cc17473"},
+]
+
 [[package]]
 name = "types-flask-cors"
-version = "4.0.0.20240828"
+version = "5.0.0.20240902"
 description = "Typing stubs for Flask-Cors"
 optional = false
 python-versions = ">=3.8"
 groups = ["dev"]
 files = [
-    {file = "types-Flask-Cors-4.0.0.20240828.tar.gz", hash = "sha256:f48ecf6366da923331311907cde3500e1435e07df01397ce0ef2306e263a5e85"},
+    {file = "types-Flask-Cors-5.0.0.20240902.tar.gz", hash = "sha256:8921b273bf7cd9636df136b66408efcfa6338a935e5c8f53f5eff1cee03f3394"},
-    {file = "types_Flask_Cors-4.0.0.20240828-py3-none-any.whl", hash = "sha256:36b752e88d6517fb82973b4240fe9bde44d29485bbd92dfff762a7101bdac3a0"},
+    {file = "types_Flask_Cors-5.0.0.20240902-py3-none-any.whl", hash = "sha256:595e5f36056cd128ab905832e055f2e5d116fbdc685356eea4490bc77df82137"},
 ]
 
 [package.dependencies]
@@ -8929,6 +8962,34 @@ files = [
 Flask = ">=2.0.0"
 Flask-SQLAlchemy = ">=3.0.1"
 
+[[package]]
+name = "types-gevent"
+version = "24.11.0.20250401"
+description = "Typing stubs for gevent"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_gevent-24.11.0.20250401-py3-none-any.whl", hash = "sha256:6764faf861ea99250c38179c58076392c44019ac3393029f71b06c4a15e8c1d1"},
+    {file = "types_gevent-24.11.0.20250401.tar.gz", hash = "sha256:1443f796a442062698e67d818fca50aa88067dee4021d457a7c0c6bedd6f46ca"},
+]
+
+[package.dependencies]
+types-greenlet = "*"
+types-psutil = "*"
+
+[[package]]
+name = "types-greenlet"
+version = "3.1.0.20250401"
+description = "Typing stubs for greenlet"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_greenlet-3.1.0.20250401-py3-none-any.whl", hash = "sha256:77987f3249b0f21415dc0254057e1ae4125a696a9bba28b0bcb67ee9e3dc14f6"},
+    {file = "types_greenlet-3.1.0.20250401.tar.gz", hash = "sha256:949389b64c34ca9472f6335189e9fe0b2e9704436d4f0850e39e9b7145909082"},
+]
+
 [[package]]
 name = "types-html5lib"
 version = "1.1.11.20241018"
@@ -8941,6 +9002,54 @@ files = [
     {file = "types_html5lib-1.1.11.20241018-py3-none-any.whl", hash = "sha256:3f1e064d9ed2c289001ae6392c84c93833abb0816165c6ff0abfc304a779f403"},
 ]
 
+[[package]]
+name = "types-markdown"
+version = "3.7.0.20250322"
+description = "Typing stubs for Markdown"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_markdown-3.7.0.20250322-py3-none-any.whl", hash = "sha256:7e855503027b4290355a310fb834871940d9713da7c111f3e98a5e1cbc77acfb"},
+    {file = "types_markdown-3.7.0.20250322.tar.gz", hash = "sha256:a48ed82dfcb6954592a10f104689d2d44df9125ce51b3cee20e0198a5216d55c"},
+]
+
+[[package]]
+name = "types-oauthlib"
+version = "3.2.0.20250403"
+description = "Typing stubs for oauthlib"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_oauthlib-3.2.0.20250403-py3-none-any.whl", hash = "sha256:02466f91a01522adfa4aaf0d7e76274f00a102eed40034117c5ecae768a2571e"},
+    {file = "types_oauthlib-3.2.0.20250403.tar.gz", hash = "sha256:40a4fcfb2e95235e399b5c0dd1cbe9d8c4b19415c09fb54c648d3397e02e0425"},
+]
+
+[[package]]
+name = "types-objgraph"
+version = "3.6.0.20240907"
+description = "Typing stubs for objgraph"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "types-objgraph-3.6.0.20240907.tar.gz", hash = "sha256:2e3dee675843ae387889731550b0ddfed06e9420946cf78a4bca565b5fc53634"},
+    {file = "types_objgraph-3.6.0.20240907-py3-none-any.whl", hash = "sha256:67207633a9b5789ee1911d740b269c3371081b79c0d8f68b00e7b8539f5c43f5"},
+]
+
+[[package]]
+name = "types-olefile"
+version = "0.47.0.20240806"
+description = "Typing stubs for olefile"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "types-olefile-0.47.0.20240806.tar.gz", hash = "sha256:96490f208cbb449a52283855319d73688ba9167ae58858ef8c506bf7ca2c6b67"},
+    {file = "types_olefile-0.47.0.20240806-py3-none-any.whl", hash = "sha256:c760a3deab7adb87a80d33b0e4edbbfbab865204a18d5121746022d7f8555118"},
+]
+
 [[package]]
 name = "types-openpyxl"
 version = "3.1.5.20250306"
@@ -8954,27 +9063,39 @@ files = [
 ]
 
 [[package]]
-name = "types-protobuf"
+name = "types-pexpect"
-version = "4.25.0.20240417"
+version = "4.9.0.20241208"
-description = "Typing stubs for protobuf"
+description = "Typing stubs for pexpect"
 optional = false
 python-versions = ">=3.8"
 groups = ["dev"]
 files = [
-    {file = "types-protobuf-4.25.0.20240417.tar.gz", hash = "sha256:c34eff17b9b3a0adb6830622f0f302484e4c089f533a46e3f147568313544352"},
+    {file = "types_pexpect-4.9.0.20241208-py3-none-any.whl", hash = "sha256:1928f478528454f0fea3495c16cf1ee2e67fca5c9fe97d60b868ac48c1fd5633"},
-    {file = "types_protobuf-4.25.0.20240417-py3-none-any.whl", hash = "sha256:e9b613227c2127e3d4881d75d93c93b4d6fd97b5f6a099a0b654a05351c8685d"},
+    {file = "types_pexpect-4.9.0.20241208.tar.gz", hash = "sha256:bbca0d0819947a719989a5cfe83641d9212bef893e2f0a7a01e47926bc82401d"},
+]
+
+[[package]]
+name = "types-protobuf"
+version = "5.29.1.20250403"
+description = "Typing stubs for protobuf"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_protobuf-5.29.1.20250403-py3-none-any.whl", hash = "sha256:c71de04106a2d54e5b2173d0a422058fae0ef2d058d70cf369fb797bf61ffa59"},
+    {file = "types_protobuf-5.29.1.20250403.tar.gz", hash = "sha256:7ff44f15022119c9d7558ce16e78b2d485bf7040b4fadced4dd069bb5faf77a2"},
 ]
 
 [[package]]
 name = "types-psutil"
-version = "7.0.0.20250218"
+version = "7.0.0.20250401"
 description = "Typing stubs for psutil"
 optional = false
 python-versions = ">=3.9"
 groups = ["dev"]
 files = [
-    {file = "types_psutil-7.0.0.20250218-py3-none-any.whl", hash = "sha256:1447a30c282aafefcf8941ece854e1100eee7b0296a9d9be9977292f0269b121"},
+    {file = "types_psutil-7.0.0.20250401-py3-none-any.whl", hash = "sha256:ed23f7140368104afe4e05a6085a5fa56fbe8c880a0f4dfe8d63e041106071ed"},
-    {file = "types_psutil-7.0.0.20250218.tar.gz", hash = "sha256:1e642cdafe837b240295b23b1cbd4691d80b08a07d29932143cbbae30eb0db9c"},
+    {file = "types_psutil-7.0.0.20250401.tar.gz", hash = "sha256:2a7d663c0888a079fc1643ebc109ad12e57a21c9552a9e2035da504191336dbf"},
 ]
 
 [[package]]
@@ -8989,6 +9110,33 @@ files = [
     {file = "types_psycopg2-2.9.21.20250318.tar.gz", hash = "sha256:eb6eac5bfb16adfd5f16b818918b9e26a40ede147e0f2bbffdf53a6ef7025a87"},
 ]
 
+[[package]]
+name = "types-pygments"
+version = "2.19.0.20250305"
+description = "Typing stubs for Pygments"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_pygments-2.19.0.20250305-py3-none-any.whl", hash = "sha256:ca88aae5ec426f9b107c0f7adc36dc096d2882d930a49f679eaf4b8b643db35d"},
+    {file = "types_pygments-2.19.0.20250305.tar.gz", hash = "sha256:044c50e80ecd4128c00a7268f20355e16f5c55466d3d49dfda09be920af40b4b"},
+]
+
+[package.dependencies]
+types-docutils = "*"
+
+[[package]]
+name = "types-pymysql"
+version = "1.1.0.20241103"
+description = "Typing stubs for PyMySQL"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "types-PyMySQL-1.1.0.20241103.tar.gz", hash = "sha256:a7628542919a0ba87625fb79eefb2a2de45fb4ad32afe6e561e8f2f27fb58b8c"},
+    {file = "types_PyMySQL-1.1.0.20241103-py3-none-any.whl", hash = "sha256:1a32efd8a74b5bf74c4de92a86c1cc6edaf3802dcfd5546635ab501eb5e3c096"},
+]
+
 [[package]]
 name = "types-python-dateutil"
 version = "2.9.0.20241206"
@@ -9007,78 +9155,162 @@ version = "2025.1.0.20250318"
 description = "Typing stubs for pytz"
 optional = false
 python-versions = ">=3.9"
-groups = ["main", "dev"]
+groups = ["main"]
 files = [
     {file = "types_pytz-2025.1.0.20250318-py3-none-any.whl", hash = "sha256:04dba4907c5415777083f9548693c6d9f80ec53adcaff55a38526a3f8ddcae04"},
     {file = "types_pytz-2025.1.0.20250318.tar.gz", hash = "sha256:97e0e35184c6fe14e3a5014512057f2c57bb0c6582d63c1cfcc4809f82180449"},
 ]
 
 [[package]]
-name = "types-pyyaml"
+name = "types-pywin32"
-version = "6.0.12.20241230"
+version = "310.0.0.20250319"
-description = "Typing stubs for PyYAML"
+description = "Typing stubs for pywin32"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
 groups = ["dev"]
 files = [
-    {file = "types_PyYAML-6.0.12.20241230-py3-none-any.whl", hash = "sha256:fa4d32565219b68e6dee5f67534c722e53c00d1cfc09c435ef04d7353e1e96e6"},
+    {file = "types_pywin32-310.0.0.20250319-py3-none-any.whl", hash = "sha256:baeb558a82251f7d430d135036b054740893902fdee3f9fe568322730ff49779"},
-    {file = "types_pyyaml-6.0.12.20241230.tar.gz", hash = "sha256:7f07622dbd34bb9c8b264fe860a17e0efcad00d50b5f27e93984909d9363498c"},
+    {file = "types_pywin32-310.0.0.20250319.tar.gz", hash = "sha256:4d28fb85b3f268a92905a7242df48c530c847cfe4cdb112386101ab6407660d8"},
+]
+
+[[package]]
+name = "types-pyyaml"
+version = "6.0.12.20250402"
+description = "Typing stubs for PyYAML"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_pyyaml-6.0.12.20250402-py3-none-any.whl", hash = "sha256:652348fa9e7a203d4b0d21066dfb00760d3cbd5a15ebb7cf8d33c88a49546681"},
+    {file = "types_pyyaml-6.0.12.20250402.tar.gz", hash = "sha256:d7c13c3e6d335b6af4b0122a01ff1d270aba84ab96d1a1a1063ecba3e13ec075"},
 ]
 
 [[package]]
 name = "types-regex"
-version = "2024.11.6.20250318"
+version = "2024.11.6.20250403"
 description = "Typing stubs for regex"
 optional = false
 python-versions = ">=3.9"
 groups = ["dev"]
 files = [
-    {file = "types_regex-2024.11.6.20250318-py3-none-any.whl", hash = "sha256:9309fe5918ee7ffe859c04c18040697655fade366c4dc844bbebe86976a9980b"},
+    {file = "types_regex-2024.11.6.20250403-py3-none-any.whl", hash = "sha256:e22c0f67d73f4b4af6086a340f387b6f7d03bed8a0bb306224b75c51a29b0001"},
-    {file = "types_regex-2024.11.6.20250318.tar.gz", hash = "sha256:6d472d0acf37b138cb32f67bd5ab1e7a200e94da8c1aa93ca3625a63e2efe1f3"},
+    {file = "types_regex-2024.11.6.20250403.tar.gz", hash = "sha256:3fdf2a70bbf830de4b3a28e9649a52d43dabb57cdb18fbfe2252eefb53666665"},
 ]
 
 [[package]]
 name = "types-requests"
-version = "2.31.0.20240406"
+version = "2.32.0.20250328"
 description = "Typing stubs for requests"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
 groups = ["dev"]
 files = [
-    {file = "types-requests-2.31.0.20240406.tar.gz", hash = "sha256:4428df33c5503945c74b3f42e82b181e86ec7b724620419a2966e2de604ce1a1"},
+    {file = "types_requests-2.32.0.20250328-py3-none-any.whl", hash = "sha256:72ff80f84b15eb3aa7a8e2625fffb6a93f2ad5a0c20215fc1dcfa61117bcb2a2"},
-    {file = "types_requests-2.31.0.20240406-py3-none-any.whl", hash = "sha256:6216cdac377c6b9a040ac1c0404f7284bd13199c0e1bb235f4324627e8898cf5"},
+    {file = "types_requests-2.32.0.20250328.tar.gz", hash = "sha256:c9e67228ea103bd811c96984fac36ed2ae8da87a36a633964a21f199d60baf32"},
 ]
 
 [package.dependencies]
 urllib3 = ">=2"
 
+[[package]]
+name = "types-requests-oauthlib"
+version = "2.0.0.20250306"
+description = "Typing stubs for requests-oauthlib"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_requests_oauthlib-2.0.0.20250306-py3-none-any.whl", hash = "sha256:37707de81d9ce54894afcccd70d4a845dbe4c59e747908faaeba59a96453d993"},
+    {file = "types_requests_oauthlib-2.0.0.20250306.tar.gz", hash = "sha256:92e5f1ed35689b1804fdcd60b7ac39b0bd440a4b96693685879bc835b334797f"},
+]
+
+[package.dependencies]
+types-oauthlib = "*"
+types-requests = "*"
+
+[[package]]
+name = "types-shapely"
+version = "2.0.0.20250404"
+description = "Typing stubs for shapely"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_shapely-2.0.0.20250404-py3-none-any.whl", hash = "sha256:170fb92f5c168a120db39b3287697fdec5c93ef3e1ad15e52552c36b25318821"},
+    {file = "types_shapely-2.0.0.20250404.tar.gz", hash = "sha256:863f540b47fa626c33ae64eae06df171f9ab0347025d4458d2df496537296b4f"},
+]
+
+[package.dependencies]
+numpy = ">=1.20"
+
+[[package]]
+name = "types-simplejson"
+version = "3.20.0.20250326"
+description = "Typing stubs for simplejson"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_simplejson-3.20.0.20250326-py3-none-any.whl", hash = "sha256:db1ddea7b8f7623b27a137578f22fc6c618db8c83ccfb1828ca0d2f0ec11efa7"},
+    {file = "types_simplejson-3.20.0.20250326.tar.gz", hash = "sha256:b2689bc91e0e672d7a5a947b4cb546b76ae7ddc2899c6678e72a10bf96cd97d2"},
+]
+
 [[package]]
 name = "types-six"
-version = "1.17.0.20250304"
+version = "1.17.0.20250403"
 description = "Typing stubs for six"
 optional = false
 python-versions = ">=3.9"
 groups = ["dev"]
 files = [
-    {file = "types_six-1.17.0.20250304-py3-none-any.whl", hash = "sha256:e482df1d439375f4b7c1f2540b1b8584aea82850164a296203ead4a7024fe14f"},
+    {file = "types_six-1.17.0.20250403-py3-none-any.whl", hash = "sha256:0bbb20fc34a18163afe7cac70b85864bd6937e6d73413c5b8f424def28760ae8"},
-    {file = "types_six-1.17.0.20250304.tar.gz", hash = "sha256:eeb240f9faec63ddd0498d6c0b6abd0496b154a66f960c004d4d733cf31bb4bd"},
+    {file = "types_six-1.17.0.20250403.tar.gz", hash = "sha256:82076f86e6e672a95adbf8b52625b1b3c72a8b9a893180344c1a02a6daabead6"},
 ]
 
+[[package]]
+name = "types-tensorflow"
+version = "2.18.0.20250404"
+description = "Typing stubs for tensorflow"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_tensorflow-2.18.0.20250404-py3-none-any.whl", hash = "sha256:4ad86534e6cfd6b36b2c97239ef9d122c44b167b25630b7c873a1483f9befd15"},
+    {file = "types_tensorflow-2.18.0.20250404.tar.gz", hash = "sha256:b38a427bbec805e4879d248f070baea802673c04cc5ccbe5979d742faa160670"},
+]
+
+[package.dependencies]
+numpy = ">=1.20"
+types-protobuf = "*"
+types-requests = "*"
+
 [[package]]
 name = "types-tqdm"
-version = "4.67.0.20250301"
+version = "4.67.0.20250404"
 description = "Typing stubs for tqdm"
 optional = false
 python-versions = ">=3.9"
 groups = ["dev"]
 files = [
-    {file = "types_tqdm-4.67.0.20250301-py3-none-any.whl", hash = "sha256:8af97deb8e6874af833555dc1fe0fcd456b1a789470bf6cd8813d4e7ee4f6c5b"},
+    {file = "types_tqdm-4.67.0.20250404-py3-none-any.whl", hash = "sha256:4a9b897eb4036f757240f4cb4a794f296265c04de46fdd058e453891f0186eed"},
-    {file = "types_tqdm-4.67.0.20250301.tar.gz", hash = "sha256:5e89a38ad89b867823368eb97d9f90d2fc69806bb055dde62716a05da62b5e0d"},
+    {file = "types_tqdm-4.67.0.20250404.tar.gz", hash = "sha256:e9997c655ffbba3ab78f4418b5511c05a54e76824d073d212166dc73aa56c768"},
 ]
 
 [package.dependencies]
 types-requests = "*"
 
+[[package]]
+name = "types-ujson"
+version = "5.10.0.20250326"
+description = "Typing stubs for ujson"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "types_ujson-5.10.0.20250326-py3-none-any.whl", hash = "sha256:acc0913f569def62ef6a892c8a47703f65d05669a3252391a97765cf207dca5b"},
+    {file = "types_ujson-5.10.0.20250326.tar.gz", hash = "sha256:5469e05f2c31ecb3c4c0267cc8fe41bcd116826fbb4ded69801a645c687dd014"},
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.12.2"
@@ -10160,4 +10392,4 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.11,<3.13"
-content-hash = "5aaeadfb807cfc51a987065213562eba1f0d9552733abbff915062c6a3483a9d"
+content-hash = "8ffd55020aab4ae38c8b5a4fffe670ce2a6a78bbbe15f70a5bc6ade0af339f3e"

api/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "dify-api"
 requires-python = ">=3.11,<3.13"
-dynamic = [ "dependencies" ]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["poetry-core>=2.0.0"]
@@ -74,7 +74,7 @@ starlette = "0.41.0"
 tiktoken = "~0.8.0"
 tokenizers = "~0.15.0"
 transformers = "~4.35.0"
-unstructured = { version = "~0.16.1", extras = ["docx", "epub", "md", "msg", "ppt", "pptx"] }
+unstructured = { version = "~0.16.1", extras = ["docx", "epub", "md", "ppt", "pptx"] }
 validators = "0.21.0"
 yarl = "~1.18.3"
 # Before adding new dependency, consider place it in alphabet order (a-z) and suitable group.
@@ -148,27 +148,47 @@ optional = true
 [tool.poetry.group.dev.dependencies]
 coverage = "~7.2.4"
 faker = "~32.1.0"
-mypy = "~1.13.0"
+lxml-stubs = "~0.5.1"
+mypy = "~1.15.0"
 pytest = "~8.3.2"
 pytest-benchmark = "~4.0.0"
 pytest-env = "~1.1.3"
 pytest-mock = "~3.14.0"
+types-aiofiles = "~24.1.0"
 types-beautifulsoup4 = "~4.12.0"
+types-cachetools = "~5.5.0"
+types-colorama = "~0.4.15"
+types-defusedxml = "~0.7.0"
 types-deprecated = "~1.2.15"
-types-flask-cors = "~4.0.0"
+types-docutils = "~0.21.0"
+types-flask-cors = "~5.0.0"
 types-flask-migrate = "~4.1.0"
+types-gevent = "~24.11.0"
+types-greenlet = "~3.1.0"
 types-html5lib = "~1.1.11"
+types-markdown = "~3.7.0"
+types-oauthlib = "~3.2.0"
+types-objgraph = "~3.6.0"
+types-olefile = "~0.47.0"
 types-openpyxl = "~3.1.5"
-types-protobuf = "~4.25.0"
+types-pexpect = "~4.9.0"
+types-protobuf = "~5.29.1"
 types-psutil = "~7.0.0"
 types-psycopg2 = "~2.9.21"
+types-pygments = "~2.19.0"
+types-pymysql = "~1.1.0"
 types-python-dateutil = "~2.9.0"
-types-pytz = "~2025.1"
+types-pywin32 = "~310.0.0"
-types-pyyaml = "~6.0.2"
+types-pyyaml = "~6.0.12"
 types-regex = "~2024.11.6"
-types-requests = "~2.31.0"
+types-requests = "~2.32.0"
+types-requests-oauthlib = "~2.0.0"
+types-shapely = "~2.0.0"
+types-simplejson = "~3.20.0"
 types-six = "~1.17.0"
+types-tensorflow = "~2.18.0"
 types-tqdm = "~4.67.0"
+types-ujson = "~5.10.0"
 
 ############################################################
 # [ Lint ] dependency group

api/services/dataset_service.py

@@ -1786,12 +1786,8 @@ class SegmentService:
                     )
                 elif document.doc_form in (IndexType.PARAGRAPH_INDEX, IndexType.QA_INDEX):
                     if args.enabled or keyword_changed:
-                        VectorService.create_segments_vector(
+                        # update segment vector index
-                            [args.keywords] if args.keywords else None,
+                        VectorService.update_segment_vector(args.keywords, segment, dataset)
-                            [segment],
-                            dataset,
-                            document.doc_form,
-                        )
             else:
                 segment_hash = helper.generate_text_hash(content)
                 tokens = 0

api/services/file_service.py

@@ -92,6 +92,11 @@ class FileService:
         db.session.add(upload_file)
         db.session.commit()
 
+        if not upload_file.source_url:
+            upload_file.source_url = file_helpers.get_signed_file_url(upload_file_id=upload_file.id)
+            db.session.add(upload_file)
+            db.session.commit()
+
         return upload_file
 
     @staticmethod

api/services/hit_testing_service.py

@@ -29,15 +29,6 @@ class HitTestingService:
         external_retrieval_model: dict,
         limit: int = 10,
     ) -> dict:
-        if dataset.available_document_count == 0 or dataset.available_segment_count == 0:
-            return {
-                "query": {
-                    "content": query,
-                    "tsne_position": {"x": 0, "y": 0},
-                },
-                "records": [],
-            }
-
         start = time.perf_counter()
 
         # get retrieval model , if the model is not setting , using default

api/services/plugin/data_migration.py

@@ -127,18 +127,32 @@ limit 1000"""
 
         processed_count = 0
         failed_ids = []
+        last_id = "00000000-0000-0000-0000-000000000000"
+
         while True:
-            sql = f"""select id, {provider_column_name} as provider_name from {table_name}
+            sql = f"""
-where {provider_column_name} not like '%/%' and {provider_column_name} is not null and {provider_column_name} != ''
+                SELECT id, {provider_column_name} AS provider_name 
-limit 1000"""
+                FROM {table_name}
+                WHERE {provider_column_name} NOT LIKE '%/%' 
+                    AND {provider_column_name} IS NOT NULL 
+                    AND {provider_column_name} != ''
+                    AND id > :last_id
+                ORDER BY id ASC
+                LIMIT 5000
+            """
+            params = {"last_id": last_id or ""}
+
             with db.engine.begin() as conn:
-                rs = conn.execute(db.text(sql))
+                rs = conn.execute(db.text(sql), params)
 
                 current_iter_count = 0
+                batch_updates = []
+
                 for i in rs:
                     current_iter_count += 1
                     processed_count += 1
                     record_id = str(i.id)
+                    last_id = record_id
                     provider_name = str(i.provider_name)
 
                     if record_id in failed_ids:
@@ -152,19 +166,9 @@ limit 1000"""
                     )
 
                     try:
-                        # update provider name append with "langgenius/{provider_name}/{provider_name}"
+                        updated_value = f"{DEFAULT_PLUGIN_ID}/{provider_name}/{provider_name}"
-                        sql = f"""update {table_name} 
+                        batch_updates.append((updated_value, record_id))
-                        set {provider_column_name} = 
+                    except Exception as e:
-                        concat('{DEFAULT_PLUGIN_ID}/', {provider_column_name}, '/', {provider_column_name}) 
-                        where id = :record_id"""
-                        conn.execute(db.text(sql), {"record_id": record_id})
-                        click.echo(
-                            click.style(
-                                f"[{processed_count}] Migrated [{table_name}] {record_id} ({provider_name})",
-                                fg="green",
-                            )
-                        )
-                    except Exception:
                         failed_ids.append(record_id)
                         click.echo(
                             click.style(
@@ -177,6 +181,20 @@ limit 1000"""
                         )
                         continue
 
+                if batch_updates:
+                    update_sql = f"""
+                        UPDATE {table_name} 
+                        SET {provider_column_name} = :updated_value 
+                        WHERE id = :record_id
+                    """
+                    conn.execute(db.text(update_sql), [{"updated_value": u, "record_id": r} for u, r in batch_updates])
+                    click.echo(
+                        click.style(
+                            f"[{processed_count}] Batch migrated [{len(batch_updates)}] records from [{table_name}]",
+                            fg="green",
+                        )
+                    )
+
             if not current_iter_count:
                 break
 

api/services/plugin/plugin_service.py

@@ -94,6 +94,13 @@ class PluginService:
         manager = PluginDebuggingManager()
         return manager.get_debugging_key(tenant_id)
 
+    @staticmethod
+    def list_latest_versions(plugin_ids: Sequence[str]) -> Mapping[str, Optional[LatestPluginCache]]:
+        """
+        List the latest versions of the plugins
+        """
+        return PluginService.fetch_latest_plugin_version(plugin_ids)
+
     @staticmethod
     def list(tenant_id: str) -> list[PluginEntity]:
         """
@@ -101,22 +108,6 @@ class PluginService:
         """
         manager = PluginInstallationManager()
         plugins = manager.list_plugins(tenant_id)
-        plugin_ids = [plugin.plugin_id for plugin in plugins if plugin.source == PluginInstallationSource.Marketplace]
-        try:
-            manifests = PluginService.fetch_latest_plugin_version(plugin_ids)
-        except Exception:
-            manifests = {}
-            logger.exception("failed to fetch plugin manifests")
-
-        for plugin in plugins:
-            if plugin.source == PluginInstallationSource.Marketplace:
-                if plugin.plugin_id in manifests:
-                    latest_plugin_cache = manifests[plugin.plugin_id]
-                    if latest_plugin_cache:
-                        # set latest_version
-                        plugin.latest_version = latest_plugin_cache.version
-                        plugin.latest_unique_identifier = latest_plugin_cache.unique_identifier
-
         return plugins
 
     @staticmethod

api/tasks/add_document_to_index_task.py

@@ -37,6 +37,10 @@ def add_document_to_index_task(dataset_document_id: str):
     indexing_cache_key = "document_{}_indexing".format(dataset_document.id)
 
     try:
+        dataset = dataset_document.dataset
+        if not dataset:
+            raise Exception(f"Document {dataset_document.id} dataset {dataset_document.dataset_id} doesn't exist.")
+
         segments = (
             db.session.query(DocumentSegment)
             .filter(
@@ -77,11 +81,6 @@ def add_document_to_index_task(dataset_document_id: str):
                     document.children = child_documents
             documents.append(document)
 
-        dataset = dataset_document.dataset
-
-        if not dataset:
-            raise Exception("Document has no dataset")
-
         index_type = dataset.doc_form
         index_processor = IndexProcessorFactory(index_type).init_index_processor()
         index_processor.load(dataset, documents)

api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py

@@ -5,6 +5,7 @@ from typing import Optional
 from unittest.mock import MagicMock
 
 from core.app.entities.app_invoke_entities import InvokeFrom
+from core.model_runtime.entities import AssistantPromptMessage
 from core.workflow.entities.variable_pool import VariablePool
 from core.workflow.enums import SystemVariableKey
 from core.workflow.graph_engine.entities.graph import Graph
@@ -311,6 +312,46 @@ def test_extract_json_response():
     assert result["location"] == "kawaii"
 
 
+def test_extract_json_from_tool_call():
+    """
+    Test extract json response.
+    """
+
+    node = init_parameter_extractor_node(
+        config={
+            "id": "llm",
+            "data": {
+                "title": "123",
+                "type": "parameter-extractor",
+                "model": {
+                    "provider": "langgenius/openai/openai",
+                    "name": "gpt-3.5-turbo-instruct",
+                    "mode": "completion",
+                    "completion_params": {},
+                },
+                "query": ["sys", "query"],
+                "parameters": [{"name": "location", "type": "string", "description": "location", "required": True}],
+                "reasoning_mode": "prompt",
+                "instruction": "{{#sys.query#}}",
+                "memory": None,
+            },
+        },
+    )
+
+    result = node._extract_json_from_tool_call(
+        AssistantPromptMessage.ToolCall(
+            id="llm",
+            type="parameter-extractor",
+            function=AssistantPromptMessage.ToolCall.ToolCallFunction(
+                name="foo", arguments="""{"location":"kawaii"}{"location": 1}"""
+            ),
+        )
+    )
+
+    assert result is not None
+    assert result["location"] == "kawaii"
+
+
 def test_chat_parameter_extractor_with_memory(setup_model_mock):
     """
     Test chat parameter extractor with memory.

dev/reformat

@@ -16,3 +16,6 @@ poetry run -C api ruff format ./
 
 # run dotenv-linter linter
 poetry run -P api dotenv-linter ./api/.env.example ./web/.env.example
+
+# run mypy check
+dev/run-mypy

dev/run-mypy

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -x
+
+if ! command -v mypy &> /dev/null; then
+    poetry install -C api --with dev
+fi
+
+# run mypy checks
+poetry run -C api \
+  python -m mypy --install-types --non-interactive .

docker/.env.example

@@ -621,6 +621,10 @@ CODE_GENERATION_MAX_TOKENS=1024
 # when generating structured output in the structured output tool.
 # Default: 1024 tokens.
 STRUCTURED_OUTPUT_MAX_TOKENS=1024
+# Enable or disable plugin based token counting. If disabled, token counting will return 0.
+# This can improve performance by skipping token counting operations.
+# Default: false (disabled).
+PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false
 
 # ------------------------------
 # Multi-modal Configuration
@@ -1011,3 +1015,28 @@ PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120
 PLUGIN_MAX_EXECUTION_TIMEOUT=600
 # PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple
 PIP_MIRROR_URL=
+
+# https://github.com/langgenius/dify-plugin-daemon/blob/main/.env.example
+# Plugin storage type, local aws_s3 tencent_cos azure_blob
+PLUGIN_STORAGE_TYPE=local
+PLUGIN_STORAGE_LOCAL_ROOT=/app/storage
+PLUGIN_WORKING_PATH=/app/storage/cwd
+PLUGIN_INSTALLED_PATH=plugin
+PLUGIN_PACKAGE_CACHE_PATH=plugin_packages
+PLUGIN_MEDIA_CACHE_PATH=assets
+# Plugin oss bucket
+PLUGIN_STORAGE_OSS_BUCKET=
+# Plugin oss s3 credentials
+PLUGIN_S3_USE_AWS_MANAGED_IAM=false
+PLUGIN_S3_ENDPOINT=
+PLUGIN_S3_USE_PATH_STYLE=false
+PLUGIN_AWS_ACCESS_KEY=
+PLUGIN_AWS_SECRET_KEY=
+PLUGIN_AWS_REGION=
+# Plugin oss azure blob
+PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME=
+PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING=
+# Plugin oss tencent cos
+PLUGIN_TENCENT_COS_SECRET_KEY=
+PLUGIN_TENCENT_COS_SECRET_ID=
+PLUGIN_TENCENT_COS_REGION=

docker/docker-compose-template.yaml

@@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env
 services:
   # API service
   api:
-    image: langgenius/dify-api:1.1.3
+    image: langgenius/dify-api:1.2.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -29,7 +29,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:1.1.3
+    image: langgenius/dify-api:1.2.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -53,7 +53,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:1.1.3
+    image: langgenius/dify-web:1.2.0
     restart: always
     environment:
       CONSOLE_API_URL: ${CONSOLE_API_URL:-}
@@ -134,7 +134,7 @@ services:
 
   # plugin daemon
   plugin_daemon:
-    image: langgenius/dify-plugin-daemon:0.0.6-local
+    image: langgenius/dify-plugin-daemon:0.0.7-local
     restart: always
     environment:
       # Use the shared environment variables.
@@ -153,6 +153,23 @@ services:
       PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
       PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
       PIP_MIRROR_URL: ${PIP_MIRROR_URL:-}
+      PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local}
+      PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage}
+      PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin}
+      PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages}
+      PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets}
+      PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-}
+      S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false}
+      S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-}
+      S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false}
+      AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-}
+      PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-}
+      AWS_REGION: ${PLUGIN_AWS_REGION:-}
+      AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-}
+      AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-}
+      TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-}
+      TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-}
+      TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-}
     ports:
       - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}"
     volumes:

docker/docker-compose.middleware.yaml

@@ -29,6 +29,8 @@ services:
   redis:
     image: redis:6-alpine
     restart: always
+    env_file:
+      - ./middleware.env
     environment:
       REDISCLI_AUTH: ${REDIS_PASSWORD:-difyai123456}
     volumes:
@@ -45,6 +47,8 @@ services:
   sandbox:
     image: langgenius/dify-sandbox:0.2.11
     restart: always
+    env_file:
+      - ./middleware.env
     environment:
       # The DifySandbox configurations
       # Make sure you are changing this key for your deployment with a strong key.
@@ -66,8 +70,10 @@ services:
 
   # plugin daemon
   plugin_daemon:
-    image: langgenius/dify-plugin-daemon:0.0.6-local
+    image: langgenius/dify-plugin-daemon:0.0.7-local
     restart: always
+    env_file:
+      - ./middleware.env
     environment:
       # Use the shared environment variables.
       DB_HOST: ${DB_HOST:-db}
@@ -91,6 +97,23 @@ services:
       PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
       PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
       PIP_MIRROR_URL: ${PIP_MIRROR_URL:-}
+      PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local}
+      PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage}
+      PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin}
+      PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages}
+      PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets}
+      PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-}
+      S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false}
+      S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-}
+      S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false}
+      AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-}
+      PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-}
+      AWS_REGION: ${PLUGIN_AWS_REGION:-}
+      AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-}
+      AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-}
+      TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-}
+      TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-}
+      TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-}
     ports:
       - "${EXPOSE_PLUGIN_DAEMON_PORT:-5002}:${PLUGIN_DAEMON_PORT:-5002}"
       - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}"
@@ -107,6 +130,8 @@ services:
       - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template
       - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh
     entrypoint: [ "sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ]
+    env_file:
+      - ./middleware.env
     environment:
       # pls clearly modify the squid env vars to fit your network environment.
       HTTP_PORT: ${SSRF_HTTP_PORT:-3128}

docker/docker-compose.yaml

@@ -277,6 +277,7 @@ x-shared-env: &shared-api-worker-env
   PROMPT_GENERATION_MAX_TOKENS: ${PROMPT_GENERATION_MAX_TOKENS:-512}
   CODE_GENERATION_MAX_TOKENS: ${CODE_GENERATION_MAX_TOKENS:-1024}
   STRUCTURED_OUTPUT_MAX_TOKENS: ${STRUCTURED_OUTPUT_MAX_TOKENS:-1024}
+  PLUGIN_BASED_TOKEN_COUNTING_ENABLED: ${PLUGIN_BASED_TOKEN_COUNTING_ENABLED:-false}
   MULTIMODAL_SEND_FORMAT: ${MULTIMODAL_SEND_FORMAT:-base64}
   UPLOAD_IMAGE_FILE_SIZE_LIMIT: ${UPLOAD_IMAGE_FILE_SIZE_LIMIT:-10}
   UPLOAD_VIDEO_FILE_SIZE_LIMIT: ${UPLOAD_VIDEO_FILE_SIZE_LIMIT:-100}
@@ -437,11 +438,29 @@ x-shared-env: &shared-api-worker-env
   PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
   PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
   PIP_MIRROR_URL: ${PIP_MIRROR_URL:-}
+  PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local}
+  PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage}
+  PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd}
+  PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin}
+  PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages}
+  PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets}
+  PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-}
+  PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false}
+  PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-}
+  PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false}
+  PLUGIN_AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-}
+  PLUGIN_AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-}
+  PLUGIN_AWS_REGION: ${PLUGIN_AWS_REGION:-}
+  PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-}
+  PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-}
+  PLUGIN_TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-}
+  PLUGIN_TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-}
+  PLUGIN_TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-}
 
 services:
   # API service
   api:
-    image: langgenius/dify-api:1.1.3
+    image: langgenius/dify-api:1.2.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -468,7 +487,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:1.1.3
+    image: langgenius/dify-api:1.2.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -492,7 +511,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:1.1.3
+    image: langgenius/dify-web:1.2.0
     restart: always
     environment:
       CONSOLE_API_URL: ${CONSOLE_API_URL:-}
@@ -529,7 +548,7 @@ services:
     volumes:
       - ./volumes/db/data:/var/lib/postgresql/data
     healthcheck:
-      test: [ 'CMD', 'pg_isready' ]
+      test: ["CMD", "pg_isready"]
       interval: 1s
       timeout: 3s
       retries: 30
@@ -546,7 +565,7 @@ services:
     # Set the redis password when startup redis server.
     command: redis-server --requirepass ${REDIS_PASSWORD:-difyai123456}
     healthcheck:
-      test: [ 'CMD', 'redis-cli', 'ping' ]
+      test: ["CMD", "redis-cli", "ping"]
 
   # The DifySandbox
   sandbox:
@@ -567,13 +586,13 @@ services:
       - ./volumes/sandbox/dependencies:/dependencies
       - ./volumes/sandbox/conf:/conf
     healthcheck:
-      test: [ 'CMD', 'curl', '-f', 'http://localhost:8194/health' ]
+      test: ["CMD", "curl", "-f", "http://localhost:8194/health"]
     networks:
       - ssrf_proxy_network
 
   # plugin daemon
   plugin_daemon:
-    image: langgenius/dify-plugin-daemon:0.0.6-local
+    image: langgenius/dify-plugin-daemon:0.0.7-local
     restart: always
     environment:
       # Use the shared environment variables.
@@ -592,6 +611,23 @@ services:
       PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
       PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
       PIP_MIRROR_URL: ${PIP_MIRROR_URL:-}
+      PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local}
+      PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage}
+      PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin}
+      PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages}
+      PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets}
+      PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-}
+      S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false}
+      S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-}
+      S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false}
+      AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-}
+      PAWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-}
+      AWS_REGION: ${PLUGIN_AWS_REGION:-}
+      AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-}
+      AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-}
+      TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-}
+      TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-}
+      TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-}
     ports:
       - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}"
     volumes:
@@ -608,7 +644,12 @@ services:
     volumes:
       - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template
       - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh
-    entrypoint: [ 'sh', '-c', "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ]
+    entrypoint:
+      [
+        "sh",
+        "-c",
+        "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh",
+      ]
     environment:
       # pls clearly modify the squid env vars to fit your network environment.
       HTTP_PORT: ${SSRF_HTTP_PORT:-3128}
@@ -637,8 +678,8 @@ services:
       - CERTBOT_EMAIL=${CERTBOT_EMAIL}
       - CERTBOT_DOMAIN=${CERTBOT_DOMAIN}
       - CERTBOT_OPTIONS=${CERTBOT_OPTIONS:-}
-    entrypoint: [ '/docker-entrypoint.sh' ]
+    entrypoint: ["/docker-entrypoint.sh"]
-    command: [ 'tail', '-f', '/dev/null' ]
+    command: ["tail", "-f", "/dev/null"]
 
   # The nginx reverse proxy.
   # used for reverse proxying the API service and Web service.
@@ -655,7 +696,12 @@ services:
       - ./volumes/certbot/conf/live:/etc/letsencrypt/live # cert dir (with certbot container)
       - ./volumes/certbot/conf:/etc/letsencrypt
       - ./volumes/certbot/www:/var/www/html
-    entrypoint: [ 'sh', '-c', "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ]
+    entrypoint:
+      [
+        "sh",
+        "-c",
+        "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh",
+      ]
     environment:
       NGINX_SERVER_NAME: ${NGINX_SERVER_NAME:-_}
       NGINX_HTTPS_ENABLED: ${NGINX_HTTPS_ENABLED:-false}
@@ -677,14 +723,14 @@ services:
       - api
       - web
     ports:
-      - '${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}'
+      - "${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}"
-      - '${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}'
+      - "${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}"
 
   # The Weaviate vector store.
   weaviate:
     image: semitechnologies/weaviate:1.19.0
     profiles:
-      - ''
+      - ""
       - weaviate
     restart: always
     volumes:
@@ -737,13 +783,17 @@ services:
     working_dir: /opt/couchbase
     stdin_open: true
     tty: true
-    entrypoint: [ "" ]
+    entrypoint: [""]
     command: sh -c "/opt/couchbase/init/init-cbserver.sh"
     volumes:
       - ./volumes/couchbase/data:/opt/couchbase/var/lib/couchbase/data
     healthcheck:
       # ensure bucket was created before proceeding
-      test: [ "CMD-SHELL", "curl -s -f -u Administrator:password http://localhost:8091/pools/default/buckets | grep -q '\\[{' || exit 1" ]
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s -f -u Administrator:password http://localhost:8091/pools/default/buckets | grep -q '\\[{' || exit 1",
+        ]
       interval: 10s
       retries: 10
       start_period: 30s
@@ -769,9 +819,9 @@ services:
     volumes:
       - ./volumes/pgvector/data:/var/lib/postgresql/data
       - ./pgvector/docker-entrypoint.sh:/docker-entrypoint.sh
-    entrypoint: [ '/docker-entrypoint.sh' ]
+    entrypoint: ["/docker-entrypoint.sh"]
     healthcheck:
-      test: [ 'CMD', 'pg_isready' ]
+      test: ["CMD", "pg_isready"]
       interval: 1s
       timeout: 3s
       retries: 30
@@ -793,7 +843,7 @@ services:
     volumes:
       - ./volumes/pgvecto_rs/data:/var/lib/postgresql/data
     healthcheck:
-      test: [ 'CMD', 'pg_isready' ]
+      test: ["CMD", "pg_isready"]
       interval: 1s
       timeout: 3s
       retries: 30
@@ -861,7 +911,7 @@ services:
       - ./volumes/milvus/etcd:/etcd
     command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
     healthcheck:
-      test: [ 'CMD', 'etcdctl', 'endpoint', 'health' ]
+      test: ["CMD", "etcdctl", "endpoint", "health"]
       interval: 30s
       timeout: 20s
       retries: 3
@@ -880,7 +930,7 @@ services:
       - ./volumes/milvus/minio:/minio_data
     command: minio server /minio_data --console-address ":9001"
     healthcheck:
-      test: [ 'CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live' ]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
       timeout: 20s
       retries: 3
@@ -892,7 +942,7 @@ services:
     image: milvusdb/milvus:v2.5.0-beta
     profiles:
       - milvus
-    command: [ 'milvus', 'run', 'standalone' ]
+    command: ["milvus", "run", "standalone"]
     environment:
       ETCD_ENDPOINTS: ${ETCD_ENDPOINTS:-etcd:2379}
       MINIO_ADDRESS: ${MINIO_ADDRESS:-minio:9000}
@@ -900,7 +950,7 @@ services:
     volumes:
       - ./volumes/milvus/milvus:/var/lib/milvus
     healthcheck:
-      test: [ 'CMD', 'curl', '-f', 'http://localhost:9091/healthz' ]
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
       interval: 30s
       start_period: 90s
       timeout: 20s
@@ -1007,18 +1057,19 @@ services:
       node.name: dify-es0
       discovery.type: single-node
       xpack.license.self_generated.type: basic
-      xpack.security.enabled: 'true'
+      xpack.security.enabled: "true"
-      xpack.security.enrollment.enabled: 'false'
+      xpack.security.enrollment.enabled: "false"
-      xpack.security.http.ssl.enabled: 'false'
+      xpack.security.http.ssl.enabled: "false"
     ports:
       - ${ELASTICSEARCH_PORT:-9200}:9200
     deploy:
       resources:
         limits:
           memory: 2g
-    entrypoint: [ 'sh', '-c', "sh /docker-entrypoint-mount.sh" ]
+    entrypoint: ["sh", "-c", "sh /docker-entrypoint-mount.sh"]
     healthcheck:
-      test: [ 'CMD', 'curl', '-s', 'http://localhost:9200/_cluster/health?pretty' ]
+      test:
+        ["CMD", "curl", "-s", "http://localhost:9200/_cluster/health?pretty"]
       interval: 30s
       timeout: 10s
       retries: 50
@@ -1036,17 +1087,17 @@ services:
     environment:
       XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: d1a66dfd-c4d3-4a0a-8290-2abcb83ab3aa
       NO_PROXY: localhost,127.0.0.1,elasticsearch,kibana
-      XPACK_SECURITY_ENABLED: 'true'
+      XPACK_SECURITY_ENABLED: "true"
-      XPACK_SECURITY_ENROLLMENT_ENABLED: 'false'
+      XPACK_SECURITY_ENROLLMENT_ENABLED: "false"
-      XPACK_SECURITY_HTTP_SSL_ENABLED: 'false'
+      XPACK_SECURITY_HTTP_SSL_ENABLED: "false"
-      XPACK_FLEET_ISAIRGAPPED: 'true'
+      XPACK_FLEET_ISAIRGAPPED: "true"
       I18N_LOCALE: zh-CN
-      SERVER_PORT: '5601'
+      SERVER_PORT: "5601"
       ELASTICSEARCH_HOSTS: http://elasticsearch:9200
     ports:
       - ${KIBANA_PORT:-5601}:5601
     healthcheck:
-      test: [ 'CMD-SHELL', 'curl -s http://localhost:5601 >/dev/null || exit 1' ]
+      test: ["CMD-SHELL", "curl -s http://localhost:5601 >/dev/null || exit 1"]
       interval: 30s
       timeout: 10s
       retries: 3

docker/middleware.env.example

@@ -120,3 +120,28 @@ PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120
 PLUGIN_MAX_EXECUTION_TIMEOUT=600
 # PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple
 PIP_MIRROR_URL=
+
+# https://github.com/langgenius/dify-plugin-daemon/blob/main/.env.example
+# Plugin storage type, local aws_s3 tencent_cos azure_blob
+PLUGIN_STORAGE_TYPE=local
+PLUGIN_STORAGE_LOCAL_ROOT=/app/storage
+PLUGIN_WORKING_PATH=/app/storage/cwd
+PLUGIN_INSTALLED_PATH=plugin
+PLUGIN_PACKAGE_CACHE_PATH=plugin_packages
+PLUGIN_MEDIA_CACHE_PATH=assets
+# Plugin oss bucket
+PLUGIN_STORAGE_OSS_BUCKET=
+# Plugin oss s3 credentials
+PLUGIN_S3_USE_AWS_MANAGED_IAM=false
+PLUGIN_S3_ENDPOINT=
+PLUGIN_S3_USE_PATH_STYLE=false
+PLUGIN_AWS_ACCESS_KEY=
+PLUGIN_AWS_SECRET_KEY=
+PLUGIN_AWS_REGION=
+# Plugin oss azure blob
+PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME=
+PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING=
+# Plugin oss tencent cos
+PLUGIN_TENCENT_COS_SECRET_KEY=
+PLUGIN_TENCENT_COS_SECRET_ID=
+PLUGIN_TENCENT_COS_REGION=

web/.vscode/extensions.json

@@ -1,6 +1,7 @@
 {
   "recommendations": [
     "bradlc.vscode-tailwindcss",
-    "firsttris.vscode-jest-runner"
+    "firsttris.vscode-jest-runner",
+    "kisstkondoros.vscode-codemetrics"
   ]
 }

web/Dockerfile

@@ -1,12 +1,12 @@
 # base image
-FROM node:20-alpine3.20 AS base
+FROM node:22-alpine3.21 AS base
 LABEL maintainer="takatost@gmail.com"
 
 # if you located in China, you can use aliyun mirror to speed up
 # RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
 
 RUN apk add --no-cache tzdata
-RUN npm install -g pnpm@9.12.2
+RUN npm install -g pnpm@10.8.0
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 

web/README.md

@@ -6,7 +6,9 @@ This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next
 
 ### Run by source code
 
-To start the web frontend service, you will need [Node.js v18.x (LTS)](https://nodejs.org/en) and [pnpm version 9.12.2](https://pnpm.io).
+Before starting the web frontend service, please make sure the following environment is ready.
+- [Node.js](https://nodejs.org) >= v18.x
+- [pnpm](https://pnpm.io) v10.x
 
 First, install the dependencies:
 

web/app/(commonLayout)/apps/Apps.tsx

@@ -66,6 +66,7 @@ const Apps = () => {
   const [isCreatedByMe, setIsCreatedByMe] = useState(queryIsCreatedByMe)
   const [tagFilterValue, setTagFilterValue] = useState<string[]>(tagIDs)
   const [searchKeywords, setSearchKeywords] = useState(keywords)
+  const newAppCardRef = useRef<HTMLDivElement>(null)
   const setKeywords = useCallback((keywords: string) => {
     setQuery(prev => ({ ...prev, keywords }))
   }, [setQuery])
@@ -73,10 +74,15 @@ const Apps = () => {
     setQuery(prev => ({ ...prev, tagIDs }))
   }, [setQuery])
 
-  const { data, isLoading, setSize, mutate } = useSWRInfinite(
+  const { data, isLoading, error, setSize, mutate } = useSWRInfinite(
     (pageIndex: number, previousPageData: AppListResponse) => getKey(pageIndex, previousPageData, activeTab, isCreatedByMe, tagIDs, searchKeywords),
     fetchAppList,
-    { revalidateFirstPage: true },
+    {
+      revalidateFirstPage: true,
+      shouldRetryOnError: false,
+      dedupingInterval: 500,
+      errorRetryCount: 3,
+    },
   )
 
   const anchorRef = useRef<HTMLDivElement>(null)
@@ -105,15 +111,22 @@ const Apps = () => {
   useEffect(() => {
     const hasMore = data?.at(-1)?.has_more ?? true
     let observer: IntersectionObserver | undefined
+
+    if (error) {
+      if (observer)
+        observer.disconnect()
+      return
+    }
+
     if (anchorRef.current) {
       observer = new IntersectionObserver((entries) => {
-        if (entries[0].isIntersecting && !isLoading && hasMore)
+        if (entries[0].isIntersecting && !isLoading && !error && hasMore)
           setSize((size: number) => size + 1)
       }, { rootMargin: '100px' })
       observer.observe(anchorRef.current)
     }
     return () => observer?.disconnect()
-  }, [isLoading, setSize, anchorRef, mutate, data])
+  }, [isLoading, setSize, anchorRef, mutate, data, error])
 
   const { run: handleSearch } = useDebounceFn(() => {
     setSearchKeywords(keywords)
@@ -166,14 +179,14 @@ const Apps = () => {
       {(data && data[0].total > 0)
         ? <div className='relative grid grow grid-cols-1 content-start gap-4 px-12 pt-2 sm:grid-cols-1 md:grid-cols-2 xl:grid-cols-4 2xl:grid-cols-5 2k:grid-cols-6'>
           {isCurrentWorkspaceEditor
-            && <NewAppCard onSuccess={mutate} />}
+            && <NewAppCard ref={newAppCardRef} onSuccess={mutate} />}
           {data.map(({ data: apps }) => apps.map(app => (
             <AppCard key={app.id} app={app} onRefresh={mutate} />
           )))}
         </div>
         : <div className='relative grid grow grid-cols-1 content-start gap-4 overflow-hidden px-12 pt-2 sm:grid-cols-1 md:grid-cols-2 xl:grid-cols-4 2xl:grid-cols-5 2k:grid-cols-6'>
           {isCurrentWorkspaceEditor
-            && <NewAppCard className='z-10' onSuccess={mutate} />}
+            && <NewAppCard ref={newAppCardRef} className='z-10' onSuccess={mutate} />}
           <NoAppsFound />
         </div>}
       <CheckModal />

web/app/(commonLayout)/datasets/template/template.en.mdx

@@ -386,11 +386,20 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
   <Col>
     ### Query
     <Properties>
+      <Property name='keyword' type='string' key='keyword'>
+        Search keyword, optional
+      </Property>
+      <Property name='tag_ids' type='array[string]' key='tag_ids'>
+        Tag ID list, optional
+      </Property>
       <Property name='page' type='string' key='page'>
-        Page number
+        Page number, optional, default 1
       </Property>
       <Property name='limit' type='string' key='limit'>
-        Number of items returned, default 20, range 1-100
+        Number of items returned, optional, default 20, range 1-100
+      </Property>
+      <Property name='include_all' type='boolean' key='include_all'>
+        Whether to include all datasets (only effective for owners), optional, defaults to false
       </Property>
     </Properties>
   </Col>

web/app/(commonLayout)/datasets/template/template.ja.mdx

@@ -386,11 +386,20 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
   <Col>
     ### クエリ
     <Properties>
+      <Property name='keyword' type='string' key='keyword'>
+        検索キーワード、オプション
+      </Property>
+      <Property name='tag_ids' type='array[string]' key='tag_ids'>
+        タグIDリスト、オプション
+      </Property>
       <Property name='page' type='string' key='page'>
-        ページ番号
+        ページ番号、オプション、デフォルト1
       </Property>
       <Property name='limit' type='string' key='limit'>
-        返されるアイテム数、デフォルトは 20、範囲は 1-100
+        返されるアイテム数、オプション、デフォルト20、範囲1-100
+      </Property>
+      <Property name='include_all' type='boolean' key='include_all'>
+        すべてのデータセットを含めるかどうか（所有者のみ有効）、オプション、デフォルトはfalse
       </Property>
     </Properties>
   </Col>

web/app/(commonLayout)/datasets/template/template.zh.mdx

@@ -387,11 +387,20 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
   <Col>
     ### Query
     <Properties>
-      <Property name='page' type='string' key='page'>
+      <Property name='keyword' type='string' key='keyword'>
-        页码
+        搜索关键词，可选
+      </Property>
+      <Property name='tag_ids' type='array[string]' key='tag_ids'>
+        标签 ID 列表，可选
+      </Property>
+      <Property name='page' type='integer' key='page'>
+        页码，可选，默认为 1
       </Property>
       <Property name='limit' type='string' key='limit'>
-        返回条数，默认 20，范围 1-100
+        返回条数，可选，默认 20，范围 1-100
+      </Property>
+      <Property name='include_all' type='boolean' key='include_all'>
+        是否包含所有数据集（仅对所有者生效），可选，默认为 false
       </Property>
     </Properties>
   </Col>

web/app/(commonLayout)/plugins/page.tsx

@@ -8,7 +8,7 @@ const PluginList = async () => {
   return (
     <PluginPage
       plugins={<PluginsPanel />}
-      marketplace={<Marketplace locale={locale} pluginTypeSwitchClassName='top-[60px]' searchBoxAutoAnimate={false} />}
+      marketplace={<Marketplace locale={locale} pluginTypeSwitchClassName='top-[60px]' searchBoxAutoAnimate={false} showSearchParams={false} />}
     />
   )
 }

web/app/components/app/configuration/config-var/config-select/index.tsx

@@ -1,12 +1,10 @@
 'use client'
 import type { FC } from 'react'
-import React from 'react'
+import React, { useState } from 'react'
+import { RiAddLine, RiDeleteBinLine, RiDraggable } from '@remixicon/react'
 import { useTranslation } from 'react-i18next'
-import { PlusIcon } from '@heroicons/react/24/outline'
 import { ReactSortable } from 'react-sortablejs'
-import RemoveIcon from '../../base/icons/remove-icon'
+import cn from '@/utils/classnames'
-
-import s from './style.module.css'
 
 export type Options = string[]
 export type IConfigSelectProps = {
@@ -19,6 +17,8 @@ const ConfigSelect: FC<IConfigSelectProps> = ({
   onChange,
 }) => {
   const { t } = useTranslation()
+  const [focusID, setFocusID] = useState<number | null>(null)
+  const [deletingID, setDeletingID] = useState<number | null>(null)
 
   const optionList = options.map((content, index) => {
     return ({
@@ -40,12 +40,15 @@ const ConfigSelect: FC<IConfigSelectProps> = ({
             animation={150}
           >
             {options.map((o, index) => (
-              <div className={`${s.inputWrap} relative`} key={index}>
+              <div
-                <div className='handle flex h-4 w-4 cursor-grab items-center justify-center'>
+                className={cn(
-                  <svg width="6" height="10" viewBox="0 0 6 10" fill="none" xmlns="http://www.w3.org/2000/svg">
+                  'group relative flex items-center rounded-lg border border-components-panel-border-subtle bg-components-panel-on-panel-item-bg pl-2.5 hover:bg-components-panel-on-panel-item-bg-hover',
-                    <path fillRule="evenodd" clipRule="evenodd" d="M1 2C1.55228 2 2 1.55228 2 1C2 0.447715 1.55228 0 1 0C0.447715 0 0 0.447715 0 1C0 1.55228 0.447715 2 1 2ZM1 6C1.55228 6 2 5.55228 2 5C2 4.44772 1.55228 4 1 4C0.447715 4 0 4.44772 0 5C0 5.55228 0.447715 6 1 6ZM6 1C6 1.55228 5.55228 2 5 2C4.44772 2 4 1.55228 4 1C4 0.447715 4.44772 0 5 0C5.55228 0 6 0.447715 6 1ZM5 6C5.55228 6 6 5.55228 6 5C6 4.44772 5.55228 4 5 4C4.44772 4 4 4.44772 4 5C4 5.55228 4.44772 6 5 6ZM2 9C2 9.55229 1.55228 10 1 10C0.447715 10 0 9.55229 0 9C0 8.44771 0.447715 8 1 8C1.55228 8 2 8.44771 2 9ZM5 10C5.55228 10 6 9.55229 6 9C6 8.44771 5.55228 8 5 8C4.44772 8 4 8.44771 4 9C4 9.55229 4.44772 10 5 10Z" fill="#98A2B3" />
+                  focusID === index && 'border-components-input-border-active bg-components-input-bg-active hover:border-components-input-border-active hover:bg-components-input-bg-active',
-                  </svg>
+                  deletingID === index && 'border-components-input-border-destructive bg-state-destructive-hover hover:border-components-input-border-destructive hover:bg-state-destructive-hover',
-                </div>
+                )}
+                key={index}
+              >
+                <RiDraggable className='handle h-4 w-4 cursor-grab text-text-quaternary' />
                 <input
                   key={index}
                   type="input"
@@ -59,14 +62,20 @@ const ConfigSelect: FC<IConfigSelectProps> = ({
                       return item
                     }))
                   }}
-                  className={'h-9 w-full grow cursor-pointer border-0 bg-transparent pl-1.5 pr-8 text-sm leading-9 text-gray-900 focus:outline-none'}
+                  className={'h-9 w-full grow cursor-pointer overflow-x-auto rounded-lg border-0 bg-transparent pl-1.5 pr-8 text-sm leading-9 text-text-secondary focus:outline-none'}
+                  onFocus={() => setFocusID(index)}
+                  onBlur={() => setFocusID(null)}
                 />
-                <RemoveIcon
+                <div
-                  className={`${s.deleteBtn} absolute right-1.5 top-1/2 h-6 w-6 translate-y-[-50%] cursor-pointer items-center justify-center rounded-md hover:bg-[#FEE4E2]`}
+                  className='absolute right-1.5 top-1/2 block translate-y-[-50%] cursor-pointer rounded-md p-1 text-text-tertiary hover:bg-state-destructive-hover hover:text-text-destructive'
                   onClick={() => {
                     onChange(options.filter((_, i) => index !== i))
                   }}
-                />
+                  onMouseEnter={() => setDeletingID(index)}
+                  onMouseLeave={() => setDeletingID(null)}
+                >
+                  <RiDeleteBinLine className='h-3.5 w-3.5' />
+                </div>
               </div>
             ))}
           </ReactSortable>
@@ -75,9 +84,9 @@ const ConfigSelect: FC<IConfigSelectProps> = ({
 
       <div
         onClick={() => { onChange([...options, '']) }}
-        className='flex h-9 cursor-pointer items-center gap-2 rounded-lg bg-gray-100 px-3  text-gray-400'>
+        className='mt-1 flex h-9 cursor-pointer items-center gap-2 rounded-lg bg-components-button-tertiary-bg px-3  text-components-button-tertiary-text hover:bg-components-button-tertiary-bg-hover'>
-        <PlusIcon width={16} height={16}></PlusIcon>
+        <RiAddLine className='h-4 w-4' />
-        <div className='text-[13px] text-gray-500'>{t('appDebug.variableConfig.addOption')}</div>
+        <div className='system-sm-medium text-[13px]'>{t('appDebug.variableConfig.addOption')}</div>
       </div>
     </div>
   )

web/app/components/app/configuration/config-var/config-select/style.module.css

@@ -1,21 +0,0 @@
-.inputWrap {
-  display: flex;
-  align-items: center;
-  border-radius: 8px;
-  border: 1px solid #EAECF0;
-  padding-left: 10px;
-  cursor: pointer;
-}
-
-.deleteBtn {
-  display: none;
-  display: flex;
-}
-
-.inputWrap:hover {
-  box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05);
-}
-
-.inputWrap:hover .deleteBtn {
-  display: flex;
-}

web/app/components/app/configuration/dataset-config/index.tsx

@@ -270,7 +270,7 @@ const DatasetConfig: FC = () => {
           handleMetadataModelChange={handleMetadataModelChange}
           handleMetadataCompletionParamsChange={handleMetadataCompletionParamsChange}
           isCommonVariable
-          availableCommonStringVars={promptVariablesToSelect.filter(item => item.type === MetadataFilteringVariableType.string)}
+          availableCommonStringVars={promptVariablesToSelect.filter(item => item.type === MetadataFilteringVariableType.string || item.type === MetadataFilteringVariableType.select)}
           availableCommonNumberVars={promptVariablesToSelect.filter(item => item.type === MetadataFilteringVariableType.number)}
         />
       </div>

web/app/components/app/configuration/dataset-config/params-config/config-content.tsx

@@ -25,6 +25,7 @@ import { useSelectedDatasetsMode } from '@/app/components/workflow/nodes/knowled
 import Switch from '@/app/components/base/switch'
 import Toast from '@/app/components/base/toast'
 import Divider from '@/app/components/base/divider'
+import { noop } from 'lodash-es'
 
 type Props = {
   datasetConfigs: DatasetConfigs
@@ -41,8 +42,8 @@ const ConfigContent: FC<Props> = ({
   onChange,
   isInWorkflow,
   singleRetrievalModelConfig: singleRetrievalConfig = {} as ModelConfig,
-  onSingleRetrievalModelChange = () => { },
+  onSingleRetrievalModelChange = noop,
-  onSingleRetrievalModelParamsChange = () => { },
+  onSingleRetrievalModelParamsChange = noop,
   selectedDatasets = [],
 }) => {
   const { t } = useTranslation()

web/app/components/app/configuration/index.tsx

@@ -197,9 +197,6 @@ const Configuration: FC = () => {
   const isOpenAI = modelConfig.provider === 'langgenius/openai/openai'
 
   const [collectionList, setCollectionList] = useState<Collection[]>([])
-  useEffect(() => {
-
-  }, [])
   const [datasetConfigs, doSetDatasetConfigs] = useState<DatasetConfigs>({
     retrieval_model: RETRIEVE_TYPE.multiWay,
     reranking_model: {

web/app/components/app/create-app-dialog/index.tsx

@@ -1,4 +1,6 @@
 'use client'
+import { useCallback } from 'react'
+import { useKeyPress } from 'ahooks'
 import AppList from './app-list'
 import FullScreenModal from '@/app/components/base/fullscreen-modal'
 
@@ -10,6 +12,13 @@ type CreateAppDialogProps = {
 }
 
 const CreateAppTemplateDialog = ({ show, onSuccess, onClose, onCreateFromBlank }: CreateAppDialogProps) => {
+  const handleEscKeyPress = useCallback(() => {
+    if (show)
+      onClose()
+  }, [show, onClose])
+
+  useKeyPress('esc', handleEscKeyPress)
+
   return (
     <FullScreenModal
       open={show}

web/app/components/app/create-app-modal/index.tsx

@@ -1,9 +1,9 @@
 'use client'
 
-import { useCallback, useRef, useState } from 'react'
+import { useCallback, useEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 
-import { useRouter } from 'next/navigation'
+import { useRouter, useSearchParams } from 'next/navigation'
 import { useContext, useContextSelector } from 'use-context-selector'
 import { RiArrowRightLine, RiCommandLine, RiCornerDownLeftLine, RiExchange2Fill } from '@remixicon/react'
 import Link from 'next/link'
@@ -18,6 +18,7 @@ import AppsContext, { useAppContext } from '@/context/app-context'
 import { useProviderContext } from '@/context/provider-context'
 import { ToastContext } from '@/app/components/base/toast'
 import type { AppMode } from '@/types/app'
+import { AppModes } from '@/types/app'
 import { createApp } from '@/service/apps'
 import Input from '@/app/components/base/input'
 import Textarea from '@/app/components/base/textarea'
@@ -53,6 +54,14 @@ function CreateApp({ onClose, onSuccess, onCreateFromTemplate }: CreateAppProps)
 
   const isCreatingRef = useRef(false)
 
+  const searchParams = useSearchParams()
+
+  useEffect(() => {
+    const category = searchParams.get('category')
+    if (category && AppModes.includes(category as AppMode))
+      setAppMode(category as AppMode)
+  }, [searchParams])
+
   const onCreate = useCallback(async () => {
     if (!appMode) {
       notify({ type: 'error', message: t('app.newApp.appTypeRequired') })
@@ -148,7 +157,6 @@ function CreateApp({ onClose, onSuccess, onCreateFromTemplate }: CreateAppProps)
               </div>
               <div className='flex flex-row gap-2'>
                 <AppTypeCard
-                  beta
                   active={appMode === 'advanced-chat'}
                   title={t('app.types.advanced')}
                   description={t('app.newApp.advancedShortDescription')}
@@ -159,7 +167,6 @@ function CreateApp({ onClose, onSuccess, onCreateFromTemplate }: CreateAppProps)
                     setAppMode('advanced-chat')
                   }} />
                 <AppTypeCard
-                  beta
                   active={appMode === 'workflow'}
                   title={t('app.types.workflow')}
                   description={t('app.newApp.workflowShortDescription')}
@@ -274,13 +281,12 @@ export default CreateAppModal
 
 type AppTypeCardProps = {
   icon: React.JSX.Element
-  beta?: boolean
   title: string
   description: string
   active: boolean
   onClick: () => void
 }
-function AppTypeCard({ icon, title, beta = false, description, active, onClick }: AppTypeCardProps) {
+function AppTypeCard({ icon, title, description, active, onClick }: AppTypeCardProps) {
   const { t } = useTranslation()
   return <div
     className={
@@ -292,9 +298,6 @@ function AppTypeCard({ icon, title, beta = false, description, active, onClick }
     }
     onClick={onClick}
   >
-    {beta && <div className='system-2xs-medium-uppercase absolute
-      right-3 top-3 min-w-[18px] rounded-[5px] border
-      border-divider-deep px-[5px] py-[3px] text-text-tertiary'>{t('common.menus.status')}</div>}
     {icon}
     <div className='system-sm-semibold mb-0.5 mt-2 text-text-secondary'>{title}</div>
     <div className='system-xs-regular text-text-tertiary'>{description}</div>

web/app/components/app/create-from-dsl-modal/index.tsx

@@ -5,7 +5,8 @@ import { useMemo, useRef, useState } from 'react'
 import { useRouter } from 'next/navigation'
 import { useContext } from 'use-context-selector'
 import { useTranslation } from 'react-i18next'
-import { RiCloseLine } from '@remixicon/react'
+import { RiCloseLine, RiCommandLine, RiCornerDownLeftLine } from '@remixicon/react'
+import { useDebounceFn, useKeyPress } from 'ahooks'
 import Uploader from './uploader'
 import Button from '@/app/components/base/button'
 import Input from '@/app/components/base/input'
@@ -143,6 +144,18 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS
     isCreatingRef.current = false
   }
 
+  const { run: handleCreateApp } = useDebounceFn(onCreate, { wait: 300 })
+
+  useKeyPress(['meta.enter', 'ctrl.enter'], () => {
+    if (show && !isAppsFull && ((currentTab === CreateFromDSLModalTab.FROM_FILE && currentFile) || (currentTab === CreateFromDSLModalTab.FROM_URL && dslUrlValue)))
+      handleCreateApp()
+  })
+
+  useKeyPress('esc', () => {
+    if (show && !showErrorModal)
+      onClose()
+  })
+
   const onDSLConfirm: MouseEventHandler = async () => {
     try {
       if (!importId)
@@ -266,7 +279,18 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS
         )}
         <div className='flex justify-end px-6 py-5'>
           <Button className='mr-2' onClick={onClose}>{t('app.newApp.Cancel')}</Button>
-          <Button disabled={buttonDisabled} variant="primary" onClick={onCreate}>{t('app.newApp.Create')}</Button>
+          <Button
+            disabled={buttonDisabled}
+            variant="primary"
+            onClick={handleCreateApp}
+            className="gap-1"
+          >
+            <span>{t('app.newApp.Create')}</span>
+            <div className='flex gap-0.5'>
+              <RiCommandLine size={14} className='system-kbd rounded-sm bg-components-kbd-bg-white p-0.5' />
+              <RiCornerDownLeftLine size={14} className='system-kbd rounded-sm bg-components-kbd-bg-white p-0.5' />
+            </div>
+          </Button>
         </div>
       </Modal>
       <Modal

web/app/components/base/audio-btn/audio.player.manager.ts

@@ -12,9 +12,6 @@ export class AudioPlayerManager {
   private audioPlayers: AudioPlayer | null = null
   private msgId: string | undefined
 
-  private constructor() {
-  }
-
   public static getInstance(): AudioPlayerManager {
     if (!AudioPlayerManager.instance) {
       AudioPlayerManager.instance = new AudioPlayerManager()
@@ -24,7 +21,7 @@ export class AudioPlayerManager {
     return AudioPlayerManager.instance
   }
 
-  public getAudioPlayer(url: string, isPublic: boolean, id: string | undefined, msgContent: string | null | undefined, voice: string | undefined, callback: ((event: string) => {}) | null): AudioPlayer {
+  public getAudioPlayer(url: string, isPublic: boolean, id: string | undefined, msgContent: string | null | undefined, voice: string | undefined, callback: ((event: string) => void) | null): AudioPlayer {
     if (this.msgId && this.msgId === id && this.audioPlayers) {
       this.audioPlayers.setCallback(callback)
       return this.audioPlayers

web/app/components/base/audio-btn/audio.ts

@@ -21,9 +21,9 @@ export default class AudioPlayer {
   isLoadData = false
   url: string
   isPublic: boolean
-  callback: ((event: string) => {}) | null
+  callback: ((event: string) => void) | null
 
-  constructor(streamUrl: string, isPublic: boolean, msgId: string | undefined, msgContent: string | null | undefined, voice: string | undefined, callback: ((event: string) => {}) | null) {
+  constructor(streamUrl: string, isPublic: boolean, msgId: string | undefined, msgContent: string | null | undefined, voice: string | undefined, callback: ((event: string) => void) | null) {
     this.audioContext = new AudioContext()
     this.msgId = msgId
     this.msgContent = msgContent
@@ -68,7 +68,7 @@ export default class AudioPlayer {
     })
   }
 
-  public setCallback(callback: ((event: string) => {}) | null) {
+  public setCallback(callback: ((event: string) => void) | null) {
     this.callback = callback
     if (callback) {
       this.audio.addEventListener('ended', () => {
@@ -211,10 +211,6 @@ export default class AudioPlayer {
     this.audioContext.suspend()
   }
 
-  private cancer() {
-
-  }
-
   private receiveAudioData(unit8Array: Uint8Array) {
     if (!unit8Array) {
       this.finishStream()

web/app/components/base/chat/chat-with-history/hooks.tsx

@@ -266,7 +266,7 @@ export const useChatWithHistory = (installedAppInfo?: InstalledApp) => {
 
   const currentConversationLatestInputs = useMemo(() => {
     if (!currentConversationId || !appChatListData?.data.length)
-      return {}
+      return newConversationInputsRef.current || {}
     return appChatListData.data.slice().pop().inputs || {}
   }, [appChatListData, currentConversationId])
   const [currentConversationInputs, setCurrentConversationInputs] = useState<Record<string, any>>(currentConversationLatestInputs || {})

web/app/components/base/chat/chat/hooks.ts

@@ -34,6 +34,7 @@ import {
   getProcessedFiles,
   getProcessedFilesFromResponse,
 } from '@/app/components/base/file-uploader/utils'
+import { noop } from 'lodash-es'
 
 type GetAbortController = (abortController: AbortController) => void
 type SendCallback = {
@@ -308,7 +309,7 @@ export const useChat = (
       else
         ttsUrl = `/apps/${params.appId}/text-to-audio`
     }
-    const player = AudioPlayerManager.getInstance().getAudioPlayer(ttsUrl, ttsIsPublic, uuidV4(), 'none', 'none', (_: any): any => { })
+    const player = AudioPlayerManager.getInstance().getAudioPlayer(ttsUrl, ttsIsPublic, uuidV4(), 'none', 'none', noop)
     ssePost(
       url,
       {

web/app/components/base/chat/embedded-chatbot/hooks.tsx

@@ -242,7 +242,7 @@ export const useEmbeddedChatbot = () => {
 
   const currentConversationLatestInputs = useMemo(() => {
     if (!currentConversationId || !appChatListData?.data.length)
-      return {}
+      return newConversationInputsRef.current || {}
     return appChatListData.data.slice().pop().inputs || {}
   }, [appChatListData, currentConversationId])
   const [currentConversationInputs, setCurrentConversationInputs] = useState<Record<string, any>>(currentConversationLatestInputs || {})

web/app/components/base/markdown.tsx

@@ -222,19 +222,21 @@ const Paragraph = (paragraph: any) => {
   const children_node = node.children
   if (children_node && children_node[0] && 'tagName' in children_node[0] && children_node[0].tagName === 'img') {
     return (
-      <>
+      <div className="markdown-img-wrapper">
         <ImageGallery srcs={[children_node[0].properties.src]} />
         {
-          Array.isArray(paragraph.children) ? <p>{paragraph.children.slice(1)}</p> : null
+          Array.isArray(paragraph.children) && paragraph.children.length > 1 && (
+            <div className="mt-2">{paragraph.children.slice(1)}</div>
+          )
         }
-      </>
+      </div>
     )
   }
   return <p>{paragraph.children}</p>
 }
 
 const Img = ({ src }: any) => {
-  return (<ImageGallery srcs={[src]} />)
+  return <div className="markdown-img-wrapper"><ImageGallery srcs={[src]} /></div>
 }
 
 const Link = ({ node, ...props }: any) => {

web/app/components/base/pagination/pagination.tsx

@@ -7,10 +7,11 @@ import type {
   IPaginationProps,
   PageButtonProps,
 } from './type'
+import { noop } from 'lodash-es'
 
 const defaultState: IPagination = {
   currentPage: 0,
-  setCurrentPage: () => {},
+  setCurrentPage: noop,
   truncableText: '...',
   truncableClassName: '',
   pages: [],

web/app/components/base/prompt-editor/utils.ts

@@ -296,7 +296,7 @@ export function $splitNodeContainingQuery(match: MenuTextMatch): TextNode | null
 }
 
 export function textToEditorState(text: string) {
-  const paragraph = text ? text.split('\n') : ['']
+  const paragraph = text && (typeof text === 'string') ? text.split('\n') : ['']
 
   return JSON.stringify({
     root: {

web/app/components/base/tag-input/index.tsx

@@ -70,7 +70,7 @@ const TagInput: FC<TagInputProps> = ({
   }
 
   return (
-    <div className={cn('flex flex-wrap', !isInWorkflow && 'min-w-[200px]', isSpecialMode ? 'rounded-lg bg-gray-100 pb-1 pl-1' : '')}>
+    <div className={cn('flex flex-wrap', !isInWorkflow && 'min-w-[200px]', isSpecialMode ? 'rounded-lg bg-components-input-bg-normal pb-1 pl-1' : '')}>
       {
         (items || []).map((item, index) => (
           <div

web/app/components/datasets/create/step-two/index.tsx

@@ -62,6 +62,7 @@ import Tooltip from '@/app/components/base/tooltip'
 import CustomDialog from '@/app/components/base/dialog'
 import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem'
 import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
+import { noop } from 'lodash-es'
 
 const TextLabel: FC<PropsWithChildren> = (props) => {
   return <label className='system-sm-semibold text-text-secondary'>{props.children}</label>
@@ -169,12 +170,11 @@ const StepTwo = ({
   const [rules, setRules] = useState<PreProcessingRule[]>([])
   const [defaultConfig, setDefaultConfig] = useState<Rules>()
   const hasSetIndexType = !!indexingType
-  const [indexType, setIndexType] = useState<IndexingType>(
+  const [indexType, setIndexType] = useState<IndexingType>(() => {
-    (indexingType
+    if (hasSetIndexType)
-      || isAPIKeySet)
+      return indexingType
-      ? IndexingType.QUALIFIED
+    return isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL
-      : IndexingType.ECONOMICAL,
+  })
-  )
 
   const [previewFile, setPreviewFile] = useState<DocumentItem>(
     (datasetId && documentDetail)
@@ -421,6 +421,13 @@ const StepTwo = ({
     }
     else { // create
       const indexMethod = getIndexing_technique()
+      if (indexMethod === IndexingType.QUALIFIED && (!embeddingModel.model || !embeddingModel.provider)) {
+        Toast.notify({
+          type: 'error',
+          message: t('appDebug.datasetConfig.embeddingModelRequired'),
+        })
+        return
+      }
       if (
         !isReRankModelSelected({
           rerankModelList,
@@ -568,7 +575,6 @@ const StepTwo = ({
     // get indexing type by props
     if (indexingType)
       setIndexType(indexingType as IndexingType)
-
     else
       setIndexType(isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL)
   }, [isAPIKeySet, indexingType, datasetId])
@@ -848,9 +854,8 @@ const StepTwo = ({
               description={t('datasetCreation.stepTwo.qualifiedTip')}
               icon={<Image src={indexMethodIcon.high_quality} alt='' />}
               isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED}
-              disabled={!isAPIKeySet || hasSetIndexType}
+              disabled={hasSetIndexType}
               onSwitched={() => {
-                if (isAPIKeySet)
                 setIndexType(IndexingType.QUALIFIED)
               }}
             />
@@ -894,10 +899,9 @@ const StepTwo = ({
                     description={t('datasetCreation.stepTwo.economicalTip')}
                     icon={<Image src={indexMethodIcon.economical} alt='' />}
                     isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL}
-                    disabled={!isAPIKeySet || hasSetIndexType || docForm !== ChunkingMode.text}
+                    disabled={hasSetIndexType || docForm !== ChunkingMode.text}
                     ref={economyDomRef}
                     onSwitched={() => {
-                      if (isAPIKeySet && docForm === ChunkingMode.text)
                       setIndexType(IndexingType.ECONOMICAL)
                     }}
                   />
@@ -1007,7 +1011,7 @@ const StepTwo = ({
             </div>
           )}
       </div>
-      <FloatRightContainer isMobile={isMobile} isOpen={true} onClose={() => { }} footer={null}>
+      <FloatRightContainer isMobile={isMobile} isOpen={true} onClose={noop} footer={null}>
         <PreviewContainer
           header={<PreviewHeader
             title={t('datasetCreation.stepTwo.preview')}

web/app/components/datasets/documents/list.tsx

@@ -46,6 +46,7 @@ import { useDocumentArchive, useDocumentDelete, useDocumentDisable, useDocumentE
 import { extensionToFileType } from '@/app/components/datasets/hit-testing/utils/extension-to-file-type'
 import useBatchEditDocumentMetadata from '../metadata/hooks/use-batch-edit-document-metadata'
 import EditMetadataBatchModal from '@/app/components/datasets/metadata/edit-metadata-batch/modal'
+import { noop } from 'lodash-es'
 
 export const useIndexStatus = () => {
   const { t } = useTranslation()
@@ -265,7 +266,7 @@ export const OperationAction: FC<{
 
   return <div className='flex items-center' onClick={e => e.stopPropagation()}>
     {isListScene && !embeddingAvailable && (
-      <Switch defaultValue={false} onChange={() => { }} disabled={true} size='md' />
+      <Switch defaultValue={false} onChange={noop} disabled={true} size='md' />
     )}
     {isListScene && embeddingAvailable && (
       <>
@@ -276,7 +277,7 @@ export const OperationAction: FC<{
             needsDelay
           >
             <div>
-              <Switch defaultValue={false} onChange={() => { }} disabled={true} size='md' />
+              <Switch defaultValue={false} onChange={noop} disabled={true} size='md' />
             </div>
           </Tooltip>
           : <Switch defaultValue={enabled} onChange={v => handleSwitch(v ? 'enable' : 'disable')} size='md' />

web/app/components/explore/create-app-modal/index.tsx

@@ -1,7 +1,8 @@
 'use client'
-import React, { useState } from 'react'
+import React, { useCallback, useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import { RiCloseLine } from '@remixicon/react'
+import { RiCloseLine, RiCommandLine, RiCornerDownLeftLine } from '@remixicon/react'
+import { useDebounceFn, useKeyPress } from 'ahooks'
 import AppIconPicker from '../../base/app-icon-picker'
 import Modal from '@/app/components/base/modal'
 import Button from '@/app/components/base/button'
@@ -13,6 +14,7 @@ import AppIcon from '@/app/components/base/app-icon'
 import { useProviderContext } from '@/context/provider-context'
 import AppsFull from '@/app/components/billing/apps-full-in-dialog'
 import type { AppIconType } from '@/types/app'
+import { noop } from 'lodash-es'
 
 export type CreateAppModalProps = {
   show: boolean
@@ -65,7 +67,7 @@ const CreateAppModal = ({
   const { plan, enableBilling } = useProviderContext()
   const isAppsFull = (enableBilling && plan.usage.buildApps >= plan.total.buildApps)
 
-  const submit = () => {
+  const submit = useCallback(() => {
     if (!name.trim()) {
       Toast.notify({ type: 'error', message: t('explore.appCustomize.nameRequired') })
       return
@@ -79,13 +81,25 @@ const CreateAppModal = ({
       use_icon_as_answer_icon: useIconAsAnswerIcon,
     })
     onHide()
-  }
+  }, [name, appIcon, description, useIconAsAnswerIcon, onConfirm, onHide, t])
+
+  const { run: handleSubmit } = useDebounceFn(submit, { wait: 300 })
+
+  useKeyPress(['meta.enter', 'ctrl.enter'], () => {
+    if (show && !(!isEditModal && isAppsFull) && name.trim())
+      handleSubmit()
+  })
+
+  useKeyPress('esc', () => {
+    if (show)
+      onHide()
+  })
 
   return (
     <>
       <Modal
         isShow={show}
-        onClose={() => {}}
+        onClose={noop}
         className='relative !max-w-[480px] px-8'
       >
         <div className='absolute right-4 top-4 cursor-pointer p-2' onClick={onHide}>
@@ -145,7 +159,18 @@ const CreateAppModal = ({
           {!isEditModal && isAppsFull && <AppsFull className='mt-4' loc='app-explore-create' />}
         </div>
         <div className='flex flex-row-reverse'>
-          <Button disabled={!isEditModal && isAppsFull} className='ml-2 w-24' variant='primary' onClick={submit}>{!isEditModal ? t('common.operation.create') : t('common.operation.save')}</Button>
+          <Button
+            disabled={(!isEditModal && isAppsFull) || !name.trim()}
+            className='ml-2 w-24 gap-1'
+            variant='primary'
+            onClick={handleSubmit}
+          >
+            <span>{!isEditModal ? t('common.operation.create') : t('common.operation.save')}</span>
+            <div className='flex gap-0.5'>
+              <RiCommandLine size={14} className='system-kbd rounded-sm bg-components-kbd-bg-white p-0.5' />
+              <RiCornerDownLeftLine size={14} className='system-kbd rounded-sm bg-components-kbd-bg-white p-0.5' />
+            </div>
+          </Button>
           <Button className='w-24' onClick={onHide}>{t('common.operation.cancel')}</Button>
         </div>
       </Modal>

web/app/components/header/account-setting/data-source-page/panel/config-item.tsx

@@ -10,6 +10,7 @@ import Operate from '../data-source-notion/operate'
 import { DataSourceType } from './types'
 import s from './style.module.css'
 import cn from '@/utils/classnames'
+import { noop } from 'lodash-es'
 
 export type ConfigItemType = {
   id: string
@@ -41,7 +42,7 @@ const ConfigItem: FC<Props> = ({
   const { t } = useTranslation()
   const isNotion = type === DataSourceType.notion
   const isWebsite = type === DataSourceType.website
-  const onChangeAuthorizedPage = notionActions?.onChangeAuthorizedPage || function () { }
+  const onChangeAuthorizedPage = notionActions?.onChangeAuthorizedPage || noop
 
   return (
     <div className={cn(s['workspace-item'], 'mb-1 flex items-center rounded-lg bg-components-panel-on-panel-item-bg py-1 pr-1')} key={payload.id}>

web/app/components/header/account-setting/model-provider-page/hooks.spec.ts

@@ -0,0 +1,90 @@
+import { renderHook } from '@testing-library/react'
+import { useLanguage } from './hooks'
+import { useContext } from 'use-context-selector'
+import { after } from 'node:test'
+
+jest.mock('swr', () => ({
+  __esModule: true,
+  default: jest.fn(), // mock useSWR
+  useSWRConfig: jest.fn(),
+}))
+
+// mock use-context-selector
+jest.mock('use-context-selector', () => ({
+  useContext: jest.fn(),
+}))
+
+// mock service/common functions
+jest.mock('@/service/common', () => ({
+  fetchDefaultModal: jest.fn(),
+  fetchModelList: jest.fn(),
+  fetchModelProviderCredentials: jest.fn(),
+  fetchModelProviders: jest.fn(),
+  getPayUrl: jest.fn(),
+}))
+
+// mock context hooks
+jest.mock('@/context/i18n', () => ({
+  __esModule: true,
+  default: jest.fn(),
+}))
+
+jest.mock('@/context/provider-context', () => ({
+  useProviderContext: jest.fn(),
+}))
+
+jest.mock('@/context/modal-context', () => ({
+  useModalContextSelector: jest.fn(),
+}))
+
+jest.mock('@/context/event-emitter', () => ({
+  useEventEmitterContextContext: jest.fn(),
+}))
+
+// mock plugins
+jest.mock('@/app/components/plugins/marketplace/hooks', () => ({
+  useMarketplacePlugins: jest.fn(),
+}))
+
+jest.mock('@/app/components/plugins/marketplace/utils', () => ({
+  getMarketplacePluginsByCollectionId: jest.fn(),
+}))
+
+jest.mock('./provider-added-card', () => {
+  // eslint-disable-next-line no-labels, ts/no-unused-expressions
+  UPDATE_MODEL_PROVIDER_CUSTOM_MODEL_LIST: []
+})
+
+after(() => {
+  jest.resetModules()
+  jest.clearAllMocks()
+})
+
+describe('useLanguage', () => {
+  it('should replace hyphen with underscore in locale', () => {
+    (useContext as jest.Mock).mockReturnValue({
+      locale: 'en-US',
+    })
+    const { result } = renderHook(() => useLanguage())
+    expect(result.current).toBe('en_US')
+  })
+
+  it('should return locale as is if no hyphen exists', () => {
+    (useContext as jest.Mock).mockReturnValue({
+      locale: 'enUS',
+    })
+
+    const { result } = renderHook(() => useLanguage())
+    expect(result.current).toBe('enUS')
+  })
+
+  it('should handle multiple hyphens', () => {
+    // Mock the I18n context return value
+    (useContext as jest.Mock).mockReturnValue({
+      locale: 'zh-Hans-CN',
+    })
+
+    const { result } = renderHook(() => useLanguage())
+    expect(result.current).toBe('zh_Hans-CN')
+  })
+})

web/app/components/header/account-setting/model-provider-page/model-icon/index.tsx

@@ -7,7 +7,7 @@ import { useLanguage } from '../hooks'
 import { Group } from '@/app/components/base/icons/src/vender/other'
 import { OpenaiBlue, OpenaiViolet } from '@/app/components/base/icons/src/public/llm'
 import cn from '@/utils/classnames'
-import { renderI18nObject } from '@/hooks/use-i18n'
+import { renderI18nObject } from '@/i18n'
 
 type ModelIconProps = {
   provider?: Model | ModelProvider

web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx

@@ -270,8 +270,7 @@ const ModelModal: FC<ModelModalProps> = ({
   }
 
   const renderTitlePrefix = () => {
-    const prefix = configurateMethod === ConfigurationMethodEnum.customizableModel ? t('common.operation.add') : t('common.operation.setup')
+    const prefix = isEditMode ? t('common.operation.setup') : t('common.operation.add')
-
     return `${prefix} ${provider.label[language] || provider.label.en_US}`
   }
 

web/app/components/header/account-setting/model-provider-page/model-parameter-modal/trigger.tsx

@@ -47,7 +47,7 @@ const Trigger: FC<TriggerProps> = ({
         'relative flex h-8 cursor-pointer items-center rounded-lg  px-2',
         !isInWorkflow && 'border ring-inset hover:ring-[0.5px]',
         !isInWorkflow && (disabled ? 'border-text-warning bg-state-warning-hover ring-text-warning' : 'border-util-colors-indigo-indigo-600 bg-state-accent-hover ring-util-colors-indigo-indigo-600'),
-        isInWorkflow && 'border border-workflow-block-parma-bg bg-workflow-block-parma-bg pr-[30px]  hover:border-gray-200',
+        isInWorkflow && 'border border-workflow-block-parma-bg bg-workflow-block-parma-bg pr-[30px]  hover:border-components-input-border-active',
       )}
     >
       {

web/app/components/header/account-setting/model-provider-page/provider-icon/index.tsx

@@ -3,7 +3,7 @@ import type { ModelProvider } from '../declarations'
 import { useLanguage } from '../hooks'
 import { Openai } from '@/app/components/base/icons/src/vender/other'
 import { AnthropicDark, AnthropicLight } from '@/app/components/base/icons/src/public/llm'
-import { renderI18nObject } from '@/hooks/use-i18n'
+import { renderI18nObject } from '@/i18n'
 import { Theme } from '@/types/app'
 import cn from '@/utils/classnames'
 import useTheme from '@/hooks/use-theme'

web/app/components/plugins/card/index.tsx

@@ -11,7 +11,7 @@ import cn from '@/utils/classnames'
 import { useGetLanguage } from '@/context/i18n'
 import { getLanguage } from '@/i18n/language'
 import { useSingleCategories } from '../hooks'
-import { renderI18nObject } from '@/hooks/use-i18n'
+import { renderI18nObject } from '@/i18n'
 import { useMixedTranslation } from '@/app/components/plugins/marketplace/hooks'
 import Partner from '../base/badges/partner'
 import Verified from '../base/badges/verified'

web/app/components/plugins/hooks.ts

@@ -92,3 +92,17 @@ export const useSingleCategories = (translateFromOut?: TFunction) => {
     categoriesMap,
   }
 }
+
+export const PLUGIN_PAGE_TABS_MAP = {
+  plugins: 'plugins',
+  marketplace: 'discover',
+}
+
+export const usePluginPageTabs = () => {
+  const { t } = useTranslation()
+  const tabs = [
+    { value: PLUGIN_PAGE_TABS_MAP.plugins, text: t('common.menus.plugins') },
+    { value: PLUGIN_PAGE_TABS_MAP.marketplace, text: t('common.menus.exploreMarketplace') },
+  ]
+  return tabs
+}

web/app/components/plugins/marketplace/context.tsx

@@ -35,9 +35,10 @@ import {
 import {
   getMarketplaceListCondition,
   getMarketplaceListFilterType,
+  updateSearchParams,
 } from './utils'
 import { useInstalledPluginList } from '@/service/use-plugins'
-import { noop } from 'lodash-es'
+import { debounce, noop } from 'lodash-es'
 
 export type MarketplaceContextValue = {
   intersected: boolean
@@ -96,6 +97,7 @@ type MarketplaceContextProviderProps = {
   searchParams?: SearchParams
   shouldExclude?: boolean
   scrollContainerId?: string
+  showSearchParams?: boolean
 }
 
 export function useMarketplaceContext(selector: (value: MarketplaceContextValue) => any) {
@@ -107,6 +109,7 @@ export const MarketplaceContextProvider = ({
   searchParams,
   shouldExclude,
   scrollContainerId,
+  showSearchParams,
 }: MarketplaceContextProviderProps) => {
   const { data, isSuccess } = useInstalledPluginList(!shouldExclude)
   const exclude = useMemo(() => {
@@ -159,7 +162,10 @@ export const MarketplaceContextProvider = ({
         type: getMarketplaceListFilterType(activePluginTypeRef.current),
         page: pageRef.current,
       })
-      history.pushState({}, '', `/${searchParams?.language ? `?language=${searchParams?.language}` : ''}`)
+      const url = new URL(window.location.href)
+      if (searchParams?.language)
+        url.searchParams.set('language', searchParams?.language)
+      history.replaceState({}, '', url)
     }
     else {
       if (shouldExclude && isSuccess) {
@@ -182,7 +188,31 @@ export const MarketplaceContextProvider = ({
     resetPlugins()
   }, [exclude, queryMarketplaceCollectionsAndPlugins, resetPlugins])
 
+  const debouncedUpdateSearchParams = useMemo(() => debounce(() => {
+    updateSearchParams({
+      query: searchPluginTextRef.current,
+      category: activePluginTypeRef.current,
+      tags: filterPluginTagsRef.current,
+    })
+  }, 500), [])
+
+  const handleUpdateSearchParams = useCallback((debounced?: boolean) => {
+    if (!showSearchParams)
+      return
+    if (debounced) {
+      debouncedUpdateSearchParams()
+    }
+    else {
+      updateSearchParams({
+        query: searchPluginTextRef.current,
+        category: activePluginTypeRef.current,
+        tags: filterPluginTagsRef.current,
+      })
+    }
+  }, [debouncedUpdateSearchParams, showSearchParams])
+
   const handleQueryPlugins = useCallback((debounced?: boolean) => {
+    handleUpdateSearchParams(debounced)
     if (debounced) {
       queryPluginsWithDebounced({
         query: searchPluginTextRef.current,
@@ -207,17 +237,18 @@ export const MarketplaceContextProvider = ({
         page: pageRef.current,
       })
     }
-  }, [exclude, queryPluginsWithDebounced, queryPlugins])
+  }, [exclude, queryPluginsWithDebounced, queryPlugins, handleUpdateSearchParams])
 
   const handleQuery = useCallback((debounced?: boolean) => {
     if (!searchPluginTextRef.current && !filterPluginTagsRef.current.length) {
+      handleUpdateSearchParams(debounced)
       cancelQueryPluginsWithDebounced()
       handleQueryMarketplaceCollectionsAndPlugins()
       return
     }
 
     handleQueryPlugins(debounced)
-  }, [handleQueryMarketplaceCollectionsAndPlugins, handleQueryPlugins, cancelQueryPluginsWithDebounced])
+  }, [handleQueryMarketplaceCollectionsAndPlugins, handleQueryPlugins, cancelQueryPluginsWithDebounced, handleUpdateSearchParams])
 
   const handleSearchPluginTextChange = useCallback((text: string) => {
     setSearchPluginText(text)
@@ -242,11 +273,9 @@ export const MarketplaceContextProvider = ({
     activePluginTypeRef.current = type
     setPage(1)
     pageRef.current = 1
-  }, [])
 
-  useEffect(() => {
     handleQuery()
-  }, [activePluginType, handleQuery])
+  }, [handleQuery])
 
   const handleSortChange = useCallback((sort: PluginsSort) => {
     setSort(sort)

web/app/components/plugins/marketplace/index.tsx

@@ -17,6 +17,7 @@ type MarketplaceProps = {
   pluginTypeSwitchClassName?: string
   intersectionContainerId?: string
   scrollContainerId?: string
+  showSearchParams?: boolean
 }
 const Marketplace = async ({
   locale,
@@ -27,6 +28,7 @@ const Marketplace = async ({
   pluginTypeSwitchClassName,
   intersectionContainerId,
   scrollContainerId,
+  showSearchParams = true,
 }: MarketplaceProps) => {
   let marketplaceCollections: any = []
   let marketplaceCollectionPluginsMap = {}
@@ -42,6 +44,7 @@ const Marketplace = async ({
         searchParams={searchParams}
         shouldExclude={shouldExclude}
         scrollContainerId={scrollContainerId}
+        showSearchParams={showSearchParams}
       >
         <Description locale={locale} />
         <IntersectionLine intersectionContainerId={intersectionContainerId} />
@@ -53,6 +56,7 @@ const Marketplace = async ({
           locale={locale}
           className={pluginTypeSwitchClassName}
           searchBoxAutoAnimate={searchBoxAutoAnimate}
+          showSearchParams={showSearchParams}
         />
         <ListWrapper
           locale={locale}

web/app/components/plugins/marketplace/plugin-type-switch.tsx

@@ -13,6 +13,7 @@ import {
   useSearchBoxAutoAnimate,
 } from './hooks'
 import cn from '@/utils/classnames'
+import { useCallback, useEffect } from 'react'
 
 export const PLUGIN_TYPE_SEARCH_MAP = {
   all: 'all',
@@ -26,11 +27,13 @@ type PluginTypeSwitchProps = {
   locale?: string
   className?: string
   searchBoxAutoAnimate?: boolean
+  showSearchParams?: boolean
 }
 const PluginTypeSwitch = ({
   locale,
   className,
   searchBoxAutoAnimate,
+  showSearchParams,
 }: PluginTypeSwitchProps) => {
   const { t } = useMixedTranslation(locale)
   const activePluginType = useMarketplaceContext(s => s.activePluginType)
@@ -70,6 +73,23 @@ const PluginTypeSwitch = ({
     },
   ]
 
+  const handlePopState = useCallback(() => {
+    if (!showSearchParams)
+      return
+    const url = new URL(window.location.href)
+    const category = url.searchParams.get('category') || PLUGIN_TYPE_SEARCH_MAP.all
+    handleActivePluginTypeChange(category)
+  }, [showSearchParams, handleActivePluginTypeChange])
+
+  useEffect(() => {
+    window.addEventListener('popstate', () => {
+      handlePopState()
+    })
+    return () => {
+      window.removeEventListener('popstate', handlePopState)
+    }
+  }, [handlePopState])
+
   return (
     <div className={cn(
       'flex shrink-0 items-center justify-center space-x-2 bg-background-body py-3',

web/app/components/plugins/marketplace/utils.ts

@@ -4,6 +4,7 @@ import { PluginType } from '@/app/components/plugins/types'
 import type {
   CollectionsAndPluginsSearchParams,
   MarketplaceCollection,
+  PluginsSearchParams,
 } from '@/app/components/plugins/marketplace/types'
 import {
   MARKETPLACE_API_PREFIX,
@@ -125,3 +126,22 @@ export const getMarketplaceListFilterType = (category: string) => {
 
   return 'plugin'
 }
+
+export const updateSearchParams = (pluginsSearchParams: PluginsSearchParams) => {
+  const { query, category, tags } = pluginsSearchParams
+  const url = new URL(window.location.href)
+  const categoryChanged = url.searchParams.get('category') !== category
+  if (query)
+    url.searchParams.set('q', query)
+  else
+    url.searchParams.delete('q')
+  if (category)
+    url.searchParams.set('category', category)
+  else
+    url.searchParams.delete('category')
+  if (tags && tags.length)
+    url.searchParams.set('tags', tags.join(','))
+  else
+    url.searchParams.delete('tags')
+  history[`${categoryChanged ? 'pushState' : 'replaceState'}`]({}, '', url)
+}

web/app/components/plugins/plugin-page/context.tsx

@@ -12,9 +12,9 @@ import {
 } from 'use-context-selector'
 import { useSelector as useAppContextSelector } from '@/context/app-context'
 import type { FilterState } from './filter-management'
-import { useTranslation } from 'react-i18next'
 import { useTabSearchParams } from '@/hooks/use-tab-searchparams'
 import { noop } from 'lodash-es'
+import { PLUGIN_PAGE_TABS_MAP, usePluginPageTabs } from '../hooks'
 
 export type PluginPageContextValue = {
   containerRef: React.RefObject<HTMLDivElement>
@@ -53,7 +53,6 @@ export function usePluginPageContext(selector: (value: PluginPageContextValue) =
 export const PluginPageContextProvider = ({
   children,
 }: PluginPageContextProviderProps) => {
-  const { t } = useTranslation()
   const containerRef = useRef<HTMLDivElement>(null)
   const [filters, setFilters] = useState<FilterState>({
     categories: [],
@@ -63,16 +62,10 @@ export const PluginPageContextProvider = ({
   const [currentPluginID, setCurrentPluginID] = useState<string | undefined>()
 
   const { enable_marketplace } = useAppContextSelector(s => s.systemFeatures)
+  const tabs = usePluginPageTabs()
   const options = useMemo(() => {
-    return [
+    return enable_marketplace ? tabs : tabs.filter(tab => tab.value !== PLUGIN_PAGE_TABS_MAP.marketplace)
-      { value: 'plugins', text: t('common.menus.plugins') },
+  }, [tabs, enable_marketplace])
-      ...(
-        enable_marketplace
-          ? [{ value: 'discover', text: t('common.menus.exploreMarketplace') }]
-          : []
-      ),
-    ]
-  }, [t, enable_marketplace])
   const [activeTab, setActiveTab] = useTabSearchParams({
     defaultTab: options[0].value,
   })

web/app/components/plugins/plugin-page/index.tsx

@@ -40,6 +40,8 @@ import { SUPPORT_INSTALL_LOCAL_FILE_EXTENSIONS } from '@/config'
 import { LanguagesSupported } from '@/i18n/language'
 import I18n from '@/context/i18n'
 import { noop } from 'lodash-es'
+import { PLUGIN_TYPE_SEARCH_MAP } from '../marketplace/plugin-type-switch'
+import { PLUGIN_PAGE_TABS_MAP } from '../hooks'
 
 const PACKAGE_IDS_KEY = 'package-ids'
 const BUNDLE_INFO_KEY = 'bundle-info'
@@ -136,40 +138,45 @@ const PluginPage = ({
   const setActiveTab = usePluginPageContext(v => v.setActiveTab)
   const { enable_marketplace } = useAppContextSelector(s => s.systemFeatures)
 
+  const isPluginsTab = useMemo(() => activeTab === PLUGIN_PAGE_TABS_MAP.plugins, [activeTab])
+  const isExploringMarketplace = useMemo(() => {
+    const values = Object.values(PLUGIN_TYPE_SEARCH_MAP)
+    return activeTab === PLUGIN_PAGE_TABS_MAP.marketplace || values.includes(activeTab)
+  }, [activeTab])
+
   const uploaderProps = useUploader({
     onFileChange: setCurrentFile,
     containerRef,
-    enabled: activeTab === 'plugins',
+    enabled: isPluginsTab,
   })
 
   const { dragging, fileUploader, fileChangeHandle, removeFile } = uploaderProps
-
   return (
     <div
       id='marketplace-container'
       ref={containerRef}
       style={{ scrollbarGutter: 'stable' }}
-      className={cn('relative flex grow flex-col overflow-y-auto border-t border-divider-subtle', activeTab === 'plugins'
+      className={cn('relative flex grow flex-col overflow-y-auto border-t border-divider-subtle', isPluginsTab
         ? 'rounded-t-xl bg-components-panel-bg'
         : 'bg-background-body',
       )}
     >
       <div
         className={cn(
-          'sticky top-0 z-10 flex min-h-[60px] items-center gap-1 self-stretch bg-components-panel-bg px-12 pb-2 pt-4', activeTab === 'discover' && 'bg-background-body',
+          'sticky top-0 z-10 flex min-h-[60px] items-center gap-1 self-stretch bg-components-panel-bg px-12 pb-2 pt-4', isExploringMarketplace && 'bg-background-body',
         )}
       >
         <div className='flex w-full items-center justify-between'>
           <div className='flex-1'>
             <TabSlider
-              value={activeTab}
+              value={isPluginsTab ? PLUGIN_PAGE_TABS_MAP.plugins : PLUGIN_PAGE_TABS_MAP.marketplace}
               onChange={setActiveTab}
               options={options}
             />
           </div>
           <div className='flex shrink-0 items-center gap-1'>
             {
-              activeTab === 'discover' && (
+              isExploringMarketplace && (
                 <>
                   <Link
                     href={`https://docs.dify.ai/${locale === LanguagesSupported[1] ? 'v/zh-hans/' : ''}plugins/publish-plugins/publish-to-dify-marketplace`}
@@ -215,7 +222,7 @@ const PluginPage = ({
           </div>
         </div>
       </div>
-      {activeTab === 'plugins' && (
+      {isPluginsTab && (
         <>
           {plugins}
           {dragging && (
@@ -246,7 +253,7 @@ const PluginPage = ({
         </>
       )}
       {
-        activeTab === 'discover' && enable_marketplace && marketplace
+        isExploringMarketplace && enable_marketplace && marketplace
       }
 
       {showPluginSettingModal && (

web/app/components/plugins/plugin-page/plugins-panel.tsx

@@ -3,17 +3,23 @@ import { useMemo } from 'react'
 import type { FilterState } from './filter-management'
 import FilterManagement from './filter-management'
 import List from './list'
-import { useInstalledPluginList, useInvalidateInstalledPluginList } from '@/service/use-plugins'
+import { useInstalledLatestVersion, useInstalledPluginList, useInvalidateInstalledPluginList } from '@/service/use-plugins'
 import PluginDetailPanel from '@/app/components/plugins/plugin-detail-panel'
 import { usePluginPageContext } from './context'
 import { useDebounceFn } from 'ahooks'
 import Empty from './empty'
 import Loading from '../../base/loading'
+import { PluginSource } from '../types'
 
 const PluginsPanel = () => {
   const filters = usePluginPageContext(v => v.filters) as FilterState
   const setFilters = usePluginPageContext(v => v.setFilters)
   const { data: pluginList, isLoading: isPluginListLoading } = useInstalledPluginList()
+  const { data: installedLatestVersion } = useInstalledLatestVersion(
+    pluginList?.plugins
+      .filter(plugin => plugin.source === PluginSource.marketplace)
+      .map(plugin => plugin.plugin_id) ?? [],
+  )
   const invalidateInstalledPluginList = useInvalidateInstalledPluginList()
   const currentPluginID = usePluginPageContext(v => v.currentPluginID)
   const setCurrentPluginID = usePluginPageContext(v => v.setCurrentPluginID)
@@ -22,9 +28,17 @@ const PluginsPanel = () => {
     setFilters(filters)
   }, { wait: 500 })
 
+  const pluginListWithLatestVersion = useMemo(() => {
+    return pluginList?.plugins.map(plugin => ({
+      ...plugin,
+      latest_version: installedLatestVersion?.versions[plugin.plugin_id]?.version ?? '',
+      latest_unique_identifier: installedLatestVersion?.versions[plugin.plugin_id]?.unique_identifier ?? '',
+    })) || []
+  }, [pluginList, installedLatestVersion])
+
   const filteredList = useMemo(() => {
     const { categories, searchQuery, tags } = filters
-    const filteredList = pluginList?.plugins.filter((plugin) => {
+    const filteredList = pluginListWithLatestVersion.filter((plugin) => {
       return (
         (categories.length === 0 || categories.includes(plugin.declaration.category))
         && (tags.length === 0 || tags.some(tag => plugin.declaration.tags.includes(tag)))
@@ -32,12 +46,12 @@ const PluginsPanel = () => {
       )
     })
     return filteredList
-  }, [pluginList, filters])
+  }, [pluginListWithLatestVersion, filters])
 
   const currentPluginDetail = useMemo(() => {
-    const detail = pluginList?.plugins.find(plugin => plugin.plugin_id === currentPluginID)
+    const detail = pluginListWithLatestVersion.find(plugin => plugin.plugin_id === currentPluginID)
     return detail
-  }, [currentPluginID, pluginList?.plugins])
+  }, [currentPluginID, pluginListWithLatestVersion])
 
   const handleHide = () => setCurrentPluginID(undefined)
 

web/app/components/plugins/types.ts

@@ -318,6 +318,15 @@ export type InstalledPluginListResponse = {
   plugins: PluginDetail[]
 }
 
+export type InstalledLatestVersionResponse = {
+  versions: {
+    [plugin_id: string]: {
+      unique_identifier: string
+      version: string
+    } | null
+  }
+}
+
 export type UninstallPluginResponse = {
   success: boolean
 }

web/app/components/share/text-generation/run-once/index.tsx

@@ -1,4 +1,4 @@
-import type { FC, FormEvent } from 'react'
+import type { ChangeEvent, FC, FormEvent } from 'react'
 import { useEffect } from 'react'
 import React, { useCallback } from 'react'
 import { useTranslation } from 'react-i18next'
@@ -66,17 +66,15 @@ const RunOnce: FC<IRunOnceProps> = ({
       newInputs[item.key] = ''
     })
     onInputsChange(newInputs)
-  }, [promptConfig.prompt_variables])
+  }, [promptConfig.prompt_variables, onInputsChange])
-
-  if (inputs === null || inputs === undefined || Object.keys(inputs).length === 0)
-    return null
 
   return (
     <div className="">
       <section>
         {/* input form */}
         <form onSubmit={onSubmit}>
-          {promptConfig.prompt_variables.map(item => (
+          {(inputs === null || inputs === undefined || Object.keys(inputs).length === 0) ? null
+            : promptConfig.prompt_variables.map(item => (
               <div className='mt-4 w-full' key={item.key}>
                 <label className='system-md-semibold flex h-6 items-center text-text-secondary'>{item.name}</label>
                 <div className='mt-1'>
@@ -94,7 +92,7 @@ const RunOnce: FC<IRunOnceProps> = ({
                       type="text"
                       placeholder={`${item.name}${!item.required ? `(${t('appDebug.variableTable.optional')})` : ''}`}
                       value={inputs[item.key]}
-                    onChange={(e) => { handleInputsChange({ ...inputsRef.current, [item.key]: e.target.value }) }}
+                      onChange={(e: ChangeEvent<HTMLInputElement>) => { handleInputsChange({ ...inputsRef.current, [item.key]: e.target.value }) }}
                       maxLength={item.max_length || DEFAULT_VALUE_MAX_LEN}
                     />
                   )}
@@ -103,7 +101,7 @@ const RunOnce: FC<IRunOnceProps> = ({
                       className='h-[104px] sm:text-xs'
                       placeholder={`${item.name}${!item.required ? `(${t('appDebug.variableTable.optional')})` : ''}`}
                       value={inputs[item.key]}
-                    onChange={(e) => { handleInputsChange({ ...inputsRef.current, [item.key]: e.target.value }) }}
+                      onChange={(e: ChangeEvent<HTMLInputElement>) => { handleInputsChange({ ...inputsRef.current, [item.key]: e.target.value }) }}
                     />
                   )}
                   {item.type === 'number' && (
@@ -111,7 +109,7 @@ const RunOnce: FC<IRunOnceProps> = ({
                       type="number"
                       placeholder={`${item.name}${!item.required ? `(${t('appDebug.variableTable.optional')})` : ''}`}
                       value={inputs[item.key]}
-                    onChange={(e) => { handleInputsChange({ ...inputsRef.current, [item.key]: e.target.value }) }}
+                      onChange={(e: ChangeEvent<HTMLInputElement>) => { handleInputsChange({ ...inputsRef.current, [item.key]: e.target.value }) }}
                     />
                   )}
                   {item.type === 'file' && (

web/app/components/workflow/block-selector/index.tsx

@@ -128,7 +128,7 @@ const NodeSelector: FC<NodeSelectorProps> = ({
         }
       </PortalToFollowElemTrigger>
       <PortalToFollowElemContent className='z-[1000]'>
-        <div className={`rounded-lg border-[0.5px] border-gray-200 bg-white shadow-lg ${popupClassName}`}>
+        <div className={`rounded-lg border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-lg ${popupClassName}`}>
           <div className='px-2 pt-2' onClick={e => e.stopPropagation()}>
             {activeTab === TabsEnum.Blocks && (
               <Input

web/app/components/workflow/block-selector/market-place-plugin/list.tsx

@@ -9,6 +9,7 @@ import Link from 'next/link'
 import { marketplaceUrlPrefix } from '@/config'
 import { RiArrowRightUpLine, RiSearchLine } from '@remixicon/react'
 // import { RiArrowRightUpLine } from '@remixicon/react'
+import { noop } from 'lodash-es'
 
 type Props = {
   wrapElemRef: React.RefObject<HTMLElement>
@@ -107,7 +108,7 @@ const List = (
           <Item
             key={index}
             payload={item}
-            onAction={() => { }}
+            onAction={noop}
           />
         ))}
         <div className='mb-3 mt-2 flex items-center justify-center space-x-2'>

web/app/components/workflow/header/checklist.tsx

@@ -79,37 +79,37 @@ const WorkflowChecklist = ({
       </PortalToFollowElemTrigger>
       <PortalToFollowElemContent className='z-[12]'>
         <div
-          className='w-[420px] overflow-y-auto rounded-2xl border-[0.5px] border-black/5 bg-white shadow-lg'
+          className='w-[420px] overflow-y-auto rounded-2xl border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-lg'
           style={{
             maxHeight: 'calc(2 / 3 * 100vh)',
           }}
         >
-          <div className='text-md sticky top-0 z-[1] flex h-[44px] items-center bg-white pl-4 pr-3 pt-3 font-semibold text-gray-900'>
+          <div className='text-md sticky top-0 z-[1] flex h-[44px] items-center bg-components-panel-bg pl-4 pr-3 pt-3 font-semibold text-text-primary'>
             <div className='grow'>{t('workflow.panel.checklist')}{needWarningNodes.length ? `(${needWarningNodes.length})` : ''}</div>
             <div
               className='flex h-6 w-6 shrink-0 cursor-pointer items-center justify-center'
               onClick={() => setOpen(false)}
             >
-              <RiCloseLine className='h-4 w-4 text-gray-500' />
+              <RiCloseLine className='h-4 w-4 text-text-tertiary' />
             </div>
           </div>
           <div className='py-2'>
             {
               !!needWarningNodes.length && (
                 <>
-                  <div className='px-4 text-xs text-gray-400'>{t('workflow.panel.checklistTip')}</div>
+                  <div className='px-4 text-xs text-text-tertiary'>{t('workflow.panel.checklistTip')}</div>
                   <div className='px-4 py-2'>
                     {
                       needWarningNodes.map(node => (
                         <div
                           key={node.id}
-                          className='mb-2 cursor-pointer rounded-lg border-[0.5px] border-gray-200 bg-white shadow-xs last-of-type:mb-0'
+                          className='mb-2 cursor-pointer rounded-lg border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-xs last-of-type:mb-0'
                           onClick={() => {
                             handleNodeSelect(node.id)
                             setOpen(false)
                           }}
                         >
-                          <div className='flex h-9 items-center p-2 text-xs font-medium text-gray-700'>
+                          <div className='flex h-9 items-center p-2 text-xs font-medium text-text-secondary'>
                             <BlockIcon
                               type={node.type}
                               className='mr-1.5'
@@ -119,11 +119,11 @@ const WorkflowChecklist = ({
                               {node.title}
                             </span>
                           </div>
-                          <div className='border-t-[0.5px] border-t-black/2'>
+                          <div className='border-t-[0.5px] border-divider-regular'>
                             {
                               node.unConnected && (
-                                <div className='rounded-b-lg bg-gray-25 px-3 py-2'>
+                                <div className='px-3 py-2 last:rounded-b-lg'>
-                                  <div className='flex text-xs leading-[18px] text-gray-500'>
+                                  <div className='flex text-xs leading-[18px] text-text-tertiary'>
                                     <AlertTriangle className='mr-2 mt-[3px] h-3 w-3 text-[#F79009]' />
                                     {t('workflow.common.needConnectTip')}
                                   </div>
@@ -132,8 +132,8 @@ const WorkflowChecklist = ({
                             }
                             {
                               node.errorMessage && (
-                                <div className='rounded-b-lg bg-gray-25 px-3 py-2'>
+                                <div className='px-3 py-2 last:rounded-b-lg'>
-                                  <div className='flex text-xs leading-[18px] text-gray-500'>
+                                  <div className='flex text-xs leading-[18px] text-text-tertiary'>
                                     <AlertTriangle className='mr-2 mt-[3px] h-3 w-3 text-[#F79009]' />
                                     {node.errorMessage}
                                   </div>
@@ -150,8 +150,8 @@ const WorkflowChecklist = ({
             }
             {
               !needWarningNodes.length && (
-                <div className='mx-4 mb-3 rounded-lg bg-gray-50 py-4 text-center text-xs text-gray-400'>
+                <div className='mx-4 mb-3 rounded-lg bg-components-panel-bg py-4 text-center text-xs text-text-tertiary'>
-                  <ChecklistSquare className='mx-auto mb-[5px] h-8 w-8 text-gray-300' />
+                  <ChecklistSquare className='mx-auto mb-[5px] h-8 w-8 text-text-quaternary' />
                   {t('workflow.panel.checklistResolved')}
                 </div>
               )

web/app/components/workflow/header/view-history.tsx

@@ -87,9 +87,9 @@ const ViewHistory = ({
           {
             withText && (
               <div className={cn(
-                'flex h-8 items-center rounded-lg border-[0.5px] border-gray-200 bg-white px-3 shadow-xs',
+                'flex h-8 items-center rounded-lg border-[0.5px] border-components-button-secondary-border bg-components-button-secondary-bg px-3 shadow-xs',
-                'cursor-pointer text-[13px] font-medium text-primary-600',
+                'cursor-pointer text-[13px] font-medium text-components-button-secondary-text hover:bg-components-button-secondary-bg-hover',
-                open && '!bg-primary-50',
+                open && 'bg-components-button-secondary-bg-hover',
               )}>
                 <ClockPlay
                   className={'mr-1 h-4 w-4'}
@@ -118,12 +118,12 @@ const ViewHistory = ({
         </PortalToFollowElemTrigger>
         <PortalToFollowElemContent className='z-[12]'>
           <div
-            className='ml-2 flex w-[240px] flex-col overflow-y-auto rounded-xl border-[0.5px] border-gray-200 bg-white shadow-xl'
+            className='ml-2 flex w-[240px] flex-col overflow-y-auto rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-xl'
             style={{
               maxHeight: 'calc(2 / 3 * 100vh)',
             }}
           >
-            <div className='sticky top-0 flex items-center justify-between bg-white px-4 pt-3 text-base font-semibold text-gray-900'>
+            <div className='sticky top-0 flex items-center justify-between bg-components-panel-bg px-4 pt-3 text-base font-semibold text-text-primary'>
               <div className='grow'>{t('workflow.common.runHistory')}</div>
               <div
                 className='flex h-6 w-6 shrink-0 cursor-pointer items-center justify-center'
@@ -133,7 +133,7 @@ const ViewHistory = ({
                   setOpen(false)
                 }}
               >
-                <RiCloseLine className='h-4 w-4 text-gray-500' />
+                <RiCloseLine className='h-4 w-4 text-text-tertiary' />
               </div>
             </div>
             {
@@ -149,8 +149,8 @@ const ViewHistory = ({
                   {
                     !data?.data.length && (
                       <div className='py-12'>
-                        <ClockPlaySlim className='mx-auto mb-2 h-8 w-8 text-gray-300' />
+                        <ClockPlaySlim className='mx-auto mb-2 h-8 w-8 text-text-quaternary' />
-                        <div className='text-center text-[13px] text-gray-400'>
+                        <div className='text-center text-[13px] text-text-quaternary'>
                           {t('workflow.common.notRunning')}
                         </div>
                       </div>
@@ -161,8 +161,8 @@ const ViewHistory = ({
                       <div
                         key={item.id}
                         className={cn(
-                          'mb-0.5 flex cursor-pointer rounded-lg px-2 py-[7px] hover:bg-primary-50',
+                          'mb-0.5 flex cursor-pointer rounded-lg px-2 py-[7px] hover:bg-state-base-hover',
-                          item.id === historyWorkflowData?.id && 'bg-primary-50',
+                          item.id === historyWorkflowData?.id && 'bg-state-accent-hover hover:bg-state-accent-hover',
                         )}
                         onClick={() => {
                           workflowStore.setState({
@@ -195,13 +195,13 @@ const ViewHistory = ({
                         <div>
                           <div
                             className={cn(
-                              'flex items-center text-[13px] font-medium leading-[18px]',
+                              'flex items-center text-[13px] font-medium leading-[18px] text-text-primary',
-                              item.id === historyWorkflowData?.id && 'text-primary-600',
+                              item.id === historyWorkflowData?.id && 'text-text-accent',
                             )}
                           >
-                            {`Test ${isChatMode ? 'Chat' : 'Run'}#${item.sequence_number}`}
+                            {`Test ${isChatMode ? 'Chat' : 'Run'} #${item.sequence_number}`}
                           </div>
-                          <div className='flex items-center text-xs leading-[18px] text-gray-500'>
+                          <div className='flex items-center text-xs leading-[18px] text-text-tertiary'>
                             {item.created_by_account?.name} · {formatTimeFromNow((item.finished_at || item.created_at) * 1000)}
                           </div>
                         </div>

web/app/components/workflow/hooks/use-workflow-run.ts

@@ -18,6 +18,7 @@ import { stopWorkflowRun } from '@/service/workflow'
 import { useFeaturesStore } from '@/app/components/base/features/hooks'
 import { AudioPlayerManager } from '@/app/components/base/audio-btn/audio.player.manager'
 import type { VersionHistory } from '@/types/workflow'
+import { noop } from 'lodash-es'
 
 export const useWorkflowRun = () => {
   const store = useStoreApi()
@@ -168,7 +169,7 @@ export const useWorkflowRun = () => {
       else
         ttsUrl = `/apps/${params.appId}/text-to-audio`
     }
-    const player = AudioPlayerManager.getInstance().getAudioPlayer(ttsUrl, ttsIsPublic, uuidV4(), 'none', 'none', (_: any): any => { })
+    const player = AudioPlayerManager.getInstance().getAudioPlayer(ttsUrl, ttsIsPublic, uuidV4(), 'none', 'none', noop)
 
     ssePost(
       url,

web/app/components/workflow/nodes/_base/components/add-variable-popup.tsx

@@ -18,8 +18,8 @@ export const AddVariablePopup = ({
   const { t } = useTranslation()
 
   return (
-    <div className='w-[240px] rounded-lg border-[0.5px] border-gray-200 bg-white shadow-lg'>
+    <div className='w-[240px] rounded-lg border-[0.5px] border-components-panel-border bg-components-panel-bg shadow-lg'>
-      <div className='flex h-[34px] items-center border-b-[0.5px] border-b-gray-200 px-4 text-[13px] font-semibold text-gray-700'>
+      <div className='flex h-[34px] items-center border-b-[0.5px] border-b-divider-regular px-4 text-[13px] font-semibold text-text-secondary'>
         {t('workflow.nodes.variableAssigner.setAssignVariable')}
       </div>
       <div className='p-1'>

web/app/components/workflow/nodes/_base/components/before-run-form/index.tsx

@@ -139,18 +139,16 @@ const BeforeRunForm: FC<BeforeRunFormProps> = ({
     onRun(submitData)
   }, [forms, onRun, t])
   return (
-    <div className='absolute inset-0 z-10 rounded-2xl pt-10' style={{
+    <div className='absolute inset-0 z-10 rounded-2xl bg-background-overlay-alt pt-10'>
-      backgroundColor: 'rgba(16, 24, 40, 0.20)',
+      <div className='flex h-full flex-col rounded-2xl bg-components-panel-bg'>
-    }}>
-      <div className='flex h-full flex-col rounded-2xl bg-white'>
         <div className='flex h-8 shrink-0 items-center justify-between pl-4 pr-3 pt-3'>
-          <div className='truncate text-base font-semibold text-gray-900'>
+          <div className='truncate text-base font-semibold text-text-primary'>
             {t(`${i18nPrefix}.testRun`)} {nodeName}
           </div>
           <div className='ml-2 shrink-0 cursor-pointer p-1' onClick={() => {
             onHide()
           }}>
-            <RiCloseLine className='h-4 w-4 text-gray-500 ' />
+            <RiCloseLine className='h-4 w-4 text-text-tertiary ' />
           </div>
         </div>
         {
@@ -178,14 +176,14 @@ const BeforeRunForm: FC<BeforeRunFormProps> = ({
               <div className='mt-4 flex justify-between space-x-2 px-4' >
                 {isRunning && (
                   <div
-                    className='cursor-pointer rounded-lg border border-gray-200 bg-white p-2 shadow-xs'
+                    className='cursor-pointer rounded-lg border border-divider-regular bg-components-button-secondary-bg p-2 shadow-xs'
                     onClick={onStop}
                   >
-                    <StopCircle className='h-4 w-4 text-gray-500' />
+                    <StopCircle className='h-4 w-4 text-text-tertiary' />
                   </div>
                 )}
                 <Button disabled={!isFileLoaded || isRunning} variant='primary' className='w-0 grow space-x-2' onClick={handleRun}>
-                  {isRunning && <RiLoader2Line className='h-4 w-4 animate-spin text-white' />}
+                  {isRunning && <RiLoader2Line className='h-4 w-4 animate-spin' />}
                   <div>{t(`${i18nPrefix}.${isRunning ? 'running' : 'startRun'}`)}</div>
                 </Button>
               </div>

web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx

@@ -8,6 +8,8 @@ import { CodeLanguage } from '@/app/components/workflow/nodes/code/types'
 import {
   getFilesInLogs,
 } from '@/app/components/base/file-uploader/utils'
+import { Theme } from '@/types/app'
+import useTheme from '@/hooks/use-theme'
 import './style.css'
 import { noop } from 'lodash-es'
 
@@ -43,15 +45,6 @@ export const languageMap = {
   [CodeLanguage.json]: 'json',
 }
 
-const DEFAULT_THEME = {
-  base: 'vs',
-  inherit: true,
-  rules: [],
-  colors: {
-    'editor.background': '#F2F4F7', // #00000000 transparent. But it will has a blue border
-  },
-}
-
 const CodeEditor: FC<Props> = ({
   value = '',
   placeholder = '',
@@ -76,7 +69,7 @@ const CodeEditor: FC<Props> = ({
   const [isMounted, setIsMounted] = React.useState(false)
   const minHeight = height || 200
   const [editorContentHeight, setEditorContentHeight] = useState(56)
-
+  const { theme: appTheme } = useTheme()
   const valueRef = useRef(value)
   useEffect(() => {
     valueRef.current = value
@@ -114,27 +107,7 @@ const CodeEditor: FC<Props> = ({
       setIsFocus(false)
     })
 
-    monaco.editor.defineTheme('default-theme', DEFAULT_THEME)
+    monaco.editor.setTheme(appTheme === Theme.light ? 'light' : 'vs-dark') // Fix: sometimes not load the default theme
-
-    monaco.editor.defineTheme('blur-theme', {
-      base: 'vs',
-      inherit: true,
-      rules: [],
-      colors: {
-        'editor.background': '#F2F4F7',
-      },
-    })
-
-    monaco.editor.defineTheme('focus-theme', {
-      base: 'vs',
-      inherit: true,
-      rules: [],
-      colors: {
-        'editor.background': '#ffffff',
-      },
-    })
-
-    monaco.editor.setTheme('default-theme') // Fix: sometimes not load the default theme
 
     onMount?.(editor, monaco)
     setIsMounted(true)
@@ -151,12 +124,11 @@ const CodeEditor: FC<Props> = ({
     }
   })()
 
-  const theme = (() => {
+  const theme = useMemo(() => {
-    if (noWrapper)
+    if (appTheme === Theme.light)
-      return 'default-theme'
+      return 'light'
-
+    return 'vs-dark'
-    return isFocus ? 'focus-theme' : 'blur-theme'
+  }, [appTheme])
-  })()
 
   const main = (
     <>

web/app/components/workflow/nodes/_base/components/editor/code-editor/style.css

@@ -6,6 +6,17 @@
   padding-left: 0;
 }
 
+.monaco-editor {
+  background-color: transparent !important;
+  outline: none !important;
+}
+.monaco-editor .monaco-editor-background {
+  background-color: transparent !important;
+}
+.monaco-editor .margin {
+  background-color: transparent !important;
+}
+
 /* hide readonly tooltip */
 .monaco-editor-overlaymessage {
   display: none !important;