From f191d372f0c90571c1e388e2d6fe6924b1657da4 Mon Sep 17 00:00:00 2001 From: Novice <857526207@qq.com> Date: Mon, 21 Apr 2025 15:09:49 +0800 Subject: [PATCH 001/166] fix(promptMessage): correct field_serializer implementation for content serialization (#18458) --- .../model_runtime/entities/message_entities.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/api/core/model_runtime/entities/message_entities.py b/api/core/model_runtime/entities/message_entities.py index 977678b893..3bed2460dd 100644 --- a/api/core/model_runtime/entities/message_entities.py +++ b/api/core/model_runtime/entities/message_entities.py @@ -1,8 +1,8 @@ from collections.abc import Sequence from enum import Enum, StrEnum -from typing import Optional +from typing import Any, Optional, Union -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_serializer, field_validator class PromptMessageRole(Enum): @@ -135,6 +135,16 @@ class PromptMessage(BaseModel): """ return not self.content + @field_serializer("content") + def serialize_content( + self, content: Optional[Union[str, Sequence[PromptMessageContent]]] + ) -> Optional[str | list[dict[str, Any] | PromptMessageContent] | Sequence[PromptMessageContent]]: + if content is None or isinstance(content, str): + return content + if isinstance(content, list): + return [item.model_dump() if hasattr(item, "model_dump") else item for item in content] + return content + class UserPromptMessage(PromptMessage): """ From 30c051d48553abf3c0c6793ab1277769e7338c90 Mon Sep 17 00:00:00 2001 From: doskoi <50610194+t-daisuke@users.noreply.github.com> Date: Mon, 21 Apr 2025 18:56:31 +0900 Subject: [PATCH 002/166] =?UTF-8?q?fix:=20update=20Japanese=20translation?= =?UTF-8?q?=20for=20'switchVersion'=20in=20plugin.ts=20to=20=E2=80=A6=20(#?= =?UTF-8?q?18469)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/i18n/ja-JP/plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/i18n/ja-JP/plugin.ts b/web/i18n/ja-JP/plugin.ts index cb2a5f4f80..1d2f1a2fb5 100644 --- a/web/i18n/ja-JP/plugin.ts +++ b/web/i18n/ja-JP/plugin.ts @@ -81,7 +81,7 @@ const translation = { endpointDeleteContent: '{{name}}を削除しますか?', actionNum: '{{num}} {{action}} が含まれています', endpointsDocLink: 'ドキュメントを表示する', - switchVersion: 'スイッチ版', + switchVersion: 'バージョンの切り替え', }, debugInfo: { title: 'デバッグ', From 7b6523e54dec4a181f5ffac57932a99e706a2e64 Mon Sep 17 00:00:00 2001 From: tmuife <43266626@qq.com> Date: Mon, 21 Apr 2025 17:56:57 +0800 Subject: [PATCH 003/166] Update Oracle db connection library and change connection pool to single connection (#18466) --- .../rag/datasource/vdb/oracle/oraclevector.py | 176 ++++++++++-------- api/pyproject.toml | 2 +- api/uv.lock | 26 +-- 3 files changed, 117 insertions(+), 87 deletions(-) diff --git a/api/core/rag/datasource/vdb/oracle/oraclevector.py b/api/core/rag/datasource/vdb/oracle/oraclevector.py index 4af2578197..63695e6f3f 100644 --- a/api/core/rag/datasource/vdb/oracle/oraclevector.py +++ b/api/core/rag/datasource/vdb/oracle/oraclevector.py @@ -2,12 +2,12 @@ import array import json import re import uuid -from contextlib import contextmanager from typing import Any import jieba.posseg as pseg # type: ignore import numpy import oracledb +from oracledb.connection import Connection from pydantic import BaseModel, model_validator from configs import dify_config @@ -70,6 +70,7 @@ class OracleVector(BaseVector): super().__init__(collection_name) self.pool = self._create_connection_pool(config) self.table_name = f"embedding_{collection_name}" + self.config = config def get_type(self) -> str: return VectorType.ORACLE @@ -107,16 +108,19 @@ class OracleVector(BaseVector): outconverter=self.numpy_converter_out, ) + def _get_connection(self) -> Connection: + connection = oracledb.connect(user=self.config.user, password=self.config.password, dsn=self.config.dsn) + return connection + def _create_connection_pool(self, config: OracleVectorConfig): pool_params = { "user": config.user, "password": config.password, "dsn": config.dsn, "min": 1, - "max": 50, + "max": 5, "increment": 1, } - if config.is_autonomous: pool_params.update( { @@ -125,22 +129,8 @@ class OracleVector(BaseVector): "wallet_password": config.wallet_password, } ) - return oracledb.create_pool(**pool_params) - @contextmanager - def _get_cursor(self): - conn = self.pool.acquire() - conn.inputtypehandler = self.input_type_handler - conn.outputtypehandler = self.output_type_handler - cur = conn.cursor() - try: - yield cur - finally: - cur.close() - conn.commit() - conn.close() - def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): dimension = len(embeddings[0]) self._create_collection(dimension) @@ -162,41 +152,68 @@ class OracleVector(BaseVector): numpy.array(embeddings[i]), ) ) - # print(f"INSERT INTO {self.table_name} (id, text, meta, embedding) VALUES (:1, :2, :3, :4)") - with self._get_cursor() as cur: - cur.executemany( - f"INSERT INTO {self.table_name} (id, text, meta, embedding) VALUES (:1, :2, :3, :4)", values - ) + with self._get_connection() as conn: + conn.inputtypehandler = self.input_type_handler + conn.outputtypehandler = self.output_type_handler + # with conn.cursor() as cur: + # cur.executemany( + # f"INSERT INTO {self.table_name} (id, text, meta, embedding) VALUES (:1, :2, :3, :4)", values + # ) + # conn.commit() + for value in values: + with conn.cursor() as cur: + try: + cur.execute( + f"""INSERT INTO {self.table_name} (id, text, meta, embedding) + VALUES (:1, :2, :3, :4)""", + value, + ) + conn.commit() + except Exception as e: + print(e) + conn.close() return pks def text_exists(self, id: str) -> bool: - with self._get_cursor() as cur: - cur.execute(f"SELECT id FROM {self.table_name} WHERE id = '%s'" % (id,)) - return cur.fetchone() is not None + with self._get_connection() as conn: + with conn.cursor() as cur: + cur.execute(f"SELECT id FROM {self.table_name} WHERE id = '%s'" % (id,)) + return cur.fetchone() is not None + conn.close() def get_by_ids(self, ids: list[str]) -> list[Document]: - with self._get_cursor() as cur: - cur.execute(f"SELECT meta, text FROM {self.table_name} WHERE id IN %s", (tuple(ids),)) - docs = [] - for record in cur: - docs.append(Document(page_content=record[1], metadata=record[0])) + with self._get_connection() as conn: + with conn.cursor() as cur: + cur.execute(f"SELECT meta, text FROM {self.table_name} WHERE id IN %s", (tuple(ids),)) + docs = [] + for record in cur: + docs.append(Document(page_content=record[1], metadata=record[0])) + self.pool.release(connection=conn) + conn.close() return docs def delete_by_ids(self, ids: list[str]) -> None: if not ids: return - with self._get_cursor() as cur: - cur.execute(f"DELETE FROM {self.table_name} WHERE id IN %s" % (tuple(ids),)) + with self._get_connection() as conn: + with conn.cursor() as cur: + cur.execute(f"DELETE FROM {self.table_name} WHERE id IN %s" % (tuple(ids),)) + conn.commit() + conn.close() def delete_by_metadata_field(self, key: str, value: str) -> None: - with self._get_cursor() as cur: - cur.execute(f"DELETE FROM {self.table_name} WHERE meta->>%s = %s", (key, value)) + with self._get_connection() as conn: + with conn.cursor() as cur: + cur.execute(f"DELETE FROM {self.table_name} WHERE meta->>%s = %s", (key, value)) + conn.commit() + conn.close() def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: """ Search the nearest neighbors to a vector. :param query_vector: The input vector to search for similar items. + :param top_k: The number of nearest neighbors to return, default is 5. :return: List of Documents that are nearest to the query vector. """ top_k = kwargs.get("top_k", 4) @@ -205,20 +222,25 @@ class OracleVector(BaseVector): if document_ids_filter: document_ids = ", ".join(f"'{id}'" for id in document_ids_filter) where_clause = f"WHERE metadata->>'document_id' in ({document_ids})" - with self._get_cursor() as cur: - cur.execute( - f"SELECT meta, text, vector_distance(embedding,:1) AS distance FROM {self.table_name}" - f" {where_clause} ORDER BY distance fetch first {top_k} rows only", - [numpy.array(query_vector)], - ) - docs = [] - score_threshold = float(kwargs.get("score_threshold") or 0.0) - for record in cur: - metadata, text, distance = record - score = 1 - distance - metadata["score"] = score - if score > score_threshold: - docs.append(Document(page_content=text, metadata=metadata)) + with self._get_connection() as conn: + conn.inputtypehandler = self.input_type_handler + conn.outputtypehandler = self.output_type_handler + with conn.cursor() as cur: + cur.execute( + f"""SELECT meta, text, vector_distance(embedding,(select to_vector(:1) from dual),cosine) + AS distance FROM {self.table_name} + {where_clause} ORDER BY distance fetch first {top_k} rows only""", + [numpy.array(query_vector)], + ) + docs = [] + score_threshold = float(kwargs.get("score_threshold") or 0.0) + for record in cur: + metadata, text, distance = record + score = 1 - distance + metadata["score"] = score + if score > score_threshold: + docs.append(Document(page_content=text, metadata=metadata)) + conn.close() return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: @@ -228,7 +250,7 @@ class OracleVector(BaseVector): top_k = kwargs.get("top_k", 5) # just not implement fetch by score_threshold now, may be later - # score_threshold = float(kwargs.get("score_threshold") or 0.0) + score_threshold = float(kwargs.get("score_threshold") or 0.0) if len(query) > 0: # Check which language the query is in zh_pattern = re.compile("[\u4e00-\u9fa5]+") @@ -239,7 +261,7 @@ class OracleVector(BaseVector): words = pseg.cut(query) current_entity = "" for word, pos in words: - if pos in {"nr", "Ng", "eng", "nz", "n", "ORG", "v"}: # nr: 人名,ns: 地名,nt: 机构名 + if pos in {"nr", "Ng", "eng", "nz", "n", "ORG", "v"}: # nr: 人名, ns: 地名, nt: 机构名 current_entity += word else: if current_entity: @@ -260,30 +282,35 @@ class OracleVector(BaseVector): for token in all_tokens: if token not in stop_words: entities.append(token) - with self._get_cursor() as cur: - document_ids_filter = kwargs.get("document_ids_filter") - where_clause = "" - if document_ids_filter: - document_ids = ", ".join(f"'{id}'" for id in document_ids_filter) - where_clause = f" AND metadata->>'document_id' in ({document_ids}) " - cur.execute( - f"select meta, text, embedding FROM {self.table_name}" - f"WHERE CONTAINS(text, :1, 1) > 0 {where_clause} " - f"order by score(1) desc fetch first {top_k} rows only", - [" ACCUM ".join(entities)], - ) - docs = [] - for record in cur: - metadata, text, embedding = record - docs.append(Document(page_content=text, vector=embedding, metadata=metadata)) + with self._get_connection() as conn: + with conn.cursor() as cur: + document_ids_filter = kwargs.get("document_ids_filter") + where_clause = "" + if document_ids_filter: + document_ids = ", ".join(f"'{id}'" for id in document_ids_filter) + where_clause = f" AND metadata->>'document_id' in ({document_ids}) " + cur.execute( + f"""select meta, text, embedding FROM {self.table_name} + WHERE CONTAINS(text, :kk, 1) > 0 {where_clause} + order by score(1) desc fetch first {top_k} rows only""", + kk=" ACCUM ".join(entities), + ) + docs = [] + for record in cur: + metadata, text, embedding = record + docs.append(Document(page_content=text, vector=embedding, metadata=metadata)) + conn.close() return docs else: return [Document(page_content="", metadata={})] return [] def delete(self) -> None: - with self._get_cursor() as cur: - cur.execute(f"DROP TABLE IF EXISTS {self.table_name} cascade constraints") + with self._get_connection() as conn: + with conn.cursor() as cur: + cur.execute(f"DROP TABLE IF EXISTS {self.table_name} cascade constraints") + conn.commit() + conn.close() def _create_collection(self, dimension: int): cache_key = f"vector_indexing_{self._collection_name}" @@ -293,11 +320,14 @@ class OracleVector(BaseVector): if redis_client.get(collection_exist_cache_key): return - with self._get_cursor() as cur: - cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name)) - redis_client.set(collection_exist_cache_key, 1, ex=3600) - with self._get_cursor() as cur: - cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name)) + with self._get_connection() as conn: + with conn.cursor() as cur: + cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name)) + redis_client.set(collection_exist_cache_key, 1, ex=3600) + with conn.cursor() as cur: + cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name)) + conn.commit() + conn.close() class OracleVectorFactory(AbstractVectorFactory): diff --git a/api/pyproject.toml b/api/pyproject.toml index 08f9c1e229..4992178423 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -178,7 +178,7 @@ vdb = [ "couchbase~=4.3.0", "elasticsearch==8.14.0", "opensearch-py==2.4.0", - "oracledb~=2.2.1", + "oracledb==3.0.0", "pgvecto-rs[sqlalchemy]~=0.2.1", "pgvector==0.2.5", "pymilvus~=2.5.0", diff --git a/api/uv.lock b/api/uv.lock index 4384e1abb5..6c8699dd7c 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -1471,7 +1471,7 @@ vdb = [ { name = "couchbase", specifier = "~=4.3.0" }, { name = "elasticsearch", specifier = "==8.14.0" }, { name = "opensearch-py", specifier = "==2.4.0" }, - { name = "oracledb", specifier = "~=2.2.1" }, + { name = "oracledb", specifier = "==3.0.0" }, { name = "pgvecto-rs", extras = ["sqlalchemy"], specifier = "~=0.2.1" }, { name = "pgvector", specifier = "==0.2.5" }, { name = "pymilvus", specifier = "~=2.5.0" }, @@ -3600,23 +3600,23 @@ wheels = [ [[package]] name = "oracledb" -version = "2.2.1" +version = "3.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/36/fb/3fbacb351833dd794abb184303a5761c4bb33df9d770fd15d01ead2ff738/oracledb-2.2.1.tar.gz", hash = "sha256:8464c6f0295f3318daf6c2c72c83c2dcbc37e13f8fd44e3e39ff8665f442d6b6", size = 580818 } +sdist = { url = "https://files.pythonhosted.org/packages/bf/39/712f797b75705c21148fa1d98651f63c2e5cc6876e509a0a9e2f5b406572/oracledb-3.0.0.tar.gz", hash = "sha256:64dc86ee5c032febc556798b06e7b000ef6828bb0252084f6addacad3363db85", size = 840431 } wheels = [ - { url = "https://files.pythonhosted.org/packages/74/b7/a4238295944670fb8cc50a8cc082e0af5a0440bfb1c2bac2b18429c0a579/oracledb-2.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fb6d9a4d7400398b22edb9431334f9add884dec9877fd9c4ae531e1ccc6ee1fd", size = 3551303 }, - { url = "https://files.pythonhosted.org/packages/4f/5f/98481d44976cd2b3086361f2d50026066b24090b0e6cd1f2a12c824e9717/oracledb-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07757c240afbb4f28112a6affc2c5e4e34b8a92e5bb9af81a40fba398da2b028", size = 12258455 }, - { url = "https://files.pythonhosted.org/packages/e9/54/06b2540286e2b63f60877d6f3c6c40747e216b6eeda0756260e194897076/oracledb-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63daec72f853c47179e98493e9b732909d96d495bdceb521c5973a3940d28142", size = 12317476 }, - { url = "https://files.pythonhosted.org/packages/4d/1a/67814439a4e24df83281a72cb0ba433d6b74e1bff52a9975b87a725bcba5/oracledb-2.2.1-cp311-cp311-win32.whl", hash = "sha256:fec5318d1e0ada7e4674574cb6c8d1665398e8b9c02982279107212f05df1660", size = 1369368 }, - { url = "https://files.pythonhosted.org/packages/e3/b8/b2a8f0607be17f58ec6689ad5fd15c2956f4996c64547325e96439570edf/oracledb-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:5134dccb5a11bc755abf02fd49be6dc8141dfcae4b650b55d40509323d00b5c2", size = 1655035 }, - { url = "https://files.pythonhosted.org/packages/24/5b/2fff762243030f31a6b1561fc8eeb142e69ba6ebd3e7fbe4a2c82f0eb6f0/oracledb-2.2.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ac5716bc9a48247fdf563f5f4ec097f5c9f074a60fd130cdfe16699208ca29b5", size = 3583960 }, - { url = "https://files.pythonhosted.org/packages/e6/88/34117ae830e7338af7c0481f1c0fc6eda44d558e12f9203b45b491e53071/oracledb-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c150bddb882b7c73fb462aa2d698744da76c363e404570ed11d05b65811d96c3", size = 11749006 }, - { url = "https://files.pythonhosted.org/packages/9d/58/bac788f18c21f727955652fe238de2d24a12c2b455ed4db18a6d23ff781e/oracledb-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193e1888411bc21187ade4b16b76820bd1e8f216e25602f6cd0a97d45723c1dc", size = 11950663 }, - { url = "https://files.pythonhosted.org/packages/3b/e2/005f66ae919c6f7c73e06863256cf43aa844330e2dc61a5f9779ae44a801/oracledb-2.2.1-cp312-cp312-win32.whl", hash = "sha256:44a960f8bbb0711af222e0a9690e037b6a2a382e0559ae8eeb9cfafe26c7a3bc", size = 1324255 }, - { url = "https://files.pythonhosted.org/packages/e6/25/759eb2143134513382e66d874c4aacfd691dec3fef7141170cfa6c1b154f/oracledb-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:470136add32f0d0084225c793f12a52b61b52c3dc00c9cd388ec6a3db3a7643e", size = 1613047 }, + { url = "https://files.pythonhosted.org/packages/fa/bf/d872c4b3fc15cd3261fe0ea72b21d181700c92dbc050160e161654987062/oracledb-3.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:52daa9141c63dfa75c07d445e9bb7f69f43bfb3c5a173ecc48c798fe50288d26", size = 4312963 }, + { url = "https://files.pythonhosted.org/packages/b1/ea/01ee29e76a610a53bb34fdc1030f04b7669c3f80b25f661e07850fc6160e/oracledb-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af98941789df4c6aaaf4338f5b5f6b7f2c8c3fe6f8d6a9382f177f350868747a", size = 2661536 }, + { url = "https://files.pythonhosted.org/packages/3d/8e/ad380e34a46819224423b4773e58c350bc6269643c8969604097ced8c3bc/oracledb-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9812bb48865aaec35d73af54cd1746679f2a8a13cbd1412ab371aba2e39b3943", size = 2867461 }, + { url = "https://files.pythonhosted.org/packages/96/09/ecc4384a27fd6e1e4de824ae9c160e4ad3aaebdaade5b4bdcf56a4d1ff63/oracledb-3.0.0-cp311-cp311-win32.whl", hash = "sha256:6c27fe0de64f2652e949eb05b3baa94df9b981a4a45fa7f8a991e1afb450c8e2", size = 1752046 }, + { url = "https://files.pythonhosted.org/packages/62/e8/f34bde24050c6e55eeba46b23b2291f2dd7fd272fa8b322dcbe71be55778/oracledb-3.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:f922709672002f0b40997456f03a95f03e5712a86c61159951c5ce09334325e0", size = 2101210 }, + { url = "https://files.pythonhosted.org/packages/6f/fc/24590c3a3d41e58494bd3c3b447a62835138e5f9b243d9f8da0cfb5da8dc/oracledb-3.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:acd0e747227dea01bebe627b07e958bf36588a337539f24db629dc3431d3f7eb", size = 4351993 }, + { url = "https://files.pythonhosted.org/packages/b7/b6/1f3b0b7bb94d53e8857d77b2e8dbdf6da091dd7e377523e24b79dac4fd71/oracledb-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f8b402f77c22af031cd0051aea2472ecd0635c1b452998f511aa08b7350c90a4", size = 2532640 }, + { url = "https://files.pythonhosted.org/packages/72/1a/1815f6c086ab49c00921cf155ff5eede5267fb29fcec37cb246339a5ce4d/oracledb-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:378a27782e9a37918bd07a5a1427a77cb6f777d0a5a8eac9c070d786f50120ef", size = 2765949 }, + { url = "https://files.pythonhosted.org/packages/33/8d/208900f8d372909792ee70b2daad3f7361181e55f2217c45ed9dff658b54/oracledb-3.0.0-cp312-cp312-win32.whl", hash = "sha256:54a28c2cb08316a527cd1467740a63771cc1c1164697c932aa834c0967dc4efc", size = 1709373 }, + { url = "https://files.pythonhosted.org/packages/0c/5e/c21754f19c896102793c3afec2277e2180aa7d505e4d7fcca24b52d14e4f/oracledb-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:8289bad6d103ce42b140e40576cf0c81633e344d56e2d738b539341eacf65624", size = 2056452 }, ] [[package]] From 3136eb8e4b84d4a7614ce15db60f3627138a8cd0 Mon Sep 17 00:00:00 2001 From: KVOJJJin Date: Mon, 21 Apr 2025 17:58:01 +0800 Subject: [PATCH 004/166] Fix: json update in conversation variable (#18483) --- .../panel/chat-variable-panel/components/variable-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/workflow/panel/chat-variable-panel/components/variable-modal.tsx b/web/app/components/workflow/panel/chat-variable-panel/components/variable-modal.tsx index 947007e93e..d8da0e69a3 100644 --- a/web/app/components/workflow/panel/chat-variable-panel/components/variable-modal.tsx +++ b/web/app/components/workflow/panel/chat-variable-panel/components/variable-modal.tsx @@ -123,7 +123,7 @@ const ChatVariableModal = ({ case ChatVarType.Number: return value || 0 case ChatVarType.Object: - return formatValueFromObject(objectValue) + return editInJSON ? value : formatValueFromObject(objectValue) case ChatVarType.ArrayString: case ChatVarType.ArrayNumber: case ChatVarType.ArrayObject: From 2543162dec3768ae807ed6cf40b59d1bebfd2e95 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Mon, 21 Apr 2025 18:58:22 +0900 Subject: [PATCH 005/166] fix: cannot delete workflow version if other version is published as a tool (#18486) Signed-off-by: -LAN- --- api/models/workflow.py | 7 +++++++ api/services/workflow_service.py | 16 ++++++++++++++-- .../services/workflow/test_workflow_deletion.py | 11 ++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/api/models/workflow.py b/api/models/workflow.py index 51f2f4cc9f..5a67fa47a8 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -245,6 +245,13 @@ class Workflow(Base): @property def tool_published(self) -> bool: + """ + DEPRECATED: This property is not accurate for determining if a workflow is published as a tool. + It only checks if there's a WorkflowToolProvider for the app, not if this specific workflow version + is the one being used by the tool. + + For accurate checking, use a direct query with tenant_id, app_id, and version. + """ from models.tools import WorkflowToolProvider return ( diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index b88c7b296d..5cd5c55746 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -28,6 +28,7 @@ from extensions.ext_database import db from models.account import Account from models.enums import CreatedByRole from models.model import App, AppMode +from models.tools import WorkflowToolProvider from models.workflow import ( Workflow, WorkflowNodeExecution, @@ -523,8 +524,19 @@ class WorkflowService: # Cannot delete a workflow that's currently in use by an app raise WorkflowInUseError(f"Cannot delete workflow that is currently in use by app '{app.name}'") - # Check if this workflow is published as a tool - if workflow.tool_published: + # Don't use workflow.tool_published as it's not accurate for specific workflow versions + # Check if there's a tool provider using this specific workflow version + tool_provider = ( + session.query(WorkflowToolProvider) + .filter( + WorkflowToolProvider.tenant_id == workflow.tenant_id, + WorkflowToolProvider.app_id == workflow.app_id, + WorkflowToolProvider.version == workflow.version, + ) + .first() + ) + + if tool_provider: # Cannot delete a workflow that's published as a tool raise WorkflowInUseError("Cannot delete workflow that is published as a tool") diff --git a/api/tests/unit_tests/services/workflow/test_workflow_deletion.py b/api/tests/unit_tests/services/workflow/test_workflow_deletion.py index 56efcccc78..223020c2c5 100644 --- a/api/tests/unit_tests/services/workflow/test_workflow_deletion.py +++ b/api/tests/unit_tests/services/workflow/test_workflow_deletion.py @@ -40,6 +40,10 @@ def workflow_setup(): def test_delete_workflow_success(workflow_setup): # Setup mocks + + # Mock the tool provider query to return None (not published as a tool) + workflow_setup["session"].query.return_value.filter.return_value.first.return_value = None + workflow_setup["session"].scalar = MagicMock( side_effect=[workflow_setup["workflow"], None] ) # Return workflow first, then None for app @@ -97,7 +101,12 @@ def test_delete_workflow_in_use_by_app_error(workflow_setup): def test_delete_workflow_published_as_tool_error(workflow_setup): # Setup mocks - workflow_setup["workflow"].tool_published = True + from models.tools import WorkflowToolProvider + + # Mock the tool provider query + mock_tool_provider = MagicMock(spec=WorkflowToolProvider) + workflow_setup["session"].query.return_value.filter.return_value.first.return_value = mock_tool_provider + workflow_setup["session"].scalar = MagicMock( side_effect=[workflow_setup["workflow"], None] ) # Return workflow first, then None for app From be964c78ecf2d98cf8d484b31799eea5b751f7b4 Mon Sep 17 00:00:00 2001 From: GuanMu Date: Mon, 21 Apr 2025 21:00:04 +0800 Subject: [PATCH 006/166] fix: update document link based on client locale (#18493) --- web/app/components/datasets/documents/index.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/web/app/components/datasets/documents/index.tsx b/web/app/components/datasets/documents/index.tsx index 20e14a994b..854c984559 100644 --- a/web/app/components/datasets/documents/index.tsx +++ b/web/app/components/datasets/documents/index.tsx @@ -29,6 +29,8 @@ import { useChildSegmentListKey, useSegmentListKey } from '@/service/knowledge/u import useEditDocumentMetadata from '../metadata/hooks/use-edit-dataset-metadata' import DatasetMetadataDrawer from '../metadata/metadata-dataset/dataset-metadata-drawer' import StatusWithAction from '../common/document-status-with-action/status-with-action' +import { LanguagesSupported } from '@/i18n/language' +import { getLocaleOnClient } from '@/i18n' const FolderPlusIcon = ({ className }: React.SVGProps) => { return @@ -98,7 +100,7 @@ const Documents: FC = ({ datasetId }) => { const isDataSourceWeb = dataset?.data_source_type === DataSourceType.WEB const isDataSourceFile = dataset?.data_source_type === DataSourceType.FILE const embeddingAvailable = !!dataset?.embedding_available - + const locale = getLocaleOnClient() const debouncedSearchValue = useDebounce(searchValue, { wait: 500 }) const { data: documentsRes, isFetching: isListLoading } = useDocumentList({ @@ -260,7 +262,12 @@ const Documents: FC = ({ datasetId }) => { + href={ + locale === LanguagesSupported[1] + ? 'https://docs.dify.ai/v/zh-hans/guides/knowledge-base/integrate-knowledge-within-application' + : 'https://docs.dify.ai/guides/knowledge-base/integrate-knowledge-within-application' + } + > {t('datasetDocuments.list.learnMore')} From ee30497237d9963c2bf4f8b0d8f37b0192517231 Mon Sep 17 00:00:00 2001 From: Wunmi Sogunle Date: Tue, 22 Apr 2025 02:56:53 +0100 Subject: [PATCH 007/166] fix(markdown): correctly render links with inline code (#18500) --- web/app/components/base/markdown.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/components/base/markdown.tsx b/web/app/components/base/markdown.tsx index 52b880affa..6ea84a2842 100644 --- a/web/app/components/base/markdown.tsx +++ b/web/app/components/base/markdown.tsx @@ -252,7 +252,7 @@ const Img = ({ src }: any) => { return
} -const Link = ({ node, ...props }: any) => { +const Link = ({ node, children, ...props }: any) => { if (node.properties?.href && node.properties.href?.toString().startsWith('abbr')) { // eslint-disable-next-line react-hooks/rules-of-hooks const { onSend } = useChatContext() @@ -261,7 +261,7 @@ const Link = ({ node, ...props }: any) => { return onSend?.(hidden_text)} title={node.children[0]?.value}>{node.children[0]?.value} } else { - return {node.children[0] ? node.children[0]?.value : 'Download'} + return {children || 'Download'} } } From 80f5ee1eb2d12f2a0aba9a91025c1b965b41decb Mon Sep 17 00:00:00 2001 From: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Date: Tue, 22 Apr 2025 09:59:14 +0800 Subject: [PATCH 008/166] fix: fix workflow as a tool confirm dialog layout issue (#18494) --- .../agent/agent-tools/setting-built-in-tool.tsx | 2 +- .../dataset-config/card-item/item.tsx | 2 +- web/app/components/app/log/list.tsx | 2 +- web/app/components/app/workflow-log/list.tsx | 2 +- web/app/components/base/drawer-plus/index.tsx | 8 +++++++- web/app/components/base/drawer/index.tsx | 14 +++++++++----- .../detail/completed/common/full-screen-drawer.tsx | 2 +- .../components/datasets/documents/detail/index.tsx | 2 +- web/app/components/datasets/hit-testing/index.tsx | 4 ++-- .../metadata-dataset/dataset-metadata-drawer.tsx | 2 +- .../plugins/plugin-detail-panel/endpoint-modal.tsx | 2 +- .../plugins/plugin-detail-panel/index.tsx | 2 +- .../plugin-detail-panel/strategy-detail.tsx | 2 +- web/app/components/tools/add-tool-modal/index.tsx | 2 +- .../config-credentials.tsx | 4 +++- web/app/components/tools/provider/detail.tsx | 2 +- .../components/dataset-item.tsx | 2 +- 17 files changed, 34 insertions(+), 22 deletions(-) diff --git a/web/app/components/app/configuration/config/agent/agent-tools/setting-built-in-tool.tsx b/web/app/components/app/configuration/config/agent/agent-tools/setting-built-in-tool.tsx index 75183ab5a7..952ad66fc4 100644 --- a/web/app/components/app/configuration/config/agent/agent-tools/setting-built-in-tool.tsx +++ b/web/app/components/app/configuration/config/agent/agent-tools/setting-built-in-tool.tsx @@ -163,7 +163,7 @@ const SettingBuiltInTool: FC = ({ footer={null} mask={false} positionCenter={false} - panelClassname={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} + panelClassName={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} > <> {isLoading && } diff --git a/web/app/components/app/configuration/dataset-config/card-item/item.tsx b/web/app/components/app/configuration/dataset-config/card-item/item.tsx index d44fb145bb..65ad2ca941 100644 --- a/web/app/components/app/configuration/dataset-config/card-item/item.tsx +++ b/web/app/components/app/configuration/dataset-config/card-item/item.tsx @@ -97,7 +97,7 @@ const Item: FC = ({ - setShowSettingsModal(false)} footer={null} mask={isMobile} panelClassname='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'> + setShowSettingsModal(false)} footer={null} mask={isMobile} panelClassName='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'> setShowSettingsModal(false)} diff --git a/web/app/components/app/log/list.tsx b/web/app/components/app/log/list.tsx index b78af5cdba..056ce84f1e 100644 --- a/web/app/components/app/log/list.tsx +++ b/web/app/components/app/log/list.tsx @@ -743,7 +743,7 @@ const ConversationList: FC = ({ logs, appDetail, onRefresh }) onClose={onCloseDrawer} mask={isMobile} footer={null} - panelClassname='mt-16 mx-2 sm:mr-2 mb-4 !p-0 !max-w-[640px] rounded-xl bg-components-panel-bg' + panelClassName='mt-16 mx-2 sm:mr-2 mb-4 !p-0 !max-w-[640px] rounded-xl bg-components-panel-bg' > = ({ logs, appDetail, onRefresh }) => { onClose={onCloseDrawer} mask={isMobile} footer={null} - panelClassname='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[600px] rounded-xl border border-components-panel-border' + panelClassName='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[600px] rounded-xl border border-components-panel-border' > diff --git a/web/app/components/base/drawer-plus/index.tsx b/web/app/components/base/drawer-plus/index.tsx index bb022acdcb..33a1948181 100644 --- a/web/app/components/base/drawer-plus/index.tsx +++ b/web/app/components/base/drawer-plus/index.tsx @@ -9,6 +9,8 @@ import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' type Props = { isShow: boolean onHide: () => void + dialogClassName?: string + dialogBackdropClassName?: string panelClassName?: string maxWidthClassName?: string contentClassName?: string @@ -26,6 +28,8 @@ type Props = { const DrawerPlus: FC = ({ isShow, onHide, + dialogClassName = '', + dialogBackdropClassName = '', panelClassName = '', maxWidthClassName = '!max-w-[640px]', height = 'calc(100vh - 72px)', @@ -55,7 +59,9 @@ const DrawerPlus: FC = ({ footer={null} mask={isMobile || isShowMask} positionCenter={positionCenter} - panelClassname={cn('mx-2 mb-3 mt-16 rounded-xl !p-0 sm:mr-2', panelClassName, maxWidthClassName)} + dialogClassName={dialogClassName} + dialogBackdropClassName={dialogBackdropClassName} + panelClassName={cn('mx-2 mb-3 mt-16 rounded-xl !p-0 sm:mr-2', panelClassName, maxWidthClassName)} >
!clickOutsideNotOpen && onClose()} - className="fixed inset-0 z-[80] overflow-y-auto" + className={cn('fixed inset-0 z-[30] overflow-y-auto', dialogClassName)} >
{/* mask */} { !clickOutsideNotOpen && onClose() }} /> -
+
<>
{title && = ({ = ({ datasetId, documentId }) => { }
} - setShowMetadata(false)} isMobile={isMobile} panelClassname='!justify-start' footer={null}> + setShowMetadata(false)} isMobile={isMobile} panelClassName='!justify-start' footer={null}> = ({ datasetId }: Props) => { )}
- +
{/* {renderHitResults(generalResultData)} */} {submitLoading @@ -197,7 +197,7 @@ const HitTestingPage: FC = ({ datasetId }: Props) => { }
- setIsShowModifyRetrievalModal(false)} footer={null} mask={isMobile} panelClassname='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'> + setIsShowModifyRetrievalModal(false)} footer={null} mask={isMobile} panelClassName='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'> = ({ showClose title={t('dataset.metadata.metadata')} footer={null} - panelClassname='px-4 block !max-w-[420px] my-2 rounded-l-2xl' + panelClassName='px-4 block !max-w-[420px] my-2 rounded-l-2xl' >
{t(`${i18nPrefix}.description`)}
diff --git a/web/app/components/plugins/plugin-detail-panel/endpoint-modal.tsx b/web/app/components/plugins/plugin-detail-panel/endpoint-modal.tsx index 46aaf6a7d6..fd862720af 100644 --- a/web/app/components/plugins/plugin-detail-panel/endpoint-modal.tsx +++ b/web/app/components/plugins/plugin-detail-panel/endpoint-modal.tsx @@ -46,7 +46,7 @@ const EndpointModal: FC = ({ footer={null} mask positionCenter={false} - panelClassname={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} + panelClassName={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} > <>
diff --git a/web/app/components/plugins/plugin-detail-panel/index.tsx b/web/app/components/plugins/plugin-detail-panel/index.tsx index 70bd9edabc..3ec867faae 100644 --- a/web/app/components/plugins/plugin-detail-panel/index.tsx +++ b/web/app/components/plugins/plugin-detail-panel/index.tsx @@ -38,7 +38,7 @@ const PluginDetailPanel: FC = ({ footer={null} mask={false} positionCenter={false} - panelClassname={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} + panelClassName={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} > {detail && ( <> diff --git a/web/app/components/plugins/plugin-detail-panel/strategy-detail.tsx b/web/app/components/plugins/plugin-detail-panel/strategy-detail.tsx index 89ee850e03..00794d83ed 100644 --- a/web/app/components/plugins/plugin-detail-panel/strategy-detail.tsx +++ b/web/app/components/plugins/plugin-detail-panel/strategy-detail.tsx @@ -78,7 +78,7 @@ const StrategyDetail: FC = ({ footer={null} mask={false} positionCenter={false} - panelClassname={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} + panelClassName={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} > <> {/* header */} diff --git a/web/app/components/tools/add-tool-modal/index.tsx b/web/app/components/tools/add-tool-modal/index.tsx index 1129fe55ce..c45313fc09 100644 --- a/web/app/components/tools/add-tool-modal/index.tsx +++ b/web/app/components/tools/add-tool-modal/index.tsx @@ -178,7 +178,7 @@ const AddToolModal: FC = ({ clickOutsideNotOpen onClose={onHide} footer={null} - panelClassname={cn('mx-2 mb-3 mt-16 rounded-xl !p-0 sm:mr-2', 'mt-2 !w-[640px]', '!max-w-[640px]')} + panelClassName={cn('mx-2 mb-3 mt-16 rounded-xl !p-0 sm:mr-2', 'mt-2 !w-[640px]', '!max-w-[640px]')} >
= ({ positionCenter={positionCenter} onHide={onHide} title={t('tools.createTool.authMethod.title')!} - panelClassName='mt-2 !w-[520px] h-fit' + dialogClassName='z-[60]' + dialogBackdropClassName='z-[70]' + panelClassName='mt-2 !w-[520px] h-fit z-[80]' maxWidthClassName='!max-w-[520px]' height={'fit-content'} headerClassName='!border-b-divider-regular' diff --git a/web/app/components/tools/provider/detail.tsx b/web/app/components/tools/provider/detail.tsx index 5d3a1794d8..21ea8bc464 100644 --- a/web/app/components/tools/provider/detail.tsx +++ b/web/app/components/tools/provider/detail.tsx @@ -234,7 +234,7 @@ const ProviderDetail = ({ footer={null} mask={false} positionCenter={false} - panelClassname={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} + panelClassName={cn('mb-2 mr-2 mt-[64px] !w-[420px] !max-w-[420px] justify-start rounded-2xl border-[0.5px] border-components-panel-border !bg-components-panel-bg !p-0 shadow-xl')} >
diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx index e424ea8e1f..f8d2dcfc75 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx @@ -111,7 +111,7 @@ const DatasetItem: FC = ({ } {isShowSettingsModal && ( - + Date: Tue, 22 Apr 2025 10:13:22 +0800 Subject: [PATCH 009/166] fix: filter empty marketplace collection (#18511) --- .../plugins/marketplace/list/list-with-collection.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/app/components/plugins/marketplace/list/list-with-collection.tsx b/web/app/components/plugins/marketplace/list/list-with-collection.tsx index e18356cd85..4c396c565f 100644 --- a/web/app/components/plugins/marketplace/list/list-with-collection.tsx +++ b/web/app/components/plugins/marketplace/list/list-with-collection.tsx @@ -32,7 +32,9 @@ const ListWithCollection = ({ return ( <> { - marketplaceCollections.map(collection => ( + marketplaceCollections.filter((collection) => { + return marketplaceCollectionPluginsMap[collection.name]?.length + }).map(collection => (
Date: Tue, 22 Apr 2025 11:00:22 +0800 Subject: [PATCH 010/166] fix: adjust padding and background for sticky header (#18515) --- .../datasets/settings/permission-selector/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/components/datasets/settings/permission-selector/index.tsx b/web/app/components/datasets/settings/permission-selector/index.tsx index 71a46087af..9bb6f812d4 100644 --- a/web/app/components/datasets/settings/permission-selector/index.tsx +++ b/web/app/components/datasets/settings/permission-selector/index.tsx @@ -150,8 +150,8 @@ const PermissionSelector = ({ disabled, permission, value, memberList, onChange,
{isPartialMembers && ( -
-
+
+
Date: Mon, 21 Apr 2025 23:03:01 -0400 Subject: [PATCH 011/166] feat(embedded-chatbot): support overriding locale via URL params (#18509) --- .../base/chat/embedded-chatbot/hooks.tsx | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/web/app/components/base/chat/embedded-chatbot/hooks.tsx b/web/app/components/base/chat/embedded-chatbot/hooks.tsx index d6a7b230e4..0f2529152c 100644 --- a/web/app/components/base/chat/embedded-chatbot/hooks.tsx +++ b/web/app/components/base/chat/embedded-chatbot/hooks.tsx @@ -80,8 +80,30 @@ export const useEmbeddedChatbot = () => { }, []) useEffect(() => { - if (appInfo?.site.default_language) - changeLanguage(appInfo.site.default_language) + const setLanguageFromParams = async () => { + // Check URL parameters for language override + const urlParams = new URLSearchParams(window.location.search) + const localeParam = urlParams.get('locale') + + // Check for encoded system variables + const systemVariables = await getProcessedSystemVariablesFromUrlParams() + const localeFromSysVar = systemVariables.locale + + if (localeParam) { + // If locale parameter exists in URL, use it instead of default + changeLanguage(localeParam) + } + else if (localeFromSysVar) { + // If locale is set as a system variable, use that + changeLanguage(localeFromSysVar) + } + else if (appInfo?.site.default_language) { + // Otherwise use the default from app config + changeLanguage(appInfo.site.default_language) + } + } + + setLanguageFromParams() }, [appInfo]) const [conversationIdInfo, setConversationIdInfo] = useLocalStorageState>>(CONVERSATION_ID_INFO, { From 67eefd0ba19dbcd2d36a25ba6163e61ce08a2a94 Mon Sep 17 00:00:00 2001 From: GuanMu Date: Tue, 22 Apr 2025 11:06:36 +0800 Subject: [PATCH 012/166] fix: update search model placeholder and add translations f (#18518) --- .../model-provider-page/model-selector/popup.tsx | 2 +- web/i18n/de-DE/dataset-settings.ts | 1 + web/i18n/en-US/dataset-settings.ts | 1 + web/i18n/es-ES/dataset-settings.ts | 1 + web/i18n/fa-IR/dataset-settings.ts | 1 + web/i18n/fr-FR/dataset-settings.ts | 1 + web/i18n/hi-IN/dataset-settings.ts | 1 + web/i18n/it-IT/dataset-settings.ts | 1 + web/i18n/ja-JP/dataset-settings.ts | 1 + web/i18n/ko-KR/dataset-settings.ts | 1 + web/i18n/pl-PL/dataset-settings.ts | 1 + web/i18n/pt-BR/dataset-settings.ts | 1 + web/i18n/ro-RO/dataset-settings.ts | 1 + web/i18n/ru-RU/dataset-settings.ts | 1 + web/i18n/sl-SI/dataset-settings.ts | 1 + web/i18n/th-TH/dataset-settings.ts | 1 + web/i18n/tr-TR/dataset-settings.ts | 1 + web/i18n/uk-UA/dataset-settings.ts | 1 + web/i18n/vi-VN/dataset-settings.ts | 1 + web/i18n/zh-Hans/dataset-settings.ts | 1 + web/i18n/zh-Hant/dataset-settings.ts | 1 + 21 files changed, 21 insertions(+), 1 deletion(-) diff --git a/web/app/components/header/account-setting/model-provider-page/model-selector/popup.tsx b/web/app/components/header/account-setting/model-provider-page/model-selector/popup.tsx index 6a336fb6f7..63849bddda 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-selector/popup.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-selector/popup.tsx @@ -74,7 +74,7 @@ const Popup: FC = ({ /> setSearchText(e.target.value)} /> diff --git a/web/i18n/de-DE/dataset-settings.ts b/web/i18n/de-DE/dataset-settings.ts index c871e13d4b..24cb1207b8 100644 --- a/web/i18n/de-DE/dataset-settings.ts +++ b/web/i18n/de-DE/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { upgradeHighQualityTip: 'Nach dem Upgrade auf den Modus "Hohe Qualität" ist das Zurücksetzen auf den Modus "Wirtschaftlich" nicht mehr möglich', helpText: 'Erfahren Sie, wie Sie eine gute Datensatzbeschreibung schreiben.', indexMethodChangeToEconomyDisabledTip: 'Nicht verfügbar für ein Downgrade von HQ auf ECO', + searchModel: 'Modell suchen', }, } diff --git a/web/i18n/en-US/dataset-settings.ts b/web/i18n/en-US/dataset-settings.ts index dffb96144d..bf10bed436 100644 --- a/web/i18n/en-US/dataset-settings.ts +++ b/web/i18n/en-US/dataset-settings.ts @@ -36,6 +36,7 @@ const translation = { retrievalSettings: 'Retrieval Settings', save: 'Save', indexMethodChangeToEconomyDisabledTip: 'Not available for downgrading from HQ to ECO', + searchModel: 'Search model', }, } diff --git a/web/i18n/es-ES/dataset-settings.ts b/web/i18n/es-ES/dataset-settings.ts index 211a23edd1..ee8072e278 100644 --- a/web/i18n/es-ES/dataset-settings.ts +++ b/web/i18n/es-ES/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { indexMethodChangeToEconomyDisabledTip: 'No disponible para degradar de HQ a ECO', helpText: 'Aprenda a escribir una buena descripción del conjunto de datos.', upgradeHighQualityTip: 'Una vez que se actualiza al modo de alta calidad, no está disponible volver al modo económico', + searchModel: 'Buscar modelo', }, } diff --git a/web/i18n/fa-IR/dataset-settings.ts b/web/i18n/fa-IR/dataset-settings.ts index 1ddee95e9b..0243929c36 100644 --- a/web/i18n/fa-IR/dataset-settings.ts +++ b/web/i18n/fa-IR/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { indexMethodChangeToEconomyDisabledTip: 'برای تنزل رتبه از HQ به ECO در دسترس نیست', helpText: 'یاد بگیرید که چگونه یک توضیحات مجموعه داده خوب بنویسید.', upgradeHighQualityTip: 'پس از ارتقاء به حالت کیفیت بالا، بازگشت به حالت اقتصادی در دسترس نیست', + searchModel: 'جستجوی مدل', }, } diff --git a/web/i18n/fr-FR/dataset-settings.ts b/web/i18n/fr-FR/dataset-settings.ts index 101214d288..20d8c47149 100644 --- a/web/i18n/fr-FR/dataset-settings.ts +++ b/web/i18n/fr-FR/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { indexMethodChangeToEconomyDisabledTip: 'Non disponible pour le déclassement de HQ à ECO', upgradeHighQualityTip: 'Une fois la mise à niveau vers le mode Haute Qualité, il n’est pas possible de revenir au mode Économique', helpText: 'Apprenez à rédiger une bonne description de jeu de données.', + searchModel: 'Rechercher un modèle', }, } diff --git a/web/i18n/hi-IN/dataset-settings.ts b/web/i18n/hi-IN/dataset-settings.ts index ff324dcb43..e7a383690c 100644 --- a/web/i18n/hi-IN/dataset-settings.ts +++ b/web/i18n/hi-IN/dataset-settings.ts @@ -40,6 +40,7 @@ const translation = { indexMethodChangeToEconomyDisabledTip: 'मुख्यालय से ईसीओ में डाउनग्रेड करने के लिए उपलब्ध नहीं है', helpText: 'एक अच्छा डेटासेट विवरण लिखना सीखें।', upgradeHighQualityTip: 'एक बार उच्च गुणवत्ता मोड में अपग्रेड करने के बाद, किफायती मोड में वापस जाना उपलब्ध नहीं है', + searchModel: 'मॉडल खोजें', }, } diff --git a/web/i18n/it-IT/dataset-settings.ts b/web/i18n/it-IT/dataset-settings.ts index 66c13bd3b4..c799872975 100644 --- a/web/i18n/it-IT/dataset-settings.ts +++ b/web/i18n/it-IT/dataset-settings.ts @@ -40,6 +40,7 @@ const translation = { helpText: 'Scopri come scrivere una buona descrizione del set di dati.', upgradeHighQualityTip: 'Una volta effettuato l\'aggiornamento alla modalità Alta qualità, il ripristino della modalità Risparmio non è disponibile', indexMethodChangeToEconomyDisabledTip: 'Non disponibile per il downgrade da HQ a ECO', + searchModel: 'Cerca modello', }, } diff --git a/web/i18n/ja-JP/dataset-settings.ts b/web/i18n/ja-JP/dataset-settings.ts index 9ea6aba9eb..6b809ddd43 100644 --- a/web/i18n/ja-JP/dataset-settings.ts +++ b/web/i18n/ja-JP/dataset-settings.ts @@ -36,6 +36,7 @@ const translation = { retrievalSettings: '取得設定', externalKnowledgeAPI: '外部ナレッジベースAPI', indexMethodChangeToEconomyDisabledTip: 'HQからECOへのダウングレードはできません。', + searchModel: 'モデル検索', }, } diff --git a/web/i18n/ko-KR/dataset-settings.ts b/web/i18n/ko-KR/dataset-settings.ts index 22e9733ed8..c15fff8db6 100644 --- a/web/i18n/ko-KR/dataset-settings.ts +++ b/web/i18n/ko-KR/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { upgradeHighQualityTip: '고품질 모드로 업그레이드한 후에는 경제적 모드로 되돌릴 수 없습니다.', indexMethodChangeToEconomyDisabledTip: 'HQ에서 ECO로 다운그레이드할 수 없습니다.', helpText: '좋은 데이터 세트 설명을 작성하는 방법을 알아보세요.', + searchModel: '모델 검색', }, } diff --git a/web/i18n/pl-PL/dataset-settings.ts b/web/i18n/pl-PL/dataset-settings.ts index ff2a2e5d5f..94099708b7 100644 --- a/web/i18n/pl-PL/dataset-settings.ts +++ b/web/i18n/pl-PL/dataset-settings.ts @@ -40,6 +40,7 @@ const translation = { helpText: 'Dowiedz się, jak napisać dobry opis zestawu danych.', upgradeHighQualityTip: 'Po uaktualnieniu do trybu wysokiej jakości powrót do trybu ekonomicznego nie jest dostępny', indexMethodChangeToEconomyDisabledTip: 'Niedostępne w przypadku zmiany z HQ na ECO', + searchModel: 'Szukaj modelu', }, } diff --git a/web/i18n/pt-BR/dataset-settings.ts b/web/i18n/pt-BR/dataset-settings.ts index b8176d222a..a9346c4dd0 100644 --- a/web/i18n/pt-BR/dataset-settings.ts +++ b/web/i18n/pt-BR/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { indexMethodChangeToEconomyDisabledTip: 'Não disponível para rebaixamento de HQ para ECO', helpText: 'Aprenda a escrever uma boa descrição do conjunto de dados.', upgradeHighQualityTip: 'Depois de atualizar para o modo de alta qualidade, reverter para o modo econômico não está disponível', + searchModel: 'Pesquisar modelo', }, } diff --git a/web/i18n/ro-RO/dataset-settings.ts b/web/i18n/ro-RO/dataset-settings.ts index baf86c7a8e..0627b08b79 100644 --- a/web/i18n/ro-RO/dataset-settings.ts +++ b/web/i18n/ro-RO/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { indexMethodChangeToEconomyDisabledTip: 'Nu este disponibil pentru retrogradarea de la HQ la ECO', upgradeHighQualityTip: 'După ce faceți upgrade la modul Înaltă calitate, revenirea la modul Economic nu este disponibilă', helpText: 'Aflați cum să scrieți o descriere bună a setului de date.', + searchModel: 'Căutare model', }, } diff --git a/web/i18n/ru-RU/dataset-settings.ts b/web/i18n/ru-RU/dataset-settings.ts index 82c2fafe2d..b3b8347dd2 100644 --- a/web/i18n/ru-RU/dataset-settings.ts +++ b/web/i18n/ru-RU/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { helpText: 'Узнайте, как написать хорошее описание набора данных.', upgradeHighQualityTip: 'После обновления до режима «Высокое качество» возврат к экономичному режиму невозможен', indexMethodChangeToEconomyDisabledTip: 'Недоступно для понижения уровня с HQ до ECO', + searchModel: 'Поиск модели', }, } diff --git a/web/i18n/sl-SI/dataset-settings.ts b/web/i18n/sl-SI/dataset-settings.ts index 5cd7a72a27..dc131c154e 100644 --- a/web/i18n/sl-SI/dataset-settings.ts +++ b/web/i18n/sl-SI/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { indexMethodChangeToEconomyDisabledTip: 'Ni na voljo za pregradnjo iz HQ v ECO', upgradeHighQualityTip: 'Ko nadgradite na način visoke kakovosti, vrnitev v ekonomični način ni na voljo', helpText: 'Naučite se napisati dober opis nabora podatkov.', + searchModel: 'Išči model', }, } diff --git a/web/i18n/th-TH/dataset-settings.ts b/web/i18n/th-TH/dataset-settings.ts index ec05db6824..e91834ced2 100644 --- a/web/i18n/th-TH/dataset-settings.ts +++ b/web/i18n/th-TH/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { indexMethodChangeToEconomyDisabledTip: 'ไม่สามารถดาวน์เกรดจาก HQ เป็น ECO ได้', helpText: 'เรียนรู้วิธีเขียนคําอธิบายชุดข้อมูลที่ดี', upgradeHighQualityTip: 'เมื่ออัปเกรดเป็นโหมดคุณภาพสูงแล้ว จะไม่สามารถเปลี่ยนกลับเป็นโหมดประหยัดได้', + searchModel: 'ค้นหารุ่น', }, } diff --git a/web/i18n/tr-TR/dataset-settings.ts b/web/i18n/tr-TR/dataset-settings.ts index d173563da8..554f3c7a5c 100644 --- a/web/i18n/tr-TR/dataset-settings.ts +++ b/web/i18n/tr-TR/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { upgradeHighQualityTip: 'Yüksek Kalite moduna yükselttikten sonra Ekonomik moda geri dönülemez', indexMethodChangeToEconomyDisabledTip: 'Genel Merkezden ECO\'ya düşürme için mevcut değil', helpText: 'İyi bir veri kümesi açıklamasının nasıl yazılacağını öğrenin.', + searchModel: 'Model Ara', }, } diff --git a/web/i18n/uk-UA/dataset-settings.ts b/web/i18n/uk-UA/dataset-settings.ts index ef3bd5eaa6..c56473896c 100644 --- a/web/i18n/uk-UA/dataset-settings.ts +++ b/web/i18n/uk-UA/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { helpText: 'Дізнайтеся, як написати хороший опис набору даних.', indexMethodChangeToEconomyDisabledTip: 'Недоступно для пониження з HQ до ECO', upgradeHighQualityTip: 'Після оновлення до режиму високої якості повернення до економного режиму недоступне', + searchModel: 'Пошук моделі', }, } diff --git a/web/i18n/vi-VN/dataset-settings.ts b/web/i18n/vi-VN/dataset-settings.ts index 790fd05ca8..7add91884e 100644 --- a/web/i18n/vi-VN/dataset-settings.ts +++ b/web/i18n/vi-VN/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { helpText: 'Tìm hiểu cách viết mô tả tập dữ liệu tốt.', indexMethodChangeToEconomyDisabledTip: 'Không khả dụng để hạ cấp từ HQ xuống ECO', upgradeHighQualityTip: 'Sau khi nâng cấp lên chế độ Chất lượng cao, không thể hoàn nguyên về chế độ Tiết kiệm', + searchModel: 'Tìm kiếm mô hình', }, } diff --git a/web/i18n/zh-Hans/dataset-settings.ts b/web/i18n/zh-Hans/dataset-settings.ts index 4ed0645e0f..f23355dbe1 100644 --- a/web/i18n/zh-Hans/dataset-settings.ts +++ b/web/i18n/zh-Hans/dataset-settings.ts @@ -36,6 +36,7 @@ const translation = { save: '保存', retrievalSettings: '检索设置', indexMethodChangeToEconomyDisabledTip: '无法从高质量降级为经济', + searchModel: '搜索模型', }, } diff --git a/web/i18n/zh-Hant/dataset-settings.ts b/web/i18n/zh-Hant/dataset-settings.ts index b22f899f32..768937c168 100644 --- a/web/i18n/zh-Hant/dataset-settings.ts +++ b/web/i18n/zh-Hant/dataset-settings.ts @@ -35,6 +35,7 @@ const translation = { indexMethodChangeToEconomyDisabledTip: '不適用於從 HQ 降級到 ECO', upgradeHighQualityTip: '升級到高品質模式后,無法恢復到經濟模式', helpText: '瞭解如何編寫良好的數據集描述。', + searchModel: '搜索模型', }, } From 94e22ba0fdc39f539f75d07e4c166f56772b7ccd Mon Sep 17 00:00:00 2001 From: allenZhang <58501701+441126098@users.noreply.github.com> Date: Tue, 22 Apr 2025 11:07:18 +0800 Subject: [PATCH 013/166] feat: add search input field (#18409) --- .../plugins/component-picker-block/index.tsx | 93 +++++++++++-------- .../plugins/on-blur-or-focus-block.tsx | 18 ++-- .../variable/var-reference-vars.tsx | 16 +++- 3 files changed, 77 insertions(+), 50 deletions(-) diff --git a/web/app/components/base/prompt-editor/plugins/component-picker-block/index.tsx b/web/app/components/base/prompt-editor/plugins/component-picker-block/index.tsx index d7a3a81417..562bb8c0d9 100644 --- a/web/app/components/base/prompt-editor/plugins/component-picker-block/index.tsx +++ b/web/app/components/base/prompt-editor/plugins/component-picker-block/index.tsx @@ -31,6 +31,7 @@ import { useOptions } from './hooks' import type { PickerBlockMenuOption } from './menu' import VarReferenceVars from '@/app/components/workflow/nodes/_base/components/variable/var-reference-vars' import { useEventEmitterContextContext } from '@/context/event-emitter' +import { KEY_ESCAPE_COMMAND } from 'lexical' type ComponentPickerProps = { triggerString: string @@ -118,6 +119,13 @@ const ComponentPicker = ({ editor.dispatchCommand(INSERT_WORKFLOW_VARIABLE_BLOCK_COMMAND, variables) }, [editor, checkForTriggerMatch, triggerString]) + const handleClose = useCallback(() => { + ReactDOM.flushSync(() => { + const escapeEvent = new KeyboardEvent('keydown', { key: 'Escape' }) + editor.dispatchCommand(KEY_ESCAPE_COMMAND, escapeEvent) + }) + }, [editor]) + const renderMenu = useCallback>(( anchorElementRef, { options, selectedIndex, selectOptionAndCleanUp, setHighlightedIndex }, @@ -141,51 +149,54 @@ const ComponentPicker = ({ visibility: isPositioned ? 'visible' : 'hidden', }} ref={refs.setFloating} + data-testid="component-picker-container" > - { - options.map((option, index) => ( - - { - // Divider - index !== 0 && options.at(index - 1)?.group !== option.group && ( -
- ) - } - {option.renderMenuOption({ - queryString, - isSelected: selectedIndex === index, - onSelect: () => { - selectOptionAndCleanUp(option) - }, - onSetHighlight: () => { - setHighlightedIndex(index) - }, - })} -
- )) - } { workflowVariableBlock?.show && ( - <> - { - (!!options.length) && ( -
- ) - } -
- { - handleSelectWorkflowVariable(variables) - }} - maxHeightClass='max-h-[34vh]' - isSupportFileVar={isSupportFileVar} - /> -
- +
+ { + handleSelectWorkflowVariable(variables) + }} + maxHeightClass='max-h-[34vh]' + isSupportFileVar={isSupportFileVar} + onClose={handleClose} + onBlur={handleClose} + /> +
) } + { + workflowVariableBlock?.show && !!options.length && ( +
+ ) + } +
+ { + options.map((option, index) => ( + + { + // Divider + index !== 0 && options.at(index - 1)?.group !== option.group && ( +
+ ) + } + {option.renderMenuOption({ + queryString, + isSelected: selectedIndex === index, + onSelect: () => { + selectOptionAndCleanUp(option) + }, + onSetHighlight: () => { + setHighlightedIndex(index) + }, + })} +
+ )) + } +
, anchorElementRef.current, @@ -193,7 +204,7 @@ const ComponentPicker = ({ } ) - }, [allFlattenOptions.length, workflowVariableBlock?.show, refs, isPositioned, floatingStyles, queryString, workflowVariableOptions, handleSelectWorkflowVariable]) + }, [allFlattenOptions.length, workflowVariableBlock?.show, refs, isPositioned, floatingStyles, queryString, workflowVariableOptions, handleSelectWorkflowVariable, handleClose, isSupportFileVar]) return ( = ({ ), editor.registerCommand( BLUR_COMMAND, - () => { - ref.current = setTimeout(() => { - editor.dispatchCommand(KEY_ESCAPE_COMMAND, new KeyboardEvent('keydown', { key: 'Escape' })) - }, 200) - - if (onBlur) - onBlur() - + (event) => { + // Check if the clicked target element is var-search-input + const target = event?.relatedTarget as HTMLElement + if (!target?.classList?.contains('var-search-input')) { + ref.current = setTimeout(() => { + editor.dispatchCommand(KEY_ESCAPE_COMMAND, new KeyboardEvent('keydown', { key: 'Escape' })) + }, 200) + if (onBlur) + onBlur() + } return true }, COMMAND_PRIORITY_EDITOR, diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx index 751e1990cf..023916ec5b 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx @@ -258,6 +258,8 @@ type Props = { onChange: (value: ValueSelector, item: Var) => void itemWidth?: number maxHeightClass?: string + onClose?: () => void + onBlur?: () => void } const VarReferenceVars: FC = ({ hideSearch, @@ -267,10 +269,19 @@ const VarReferenceVars: FC = ({ onChange, itemWidth, maxHeightClass, + onClose, + onBlur, }) => { const { t } = useTranslation() const [searchText, setSearchText] = useState('') + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === 'Escape') { + e.preventDefault() + onClose?.() + } + } + const filteredVars = vars.filter((v) => { const children = v.vars.filter(v => checkKeys([v.variable], false).isValid || v.variable.startsWith('sys.') || v.variable.startsWith('env.') || v.variable.startsWith('conversation.')) return children.length > 0 @@ -301,14 +312,17 @@ const VarReferenceVars: FC = ({ { !hideSearch && ( <> -
e.stopPropagation()}> +
e.stopPropagation()}> setSearchText(e.target.value)} + onKeyDown={handleKeyDown} onClear={() => setSearchText('')} + onBlur={onBlur} autoFocus />
From e0e92921b5cb7d3ae1700e4c700bcbdbd712a705 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Tue, 22 Apr 2025 11:29:45 +0800 Subject: [PATCH 014/166] fix: external knowledge setting in knowledge selector (#18519) --- .../dataset-config/settings-modal/index.tsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx b/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx index 90885dacc8..645f6045f0 100644 --- a/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx +++ b/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx @@ -62,13 +62,13 @@ const SettingsModal: FC = ({ const { notify } = useToastContext() const ref = useRef(null) const isExternal = currentDataset.provider === 'external' - const [topK, setTopK] = useState(currentDataset?.external_retrieval_model.top_k ?? 2) - const [scoreThreshold, setScoreThreshold] = useState(currentDataset?.external_retrieval_model.score_threshold ?? 0.5) - const [scoreThresholdEnabled, setScoreThresholdEnabled] = useState(currentDataset?.external_retrieval_model.score_threshold_enabled ?? false) const { setShowAccountSettingModal } = useModalContext() const [loading, setLoading] = useState(false) const { isCurrentWorkspaceDatasetOperator } = useAppContext() const [localeCurrentDataset, setLocaleCurrentDataset] = useState({ ...currentDataset }) + const [topK, setTopK] = useState(localeCurrentDataset?.external_retrieval_model.top_k ?? 2) + const [scoreThreshold, setScoreThreshold] = useState(localeCurrentDataset?.external_retrieval_model.score_threshold ?? 0.5) + const [scoreThresholdEnabled, setScoreThresholdEnabled] = useState(localeCurrentDataset?.external_retrieval_model.score_threshold_enabled ?? false) const [selectedMemberIDs, setSelectedMemberIDs] = useState(currentDataset.partial_member_list || []) const [memberList, setMemberList] = useState([]) @@ -88,6 +88,14 @@ const SettingsModal: FC = ({ setScoreThreshold(data.score_threshold) if (data.score_threshold_enabled !== undefined) setScoreThresholdEnabled(data.score_threshold_enabled) + + setLocaleCurrentDataset({ + ...localeCurrentDataset, + external_retrieval_model: { + ...localeCurrentDataset?.external_retrieval_model, + ...data, + }, + }) } const handleSave = async () => { From 18e4f42c3cd0423ede63035ae099c55a3b6f75db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?= Date: Tue, 22 Apr 2025 13:02:38 +0800 Subject: [PATCH 015/166] fix draft run node exception (#18520) --- api/services/workflow_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 5cd5c55746..63e3791147 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -289,7 +289,7 @@ class WorkflowService: params={ "tenant_id": app_model.tenant_id, "app_id": app_model.id, - "session_factory": db.session.get_bind, + "session_factory": db.session.get_bind(), } ) repository.save(workflow_node_execution) From eb1ce3dd6bc7e8c2883e37da1812a1dac8382c92 Mon Sep 17 00:00:00 2001 From: lauding <719880851@qq.com> Date: Tue, 22 Apr 2025 13:03:35 +0800 Subject: [PATCH 016/166] feat: support huawei cloud vector database (#16141) --- api/configs/middleware/__init__.py | 2 + .../middleware/vdb/huawei_cloud_config.py | 25 ++ api/controllers/console/datasets/datasets.py | 2 + .../rag/datasource/vdb/huawei/__init__.py | 0 .../vdb/huawei/huawei_cloud_vector.py | 215 ++++++++++++++++++ api/core/rag/datasource/vdb/vector_factory.py | 4 + api/core/rag/datasource/vdb/vector_type.py | 1 + .../vdb/__mock/huaweicloudvectordb.py | 88 +++++++ .../integration_tests/vdb/huawei/__init__.py | 0 .../vdb/huawei/test_huawei_cloud.py | 28 +++ dev/pytest/pytest_vdb.sh | 1 + docker/.env.example | 5 + docker/docker-compose.yaml | 3 + 13 files changed, 374 insertions(+) create mode 100644 api/configs/middleware/vdb/huawei_cloud_config.py create mode 100644 api/core/rag/datasource/vdb/huawei/__init__.py create mode 100644 api/core/rag/datasource/vdb/huawei/huawei_cloud_vector.py create mode 100644 api/tests/integration_tests/vdb/__mock/huaweicloudvectordb.py create mode 100644 api/tests/integration_tests/vdb/huawei/__init__.py create mode 100644 api/tests/integration_tests/vdb/huawei/test_huawei_cloud.py diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index 15dfe0063b..c2ad24094a 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -22,6 +22,7 @@ from .vdb.baidu_vector_config import BaiduVectorDBConfig from .vdb.chroma_config import ChromaConfig from .vdb.couchbase_config import CouchbaseConfig from .vdb.elasticsearch_config import ElasticsearchConfig +from .vdb.huawei_cloud_config import HuaweiCloudConfig from .vdb.lindorm_config import LindormConfig from .vdb.milvus_config import MilvusConfig from .vdb.myscale_config import MyScaleConfig @@ -263,6 +264,7 @@ class MiddlewareConfig( VectorStoreConfig, AnalyticdbConfig, ChromaConfig, + HuaweiCloudConfig, MilvusConfig, MyScaleConfig, OpenSearchConfig, diff --git a/api/configs/middleware/vdb/huawei_cloud_config.py b/api/configs/middleware/vdb/huawei_cloud_config.py new file mode 100644 index 0000000000..2290c60499 --- /dev/null +++ b/api/configs/middleware/vdb/huawei_cloud_config.py @@ -0,0 +1,25 @@ +from typing import Optional + +from pydantic import Field +from pydantic_settings import BaseSettings + + +class HuaweiCloudConfig(BaseSettings): + """ + Configuration settings for Huawei cloud search service + """ + + HUAWEI_CLOUD_HOSTS: Optional[str] = Field( + description="Hostname or IP address of the Huawei cloud search service instance", + default=None, + ) + + HUAWEI_CLOUD_USER: Optional[str] = Field( + description="Username for authenticating with Huawei cloud search service", + default=None, + ) + + HUAWEI_CLOUD_PASSWORD: Optional[str] = Field( + description="Password for authenticating with Huawei cloud search service", + default=None, + ) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 4644ac6299..752d124735 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -664,6 +664,7 @@ class DatasetRetrievalSettingApi(Resource): | VectorType.OPENGAUSS | VectorType.OCEANBASE | VectorType.TABLESTORE + | VectorType.HUAWEI_CLOUD | VectorType.TENCENT ): return { @@ -710,6 +711,7 @@ class DatasetRetrievalSettingMockApi(Resource): | VectorType.OCEANBASE | VectorType.TABLESTORE | VectorType.TENCENT + | VectorType.HUAWEI_CLOUD ): return { "retrieval_method": [ diff --git a/api/core/rag/datasource/vdb/huawei/__init__.py b/api/core/rag/datasource/vdb/huawei/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/rag/datasource/vdb/huawei/huawei_cloud_vector.py b/api/core/rag/datasource/vdb/huawei/huawei_cloud_vector.py new file mode 100644 index 0000000000..89423eb160 --- /dev/null +++ b/api/core/rag/datasource/vdb/huawei/huawei_cloud_vector.py @@ -0,0 +1,215 @@ +import json +import logging +import ssl +from typing import Any, Optional + +from elasticsearch import Elasticsearch +from pydantic import BaseModel, model_validator + +from configs import dify_config +from core.rag.datasource.vdb.field import Field +from core.rag.datasource.vdb.vector_base import BaseVector +from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory +from core.rag.datasource.vdb.vector_type import VectorType +from core.rag.embedding.embedding_base import Embeddings +from core.rag.models.document import Document +from extensions.ext_redis import redis_client +from models.dataset import Dataset + +logger = logging.getLogger(__name__) + + +def create_ssl_context() -> ssl.SSLContext: + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + return ssl_context + + +class HuaweiCloudVectorConfig(BaseModel): + hosts: str + username: str | None + password: str | None + + @model_validator(mode="before") + @classmethod + def validate_config(cls, values: dict) -> dict: + if not values["hosts"]: + raise ValueError("config HOSTS is required") + return values + + def to_elasticsearch_params(self) -> dict[str, Any]: + params = { + "hosts": self.hosts.split(","), + "verify_certs": False, + "ssl_show_warn": False, + "request_timeout": 30000, + "retry_on_timeout": True, + "max_retries": 10, + } + if self.username and self.password: + params["basic_auth"] = (self.username, self.password) + return params + + +class HuaweiCloudVector(BaseVector): + def __init__(self, index_name: str, config: HuaweiCloudVectorConfig): + super().__init__(index_name.lower()) + self._client = Elasticsearch(**config.to_elasticsearch_params()) + + def get_type(self) -> str: + return VectorType.HUAWEI_CLOUD + + def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs): + uuids = self._get_uuids(documents) + for i in range(len(documents)): + self._client.index( + index=self._collection_name, + id=uuids[i], + document={ + Field.CONTENT_KEY.value: documents[i].page_content, + Field.VECTOR.value: embeddings[i] or None, + Field.METADATA_KEY.value: documents[i].metadata or {}, + }, + ) + self._client.indices.refresh(index=self._collection_name) + return uuids + + def text_exists(self, id: str) -> bool: + return bool(self._client.exists(index=self._collection_name, id=id)) + + def delete_by_ids(self, ids: list[str]) -> None: + if not ids: + return + for id in ids: + self._client.delete(index=self._collection_name, id=id) + + def delete_by_metadata_field(self, key: str, value: str) -> None: + query_str = {"query": {"match": {f"metadata.{key}": f"{value}"}}} + results = self._client.search(index=self._collection_name, body=query_str) + ids = [hit["_id"] for hit in results["hits"]["hits"]] + if ids: + self.delete_by_ids(ids) + + def delete(self) -> None: + self._client.indices.delete(index=self._collection_name) + + def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: + top_k = kwargs.get("top_k", 4) + + query = { + "size": top_k, + "query": { + "vector": { + Field.VECTOR.value: { + "vector": query_vector, + "topk": top_k, + } + } + }, + } + + results = self._client.search(index=self._collection_name, body=query) + + docs_and_scores = [] + for hit in results["hits"]["hits"]: + docs_and_scores.append( + ( + Document( + page_content=hit["_source"][Field.CONTENT_KEY.value], + vector=hit["_source"][Field.VECTOR.value], + metadata=hit["_source"][Field.METADATA_KEY.value], + ), + hit["_score"], + ) + ) + + docs = [] + for doc, score in docs_and_scores: + score_threshold = float(kwargs.get("score_threshold") or 0.0) + if score > score_threshold: + if doc.metadata is not None: + doc.metadata["score"] = score + docs.append(doc) + + return docs + + def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: + query_str = {"match": {Field.CONTENT_KEY.value: query}} + results = self._client.search(index=self._collection_name, query=query_str, size=kwargs.get("top_k", 4)) + docs = [] + for hit in results["hits"]["hits"]: + docs.append( + Document( + page_content=hit["_source"][Field.CONTENT_KEY.value], + vector=hit["_source"][Field.VECTOR.value], + metadata=hit["_source"][Field.METADATA_KEY.value], + ) + ) + + return docs + + def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): + metadatas = [d.metadata if d.metadata is not None else {} for d in texts] + self.create_collection(embeddings, metadatas) + self.add_texts(texts, embeddings, **kwargs) + + def create_collection( + self, + embeddings: list[list[float]], + metadatas: Optional[list[dict[Any, Any]]] = None, + index_params: Optional[dict] = None, + ): + lock_name = f"vector_indexing_lock_{self._collection_name}" + with redis_client.lock(lock_name, timeout=20): + collection_exist_cache_key = f"vector_indexing_{self._collection_name}" + if redis_client.get(collection_exist_cache_key): + logger.info(f"Collection {self._collection_name} already exists.") + return + + if not self._client.indices.exists(index=self._collection_name): + dim = len(embeddings[0]) + mappings = { + "properties": { + Field.CONTENT_KEY.value: {"type": "text"}, + Field.VECTOR.value: { # Make sure the dimension is correct here + "type": "vector", + "dimension": dim, + "indexing": True, + "algorithm": "GRAPH", + "metric": "cosine", + "neighbors": 32, + "efc": 128, + }, + Field.METADATA_KEY.value: { + "type": "object", + "properties": { + "doc_id": {"type": "keyword"} # Map doc_id to keyword type + }, + }, + } + } + settings = {"index.vector": True} + self._client.indices.create(index=self._collection_name, mappings=mappings, settings=settings) + + redis_client.set(collection_exist_cache_key, 1, ex=3600) + + +class HuaweiCloudVectorFactory(AbstractVectorFactory): + def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> HuaweiCloudVector: + if dataset.index_struct_dict: + class_prefix: str = dataset.index_struct_dict["vector_store"]["class_prefix"] + collection_name = class_prefix.lower() + else: + dataset_id = dataset.id + collection_name = Dataset.gen_collection_name_by_id(dataset_id).lower() + dataset.index_struct = json.dumps(self.gen_index_struct_dict(VectorType.HUAWEI_CLOUD, collection_name)) + + return HuaweiCloudVector( + index_name=collection_name, + config=HuaweiCloudVectorConfig( + hosts=dify_config.HUAWEI_CLOUD_HOSTS or "http://localhost:9200", + username=dify_config.HUAWEI_CLOUD_USER, + password=dify_config.HUAWEI_CLOUD_PASSWORD, + ), + ) diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index 00601c38a1..05158cc7ca 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -156,6 +156,10 @@ class Vector: from core.rag.datasource.vdb.tablestore.tablestore_vector import TableStoreVectorFactory return TableStoreVectorFactory + case VectorType.HUAWEI_CLOUD: + from core.rag.datasource.vdb.huawei.huawei_cloud_vector import HuaweiCloudVectorFactory + + return HuaweiCloudVectorFactory case _: raise ValueError(f"Vector store {vector_type} is not supported.") diff --git a/api/core/rag/datasource/vdb/vector_type.py b/api/core/rag/datasource/vdb/vector_type.py index 940f12caef..0421be3458 100644 --- a/api/core/rag/datasource/vdb/vector_type.py +++ b/api/core/rag/datasource/vdb/vector_type.py @@ -26,3 +26,4 @@ class VectorType(StrEnum): OCEANBASE = "oceanbase" OPENGAUSS = "opengauss" TABLESTORE = "tablestore" + HUAWEI_CLOUD = "huawei_cloud" diff --git a/api/tests/integration_tests/vdb/__mock/huaweicloudvectordb.py b/api/tests/integration_tests/vdb/__mock/huaweicloudvectordb.py new file mode 100644 index 0000000000..e1aba4e2c1 --- /dev/null +++ b/api/tests/integration_tests/vdb/__mock/huaweicloudvectordb.py @@ -0,0 +1,88 @@ +import os + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from api.core.rag.datasource.vdb.field import Field +from elasticsearch import Elasticsearch + + +class MockIndicesClient: + def __init__(self): + pass + + def create(self, index, mappings, settings): + return {"acknowledge": True} + + def refresh(self, index): + return {"acknowledge": True} + + def delete(self, index): + return {"acknowledge": True} + + def exists(self, index): + return True + + +class MockClient: + def __init__(self, **kwargs): + self.indices = MockIndicesClient() + + def index(self, **kwargs): + return {"acknowledge": True} + + def exists(self, **kwargs): + return True + + def delete(self, **kwargs): + return {"acknowledge": True} + + def search(self, **kwargs): + return { + "took": 1, + "hits": { + "hits": [ + { + "_source": { + Field.CONTENT_KEY.value: "abcdef", + Field.VECTOR.value: [1, 2], + Field.METADATA_KEY.value: {}, + }, + "_score": 1.0, + }, + { + "_source": { + Field.CONTENT_KEY.value: "123456", + Field.VECTOR.value: [2, 2], + Field.METADATA_KEY.value: {}, + }, + "_score": 0.9, + }, + { + "_source": { + Field.CONTENT_KEY.value: "a1b2c3", + Field.VECTOR.value: [3, 2], + Field.METADATA_KEY.value: {}, + }, + "_score": 0.8, + }, + ] + }, + } + + +MOCK = os.getenv("MOCK_SWITCH", "false").lower() == "true" + + +@pytest.fixture +def setup_client_mock(request, monkeypatch: MonkeyPatch): + if MOCK: + monkeypatch.setattr(Elasticsearch, "__init__", MockClient.__init__) + monkeypatch.setattr(Elasticsearch, "index", MockClient.index) + monkeypatch.setattr(Elasticsearch, "exists", MockClient.exists) + monkeypatch.setattr(Elasticsearch, "delete", MockClient.delete) + monkeypatch.setattr(Elasticsearch, "search", MockClient.search) + + yield + + if MOCK: + monkeypatch.undo() diff --git a/api/tests/integration_tests/vdb/huawei/__init__.py b/api/tests/integration_tests/vdb/huawei/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/integration_tests/vdb/huawei/test_huawei_cloud.py b/api/tests/integration_tests/vdb/huawei/test_huawei_cloud.py new file mode 100644 index 0000000000..943b2bc877 --- /dev/null +++ b/api/tests/integration_tests/vdb/huawei/test_huawei_cloud.py @@ -0,0 +1,28 @@ +from core.rag.datasource.vdb.huawei.huawei_cloud_vector import HuaweiCloudVector, HuaweiCloudVectorConfig +from tests.integration_tests.vdb.__mock.huaweicloudvectordb import setup_client_mock +from tests.integration_tests.vdb.test_vector_store import AbstractVectorTest, get_example_text, setup_mock_redis + + +class HuaweiCloudVectorTest(AbstractVectorTest): + def __init__(self): + super().__init__() + self.vector = HuaweiCloudVector( + "dify", + HuaweiCloudVectorConfig( + hosts="https://127.0.0.1:9200", + username="dify", + password="dify", + ), + ) + + def search_by_vector(self): + hits_by_vector = self.vector.search_by_vector(query_vector=self.example_embedding) + assert len(hits_by_vector) == 3 + + def search_by_full_text(self): + hits_by_full_text = self.vector.search_by_full_text(query=get_example_text()) + assert len(hits_by_full_text) == 3 + + +def test_huawei_cloud_vector(setup_mock_redis, setup_client_mock): + HuaweiCloudVectorTest().run_all_tests() diff --git a/dev/pytest/pytest_vdb.sh b/dev/pytest/pytest_vdb.sh index c68a94c79b..dd03ca3514 100755 --- a/dev/pytest/pytest_vdb.sh +++ b/dev/pytest/pytest_vdb.sh @@ -15,3 +15,4 @@ pytest api/tests/integration_tests/vdb/chroma \ api/tests/integration_tests/vdb/couchbase \ api/tests/integration_tests/vdb/oceanbase \ api/tests/integration_tests/vdb/tidb_vector \ + api/tests/integration_tests/vdb/huawei \ diff --git a/docker/.env.example b/docker/.env.example index 82ef4174c2..f8310a10f1 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -574,6 +574,11 @@ OPENGAUSS_MIN_CONNECTION=1 OPENGAUSS_MAX_CONNECTION=5 OPENGAUSS_ENABLE_PQ=false +# huawei cloud search service vector configurations, only available when VECTOR_STORE is `huawei_cloud` +HUAWEI_CLOUD_HOSTS=https://127.0.0.1:9200 +HUAWEI_CLOUD_USER=admin +HUAWEI_CLOUD_PASSWORD=admin + # Upstash Vector configuration, only available when VECTOR_STORE is `upstash` UPSTASH_VECTOR_URL=https://xxx-vector.upstash.io UPSTASH_VECTOR_TOKEN=dify diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index def4b77c65..d8ff7d841a 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -266,6 +266,9 @@ x-shared-env: &shared-api-worker-env OPENGAUSS_MIN_CONNECTION: ${OPENGAUSS_MIN_CONNECTION:-1} OPENGAUSS_MAX_CONNECTION: ${OPENGAUSS_MAX_CONNECTION:-5} OPENGAUSS_ENABLE_PQ: ${OPENGAUSS_ENABLE_PQ:-false} + HUAWEI_CLOUD_HOSTS: ${HUAWEI_CLOUD_HOSTS:-https://127.0.0.1:9200} + HUAWEI_CLOUD_USER: ${HUAWEI_CLOUD_USER:-admin} + HUAWEI_CLOUD_PASSWORD: ${HUAWEI_CLOUD_PASSWORD:-admin} UPSTASH_VECTOR_URL: ${UPSTASH_VECTOR_URL:-https://xxx-vector.upstash.io} UPSTASH_VECTOR_TOKEN: ${UPSTASH_VECTOR_TOKEN:-dify} TABLESTORE_ENDPOINT: ${TABLESTORE_ENDPOINT:-https://instance-name.cn-hangzhou.ots.aliyuncs.com} From 413271eaa65e758b9d5573565ae246630a0819c9 Mon Sep 17 00:00:00 2001 From: Dongyu Li <544104925@qq.com> Date: Tue, 22 Apr 2025 13:05:42 +0800 Subject: [PATCH 017/166] =?UTF-8?q?feat[plugin]:The=20plugin=20upload=20fi?= =?UTF-8?q?le=20change=20to=20be=20stored=20as=20a=20toolfile=E2=80=A6=20(?= =?UTF-8?q?#18277)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/controllers/files/upload.py | 23 ++++++++++++++++------- api/fields/file_fields.py | 1 + 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/api/controllers/files/upload.py b/api/controllers/files/upload.py index ca5ea54435..28ee0eecf4 100644 --- a/api/controllers/files/upload.py +++ b/api/controllers/files/upload.py @@ -1,3 +1,5 @@ +from mimetypes import guess_extension + from flask import request from flask_restful import Resource, marshal_with # type: ignore from werkzeug.exceptions import Forbidden @@ -9,8 +11,8 @@ from controllers.files.error import UnsupportedFileTypeError from controllers.inner_api.plugin.wraps import get_user from controllers.service_api.app.error import FileTooLargeError from core.file.helpers import verify_plugin_file_signature +from core.tools.tool_file_manager import ToolFileManager from fields.file_fields import file_fields -from services.file_service import FileService class PluginUploadFileApi(Resource): @@ -51,19 +53,26 @@ class PluginUploadFileApi(Resource): raise Forbidden("Invalid request.") try: - upload_file = FileService.upload_file( - filename=filename, - content=file.read(), + tool_file = ToolFileManager.create_file_by_raw( + user_id=user.id, + tenant_id=tenant_id, + file_binary=file.read(), mimetype=mimetype, - user=user, - source=None, + filename=filename, + conversation_id=None, ) + + extension = guess_extension(tool_file.mimetype) or ".bin" + preview_url = ToolFileManager.sign_file(tool_file_id=tool_file.id, extension=extension) + tool_file.mime_type = mimetype + tool_file.extension = extension + tool_file.preview_url = preview_url except services.errors.file.FileTooLargeError as file_too_large_error: raise FileTooLargeError(file_too_large_error.description) except services.errors.file.UnsupportedFileTypeError: raise UnsupportedFileTypeError() - return upload_file, 201 + return tool_file, 201 api.add_resource(PluginUploadFileApi, "/files/upload/for-plugin") diff --git a/api/fields/file_fields.py b/api/fields/file_fields.py index f896c15f0f..dfc1b623d5 100644 --- a/api/fields/file_fields.py +++ b/api/fields/file_fields.py @@ -19,6 +19,7 @@ file_fields = { "mime_type": fields.String, "created_by": fields.String, "created_at": TimestampField, + "preview_url": fields.String, } remote_file_info_fields = { From ef188564f30ab4feafe1dda015642cf2af800305 Mon Sep 17 00:00:00 2001 From: "Charlie.Wei" Date: Tue, 22 Apr 2025 13:06:47 +0800 Subject: [PATCH 018/166] Mermaid analysis optimization (#18089) Co-authored-by: luowei Co-authored-by: crazywoola <427733928@qq.com> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- web/app/components/base/mermaid/index.tsx | 645 ++++++++++++++++++---- web/app/components/base/mermaid/utils.ts | 233 ++++++++ 2 files changed, 776 insertions(+), 102 deletions(-) diff --git a/web/app/components/base/mermaid/index.tsx b/web/app/components/base/mermaid/index.tsx index 6ed5cfab23..8fd8ae8b59 100644 --- a/web/app/components/base/mermaid/index.tsx +++ b/web/app/components/base/mermaid/index.tsx @@ -1,116 +1,528 @@ -import React, { useCallback, useEffect, useRef, useState } from 'react' +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' import mermaid from 'mermaid' -import { usePrevious } from 'ahooks' import { useTranslation } from 'react-i18next' import { ExclamationTriangleIcon } from '@heroicons/react/24/outline' -import { cleanUpSvgCode } from './utils' +import { MoonIcon, SunIcon } from '@heroicons/react/24/solid' +import { + cleanUpSvgCode, + isMermaidCodeComplete, + prepareMermaidCode, + processSvgForTheme, + svgToBase64, + waitForDOMElement, +} from './utils' import LoadingAnim from '@/app/components/base/chat/chat/loading-anim' import cn from '@/utils/classnames' import ImagePreview from '@/app/components/base/image-uploader/image-preview' +import { Theme } from '@/types/app' -let mermaidAPI: any -mermaidAPI = null +// Global flags and cache for mermaid +let isMermaidInitialized = false +const diagramCache = new Map() +let mermaidAPI: any = null if (typeof window !== 'undefined') mermaidAPI = mermaid.mermaidAPI -const svgToBase64 = (svgGraph: string) => { - const svgBytes = new TextEncoder().encode(svgGraph) - const blob = new Blob([svgBytes], { type: 'image/svg+xml;charset=utf-8' }) - return new Promise((resolve, reject) => { - const reader = new FileReader() - reader.onloadend = () => resolve(reader.result) - reader.onerror = reject - reader.readAsDataURL(blob) - }) +// Theme configurations +const THEMES = { + light: { + name: 'Light Theme', + background: '#ffffff', + primaryColor: '#ffffff', + primaryBorderColor: '#000000', + primaryTextColor: '#000000', + secondaryColor: '#ffffff', + tertiaryColor: '#ffffff', + nodeColors: [ + { bg: '#f0f9ff', color: '#0369a1' }, + { bg: '#f0fdf4', color: '#166534' }, + { bg: '#fef2f2', color: '#b91c1c' }, + { bg: '#faf5ff', color: '#7e22ce' }, + { bg: '#fffbeb', color: '#b45309' }, + ], + connectionColor: '#74a0e0', + }, + dark: { + name: 'Dark Theme', + background: '#1e293b', + primaryColor: '#334155', + primaryBorderColor: '#94a3b8', + primaryTextColor: '#e2e8f0', + secondaryColor: '#475569', + tertiaryColor: '#334155', + nodeColors: [ + { bg: '#164e63', color: '#e0f2fe' }, + { bg: '#14532d', color: '#dcfce7' }, + { bg: '#7f1d1d', color: '#fee2e2' }, + { bg: '#581c87', color: '#f3e8ff' }, + { bg: '#78350f', color: '#fef3c7' }, + ], + connectionColor: '#60a5fa', + }, } -const Flowchart = ( - { - ref, - ...props - }: { - PrimitiveCode: string - } & { - ref: React.RefObject; - }, -) => { - const { t } = useTranslation() - const [svgCode, setSvgCode] = useState(null) - const [look, setLook] = useState<'classic' | 'handDrawn'>('classic') - - const prevPrimitiveCode = usePrevious(props.PrimitiveCode) - const [isLoading, setIsLoading] = useState(true) - const timeRef = useRef(0) - const [errMsg, setErrMsg] = useState('') - const [imagePreviewUrl, setImagePreviewUrl] = useState('') - - const renderFlowchart = useCallback(async (PrimitiveCode: string) => { - setSvgCode(null) - setIsLoading(true) - +/** + * Initializes mermaid library with default configuration + */ +const initMermaid = () => { + if (typeof window !== 'undefined' && !isMermaidInitialized) { try { - if (typeof window !== 'undefined' && mermaidAPI) { - const svgGraph = await mermaidAPI.render('flowchart', PrimitiveCode) - const base64Svg: any = await svgToBase64(cleanUpSvgCode(svgGraph.svg)) - setSvgCode(base64Svg) - setIsLoading(false) - } - } - catch (error) { - if (prevPrimitiveCode === props.PrimitiveCode) { - setIsLoading(false) - setErrMsg((error as Error).message) - } - } - }, [props.PrimitiveCode]) - - useEffect(() => { - if (typeof window !== 'undefined') { mermaid.initialize({ - startOnLoad: true, - theme: 'neutral', - look, + startOnLoad: false, + fontFamily: 'sans-serif', + securityLevel: 'loose', flowchart: { htmlLabels: true, useMaxWidth: true, + diagramPadding: 10, + curve: 'basis', + nodeSpacing: 50, + rankSpacing: 70, }, + gantt: { + titleTopMargin: 25, + barHeight: 20, + barGap: 4, + topPadding: 50, + leftPadding: 75, + gridLineStartPadding: 35, + fontSize: 11, + numberSectionStyles: 4, + axisFormat: '%Y-%m-%d', + }, + maxTextSize: 50000, }) - - renderFlowchart(props.PrimitiveCode) + isMermaidInitialized = true } - }, [look]) + catch (error) { + console.error('Mermaid initialization error:', error) + return null + } + } + return isMermaidInitialized +} +const Flowchart = React.forwardRef((props: { + PrimitiveCode: string + theme?: 'light' | 'dark' +}, ref) => { + const { t } = useTranslation() + const [svgCode, setSvgCode] = useState(null) + const [look, setLook] = useState<'classic' | 'handDrawn'>('classic') + const [isInitialized, setIsInitialized] = useState(false) + const [currentTheme, setCurrentTheme] = useState<'light' | 'dark'>(props.theme || 'light') + const containerRef = useRef(null) + const chartId = useRef(`mermaid-chart-${Math.random().toString(36).substr(2, 9)}`).current + const [isLoading, setIsLoading] = useState(true) + const renderTimeoutRef = useRef() + const [errMsg, setErrMsg] = useState('') + const [imagePreviewUrl, setImagePreviewUrl] = useState('') + const [isCodeComplete, setIsCodeComplete] = useState(false) + const codeCompletionCheckRef = useRef() + + // Create cache key from code, style and theme + const cacheKey = useMemo(() => { + return `${props.PrimitiveCode}-${look}-${currentTheme}` + }, [props.PrimitiveCode, look, currentTheme]) + + /** + * Renders Mermaid chart + */ + const renderMermaidChart = async (code: string, style: 'classic' | 'handDrawn') => { + if (style === 'handDrawn') { + // Special handling for hand-drawn style + if (containerRef.current) + containerRef.current.innerHTML = `
` + await new Promise(resolve => setTimeout(resolve, 30)) + + if (typeof window !== 'undefined' && mermaidAPI) { + // Prefer using mermaidAPI directly for hand-drawn style + return await mermaidAPI.render(chartId, code) + } + else { + // Fall back to standard rendering if mermaidAPI is not available + const { svg } = await mermaid.render(chartId, code) + return { svg } + } + } + else { + // Standard rendering for classic style - using the extracted waitForDOMElement function + const renderWithRetry = async () => { + if (containerRef.current) + containerRef.current.innerHTML = `
` + await new Promise(resolve => setTimeout(resolve, 30)) + const { svg } = await mermaid.render(chartId, code) + return { svg } + } + return await waitForDOMElement(renderWithRetry) + } + } + + /** + * Handle rendering errors + */ + const handleRenderError = (error: any) => { + console.error('Mermaid rendering error:', error) + const errorMsg = (error as Error).message + + if (errorMsg.includes('getAttribute')) { + diagramCache.clear() + mermaid.initialize({ + startOnLoad: false, + securityLevel: 'loose', + }) + } + else { + setErrMsg(`Rendering chart failed, please refresh and try again ${look === 'handDrawn' ? 'Or try using classic mode' : ''}`) + } + + if (look === 'handDrawn') { + try { + // Clear possible cache issues + diagramCache.delete(`${props.PrimitiveCode}-handDrawn-${currentTheme}`) + + // Reset mermaid configuration + mermaid.initialize({ + startOnLoad: false, + securityLevel: 'loose', + theme: 'default', + maxTextSize: 50000, + }) + + // Try rendering with standard mode + setLook('classic') + setErrMsg('Hand-drawn mode is not supported for this diagram. Switched to classic mode.') + + // Delay error clearing + setTimeout(() => { + if (containerRef.current) { + // Try rendering again with standard mode, but can't call renderFlowchart directly due to circular dependency + // Instead set state to trigger re-render + setIsCodeComplete(true) // This will trigger useEffect re-render + } + }, 500) + } + catch (e) { + console.error('Reset after handDrawn error failed:', e) + } + } + + setIsLoading(false) + } + + // Initialize mermaid useEffect(() => { - if (timeRef.current) - window.clearTimeout(timeRef.current) + const api = initMermaid() + if (api) + setIsInitialized(true) + }, []) - timeRef.current = window.setTimeout(() => { - renderFlowchart(props.PrimitiveCode) + // Update theme when prop changes + useEffect(() => { + if (props.theme) + setCurrentTheme(props.theme) + }, [props.theme]) + + // Validate mermaid code and check for completeness + useEffect(() => { + if (codeCompletionCheckRef.current) + clearTimeout(codeCompletionCheckRef.current) + + // Reset code complete status when code changes + setIsCodeComplete(false) + + // If no code or code is extremely short, don't proceed + if (!props.PrimitiveCode || props.PrimitiveCode.length < 10) + return + + // Check if code already in cache - if so we know it's valid + if (diagramCache.has(cacheKey)) { + setIsCodeComplete(true) + return + } + + // Initial check using the extracted isMermaidCodeComplete function + const isComplete = isMermaidCodeComplete(props.PrimitiveCode) + if (isComplete) { + setIsCodeComplete(true) + return + } + + // Set a delay to check again in case code is still being generated + codeCompletionCheckRef.current = setTimeout(() => { + setIsCodeComplete(isMermaidCodeComplete(props.PrimitiveCode)) }, 300) - }, [props.PrimitiveCode]) + + return () => { + if (codeCompletionCheckRef.current) + clearTimeout(codeCompletionCheckRef.current) + } + }, [props.PrimitiveCode, cacheKey]) + + /** + * Renders flowchart based on provided code + */ + const renderFlowchart = useCallback(async (primitiveCode: string) => { + if (!isInitialized || !containerRef.current) { + setIsLoading(false) + setErrMsg(!isInitialized ? 'Mermaid initialization failed' : 'Container element not found') + return + } + + // Don't render if code is not complete yet + if (!isCodeComplete) { + setIsLoading(true) + return + } + + // Return cached result if available + if (diagramCache.has(cacheKey)) { + setSvgCode(diagramCache.get(cacheKey) || null) + setIsLoading(false) + return + } + + setIsLoading(true) + setErrMsg('') + + try { + let finalCode: string + + // Check if it's a gantt chart + const isGanttChart = primitiveCode.trim().startsWith('gantt') + + if (isGanttChart) { + // For gantt charts, ensure each task is on its own line + // and preserve exact whitespace/format + finalCode = primitiveCode.trim() + } + else { + // Step 1: Clean and prepare Mermaid code using the extracted prepareMermaidCode function + finalCode = prepareMermaidCode(primitiveCode, look) + } + + // Step 2: Render chart + const svgGraph = await renderMermaidChart(finalCode, look) + + // Step 3: Apply theme to SVG using the extracted processSvgForTheme function + const processedSvg = processSvgForTheme( + svgGraph.svg, + currentTheme === Theme.dark, + look === 'handDrawn', + THEMES, + ) + + // Step 4: Clean SVG code and convert to base64 using the extracted functions + const cleanedSvg = cleanUpSvgCode(processedSvg) + const base64Svg = await svgToBase64(cleanedSvg) + + if (base64Svg && typeof base64Svg === 'string') { + diagramCache.set(cacheKey, base64Svg) + setSvgCode(base64Svg) + } + + setIsLoading(false) + } + catch (error) { + // Error handling + handleRenderError(error) + } + }, [chartId, isInitialized, cacheKey, isCodeComplete, look, currentTheme, t]) + + /** + * Configure mermaid based on selected style and theme + */ + const configureMermaid = useCallback(() => { + if (typeof window !== 'undefined' && isInitialized) { + const themeVars = THEMES[currentTheme] + const config: any = { + startOnLoad: false, + securityLevel: 'loose', + fontFamily: 'sans-serif', + maxTextSize: 50000, + gantt: { + titleTopMargin: 25, + barHeight: 20, + barGap: 4, + topPadding: 50, + leftPadding: 75, + gridLineStartPadding: 35, + fontSize: 11, + numberSectionStyles: 4, + axisFormat: '%Y-%m-%d', + }, + } + + if (look === 'classic') { + config.theme = currentTheme === 'dark' ? 'dark' : 'neutral' + config.flowchart = { + htmlLabels: true, + useMaxWidth: true, + diagramPadding: 12, + nodeSpacing: 60, + rankSpacing: 80, + curve: 'linear', + ranker: 'tight-tree', + } + } + else { + config.theme = 'default' + config.themeCSS = ` + .node rect { fill-opacity: 0.85; } + .edgePath .path { stroke-width: 1.5px; } + .label { font-family: 'sans-serif'; } + .edgeLabel { font-family: 'sans-serif'; } + .cluster rect { rx: 5px; ry: 5px; } + ` + config.themeVariables = { + fontSize: '14px', + fontFamily: 'sans-serif', + } + config.flowchart = { + htmlLabels: true, + useMaxWidth: true, + diagramPadding: 10, + nodeSpacing: 40, + rankSpacing: 60, + curve: 'basis', + } + config.themeVariables.primaryBorderColor = currentTheme === 'dark' ? THEMES.dark.connectionColor : THEMES.light.connectionColor + } + + if (currentTheme === 'dark' && !config.themeVariables) { + config.themeVariables = { + background: themeVars.background, + primaryColor: themeVars.primaryColor, + primaryBorderColor: themeVars.primaryBorderColor, + primaryTextColor: themeVars.primaryTextColor, + secondaryColor: themeVars.secondaryColor, + tertiaryColor: themeVars.tertiaryColor, + fontFamily: 'sans-serif', + } + } + + try { + mermaid.initialize(config) + return true + } + catch (error) { + console.error('Config error:', error) + return false + } + } + return false + }, [currentTheme, isInitialized, look]) + + // Effect for theme and style configuration + useEffect(() => { + if (diagramCache.has(cacheKey)) { + setSvgCode(diagramCache.get(cacheKey) || null) + setIsLoading(false) + return + } + + if (configureMermaid() && containerRef.current && isCodeComplete) + renderFlowchart(props.PrimitiveCode) + }, [look, props.PrimitiveCode, renderFlowchart, isInitialized, cacheKey, currentTheme, isCodeComplete, configureMermaid]) + + // Effect for rendering with debounce + useEffect(() => { + if (diagramCache.has(cacheKey)) { + setSvgCode(diagramCache.get(cacheKey) || null) + setIsLoading(false) + return + } + + if (renderTimeoutRef.current) + clearTimeout(renderTimeoutRef.current) + + if (isCodeComplete) { + renderTimeoutRef.current = setTimeout(() => { + if (isInitialized) + renderFlowchart(props.PrimitiveCode) + }, 300) + } + else { + setIsLoading(true) + } + + return () => { + if (renderTimeoutRef.current) + clearTimeout(renderTimeoutRef.current) + } + }, [props.PrimitiveCode, renderFlowchart, isInitialized, cacheKey, isCodeComplete]) + + // Cleanup on unmount + useEffect(() => { + return () => { + if (containerRef.current) + containerRef.current.innerHTML = '' + if (renderTimeoutRef.current) + clearTimeout(renderTimeoutRef.current) + if (codeCompletionCheckRef.current) + clearTimeout(codeCompletionCheckRef.current) + } + }, []) + + const toggleTheme = () => { + setCurrentTheme(prevTheme => prevTheme === 'light' ? Theme.dark : Theme.light) + diagramCache.clear() + } + + // Style classes for theme-dependent elements + const themeClasses = { + container: cn('relative', { + 'bg-white': currentTheme === Theme.light, + 'bg-slate-900': currentTheme === Theme.dark, + }), + mermaidDiv: cn('mermaid cursor-pointer h-auto w-full relative', { + 'bg-white': currentTheme === Theme.light, + 'bg-slate-900': currentTheme === Theme.dark, + }), + errorMessage: cn('py-4 px-[26px]', { + 'text-red-500': currentTheme === Theme.light, + 'text-red-400': currentTheme === Theme.dark, + }), + errorIcon: cn('w-6 h-6', { + 'text-red-500': currentTheme === Theme.light, + 'text-red-400': currentTheme === Theme.dark, + }), + segmented: cn('msh-segmented msh-segmented-sm css-23bs09 css-var-r1', { + 'text-gray-700': currentTheme === Theme.light, + 'text-gray-300': currentTheme === Theme.dark, + }), + themeToggle: cn('flex items-center justify-center w-10 h-10 rounded-full transition-all duration-300 shadow-md backdrop-blur-sm', { + 'bg-white/80 hover:bg-white hover:shadow-lg text-gray-700 border border-gray-200': currentTheme === Theme.light, + 'bg-slate-800/80 hover:bg-slate-700 hover:shadow-lg text-yellow-300 border border-slate-600': currentTheme === Theme.dark, + }), + } + + // Style classes for look options + const getLookButtonClass = (lookType: 'classic' | 'handDrawn') => { + return cn( + 'flex items-center justify-center mb-4 w-[calc((100%-8px)/2)] h-8 rounded-lg border border-components-option-card-option-border bg-components-option-card-option-bg cursor-pointer system-sm-medium text-text-secondary', + look === lookType && 'border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary', + currentTheme === Theme.dark && 'border-slate-600 bg-slate-800 text-slate-300', + look === lookType && currentTheme === Theme.dark && 'border-blue-500 bg-slate-700 text-white', + ) + } return ( - // eslint-disable-next-line ts/ban-ts-comment - // @ts-expect-error - (
-
+
} className={themeClasses.container}> +
-