diff --git a/.github/DISCUSSION_TEMPLATE/general.yml b/.github/DISCUSSION_TEMPLATE/general.yml new file mode 100644 index 0000000000..5af61ea64c --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/general.yml @@ -0,0 +1,24 @@ +title: "General Discussion" +body: + - type: checkboxes + attributes: + label: Self Checks + description: "To make sure we get to you in time, please check the following :)" + options: + - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones. + required: true + - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)). + required: true + - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）" + required: true + - label: "Please do not modify this template :) and fill in all the required fields." + required: true + - type: textarea + attributes: + label: Content + placeholder: Please describe the content you would like to discuss. + validations: + required: true + - type: markdown + attributes: + value: Please limit one request per issue. diff --git a/.github/DISCUSSION_TEMPLATE/help.yml b/.github/DISCUSSION_TEMPLATE/help.yml new file mode 100644 index 0000000000..abebaa9727 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/help.yml @@ -0,0 +1,30 @@ +title: "Help" +body: + - type: checkboxes + attributes: + label: Self Checks + description: "To make sure we get to you in time, please check the following :)" + options: + - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones. + required: true + - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)). + required: true + - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）" + required: true + - label: "Please do not modify this template :) and fill in all the required fields." + required: true + - type: textarea + attributes: + label: 1. Is this request related to a challenge you're experiencing? Tell me about your story. + placeholder: Please describe the specific scenario or problem you're facing as clearly as possible. For instance "I was trying to use [feature] for [specific task], and [what happened]... It was frustrating because...." + validations: + required: true + - type: textarea + attributes: + label: 2. Additional context or comments + placeholder: (Any other information, comments, documentations, links, or screenshots that would provide more clarity. This is the place to add anything else not covered above.) + validations: + required: false + - type: markdown + attributes: + value: Please limit one request per issue. diff --git a/.github/DISCUSSION_TEMPLATE/suggestion.yml b/.github/DISCUSSION_TEMPLATE/suggestion.yml new file mode 100644 index 0000000000..0893a10b2d --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/suggestion.yml @@ -0,0 +1,37 @@ +title: Suggestions for New Features +body: + - type: checkboxes + attributes: + label: Self Checks + description: "To make sure we get to you in time, please check the following :)" + options: + - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones. + required: true + - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)). + required: true + - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）" + required: true + - label: "Please do not modify this template :) and fill in all the required fields." + required: true + - type: textarea + attributes: + label: 1. Is this request related to a challenge you're experiencing? Tell me about your story. + placeholder: Please describe the specific scenario or problem you're facing as clearly as possible. For instance "I was trying to use [feature] for [specific task], and [what happened]... It was frustrating because...." + validations: + required: true + - type: textarea + attributes: + label: 2. Additional context or comments + placeholder: (Any other information, comments, documentations, links, or screenshots that would provide more clarity. This is the place to add anything else not covered above.) + validations: + required: false + - type: checkboxes + attributes: + label: 3. Can you help us with this feature? + description: Let us know! This is not a commitment, but a starting point for collaboration. + options: + - label: I am interested in contributing to this feature. + required: false + - type: markdown + attributes: + value: Please limit one request per issue. diff --git a/README.md b/README.md index 40a6837c42..5d8221bcbd 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,6 @@ At the same time, please consider supporting Dify by sharing it on social media * [Github Discussion](https://github.com/langgenius/dify/discussions). Best for: sharing feedback and asking questions. * [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [Email](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Best for: questions you have about using Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community. * [Twitter](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community. diff --git a/README_AR.md b/README_AR.md index 35be2ba9b6..c91602721e 100644 --- a/README_AR.md +++ b/README_AR.md @@ -199,7 +199,6 @@ docker compose up -d ## المجتمع والاتصال * [مناقشة Github](https://github.com/langgenius/dify/discussions). الأفضل لـ: مشاركة التعليقات وطرح الأسئلة. * [المشكلات على GitHub](https://github.com/langgenius/dify/issues). الأفضل لـ: الأخطاء التي تواجهها في استخدام Dify.AI، واقتراحات الميزات. انظر [دليل المساهمة](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [البريد الإلكتروني](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). الأفضل لـ: الأسئلة التي تتعلق باستخدام Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع. * [تويتر](https://twitter.com/dify_ai). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع. diff --git a/README_ES.md b/README_ES.md index ed613be8d4..84c06a2503 100644 --- a/README_ES.md +++ b/README_ES.md @@ -224,7 +224,6 @@ Al mismo tiempo, considera apoyar a Dify compartiéndolo en redes sociales y en * [Discusión en GitHub](https://github.com/langgenius/dify/discussions). Lo mejor para: compartir comentarios y hacer preguntas. * [Reporte de problemas en GitHub](https://github.com/langgenius/dify/issues). Lo mejor para: errores que encuentres usando Dify.AI y propuestas de características. Consulta nuestra [Guía de contribución](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [Correo electrónico](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Lo mejor para: preguntas que tengas sobre el uso de Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). Lo mejor para: compartir tus aplicaciones y pasar el rato con la comunidad. * [Twitter](https://twitter.com/dify_ai). Lo mejor para: compartir tus aplicaciones y pasar el rato con la comunidad. diff --git a/README_FR.md b/README_FR.md index 6f09773bf2..768c9390d8 100644 --- a/README_FR.md +++ b/README_FR.md @@ -222,7 +222,6 @@ Dans le même temps, veuillez envisager de soutenir Dify en le partageant sur le * [Discussion GitHub](https://github.com/langgenius/dify/discussions). Meilleur pour: partager des commentaires et poser des questions. * [Problèmes GitHub](https://github.com/langgenius/dify/issues). Meilleur pour: les bogues que vous rencontrez en utilisant Dify.AI et les propositions de fonctionnalités. Consultez notre [Guide de contribution](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [E-mail](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Meilleur pour: les questions que vous avez sur l'utilisation de Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). Meilleur pour: partager vos applications et passer du temps avec la communauté. * [Twitter](https://twitter.com/dify_ai). Meilleur pour: partager vos applications et passer du temps avec la communauté. diff --git a/README_JA.md b/README_JA.md index 55f6e173fd..f4cccd5271 100644 --- a/README_JA.md +++ b/README_JA.md @@ -221,7 +221,6 @@ docker compose up -d * [Github Discussion](https://github.com/langgenius/dify/discussions). 主に: フィードバックの共有や質問。 * [GitHub Issues](https://github.com/langgenius/dify/issues). 主に: Dify.AIを使用する際に発生するエラーや問題については、[貢献ガイド](CONTRIBUTING_JA.md)を参照してください -* [Email](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). 主に: Dify.AIの使用に関する質問。 * [Discord](https://discord.gg/FngNHpbcY7). 主に: アプリケーションの共有やコミュニティとの交流。 * [Twitter](https://twitter.com/dify_ai). 主に: アプリケーションの共有やコミュニティとの交流。 @@ -239,7 +238,7 @@ docker compose up -d 無料の30分間のミーティングをスケジュール - 技術サポート + 技術サポート 技術的な問題やサポートに関する質問 diff --git a/README_KL.md b/README_KL.md index 7fdc0b5181..6a15f39bc6 100644 --- a/README_KL.md +++ b/README_KL.md @@ -224,7 +224,6 @@ At the same time, please consider supporting Dify by sharing it on social media ). Best for: sharing feedback and asking questions. * [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). -* [Email](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Best for: questions you have about using Dify.AI. * [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community. * [Twitter](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community. diff --git a/README_KR.md b/README_KR.md index fa1980a99f..bb15fac8ef 100644 --- a/README_KR.md +++ b/README_KR.md @@ -214,7 +214,6 @@ Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했 * [Github 토론](https://github.com/langgenius/dify/discussions). 피드백 공유 및 질문하기에 적합합니다. * [GitHub 이슈](https://github.com/langgenius/dify/issues). Dify.AI 사용 중 발견한 버그와 기능 제안에 적합합니다. [기여 가이드](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md)를 참조하세요. -* [이메일](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). Dify.AI 사용에 대한 질문하기에 적합합니다. * [디스코드](https://discord.gg/FngNHpbcY7). 애플리케이션 공유 및 커뮤니티와 소통하기에 적합합니다. * [트위터](https://twitter.com/dify_ai). 애플리케이션 공유 및 커뮤니티와 소통하기에 적합합니다. diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 934b6413ae..c446f523b6 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -543,13 +543,13 @@ class DatasetRetrievalSettingApi(Resource): def get(self): vector_type = dify_config.VECTOR_STORE match vector_type: - case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE: + case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT: return { 'retrieval_method': [ RetrievalMethod.SEMANTIC_SEARCH.value ] } - case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH | VectorType.ANALYTICDB | VectorType.MYSCALE: + case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH | VectorType.ANALYTICDB | VectorType.MYSCALE | VectorType.ORACLE: return { 'retrieval_method': [ RetrievalMethod.SEMANTIC_SEARCH.value, @@ -567,13 +567,13 @@ class DatasetRetrievalSettingMockApi(Resource): @account_initialization_required def get(self, vector_type): match vector_type: - case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE: + case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT: return { 'retrieval_method': [ RetrievalMethod.SEMANTIC_SEARCH.value ] } - case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH| VectorType.ANALYTICDB | VectorType.MYSCALE: + case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH| VectorType.ANALYTICDB | VectorType.MYSCALE | VectorType.ORACLE: return { 'retrieval_method': [ RetrievalMethod.SEMANTIC_SEARCH.value, diff --git a/api/core/model_manager.py b/api/core/model_manager.py index dc7556f09a..8e99ad3dec 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -410,10 +410,9 @@ class LBModelManager: self._model = model self._load_balancing_configs = load_balancing_configs - for load_balancing_config in self._load_balancing_configs: + for load_balancing_config in self._load_balancing_configs[:]: # Iterate over a shallow copy of the list if load_balancing_config.name == "__inherit__": if not managed_credentials: - # FIXME: Mutation to loop iterable `self._load_balancing_configs` during iteration # remove __inherit__ if managed credentials is not provided self._load_balancing_configs.remove(load_balancing_config) else: diff --git a/api/core/rag/datasource/keyword/keyword_base.py b/api/core/rag/datasource/keyword/keyword_base.py index 67bc6df6fd..b77c6562b2 100644 --- a/api/core/rag/datasource/keyword/keyword_base.py +++ b/api/core/rag/datasource/keyword/keyword_base.py @@ -38,11 +38,10 @@ class BaseKeyword(ABC): raise NotImplementedError def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: - for text in texts: + for text in texts[:]: doc_id = text.metadata['doc_id'] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: - # FIXME: Mutation to loop iterable `texts` during iteration texts.remove(text) return texts diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index 8814c61433..702dcec314 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -110,7 +110,7 @@ class RetrievalService: ) documents = keyword.search( - query, + cls.escape_query_for_search(query), top_k=top_k ) all_documents.extend(documents) @@ -132,7 +132,7 @@ class RetrievalService: ) documents = vector.search_by_vector( - query, + cls.escape_query_for_search(query), search_type='similarity_score_threshold', top_k=top_k, score_threshold=score_threshold, @@ -170,7 +170,7 @@ class RetrievalService: ) documents = vector_processor.search_by_full_text( - query, + cls.escape_query_for_search(query), top_k=top_k ) if documents: @@ -186,3 +186,7 @@ class RetrievalService: all_documents.extend(documents) except Exception as e: exceptions.append(str(e)) + + @staticmethod + def escape_query_for_search(query: str) -> str: + return query.replace('"', '\\"') \ No newline at end of file diff --git a/api/core/rag/datasource/vdb/milvus/milvus_vector.py b/api/core/rag/datasource/vdb/milvus/milvus_vector.py index 5f2ab7c5fc..b679577c04 100644 --- a/api/core/rag/datasource/vdb/milvus/milvus_vector.py +++ b/api/core/rag/datasource/vdb/milvus/milvus_vector.py @@ -115,7 +115,8 @@ class MilvusVector(BaseVector): uri = "https://" + str(self._client_config.host) + ":" + str(self._client_config.port) else: uri = "http://" + str(self._client_config.host) + ":" + str(self._client_config.port) - connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password) + connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password, + db_name=self._client_config.database) from pymilvus import utility if utility.has_collection(self._collection_name, using=alias): @@ -130,7 +131,8 @@ class MilvusVector(BaseVector): uri = "https://" + str(self._client_config.host) + ":" + str(self._client_config.port) else: uri = "http://" + str(self._client_config.host) + ":" + str(self._client_config.port) - connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password) + connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password, + db_name=self._client_config.database) from pymilvus import utility if utility.has_collection(self._collection_name, using=alias): diff --git a/api/core/rag/datasource/vdb/oracle/oraclevector.py b/api/core/rag/datasource/vdb/oracle/oraclevector.py index f75310205c..f65f57da60 100644 --- a/api/core/rag/datasource/vdb/oracle/oraclevector.py +++ b/api/core/rag/datasource/vdb/oracle/oraclevector.py @@ -1,11 +1,15 @@ import array import json +import re import uuid from contextlib import contextmanager from typing import Any +import jieba.posseg as pseg +import nltk import numpy import oracledb +from nltk.corpus import stopwords from pydantic import BaseModel, model_validator from configs import dify_config @@ -50,6 +54,11 @@ CREATE TABLE IF NOT EXISTS {table_name} ( ,embedding vector NOT NULL ) """ +SQL_CREATE_INDEX = """ +CREATE INDEX idx_docs_{table_name} ON {table_name}(text) +INDEXTYPE IS CTXSYS.CONTEXT PARAMETERS +('FILTER CTXSYS.NULL_FILTER SECTION GROUP CTXSYS.HTML_SECTION_GROUP LEXER sys.my_chinese_vgram_lexer') +""" class OracleVector(BaseVector): @@ -188,7 +197,53 @@ class OracleVector(BaseVector): return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: - # do not support bm25 search + top_k = kwargs.get("top_k", 5) + # just not implement fetch by score_threshold now, may be later + score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + if len(query) > 0: + # Check which language the query is in + zh_pattern = re.compile('[\u4e00-\u9fa5]+') + match = zh_pattern.search(query) + entities = [] + # match: query condition maybe is a chinese sentence, so using Jieba split,else using nltk split + if match: + words = pseg.cut(query) + current_entity = "" + for word, pos in words: + if pos == 'nr' or pos == 'Ng' or pos == 'eng' or pos == 'nz' or pos == 'n' or pos == 'ORG' or pos == 'v': # nr: 人名, ns: 地名, nt: 机构名 + current_entity += word + else: + if current_entity: + entities.append(current_entity) + current_entity = "" + if current_entity: + entities.append(current_entity) + else: + try: + nltk.data.find('tokenizers/punkt') + nltk.data.find('corpora/stopwords') + except LookupError: + nltk.download('punkt') + nltk.download('stopwords') + print("run download") + e_str = re.sub(r'[^\w ]', '', query) + all_tokens = nltk.word_tokenize(e_str) + stop_words = stopwords.words('english') + for token in all_tokens: + if token not in stop_words: + entities.append(token) + with self._get_cursor() as cur: + cur.execute( + f"select meta, text FROM {self.table_name} WHERE CONTAINS(text, :1, 1) > 0 order by score(1) desc fetch first {top_k} rows only", + [" ACCUM ".join(entities)] + ) + docs = [] + for record in cur: + metadata, text = record + docs.append(Document(page_content=text, metadata=metadata)) + return docs + else: + return [Document(page_content="", metadata="")] return [] def delete(self) -> None: @@ -206,6 +261,8 @@ class OracleVector(BaseVector): with self._get_cursor() as cur: cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name)) redis_client.set(collection_exist_cache_key, 1, ex=3600) + with self._get_cursor() as cur: + cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name)) class OracleVectorFactory(AbstractVectorFactory): diff --git a/api/core/rag/datasource/vdb/vector_base.py b/api/core/rag/datasource/vdb/vector_base.py index 0b1d58856c..3f70e8b608 100644 --- a/api/core/rag/datasource/vdb/vector_base.py +++ b/api/core/rag/datasource/vdb/vector_base.py @@ -57,11 +57,10 @@ class BaseVector(ABC): raise NotImplementedError def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: - for text in texts: + for text in texts[:]: doc_id = text.metadata['doc_id'] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: - # FIXME: Mutation to loop iterable `texts` during iteration texts.remove(text) return texts diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index 509273e8ea..fad60ecf45 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -153,11 +153,10 @@ class Vector: return CacheEmbedding(embedding_model) def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: - for text in texts: + for text in texts[:]: doc_id = text.metadata['doc_id'] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: - # FIXME: Mutation to loop iterable `texts` during iteration texts.remove(text) return texts diff --git a/api/core/tools/provider/builtin/hap/_assets/icon.svg b/api/core/tools/provider/builtin/hap/_assets/icon.svg new file mode 100644 index 0000000000..0fa6f0886f --- /dev/null +++ b/api/core/tools/provider/builtin/hap/_assets/icon.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/api/core/tools/provider/builtin/hap/hap.py b/api/core/tools/provider/builtin/hap/hap.py new file mode 100644 index 0000000000..e0a48e05a5 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/hap.py @@ -0,0 +1,8 @@ +from typing import Any + +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class HapProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + pass \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/hap.yaml b/api/core/tools/provider/builtin/hap/hap.yaml new file mode 100644 index 0000000000..25b473cf9d --- /dev/null +++ b/api/core/tools/provider/builtin/hap/hap.yaml @@ -0,0 +1,15 @@ +identity: + author: Mingdao + name: hap + label: + en_US: HAP + zh_Hans: HAP + pt_BR: HAP + description: + en_US: "Hyper application platform that is particularly friendly to AI" + zh_Hans: "对 AI 特别友好的超级应用平台" + pt_BR: "Plataforma de aplicação hiper que é particularmente amigável à IA" + icon: icon.svg + tags: + - productivity +credentials_for_provider: diff --git a/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.py b/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.py new file mode 100644 index 0000000000..0e101dc67d --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.py @@ -0,0 +1,53 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class AddWorksheetRecordTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + record_data = tool_parameters.get('record_data', '') + if not record_data: + return self.create_text_message('Invalid parameter Record Row Data') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/v2/open/worksheet/addRow" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id} + + try: + payload['controls'] = json.loads(record_data) + res = httpx.post(url, headers=headers, json=payload, timeout=60) + res.raise_for_status() + res_json = res.json() + if res_json.get('error_code') != 1: + return self.create_text_message(f"Failed to add the new record. {res_json['error_msg']}") + return self.create_text_message(f"New record added successfully. The record ID is {res_json['data']}.") + except httpx.RequestError as e: + return self.create_text_message(f"Failed to add the new record, request error: {e}") + except json.JSONDecodeError as e: + return self.create_text_message(f"Failed to parse JSON response: {e}") + except Exception as e: + return self.create_text_message(f"Failed to add the new record, unexpected error: {e}") diff --git a/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.yaml b/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.yaml new file mode 100644 index 0000000000..add7742cd7 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/add_worksheet_record.yaml @@ -0,0 +1,78 @@ +identity: + name: add_worksheet_record + author: Ryan Tian + label: + en_US: Add Worksheet Record + zh_Hans: 新增一条工作表记录 +description: + human: + en_US: Adds a new record to the specified worksheet + zh_Hans: 向指定的工作表新增一条记录数据 + llm: A tool to append a new data entry into a specified worksheet. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: record_data + type: string + required: true + label: + en_US: Record Row Data + zh_Hans: 记录数据 + human_description: + en_US: The fields with data of the specified record + zh_Hans: 要新增的记录数据，JSON 对象数组格式。数组元素属性：controlId-字段ID，value-字段值 + llm_description: | + The fields with data of the specified record which to be created. It is in the format of an array of JSON objects, and the structure is defined as follows: + ``` + type RowData = { + controlId: string; // Field ID to be updated + value: string; // Field value to be updated + }[]; + ``` + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.py b/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.py new file mode 100644 index 0000000000..ba25952c9f --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.py @@ -0,0 +1,49 @@ +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class DeleteWorksheetRecordTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + row_id = tool_parameters.get('row_id', '') + if not row_id: + return self.create_text_message('Invalid parameter Record Row ID') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/v2/open/worksheet/deleteRow" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id, "rowId": row_id} + + try: + res = httpx.post(url, headers=headers, json=payload, timeout=30) + res.raise_for_status() + res_json = res.json() + if res_json.get('error_code') != 1: + return self.create_text_message(f"Failed to delete the record. {res_json['error_msg']}") + return self.create_text_message("Successfully deleted the record.") + except httpx.RequestError as e: + return self.create_text_message(f"Failed to delete the record, request error: {e}") + except Exception as e: + return self.create_text_message(f"Failed to delete the record, unexpected error: {e}") \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.yaml b/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.yaml new file mode 100644 index 0000000000..7c0c2a6439 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/delete_worksheet_record.yaml @@ -0,0 +1,71 @@ +identity: + name: delete_worksheet_record + author: Ryan Tian + label: + en_US: Delete Worksheet Record + zh_Hans: 删除指定的一条工作表记录 +description: + human: + en_US: Deletes a single record from a worksheet based on the specified record row ID + zh_Hans: 根据指定的记录ID删除一条工作表记录数据 + llm: A tool to remove a particular record from a worksheet by specifying its unique record identifier. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: row_id + type: string + required: true + label: + en_US: Record Row ID + zh_Hans: 记录 ID + human_description: + en_US: The row ID of the specified record + zh_Hans: 要删除的记录 ID + llm_description: The row ID of the specified record which to be deleted. + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py new file mode 100644 index 0000000000..2c46d9dd4e --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.py @@ -0,0 +1,148 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GetWorksheetFieldsTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/v2/open/worksheet/getWorksheetInfo" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id} + + try: + res = httpx.post(url, headers=headers, json=payload, timeout=60) + res.raise_for_status() + res_json = res.json() + if res_json.get('error_code') != 1: + return self.create_text_message(f"Failed to get the worksheet information. {res_json['error_msg']}") + + fields_json, fields_table = self.get_controls(res_json['data']['controls']) + result_type = tool_parameters.get('result_type', 'table') + return self.create_text_message( + text=json.dumps(fields_json, ensure_ascii=False) if result_type == 'json' else fields_table + ) + except httpx.RequestError as e: + return self.create_text_message(f"Failed to get the worksheet information, request error: {e}") + except json.JSONDecodeError as e: + return self.create_text_message(f"Failed to parse JSON response: {e}") + except Exception as e: + return self.create_text_message(f"Failed to get the worksheet information, unexpected error: {e}") + + def get_field_type_by_id(self, field_type_id: int) -> str: + field_type_map = { + 2: "Text", + 3: "Text-Phone", + 4: "Text-Phone", + 5: "Text-Email", + 6: "Number", + 7: "Text", + 8: "Number", + 9: "Option-Single Choice", + 10: "Option-Multiple Choices", + 11: "Option-Single Choice", + 15: "Date", + 16: "Date", + 24: "Option-Region", + 25: "Text", + 26: "Option-Member", + 27: "Option-Department", + 28: "Number", + 29: "Option-Linked Record", + 30: "Unknown Type", + 31: "Number", + 32: "Text", + 33: "Text", + 35: "Option-Linked Record", + 36: "Number-Yes1/No0", + 37: "Number", + 38: "Date", + 40: "Location", + 41: "Text", + 46: "Time", + 48: "Option-Organizational Role", + 50: "Text", + 51: "Query Record", + } + return field_type_map.get(field_type_id, '') + + def get_controls(self, controls: list) -> dict: + fields = [] + fields_list = ['|fieldId|fieldName|fieldType|fieldTypeId|description|options|','|'+'---|'*6] + for control in controls: + if control['type'] in self._get_ignore_types(): + continue + field_type_id = control['type'] + field_type = self.get_field_type_by_id(control['type']) + if field_type_id == 30: + source_type = control['sourceControl']['type'] + if source_type in self._get_ignore_types(): + continue + else: + field_type_id = source_type + field_type = self.get_field_type_by_id(source_type) + field = { + 'id': control['controlId'], + 'name': control['controlName'], + 'type': field_type, + 'typeId': field_type_id, + 'description': control['remark'].replace('\n', ' ').replace('\t', ' '), + 'options': self._extract_options(control), + } + fields.append(field) + fields_list.append(f"|{field['id']}|{field['name']}|{field['type']}|{field['typeId']}|{field['description']}|{field['options'] if field['options'] else ''}|") + + fields.append({ + 'id': 'ctime', + 'name': 'Created Time', + 'type': self.get_field_type_by_id(16), + 'typeId': 16, + 'description': '', + 'options': [] + }) + fields_list.append("|ctime|Created Time|Date|16|||") + return fields, '\n'.join(fields_list) + + def _extract_options(self, control: dict) -> list: + options = [] + if control['type'] in [9, 10, 11]: + options.extend([{"key": opt['key'], "value": opt['value']} for opt in control.get('options', [])]) + elif control['type'] in [28, 36]: + itemnames = control['advancedSetting'].get('itemnames') + if itemnames and itemnames.startswith('[{'): + try: + options = json.loads(itemnames) + except json.JSONDecodeError: + pass + elif control['type'] == 30: + source_type = control['sourceControl']['type'] + if source_type not in self._get_ignore_types(): + options.extend([{"key": opt['key'], "value": opt['value']} for opt in control.get('options', [])]) + return options + + def _get_ignore_types(self): + return {14, 21, 22, 34, 42, 43, 45, 47, 49, 10010} \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.yaml b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.yaml new file mode 100644 index 0000000000..f0d4973e85 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_fields.yaml @@ -0,0 +1,80 @@ +identity: + name: get_worksheet_fields + author: Ryan Tian + label: + en_US: Get Worksheet Fields + zh_Hans: 获取工作表字段结构 +description: + human: + en_US: Get fields information of the worksheet + zh_Hans: 获取指定工作表的所有字段结构信息 + llm: A tool to get fields information of the specific worksheet. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form + + - name: result_type + type: select + required: true + options: + - value: table + label: + en_US: table text + zh_Hans: 表格文本 + - value: json + label: + en_US: json text + zh_Hans: JSON文本 + default: table + label: + en_US: Result type + zh_Hans: 结果类型 + human_description: + en_US: used for selecting the result type, table styled text or json text + zh_Hans: 用于选择结果类型，使用表格格式文本还是JSON格式文本 + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py new file mode 100644 index 0000000000..6bf1caa65e --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.py @@ -0,0 +1,130 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GetWorksheetPivotDataTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + x_column_fields = tool_parameters.get('x_column_fields', '') + if not x_column_fields or not x_column_fields.startswith('['): + return self.create_text_message('Invalid parameter Column Fields') + y_row_fields = tool_parameters.get('y_row_fields', '') + if y_row_fields and not y_row_fields.strip().startswith('['): + return self.create_text_message('Invalid parameter Row Fields') + elif not y_row_fields: + y_row_fields = '[]' + value_fields = tool_parameters.get('value_fields', '') + if not value_fields or not value_fields.strip().startswith('['): + return self.create_text_message('Invalid parameter Value Fields') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/report/getPivotData" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id, "options": {"showTotal": True}} + + try: + x_column_fields = json.loads(x_column_fields) + payload['columns'] = x_column_fields + y_row_fields = json.loads(y_row_fields) + if y_row_fields: payload['rows'] = y_row_fields + value_fields = json.loads(value_fields) + payload['values'] = value_fields + sort_fields = tool_parameters.get('sort_fields', '') + if not sort_fields: sort_fields = '[]' + sort_fields = json.loads(sort_fields) + if sort_fields: payload['options']['sort'] = sort_fields + res = httpx.post(url, headers=headers, json=payload, timeout=60) + res.raise_for_status() + res_json = res.json() + if res_json.get('status') != 1: + return self.create_text_message(f"Failed to get the worksheet pivot data. {res_json['msg']}") + + pivot_json = self.generate_pivot_json(res_json['data']) + pivot_table = self.generate_pivot_table(res_json['data']) + result_type = tool_parameters.get('result_type', '') + text = pivot_table if result_type == 'table' else json.dumps(pivot_json, ensure_ascii=False) + return self.create_text_message(text) + except httpx.RequestError as e: + return self.create_text_message(f"Failed to get the worksheet pivot data, request error: {e}") + except json.JSONDecodeError as e: + return self.create_text_message(f"Failed to parse JSON response: {e}") + except Exception as e: + return self.create_text_message(f"Failed to get the worksheet pivot data, unexpected error: {e}") + + def generate_pivot_table(self, data: dict[str, Any]) -> str: + columns = data['metadata']['columns'] + rows = data['metadata']['rows'] + values = data['metadata']['values'] + + rows_data = data['data'] + + header = ([row['displayName'] for row in rows] if rows else []) + [column['displayName'] for column in columns] + [value['displayName'] for value in values] + line = (['---'] * len(rows) if rows else []) + ['---'] * len(columns) + ['--:'] * len(values) + + table = [header, line] + for row in rows_data: + row_data = [self.replace_pipe(row['rows'][r['controlId']]) for r in rows] if rows else [] + row_data.extend([self.replace_pipe(row['columns'][column['controlId']]) for column in columns]) + row_data.extend([self.replace_pipe(str(row['values'][value['controlId']])) for value in values]) + table.append(row_data) + + return '\n'.join([('|'+'|'.join(row) +'|') for row in table]) + + def replace_pipe(self, text: str) -> str: + return text.replace('|', '▏').replace('\n', ' ') + + def generate_pivot_json(self, data: dict[str, Any]) -> dict: + fields = { + "x-axis": [ + {"fieldId": column["controlId"], "fieldName": column["displayName"]} + for column in data["metadata"]["columns"] + ], + "y-axis": [ + {"fieldId": row["controlId"], "fieldName": row["displayName"]} + for row in data["metadata"]["rows"] + ] if data["metadata"]["rows"] else [], + "values": [ + {"fieldId": value["controlId"], "fieldName": value["displayName"]} + for value in data["metadata"]["values"] + ] + } + # fields = ([ + # {"fieldId": row["controlId"], "fieldName": row["displayName"]} + # for row in data["metadata"]["rows"] + # ] if data["metadata"]["rows"] else []) + [ + # {"fieldId": column["controlId"], "fieldName": column["displayName"]} + # for column in data["metadata"]["columns"] + # ] + [ + # {"fieldId": value["controlId"], "fieldName": value["displayName"]} + # for value in data["metadata"]["values"] + # ] + rows = [] + for row in data["data"]: + row_data = row["rows"] if row["rows"] else {} + row_data.update(row["columns"]) + row_data.update(row["values"]) + rows.append(row_data) + return {"fields": fields, "rows": rows, "summary": data["metadata"]["totalRow"]} \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.yaml b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.yaml new file mode 100644 index 0000000000..cf8c57b262 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/get_worksheet_pivot_data.yaml @@ -0,0 +1,248 @@ +identity: + name: get_worksheet_pivot_data + author: Ryan Tian + label: + en_US: Get Worksheet Pivot Data + zh_Hans: 获取工作表统计透视数据 +description: + human: + en_US: Retrieve statistical pivot table data from a specified worksheet + zh_Hans: 从指定的工作表中检索统计透视表数据 + llm: A tool for extracting statistical pivot table data from a specific worksheet, providing summarized information for analysis and reporting purposes. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: x_column_fields + type: string + required: true + label: + en_US: Columns (X-axis) + zh_Hans: 统计列字段（X轴） + human_description: + en_US: The column fields that make up the pivot table's X-axis groups or other dimensions for the X-axis in pivot charts + zh_Hans: 组成透视表的统计列或者统计图表的X轴分组及X轴其它维度。JSON 对象数组格式，数组元素属性：controlId-列ID，displayName-显示名称，particleSize(可选)-字段类型是日期或者地区时，通过此参数设置统计维度（日期时间：1-日，2-周，3-月；地区：1-全国，2-省，3-市） + llm_description: | + This parameter allows you to specify the columns that make up the pivot table's X-axis groups or other dimensions for the X-axis in pivot charts. It is formatted as a JSON array, with its structure defined as follows: + ``` + type XColumnFields = { // X-axis or column object array + controlId: string; // fieldId + displayName: string; // displayName + particleSize?: number; // field type is date or area, set the statistical dimension (date time: 1-day, 2-week, 3-month; area: 1-nation, 2-province, 3-city) + }[]; + ``` + form: llm + + - name: y_row_fields + type: string + required: false + label: + en_US: Rows (Y-axis) + zh_Hans: 统计行字段（Y轴） + human_description: + en_US: The row fields that make up the pivot table's Y-axis groups or other dimensions for the Y-axis in pivot charts + zh_Hans: 组成透视表的统计行或者统计图表的Y轴分组及Y轴其它维度。JSON 对象数组格式，数组元素属性：controlId-列ID，displayName-显示名称，particleSize(可选)-字段类型是日期或者地区时，通过此参数设置统计维度（日期时间：1-日，2-周，3-月；地区：1-全国，2-省，3-市） + llm_description: | + This parameter allows you to specify the rows that make up the pivot table's Y-axis groups or other dimensions for the Y-axis in pivot charts. It is formatted as a JSON array, with its structure defined as follows: + ``` + type YRowFields = { // Y-axis or row object array + controlId: string; // fieldId + displayName: string; // displayName + particleSize?: number; // field type is date or area, set the statistical dimension (date time: 1-day, 2-week, 3-month; area: 1-nation, 2-province, 3-city) + }[]; + ``` + form: llm + + - name: value_fields + type: string + required: true + label: + en_US: Aggregated Values + zh_Hans: 统计值字段 + human_description: + en_US: The aggregated value fields in the pivot table + zh_Hans: 透视表中经过聚合计算后的统计值字段。JSON 对象数组格式，数组元素属性：controlId-列ID，displayName-显示名称，aggregation-聚合方式（SUM,AVG,MIN,MAX,COUNT） + llm_description: | + This parameter allows you to specify the aggregated value fields in the pivot table. It is formatted as a JSON array, with its structure defined as follows: + ``` + type ValueFields = { // aggregated value object array + controlId: string; // fieldId + displayName: string; // displayName + aggregation: string; // aggregation method, e.g.: SUM, AVG, MIN, MAX, COUNT + }[]; + ``` + form: llm + + - name: filters + type: string + required: false + label: + en_US: Filter Set + zh_Hans: 筛选器组合 + human_description: + en_US: A combination of filters applied to query records, formatted as a JSON array. See the application's API documentation for details on its structure and usage. + zh_Hans: 查询记录的筛选条件组合，格式为 JSON 数组，可以从应用 API 文档中了解参数结构详情 + llm_description: | + This parameter allows you to specify a set of conditions that records must meet to be included in the result set. It is formatted as a JSON array, with its structure defined as follows: + ``` + type Filters = { // filter object array + controlId: string; // fieldId + dataType: number; // fieldTypeId + spliceType: number; // condition concatenation method, 1: And, 2: Or + filterType: number; // expression type, refer to the for enumerable values + values?: string[]; // values in the condition, for option-type fields, multiple values can be passed + value?: string; // value in the condition, a single value can be passed according to the field type + dateRange?: number; // date range, mandatory when filterType is 17 or 18, refer to the for enumerable values + minValue?: string; // minimum value for custom range + maxValue?: string; // maximum value for custom range + isAsc?: boolean; // ascending order, false: descending, true: ascending + }[]; + ``` + For option-type fields, if this option field has `options`, then you need to get the corresponding `key` value from the `options` in the current field information via `value`, and pass it into `values` in array format. Do not use the `options` value of other fields as input conditions. + + ### FilterTypeEnum Reference + ``` + Enum Value, Enum Character, Description + 1, Like, Contains + 2, Eq, Is (Equal) + 3, Start, Starts With + 4, End, Ends With + 5, NotLike, Does Not Contain + 6, Ne, Is Not (Not Equal) + 7, IsEmpty, Empty + 8, HasValue, Not Empty + 11, Between, Within Range + 12, NotBetween, Outside Range + 13, Gt, Greater Than + 14, Gte, Greater Than or Equal To + 15, Lt, Less Than + 16, Lte, Less Than or Equal To + 17, DateEnum, Date Is + 18, NotDateEnum, Date Is Not + 21, MySelf, Owned by Me + 22, UnRead, Unread + 23, Sub, Owned by Subordinate + 24, RCEq, Associated Field Is + 25, RCNe, Associated Field Is Not + 26, ArrEq, Array Equals + 27, ArrNe, Array Does Not Equal + 31, DateBetween, Date Within Range (can only be used with minValue and maxValue) + 32, DateNotBetween, Date Not Within Range (can only be used with minValue and maxValue) + 33, DateGt, Date Later Than + 34, DateGte, Date Later Than or Equal To + 35, DateLt, Date Earlier Than + 36, DateLte, Date Earlier Than or Equal To + ``` + + ### DateRangeEnum Reference + ``` + Enum Value, Enum Character, Description + 1, Today, Today + 2, Yesterday, Yesterday + 3, Tomorrow, Tomorrow + 4, ThisWeek, This Week + 5, LastWeek, Last Week + 6, NextWeek, Next Week + 7, ThisMonth, This Month + 8, LastMonth, Last Month + 9, NextMonth, Next Month + 12, ThisQuarter, This Quarter + 13, LastQuarter, Last Quarter + 14, NextQuarter, Next Quarter + 15, ThisYear, This Year + 16, LastYear, Last Year + 17, NextYear, Next Year + 18, Customize, Custom + 21, Last7Day, Past 7 Days + 22, Last14Day, Past 14 Days + 23, Last30Day, Past 30 Days + 31, Next7Day, Next 7 Days + 32, Next14Day, Next 14 Days + 33, Next33Day, Next 33 Days + ``` + form: llm + + - name: sort_fields + type: string + required: false + label: + en_US: Sort Fields + zh_Hans: 排序字段 + human_description: + en_US: The fields to used for sorting + zh_Hans: 用于确定排序的字段，不超过3个 + llm_description: | + This optional parameter specifies the unique identifier of the fields that will be used to sort the results. It is in the format of an array of JSON objects, and its structure is defined as follows: + ``` + type SortByFields = { + controlId: string; // Field ID used for sorting + isAsc: boolean; // Sorting direction, true indicates ascending order, false indicates descending order + }[]; + ``` + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form + + - name: result_type + type: select + required: true + options: + - value: table + label: + en_US: table text + zh_Hans: 表格文本 + - value: json + label: + en_US: json text + zh_Hans: JSON文本 + default: table + label: + en_US: Result type + zh_Hans: 结果类型 + human_description: + en_US: used for selecting the result type, table styled text or json text + zh_Hans: 用于选择结果类型，使用表格格式文本还是JSON格式文本 + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py new file mode 100644 index 0000000000..dddc041cc1 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.py @@ -0,0 +1,209 @@ +import json +import re +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class ListWorksheetRecordsTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not (host.startswith("http://") or host.startswith("https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url_fields = f"{host}/v2/open/worksheet/getWorksheetInfo" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id} + + field_ids = tool_parameters.get('field_ids', '') + + try: + res = httpx.post(url_fields, headers=headers, json=payload, timeout=30) + res_json = res.json() + if res.is_success: + if res_json['error_code'] != 1: + return self.create_text_message("Failed to get the worksheet information. {}".format(res_json['error_msg'])) + else: + worksheet_name = res_json['data']['name'] + fields, schema, table_header = self.get_schema(res_json['data']['controls'], field_ids) + else: + return self.create_text_message( + f"Failed to get the worksheet information, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to get the worksheet information, something went wrong: {}".format(e)) + + if field_ids: + payload['controls'] = [v.strip() for v in field_ids.split(',')] if field_ids else [] + filters = tool_parameters.get('filters', '') + if filters: + payload['filters'] = json.loads(filters) + sort_id = tool_parameters.get('sort_id', '') + sort_is_asc = tool_parameters.get('sort_is_asc', False) + if sort_id: + payload['sortId'] = sort_id + payload['isAsc'] = sort_is_asc + limit = tool_parameters.get('limit', 50) + payload['pageSize'] = limit + page_index = tool_parameters.get('page_index', 1) + payload['pageIndex'] = page_index + payload['useControlId'] = True + payload['listType'] = 1 + + url = f"{host}/v2/open/worksheet/getFilterRows" + try: + res = httpx.post(url, headers=headers, json=payload, timeout=90) + res_json = res.json() + if res.is_success: + if res_json['error_code'] != 1: + return self.create_text_message("Failed to get the records. {}".format(res_json['error_msg'])) + else: + result = { + "fields": fields, + "rows": [], + "total": res_json.get("data", {}).get("total"), + "payload": {key: payload[key] for key in ['worksheetId', 'controls', 'filters', 'sortId', 'isAsc', 'pageSize', 'pageIndex'] if key in payload} + } + rows = res_json.get("data", {}).get("rows", []) + result_type = tool_parameters.get('result_type', '') + if not result_type: result_type = 'table' + if result_type == 'json': + for row in rows: + result['rows'].append(self.get_row_field_value(row, schema)) + return self.create_text_message(json.dumps(result, ensure_ascii=False)) + else: + result_text = f"Found {result['total']} rows in worksheet \"{worksheet_name}\"." + if result['total'] > 0: + result_text += f" The following are {result['total'] if result['total'] < limit else limit} pieces of data presented in a table format:\n\n{table_header}" + for row in rows: + result_values = [] + for f in fields: + result_values.append(self.handle_value_type(row[f['fieldId']], schema[f['fieldId']])) + result_text += '\n|'+'|'.join(result_values)+'|' + return self.create_text_message(result_text) + else: + return self.create_text_message( + f"Failed to get the records, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to get the records, something went wrong: {}".format(e)) + + + def get_row_field_value(self, row: dict, schema: dict): + row_value = {"rowid": row["rowid"]} + for field in schema: + row_value[field] = self.handle_value_type(row[field], schema[field]) + return row_value + + + def get_schema(self, controls: list, fieldids: str): + allow_fields = {v.strip() for v in fieldids.split(',')} if fieldids else set() + fields = [] + schema = {} + field_names = [] + for control in controls: + control_type_id = self.get_real_type_id(control) + if (control_type_id in self._get_ignore_types()) or (allow_fields and not control['controlId'] in allow_fields): + continue + else: + fields.append({'fieldId': control['controlId'], 'fieldName': control['controlName']}) + schema[control['controlId']] = {'typeId': control_type_id, 'options': self.set_option(control)} + field_names.append(control['controlName']) + if (not allow_fields or ('ctime' in allow_fields)): + fields.append({'fieldId': 'ctime', 'fieldName': 'Created Time'}) + schema['ctime'] = {'typeId': 16, 'options': {}} + field_names.append("Created Time") + fields.append({'fieldId':'rowid', 'fieldName': 'Record Row ID'}) + schema['rowid'] = {'typeId': 2, 'options': {}} + field_names.append("Record Row ID") + return fields, schema, '|'+'|'.join(field_names)+'|\n|'+'---|'*len(field_names) + + def get_real_type_id(self, control: dict) -> int: + return control['sourceControlType'] if control['type'] == 30 else control['type'] + + def set_option(self, control: dict) -> dict: + options = {} + if control.get('options'): + options = {option['key']: option['value'] for option in control['options']} + elif control.get('advancedSetting', {}).get('itemnames'): + try: + itemnames = json.loads(control['advancedSetting']['itemnames']) + options = {item['key']: item['value'] for item in itemnames} + except json.JSONDecodeError: + pass + return options + + def _get_ignore_types(self): + return {14, 21, 22, 34, 42, 43, 45, 47, 49, 10010} + + def handle_value_type(self, value, field): + type_id = field.get("typeId") + if type_id == 10: + value = value if isinstance(value, str) else "、".join(value) + elif type_id in [28, 36]: + value = field.get("options", {}).get(value, value) + elif type_id in [26, 27, 48, 14]: + value = self.process_value(value) + elif type_id in [35, 29]: + value = self.parse_cascade_or_associated(field, value) + elif type_id == 40: + value = self.parse_location(value) + return self.rich_text_to_plain_text(value) if value else '' + + def process_value(self, value): + if isinstance(value, str): + if value.startswith("[{\"accountId\""): + value = json.loads(value) + value = ', '.join([item['fullname'] for item in value]) + elif value.startswith("[{\"departmentId\""): + value = json.loads(value) + value = '、'.join([item['departmentName'] for item in value]) + elif value.startswith("[{\"organizeId\""): + value = json.loads(value) + value = '、'.join([item['organizeName'] for item in value]) + elif value.startswith("[{\"file_id\""): + value = '' + elif value == '[]': + value = '' + elif hasattr(value, 'accountId'): + value = value['fullname'] + return value + + def parse_cascade_or_associated(self, field, value): + if (field['typeId'] == 35 and value.startswith('[')) or (field['typeId'] == 29 and value.startswith('[{')): + value = json.loads(value) + value = value[0]['name'] if len(value) > 0 else '' + else: + value = '' + return value + + def parse_location(self, value): + if len(value) > 10: + parsed_value = json.loads(value) + value = parsed_value.get("address", "") + else: + value = "" + return value + + def rich_text_to_plain_text(self, rich_text): + text = re.sub(r'<[^>]+>', '', rich_text) if '<' in rich_text else rich_text + return text.replace("|", "▏").replace("\n", " ") \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.yaml b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.yaml new file mode 100644 index 0000000000..3c37746b92 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheet_records.yaml @@ -0,0 +1,226 @@ +identity: + name: list_worksheet_records + author: Ryan Tian + label: + en_US: List Worksheet Records + zh_Hans: 查询工作表记录数据 +description: + human: + en_US: List records from the worksheet + zh_Hans: 查询工作表的记录列表数据，一次最多1000行，可分页获取 + llm: A tool to retrieve record data from the specific worksheet. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the worksheet from which to retrieve record data + zh_Hans: 要获取记录数据的工作表 ID + llm_description: This parameter specifies the ID of the worksheet where the records are stored. + form: llm + + - name: field_ids + type: string + required: false + label: + en_US: Field IDs + zh_Hans: 字段 ID 列表 + human_description: + en_US: A comma-separated list of field IDs whose data to retrieve. If not provided, all fields' data will be fetched + zh_Hans: 要获取记录数据的字段 ID，多个 ID 间用英文逗号隔开，不传此参数则将获取所有字段的数据 + llm_description: This optional parameter lets you specify a comma-separated list of field IDs. Unless the user explicitly requests to output the specified field in the question, this parameter should usually be omitted. If this parameter is omitted, the API will return data for all fields by default. When provided, only the data associated with these fields will be included in the response. + form: llm + + - name: filters + type: string + required: false + label: + en_US: Filter Set + zh_Hans: 筛选器组合 + human_description: + en_US: A combination of filters applied to query records, formatted as a JSON array. See the application's API documentation for details on its structure and usage. + zh_Hans: 查询记录的筛选条件组合，格式为 JSON 数组，可以从应用 API 文档中了解参数结构详情 + llm_description: | + This parameter allows you to specify a set of conditions that records must meet to be included in the result set. It is formatted as a JSON array, with its structure defined as follows: + ``` + type Filters = { // filter object array + controlId: string; // fieldId + dataType: number; // fieldTypeId + spliceType: number; // condition concatenation method, 1: And, 2: Or + filterType: number; // expression type, refer to the for enumerable values + values?: string[]; // values in the condition, for option-type fields, multiple values can be passed + value?: string; // value in the condition, a single value can be passed according to the field type + dateRange?: number; // date range, mandatory when filterType is 17 or 18, refer to the for enumerable values + minValue?: string; // minimum value for custom range + maxValue?: string; // maximum value for custom range + isAsc?: boolean; // ascending order, false: descending, true: ascending + }[]; + ``` + For option-type fields, if this option field has `options`, then you need to get the corresponding `key` value from the `options` in the current field information via `value`, and pass it into `values` in array format. Do not use the `options` value of other fields as input conditions. + + ### FilterTypeEnum Reference + ``` + Enum Value, Enum Character, Description + 1, Like, Contains(Include) + 2, Eq, Is (Equal) + 3, Start, Starts With + 4, End, Ends With + 5, NotLike, Does Not Contain(Not Include) + 6, Ne, Is Not (Not Equal) + 7, IsEmpty, Empty + 8, HasValue, Not Empty + 11, Between, Within Range(Belong to) + 12, NotBetween, Outside Range(Not belong to) + 13, Gt, Greater Than + 14, Gte, Greater Than or Equal To + 15, Lt, Less Than + 16, Lte, Less Than or Equal To + 17, DateEnum, Date Is + 18, NotDateEnum, Date Is Not + 24, RCEq, Associated Field Is + 25, RCNe, Associated Field Is Not + 26, ArrEq, Array Equals + 27, ArrNe, Array Does Not Equal + 31, DateBetween, Date Within Range (can only be used with minValue and maxValue) + 32, DateNotBetween, Date Not Within Range (can only be used with minValue and maxValue) + 33, DateGt, Date Later Than + 34, DateGte, Date Later Than or Equal To + 35, DateLt, Date Earlier Than + 36, DateLte, Date Earlier Than or Equal To + ``` + + ### DateRangeEnum Reference + ``` + Enum Value, Enum Character, Description + 1, Today, Today + 2, Yesterday, Yesterday + 3, Tomorrow, Tomorrow + 4, ThisWeek, This Week + 5, LastWeek, Last Week + 6, NextWeek, Next Week + 7, ThisMonth, This Month + 8, LastMonth, Last Month + 9, NextMonth, Next Month + 12, ThisQuarter, This Quarter + 13, LastQuarter, Last Quarter + 14, NextQuarter, Next Quarter + 15, ThisYear, This Year + 16, LastYear, Last Year + 17, NextYear, Next Year + 18, Customize, Custom + 21, Last7Day, Past 7 Days + 22, Last14Day, Past 14 Days + 23, Last30Day, Past 30 Days + 31, Next7Day, Next 7 Days + 32, Next14Day, Next 14 Days + 33, Next33Day, Next 33 Days + ``` + form: llm + + - name: sort_id + type: string + required: false + label: + en_US: Sort Field ID + zh_Hans: 排序字段 ID + human_description: + en_US: The ID of the field used for sorting + zh_Hans: 用以排序的字段 ID + llm_description: This optional parameter specifies the unique identifier of the field that will be used to sort the results. It should be set to the ID of an existing field within your data structure. + form: llm + + - name: sort_is_asc + type: boolean + required: false + label: + en_US: Ascending Order + zh_Hans: 是否升序排列 + human_description: + en_US: Determines whether the sorting is in ascending (true) or descending (false) order + zh_Hans: 排序字段的排序方式：true-升序，false-降序 + llm_description: This optional parameter controls the direction of the sort. If set to true, the results will be sorted in ascending order; if false, they will be sorted in descending order. + form: llm + + - name: limit + type: number + required: false + label: + en_US: Record Limit + zh_Hans: 记录数量限制 + human_description: + en_US: The maximum number of records to retrieve + zh_Hans: 要获取的记录数量限制条数 + llm_description: This optional parameter allows you to specify the maximum number of records that should be returned in the result set. When retrieving paginated record data, this parameter indicates the number of rows to fetch per page, and must be used in conjunction with the `page_index` parameter. + form: llm + + - name: page_index + type: number + required: false + label: + en_US: Page Index + zh_Hans: 页码 + human_description: + en_US: The page number when paginating through a list of records + zh_Hans: 分页读取记录列表时的页码 + llm_description: This parameter is used when you need to paginate through a large set of records. The default value is 1, which refers to the first page. When it is used, the meaning of the `limit` parameter becomes the number of records per page. + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form + + - name: result_type + type: select + required: true + options: + - value: table + label: + en_US: table text + zh_Hans: 表格文本 + - value: json + label: + en_US: json text + zh_Hans: JSON文本 + default: table + label: + en_US: Result type + zh_Hans: 结果类型 + human_description: + en_US: used for selecting the result type, table styled text or json text + zh_Hans: 用于选择结果类型，使用表格格式文本还是JSON格式文本 + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheets.py b/api/core/tools/provider/builtin/hap/tools/list_worksheets.py new file mode 100644 index 0000000000..960cbd10ac --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheets.py @@ -0,0 +1,82 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class ListWorksheetsTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not (host.startswith("http://") or host.startswith("https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + url = f"{host}/v1/open/app/get" + + result_type = tool_parameters.get('result_type', '') + if not result_type: + result_type = 'table' + + headers = { 'Content-Type': 'application/json' } + params = { "appKey": appkey, "sign": sign, } + try: + res = httpx.get(url, headers=headers, params=params, timeout=30) + res_json = res.json() + if res.is_success: + if res_json['error_code'] != 1: + return self.create_text_message("Failed to access the application. {}".format(res_json['error_msg'])) + else: + if result_type == 'json': + worksheets = [] + for section in res_json['data']['sections']: + worksheets.extend(self._extract_worksheets(section, result_type)) + return self.create_text_message(text=json.dumps(worksheets, ensure_ascii=False)) + else: + worksheets = '|worksheetId|worksheetName|description|\n|---|---|---|' + for section in res_json['data']['sections']: + worksheets += self._extract_worksheets(section, result_type) + return self.create_text_message(worksheets) + + else: + return self.create_text_message( + f"Failed to list worksheets, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to list worksheets, something went wrong: {}".format(e)) + + def _extract_worksheets(self, section, type): + items = [] + tables = '' + for item in section.get('items', []): + if item.get('type') == 0 and (not 'notes' in item or item.get('notes') != 'NO'): + if type == 'json': + filtered_item = { + 'id': item['id'], + 'name': item['name'], + 'notes': item.get('notes', '') + } + items.append(filtered_item) + else: + tables += f"\n|{item['id']}|{item['name']}|{item.get('notes', '')}|" + + for child_section in section.get('childSections', []): + if type == 'json': + items.extend(self._extract_worksheets(child_section, 'json')) + else: + tables += self._extract_worksheets(child_section, 'table') + + return items if type == 'json' else tables \ No newline at end of file diff --git a/api/core/tools/provider/builtin/hap/tools/list_worksheets.yaml b/api/core/tools/provider/builtin/hap/tools/list_worksheets.yaml new file mode 100644 index 0000000000..935b72a895 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/list_worksheets.yaml @@ -0,0 +1,68 @@ +identity: + name: list_worksheets + author: Ryan Tian + label: + en_US: List Worksheets + zh_Hans: 获取应用下所有工作表 +description: + human: + en_US: List worksheets within an application + zh_Hans: 获取应用下的所有工作表和说明信息 + llm: A tool to list worksheets info within an application, imported parameter is AppKey and Sign of the application. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form + + - name: result_type + type: select + required: true + options: + - value: table + label: + en_US: table text + zh_Hans: 表格文本 + - value: json + label: + en_US: json text + zh_Hans: JSON文本 + default: table + label: + en_US: Result type + zh_Hans: 结果类型 + human_description: + en_US: used for selecting the result type, table styled text or json text + zh_Hans: 用于选择结果类型，使用表格格式文本还是JSON格式文本 + form: form diff --git a/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.py b/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.py new file mode 100644 index 0000000000..6ca1b98d90 --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.py @@ -0,0 +1,56 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class UpdateWorksheetRecordTool(BuiltinTool): + + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + appkey = tool_parameters.get('appkey', '') + if not appkey: + return self.create_text_message('Invalid parameter App Key') + sign = tool_parameters.get('sign', '') + if not sign: + return self.create_text_message('Invalid parameter Sign') + worksheet_id = tool_parameters.get('worksheet_id', '') + if not worksheet_id: + return self.create_text_message('Invalid parameter Worksheet ID') + row_id = tool_parameters.get('row_id', '') + if not row_id: + return self.create_text_message('Invalid parameter Record Row ID') + record_data = tool_parameters.get('record_data', '') + if not record_data: + return self.create_text_message('Invalid parameter Record Row Data') + + host = tool_parameters.get('host', '') + if not host: + host = 'https://api.mingdao.com' + elif not host.startswith(("http://", "https://")): + return self.create_text_message('Invalid parameter Host Address') + else: + host = f"{host[:-1] if host.endswith('/') else host}/api" + + url = f"{host}/v2/open/worksheet/editRow" + headers = {'Content-Type': 'application/json'} + payload = {"appKey": appkey, "sign": sign, "worksheetId": worksheet_id, "rowId": row_id} + + try: + payload['controls'] = json.loads(record_data) + res = httpx.post(url, headers=headers, json=payload, timeout=60) + res.raise_for_status() + res_json = res.json() + if res_json.get('error_code') != 1: + return self.create_text_message(f"Failed to update the record. {res_json['error_msg']}") + return self.create_text_message("Record updated successfully.") + except httpx.RequestError as e: + return self.create_text_message(f"Failed to update the record, request error: {e}") + except json.JSONDecodeError as e: + return self.create_text_message(f"Failed to parse JSON response: {e}") + except Exception as e: + return self.create_text_message(f"Failed to update the record, unexpected error: {e}") diff --git a/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.yaml b/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.yaml new file mode 100644 index 0000000000..fe1f8f671a --- /dev/null +++ b/api/core/tools/provider/builtin/hap/tools/update_worksheet_record.yaml @@ -0,0 +1,90 @@ +identity: + name: update_worksheet_record + author: Ryan Tian + label: + en_US: Update Worksheet Record + zh_Hans: 更新指定的一条工作表记录 +description: + human: + en_US: Updates a single record in a worksheet based on the specified record row ID + zh_Hans: 根据指定的记录ID更新一条工作表记录数据 + llm: A tool to modify existing information within a particular record of a worksheet by referencing its unique identifier. +parameters: + - name: appkey + type: secret-input + required: true + label: + en_US: App Key + zh_Hans: App Key + human_description: + en_US: The AppKey parameter for the HAP application, typically found in the application's API documentation. + zh_Hans: HAP 应用的 AppKey 参数，可以从应用 API 文档中查找到 + llm_description: the AppKey parameter for the HAP application + form: form + + - name: sign + type: secret-input + required: true + label: + en_US: Sign + zh_Hans: Sign + human_description: + en_US: The Sign parameter for the HAP application + zh_Hans: HAP 应用的 Sign 参数 + llm_description: the Sign parameter for the HAP application + form: form + + - name: worksheet_id + type: string + required: true + label: + en_US: Worksheet ID + zh_Hans: 工作表 ID + human_description: + en_US: The ID of the specified worksheet + zh_Hans: 要获取字段信息的工作表 ID + llm_description: The ID of the specified worksheet which to get the fields information. + form: llm + + - name: row_id + type: string + required: true + label: + en_US: Record Row ID + zh_Hans: 记录 ID + human_description: + en_US: The row ID of the specified record + zh_Hans: 要更新的记录 ID + llm_description: The row ID of the specified record which to be updated. + form: llm + + - name: record_data + type: string + required: true + label: + en_US: Record Row Data + zh_Hans: 记录数据 + human_description: + en_US: The fields with data of the specified record + zh_Hans: 要更新的记录数据，JSON 对象数组格式。数组元素属性：controlId-字段ID，value-字段值 + llm_description: | + The fields with data of the specified record which to be updated. It is in the format of an array of JSON objects, and the structure is defined as follows: + ``` + type RowData = { + controlId: string; // Field ID to be updated + value: string; // Field value to be updated + }[]; + ``` + form: llm + + - name: host + type: string + required: false + label: + en_US: Host Address + zh_Hans: 服务器地址 + human_description: + en_US: The address for the privately deployed HAP server. + zh_Hans: 私有部署 HAP 服务器地址，公有云无需填写 + llm_description: the address for the privately deployed HAP server. + form: form diff --git a/api/core/workflow/nodes/code/code_node.py b/api/core/workflow/nodes/code/code_node.py index e3602fcd35..5069571511 100644 --- a/api/core/workflow/nodes/code/code_node.py +++ b/api/core/workflow/nodes/code/code_node.py @@ -57,8 +57,9 @@ class CodeNode(BaseNode): variables = {} for variable_selector in node_data.variables: variable = variable_selector.variable - value = self.graph_runtime_state.variable_pool.get(variable_selector.value_selector) - variables[variable] = value.value if value else None + value = variable_pool.get_any(variable_selector.value_selector) + + variables[variable] = value # Run code try: result = CodeExecutor.execute_workflow_code_template( diff --git a/api/core/workflow/nodes/end/end_node.py b/api/core/workflow/nodes/end/end_node.py index 796fa84373..9ab6ef610e 100644 --- a/api/core/workflow/nodes/end/end_node.py +++ b/api/core/workflow/nodes/end/end_node.py @@ -22,8 +22,8 @@ class EndNode(BaseNode): outputs = {} for variable_selector in output_variables: - value = self.graph_runtime_state.variable_pool.get(variable_selector.value_selector) - outputs[variable_selector.variable] = value.value if value else None + value = self.graph_runtime_state.variable_pool.get_any(variable_selector.value_selector) + outputs[variable_selector.variable] = value return NodeRunResult( status=WorkflowNodeExecutionStatus.SUCCEEDED, diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 473d85f073..3c24c0a018 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -333,13 +333,13 @@ class HttpExecutor: if variable_pool: variable_value_mapping = {} for variable_selector in variable_selectors: - variable = variable_pool.get(variable_selector.value_selector) + variable = variable_pool.get_any(variable_selector.value_selector) if variable is None: raise ValueError(f'Variable {variable_selector.variable} not found') - if escape_quotes and isinstance(variable.value, str): - value = variable.value.replace('"', '\\"') + if escape_quotes and isinstance(variable, str): + value = variable.replace('"', '\\"') else: - value = variable.value + value = variable variable_value_mapping[variable_selector.variable] = value return variable_template_parser.format(variable_value_mapping), variable_selectors diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index bc3e737e22..779986f43a 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -40,8 +40,8 @@ class KnowledgeRetrievalNode(BaseNode): node_data = cast(KnowledgeRetrievalNodeData, self.node_data) # extract variables - variable = self.graph_runtime_state.variable_pool.get(node_data.query_variable_selector) - query = variable.value if variable else None + variable = self.graph_runtime_state.variable_pool.get_any(node_data.query_variable_selector) + query = variable variables = { 'query': query } diff --git a/api/core/workflow/nodes/llm/llm_node.py b/api/core/workflow/nodes/llm/llm_node.py index 6726155487..0250a01131 100644 --- a/api/core/workflow/nodes/llm/llm_node.py +++ b/api/core/workflow/nodes/llm/llm_node.py @@ -405,7 +405,7 @@ class LLMNode(BaseNode): if 'content' not in item: raise ValueError(f'Invalid context structure: {item}') - context_str += item['content'] + '\n' + context_str += item['content'].text + '\n' retriever_resource = self._convert_to_original_retriever_resource(item) if retriever_resource: diff --git a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py index 6d4b957671..8f3ecf3793 100644 --- a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py +++ b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py @@ -71,10 +71,10 @@ class ParameterExtractorNode(LLMNode): Run the node. """ node_data = cast(ParameterExtractorNodeData, self.node_data) - variable = self.graph_runtime_state.variable_pool.get(node_data.query) + variable = self.graph_runtime_state.variable_pool.get_any(node_data.query) if not variable: raise ValueError("Input variable content not found or is empty") - query = variable.value + query = variable inputs = { 'query': query, @@ -568,8 +568,8 @@ class ParameterExtractorNode(LLMNode): variable_template_parser = VariableTemplateParser(instruction) inputs = {} for selector in variable_template_parser.extract_variable_selectors(): - variable = variable_pool.get(selector.value_selector) - inputs[selector.variable] = variable.value if variable else None + variable = variable_pool.get_any(selector.value_selector) + inputs[selector.variable] = variable return variable_template_parser.format(inputs) diff --git a/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py b/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py index 144cb015da..61ee59ec92 100644 --- a/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py +++ b/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py @@ -19,26 +19,26 @@ class VariableAggregatorNode(BaseNode): if not node_data.advanced_settings or not node_data.advanced_settings.group_enabled: for selector in node_data.variables: - variable = self.graph_runtime_state.variable_pool.get(selector) + variable = self.graph_runtime_state.variable_pool.get_any(selector) if variable is not None: outputs = { - "output": variable.value + "output": variable } inputs = { - '.'.join(selector[1:]): variable.value + '.'.join(selector[1:]): variable } break else: for group in node_data.advanced_settings.groups: for selector in group.variables: - variable = self.graph_runtime_state.variable_pool.get(selector) + variable = self.graph_runtime_state.variable_pool.get_any(selector) if variable is not None: outputs[group.group_name] = { - 'output': variable.value + 'output': variable } - inputs['.'.join(selector[1:])] = variable.value + inputs['.'.join(selector[1:])] = variable break return NodeRunResult( diff --git a/api/libs/helper.py b/api/libs/helper.py index 335c6688f4..15cd65dd6a 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -15,6 +15,7 @@ from zoneinfo import available_timezones from flask import Response, current_app, stream_with_context from flask_restful import fields +from core.app.features.rate_limiting.rate_limit import RateLimitGenerator from extensions.ext_redis import redis_client from models.account import Account @@ -159,7 +160,7 @@ def generate_text_hash(text: str) -> str: return sha256(hash_text.encode()).hexdigest() -def compact_generate_response(response: Union[dict, Generator]) -> Response: +def compact_generate_response(response: Union[dict, RateLimitGenerator]) -> Response: if isinstance(response, dict): return Response(response=json.dumps(response), status=200, mimetype='application/json') else: diff --git a/api/poetry.lock b/api/poetry.lock index 4b90b63e9f..353548eea7 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -7548,13 +7548,13 @@ torch = ["safetensors[numpy]", "torch (>=1.10)"] [[package]] name = "sentry-sdk" -version = "1.39.2" +version = "2.8.0" description = "Python client for Sentry (https://sentry.io)" optional = false -python-versions = "*" +python-versions = ">=3.6" files = [ - {file = "sentry-sdk-1.39.2.tar.gz", hash = "sha256:24c83b0b41c887d33328a9166f5950dc37ad58f01c9f2fbff6b87a6f1094170c"}, - {file = "sentry_sdk-1.39.2-py2.py3-none-any.whl", hash = "sha256:acaf597b30258fc7663063b291aa99e58f3096e91fe1e6634f4b79f9c1943e8e"}, + {file = "sentry_sdk-2.8.0-py2.py3-none-any.whl", hash = "sha256:6051562d2cfa8087bb8b4b8b79dc44690f8a054762a29c07e22588b1f619bfb5"}, + {file = "sentry_sdk-2.8.0.tar.gz", hash = "sha256:aa4314f877d9cd9add5a0c9ba18e3f27f99f7de835ce36bd150e48a41c7c646f"}, ] [package.dependencies] @@ -7562,28 +7562,33 @@ blinker = {version = ">=1.1", optional = true, markers = "extra == \"flask\""} certifi = "*" flask = {version = ">=0.11", optional = true, markers = "extra == \"flask\""} markupsafe = {version = "*", optional = true, markers = "extra == \"flask\""} -urllib3 = {version = ">=1.26.11", markers = "python_version >= \"3.6\""} +urllib3 = ">=1.26.11" [package.extras] aiohttp = ["aiohttp (>=3.5)"] +anthropic = ["anthropic (>=0.16)"] arq = ["arq (>=0.23)"] asyncpg = ["asyncpg (>=0.23)"] beam = ["apache-beam (>=2.12)"] bottle = ["bottle (>=0.12.13)"] celery = ["celery (>=3)"] +celery-redbeat = ["celery-redbeat (>=2)"] chalice = ["chalice (>=1.16.0)"] clickhouse-driver = ["clickhouse-driver (>=0.2.0)"] django = ["django (>=1.8)"] falcon = ["falcon (>=1.4)"] fastapi = ["fastapi (>=0.79.0)"] flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"] -grpcio = ["grpcio (>=1.21.1)"] +grpcio = ["grpcio (>=1.21.1)", "protobuf (>=3.8.0)"] httpx = ["httpx (>=0.16.0)"] huey = ["huey (>=2)"] +huggingface-hub = ["huggingface-hub (>=0.22)"] +langchain = ["langchain (>=0.0.210)"] loguru = ["loguru (>=0.5)"] +openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] -opentelemetry-experimental = ["opentelemetry-distro (>=0.40b0,<1.0)", "opentelemetry-instrumentation-aiohttp-client (>=0.40b0,<1.0)", "opentelemetry-instrumentation-django (>=0.40b0,<1.0)", "opentelemetry-instrumentation-fastapi (>=0.40b0,<1.0)", "opentelemetry-instrumentation-flask (>=0.40b0,<1.0)", "opentelemetry-instrumentation-requests (>=0.40b0,<1.0)", "opentelemetry-instrumentation-sqlite3 (>=0.40b0,<1.0)", "opentelemetry-instrumentation-urllib (>=0.40b0,<1.0)"] -pure-eval = ["asttokens", "executing", "pure_eval"] +opentelemetry-experimental = ["opentelemetry-instrumentation-aio-pika (==0.46b0)", "opentelemetry-instrumentation-aiohttp-client (==0.46b0)", "opentelemetry-instrumentation-aiopg (==0.46b0)", "opentelemetry-instrumentation-asgi (==0.46b0)", "opentelemetry-instrumentation-asyncio (==0.46b0)", "opentelemetry-instrumentation-asyncpg (==0.46b0)", "opentelemetry-instrumentation-aws-lambda (==0.46b0)", "opentelemetry-instrumentation-boto (==0.46b0)", "opentelemetry-instrumentation-boto3sqs (==0.46b0)", "opentelemetry-instrumentation-botocore (==0.46b0)", "opentelemetry-instrumentation-cassandra (==0.46b0)", "opentelemetry-instrumentation-celery (==0.46b0)", "opentelemetry-instrumentation-confluent-kafka (==0.46b0)", "opentelemetry-instrumentation-dbapi (==0.46b0)", "opentelemetry-instrumentation-django (==0.46b0)", "opentelemetry-instrumentation-elasticsearch (==0.46b0)", "opentelemetry-instrumentation-falcon (==0.46b0)", "opentelemetry-instrumentation-fastapi (==0.46b0)", "opentelemetry-instrumentation-flask (==0.46b0)", "opentelemetry-instrumentation-grpc (==0.46b0)", "opentelemetry-instrumentation-httpx (==0.46b0)", "opentelemetry-instrumentation-jinja2 (==0.46b0)", "opentelemetry-instrumentation-kafka-python (==0.46b0)", "opentelemetry-instrumentation-logging (==0.46b0)", "opentelemetry-instrumentation-mysql (==0.46b0)", "opentelemetry-instrumentation-mysqlclient (==0.46b0)", "opentelemetry-instrumentation-pika (==0.46b0)", "opentelemetry-instrumentation-psycopg (==0.46b0)", "opentelemetry-instrumentation-psycopg2 (==0.46b0)", "opentelemetry-instrumentation-pymemcache (==0.46b0)", "opentelemetry-instrumentation-pymongo (==0.46b0)", "opentelemetry-instrumentation-pymysql (==0.46b0)", "opentelemetry-instrumentation-pyramid (==0.46b0)", "opentelemetry-instrumentation-redis (==0.46b0)", "opentelemetry-instrumentation-remoulade (==0.46b0)", "opentelemetry-instrumentation-requests (==0.46b0)", "opentelemetry-instrumentation-sklearn (==0.46b0)", "opentelemetry-instrumentation-sqlalchemy (==0.46b0)", "opentelemetry-instrumentation-sqlite3 (==0.46b0)", "opentelemetry-instrumentation-starlette (==0.46b0)", "opentelemetry-instrumentation-system-metrics (==0.46b0)", "opentelemetry-instrumentation-threading (==0.46b0)", "opentelemetry-instrumentation-tornado (==0.46b0)", "opentelemetry-instrumentation-tortoiseorm (==0.46b0)", "opentelemetry-instrumentation-urllib (==0.46b0)", "opentelemetry-instrumentation-urllib3 (==0.46b0)", "opentelemetry-instrumentation-wsgi (==0.46b0)"] +pure-eval = ["asttokens", "executing", "pure-eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] @@ -7592,7 +7597,7 @@ sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] starlette = ["starlette (>=0.19.1)"] starlite = ["starlite (>=1.48)"] -tornado = ["tornado (>=5)"] +tornado = ["tornado (>=6)"] [[package]] name = "setuptools" @@ -9438,4 +9443,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "9b1821b6e5d6d44947cc011c2d635a366557582b4540b99e0ff53a3078a989e5" +content-hash = "ae278035ed74936930da140681218a2f0762c4bbbc6bd985d53924a56df595cf" diff --git a/api/pyproject.toml b/api/pyproject.toml index d37d4c21f0..a6fd87371d 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -163,7 +163,7 @@ redis = { version = "~5.0.3", extras = ["hiredis"] } replicate = "~0.22.0" resend = "~0.7.0" safetensors = "~0.4.3" -sentry-sdk = { version = "~1.39.2", extras = ["flask"] } +sentry-sdk = { version = "~2.8.0", extras = ["flask"] } sqlalchemy = "~2.0.29" tencentcloud-sdk-python-hunyuan = "~3.0.1158" tiktoken = "~0.7.0" diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index e894570b97..cff4ba8af9 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -21,7 +21,7 @@ class AppGenerateService: args: Any, invoke_from: InvokeFrom, streaming: bool = True, - ) -> Union[dict, Generator[dict, None, None]]: + ): """ App Content Generate :param app_model: app model diff --git a/api/services/hit_testing_service.py b/api/services/hit_testing_service.py index b83e1d8cb7..a04ffdfbbe 100644 --- a/api/services/hit_testing_service.py +++ b/api/services/hit_testing_service.py @@ -40,7 +40,7 @@ class HitTestingService: all_documents = RetrievalService.retrieve(retrival_method=retrieval_model['search_method'], dataset_id=dataset.id, - query=query, + query=cls.escape_query_for_search(query), top_k=retrieval_model['top_k'], score_threshold=retrieval_model['score_threshold'] if retrieval_model['score_threshold_enabled'] else None, @@ -104,3 +104,7 @@ class HitTestingService: if not query or len(query) > 250: raise ValueError('Query is required and cannot exceed 250 characters') + + @staticmethod + def escape_query_for_search(query: str) -> str: + return query.replace('"', '\\"') diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index 4f59b86c12..0983839996 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -131,9 +131,8 @@ class ModelLoadBalancingService: load_balancing_configs.insert(0, inherit_config) else: # move the inherit configuration to the first - for i, load_balancing_config in enumerate(load_balancing_configs): + for i, load_balancing_config in enumerate(load_balancing_configs[:]): if load_balancing_config.name == '__inherit__': - # FIXME: Mutation to loop iterable `load_balancing_configs` during iteration inherit_config = load_balancing_configs.pop(i) load_balancing_configs.insert(0, inherit_config) diff --git a/docker/startupscripts/init.sh b/docker/startupscripts/init.sh new file mode 100755 index 0000000000..ee7600850a --- /dev/null +++ b/docker/startupscripts/init.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +DB_INITIALISED="/opt/oracle/oradata/dbinit" +#[ -f ${DB_INITIALISED} ] && exit +#touch ${DB_INITIALISED} +if [ -f ${DB_INITIALISED} ]; then + echo 'File exists. Standards for have been Init' + exit +else + echo 'File does not exist. Standards for first time Strart up this DB' + "$ORACLE_HOME"/bin/sqlplus -s "/ as sysdba" @"/opt/oracle/scripts/startup/init_user.script"; + touch ${DB_INITIALISED} +fi diff --git a/docker/startupscripts/create_user.sql b/docker/startupscripts/init_user.script similarity index 70% rename from docker/startupscripts/create_user.sql rename to docker/startupscripts/init_user.script index b80e19c3b0..7aa7c28049 100755 --- a/docker/startupscripts/create_user.sql +++ b/docker/startupscripts/init_user.script @@ -3,3 +3,8 @@ ALTER SYSTEM SET PROCESSES=500 SCOPE=SPFILE; alter session set container= freepdb1; create user dify identified by dify DEFAULT TABLESPACE users quota unlimited on users; grant DB_DEVELOPER_ROLE to dify; + +BEGIN +CTX_DDL.CREATE_PREFERENCE('my_chinese_vgram_lexer','CHINESE_VGRAM_LEXER'); +END; +/ diff --git a/web/app/components/app/configuration/config-prompt/index.tsx b/web/app/components/app/configuration/config-prompt/index.tsx index 7e40fdc84e..bea4a9e455 100644 --- a/web/app/components/app/configuration/config-prompt/index.tsx +++ b/web/app/components/app/configuration/config-prompt/index.tsx @@ -19,9 +19,6 @@ export type IPromptProps = { promptTemplate: string promptVariables: PromptVariable[] readonly?: boolean - noTitle?: boolean - gradientBorder?: boolean - editorHeight?: number onChange?: (prompt: string, promptVariables: PromptVariable[]) => void } @@ -29,10 +26,7 @@ const Prompt: FC = ({ mode, promptTemplate, promptVariables, - noTitle, - gradientBorder, readonly = false, - editorHeight, onChange, }) => { const { t } = useTranslation() @@ -105,9 +99,6 @@ const Prompt: FC = ({ promptVariables={promptVariables} readonly={readonly} onChange={onChange} - noTitle={noTitle} - gradientBorder={gradientBorder} - editorHeight={editorHeight} /> ) } diff --git a/web/app/components/app/configuration/config-prompt/simple-prompt-input.tsx b/web/app/components/app/configuration/config-prompt/simple-prompt-input.tsx index b0a140fc97..a15f538227 100644 --- a/web/app/components/app/configuration/config-prompt/simple-prompt-input.tsx +++ b/web/app/components/app/configuration/config-prompt/simple-prompt-input.tsx @@ -28,7 +28,6 @@ import { useEventEmitterContextContext } from '@/context/event-emitter' import { ADD_EXTERNAL_DATA_TOOL } from '@/app/components/app/configuration/config-var' import { INSERT_VARIABLE_VALUE_BLOCK_COMMAND } from '@/app/components/base/prompt-editor/plugins/variable-block' import { PROMPT_EDITOR_UPDATE_VALUE_BY_EVENT_EMITTER } from '@/app/components/base/prompt-editor/plugins/update-block' -import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' export type ISimplePromptInput = { mode: AppType @@ -36,9 +35,6 @@ export type ISimplePromptInput = { promptVariables: PromptVariable[] readonly?: boolean onChange?: (promp: string, promptVariables: PromptVariable[]) => void - noTitle?: boolean - gradientBorder?: boolean - editorHeight?: number } const Prompt: FC = ({ @@ -47,14 +43,8 @@ const Prompt: FC = ({ promptVariables, readonly = false, onChange, - noTitle, - gradientBorder, - editorHeight: initEditorHeight, }) => { const { t } = useTranslation() - const media = useBreakpoints() - const isMobile = media === MediaType.mobile - const { eventEmitter } = useEventEmitterContextContext() const { modelConfig, @@ -126,11 +116,6 @@ const Prompt: FC = ({ const [showAutomatic, { setTrue: showAutomaticTrue, setFalse: showAutomaticFalse }] = useBoolean(false) const handleAutomaticRes = (res: AutomaticRes) => { - // put eventEmitter in first place to prevent overwrite the configs.prompt_variables.But another problem is that prompt won't hight the prompt_variables. - eventEmitter?.emit({ - type: PROMPT_EDITOR_UPDATE_VALUE_BY_EVENT_EMITTER, - payload: res.prompt, - } as any) const newModelConfig = produce(modelConfig, (draft) => { draft.configs.prompt_template = res.prompt draft.configs.prompt_variables = res.variables.map(key => ({ key, name: key, type: 'string', required: true })) @@ -140,35 +125,36 @@ const Prompt: FC = ({ if (mode !== AppType.completion) setIntroduction(res.opening_statement) showAutomaticFalse() + eventEmitter?.emit({ + type: PROMPT_EDITOR_UPDATE_VALUE_BY_EVENT_EMITTER, + payload: res.prompt, + } as any) } - const minHeight = initEditorHeight || 228 + const minHeight = 228 const [editorHeight, setEditorHeight] = useState(minHeight) return ( -
+
- {!noTitle && ( -
-
-
{mode !== AppType.completion ? t('appDebug.chatSubTitle') : t('appDebug.completionSubTitle')}
- {!readonly && ( - - {t('appDebug.promptTip')} -
} - selector='config-prompt-tooltip'> - - - )} -
-
- {!isAgent && !readonly && !isMobile && ( - - )} -
+
+
+
{mode !== AppType.completion ? t('appDebug.chatSubTitle') : t('appDebug.completionSubTitle')}
+ {!readonly && ( + + {t('appDebug.promptTip')} +
} + selector='config-prompt-tooltip'> + + + )}
- )} - +
+ {!isAgent && !readonly && ( + + )} +
+
= ({ onBlur={() => { handleChange(promptTemplate, getVars(promptTemplate)) }} - editable={!readonly} />
diff --git a/web/app/components/app/configuration/config/automatic/automatic-btn.tsx b/web/app/components/app/configuration/config/automatic/automatic-btn.tsx index f70976082d..40a9b9d799 100644 --- a/web/app/components/app/configuration/config/automatic/automatic-btn.tsx +++ b/web/app/components/app/configuration/config/automatic/automatic-btn.tsx @@ -2,21 +2,29 @@ import type { FC } from 'react' import React from 'react' import { useTranslation } from 'react-i18next' -import { Generator } from '@/app/components/base/icons/src/vender/other' export type IAutomaticBtnProps = { onClick: () => void } + +const leftIcon = ( + + + + + + +) const AutomaticBtn: FC = ({ onClick, }) => { const { t } = useTranslation() return ( -
- + {leftIcon} {t('appDebug.operation.automatic')}
) diff --git a/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx b/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx index 13cf857edf..fa58253cac 100644 --- a/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx +++ b/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx @@ -1,20 +1,8 @@ 'use client' import type { FC } from 'react' -import React, { useCallback } from 'react' +import React from 'react' import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' -import { - RiDatabase2Line, - RiFileExcel2Line, - RiGitCommitLine, - RiNewspaperLine, - RiPresentationLine, - RiRoadMapLine, - RiTerminalBoxLine, - RiTranslate, - RiUser2Line, -} from '@remixicon/react' -import s from './style.module.css' import Modal from '@/app/components/base/modal' import Button from '@/app/components/base/button' import Toast from '@/app/components/base/toast' @@ -26,97 +14,57 @@ import OpeningStatement from '@/app/components/app/configuration/features/chat-g import GroupName from '@/app/components/app/configuration/base/group-name' import Loading from '@/app/components/base/loading' import Confirm from '@/app/components/base/confirm' - // type import type { AutomaticRes } from '@/service/debug' -import { Generator } from '@/app/components/base/icons/src/vender/other' +import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' + +const noDataIcon = ( + + + +) export type IGetAutomaticResProps = { mode: AppType isShow: boolean onClose: () => void onFinished: (res: AutomaticRes) => void - isInLLMNode?: boolean } -const TryLabel: FC<{ - Icon: any - text: string - onClick: () => void -}> = ({ Icon, text, onClick }) => { - return ( -
- -
{text}
-
- ) -} +const genIcon = ( + + + + + +) const GetAutomaticRes: FC = ({ mode, isShow, onClose, - isInLLMNode, + // appId, onFinished, }) => { const { t } = useTranslation() - const tryList = [ - { - icon: RiTerminalBoxLine, - key: 'pythonDebugger', - }, - { - icon: RiTranslate, - key: 'translation', - }, - { - icon: RiPresentationLine, - key: 'meetingTakeaways', - }, - { - icon: RiNewspaperLine, - key: 'writingsPolisher', - }, - { - icon: RiUser2Line, - key: 'professionalAnalyst', - }, - { - icon: RiFileExcel2Line, - key: 'excelFormulaExpert', - }, - { - icon: RiRoadMapLine, - key: 'travelPlanning', - }, - { - icon: RiDatabase2Line, - key: 'SQLSorcerer', - }, - { - icon: RiGitCommitLine, - key: 'GitGud', - }, - ] + const media = useBreakpoints() + const isMobile = media === MediaType.mobile - const [instruction, setInstruction] = React.useState('') - const handleChooseTemplate = useCallback((key: string) => { - return () => { - const template = t(`appDebug.generate.template.${key}.instruction`) - setInstruction(template) - } - }, [t]) + const [audiences, setAudiences] = React.useState('') + const [hopingToSolve, setHopingToSolve] = React.useState('') const isValid = () => { - if (instruction.trim() === '') { + if (audiences.trim() === '') { Toast.notify({ type: 'error', - message: t('common.errorMsg.fieldRequired', { - field: t('appDebug.generate.instruction'), - }), + message: t('appDebug.automatic.audiencesRequired'), + }) + return false + } + if (hopingToSolve.trim() === '') { + Toast.notify({ + type: 'error', + message: t('appDebug.automatic.problemRequired'), }) return false } @@ -128,17 +76,14 @@ const GetAutomaticRes: FC = ({ const renderLoading = (
-
{t('appDebug.generate.loading')}
+
{t('appDebug.automatic.loading')}
) const renderNoData = (
- -
-
{t('appDebug.generate.noDataLine1')}
-
{t('appDebug.generate.noDataLine2')}
-
+ {noDataIcon} +
{t('appDebug.automatic.noData')}
) @@ -150,7 +95,8 @@ const GetAutomaticRes: FC = ({ setLoadingTrue() try { const res = await generateRule({ - instruction, + audiences, + hoping_to_solve: hopingToSolve, }) setRes(res) } @@ -161,7 +107,24 @@ const GetAutomaticRes: FC = ({ const [showConfirmOverwrite, setShowConfirmOverwrite] = React.useState(false) + const isShowAutoPromptInput = () => { + if (isMobile) { + // hide prompt panel on mobile if it is loading or has had result + if (isLoading || res) + return false + return true + } + + // always display prompt panel on desktop mode + return true + } + const isShowAutoPromptResPlaceholder = () => { + if (isMobile) { + // hide placeholder panel on mobile + return false + } + return !isLoading && !res } @@ -169,96 +132,75 @@ const GetAutomaticRes: FC = ({ -
-
-
-
{t('appDebug.generate.title')}
-
{t('appDebug.generate.description')}
-
-
-
-
{t('appDebug.generate.tryIt')}
-
-
-
- {tryList.map(item => ( - - ))} -
+
+ {isShowAutoPromptInput() &&
+
+
{t('appDebug.automatic.title')}
+
{t('appDebug.automatic.description')}
{/* inputs */} -
-
-
{t('appDebug.generate.instruction')}
-