From ccf189cb7f954be1a774ebb53c79089e0e946603 Mon Sep 17 00:00:00 2001 From: Kevin Hu Date: Wed, 13 Nov 2024 15:56:40 +0800 Subject: [PATCH] mv service_conf.yaml to conf/ and fix: add 'answer' as a parameter to 'generate' (#3379) ### What problem does this PR solve? #3373 ### Type of change - [x] Refactoring - [x] Bug fix --- agent/component/base.py | 2 +- agent/component/generate.py | 2 +- api/apps/sdk/doc.py | 2 +- api/settings.py | 2 +- conf/service_conf.yaml | 75 ++++++++++++++++++++++++++++++++++++- docker/docker-compose.yml | 1 - docker/service_conf.yaml | 74 ------------------------------------ rag/benchmark.py | 1 + rag/utils/infinity_conn.py | 2 +- 9 files changed, 80 insertions(+), 81 deletions(-) mode change 120000 => 100644 conf/service_conf.yaml delete mode 100644 docker/service_conf.yaml diff --git a/agent/component/base.py b/agent/component/base.py index dbab23c57..ae34b6155 100644 --- a/agent/component/base.py +++ b/agent/component/base.py @@ -399,7 +399,7 @@ class ComponentBase(ABC): self._param.check() def get_dependent_components(self): - cpnts = [para["component_id"] for para in self._param.query] + cpnts = [para["component_id"] for para in self._param.query if para["component_id"].lower().find("answer") < 0] return cpnts def run(self, history, **kwargs): diff --git a/agent/component/generate.py b/agent/component/generate.py index 65ce5472e..ee58fb87e 100644 --- a/agent/component/generate.py +++ b/agent/component/generate.py @@ -63,7 +63,7 @@ class Generate(ComponentBase): component_name = "Generate" def get_dependent_components(self): - cpnts = [para["component_id"] for para in self._param.parameters] + cpnts = [para["component_id"] for para in self._param.parameters if para["component_id"].lower().find("answer") < 0] return cpnts def set_cite(self, retrieval_res, answer): diff --git a/api/apps/sdk/doc.py b/api/apps/sdk/doc.py index 3383d2a1c..f9eb84271 100644 --- a/api/apps/sdk/doc.py +++ b/api/apps/sdk/doc.py @@ -1082,7 +1082,7 @@ def rm_chunk(tenant_id, dataset_id, document_id): if chunk_number != 0: DocumentService.decrement_chunk_num(document_id, dataset_id, 1, chunk_number, 0) if "chunk_ids" in req and chunk_number != len(req["chunk_ids"]): - return get_error_data_result(message=f"rm_chunk deleted chunks {chunk_number}, expect {len(req["chunk_ids"])}") + return get_error_data_result(message=f"rm_chunk deleted chunks {chunk_number}, expect {len(req['chunk_ids'])}") return get_result(message=f"deleted {chunk_number} chunks") diff --git a/api/settings.py b/api/settings.py index d2cce38b4..61efba005 100644 --- a/api/settings.py +++ b/api/settings.py @@ -191,7 +191,7 @@ AUTHENTICATION_DEFAULT_TIMEOUT = 7 * 24 * 60 * 60 # s PRIVILEGE_COMMAND_WHITELIST = [] CHECK_NODES_IDENTITY = False -if 'username' in get_base_config("es", {}): +if 'hosts' in get_base_config("es", {}): docStoreConn = rag.utils.es_conn.ESConnection() else: docStoreConn = rag.utils.infinity_conn.InfinityConnection() diff --git a/conf/service_conf.yaml b/conf/service_conf.yaml deleted file mode 120000 index 09c0c9041..000000000 --- a/conf/service_conf.yaml +++ /dev/null @@ -1 +0,0 @@ -../docker/service_conf.yaml \ No newline at end of file diff --git a/conf/service_conf.yaml b/conf/service_conf.yaml new file mode 100644 index 000000000..faa4c5cc0 --- /dev/null +++ b/conf/service_conf.yaml @@ -0,0 +1,74 @@ +ragflow: + host: 0.0.0.0 + http_port: 9380 +mysql: + name: 'rag_flow' + user: 'root' + password: 'infini_rag_flow' + host: 'mysql' + port: 3306 + max_connections: 100 + stale_timeout: 30 +minio: + user: 'rag_flow' + password: 'infini_rag_flow' + host: 'minio:9000' +es: + hosts: 'http://es01:9200' + username: 'elastic' + password: 'infini_rag_flow' +redis: + db: 1 + password: 'infini_rag_flow' + host: 'redis:6379' + +# postgres: +# name: 'rag_flow' +# user: 'rag_flow' +# password: 'infini_rag_flow' +# host: 'postgres' +# port: 5432 +# max_connections: 100 +# stale_timeout: 30 +# s3: +# endpoint: 'endpoint' +# access_key: 'access_key' +# secret_key: 'secret_key' +# region: 'region' +# azure: +# auth_type: 'sas' +# container_url: 'container_url' +# sas_token: 'sas_token' +# azure: +# auth_type: 'spn' +# account_url: 'account_url' +# client_id: 'client_id' +# secret: 'secret' +# tenant_id: 'tenant_id' +# container_name: 'container_name' +# user_default_llm: +# factory: 'Tongyi-Qianwen' +# api_key: 'sk-xxxxxxxxxxxxx' +# base_url: '' +# oauth: +# github: +# client_id: xxxxxxxxxxxxxxxxxxxxxxxxx +# secret_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxx +# url: https://github.com/login/oauth/access_token +# feishu: +# app_id: cli_xxxxxxxxxxxxxxxxxxx +# app_secret: xxxxxxxxxxxxxxxxxxxxxxxxxxxx +# app_access_token_url: https://open.feishu.cn/open-apis/auth/v3/app_access_token/internal +# user_access_token_url: https://open.feishu.cn/open-apis/authen/v1/oidc/access_token +# grant_type: 'authorization_code' +# authentication: +# client: +# switch: false +# http_app_key: +# http_secret_key: +# site: +# switch: false +# permission: +# switch: false +# component: false +# dataset: false diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b8707935c..39dceb319 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -15,7 +15,6 @@ services: - 80:80 - 443:443 volumes: - - ./service_conf.yaml:/ragflow/conf/service_conf.yaml - ./ragflow-logs:/ragflow/logs - ./nginx/ragflow.conf:/etc/nginx/conf.d/ragflow.conf - ./nginx/proxy.conf:/etc/nginx/proxy.conf diff --git a/docker/service_conf.yaml b/docker/service_conf.yaml deleted file mode 100644 index faa4c5cc0..000000000 --- a/docker/service_conf.yaml +++ /dev/null @@ -1,74 +0,0 @@ -ragflow: - host: 0.0.0.0 - http_port: 9380 -mysql: - name: 'rag_flow' - user: 'root' - password: 'infini_rag_flow' - host: 'mysql' - port: 3306 - max_connections: 100 - stale_timeout: 30 -minio: - user: 'rag_flow' - password: 'infini_rag_flow' - host: 'minio:9000' -es: - hosts: 'http://es01:9200' - username: 'elastic' - password: 'infini_rag_flow' -redis: - db: 1 - password: 'infini_rag_flow' - host: 'redis:6379' - -# postgres: -# name: 'rag_flow' -# user: 'rag_flow' -# password: 'infini_rag_flow' -# host: 'postgres' -# port: 5432 -# max_connections: 100 -# stale_timeout: 30 -# s3: -# endpoint: 'endpoint' -# access_key: 'access_key' -# secret_key: 'secret_key' -# region: 'region' -# azure: -# auth_type: 'sas' -# container_url: 'container_url' -# sas_token: 'sas_token' -# azure: -# auth_type: 'spn' -# account_url: 'account_url' -# client_id: 'client_id' -# secret: 'secret' -# tenant_id: 'tenant_id' -# container_name: 'container_name' -# user_default_llm: -# factory: 'Tongyi-Qianwen' -# api_key: 'sk-xxxxxxxxxxxxx' -# base_url: '' -# oauth: -# github: -# client_id: xxxxxxxxxxxxxxxxxxxxxxxxx -# secret_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxx -# url: https://github.com/login/oauth/access_token -# feishu: -# app_id: cli_xxxxxxxxxxxxxxxxxxx -# app_secret: xxxxxxxxxxxxxxxxxxxxxxxxxxxx -# app_access_token_url: https://open.feishu.cn/open-apis/auth/v3/app_access_token/internal -# user_access_token_url: https://open.feishu.cn/open-apis/authen/v1/oidc/access_token -# grant_type: 'authorization_code' -# authentication: -# client: -# switch: false -# http_app_key: -# http_secret_key: -# site: -# switch: false -# permission: -# switch: false -# component: false -# dataset: false diff --git a/rag/benchmark.py b/rag/benchmark.py index c26ea8251..4fb4a58fc 100644 --- a/rag/benchmark.py +++ b/rag/benchmark.py @@ -33,6 +33,7 @@ from tqdm import tqdm global max_docs max_docs = sys.maxsize + class Benchmark: def __init__(self, kb_id): self.kb_id = kb_id diff --git a/rag/utils/infinity_conn.py b/rag/utils/infinity_conn.py index f022ecdb9..db39f671c 100644 --- a/rag/utils/infinity_conn.py +++ b/rag/utils/infinity_conn.py @@ -312,7 +312,7 @@ class InfinityConnection(DocStoreConnection): for k, v in d.items(): if k.endswith("_kwd") and isinstance(v, list): d[k] = " ".join(v) - ids = [f"'{d["id"]}'" for d in documents] + ids = [f"{d['id']}" for d in documents] str_ids = ", ".join(ids) str_filter = f"id IN ({str_ids})" table_instance.delete(str_filter)