From 8e6ea4d117af5d3069de21d460e5f64cc4a83c3c Mon Sep 17 00:00:00 2001 From: shiyiyue1102 Date: Tue, 22 Apr 2025 13:12:36 +0800 Subject: [PATCH] support load .env config from nacos (#18186) --- api/configs/app_config.py | 3 + api/configs/remote_settings_sources/enums.py | 1 + .../remote_settings_sources/nacos/__init__.py | 52 ++++++++++++ .../nacos/http_request.py | 83 +++++++++++++++++++ .../remote_settings_sources/nacos/utils.py | 31 +++++++ 5 files changed, 170 insertions(+) create mode 100644 api/configs/remote_settings_sources/nacos/__init__.py create mode 100644 api/configs/remote_settings_sources/nacos/http_request.py create mode 100644 api/configs/remote_settings_sources/nacos/utils.py diff --git a/api/configs/app_config.py b/api/configs/app_config.py index cb0adb751c..3a3ad35ee7 100644 --- a/api/configs/app_config.py +++ b/api/configs/app_config.py @@ -13,6 +13,7 @@ from .observability import ObservabilityConfig from .packaging import PackagingInfo from .remote_settings_sources import RemoteSettingsSource, RemoteSettingsSourceConfig, RemoteSettingsSourceName from .remote_settings_sources.apollo import ApolloSettingsSource +from .remote_settings_sources.nacos import NacosSettingsSource logger = logging.getLogger(__name__) @@ -34,6 +35,8 @@ class RemoteSettingsSourceFactory(PydanticBaseSettingsSource): match remote_source_name: case RemoteSettingsSourceName.APOLLO: remote_source = ApolloSettingsSource(current_state) + case RemoteSettingsSourceName.NACOS: + remote_source = NacosSettingsSource(current_state) case _: logger.warning(f"Unsupported remote source: {remote_source_name}") return {} diff --git a/api/configs/remote_settings_sources/enums.py b/api/configs/remote_settings_sources/enums.py index 3081f2950f..dd998cac64 100644 --- a/api/configs/remote_settings_sources/enums.py +++ b/api/configs/remote_settings_sources/enums.py @@ -3,3 +3,4 @@ from enum import StrEnum class RemoteSettingsSourceName(StrEnum): APOLLO = "apollo" + NACOS = "nacos" diff --git a/api/configs/remote_settings_sources/nacos/__init__.py b/api/configs/remote_settings_sources/nacos/__init__.py new file mode 100644 index 0000000000..b1ce8e87bc --- /dev/null +++ b/api/configs/remote_settings_sources/nacos/__init__.py @@ -0,0 +1,52 @@ +import logging +import os +from collections.abc import Mapping +from typing import Any + +from pydantic.fields import FieldInfo + +from .http_request import NacosHttpClient + +logger = logging.getLogger(__name__) + +from configs.remote_settings_sources.base import RemoteSettingsSource + +from .utils import _parse_config + + +class NacosSettingsSource(RemoteSettingsSource): + def __init__(self, configs: Mapping[str, Any]): + self.configs = configs + self.remote_configs: dict[str, Any] = {} + self.async_init() + + def async_init(self): + data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties") + group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify") + tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "") + + params = {"dataId": data_id, "group": group, "tenant": tenant} + try: + content = NacosHttpClient().http_request("/nacos/v1/cs/configs", method="GET", headers={}, params=params) + self.remote_configs = self._parse_config(content) + except Exception as e: + logger.exception("[get-access-token] exception occurred") + raise + + def _parse_config(self, content: str) -> dict: + if not content: + return {} + try: + return _parse_config(self, content) + except Exception as e: + raise RuntimeError(f"Failed to parse config: {e}") + + def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: + if not isinstance(self.remote_configs, dict): + raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}") + + field_value = self.remote_configs.get(field_name) + if field_value is None: + return None, field_name, False + + return field_value, field_name, False diff --git a/api/configs/remote_settings_sources/nacos/http_request.py b/api/configs/remote_settings_sources/nacos/http_request.py new file mode 100644 index 0000000000..2785bd955b --- /dev/null +++ b/api/configs/remote_settings_sources/nacos/http_request.py @@ -0,0 +1,83 @@ +import base64 +import hashlib +import hmac +import logging +import os +import time + +import requests + +logger = logging.getLogger(__name__) + + +class NacosHttpClient: + def __init__(self): + self.username = os.getenv("DIFY_ENV_NACOS_USERNAME") + self.password = os.getenv("DIFY_ENV_NACOS_PASSWORD") + self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY") + self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY") + self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848") + self.token = None + self.token_ttl = 18000 + self.token_expire_time: float = 0 + + def http_request(self, url, method="GET", headers=None, params=None): + try: + self._inject_auth_info(headers, params) + response = requests.request(method, url="http://" + self.server + url, headers=headers, params=params) + response.raise_for_status() + return response.text + except requests.exceptions.RequestException as e: + return f"Request to Nacos failed: {e}" + + def _inject_auth_info(self, headers, params, module="config"): + headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"}) + + if module == "login": + return + + ts = str(int(time.time() * 1000)) + + if self.ak and self.sk: + sign_str = self.get_sign_str(params["group"], params["tenant"], ts) + headers["Spas-AccessKey"] = self.ak + headers["Spas-Signature"] = self.__do_sign(sign_str, self.sk) + headers["timeStamp"] = ts + if self.username and self.password: + self.get_access_token(force_refresh=False) + params["accessToken"] = self.token + + def __do_sign(self, sign_str, sk): + return ( + base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest()) + .decode() + .strip() + ) + + def get_sign_str(self, group, tenant, ts): + sign_str = "" + if tenant: + sign_str = tenant + "+" + if group: + sign_str = sign_str + group + "+" + if sign_str: + sign_str += ts + return sign_str + + def get_access_token(self, force_refresh=False): + current_time = time.time() + if self.token and not force_refresh and self.token_expire_time > current_time: + return self.token + + params = {"username": self.username, "password": self.password} + url = "http://" + self.server + "/nacos/v1/auth/login" + try: + resp = requests.request("POST", url, headers=None, params=params) + resp.raise_for_status() + response_data = resp.json() + self.token = response_data.get("accessToken") + self.token_ttl = response_data.get("tokenTtl", 18000) + self.token_expire_time = current_time + self.token_ttl - 10 + except Exception as e: + logger.exception("[get-access-token] exception occur") + raise diff --git a/api/configs/remote_settings_sources/nacos/utils.py b/api/configs/remote_settings_sources/nacos/utils.py new file mode 100644 index 0000000000..f3372563b1 --- /dev/null +++ b/api/configs/remote_settings_sources/nacos/utils.py @@ -0,0 +1,31 @@ +def _parse_config(self, content: str) -> dict[str, str]: + config: dict[str, str] = {} + if not content: + return config + + for line in content.splitlines(): + cleaned_line = line.strip() + if not cleaned_line or cleaned_line.startswith(("#", "!")): + continue + + separator_index = -1 + for i, c in enumerate(cleaned_line): + if c in ("=", ":") and (i == 0 or cleaned_line[i - 1] != "\\"): + separator_index = i + break + + if separator_index == -1: + continue + + key = cleaned_line[:separator_index].strip() + raw_value = cleaned_line[separator_index + 1 :].strip() + + try: + decoded_value = bytes(raw_value, "utf-8").decode("unicode_escape") + decoded_value = decoded_value.replace(r"\=", "=").replace(r"\:", ":") + except UnicodeDecodeError: + decoded_value = raw_value + + config[key] = decoded_value + + return config