From b05cc3a1e4f87a3b8137ce5c1e6087130e620ce8 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Sat, 22 Jun 2024 10:07:03 +0800 Subject: [PATCH] refactor: extract storage provider configs into dify configs (#5443) --- api/config.py | 37 ---------------- api/configs/middleware/__init__.py | 14 +++++- .../storage/aliyun_oss_storage_config.py | 39 ++++++++++++++++ .../storage/amazon_s3_storage_config.py | 44 +++++++++++++++++++ .../storage/azure_blob_storage_config.py | 29 ++++++++++++ .../storage/google_cloud_storage_config.py | 19 ++++++++ .../storage/tencent_cos_storage_config.py | 34 ++++++++++++++ 7 files changed, 178 insertions(+), 38 deletions(-) create mode 100644 api/configs/middleware/storage/aliyun_oss_storage_config.py create mode 100644 api/configs/middleware/storage/amazon_s3_storage_config.py create mode 100644 api/configs/middleware/storage/azure_blob_storage_config.py create mode 100644 api/configs/middleware/storage/google_cloud_storage_config.py create mode 100644 api/configs/middleware/storage/tencent_cos_storage_config.py diff --git a/api/config.py b/api/config.py index 4a3abc30fa..3c4721206e 100644 --- a/api/config.py +++ b/api/config.py @@ -9,8 +9,6 @@ DEFAULTS = { 'DB_PORT': '5432', 'DB_DATABASE': 'dify', 'DB_CHARSET': '', - 'S3_USE_AWS_MANAGED_IAM': 'False', - 'S3_ADDRESS_STYLE': 'auto', 'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql', 'SQLALCHEMY_POOL_SIZE': 30, 'SQLALCHEMY_MAX_OVERFLOW': 10, @@ -100,41 +98,6 @@ class Config: if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False - - # S3 Storage settings - self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM') - self.S3_ENDPOINT = get_env('S3_ENDPOINT') - self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME') - self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY') - self.S3_SECRET_KEY = get_env('S3_SECRET_KEY') - self.S3_REGION = get_env('S3_REGION') - self.S3_ADDRESS_STYLE = get_env('S3_ADDRESS_STYLE') - - # Azure Blob Storage settings - self.AZURE_BLOB_ACCOUNT_NAME = get_env('AZURE_BLOB_ACCOUNT_NAME') - self.AZURE_BLOB_ACCOUNT_KEY = get_env('AZURE_BLOB_ACCOUNT_KEY') - self.AZURE_BLOB_CONTAINER_NAME = get_env('AZURE_BLOB_CONTAINER_NAME') - self.AZURE_BLOB_ACCOUNT_URL = get_env('AZURE_BLOB_ACCOUNT_URL') - - # Aliyun Storage settings - self.ALIYUN_OSS_BUCKET_NAME = get_env('ALIYUN_OSS_BUCKET_NAME') - self.ALIYUN_OSS_ACCESS_KEY = get_env('ALIYUN_OSS_ACCESS_KEY') - self.ALIYUN_OSS_SECRET_KEY = get_env('ALIYUN_OSS_SECRET_KEY') - self.ALIYUN_OSS_ENDPOINT = get_env('ALIYUN_OSS_ENDPOINT') - self.ALIYUN_OSS_REGION = get_env('ALIYUN_OSS_REGION') - self.ALIYUN_OSS_AUTH_VERSION = get_env('ALIYUN_OSS_AUTH_VERSION') - - # Google Cloud Storage settings - self.GOOGLE_STORAGE_BUCKET_NAME = get_env('GOOGLE_STORAGE_BUCKET_NAME') - self.GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 = get_env('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64') - - # Tencent Cos Storage settings - self.TENCENT_COS_BUCKET_NAME = get_env('TENCENT_COS_BUCKET_NAME') - self.TENCENT_COS_REGION = get_env('TENCENT_COS_REGION') - self.TENCENT_COS_SECRET_ID = get_env('TENCENT_COS_SECRET_ID') - self.TENCENT_COS_SECRET_KEY = get_env('TENCENT_COS_SECRET_KEY') - self.TENCENT_COS_SCHEME = get_env('TENCENT_COS_SCHEME') - # ------------------------ # Platform Configurations. # ------------------------ diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index f2093b918d..912e26d2a1 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -3,6 +3,11 @@ from typing import Optional from pydantic import BaseModel, Field from configs.middleware.redis_config import RedisConfig +from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig +from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig +from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig +from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig +from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig from configs.middleware.vdb.chroma_configs import ChromaConfigs from configs.middleware.vdb.milvus_configs import MilvusConfigs from configs.middleware.vdb.opensearch_configs import OpenSearchConfigs @@ -48,13 +53,21 @@ class MiddlewareConfig( # place the configs in alphabet order KeywordStoreConfigs, RedisConfig, + + # configs of storage and storage providers StorageConfigs, + AliyunOSSStorageConfig, + AzureBlobStorageConfig, + GoogleCloudStorageConfig, + TencentCloudCOSStorageConfig, + S3StorageConfig, # configs of vdb and vdb providers VectorStoreConfigs, ChromaConfigs, MilvusConfigs, OpenSearchConfigs, + OracleConfigs, PGVectorConfigs, PGVectoRSConfigs, QdrantConfigs, @@ -62,6 +75,5 @@ class MiddlewareConfig( TencentVectorDBConfigs, TiDBVectorConfigs, WeaviateConfigs, - OracleConfigs, ): pass diff --git a/api/configs/middleware/storage/aliyun_oss_storage_config.py b/api/configs/middleware/storage/aliyun_oss_storage_config.py new file mode 100644 index 0000000000..67921149d6 --- /dev/null +++ b/api/configs/middleware/storage/aliyun_oss_storage_config.py @@ -0,0 +1,39 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class AliyunOSSStorageConfig(BaseModel): + """ + Aliyun storage configs + """ + + ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field( + description='Aliyun storage ', + default=None, + ) + + ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field( + description='Aliyun storage access key', + default=None, + ) + + ALIYUN_OSS_SECRET_KEY: Optional[str] = Field( + description='Aliyun storage secret key', + default=None, + ) + + ALIYUN_OSS_ENDPOINT: Optional[str] = Field( + description='Aliyun storage endpoint URL', + default=None, + ) + + ALIYUN_OSS_REGION: Optional[str] = Field( + description='Aliyun storage region', + default=None, + ) + + ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field( + description='Aliyun storage authentication version', + default=None, + ) diff --git a/api/configs/middleware/storage/amazon_s3_storage_config.py b/api/configs/middleware/storage/amazon_s3_storage_config.py new file mode 100644 index 0000000000..21fe425fa8 --- /dev/null +++ b/api/configs/middleware/storage/amazon_s3_storage_config.py @@ -0,0 +1,44 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class S3StorageConfig(BaseModel): + """ + S3 storage configs + """ + + S3_ENDPOINT: Optional[str] = Field( + description='S3 storage endpoint', + default=None, + ) + + S3_REGION: Optional[str] = Field( + description='S3 storage region', + default=None, + ) + + S3_BUCKET_NAME: Optional[str] = Field( + description='S3 storage bucket name', + default=None, + ) + + S3_ACCESS_KEY: Optional[str] = Field( + description='S3 storage access key', + default=None, + ) + + S3_SECRET_KEY: Optional[str] = Field( + description='S3 storage secret key', + default=None, + ) + + S3_ADDRESS_STYLE: str = Field( + description='S3 storage address style', + default='auto', + ) + + S3_USE_AWS_MANAGED_IAM: bool = Field( + description='whether to use aws managed IAM for S3', + default=False, + ) diff --git a/api/configs/middleware/storage/azure_blob_storage_config.py b/api/configs/middleware/storage/azure_blob_storage_config.py new file mode 100644 index 0000000000..a37aa496f1 --- /dev/null +++ b/api/configs/middleware/storage/azure_blob_storage_config.py @@ -0,0 +1,29 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class AzureBlobStorageConfig(BaseModel): + """ + Azure Blob storage configs + """ + + AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field( + description='Azure Blob account name', + default=None, + ) + + AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field( + description='Azure Blob account key', + default=None, + ) + + AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field( + description='Azure Blob container name', + default=None, + ) + + AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field( + description='Azure Blob account url', + default=None, + ) diff --git a/api/configs/middleware/storage/google_cloud_storage_config.py b/api/configs/middleware/storage/google_cloud_storage_config.py new file mode 100644 index 0000000000..1f4d9f9883 --- /dev/null +++ b/api/configs/middleware/storage/google_cloud_storage_config.py @@ -0,0 +1,19 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class GoogleCloudStorageConfig(BaseModel): + """ + Google Cloud storage configs + """ + + GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field( + description='Google Cloud storage bucket name', + default=None, + ) + + GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field( + description='Google Cloud storage service account json base64', + default=None, + ) diff --git a/api/configs/middleware/storage/tencent_cos_storage_config.py b/api/configs/middleware/storage/tencent_cos_storage_config.py new file mode 100644 index 0000000000..1bcc4b7b44 --- /dev/null +++ b/api/configs/middleware/storage/tencent_cos_storage_config.py @@ -0,0 +1,34 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class TencentCloudCOSStorageConfig(BaseModel): + """ + Tencent Cloud COS storage configs + """ + + TENCENT_COS_BUCKET_NAME: Optional[str] = Field( + description='Tencent Cloud COS bucket name', + default=None, + ) + + TENCENT_COS_REGION: Optional[str] = Field( + description='Tencent Cloud COS region', + default=None, + ) + + TENCENT_COS_SECRET_ID: Optional[str] = Field( + description='Tencent Cloud COS secret id', + default=None, + ) + + TENCENT_COS_SECRET_KEY: Optional[str] = Field( + description='Tencent Cloud COS secret key', + default=None, + ) + + TENCENT_COS_SCHEME: Optional[str] = Field( + description='Tencent Cloud COS scheme', + default=None, + )