diff --git a/api/controllers/console/datasets/file.py b/api/controllers/console/datasets/file.py index 0eba232289..23ab224731 100644 --- a/api/controllers/console/datasets/file.py +++ b/api/controllers/console/datasets/file.py @@ -11,7 +11,7 @@ from controllers.console.datasets.error import ( UnsupportedFileTypeError, ) from controllers.console.setup import setup_required -from controllers.console.wraps import account_initialization_required +from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check from fields.file_fields import file_fields, upload_config_fields from libs.login import login_required from services.file_service import ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS, FileService @@ -39,6 +39,7 @@ class FileApi(Resource): @login_required @account_initialization_required @marshal_with(file_fields) + @cloud_edition_billing_resource_check(resource='documents') def post(self): # get file from request diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index d5777a330c..84f9918470 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -56,6 +56,7 @@ def cloud_edition_billing_resource_check(resource: str, members = features.members apps = features.apps vector_space = features.vector_space + documents_upload_quota = features.documents_upload_quota annotation_quota_limit = features.annotation_quota_limit if resource == 'members' and 0 < members.limit <= members.size: @@ -64,6 +65,13 @@ def cloud_edition_billing_resource_check(resource: str, abort(403, error_msg) elif resource == 'vector_space' and 0 < vector_space.limit <= vector_space.size: abort(403, error_msg) + elif resource == 'documents' and 0 < documents_upload_quota.limit <= documents_upload_quota.size: + # The api of file upload is used in the multiple places, so we need to check the source of the request from datasets + source = request.args.get('source') + if source == 'datasets': + abort(403, error_msg) + else: + return view(*args, **kwargs) elif resource == 'workspace_custom' and not features.can_replace_logo: abort(403, error_msg) elif resource == 'annotation' and 0 < annotation_quota_limit.limit < annotation_quota_limit.size: diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.py index cbe0517ed3..becfb81da1 100644 --- a/api/controllers/service_api/dataset/document.py +++ b/api/controllers/service_api/dataset/document.py @@ -28,6 +28,7 @@ class DocumentAddByTextApi(DatasetApiResource): """Resource for documents.""" @cloud_edition_billing_resource_check('vector_space', 'dataset') + @cloud_edition_billing_resource_check('documents', 'dataset') def post(self, tenant_id, dataset_id): """Create document by text.""" parser = reqparse.RequestParser() @@ -153,6 +154,7 @@ class DocumentUpdateByTextApi(DatasetApiResource): class DocumentAddByFileApi(DatasetApiResource): """Resource for documents.""" @cloud_edition_billing_resource_check('vector_space', 'dataset') + @cloud_edition_billing_resource_check('documents', 'dataset') def post(self, tenant_id, dataset_id): """Create document by upload file.""" args = {} diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index 9819c73d37..bdcbaecbea 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -89,6 +89,7 @@ def cloud_edition_billing_resource_check(resource: str, members = features.members apps = features.apps vector_space = features.vector_space + documents_upload_quota = features.documents_upload_quota if resource == 'members' and 0 < members.limit <= members.size: raise Unauthorized(error_msg) @@ -96,6 +97,8 @@ def cloud_edition_billing_resource_check(resource: str, raise Unauthorized(error_msg) elif resource == 'vector_space' and 0 < vector_space.limit <= vector_space.size: raise Unauthorized(error_msg) + elif resource == 'documents' and 0 < documents_upload_quota.limit <= documents_upload_quota.size: + raise Unauthorized(error_msg) else: return view(*args, **kwargs) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index ad06096678..44a48af58b 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -37,7 +37,7 @@ from services.errors.account import NoPermissionError from services.errors.dataset import DatasetNameDuplicateError from services.errors.document import DocumentIndexingError from services.errors.file import FileNotExistsError -from services.feature_service import FeatureService +from services.feature_service import FeatureModel, FeatureService from services.vector_service import VectorService from tasks.clean_notion_document_task import clean_notion_document_task from tasks.deal_dataset_vector_index_task import deal_dataset_vector_index_task @@ -469,6 +469,9 @@ class DocumentService: batch_upload_limit = int(current_app.config['BATCH_UPLOAD_LIMIT']) if count > batch_upload_limit: raise ValueError(f"You have reached the batch upload limit of {batch_upload_limit}.") + + DocumentService.check_documents_upload_quota(count, features) + # if dataset is empty, update dataset data_source_type if not dataset.data_source_type: dataset.data_source_type = document_data["data_source"]["type"] @@ -619,6 +622,12 @@ class DocumentService: return documents, batch + @staticmethod + def check_documents_upload_quota(count: int, features: FeatureModel): + can_upload_size = features.documents_upload_quota.limit - features.documents_upload_quota.size + if count > can_upload_size: + raise ValueError(f'You have reached the limit of your subscription. Only {can_upload_size} documents can be uploaded.') + @staticmethod def build_document(dataset: Dataset, process_rule_id: str, data_source_type: str, document_form: str, document_language: str, data_source_info: dict, created_from: str, position: int, @@ -763,6 +772,8 @@ class DocumentService: if count > batch_upload_limit: raise ValueError(f"You have reached the batch upload limit of {batch_upload_limit}.") + DocumentService.check_documents_upload_quota(count, features) + embedding_model = None dataset_collection_binding_id = None retrieval_model = None diff --git a/api/services/feature_service.py b/api/services/feature_service.py index 14d262de7c..3cf51d11a0 100644 --- a/api/services/feature_service.py +++ b/api/services/feature_service.py @@ -25,6 +25,7 @@ class FeatureModel(BaseModel): apps: LimitationModel = LimitationModel(size=0, limit=10) vector_space: LimitationModel = LimitationModel(size=0, limit=5) annotation_quota_limit: LimitationModel = LimitationModel(size=0, limit=10) + documents_upload_quota: LimitationModel = LimitationModel(size=0, limit=50) docs_processing: str = 'standard' can_replace_logo: bool = False @@ -63,6 +64,9 @@ class FeatureService: features.vector_space.size = billing_info['vector_space']['size'] features.vector_space.limit = billing_info['vector_space']['limit'] + features.documents_upload_quota.size = billing_info['documents_upload_quota']['size'] + features.documents_upload_quota.limit = billing_info['documents_upload_quota']['limit'] + features.annotation_quota_limit.size = billing_info['annotation_quota_limit']['size'] features.annotation_quota_limit.limit = billing_info['annotation_quota_limit']['limit'] diff --git a/web/app/components/billing/config.ts b/web/app/components/billing/config.ts index 1456bdd081..e3f9fa65b1 100644 --- a/web/app/components/billing/config.ts +++ b/web/app/components/billing/config.ts @@ -16,6 +16,7 @@ export const ALL_PLANS: Record = { teamMembers: 1, buildApps: 10, vectorSpace: 5, + documentsUploadQuota: 50, documentProcessingPriority: Priority.standard, logHistory: 30, customTools: unAvailable, @@ -32,6 +33,7 @@ export const ALL_PLANS: Record = { teamMembers: 3, buildApps: 50, vectorSpace: 200, + documentsUploadQuota: 500, documentProcessingPriority: Priority.priority, logHistory: NUM_INFINITE, customTools: 10, @@ -48,6 +50,7 @@ export const ALL_PLANS: Record = { teamMembers: NUM_INFINITE, buildApps: NUM_INFINITE, vectorSpace: 1000, + documentsUploadQuota: 1000, documentProcessingPriority: Priority.topPriority, logHistory: NUM_INFINITE, customTools: NUM_INFINITE, @@ -64,6 +67,7 @@ export const ALL_PLANS: Record = { teamMembers: NUM_INFINITE, buildApps: NUM_INFINITE, vectorSpace: NUM_INFINITE, + documentsUploadQuota: NUM_INFINITE, documentProcessingPriority: Priority.topPriority, logHistory: NUM_INFINITE, customTools: NUM_INFINITE, diff --git a/web/app/components/billing/pricing/plan-item.tsx b/web/app/components/billing/pricing/plan-item.tsx index 9e694041b3..4e020b20e8 100644 --- a/web/app/components/billing/pricing/plan-item.tsx +++ b/web/app/components/billing/pricing/plan-item.tsx @@ -129,6 +129,9 @@ const PlanItem: FC = ({
+ {t('billing.plansCommon.supportItems.logoChange')}
+
+
+ {t('billing.plansCommon.supportItems.bulkUpload')}
+
+ @@ -264,6 +267,10 @@ const PlanItem: FC = ({ value={planInfo.vectorSpace === NUM_INFINITE ? t('billing.plansCommon.unlimited') as string : (planInfo.vectorSpace >= 1000 ? `${planInfo.vectorSpace / 1000}G` : `${planInfo.vectorSpace}MB`)} tooltip={t('billing.plansCommon.vectorSpaceBillingTooltip') as string} /> + void onFileListUpdate?: (files: FileItem[]) => void onPreview: (file: File) => void + notSupportBatchUpload?: boolean } const FileUploader = ({ @@ -32,6 +33,7 @@ const FileUploader = ({ onFileUpdate, onFileListUpdate, onPreview, + notSupportBatchUpload, }: IFileUploaderProps) => { const { t } = useTranslation() const { notify } = useContext(ToastContext) @@ -40,6 +42,7 @@ const FileUploader = ({ const dropRef = useRef(null) const dragRef = useRef(null) const fileUploader = useRef(null) + const hideUpload = notSupportBatchUpload && fileList.length > 0 const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig) const { data: supportFileTypesResponse } = useSWR({ url: '/files/support-type' }, fetchSupportFileTypes) @@ -131,7 +134,7 @@ const FileUploader = ({ xhr: new XMLHttpRequest(), data: formData, onprogress: onProgress, - }) + }, false, undefined, '?source=datasets') .then((res: File) => { const completeFile = { fileID: fileItem.fileID, @@ -143,8 +146,8 @@ const FileUploader = ({ onFileUpdate(completeFile, 100, fileListCopy) return Promise.resolve({ ...completeFile }) }) - .catch(() => { - notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.failed') }) + .catch((e) => { + notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') }) onFileUpdate(fileItem, -2, fileListCopy) return Promise.resolve({ ...fileItem }) }) @@ -252,30 +255,36 @@ const FileUploader = ({ return (
- + {!hideUpload && ( + + )} +
{t('datasetCreation.stepOne.uploader.title')}
-
-
- - - {t('datasetCreation.stepOne.uploader.button')} - - + {!hideUpload && ( + +
+
+ + + {t('datasetCreation.stepOne.uploader.button')} + + +
+
{t('datasetCreation.stepOne.uploader.tip', { + size: fileUploadConfig.file_size_limit, + supportTypes: supportTypesShowNames, + })}
+ {dragging &&
}
-
{t('datasetCreation.stepOne.uploader.tip', { - size: fileUploadConfig.file_size_limit, - supportTypes: supportTypesShowNames, - })}
- {dragging &&
} -
+ )}
{fileList.map((fileItem, index) => (
{ return (
- +
{t('datasetCreation.stepOne.notionSyncTitle')}
{t('datasetCreation.stepOne.notionSyncTip')}
@@ -92,7 +92,7 @@ const StepOne = ({ const hasNotin = notionPages.length > 0 const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace const isShowVectorSpaceFull = (allFileLoaded || hasNotin) && isVectorSpaceFull && enableBilling - + const notSupportBatchUpload = enableBilling && plan.type === 'sandbox' const nextDisabled = useMemo(() => { if (!files.length) return true @@ -169,6 +169,7 @@ const StepOne = ({ onFileListUpdate={updateFileList} onFileUpdate={updateFile} onPreview={updateCurrentFile} + notSupportBatchUpload={notSupportBatchUpload} /> {isShowVectorSpaceFull && (
diff --git a/web/i18n/en-US/billing.ts b/web/i18n/en-US/billing.ts index 88cde4eed6..291f3073a9 100644 --- a/web/i18n/en-US/billing.ts +++ b/web/i18n/en-US/billing.ts @@ -32,6 +32,7 @@ const translation = { vectorSpace: 'Vector Space', vectorSpaceBillingTooltip: 'Each 1MB can store about 1.2million characters of vectorized data(estimated using OpenAI Embeddings, varies across models).', vectorSpaceTooltip: 'Vector Space is the long-term memory system required for LLMs to comprehend your data.', + documentsUploadQuota: 'Documents Upload Quota', documentProcessingPriority: 'Document Processing Priority', documentProcessingPriorityTip: 'For higher document processing priority, please upgrade your plan.', documentProcessingPriorityUpgrade: 'Process more data with higher accuracy at faster speeds.', @@ -56,6 +57,7 @@ const translation = { dedicatedAPISupport: 'Dedicated API support', customIntegration: 'Custom integration and support', ragAPIRequest: 'RAG API Requests', + bulkUpload: 'Bulk upload documents', agentMode: 'Agent Mode', workflow: 'Workflow', }, diff --git a/web/i18n/zh-Hans/billing.ts b/web/i18n/zh-Hans/billing.ts index 8c051d57d8..30f2a95676 100644 --- a/web/i18n/zh-Hans/billing.ts +++ b/web/i18n/zh-Hans/billing.ts @@ -32,6 +32,7 @@ const translation = { vectorSpace: '向量空间', vectorSpaceTooltip: '向量空间是 LLMs 理解您的数据所需的长期记忆系统。', vectorSpaceBillingTooltip: '向量存储是将知识库向量化处理后为让 LLMs 理解数据而使用的长期记忆存储，1MB 大约能满足1.2 million character 的向量化后数据存储（以 OpenAI Embedding 模型估算，不同模型计算方式有差异）。在向量化过程中，实际的压缩或尺寸减小取决于内容的复杂性和冗余性。', + documentsUploadQuota: '文档上传配额', documentProcessingPriority: '文档处理优先级', documentProcessingPriorityTip: '如需更高的文档处理优先级，请升级您的套餐', documentProcessingPriorityUpgrade: '以更快的速度、更高的精度处理更多的数据。', @@ -56,6 +57,7 @@ const translation = { dedicatedAPISupport: '专用 API 支持', customIntegration: '自定义集成和支持', ragAPIRequest: 'RAG API 请求', + bulkUpload: '批量上传文档', agentMode: '代理模式', workflow: '工作流', }, diff --git a/web/service/base.ts b/web/service/base.ts index fe23649929..25cc6570b8 100644 --- a/web/service/base.ts +++ b/web/service/base.ts @@ -308,7 +308,7 @@ const baseFetch = ( ]) as Promise } -export const upload = (options: any, isPublicAPI?: boolean, url?: string): Promise => { +export const upload = (options: any, isPublicAPI?: boolean, url?: string, searchParams?: string): Promise => { const urlPrefix = isPublicAPI ? PUBLIC_API_PREFIX : API_PREFIX let token = '' if (isPublicAPI) { @@ -329,7 +329,7 @@ export const upload = (options: any, isPublicAPI?: boolean, url?: string): Promi } const defaultOptions = { method: 'POST', - url: url ? `${urlPrefix}${url}` : `${urlPrefix}/files/upload`, + url: (url ? `${urlPrefix}${url}` : `${urlPrefix}/files/upload`) + (searchParams || ''), headers: { Authorization: `Bearer ${token}`, },