From 2c427e04be3ebbdd624d7b0c85bed3336e2ae040 Mon Sep 17 00:00:00 2001
From: crazywoola <100913391+crazywoola@users.noreply.github.com>
Date: Wed, 21 Aug 2024 20:25:45 +0800
Subject: [PATCH] Feat/7134 use dataset api create a dataset with permission
(#7508)
---
api/.env.example | 4 ++--
api/controllers/console/datasets/datasets.py | 6 +++---
.../service_api/dataset/dataset.py | 7 +++++--
api/models/dataset.py | 6 ++++++
api/services/dataset_service.py | 19 ++++++++++---------
.../datasets/template/template.en.mdx | 11 +++++++++--
.../datasets/template/template.zh.mdx | 11 +++++++++--
7 files changed, 44 insertions(+), 20 deletions(-)
diff --git a/api/.env.example b/api/.env.example
index f81675fd53..edbb684cc7 100644
--- a/api/.env.example
+++ b/api/.env.example
@@ -247,8 +247,8 @@ API_TOOL_DEFAULT_READ_TIMEOUT=60
HTTP_REQUEST_MAX_CONNECT_TIMEOUT=300
HTTP_REQUEST_MAX_READ_TIMEOUT=600
HTTP_REQUEST_MAX_WRITE_TIMEOUT=600
-HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 # 10MB
-HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 # 1MB
+HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760
+HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576
# Log file path
LOG_FILE=
diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py
index f2a9a965ae..b9a1c25154 100644
--- a/api/controllers/console/datasets/datasets.py
+++ b/api/controllers/console/datasets/datasets.py
@@ -24,7 +24,7 @@ from fields.app_fields import related_app_list
from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
from fields.document_fields import document_status_fields
from libs.login import login_required
-from models.dataset import Dataset, Document, DocumentSegment
+from models.dataset import Dataset, DatasetPermissionEnum, Document, DocumentSegment
from models.model import ApiToken, UploadFile
from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService
@@ -202,7 +202,7 @@ class DatasetApi(Resource):
nullable=True,
help='Invalid indexing technique.')
parser.add_argument('permission', type=str, location='json', choices=(
- 'only_me', 'all_team_members', 'partial_members'), help='Invalid permission.'
+ DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM, DatasetPermissionEnum.PARTIAL_TEAM), help='Invalid permission.'
)
parser.add_argument('embedding_model', type=str,
location='json', help='Invalid embedding model.')
@@ -239,7 +239,7 @@ class DatasetApi(Resource):
tenant_id, dataset_id_str, data.get('partial_member_list')
)
# clear partial member list when permission is only_me or all_team_members
- elif data.get('permission') == 'only_me' or data.get('permission') == 'all_team_members':
+ elif data.get('permission') == DatasetPermissionEnum.ONLY_ME or data.get('permission') == DatasetPermissionEnum.ALL_TEAM:
DatasetPermissionService.clear_partial_member_list(dataset_id_str)
partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py
index 8dd16c0787..e0863859a2 100644
--- a/api/controllers/service_api/dataset/dataset.py
+++ b/api/controllers/service_api/dataset/dataset.py
@@ -10,7 +10,7 @@ from core.model_runtime.entities.model_entities import ModelType
from core.provider_manager import ProviderManager
from fields.dataset_fields import dataset_detail_fields
from libs.login import current_user
-from models.dataset import Dataset
+from models.dataset import Dataset, DatasetPermissionEnum
from services.dataset_service import DatasetService
@@ -78,6 +78,8 @@ class DatasetListApi(DatasetApiResource):
parser.add_argument('indexing_technique', type=str, location='json',
choices=Dataset.INDEXING_TECHNIQUE_LIST,
help='Invalid indexing technique.')
+ parser.add_argument('permission', type=str, location='json', choices=(
+ DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM, DatasetPermissionEnum.PARTIAL_TEAM), help='Invalid permission.', required=False, nullable=False)
args = parser.parse_args()
try:
@@ -85,7 +87,8 @@ class DatasetListApi(DatasetApiResource):
tenant_id=tenant_id,
name=args['name'],
indexing_technique=args['indexing_technique'],
- account=current_user
+ account=current_user,
+ permission=args['permission']
)
except services.errors.dataset.DatasetNameDuplicateError:
raise DatasetNameDuplicateError()
diff --git a/api/models/dataset.py b/api/models/dataset.py
index 0d48177eb6..203031c7b9 100644
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@@ -1,4 +1,5 @@
import base64
+import enum
import hashlib
import hmac
import json
@@ -22,6 +23,11 @@ from .model import App, Tag, TagBinding, UploadFile
from .types import StringUUID
+class DatasetPermissionEnum(str, enum.Enum):
+ ONLY_ME = 'only_me'
+ ALL_TEAM = 'all_team_members'
+ PARTIAL_TEAM = 'partial_members'
+
class Dataset(db.Model):
__tablename__ = 'datasets'
__table_args__ = (
diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py
index 12ae0e39a8..d547014866 100644
--- a/api/services/dataset_service.py
+++ b/api/services/dataset_service.py
@@ -27,6 +27,7 @@ from models.dataset import (
Dataset,
DatasetCollectionBinding,
DatasetPermission,
+ DatasetPermissionEnum,
DatasetProcessRule,
DatasetQuery,
Document,
@@ -80,21 +81,21 @@ class DatasetService:
if permitted_dataset_ids:
query = query.filter(
db.or_(
- Dataset.permission == 'all_team_members',
- db.and_(Dataset.permission == 'only_me', Dataset.created_by == user.id),
- db.and_(Dataset.permission == 'partial_members', Dataset.id.in_(permitted_dataset_ids))
+ Dataset.permission == DatasetPermissionEnum.ALL_TEAM,
+ db.and_(Dataset.permission == DatasetPermissionEnum.ONLY_ME, Dataset.created_by == user.id),
+ db.and_(Dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM, Dataset.id.in_(permitted_dataset_ids))
)
)
else:
query = query.filter(
db.or_(
- Dataset.permission == 'all_team_members',
- db.and_(Dataset.permission == 'only_me', Dataset.created_by == user.id)
+ Dataset.permission == DatasetPermissionEnum.ALL_TEAM,
+ db.and_(Dataset.permission == DatasetPermissionEnum.ONLY_ME, Dataset.created_by == user.id)
)
)
else:
# if no user, only show datasets that are shared with all team members
- query = query.filter(Dataset.permission == 'all_team_members')
+ query = query.filter(Dataset.permission == DatasetPermissionEnum.ALL_TEAM)
if search:
query = query.filter(Dataset.name.ilike(f'%{search}%'))
@@ -330,7 +331,7 @@ class DatasetService:
raise NoPermissionError(
'You do not have permission to access this dataset.'
)
- if dataset.permission == 'only_me' and dataset.created_by != user.id:
+ if dataset.permission == DatasetPermissionEnum.ONLY_ME and dataset.created_by != user.id:
logging.debug(
f'User {user.id} does not have permission to access dataset {dataset.id}'
)
@@ -351,11 +352,11 @@ class DatasetService:
@staticmethod
def check_dataset_operator_permission(user: Account = None, dataset: Dataset = None):
- if dataset.permission == 'only_me':
+ if dataset.permission == DatasetPermissionEnum.ONLY_ME:
if dataset.created_by != user.id:
raise NoPermissionError('You do not have permission to access this dataset.')
- elif dataset.permission == 'partial_members':
+ elif dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM:
if not any(
dp.dataset_id == dataset.id for dp in DatasetPermission.query.filter_by(account_id=user.id).all()
):
diff --git a/web/app/(commonLayout)/datasets/template/template.en.mdx b/web/app/(commonLayout)/datasets/template/template.en.mdx
index 44c5964d77..33451b8a0b 100644
--- a/web/app/(commonLayout)/datasets/template/template.en.mdx
+++ b/web/app/(commonLayout)/datasets/template/template.en.mdx
@@ -236,6 +236,12 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
Knowledge name
+
+ Permission
+ - only_me Only me
+ - all_team_members All team members
+ - partial_members Partial members
+
@@ -243,14 +249,15 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
title="Request"
tag="POST"
label="/datasets"
- targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name"}'`}
+ targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name", "permission": "only_me"}'`}
>
```bash {{ title: 'cURL' }}
curl --location --request POST '${apiBaseUrl}/v1/datasets' \
--header 'Authorization: Bearer {api_key}' \
--header 'Content-Type: application/json' \
--data-raw '{
- "name": "name"
+ "name": "name",
+ "permission": "only_me"
}'
```
diff --git a/web/app/(commonLayout)/datasets/template/template.zh.mdx b/web/app/(commonLayout)/datasets/template/template.zh.mdx
index 9f79b0f900..dc48f92f18 100644
--- a/web/app/(commonLayout)/datasets/template/template.zh.mdx
+++ b/web/app/(commonLayout)/datasets/template/template.zh.mdx
@@ -236,6 +236,12 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
知识库名称
+
+ 权限
+ - only_me 仅自己
+ - all_team_members 所有团队成员
+ - partial_members 部分团队成员
+
@@ -243,14 +249,15 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
title="Request"
tag="POST"
label="/datasets"
- targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name"}'`}
+ targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name", "permission": "only_me"}'`}
>
```bash {{ title: 'cURL' }}
curl --location --request POST '${props.apiBaseUrl}/datasets' \
--header 'Authorization: Bearer {api_key}' \
--header 'Content-Type: application/json' \
--data-raw '{
- "name": "name"
+ "name": "name",
+ "permission": "only_me"
}'
```