AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-14 03:35:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

generalize the generation of new collection name by dataset id (#2620)

Browse Source
This commit is contained in:
Bowen Liang 2024-02-29 12:47:10 +08:00 committed by GitHub
parent 0428f44113
commit 801d135390
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/commands.py
View File

@ -150,7 +150,7 @@ def vdb_migrate():
continue continue
if vector_type == "weaviate": if vector_type == "weaviate":
dataset_id = dataset.id dataset_id = dataset.id
collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node' collection_name = Dataset.gen_collection_name_by_id(dataset_id)
index_struct_dict = { index_struct_dict = {
"type": 'weaviate', "type": 'weaviate',
"vector_store": {"class_prefix": collection_name} "vector_store": {"class_prefix": collection_name}
@ -167,7 +167,7 @@ def vdb_migrate():
raise ValueError('Dataset Collection Bindings is not exist!') raise ValueError('Dataset Collection Bindings is not exist!')
else: else:
dataset_id = dataset.id dataset_id = dataset.id
collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node' collection_name = Dataset.gen_collection_name_by_id(dataset_id)
index_struct_dict = { index_struct_dict = {
"type": 'qdrant', "type": 'qdrant',
"vector_store": {"class_prefix": collection_name} "vector_store": {"class_prefix": collection_name}
@ -176,7 +176,7 @@ def vdb_migrate():
elif vector_type == "milvus": elif vector_type == "milvus":
dataset_id = dataset.id dataset_id = dataset.id
collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node' collection_name = Dataset.gen_collection_name_by_id(dataset_id)
index_struct_dict = { index_struct_dict = {
"type": 'milvus', "type": 'milvus',
"vector_store": {"class_prefix": collection_name} "vector_store": {"class_prefix": collection_name}

6
api/core/rag/datasource/vdb/vector_factory.py
View File

@ -39,7 +39,7 @@ class Vector:
collection_name = class_prefix collection_name = class_prefix
else: else:
dataset_id = self._dataset.id dataset_id = self._dataset.id
collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node' collection_name = Dataset.gen_collection_name_by_id(dataset_id)
index_struct_dict = { index_struct_dict = {
"type": 'weaviate', "type": 'weaviate',
"vector_store": {"class_prefix": collection_name} "vector_store": {"class_prefix": collection_name}
@ -70,7 +70,7 @@ class Vector:
collection_name = class_prefix collection_name = class_prefix
else: else:
dataset_id = self._dataset.id dataset_id = self._dataset.id
collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node' collection_name = Dataset.gen_collection_name_by_id(dataset_id)
if not self._dataset.index_struct_dict: if not self._dataset.index_struct_dict:
index_struct_dict = { index_struct_dict = {
@ -96,7 +96,7 @@ class Vector:
collection_name = class_prefix collection_name = class_prefix
else: else:
dataset_id = self._dataset.id dataset_id = self._dataset.id
collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node' collection_name = Dataset.gen_collection_name_by_id(dataset_id)
index_struct_dict = { index_struct_dict = {
"type": 'milvus', "type": 'milvus',
"vector_store": {"class_prefix": collection_name} "vector_store": {"class_prefix": collection_name}

2
api/core/rag/datasource/vdb/weaviate/weaviate_vector.py
View File

@ -70,7 +70,7 @@ class WeaviateVector(BaseVector):
return class_prefix return class_prefix
dataset_id = dataset.id dataset_id = dataset.id
return "Vector_index_" + dataset_id.replace("-", "_") + '_Node' return Dataset.gen_collection_name_by_id(dataset_id)
def to_index_struct(self) -> dict: def to_index_struct(self) -> dict:
return { return {

4
api/models/dataset.py
View File

@ -116,6 +116,10 @@ class Dataset(db.Model):
} }
return self.retrieval_model if self.retrieval_model else default_retrieval_model return self.retrieval_model if self.retrieval_model else default_retrieval_model
@staticmethod
def gen_collection_name_by_id(dataset_id: str) -> str:
normalized_dataset_id = dataset_id.replace("-", "_")
return f'Vector_index_{normalized_dataset_id}_Node'
class DatasetProcessRule(db.Model): class DatasetProcessRule(db.Model):
__tablename__ = 'dataset_process_rules' __tablename__ = 'dataset_process_rules'

2
api/services/dataset_service.py
View File

@ -1244,7 +1244,7 @@ class DatasetCollectionBindingService:
dataset_collection_binding = DatasetCollectionBinding( dataset_collection_binding = DatasetCollectionBinding(
provider_name=provider_name, provider_name=provider_name,
model_name=model_name, model_name=model_name,
collection_name="Vector_index_" + str(uuid.uuid4()).replace("-", "_") + '_Node', collection_name=Dataset.gen_collection_name_by_id(str(uuid.uuid4())),
type=collection_type type=collection_type
) )
db.session.add(dataset_collection_binding) db.session.add(dataset_collection_binding)