AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-15 01:05:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

add segment with keyword issue (#3351)

Browse Source 
Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
This commit is contained in:
Jyong 2024-04-11 16:57:02 +08:00 committed by GitHub
parent 37024afe9c
commit 1f302990c6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 89 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/core/rag/datasource/keyword/jieba/jieba.py
View File

@ -48,6 +48,9 @@ class Jieba(BaseKeyword):
text = texts[i] text = texts[i]
if keywords_list: if keywords_list:
keywords = keywords_list[i] keywords = keywords_list[i]
if not keywords:
keywords = keyword_table_handler.extract_keywords(text.page_content,
self._config.max_keywords_per_chunk)
else: else:
keywords = keyword_table_handler.extract_keywords(text.page_content, self._config.max_keywords_per_chunk) keywords = keyword_table_handler.extract_keywords(text.page_content, self._config.max_keywords_per_chunk)
self._update_segment_keywords(self.dataset.id, text.metadata['doc_id'], list(keywords)) self._update_segment_keywords(self.dataset.id, text.metadata['doc_id'], list(keywords))

4
api/services/dataset_service.py
View File

@ -1046,6 +1046,8 @@ class SegmentService:
credentials=embedding_model.credentials, credentials=embedding_model.credentials,
texts=[content] texts=[content]
) )
lock_name = 'add_segment_lock_document_id_{}'.format(document.id)
with redis_client.lock(lock_name, timeout=600):
max_position = db.session.query(func.max(DocumentSegment.position)).filter( max_position = db.session.query(func.max(DocumentSegment.position)).filter(
DocumentSegment.document_id == document.id DocumentSegment.document_id == document.id
).scalar() ).scalar()
@ -1085,6 +1087,8 @@ class SegmentService:
@classmethod @classmethod
def multi_create_segment(cls, segments: list, document: Document, dataset: Dataset): def multi_create_segment(cls, segments: list, document: Document, dataset: Dataset):
lock_name = 'multi_add_segment_lock_document_id_{}'.format(document.id)
with redis_client.lock(lock_name, timeout=600):
embedding_model = None embedding_model = None
if dataset.indexing_technique == 'high_quality': if dataset.indexing_technique == 'high_quality':
model_manager = ModelManager() model_manager = ModelManager()

2
web/app/components/base/tag-input/index.tsx
View File

@ -56,7 +56,9 @@ const TagInput: FC<TagInputProps> = ({
} }
onChange([...items, valueTrimed]) onChange([...items, valueTrimed])
setTimeout(() => {
setValue('') setValue('')
})
} }
} }