AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-13 18:09:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: Incorrect order of embedded documents in CacheEmbedding (#1671)

Browse Source
This commit is contained in:
Wen O.Y 2023-12-03 19:07:00 +08:00 committed by GitHub
parent 671a8e7972
commit a1cd043fdc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
api/core/embedding/cached_embedding.py
View File

@ -18,31 +18,30 @@ class CacheEmbedding(Embeddings):
def embed_documents(self, texts: List[str]) -> List[List[float]]: def embed_documents(self, texts: List[str]) -> List[List[float]]:
"""Embed search docs.""" """Embed search docs."""
# use doc embedding cache or store if not exists # use doc embedding cache or store if not exists
text_embeddings = [] text_embeddings = [None for _ in range(len(texts))]
embedding_queue_texts = [] embedding_queue_indices = []
for text in texts: for i, text in enumerate(texts):
hash = helper.generate_text_hash(text) hash = helper.generate_text_hash(text)
embedding = db.session.query(Embedding).filter_by(model_name=self._embeddings.name, hash=hash).first() embedding = db.session.query(Embedding).filter_by(model_name=self._embeddings.name, hash=hash).first()
if embedding: if embedding:
text_embeddings.append(embedding.get_embedding()) text_embeddings[i] = embedding.get_embedding()
else: else:
embedding_queue_texts.append(text) embedding_queue_indices.append(i)
if embedding_queue_texts: if embedding_queue_indices:
try: try:
embedding_results = self._embeddings.client.embed_documents(embedding_queue_texts) embedding_results = self._embeddings.client.embed_documents([texts[i] for i in embedding_queue_indices])
except Exception as ex: except Exception as ex:
raise self._embeddings.handle_exceptions(ex) raise self._embeddings.handle_exceptions(ex)
i = 0
normalized_embedding_results = [] for i, indice in enumerate(embedding_queue_indices):
for text in embedding_queue_texts: hash = helper.generate_text_hash(texts[indice])
hash = helper.generate_text_hash(text)
try: try:
embedding = Embedding(model_name=self._embeddings.name, hash=hash) embedding = Embedding(model_name=self._embeddings.name, hash=hash)
vector = embedding_results[i] vector = embedding_results[i]
normalized_embedding = (vector / np.linalg.norm(vector)).tolist() normalized_embedding = (vector / np.linalg.norm(vector)).tolist()
normalized_embedding_results.append(normalized_embedding) text_embeddings[indice] = normalized_embedding
embedding.set_embedding(normalized_embedding) embedding.set_embedding(normalized_embedding)
db.session.add(embedding) db.session.add(embedding)
db.session.commit() db.session.commit()
@ -52,10 +51,7 @@ class CacheEmbedding(Embeddings):
except: except:
logging.exception('Failed to add embedding to db') logging.exception('Failed to add embedding to db')
continue continue
finally:
i += 1
text_embeddings.extend(normalized_embedding_results)
return text_embeddings return text_embeddings
def embed_query(self, text: str) -> List[float]: def embed_query(self, text: str) -> List[float]: