AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-12 10:48:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: array oob in azure openai embeddings (#1905)

Browse Source
This commit is contained in:
crazywoola 2024-01-04 13:11:54 +08:00 committed by GitHub
parent 025b859c7e
commit 18af84e193
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api/core/model_runtime/model_providers/azure_openai/text_embedding/text_embedding.py
View File

@ -54,7 +54,7 @@ class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel):
_iter = range(0, len(tokens), max_chunks) _iter = range(0, len(tokens), max_chunks)
for i in _iter: for i in _iter:
embeddings, embedding_used_tokens = self._embedding_invoke( embeddings_batch, embedding_used_tokens = self._embedding_invoke(
model=model, model=model,
client=client, client=client,
texts=tokens[i: i + max_chunks], texts=tokens[i: i + max_chunks],
@ -62,7 +62,7 @@ class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel):
) )
used_tokens += embedding_used_tokens used_tokens += embedding_used_tokens
batched_embeddings += [data for data in embeddings] batched_embeddings += embeddings_batch
results: list[list[list[float]]] = [[] for _ in range(len(texts))] results: list[list[list[float]]] = [[] for _ in range(len(texts))]
num_tokens_in_batch: list[list[int]] = [[] for _ in range(len(texts))] num_tokens_in_batch: list[list[int]] = [[] for _ in range(len(texts))]
@ -73,7 +73,7 @@ class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel):
for i in range(len(texts)): for i in range(len(texts)):
_result = results[i] _result = results[i]
if len(_result) == 0: if len(_result) == 0:
embeddings, embedding_used_tokens = self._embedding_invoke( embeddings_batch, embedding_used_tokens = self._embedding_invoke(
model=model, model=model,
client=client, client=client,
texts=[""], texts=[""],
@ -81,7 +81,7 @@ class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel):
) )
used_tokens += embedding_used_tokens used_tokens += embedding_used_tokens
average = embeddings[0] average = embeddings_batch[0]
else: else:
average = np.average(_result, axis=0, weights=num_tokens_in_batch[i]) average = np.average(_result, axis=0, weights=num_tokens_in_batch[i])
embeddings[i] = (average / np.linalg.norm(average)).tolist() embeddings[i] = (average / np.linalg.norm(average)).tolist()