AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-12 09:49:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Wrong cutoff length lead to empty input in openai compatible embedding model. (#7133)

Browse Source
This commit is contained in:
Yanyi Liu 2024-08-09 19:11:57 +08:00 committed by GitHub
parent 2c188a45c8
commit 4cbeb6815b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py
View File

@ -72,7 +72,7 @@ class OllamaEmbeddingModel(TextEmbeddingModel):
num_tokens = self._get_num_tokens_by_gpt2(text) num_tokens = self._get_num_tokens_by_gpt2(text)
if num_tokens >= context_size: if num_tokens >= context_size:
cutoff = int(len(text) * (np.floor(context_size / num_tokens))) cutoff = int(np.floor(len(text) * (context_size / num_tokens)))
# if num tokens is larger than context length, only use the start # if num tokens is larger than context length, only use the start
inputs.append(text[0: cutoff]) inputs.append(text[0: cutoff])
else: else:

2
api/core/model_runtime/model_providers/openai_api_compatible/text_embedding/text_embedding.py
View File

@ -76,7 +76,7 @@ class OAICompatEmbeddingModel(_CommonOAI_API_Compat, TextEmbeddingModel):
num_tokens = self._get_num_tokens_by_gpt2(text) num_tokens = self._get_num_tokens_by_gpt2(text)
if num_tokens >= context_size: if num_tokens >= context_size:
cutoff = int(len(text) * (np.floor(context_size / num_tokens))) cutoff = int(np.floor(len(text) * (context_size / num_tokens)))
# if num tokens is larger than context length, only use the start # if num tokens is larger than context length, only use the start
inputs.append(text[0: cutoff]) inputs.append(text[0: cutoff])
else: else:

2
api/core/model_runtime/model_providers/perfxcloud/text_embedding/text_embedding.py
View File

@ -79,7 +79,7 @@ class OAICompatEmbeddingModel(_CommonOAI_API_Compat, TextEmbeddingModel):
num_tokens = self._get_num_tokens_by_gpt2(text) num_tokens = self._get_num_tokens_by_gpt2(text)
if num_tokens >= context_size: if num_tokens >= context_size:
cutoff = int(len(text) * (np.floor(context_size / num_tokens))) cutoff = int(np.floor(len(text) * (context_size / num_tokens)))
# if num tokens is larger than context length, only use the start # if num tokens is larger than context length, only use the start
inputs.append(text[0: cutoff]) inputs.append(text[0: cutoff])
else: else: