AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-14 04:25:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Add a INFO-level log when fallback to gpt2tokenizer (#12508)

Browse Source
This commit is contained in:
Gen Sato 2025-01-09 15:37:46 +09:00 committed by GitHub
parent b7a4e3903e
commit dbe7a7c4fd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py
View File

@ -1,6 +1,9 @@
import logging
from threading import Lock from threading import Lock
from typing import Any from typing import Any
logger = logging.getLogger(__name__)
_tokenizer: Any = None _tokenizer: Any = None
_lock = Lock() _lock = Lock()
@ -43,5 +46,6 @@ class GPT2Tokenizer:
base_path = abspath(__file__) base_path = abspath(__file__)
gpt2_tokenizer_path = join(dirname(base_path), "gpt2") gpt2_tokenizer_path = join(dirname(base_path), "gpt2")
_tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path) _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path)
logger.info("Fallback to Transformers' GPT-2 tokenizer from tiktoken")
return _tokenizer return _tokenizer