From daddfc9e1b60f631439aa634c817ab9ecaf596a6 Mon Sep 17 00:00:00 2001 From: Kevin Hu Date: Tue, 25 Feb 2025 12:22:37 +0800 Subject: [PATCH] Remove dup gb2312, solve currupt error. (#5326) ### What problem does this PR solve? #5252 #5325 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- rag/llm/chat_model.py | 2 ++ rag/nlp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.py index a8ba8413c..5e23c9326 100644 --- a/rag/llm/chat_model.py +++ b/rag/llm/chat_model.py @@ -47,6 +47,8 @@ class Base(ABC): model=self.model_name, messages=history, **gen_conf) + if not response.choices: + return "", 0 ans = response.choices[0].message.content.strip() if response.choices[0].finish_reason == "length": if is_chinese(ans): diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py index c98597bce..eb14adeb0 100644 --- a/rag/nlp/__init__.py +++ b/rag/nlp/__init__.py @@ -37,7 +37,7 @@ all_codecs = [ 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006', 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'euc_jp', 'euc_jis_2004', 'euc_jisx0213', 'euc_kr', - 'gb2312', 'gb18030', 'hz', 'iso2022_jp', 'iso2022_jp_1', 'iso2022_jp_2', + 'gb18030', 'hz', 'iso2022_jp', 'iso2022_jp_1', 'iso2022_jp_2', 'iso2022_jp_2004', 'iso2022_jp_3', 'iso2022_jp_ext', 'iso2022_kr', 'latin_1', 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6', 'iso8859_7', 'iso8859_8', 'iso8859_9', 'iso8859_10', 'iso8859_11', 'iso8859_13',