From bd3a9b2f8d25c3f40bc0c16dd8bba33f178625e1 Mon Sep 17 00:00:00 2001 From: takatost Date: Thu, 24 Aug 2023 14:39:34 +0800 Subject: [PATCH] fix: xinference-chat-stream-response (#991) --- api/core/third_party/langchain/llms/xinference_llm.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/core/third_party/langchain/llms/xinference_llm.py b/api/core/third_party/langchain/llms/xinference_llm.py index dda3659d8e..1a01ee55b4 100644 --- a/api/core/third_party/langchain/llms/xinference_llm.py +++ b/api/core/third_party/langchain/llms/xinference_llm.py @@ -123,7 +123,12 @@ class XinferenceLLM(Xinference): if choices: choice = choices[0] if isinstance(choice, dict): - token = choice.get("text", "") + if 'text' in choice: + token = choice.get("text", "") + elif 'delta' in choice and 'content' in choice['delta']: + token = choice.get('delta').get('content') + else: + continue log_probs = choice.get("logprobs") if run_manager: run_manager.on_llm_new_token(