From 99e80a8ed094d52559dffac1709fbf1de4dbd011 Mon Sep 17 00:00:00 2001 From: glay Date: Fri, 26 Jan 2024 15:34:29 +0800 Subject: [PATCH] fix:Bedrock llm issue #2214 (#2215) Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: Chenhe Gu --- .../bedrock/llm/anthropic.claude-v1.yaml | 4 ++-- .../bedrock/llm/anthropic.claude-v2.1.yaml | 4 ++-- .../bedrock/llm/anthropic.claude-v2.yaml | 4 ++-- .../model_providers/bedrock/llm/llm.py | 14 ++++++++------ 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v1.yaml b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v1.yaml index 1c85923335..d2eba135f9 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v1.yaml +++ b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v1.yaml @@ -8,9 +8,9 @@ model_properties: parameter_rules: - name: temperature use_template: temperature - - name: topP + - name: top_p use_template: top_p - - name: topK + - name: top_k label: zh_Hans: 取样数量 en_US: Top K diff --git a/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v2.1.yaml b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v2.1.yaml index c5502daf4a..3490e57427 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v2.1.yaml +++ b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v2.1.yaml @@ -8,9 +8,9 @@ model_properties: parameter_rules: - name: temperature use_template: temperature - - name: topP + - name: top_p use_template: top_p - - name: topK + - name: top_k label: zh_Hans: 取样数量 en_US: Top K diff --git a/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v2.yaml b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v2.yaml index d12e7fce90..f2ac2e013d 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v2.yaml +++ b/api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-v2.yaml @@ -8,9 +8,9 @@ model_properties: parameter_rules: - name: temperature use_template: temperature - - name: topP + - name: top_p use_template: top_p - - name: topK + - name: top_k label: zh_Hans: 取样数量 en_US: Top K diff --git a/api/core/model_runtime/model_providers/bedrock/llm/llm.py b/api/core/model_runtime/model_providers/bedrock/llm/llm.py index 269e814d99..19c4d25fa6 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/api/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -250,9 +250,12 @@ class BedrockLargeLanguageModel(LargeLanguageModel): invoke = runtime_client.invoke_model try: + body_jsonstr=json.dumps(payload) response = invoke( - body=json.dumps(payload), modelId=model, + contentType="application/json", + accept= "*/*", + body=body_jsonstr ) except ClientError as ex: error_code = ex.response['Error']['Code'] @@ -385,7 +388,6 @@ class BedrockLargeLanguageModel(LargeLanguageModel): if not chunk: exception_name = next(iter(event)) full_ex_msg = f"{exception_name}: {event[exception_name]['message']}" - raise self._map_client_to_invoke_error(exception_name, full_ex_msg) payload = json.loads(chunk.get('bytes').decode()) @@ -396,7 +398,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel): finish_reason = payload.get("completion_reason") elif model_prefix == "anthropic": - content_delta = payload + content_delta = payload.get("completion") finish_reason = payload.get("stop_reason") elif model_prefix == "cohere": @@ -410,12 +412,12 @@ class BedrockLargeLanguageModel(LargeLanguageModel): else: raise ValueError(f"Got unknown model prefix {model_prefix} when handling stream response") - index += 1 - + # transform assistant message to prompt message assistant_prompt_message = AssistantPromptMessage( content = content_delta if content_delta else '', ) - + index += 1 + if not finish_reason: yield LLMResultChunk( model=model,