Improve MIME type detection for image URLs (#6531)

Co-authored-by: seayon <zhaoxuyang@shouqianba.com>

api/core/model_runtime/model_providers/anthropic/llm/llm.py

@@ -1,6 +1,6 @@
 import base64
+import io
 import json
-import mimetypes
 from collections.abc import Generator
 from typing import Optional, Union, cast
 
@@ -18,6 +18,7 @@ from anthropic.types import (
 )
 from anthropic.types.beta.tools import ToolsBetaMessage
 from httpx import Timeout
+from PIL import Image
 
 from core.model_runtime.callbacks.base_callback import Callback
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta
@@ -462,7 +463,8 @@ class AnthropicLargeLanguageModel(LargeLanguageModel):
                                     # fetch image data from url
                                     try:
                                         image_content = requests.get(message_content.data).content
-                                        mime_type, _ = mimetypes.guess_type(message_content.data)
+                                        with Image.open(io.BytesIO(image_content)) as img:
+                                            mime_type = f"image/{img.format.lower()}"
                                         base64_data = base64.b64encode(image_content).decode('utf-8')
                                     except Exception as ex:
                                         raise ValueError(f"Failed to fetch image data from url {message_content.data}, {ex}")

api/core/model_runtime/model_providers/bedrock/llm/llm.py

@@ -1,8 +1,8 @@
 # standard import
 import base64
+import io
 import json
 import logging
-import mimetypes
 from collections.abc import Generator
 from typing import Optional, Union, cast
 
@@ -17,6 +17,7 @@ from botocore.exceptions import (
     ServiceNotInRegionError,
     UnknownServiceError,
 )
+from PIL.Image import Image
 
 # local import
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta
@@ -381,9 +382,8 @@ class BedrockLargeLanguageModel(LargeLanguageModel):
                             try:
                                 url = message_content.data
                                 image_content = requests.get(url).content
-                                if '?' in url:
+                                with Image.open(io.BytesIO(image_content)) as img:
-                                    url = url.split('?')[0]
+                                    mime_type = f"image/{img.format.lower()}"
-                                mime_type, _ = mimetypes.guess_type(url)
                                 base64_data = base64.b64encode(image_content).decode('utf-8')
                             except Exception as ex:
                                 raise ValueError(f"Failed to fetch image data from url {message_content.data}, {ex}")

api/core/model_runtime/model_providers/google/llm/llm.py

@@ -1,7 +1,7 @@
 import base64
+import io
 import json
 import logging
-import mimetypes
 from collections.abc import Generator
 from typing import Optional, Union, cast
 
@@ -12,6 +12,7 @@ import google.generativeai.client as client
 import requests
 from google.generativeai.types import ContentType, GenerateContentResponse, HarmBlockThreshold, HarmCategory
 from google.generativeai.types.content_types import to_part
+from PIL import Image
 
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta
 from core.model_runtime.entities.message_entities import (
@@ -371,7 +372,8 @@ class GoogleLargeLanguageModel(LargeLanguageModel):
                             # fetch image data from url
                             try:
                                 image_content = requests.get(message_content.data).content
-                                mime_type, _ = mimetypes.guess_type(message_content.data)
+                                with Image.open(io.BytesIO(image_content)) as img:
+                                    mime_type = f"image/{img.format.lower()}"
                                 base64_data = base64.b64encode(image_content).decode('utf-8')
                             except Exception as ex:
                                 raise ValueError(f"Failed to fetch image data from url {message_content.data}, {ex}")

api/core/model_runtime/model_providers/vertex_ai/llm/llm.py

@@ -1,4 +1,5 @@
 import base64
+import io
 import json
 import logging
 from collections.abc import Generator
@@ -18,6 +19,7 @@ from anthropic.types import (
 )
 from google.cloud import aiplatform
 from google.oauth2 import service_account
+from PIL import Image
 from vertexai.generative_models import HarmBlockThreshold, HarmCategory
 
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
@@ -332,7 +334,8 @@ class VertexAiLargeLanguageModel(LargeLanguageModel):
                             # fetch image data from url
                             try:
                                 image_content = requests.get(message_content.data).content
-                                mime_type, _ = mimetypes.guess_type(message_content.data)
+                                with Image.open(io.BytesIO(image_content)) as img:
+                                    mime_type = f"image/{img.format.lower()}"
                                 base64_data = base64.b64encode(image_content).decode('utf-8')
                             except Exception as ex:
                                 raise ValueError(f"Failed to fetch image data from url {message_content.data}, {ex}")