refactor some llm api using openai api format (#1692)

### What problem does this PR solve?

refactor some llm api using openai api format

### Type of change

- [x] Refactoring

---------

Co-authored-by: Zhedong Cen <cenzhedong2@126.com>

rag/llm/chat_model.py

@@ -24,6 +24,7 @@ from volcengine.maas.v2 import MaasService
 from rag.nlp import is_english
 from rag.utils import num_tokens_from_string
 from groq import Groq
+import os 
 import json
 import requests
 
@@ -60,9 +61,16 @@ class Base(ABC):
                 stream=True,
                 **gen_conf)
             for resp in response:
-                if not resp.choices or not resp.choices[0].delta.content:continue
+                if not resp.choices:continue
                 ans += resp.choices[0].delta.content
-                total_tokens += 1
+                total_tokens = (
+                    (
+                        total_tokens
+                        + num_tokens_from_string(resp.choices[0].delta.content)
+                    )
+                    if not hasattr(resp, "usage")
+                    else resp.usage["total_tokens"]
+                )
                 if resp.choices[0].finish_reason == "length":
                     ans += "...\nFor the content length reason, it stopped, continue?" if is_english(
                         [ans]) else "······\n由于长度的原因，回答被截断了，要继续吗？"
@@ -85,8 +93,13 @@ class MoonshotChat(Base):
         if not base_url: base_url="https://api.moonshot.cn/v1"
         super().__init__(key, model_name, base_url)
 
+
 class XinferenceChat(Base):
     def __init__(self, key=None, model_name="", base_url=""):
+        if not base_url:
+            raise ValueError("Local llm url cannot be None")
+        if base_url.split("/")[-1] != "v1":
+            self.base_url = os.path.join(base_url, "v1")
         key = "xxx"
         super().__init__(key, model_name, base_url)
 
@@ -349,79 +362,13 @@ class OllamaChat(Base):
 
 class LocalAIChat(Base):
     def __init__(self, key, model_name, base_url):
-        if base_url[-1] == "/":
+        if not base_url:
-            base_url = base_url[:-1]
+            raise ValueError("Local llm url cannot be None")
-        self.base_url = base_url + "/v1/chat/completions"
+        if base_url.split("/")[-1] != "v1":
+            self.base_url = os.path.join(base_url, "v1")
+        self.client = OpenAI(api_key="empty", base_url=self.base_url)
         self.model_name = model_name.split("___")[0]
 
-    def chat(self, system, history, gen_conf):
-        if system:
-            history.insert(0, {"role": "system", "content": system})
-        for k in list(gen_conf.keys()):
-            if k not in ["temperature", "top_p", "max_tokens"]:
-                del gen_conf[k]
-        headers = {
-            "Content-Type": "application/json",
-        }
-        payload = json.dumps(
-            {"model": self.model_name, "messages": history, **gen_conf}
-        )
-        try:
-            response = requests.request(
-                "POST", url=self.base_url, headers=headers, data=payload
-            )
-            response = response.json()
-            ans = response["choices"][0]["message"]["content"].strip()
-            if response["choices"][0]["finish_reason"] == "length":
-                ans += (
-                    "...\nFor the content length reason, it stopped, continue?"
-                    if is_english([ans])
-                    else "······\n由于长度的原因，回答被截断了，要继续吗？"
-                )
-            return ans, response["usage"]["total_tokens"]
-        except Exception as e:
-            return "**ERROR**: " + str(e), 0
-
-    def chat_streamly(self, system, history, gen_conf):
-        if system:
-            history.insert(0, {"role": "system", "content": system})
-        ans = ""
-        total_tokens = 0
-        try:
-            headers = {
-                "Content-Type": "application/json",
-            }
-            payload = json.dumps(
-                {
-                    "model": self.model_name,
-                    "messages": history,
-                    "stream": True,
-                    **gen_conf,
-                }
-            )
-            response = requests.request(
-                "POST",
-                url=self.base_url,
-                headers=headers,
-                data=payload,
-            )
-            for resp in response.content.decode("utf-8").split("\n\n"):
-                if "choices" not in resp:
-                    continue
-                resp = json.loads(resp[6:])
-                if "delta" in resp["choices"][0]:
-                    text = resp["choices"][0]["delta"]["content"]
-                else:
-                    continue
-                ans += text
-                total_tokens += 1
-            yield ans
-
-        except Exception as e:
-            yield ans + "\n**ERROR**: " + str(e)
-
-        yield total_tokens
-
 
 class LocalLLM(Base):
     class RPCProxy:
@@ -892,9 +839,10 @@ class GroqChat:
 ## openrouter
 class OpenRouterChat(Base):
     def __init__(self, key, model_name, base_url="https://openrouter.ai/api/v1"):
-        self.base_url = "https://openrouter.ai/api/v1"
+        if not base_url:
-        self.client = OpenAI(base_url=self.base_url, api_key=key)
+            base_url = "https://openrouter.ai/api/v1"
-        self.model_name = model_name
+        super().__init__(key, model_name, base_url)
+
 
 class StepFunChat(Base):
     def __init__(self, key, model_name, base_url="https://api.stepfun.com/v1"):
@@ -904,87 +852,17 @@ class StepFunChat(Base):
 
 
 class NvidiaChat(Base):
-    def __init__(
+    def __init__(self, key, model_name, base_url="https://integrate.api.nvidia.com/v1"):
-        self,
-        key,
-        model_name,
-        base_url="https://integrate.api.nvidia.com/v1/chat/completions",
-    ):
         if not base_url:
-            base_url = "https://integrate.api.nvidia.com/v1/chat/completions"
+            base_url = "https://integrate.api.nvidia.com/v1"
-        self.base_url = base_url
+        super().__init__(key, model_name, base_url)
-        self.model_name = model_name
-        self.api_key = key
-        self.headers = {
-            "accept": "application/json",
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json",
-        }
-
-    def chat(self, system, history, gen_conf):
-        if system:
-            history.insert(0, {"role": "system", "content": system})
-        for k in list(gen_conf.keys()):
-            if k not in ["temperature", "top_p", "max_tokens"]:
-                del gen_conf[k]
-        payload = {"model": self.model_name, "messages": history, **gen_conf}
-        try:
-            response = requests.post(
-                url=self.base_url, headers=self.headers, json=payload
-            )
-            response = response.json()
-            ans = response["choices"][0]["message"]["content"].strip()
-            return ans, response["usage"]["total_tokens"]
-        except Exception as e:
-            return "**ERROR**: " + str(e), 0
-
-    def chat_streamly(self, system, history, gen_conf):
-        if system:
-            history.insert(0, {"role": "system", "content": system})
-        for k in list(gen_conf.keys()):
-            if k not in ["temperature", "top_p", "max_tokens"]:
-                del gen_conf[k]
-        ans = ""
-        total_tokens = 0
-        payload = {
-            "model": self.model_name,
-            "messages": history,
-            "stream": True,
-            **gen_conf,
-        }
-
-        try:
-            response = requests.post(
-                url=self.base_url,
-                headers=self.headers,
-                json=payload,
-            )
-            for resp in response.text.split("\n\n"):
-                if "choices" not in resp:
-                    continue
-                resp = json.loads(resp[6:])
-                if "content" in resp["choices"][0]["delta"]:
-                    text = resp["choices"][0]["delta"]["content"]
-                else:
-                    continue
-                ans += text
-                if "usage" in resp:
-                    total_tokens = resp["usage"]["total_tokens"]
-                yield ans
-
-        except Exception as e:
-            yield ans + "\n**ERROR**: " + str(e)
-
-        yield total_tokens
 
 
 class LmStudioChat(Base):
     def __init__(self, key, model_name, base_url):
-        from os.path import join
-
         if not base_url:
             raise ValueError("Local llm url cannot be None")
         if base_url.split("/")[-1] != "v1":
-            self.base_url = join(base_url, "v1")
+            self.base_url = os.path.join(base_url, "v1")
         self.client = OpenAI(api_key="lm-studio", base_url=self.base_url)
         self.model_name = model_name

rag/llm/cv_model.py

@@ -433,27 +433,16 @@ class OllamaCV(Base):
         yield 0
 
 
-class LocalAICV(Base):
+class LocalAICV(GptV4):
     def __init__(self, key, model_name, base_url, lang="Chinese"):
+        if not base_url:
+            raise ValueError("Local cv model url cannot be None")
+        if base_url.split("/")[-1] != "v1":
+            base_url = os.path.join(base_url, "v1")
         self.client = OpenAI(api_key="empty", base_url=base_url)
         self.model_name = model_name.split("___")[0]
         self.lang = lang
 
-    def describe(self, image, max_tokens=300):
-        b64 = self.image2base64(image)
-        prompt = self.prompt(b64)
-        for i in range(len(prompt)):
-            for c in prompt[i]["content"]:
-                if "text" in c:
-                    c["type"] = "text"
-
-        res = self.client.chat.completions.create(
-            model=self.model_name,
-            messages=prompt,
-            max_tokens=max_tokens,
-        )
-        return res.choices[0].message.content.strip(), res.usage.total_tokens
-
 
 class XinferenceCV(Base):
     def __init__(self, key, model_name="", lang="Chinese", base_url=""):
@@ -549,60 +538,19 @@ class GeminiCV(Base):
         yield response._chunks[-1].usage_metadata.total_token_count
 
 
-class OpenRouterCV(Base):
+class OpenRouterCV(GptV4):
     def __init__(
         self,
         key,
         model_name,
         lang="Chinese",
-        base_url="https://openrouter.ai/api/v1/chat/completions",
+        base_url="https://openrouter.ai/api/v1",
     ):
+        if not base_url:
+            base_url = "https://openrouter.ai/api/v1"
+        self.client = OpenAI(api_key=key, base_url=base_url)
         self.model_name = model_name
         self.lang = lang
-        self.base_url = "https://openrouter.ai/api/v1/chat/completions"
-        self.key = key
-
-    def describe(self, image, max_tokens=300):
-        b64 = self.image2base64(image)
-        response = requests.post(
-            url=self.base_url,
-            headers={
-                "Authorization": f"Bearer {self.key}",
-            },
-            data=json.dumps(
-                {
-                    "model": self.model_name,
-                    "messages": self.prompt(b64),
-                    "max_tokens": max_tokens,
-                }
-            ),
-        )
-        response = response.json()
-        return (
-            response["choices"][0]["message"]["content"].strip(),
-            response["usage"]["total_tokens"],
-        )
-
-    def prompt(self, b64):
-        return [
-            {
-                "role": "user",
-                "content": [
-                    {
-                        "type": "image_url",
-                        "image_url": {"url": f"data:image/jpeg;base64,{b64}"},
-                    },
-                    {
-                        "type": "text",
-                        "text": (
-                            "请用中文详细描述一下图中的内容，比如时间，地点，人物，事情，人物心情等，如果有数据请提取出数据。"
-                            if self.lang.lower() == "chinese"
-                            else "Please describe the content of this picture, like where, when, who, what happen. If it has number data, please extract them out."
-                        ),
-                    },
-                ],
-            }
-        ]
 
 
 class LocalCV(Base):
@@ -675,12 +623,12 @@ class NvidiaCV(Base):
         ]
 
 
-class LmStudioCV(LocalAICV):
+class LmStudioCV(GptV4):
     def __init__(self, key, model_name, base_url, lang="Chinese"):
         if not base_url:
             raise ValueError("Local llm url cannot be None")
-        if base_url.split('/')[-1] != 'v1':
+        if base_url.split("/")[-1] != "v1":
-            self.base_url = os.path.join(base_url,'v1')
+            base_url = os.path.join(base_url, "v1")
-        self.client = OpenAI(api_key="lm-studio", base_url=self.base_url)
+        self.client = OpenAI(api_key="lm-studio", base_url=base_url)
         self.model_name = model_name
         self.lang = lang

rag/llm/embedding_model.py

@@ -113,21 +113,24 @@ class OpenAIEmbed(Base):
 
 class LocalAIEmbed(Base):
     def __init__(self, key, model_name, base_url):
-        self.base_url = base_url + "/embeddings"
+        if not base_url:
-        self.headers = {
+            raise ValueError("Local embedding model url cannot be None")
-            "Content-Type": "application/json",
+        if base_url.split("/")[-1] != "v1":
-        }
+            base_url = os.path.join(base_url, "v1")
+        self.client = OpenAI(api_key="empty", base_url=base_url)
         self.model_name = model_name.split("___")[0]
 
-    def encode(self, texts: list, batch_size=None):
+    def encode(self, texts: list, batch_size=32):
-        data = {"model": self.model_name, "input": texts, "encoding_type": "float"}
+        res = self.client.embeddings.create(input=texts, model=self.model_name)
-        res = requests.post(self.base_url, headers=self.headers, json=data).json()
+        return (
-
+            np.array([d.embedding for d in res.data]),
-        return np.array([d["embedding"] for d in res["data"]]), 1024
+            1024,
+        )  # local embedding for LmStudio donot count tokens
 
     def encode_queries(self, text):
-        embds, cnt = self.encode([text])
+        res = self.client.embeddings.create(text, model=self.model_name)
-        return np.array(embds[0]), cnt
+        return np.array(res.data[0].embedding), 1024
+
 
 class AzureEmbed(OpenAIEmbed):
     def __init__(self, key, model_name, **kwargs):
@@ -502,7 +505,7 @@ class NvidiaEmbed(Base):
         return np.array(embds[0]), cnt
 
 
-class LmStudioEmbed(Base):
+class LmStudioEmbed(LocalAIEmbed):
     def __init__(self, key, model_name, base_url):
         if not base_url:
             raise ValueError("Local llm url cannot be None")
@@ -510,14 +513,3 @@ class LmStudioEmbed(Base):
             self.base_url = os.path.join(base_url, "v1")
         self.client = OpenAI(api_key="lm-studio", base_url=self.base_url)
         self.model_name = model_name
-
-    def encode(self, texts: list, batch_size=32):
-        res = self.client.embeddings.create(input=texts, model=self.model_name)
-        return (
-            np.array([d.embedding for d in res.data]),
-            1024,
-        )  # local embedding for LmStudio donot count tokens
-
-    def encode_queries(self, text):
-        res = self.client.embeddings.create(text, model=self.model_name)
-        return np.array(res.data[0].embedding), 1024