add llm API (#19)

* add llm API

* refine llm API

python/conf/mapping.json

@@ -121,7 +121,6 @@
               "match": "*_vec",
               "mapping": {
                 "type": "dense_vector",
-                "dims": 1024,
                 "index": true,
                 "similarity": "cosine"
               }

python/conf/sys.cnf

@@ -1,10 +1,9 @@
 [infiniflow]
 es=http://es01:9200
-pgdb_usr=root
+postgres_user=root
-pgdb_pwd=infiniflow_docgpt
+postgres_password=infiniflow_docgpt
-pgdb_host=postgres
+postgres_host=postgres
-pgdb_port=5432
+postgres_port=5432
 minio_host=minio:9000
-minio_usr=infiniflow
+minio_user=infiniflow
-minio_pwd=infiniflow_docgpt
+minio_password=infiniflow_docgpt
-

python/llm/__init__.py

@@ -1,2 +1,21 @@
-from .embedding_model import HuEmbedding
+import os
-from .chat_model import GptTurbo
+from .embedding_model import *
+from .chat_model import *
+from .cv_model import *
+
+EmbeddingModel = None
+ChatModel = None
+CvModel = None
+
+
+if os.environ.get("OPENAI_API_KEY"):
+    EmbeddingModel = GptEmbed()
+    ChatModel = GptTurbo()
+    CvModel = GptV4()
+
+elif os.environ.get("DASHSCOPE_API_KEY"):
+    EmbeddingModel = QWenEmbd()
+    ChatModel = QWenChat()
+    CvModel = QWenCV()
+else:
+    EmbeddingModel = HuEmbedding()

python/llm/chat_model.py

@@ -1,7 +1,8 @@
 from abc import ABC
-import openapi
+from openai import OpenAI
 import os
 
+
 class Base(ABC):
     def chat(self, system, history, gen_conf):
         raise NotImplementedError("Please implement encode method!")
@@ -9,25 +10,26 @@ class Base(ABC):
 
 class GptTurbo(Base):
     def __init__(self):
-        openapi.api_key = os.environ["OPENAPI_KEY"]
+        self.client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
 
     def chat(self, system, history, gen_conf):
         history.insert(0, {"role": "system", "content": system})
-        res = openapi.ChatCompletion.create(model="gpt-3.5-turbo",
+        res = self.client.chat.completions.create(
+            model="gpt-3.5-turbo",
             messages=history,
             **gen_conf)
         return res.choices[0].message.content.strip()
 
 
-class QWen(Base):
+class QWenChat(Base):
     def chat(self, system, history, gen_conf):
         from http import HTTPStatus
         from dashscope import Generation
-        from dashscope.api_entities.dashscope_response import Role
         # export DASHSCOPE_API_KEY=YOUR_DASHSCOPE_API_KEY
+        history.insert(0, {"role": "system", "content": system})
         response = Generation.call(
             Generation.Models.qwen_turbo,
-                    messages=messages,
+            messages=history,
             result_format='message'
         )
         if response.status_code == HTTPStatus.OK:

python/llm/cv_model.py

@@ -0,0 +1,66 @@
+from abc import ABC
+from openai import OpenAI
+import os
+import base64
+from io import BytesIO
+
+
+class Base(ABC):
+    def describe(self, image, max_tokens=300):
+        raise NotImplementedError("Please implement encode method!")
+
+    def image2base64(self, image):
+        if isinstance(image, BytesIO):
+            return base64.b64encode(image.getvalue()).decode("utf-8")
+        buffered = BytesIO()
+        try:
+            image.save(buffered, format="JPEG")
+        except Exception as e:
+            image.save(buffered, format="PNG")
+        return base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+    def prompt(self, b64):
+        return [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "请用中文详细描述一下图中的内容，比如时间，地点，人物，事情，人物心情等。",
+                    },
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{b64}"
+                        },
+                    },
+                ],
+            }
+        ]
+
+
+class GptV4(Base):
+    def __init__(self):
+        self.client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+
+    def describe(self, image, max_tokens=300):
+        b64 = self.image2base64(image)
+
+        res = self.client.chat.completions.create(
+            model="gpt-4-vision-preview",
+            messages=self.prompt(b64),
+            max_tokens=max_tokens,
+        )
+        return res.choices[0].message.content.strip()
+
+
+class QWenCV(Base):
+    def describe(self, image, max_tokens=300):
+        from http import HTTPStatus
+        from dashscope import MultiModalConversation
+        # export DASHSCOPE_API_KEY=YOUR_DASHSCOPE_API_KEY
+        response = MultiModalConversation.call(model=MultiModalConversation.Models.qwen_vl_chat_v1,
+                                               messages=self.prompt(self.image2base64(image)))
+        if response.status_code == HTTPStatus.OK:
+            return response.output.choices[0]['message']['content']
+        return response.message

python/llm/embedding_model.py

@@ -1,8 +1,11 @@
 from abc import ABC
+from openai import OpenAI
 from FlagEmbedding import FlagModel
 import torch
+import os
 import numpy as np
 
+
 class Base(ABC):
     def encode(self, texts: list, batch_size=32):
         raise NotImplementedError("Please implement encode method!")
@@ -30,3 +33,29 @@ class HuEmbedding(Base):
         for i in range(0, len(texts), batch_size):
             res.extend(self.model.encode(texts[i:i + batch_size]).tolist())
         return np.array(res)
+
+
+class GptEmbed(Base):
+    def __init__(self):
+        self.client = OpenAI(api_key=os.envirement["OPENAI_API_KEY"])
+
+    def encode(self, texts: list, batch_size=32):
+        res = self.client.embeddings.create(input=texts,
+                                            model="text-embedding-ada-002")
+        return [d["embedding"] for d in res["data"]]
+
+
+class QWenEmbd(Base):
+    def encode(self, texts: list, batch_size=32, text_type="document"):
+        # export DASHSCOPE_API_KEY=YOUR_DASHSCOPE_API_KEY
+        import dashscope
+        from http import HTTPStatus
+        res = []
+        for txt in texts:
+            resp = dashscope.TextEmbedding.call(
+                model=dashscope.TextEmbedding.Models.text_embedding_v2,
+                input=txt[:2048],
+                text_type=text_type
+            )
+            res.append(resp["output"]["embeddings"][0]["embedding"])
+        return res

python/nlp/huchunk.py

@@ -372,7 +372,9 @@ class PptChunker(HuChunker):
 
     def __call__(self, fnm):
         from pptx import Presentation
-        ppt = Presentation(fnm) if isinstance(fnm, str) else Presentation(BytesIO(fnm))
+        ppt = Presentation(fnm) if isinstance(
+            fnm, str) else Presentation(
+            BytesIO(fnm))
         flds = self.Fields()
         flds.text_chunks = []
         for slide in ppt.slides:
@@ -398,7 +400,8 @@ class TextChunker(HuChunker):
         mime = magic.Magic(mime=True)
         if isinstance(file_path, str):
             file_type = mime.from_file(file_path)
-        else:file_type = mime.from_buffer(file_path) 
+        else:
+            file_type = mime.from_buffer(file_path)
         if 'text' in file_type:
             return False
         else:
@@ -406,7 +409,8 @@ class TextChunker(HuChunker):
 
     def __call__(self, fnm):
         flds = self.Fields()
-        if self.is_binary_file(fnm):return flds
+        if self.is_binary_file(fnm):
+            return flds
         with open(fnm, "r") as f:
             txt = f.read()
             flds.text_chunks = [(c, None) for c in self.naive_text_chunk(txt)]

python/nlp/search.py

@@ -9,12 +9,18 @@ from sklearn.metrics.pairwise import cosine_similarity as CosineSimilarity
 import numpy as np
 from copy import deepcopy
 
+
 def index_name(uid): return f"docgpt_{uid}"
 
+
 class Dealer:
     def __init__(self, es, emb_mdl):
         self.qryr = query.EsQueryer(es)
-        self.qryr.flds = ["title_tks^10", "title_sm_tks^5", "content_ltks^2", "content_sm_ltks"]
+        self.qryr.flds = [
+            "title_tks^10",
+            "title_sm_tks^5",
+            "content_ltks^2",
+            "content_sm_ltks"]
         self.es = es
         self.emb_mdl = emb_mdl
 
@@ -43,7 +49,8 @@ class Dealer:
         qst = req.get("question", "")
 
         bqry, keywords = self.qryr.question(qst)
-        if req.get("kb_ids"): bqry.filter.append(Q("terms", kb_id=req["kb_ids"]))
+        if req.get("kb_ids"):
+            bqry.filter.append(Q("terms", kb_id=req["kb_ids"]))
         bqry.filter.append(Q("exists", field="q_tks"))
         bqry.boost = 0.05
         print(bqry)
@@ -57,7 +64,9 @@ class Dealer:
         s = s.query(bqry)[pg * ps:(pg + 1) * ps]
         s = s.highlight("content_ltks")
         s = s.highlight("title_ltks")
-        if not qst: s = s.sort({"create_time":{"order":"desc", "unmapped_type":"date"}})
+        if not qst:
+            s = s.sort(
+                {"create_time": {"order": "desc", "unmapped_type": "date"}})
 
         s = s.highlight_options(
             fragment_size=120,
@@ -77,7 +86,8 @@ class Dealer:
         print("TOTAL: ", self.es.getTotal(res))
         if self.es.getTotal(res) == 0 and "knn" in s:
             bqry, _ = self.qryr.question(qst, min_match="10%")
-            if req.get("kb_ids"): bqry.filter.append(Q("terms", kb_id=req["kb_ids"]))
+            if req.get("kb_ids"):
+                bqry.filter.append(Q("terms", kb_id=req["kb_ids"]))
             s["query"] = bqry.to_dict()
             s["knn"]["filter"] = bqry.to_dict()
             s["knn"]["similarity"] = 0.7
@@ -87,8 +97,10 @@ class Dealer:
         for k in keywords:
             kwds.add(k)
             for kk in huqie.qieqie(k).split(" "):
-                if len(kk) < 2:continue
+                if len(kk) < 2:
-                if kk in kwds:continue
+                    continue
+                if kk in kwds:
+                    continue
                 kwds.add(kk)
 
         aggs = self.getAggregation(res, "docnm_kwd")
@@ -105,7 +117,8 @@ class Dealer:
         )
 
     def getAggregation(self, res, g):
-        if not "aggregations" in res or "aggs_"+g not in res["aggregations"]:return
+        if not "aggregations" in res or "aggs_" + g not in res["aggregations"]:
+            return
         bkts = res["aggregations"]["aggs_" + g]["buckets"]
         return [(b["key"], b["doc_count"]) for b in bkts]
 
@@ -114,8 +127,11 @@ class Dealer:
             eng = set(list("qwertyuioplkjhgfdsazxcvbnm"))
             r = []
             for t in line.split(" "):
-                if not t:continue
+                if not t:
-                if len(r)>0 and len(t)>0 and r[-1][-1] in eng and t[0] in eng:r.append(" ")
+                    continue
+                if len(r) > 0 and len(
+                        t) > 0 and r[-1][-1] in eng and t[0] in eng:
+                    r.append(" ")
                 r.append(t)
             r = "".join(r)
             return r
@@ -123,41 +139,47 @@ class Dealer:
         ans = {}
         for d in res["hits"]["hits"]:
             hlts = d.get("highlight")
-            if not hlts:continue
+            if not hlts:
+                continue
             ans[d["_id"]] = "".join([a for a in list(hlts.items())[0][1]])
         return ans
 
     def getFields(self, sres, flds):
         res = {}
-        if not flds:return {}
+        if not flds:
+            return {}
         for d in self.es.getSource(sres):
             m = {n: d.get(n) for n in flds if d.get(n) is not None}
             for n, v in m.items():
-                if type(v) == type([]):
+                if isinstance(v, type([])):
                     m[n] = "\t".join([str(vv) for vv in v])
                     continue
-                if type(v) != type(""):m[n] = str(m[n])
+                if not isinstance(v, type("")):
+                    m[n] = str(m[n])
                 m[n] = rmSpace(m[n])
 
-            if m:res[d["id"]] = m
+            if m:
+                res[d["id"]] = m
         return res
 
-
     @staticmethod
     def trans2floats(txt):
         return [float(t) for t in txt.split("\t")]
 
+    def insert_citations(self, ans, top_idx, sres,
+                         vfield="q_vec", cfield="content_ltks"):
 
-    def insert_citations(self, ans, top_idx, sres, vfield = "q_vec", cfield="content_ltks"):
+        ins_embd = [Dealer.trans2floats(
-
+            sres.field[sres.ids[i]][vfield]) for i in top_idx]
-        ins_embd = [Dealer.trans2floats(sres.field[sres.ids[i]][vfield]) for i in top_idx]
         ins_tw = [sres.field[sres.ids[i]][cfield].split(" ") for i in top_idx]
         s = 0
         e = 0
         res = ""
+
         def citeit():
             nonlocal s, e, ans, res
-            if not ins_embd:return
+            if not ins_embd:
+                return
             embd = self.emb_mdl.encode(ans[s: e])
             sim = self.qryr.hybrid_similarity(embd,
                                               ins_embd,
@@ -165,9 +187,12 @@ class Dealer:
                                               ins_tw)
             print(ans[s: e], sim)
             mx = np.max(sim) * 0.99
-            if mx < 0.55:return
+            if mx < 0.55:
-            cita = list(set([top_idx[i] for i in range(len(ins_embd)) if sim[i] >mx]))[:4]
+                return
-            for i in cita: res += f"@?{i}?@"
+            cita = list(set([top_idx[i]
+                        for i in range(len(ins_embd)) if sim[i] > mx]))[:4]
+            for i in cita:
+                res += f"@?{i}?@"
 
             return cita
 
@@ -179,7 +204,8 @@ class Dealer:
             if e - s < 12 or ans[e] not in punct:
                 e += 1
                 continue
-            if ans[e] == "." and e+1<len(ans) and re.match(r"[0-9]", ans[e+1]):
+            if ans[e] == "." and e + \
+                    1 < len(ans) and re.match(r"[0-9]", ans[e + 1]):
                 e += 1
                 continue
             if ans[e] == "." and e - 2 >= 0 and ans[e - 2] == "\n":
@@ -197,10 +223,13 @@ class Dealer:
 
         return res
 
-
+    def rerank(self, sres, query, tkweight=0.3, vtweight=0.7,
-    def rerank(self, sres, query, tkweight=0.3, vtweight=0.7, vfield="q_vec", cfield="content_ltks"):
+               vfield="q_vec", cfield="content_ltks"):
-        ins_embd = [Dealer.trans2floats(sres.field[i]["q_vec"]) for i in sres.ids]
+        ins_embd = [
-        if not ins_embd: return []
+            Dealer.trans2floats(
+                sres.field[i]["q_vec"]) for i in sres.ids]
+        if not ins_embd:
+            return []
         ins_tw = [sres.field[i][cfield].split(" ") for i in sres.ids]
         # return CosineSimilarity([sres.query_vector], ins_embd)[0]
         sim = self.qryr.hybrid_similarity(sres.query_vector,
@@ -210,7 +239,6 @@ class Dealer:
         return sim
 
 
-
 if __name__ == "__main__":
     from util import es_conn
     SE = Dealer(es_conn.HuEs("infiniflow"))
@@ -220,4 +248,5 @@ if __name__ == "__main__":
     ]
     for q in qs:
         print(">>>>>>>>>>>>>>>>>>>>", q)
-        print(SE.search({"question": q, "kb_ids": "64f072a75f3b97c865718c4a"}, "infiniflow_*"))
+        print(SE.search(
+            {"question": q, "kb_ids": "64f072a75f3b97c865718c4a"}, "infiniflow_*"))

python/parser/excel_parser.py

@@ -5,8 +5,10 @@ from io import BytesIO
 
 class HuExcelParser:
     def __call__(self, fnm):
-        if isinstance(fnm, str):wb = load_workbook(fnm)
+        if isinstance(fnm, str):
-        else: wb = load_workbook(BytesIO(fnm))
+            wb = load_workbook(fnm)
+        else:
+            wb = load_workbook(BytesIO(fnm))
         res = []
         for sheetname in wb.sheetnames:
             ws = wb[sheetname]

python/parser/pdf_parser.py

@@ -1526,7 +1526,8 @@ class HuParser:
         return "\n\n".join(res)
 
     def __call__(self, fnm, need_image=True, zoomin=3, return_html=False):
-        self.pdf = pdfplumber.open(fnm) if isinstance(fnm, str) else pdfplumber.open(BytesIO(fnm))
+        self.pdf = pdfplumber.open(fnm) if isinstance(
+            fnm, str) else pdfplumber.open(BytesIO(fnm))
         self.lefted_chars = []
         self.mean_height = []
         self.mean_width = []

python/svr/dialog_svr.py

@@ -16,11 +16,12 @@ from io import BytesIO
 from util import config
 from timeit import default_timer as timer
 from collections import OrderedDict
+from llm import ChatModel, EmbeddingModel
 
 SE = None
 CFIELD="content_ltks"
-EMBEDDING = HuEmbedding()
+EMBEDDING = EmbeddingModel
-LLM = GptTurbo()
+LLM = ChatModel
 
 def get_QA_pairs(hists):
     pa = []

python/svr/parse_user_docs.py

@@ -1,4 +1,4 @@
-import json, os, sys, hashlib, copy, time, random, re, logging, torch
+import json, os, sys, hashlib, copy, time, random, re
 from os.path import dirname, realpath
 sys.path.append(dirname(realpath(__file__)) + "/../")
 from util.es_conn import HuEs
@@ -7,10 +7,10 @@ from util.minio_conn import HuMinio
 from util import rmSpace, findMaxDt
 from FlagEmbedding import FlagModel
 from nlp import huchunk, huqie, search
-import base64, hashlib
 from io import BytesIO
 import pandas as pd
 from elasticsearch_dsl import Q
+from PIL import Image
 from parser import (
     PdfParser,
     DocxParser,
@@ -40,6 +40,15 @@ def chuck_doc(name, binary):
     if suff.find("doc") >= 0: return DOC(binary)
     if re.match(r"(xlsx|xlsm|xltx|xltm)", suff): return EXC(binary)
     if suff.find("ppt") >= 0: return PPT(binary)
+    if os.envirement.get("PARSE_IMAGE") \
+       and re.search(r"\.(jpg|jpeg|png|tif|gif|pcx|tga|exif|fpx|svg|psd|cdr|pcd|dxf|ufo|eps|ai|raw|WMF|webp|avif|apng|icon|ico)$",
+              name.lower()):
+        from llm import CvModel
+        txt = CvModel.describe(binary)
+        field = TextChunker.Fields()
+        field.text_chunks = [(txt, binary)]
+        field.table_chunks = []
+
     
     return TextChunker()(binary)
 
@@ -119,7 +128,6 @@ def build(row):
             set_progress(row["kb2doc_id"], -1, f"Internal system error: %s"%str(e).replace("'", ""))
         return []
 
-    print(row["doc_name"], obj)
     if not obj.text_chunks and not obj.table_chunks: 
         set_progress(row["kb2doc_id"], 1, "Nothing added! Mostly, file type unsupported yet.")
         return  []
@@ -146,7 +154,10 @@ def build(row):
         if not img:
             docs.append(d)
             continue
-        img.save(output_buffer, format='JPEG')
+
+        if isinstance(img, Image): img.save(output_buffer, format='JPEG')
+        else: output_buffer = BytesIO(img)
+
         MINIO.put("{}-{}".format(row["uid"], row["kb_id"]), d["_id"],
                       output_buffer.getvalue())
         d["img_id"] = "{}-{}".format(row["uid"], row["kb_id"])

python/util/__init__.py

@@ -1,19 +1,24 @@
 import re
 
+
 def rmSpace(txt):
     txt = re.sub(r"([^a-z0-9.,]) +([^ ])", r"\1\2", txt)
     return re.sub(r"([^ ]) +([^a-z0-9.,])", r"\1\2", txt)
 
+
 def findMaxDt(fnm):
     m = "1970-01-01 00:00:00"
     try:
         with open(fnm, "r") as f:
             while True:
                 l = f.readline()
-                if not l:break
+                if not l:
+                    break
                 l = l.strip("\n")
-                if l == 'nan':continue
+                if l == 'nan':
-                if l > m:m = l
+                    continue
+                if l > m:
+                    m = l
     except Exception as e:
         print("WARNING: can't find " + fnm)
     return m

python/util/config.py

@@ -1,25 +1,31 @@
 from configparser import ConfigParser
-import os,inspect
+import os
+import inspect
 
 CF = ConfigParser()
 __fnm = os.path.join(os.path.dirname(__file__), '../conf/sys.cnf')
-if not os.path.exists(__fnm):__fnm = os.path.join(os.path.dirname(__file__), '../../conf/sys.cnf')
+if not os.path.exists(__fnm):
-assert os.path.exists(__fnm), f"【EXCEPTION】can't find {__fnm}." + os.path.dirname(__file__)
+    __fnm = os.path.join(os.path.dirname(__file__), '../../conf/sys.cnf')
-if not os.path.exists(__fnm): __fnm = "./sys.cnf"
+assert os.path.exists(
+    __fnm), f"【EXCEPTION】can't find {__fnm}." + os.path.dirname(__file__)
+if not os.path.exists(__fnm):
+    __fnm = "./sys.cnf"
 
 CF.read(__fnm)
 
+
 class Config:
     def __init__(self, env):
         self.env = env
-        if env == "spark":CF.read("./cv.cnf")
+        if env == "spark":
+            CF.read("./cv.cnf")
 
     def get(self, key, default=None):
         global CF
-        return os.environ.get(key.upper(), \
+        return os.environ.get(key.upper(),
                               CF[self.env].get(key, default)
                               )
 
+
 def init(env):
     return Config(env)
-

python/util/db_conn.py

@@ -3,6 +3,7 @@ import time
 from util import config
 import pandas as pd
 
+
 class Postgres(object):
     def __init__(self, env, dbnm):
         self.config = config.init(env)
@@ -13,23 +14,30 @@ class Postgres(object):
     def __open__(self):
         import psycopg2
         try:
-            if self.conn:self.__close__()
+            if self.conn:
+                self.__close__()
             del self.conn
         except Exception as e:
             pass
 
         try:
-            self.conn = psycopg2.connect(f"dbname={self.dbnm} user={self.config.get('pgdb_usr')} password={self.config.get('pgdb_pwd')} host={self.config.get('pgdb_host')} port={self.config.get('pgdb_port')}")
+            self.conn = psycopg2.connect(f"""dbname={self.dbnm}
+                                         user={self.config.get('postgres_user')}
+                                         password={self.config.get('postgres_password')}
+                                         host={self.config.get('postgres_host')}
+                                         port={self.config.get('postgres_port')}""")
         except Exception as e:
-            logging.error("Fail to connect %s "%self.config.get("pgdb_host") + str(e))
+            logging.error(
-
+                "Fail to connect %s " %
+                self.config.get("pgdb_host") + str(e))
 
     def __close__(self):
         try:
             self.conn.close()
         except Exception as e:
-             logging.error("Fail to close %s "%self.config.get("pgdb_host") + str(e))
+            logging.error(
-
+                "Fail to close %s " %
+                self.config.get("pgdb_host") + str(e))
 
     def select(self, sql):
         for _ in range(10):
@@ -42,7 +50,6 @@ class Postgres(object):
 
         return pd.DataFrame()
 
-
     def update(self, sql):
         for _ in range(10):
             try:
@@ -58,6 +65,6 @@ class Postgres(object):
                 time.sleep(1)
         return 0
 
+
 if __name__ == "__main__":
     Postgres("infiniflow", "docgpt")
-

python/util/es_conn.py

@@ -228,7 +228,8 @@ class HuEs:
         return False
 
     def search(self, q, idxnm=None, src=False, timeout="2s"):
-        if not isinstance(q, dict): q = Search().query(q).to_dict()
+        if not isinstance(q, dict):
+            q = Search().query(q).to_dict()
         for i in range(3):
             try:
                 res = self.es.search(index=(self.idxnm if not idxnm else idxnm),
@@ -274,9 +275,10 @@ class HuEs:
 
         return False
 
-
     def updateScriptByQuery(self, q, scripts, idxnm=None):
-        ubq = UpdateByQuery(index=self.idxnm if not idxnm else idxnm).using(self.es).query(q)
+        ubq = UpdateByQuery(
+            index=self.idxnm if not idxnm else idxnm).using(
+            self.es).query(q)
         ubq = ubq.script(source=scripts)
         ubq = ubq.params(refresh=True)
         ubq = ubq.params(slices=5)
@@ -294,7 +296,6 @@ class HuEs:
 
         return False
 
-
     def deleteByQuery(self, query, idxnm=""):
         for i in range(3):
             try:

python/util/minio_conn.py

@@ -4,6 +4,7 @@ from util import config
 from minio import Minio
 from io import BytesIO
 
+
 class HuMinio(object):
     def __init__(self, env):
         self.config = config.init(env)
@@ -12,25 +13,26 @@ class HuMinio(object):
 
     def __open__(self):
         try:
-            if self.conn:self.__close__()
+            if self.conn:
+                self.__close__()
         except Exception as e:
             pass
 
         try:
             self.conn = Minio(self.config.get("minio_host"),
-                              access_key=self.config.get("minio_usr"),
+                              access_key=self.config.get("minio_user"),
-                              secret_key=self.config.get("minio_pwd"),
+                              secret_key=self.config.get("minio_password"),
                               secure=False
                               )
         except Exception as e:
-            logging.error("Fail to connect %s "%self.config.get("minio_host") + str(e))
+            logging.error(
-
+                "Fail to connect %s " %
+                self.config.get("minio_host") + str(e))
 
     def __close__(self):
         del self.conn
         self.conn = None
 
-
     def put(self, bucket, fnm, binary):
         for _ in range(10):
             try:
@@ -47,7 +49,6 @@ class HuMinio(object):
                 self.__open__()
                 time.sleep(1)
 
-
     def get(self, bucket, fnm):
         for _ in range(10):
             try:
@@ -59,7 +60,6 @@ class HuMinio(object):
                 time.sleep(1)
         return
 
-
     def get_presigned_url(self, bucket, fnm, expires):
         for _ in range(10):
             try:
@@ -71,7 +71,6 @@ class HuMinio(object):
         return
 
 
-
 if __name__ == "__main__":
     conn = HuMinio("infiniflow")
     fnm = "/opt/home/kevinhu/docgpt/upload/13/11-408.jpg"
@@ -83,4 +82,3 @@ if __name__ == "__main__":
     bts = conn.get("test", "11-408.jpg")
     img = Image.open(BytesIO(bts))
     img.save("test.jpg")
-