mirror of
https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git
synced 2025-08-13 22:55:58 +08:00
Refactor API for document and session (#2819)
### What problem does this PR solve? Refactor API for document and session. ### Type of change - [x] Refactoring --------- Co-authored-by: liuhua <10215101452@stu.ecun.edu.cn>
This commit is contained in:
liuhua
GitHub
parent
7d80fc474c
commit
6eed115723
@ -4,9 +4,11 @@ import datetime
|
||||
import json
|
||||
import traceback
|
||||
|
||||
from botocore.docs.method import document_model_driven_method
|
||||
from flask import request
|
||||
from flask_login import login_required, current_user
|
||||
from elasticsearch_dsl import Q
|
||||
from sphinx.addnodes import document
|
||||
|
||||
from rag.app.qa import rmPrefix, beAdoc
|
||||
from rag.nlp import search, rag_tokenizer, keyword_extraction
|
||||
@ -16,22 +18,22 @@ from api.db import LLMType, ParserType
|
||||
from api.db.services.knowledgebase_service import KnowledgebaseService
|
||||
from api.db.services.llm_service import TenantLLMService
|
||||
from api.db.services.user_service import UserTenantService
|
||||
from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
|
||||
from api.utils.api_utils import server_error_response, get_error_data_result, validate_request
|
||||
from api.db.services.document_service import DocumentService
|
||||
from api.settings import RetCode, retrievaler, kg_retrievaler
|
||||
from api.utils.api_utils import get_json_result
|
||||
from api.utils.api_utils import get_result
|
||||
import hashlib
|
||||
import re
|
||||
from api.utils.api_utils import get_json_result, token_required, get_data_error_result
|
||||
from api.utils.api_utils import get_result, token_required, get_error_data_result
|
||||
|
||||
from api.db.db_models import Task, File
|
||||
|
||||
from api.db.services.task_service import TaskService, queue_tasks
|
||||
from api.db.services.user_service import TenantService, UserTenantService
|
||||
|
||||
from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
|
||||
from api.utils.api_utils import server_error_response, get_error_data_result, validate_request
|
||||
|
||||
from api.utils.api_utils import get_json_result
|
||||
from api.utils.api_utils import get_result, get_result, get_error_data_result
|
||||
|
||||
from functools import partial
|
||||
from io import BytesIO
|
||||
@ -59,278 +61,136 @@ MAXIMUM_OF_UPLOADING_FILES = 256
|
||||
MAXIMUM_OF_UPLOADING_FILES = 256
|
||||
|
||||
|
||||
@manager.route('/dataset/<dataset_id>/documents/upload', methods=['POST'])
|
||||
@manager.route('/dataset/<dataset_id>/document', methods=['POST'])
|
||||
@token_required
|
||||
def upload(dataset_id, tenant_id):
|
||||
if 'file' not in request.files:
|
||||
return get_json_result(
|
||||
data=False, retmsg='No file part!', retcode=RetCode.ARGUMENT_ERROR)
|
||||
return get_error_data_result(
|
||||
retmsg='No file part!', retcode=RetCode.ARGUMENT_ERROR)
|
||||
file_objs = request.files.getlist('file')
|
||||
for file_obj in file_objs:
|
||||
if file_obj.filename == '':
|
||||
return get_json_result(
|
||||
data=False, retmsg='No file selected!', retcode=RetCode.ARGUMENT_ERROR)
|
||||
return get_result(
|
||||
retmsg='No file selected!', retcode=RetCode.ARGUMENT_ERROR)
|
||||
e, kb = KnowledgebaseService.get_by_id(dataset_id)
|
||||
if not e:
|
||||
raise LookupError(f"Can't find the knowledgebase with ID {dataset_id}!")
|
||||
err, _ = FileService.upload_document(kb, file_objs, tenant_id)
|
||||
if err:
|
||||
return get_json_result(
|
||||
data=False, retmsg="\n".join(err), retcode=RetCode.SERVER_ERROR)
|
||||
return get_json_result(data=True)
|
||||
return get_result(
|
||||
retmsg="\n".join(err), retcode=RetCode.SERVER_ERROR)
|
||||
return get_result()
|
||||
|
||||
|
||||
@manager.route('/infos', methods=['GET'])
|
||||
@manager.route('/dataset/<dataset_id>/info/<document_id>', methods=['PUT'])
|
||||
@token_required
|
||||
def docinfos(tenant_id):
|
||||
req = request.args
|
||||
if "id" not in req and "name" not in req:
|
||||
return get_data_error_result(
|
||||
retmsg="Id or name should be provided")
|
||||
doc_id=None
|
||||
if "id" in req:
|
||||
doc_id = req["id"]
|
||||
if "name" in req:
|
||||
doc_name = req["name"]
|
||||
doc_id = DocumentService.get_doc_id_by_doc_name(doc_name)
|
||||
e, doc = DocumentService.get_by_id(doc_id)
|
||||
#rename key's name
|
||||
key_mapping = {
|
||||
"chunk_num": "chunk_count",
|
||||
"kb_id": "knowledgebase_id",
|
||||
"token_num": "token_count",
|
||||
"parser_id":"parser_method",
|
||||
}
|
||||
renamed_doc = {}
|
||||
for key, value in doc.to_dict().items():
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_doc[new_key] = value
|
||||
|
||||
return get_json_result(data=renamed_doc)
|
||||
|
||||
|
||||
@manager.route('/save', methods=['POST'])
|
||||
@token_required
|
||||
def save_doc(tenant_id):
|
||||
def update_doc(tenant_id, dataset_id, document_id):
|
||||
req = request.json
|
||||
#get doc by id or name
|
||||
doc_id = None
|
||||
if "id" in req:
|
||||
doc_id = req["id"]
|
||||
elif "name" in req:
|
||||
doc_name = req["name"]
|
||||
doc_id = DocumentService.get_doc_id_by_doc_name(doc_name)
|
||||
if not doc_id:
|
||||
return get_json_result(retcode=400, retmsg="Document ID or name is required")
|
||||
e, doc = DocumentService.get_by_id(doc_id)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
#other value can't be changed
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg='You do not own the dataset.')
|
||||
doc = DocumentService.query(kb_id=dataset_id, id=document_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg='The dataset not own the document.')
|
||||
doc = doc[0]
|
||||
if "chunk_count" in req:
|
||||
if req["chunk_count"] != doc.chunk_num:
|
||||
return get_data_error_result(
|
||||
retmsg="Can't change chunk_count.")
|
||||
return get_error_data_result(retmsg="Can't change chunk_count.")
|
||||
if "token_count" in req:
|
||||
if req["token_count"] != doc.token_num:
|
||||
return get_data_error_result(
|
||||
retmsg="Can't change token_count.")
|
||||
return get_error_data_result(retmsg="Can't change token_count.")
|
||||
if "progress" in req:
|
||||
if req['progress'] != doc.progress:
|
||||
return get_data_error_result(
|
||||
retmsg="Can't change progress.")
|
||||
#change name or parse_method
|
||||
return get_error_data_result(retmsg="Can't change progress.")
|
||||
|
||||
if "name" in req and req["name"] != doc.name:
|
||||
try:
|
||||
if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
|
||||
doc.name.lower()).suffix:
|
||||
return get_json_result(
|
||||
data=False,
|
||||
retmsg="The extension of file can't be changed",
|
||||
retcode=RetCode.ARGUMENT_ERROR)
|
||||
if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
|
||||
return get_result(retmsg="The extension of file can't be changed", retcode=RetCode.ARGUMENT_ERROR)
|
||||
for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
|
||||
if d.name == req["name"]:
|
||||
return get_data_error_result(
|
||||
return get_error_data_result(
|
||||
retmsg="Duplicated document name in the same knowledgebase.")
|
||||
|
||||
if not DocumentService.update_by_id(
|
||||
doc_id, {"name": req["name"]}):
|
||||
return get_data_error_result(
|
||||
document_id, {"name": req["name"]}):
|
||||
return get_error_data_result(
|
||||
retmsg="Database error (Document rename)!")
|
||||
|
||||
informs = File2DocumentService.get_by_document_id(doc_id)
|
||||
informs = File2DocumentService.get_by_document_id(document_id)
|
||||
if informs:
|
||||
e, file = FileService.get_by_id(informs[0].file_id)
|
||||
FileService.update_by_id(file.id, {"name": req["name"]})
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
if "parser_method" in req:
|
||||
try:
|
||||
if doc.parser_id.lower() == req["parser_method"].lower():
|
||||
if "parser_config" in req:
|
||||
if req["parser_config"] == doc.parser_config:
|
||||
return get_json_result(data=True)
|
||||
return get_result(retcode=RetCode.SUCCESS)
|
||||
else:
|
||||
return get_json_result(data=True)
|
||||
return get_result(retcode=RetCode.SUCCESS)
|
||||
|
||||
if doc.type == FileType.VISUAL or re.search(
|
||||
r"\.(ppt|pptx|pages)$", doc.name):
|
||||
return get_data_error_result(retmsg="Not supported yet!")
|
||||
return get_error_data_result(retmsg="Not supported yet!")
|
||||
|
||||
e = DocumentService.update_by_id(doc.id,
|
||||
{"parser_id": req["parser_method"], "progress": 0, "progress_msg": "",
|
||||
"run": TaskStatus.UNSTART.value})
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
if "parser_config" in req:
|
||||
DocumentService.update_parser_config(doc.id, req["parser_config"])
|
||||
return get_error_data_result(retmsg="Document not found!")
|
||||
if doc.token_num > 0:
|
||||
e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1,
|
||||
doc.process_duation * -1)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
return get_error_data_result(retmsg="Document not found!")
|
||||
tenant_id = DocumentService.get_tenant_id(req["id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
return get_error_data_result(retmsg="Tenant not found!")
|
||||
ELASTICSEARCH.deleteByQuery(
|
||||
Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
return get_json_result(data=True)
|
||||
|
||||
|
||||
|
||||
@manager.route('/change_parser', methods=['POST'])
|
||||
@token_required
|
||||
def change_parser(tenant_id):
|
||||
req = request.json
|
||||
try:
|
||||
e, doc = DocumentService.get_by_id(req["doc_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
if doc.parser_id.lower() == req["parser_id"].lower():
|
||||
if "parser_config" in req:
|
||||
if req["parser_config"] == doc.parser_config:
|
||||
return get_json_result(data=True)
|
||||
else:
|
||||
return get_json_result(data=True)
|
||||
|
||||
if doc.type == FileType.VISUAL or re.search(
|
||||
r"\.(ppt|pptx|pages)$", doc.name):
|
||||
return get_data_error_result(retmsg="Not supported yet!")
|
||||
|
||||
e = DocumentService.update_by_id(doc.id,
|
||||
{"parser_id": req["parser_id"], "progress": 0, "progress_msg": "",
|
||||
"run": TaskStatus.UNSTART.value})
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
if "parser_config" in req:
|
||||
DocumentService.update_parser_config(doc.id, req["parser_config"])
|
||||
if doc.token_num > 0:
|
||||
e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1,
|
||||
doc.process_duation * -1)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
tenant_id = DocumentService.get_tenant_id(req["doc_id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
ELASTICSEARCH.deleteByQuery(
|
||||
Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
|
||||
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
|
||||
@manager.route('/rename', methods=['POST'])
|
||||
@login_required
|
||||
@validate_request("doc_id", "name")
|
||||
def rename():
|
||||
req = request.json
|
||||
try:
|
||||
e, doc = DocumentService.get_by_id(req["doc_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
if pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
|
||||
doc.name.lower()).suffix:
|
||||
return get_json_result(
|
||||
data=False,
|
||||
retmsg="The extension of file can't be changed",
|
||||
retcode=RetCode.ARGUMENT_ERROR)
|
||||
for d in DocumentService.query(name=req["name"], kb_id=doc.kb_id):
|
||||
if d.name == req["name"]:
|
||||
return get_data_error_result(
|
||||
retmsg="Duplicated document name in the same knowledgebase.")
|
||||
|
||||
if not DocumentService.update_by_id(
|
||||
req["doc_id"], {"name": req["name"]}):
|
||||
return get_data_error_result(
|
||||
retmsg="Database error (Document rename)!")
|
||||
|
||||
informs = File2DocumentService.get_by_document_id(req["doc_id"])
|
||||
if informs:
|
||||
e, file = FileService.get_by_id(informs[0].file_id)
|
||||
FileService.update_by_id(file.id, {"name": req["name"]})
|
||||
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
return get_result()
|
||||
|
||||
|
||||
@manager.route("/<document_id>", methods=["GET"])
|
||||
@manager.route('/dataset/<dataset_id>/document/<document_id>', methods=['GET'])
|
||||
@token_required
|
||||
def download_document(document_id,tenant_id):
|
||||
try:
|
||||
# Check whether there is this document
|
||||
exist, document = DocumentService.get_by_id(document_id)
|
||||
if not exist:
|
||||
return construct_json_result(message=f"This document '{document_id}' cannot be found!",
|
||||
code=RetCode.ARGUMENT_ERROR)
|
||||
|
||||
def download(tenant_id, dataset_id, document_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f'You do not own the dataset {dataset_id}.')
|
||||
doc = DocumentService.query(kb_id=dataset_id, id=document_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f'The dataset not own the document {doc.id}.')
|
||||
# The process of downloading
|
||||
doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id) # minio address
|
||||
file_stream = STORAGE_IMPL.get(doc_id, doc_location)
|
||||
if not file_stream:
|
||||
return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR)
|
||||
|
||||
file = BytesIO(file_stream)
|
||||
|
||||
# Use send_file with a proper filename and MIME type
|
||||
return send_file(
|
||||
file,
|
||||
as_attachment=True,
|
||||
download_name=document.name,
|
||||
download_name=doc[0].name,
|
||||
mimetype='application/octet-stream' # Set a default MIME type
|
||||
)
|
||||
|
||||
# Error
|
||||
except Exception as e:
|
||||
return construct_error_response(e)
|
||||
|
||||
|
||||
@manager.route('/dataset/<dataset_id>/documents', methods=['GET'])
|
||||
@manager.route('/dataset/<dataset_id>/info', methods=['GET'])
|
||||
@token_required
|
||||
def list_docs(dataset_id, tenant_id):
|
||||
kb_id = request.args.get("knowledgebase_id")
|
||||
if not kb_id:
|
||||
return get_json_result(
|
||||
data=False, retmsg='Lack of "KB ID"', retcode=RetCode.ARGUMENT_ERROR)
|
||||
tenants = UserTenantService.query(user_id=tenant_id)
|
||||
for tenant in tenants:
|
||||
if KnowledgebaseService.query(
|
||||
tenant_id=tenant.tenant_id, id=kb_id):
|
||||
break
|
||||
else:
|
||||
return get_json_result(
|
||||
data=False, retmsg=f'Only owner of knowledgebase authorized for this operation.',
|
||||
retcode=RetCode.OPERATING_ERROR)
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}. ")
|
||||
id = request.args.get("id")
|
||||
if not DocumentService.query(id=id,kb_id=dataset_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the document {id}.")
|
||||
offset = int(request.args.get("offset", 1))
|
||||
keywords = request.args.get("keywords","")
|
||||
|
||||
page_number = int(request.args.get("page", 1))
|
||||
items_per_page = int(request.args.get("page_size", 15))
|
||||
limit = int(request.args.get("limit", 1024))
|
||||
orderby = request.args.get("orderby", "create_time")
|
||||
desc = request.args.get("desc", True)
|
||||
try:
|
||||
docs, tol = DocumentService.get_by_kb_id(
|
||||
kb_id, page_number, items_per_page, orderby, desc, keywords)
|
||||
if request.args.get("desc") == "False":
|
||||
desc = False
|
||||
else:
|
||||
desc = True
|
||||
docs, tol = DocumentService.get_list(dataset_id, offset, limit, orderby, desc, keywords, id)
|
||||
|
||||
# rename key's name
|
||||
renamed_doc_list = []
|
||||
@ -346,20 +206,18 @@ def list_docs(dataset_id, tenant_id):
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_doc[new_key] = value
|
||||
renamed_doc_list.append(renamed_doc)
|
||||
return get_json_result(data={"total": tol, "docs": renamed_doc_list})
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
return get_result(data={"total": tol, "docs": renamed_doc_list})
|
||||
|
||||
|
||||
@manager.route('/delete', methods=['DELETE'])
|
||||
@manager.route('/dataset/<dataset_id>/document', methods=['DELETE'])
|
||||
@token_required
|
||||
def rm(tenant_id):
|
||||
req = request.args
|
||||
if "document_id" not in req:
|
||||
return get_data_error_result(
|
||||
retmsg="doc_id is required")
|
||||
doc_ids = req["document_id"]
|
||||
if isinstance(doc_ids, str): doc_ids = [doc_ids]
|
||||
def delete(tenant_id,dataset_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}. ")
|
||||
req = request.json
|
||||
if not req.get("ids"):
|
||||
return get_error_data_result(retmsg="ids is required")
|
||||
doc_ids = req["ids"]
|
||||
root_folder = FileService.get_root_folder(tenant_id)
|
||||
pf_id = root_folder["id"]
|
||||
FileService.init_knowledgebase_docs(pf_id, tenant_id)
|
||||
@ -368,15 +226,15 @@ def rm(tenant_id):
|
||||
try:
|
||||
e, doc = DocumentService.get_by_id(doc_id)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
return get_error_data_result(retmsg="Document not found!")
|
||||
tenant_id = DocumentService.get_tenant_id(doc_id)
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
return get_error_data_result(retmsg="Tenant not found!")
|
||||
|
||||
b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
|
||||
|
||||
if not DocumentService.remove_document(doc, tenant_id):
|
||||
return get_data_error_result(
|
||||
return get_error_data_result(
|
||||
retmsg="Database error (Document removal)!")
|
||||
|
||||
f2d = File2DocumentService.get_by_document_id(doc_id)
|
||||
@ -388,80 +246,69 @@ def rm(tenant_id):
|
||||
errors += str(e)
|
||||
|
||||
if errors:
|
||||
return get_json_result(data=False, retmsg=errors, retcode=RetCode.SERVER_ERROR)
|
||||
return get_result(retmsg=errors, retcode=RetCode.SERVER_ERROR)
|
||||
|
||||
return get_json_result(data=True, retmsg="success")
|
||||
return get_result()
|
||||
|
||||
@manager.route("/<document_id>/status", methods=["GET"])
|
||||
|
||||
@manager.route('/dataset/<dataset_id>/chunk', methods=['POST'])
|
||||
@token_required
|
||||
def show_parsing_status(tenant_id, document_id):
|
||||
try:
|
||||
# valid document
|
||||
exist, _ = DocumentService.get_by_id(document_id)
|
||||
if not exist:
|
||||
return construct_json_result(code=RetCode.DATA_ERROR,
|
||||
message=f"This document: '{document_id}' is not a valid document.")
|
||||
|
||||
_, doc = DocumentService.get_by_id(document_id) # get doc object
|
||||
doc_attributes = doc.to_dict()
|
||||
|
||||
return construct_json_result(
|
||||
data={"progress": doc_attributes["progress"], "status": TaskStatus(doc_attributes["status"]).name},
|
||||
code=RetCode.SUCCESS
|
||||
)
|
||||
except Exception as e:
|
||||
return construct_error_response(e)
|
||||
|
||||
|
||||
|
||||
@manager.route('/run', methods=['POST'])
|
||||
@token_required
|
||||
def run(tenant_id):
|
||||
def parse(tenant_id,dataset_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
req = request.json
|
||||
try:
|
||||
for id in req["document_ids"]:
|
||||
info = {"run": str(req["run"]), "progress": 0}
|
||||
if str(req["run"]) == TaskStatus.RUNNING.value:
|
||||
if not DocumentService.query(id=id,kb_id=dataset_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the document {id}.")
|
||||
info = {"run": "1", "progress": 0}
|
||||
info["progress_msg"] = ""
|
||||
info["chunk_num"] = 0
|
||||
info["token_num"] = 0
|
||||
DocumentService.update_by_id(id, info)
|
||||
# if str(req["run"]) == TaskStatus.CANCEL.value:
|
||||
tenant_id = DocumentService.get_tenant_id(id)
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
ELASTICSEARCH.deleteByQuery(
|
||||
Q("match", doc_id=id), idxnm=search.index_name(tenant_id))
|
||||
|
||||
if str(req["run"]) == TaskStatus.RUNNING.value:
|
||||
TaskService.filter_delete([Task.doc_id == id])
|
||||
e, doc = DocumentService.get_by_id(id)
|
||||
doc = doc.to_dict()
|
||||
doc["tenant_id"] = tenant_id
|
||||
bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
|
||||
queue_tasks(doc, bucket, name)
|
||||
return get_result()
|
||||
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
|
||||
|
||||
@manager.route('/chunk/list', methods=['POST'])
|
||||
@manager.route('/dataset/<dataset_id>/chunk', methods=['DELETE'])
|
||||
@token_required
|
||||
@validate_request("document_id")
|
||||
def list_chunk(tenant_id):
|
||||
def stop_parsing(tenant_id,dataset_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
req = request.json
|
||||
doc_id = req["document_id"]
|
||||
page = int(req.get("page", 1))
|
||||
size = int(req.get("size", 30))
|
||||
for id in req["document_ids"]:
|
||||
if not DocumentService.query(id=id,kb_id=dataset_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the document {id}.")
|
||||
info = {"run": "2", "progress": 0}
|
||||
DocumentService.update_by_id(id, info)
|
||||
# if str(req["run"]) == TaskStatus.CANCEL.value:
|
||||
tenant_id = DocumentService.get_tenant_id(id)
|
||||
ELASTICSEARCH.deleteByQuery(
|
||||
Q("match", doc_id=id), idxnm=search.index_name(tenant_id))
|
||||
return get_result()
|
||||
|
||||
|
||||
@manager.route('/dataset/{dataset_id}/document/{document_id}/chunk', methods=['GET'])
|
||||
@token_required
|
||||
def list_chunk(tenant_id,dataset_id,document_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
doc=DocumentService.query(id=document_id, kb_id=dataset_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f"You don't own the document {document_id}.")
|
||||
doc=doc[0]
|
||||
req = request.args
|
||||
doc_id = document_id
|
||||
page = int(req.get("offset", 1))
|
||||
size = int(req.get("limit", 30))
|
||||
question = req.get("keywords", "")
|
||||
try:
|
||||
tenant_id = DocumentService.get_tenant_id(req["document_id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
e, doc = DocumentService.get_by_id(doc_id)
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
query = {
|
||||
"doc_ids": [doc_id], "page": page, "size": size, "question": question, "sort": True
|
||||
}
|
||||
@ -506,21 +353,27 @@ def list_chunk(tenant_id):
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_chunk[new_key] = value
|
||||
res["chunks"].append(renamed_chunk)
|
||||
return get_json_result(data=res)
|
||||
return get_result(data=res)
|
||||
except Exception as e:
|
||||
if str(e).find("not_found") > 0:
|
||||
return get_json_result(data=False, retmsg=f'No chunk found!',
|
||||
return get_result(retmsg=f'No chunk found!',
|
||||
retcode=RetCode.DATA_ERROR)
|
||||
return server_error_response(e)
|
||||
|
||||
|
||||
@manager.route('/chunk/create', methods=['POST'])
|
||||
@manager.route('/dataset/{dataset_id}/document/{document_id}/chunk', methods=['POST'])
|
||||
@token_required
|
||||
@validate_request("document_id", "content")
|
||||
def create(tenant_id):
|
||||
def create(tenant_id,dataset_id,document_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
doc = DocumentService.query(id=document_id, kb_id=dataset_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f"You don't own the document {document_id}.")
|
||||
req = request.json
|
||||
if not req.get("content"):
|
||||
return get_error_data_result(retmsg="`content` is required")
|
||||
md5 = hashlib.md5()
|
||||
md5.update((req["content"] + req["document_id"]).encode("utf-8"))
|
||||
md5.update((req["content"] + document_id).encode("utf-8"))
|
||||
|
||||
chunk_id = md5.hexdigest()
|
||||
d = {"id": chunk_id, "content_ltks": rag_tokenizer.tokenize(req["content"]),
|
||||
@ -530,20 +383,10 @@ def create(tenant_id):
|
||||
d["important_tks"] = rag_tokenizer.tokenize(" ".join(req.get("important_kwd", [])))
|
||||
d["create_time"] = str(datetime.datetime.now()).replace("T", " ")[:19]
|
||||
d["create_timestamp_flt"] = datetime.datetime.now().timestamp()
|
||||
|
||||
try:
|
||||
e, doc = DocumentService.get_by_id(req["document_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
d["kb_id"] = [doc.kb_id]
|
||||
d["docnm_kwd"] = doc.name
|
||||
d["doc_id"] = doc.id
|
||||
|
||||
tenant_id = DocumentService.get_tenant_id(req["document_id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
|
||||
embd_id = DocumentService.get_embd_id(req["document_id"])
|
||||
embd_id = DocumentService.get_embd_id(document_id)
|
||||
embd_mdl = TenantLLMService.model_instance(
|
||||
tenant_id, LLMType.EMBEDDING.value, embd_id)
|
||||
|
||||
@ -571,39 +414,46 @@ def create(tenant_id):
|
||||
if key in key_mapping:
|
||||
new_key = key_mapping.get(key, key)
|
||||
renamed_chunk[new_key] = value
|
||||
return get_result(data={"chunk": renamed_chunk})
|
||||
# return get_result(data={"chunk_id": chunk_id})
|
||||
|
||||
return get_json_result(data={"chunk": renamed_chunk})
|
||||
# return get_json_result(data={"chunk_id": chunk_id})
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
|
||||
@manager.route('/chunk/rm', methods=['POST'])
|
||||
@manager.route('dataset/{dataset_id}/document/{document_id}/chunk', methods=['DELETE'])
|
||||
@token_required
|
||||
@validate_request("chunk_ids", "document_id")
|
||||
def rm_chunk(tenant_id):
|
||||
def rm_chunk(tenant_id,dataset_id,document_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
doc = DocumentService.query(id=document_id, kb_id=dataset_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f"You don't own the document {document_id}.")
|
||||
req = request.json
|
||||
try:
|
||||
if not req.get("chunk_ids"):
|
||||
return get_error_data_result("`chunk_ids` is required")
|
||||
if not ELASTICSEARCH.deleteByQuery(
|
||||
Q("ids", values=req["chunk_ids"]), search.index_name(tenant_id)):
|
||||
return get_data_error_result(retmsg="Index updating failure")
|
||||
e, doc = DocumentService.get_by_id(req["document_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
return get_error_data_result(retmsg="Index updating failure")
|
||||
deleted_chunk_ids = req["chunk_ids"]
|
||||
chunk_number = len(deleted_chunk_ids)
|
||||
DocumentService.decrement_chunk_num(doc.id, doc.kb_id, 1, chunk_number, 0)
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
return get_result()
|
||||
|
||||
@manager.route('/chunk/set', methods=['POST'])
|
||||
|
||||
|
||||
@manager.route('/dataset/{dataset_id}/document/{document_id}/chunk/{chunk_id}', methods=['PUT'])
|
||||
@token_required
|
||||
@validate_request("document_id", "chunk_id", "content",
|
||||
"important_keywords")
|
||||
def set(tenant_id):
|
||||
def set(tenant_id,dataset_id,document_id,chunk_id):
|
||||
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
|
||||
return get_error_data_result(retmsg=f"You don't own the dataset {dataset_id}.")
|
||||
doc = DocumentService.query(id=document_id, kb_id=dataset_id)
|
||||
if not doc:
|
||||
return get_error_data_result(retmsg=f"You don't own the document {document_id}.")
|
||||
req = request.json
|
||||
if not req.get("content"):
|
||||
return get_error_data_result("`content` is required")
|
||||
if not req.get("important_keywords"):
|
||||
return get_error_data_result("`important_keywords` is required")
|
||||
d = {
|
||||
"id": req["chunk_id"],
|
||||
"id": chunk_id,
|
||||
"content_with_weight": req["content"]}
|
||||
d["content_ltks"] = rag_tokenizer.tokenize(req["content"])
|
||||
d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
|
||||
@ -611,27 +461,16 @@ def set(tenant_id):
|
||||
d["important_tks"] = rag_tokenizer.tokenize(" ".join(req["important_keywords"]))
|
||||
if "available" in req:
|
||||
d["available_int"] = req["available"]
|
||||
|
||||
try:
|
||||
tenant_id = DocumentService.get_tenant_id(req["document_id"])
|
||||
if not tenant_id:
|
||||
return get_data_error_result(retmsg="Tenant not found!")
|
||||
|
||||
embd_id = DocumentService.get_embd_id(req["document_id"])
|
||||
embd_id = DocumentService.get_embd_id(document_id)
|
||||
embd_mdl = TenantLLMService.model_instance(
|
||||
tenant_id, LLMType.EMBEDDING.value, embd_id)
|
||||
|
||||
e, doc = DocumentService.get_by_id(req["document_id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Document not found!")
|
||||
|
||||
if doc.parser_id == ParserType.QA:
|
||||
arr = [
|
||||
t for t in re.split(
|
||||
r"[\n\t]",
|
||||
req["content"]) if len(t) > 1]
|
||||
if len(arr) != 2:
|
||||
return get_data_error_result(
|
||||
return get_error_data_result(
|
||||
retmsg="Q&A must be separated by TAB/ENTER key.")
|
||||
q, a = rmPrefix(arr[0]), rmPrefix(arr[1])
|
||||
d = beAdoc(d, arr[0], arr[1], not any(
|
||||
@ -641,41 +480,35 @@ def set(tenant_id):
|
||||
v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
|
||||
d["q_%d_vec" % len(v)] = v.tolist()
|
||||
ELASTICSEARCH.upsert([d], search.index_name(tenant_id))
|
||||
return get_json_result(data=True)
|
||||
except Exception as e:
|
||||
return server_error_response(e)
|
||||
return get_result()
|
||||
|
||||
@manager.route('/retrieval_test', methods=['POST'])
|
||||
|
||||
|
||||
@manager.route('/retrieval', methods=['GET'])
|
||||
@token_required
|
||||
@validate_request("knowledgebase_id", "question")
|
||||
def retrieval_test(tenant_id):
|
||||
req = request.json
|
||||
page = int(req.get("page", 1))
|
||||
size = int(req.get("size", 30))
|
||||
req = request.args
|
||||
if not req.get("datasets"):
|
||||
return get_error_data_result("`datasets` is required.")
|
||||
for id in req.get("datasets"):
|
||||
if not KnowledgebaseService.query(id=id,tenant_id=tenant_id):
|
||||
return get_error_data_result(f"You don't own the dataset {id}.")
|
||||
if not req.get("question"):
|
||||
return get_error_data_result("`question` is required.")
|
||||
page = int(req.get("offset", 1))
|
||||
size = int(req.get("limit", 30))
|
||||
question = req["question"]
|
||||
kb_id = req["knowledgebase_id"]
|
||||
kb_id = req["datasets"]
|
||||
if isinstance(kb_id, str): kb_id = [kb_id]
|
||||
doc_ids = req.get("doc_ids", [])
|
||||
doc_ids = req.get("documents", [])
|
||||
similarity_threshold = float(req.get("similarity_threshold", 0.2))
|
||||
vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
|
||||
top = int(req.get("top_k", 1024))
|
||||
|
||||
try:
|
||||
tenants = UserTenantService.query(user_id=tenant_id)
|
||||
for kid in kb_id:
|
||||
for tenant in tenants:
|
||||
if KnowledgebaseService.query(
|
||||
tenant_id=tenant.tenant_id, id=kid):
|
||||
break
|
||||
else:
|
||||
return get_json_result(
|
||||
data=False, retmsg=f'Only owner of knowledgebase authorized for this operation.',
|
||||
retcode=RetCode.OPERATING_ERROR)
|
||||
|
||||
e, kb = KnowledgebaseService.get_by_id(kb_id[0])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Knowledgebase not found!")
|
||||
|
||||
return get_error_data_result(retmsg="Knowledgebase not found!")
|
||||
embd_mdl = TenantLLMService.model_instance(
|
||||
kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
|
||||
|
||||
@ -712,9 +545,9 @@ def retrieval_test(tenant_id):
|
||||
rename_chunk[new_key] = value
|
||||
renamed_chunks.append(rename_chunk)
|
||||
ranks["chunks"] = renamed_chunks
|
||||
return get_json_result(data=ranks)
|
||||
return get_result(data=ranks)
|
||||
except Exception as e:
|
||||
if str(e).find("not_found") > 0:
|
||||
return get_json_result(data=False, retmsg=f'No chunk found! Check the chunk status please!',
|
||||
return get_result(retmsg=f'No chunk found! Check the chunk status please!',
|
||||
retcode=RetCode.DATA_ERROR)
|
||||
return server_error_response(e)
|
||||
@ -20,47 +20,18 @@ from flask import request, Response
|
||||
|
||||
from api.db import StatusEnum
|
||||
from api.db.services.dialog_service import DialogService, ConversationService, chat
|
||||
from api.settings import RetCode
|
||||
from api.utils import get_uuid
|
||||
from api.utils.api_utils import get_data_error_result
|
||||
from api.utils.api_utils import get_json_result, token_required
|
||||
from api.utils.api_utils import get_error_data_result
|
||||
from api.utils.api_utils import get_result, token_required
|
||||
|
||||
|
||||
@manager.route('/save', methods=['POST'])
|
||||
@manager.route('/chat/<chat_id>/session', methods=['POST'])
|
||||
@token_required
|
||||
def set_conversation(tenant_id):
|
||||
def create(tenant_id,chat_id):
|
||||
req = request.json
|
||||
conv_id = req.get("id")
|
||||
if "assistant_id" in req:
|
||||
req["dialog_id"] = req.pop("assistant_id")
|
||||
if "id" in req:
|
||||
del req["id"]
|
||||
conv = ConversationService.query(id=conv_id)
|
||||
if not conv:
|
||||
return get_data_error_result(retmsg="Session does not exist")
|
||||
if not DialogService.query(id=conv[0].dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_data_error_result(retmsg="You do not own the session")
|
||||
if req.get("dialog_id"):
|
||||
req["dialog_id"] = chat_id
|
||||
dia = DialogService.query(tenant_id=tenant_id, id=req["dialog_id"], status=StatusEnum.VALID.value)
|
||||
if not dia:
|
||||
return get_data_error_result(retmsg="You do not own the assistant")
|
||||
if "dialog_id" in req and not req.get("dialog_id"):
|
||||
return get_data_error_result(retmsg="assistant_id can not be empty.")
|
||||
if "message" in req:
|
||||
return get_data_error_result(retmsg="message can not be change")
|
||||
if "reference" in req:
|
||||
return get_data_error_result(retmsg="reference can not be change")
|
||||
if "name" in req and not req.get("name"):
|
||||
return get_data_error_result(retmsg="name can not be empty.")
|
||||
if not ConversationService.update_by_id(conv_id, req):
|
||||
return get_data_error_result(retmsg="Session updates error")
|
||||
return get_json_result(data=True)
|
||||
|
||||
if not req.get("dialog_id"):
|
||||
return get_data_error_result(retmsg="assistant_id is required.")
|
||||
dia = DialogService.query(tenant_id=tenant_id, id=req["dialog_id"], status=StatusEnum.VALID.value)
|
||||
if not dia:
|
||||
return get_data_error_result(retmsg="You do not own the assistant")
|
||||
return get_error_data_result(retmsg="You do not own the assistant")
|
||||
conv = {
|
||||
"id": get_uuid(),
|
||||
"dialog_id": req["dialog_id"],
|
||||
@ -68,33 +39,58 @@ def set_conversation(tenant_id):
|
||||
"message": [{"role": "assistant", "content": "Hi! I am your assistant，can I help you?"}]
|
||||
}
|
||||
if not conv.get("name"):
|
||||
return get_data_error_result(retmsg="name can not be empty.")
|
||||
return get_error_data_result(retmsg="Name can not be empty.")
|
||||
ConversationService.save(**conv)
|
||||
e, conv = ConversationService.get_by_id(conv["id"])
|
||||
if not e:
|
||||
return get_data_error_result(retmsg="Fail to new session!")
|
||||
return get_error_data_result(retmsg="Fail to create a session!")
|
||||
conv = conv.to_dict()
|
||||
conv['messages'] = conv.pop("message")
|
||||
conv["assistant_id"] = conv.pop("dialog_id")
|
||||
conv["chat_id"] = conv.pop("dialog_id")
|
||||
del conv["reference"]
|
||||
return get_json_result(data=conv)
|
||||
return get_result(data=conv)
|
||||
|
||||
|
||||
@manager.route('/completion', methods=['POST'])
|
||||
@manager.route('/chat/<chat_id>/session/<session_id>', methods=['PUT'])
|
||||
@token_required
|
||||
def completion(tenant_id):
|
||||
def update(tenant_id,chat_id,session_id):
|
||||
req = request.json
|
||||
if "dialog_id" in req and req.get("dialog_id") != chat_id:
|
||||
return get_error_data_result(retmsg="Can't change chat_id")
|
||||
if "chat_id" in req and req.get("chat_id") != chat_id:
|
||||
return get_error_data_result(retmsg="Can't change chat_id")
|
||||
req["dialog_id"] = chat_id
|
||||
conv_id = session_id
|
||||
conv = ConversationService.query(id=conv_id,dialog_id=chat_id)
|
||||
if not conv:
|
||||
return get_error_data_result(retmsg="Session does not exist")
|
||||
if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_error_data_result(retmsg="You do not own the session")
|
||||
if "message" in req or "messages" in req:
|
||||
return get_error_data_result(retmsg="Message can not be change")
|
||||
if "reference" in req:
|
||||
return get_error_data_result(retmsg="Reference can not be change")
|
||||
if "name" in req and not req.get("name"):
|
||||
return get_error_data_result(retmsg="Name can not be empty.")
|
||||
if not ConversationService.update_by_id(conv_id, req):
|
||||
return get_error_data_result(retmsg="Session updates error")
|
||||
return get_result()
|
||||
|
||||
|
||||
@manager.route('/chat/<chat_id>/session/<session_id>/completion', methods=['POST'])
|
||||
@token_required
|
||||
def completion(tenant_id,chat_id,session_id):
|
||||
req = request.json
|
||||
# req = {"conversation_id": "9aaaca4c11d311efa461fa163e197198", "messages": [
|
||||
# {"role": "user", "content": "上海有吗？"}
|
||||
# ]}
|
||||
if "session_id" not in req:
|
||||
return get_data_error_result(retmsg="session_id is required")
|
||||
conv = ConversationService.query(id=req["session_id"])
|
||||
if not req.get("question"):
|
||||
return get_error_data_result(retmsg="Please input your question.")
|
||||
conv = ConversationService.query(id=session_id,dialog_id=chat_id)
|
||||
if not conv:
|
||||
return get_data_error_result(retmsg="Session does not exist")
|
||||
return get_error_data_result(retmsg="Session does not exist")
|
||||
conv = conv[0]
|
||||
if not DialogService.query(id=conv.dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_data_error_result(retmsg="You do not own the session")
|
||||
if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_error_data_result(retmsg="You do not own the session")
|
||||
msg = []
|
||||
question = {
|
||||
"content": req.get("question"),
|
||||
@ -108,7 +104,6 @@ def completion(tenant_id):
|
||||
msg.append(m)
|
||||
message_id = msg[-1].get("id")
|
||||
e, dia = DialogService.get_by_id(conv.dialog_id)
|
||||
del req["session_id"]
|
||||
|
||||
if not conv.reference:
|
||||
conv.reference = []
|
||||
@ -130,13 +125,13 @@ def completion(tenant_id):
|
||||
try:
|
||||
for ans in chat(dia, msg, **req):
|
||||
fillin_conv(ans)
|
||||
yield "data:" + json.dumps({"retcode": 0, "retmsg": "", "data": ans}, ensure_ascii=False) + "\n\n"
|
||||
yield "data:" + json.dumps({"code": 0, "data": ans}, ensure_ascii=False) + "\n\n"
|
||||
ConversationService.update_by_id(conv.id, conv.to_dict())
|
||||
except Exception as e:
|
||||
yield "data:" + json.dumps({"retcode": 500, "retmsg": str(e),
|
||||
yield "data:" + json.dumps({"code": 500, "message": str(e),
|
||||
"data": {"answer": "**ERROR**: " + str(e), "reference": []}},
|
||||
ensure_ascii=False) + "\n\n"
|
||||
yield "data:" + json.dumps({"retcode": 0, "retmsg": "", "data": True}, ensure_ascii=False) + "\n\n"
|
||||
yield "data:" + json.dumps({"code": 0, "data": True}, ensure_ascii=False) + "\n\n"
|
||||
|
||||
if req.get("stream", True):
|
||||
resp = Response(stream(), mimetype="text/event-stream")
|
||||
@ -153,73 +148,31 @@ def completion(tenant_id):
|
||||
fillin_conv(ans)
|
||||
ConversationService.update_by_id(conv.id, conv.to_dict())
|
||||
break
|
||||
return get_json_result(data=answer)
|
||||
return get_result(data=answer)
|
||||
|
||||
|
||||
@manager.route('/get', methods=['GET'])
|
||||
@manager.route('/chat/<chat_id>/session', methods=['GET'])
|
||||
@token_required
|
||||
def get(tenant_id):
|
||||
req = request.args
|
||||
if "id" not in req:
|
||||
return get_data_error_result(retmsg="id is required")
|
||||
conv_id = req["id"]
|
||||
conv = ConversationService.query(id=conv_id)
|
||||
if not conv:
|
||||
return get_data_error_result(retmsg="Session does not exist")
|
||||
if not DialogService.query(id=conv[0].dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_data_error_result(retmsg="You do not own the session")
|
||||
if "assistant_id" in req:
|
||||
if req["assistant_id"] != conv[0].dialog_id:
|
||||
return get_data_error_result(retmsg="The session doesn't belong to the assistant")
|
||||
conv = conv[0].to_dict()
|
||||
conv['messages'] = conv.pop("message")
|
||||
conv["assistant_id"] = conv.pop("dialog_id")
|
||||
if conv["reference"]:
|
||||
messages = conv["messages"]
|
||||
message_num = 0
|
||||
chunk_num = 0
|
||||
while message_num < len(messages):
|
||||
if message_num != 0 and messages[message_num]["role"] != "user":
|
||||
chunk_list = []
|
||||
if "chunks" in conv["reference"][chunk_num]:
|
||||
chunks = conv["reference"][chunk_num]["chunks"]
|
||||
for chunk in chunks:
|
||||
new_chunk = {
|
||||
"id": chunk["chunk_id"],
|
||||
"content": chunk["content_with_weight"],
|
||||
"document_id": chunk["doc_id"],
|
||||
"document_name": chunk["docnm_kwd"],
|
||||
"knowledgebase_id": chunk["kb_id"],
|
||||
"image_id": chunk["img_id"],
|
||||
"similarity": chunk["similarity"],
|
||||
"vector_similarity": chunk["vector_similarity"],
|
||||
"term_similarity": chunk["term_similarity"],
|
||||
"positions": chunk["positions"],
|
||||
}
|
||||
chunk_list.append(new_chunk)
|
||||
chunk_num += 1
|
||||
messages[message_num]["reference"] = chunk_list
|
||||
message_num += 1
|
||||
del conv["reference"]
|
||||
return get_json_result(data=conv)
|
||||
|
||||
|
||||
@manager.route('/list', methods=["GET"])
|
||||
@token_required
|
||||
def list(tenant_id):
|
||||
assistant_id = request.args["assistant_id"]
|
||||
if not DialogService.query(tenant_id=tenant_id, id=assistant_id, status=StatusEnum.VALID.value):
|
||||
return get_json_result(
|
||||
data=False, retmsg=f"You don't own the assistant.",
|
||||
retcode=RetCode.OPERATING_ERROR)
|
||||
convs = ConversationService.query(
|
||||
dialog_id=assistant_id,
|
||||
order_by=ConversationService.model.create_time,
|
||||
reverse=True)
|
||||
convs = [d.to_dict() for d in convs]
|
||||
def list(chat_id,tenant_id):
|
||||
if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
|
||||
return get_error_data_result(retmsg=f"You don't own the assistant {chat_id}.")
|
||||
id = request.args.get("id")
|
||||
name = request.args.get("name")
|
||||
session = ConversationService.query(id=id,name=name,dialog_id=chat_id)
|
||||
if not session:
|
||||
return get_error_data_result(retmsg="The session doesn't exist")
|
||||
page_number = int(request.args.get("page", 1))
|
||||
items_per_page = int(request.args.get("page_size", 1024))
|
||||
orderby = request.args.get("orderby", "create_time")
|
||||
if request.args.get("desc") == "False":
|
||||
desc = False
|
||||
else:
|
||||
desc = True
|
||||
convs = ConversationService.get_list(chat_id,page_number,items_per_page,orderby,desc,id,name)
|
||||
if not convs:
|
||||
return get_result(data=[])
|
||||
for conv in convs:
|
||||
conv['messages'] = conv.pop("message")
|
||||
conv["assistant_id"] = conv.pop("dialog_id")
|
||||
conv["chat"] = conv.pop("dialog_id")
|
||||
if conv["reference"]:
|
||||
messages = conv["messages"]
|
||||
message_num = 0
|
||||
@ -247,20 +200,19 @@ def list(tenant_id):
|
||||
messages[message_num]["reference"] = chunk_list
|
||||
message_num += 1
|
||||
del conv["reference"]
|
||||
return get_json_result(data=convs)
|
||||
return get_result(data=convs)
|
||||
|
||||
|
||||
@manager.route('/delete', methods=["DELETE"])
|
||||
@manager.route('/chat/<chat_id>/session', methods=["DELETE"])
|
||||
@token_required
|
||||
def delete(tenant_id):
|
||||
id = request.args.get("id")
|
||||
if not id:
|
||||
return get_data_error_result(retmsg="`id` is required in deleting operation")
|
||||
conv = ConversationService.query(id=id)
|
||||
def delete(tenant_id,chat_id):
|
||||
if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_error_data_result(retmsg="You don't own the chat")
|
||||
ids = request.json.get("ids")
|
||||
if not ids:
|
||||
return get_error_data_result(retmsg="`ids` is required in deleting operation")
|
||||
for id in ids:
|
||||
conv = ConversationService.query(id=id,dialog_id=chat_id)
|
||||
if not conv:
|
||||
return get_data_error_result(retmsg="Session doesn't exist")
|
||||
conv = conv[0]
|
||||
if not DialogService.query(id=conv.dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
|
||||
return get_data_error_result(retmsg="You don't own the session")
|
||||
return get_error_data_result(retmsg="The chat doesn't own the session")
|
||||
ConversationService.delete_by_id(id)
|
||||
return get_json_result(data=True)
|
||||
return get_result()
|
||||
|
||||
@ -19,6 +19,8 @@ import json
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from timeit import default_timer as timer
|
||||
|
||||
|
||||
from api.db import LLMType, ParserType,StatusEnum
|
||||
from api.db.db_models import Dialog, Conversation,DB
|
||||
from api.db.services.common_service import CommonService
|
||||
@ -61,6 +63,22 @@ class DialogService(CommonService):
|
||||
class ConversationService(CommonService):
|
||||
model = Conversation
|
||||
|
||||
@classmethod
|
||||
@DB.connection_context()
|
||||
def get_list(cls,dialog_id,page_number, items_per_page, orderby, desc, id , name):
|
||||
sessions = cls.model.select().where(cls.model.dialog_id ==dialog_id)
|
||||
if id:
|
||||
sessions = sessions.where(cls.model.id == id)
|
||||
if name:
|
||||
sessions = sessions.where(cls.model.name == name)
|
||||
if desc:
|
||||
sessions = sessions.order_by(cls.model.getter_by(orderby).desc())
|
||||
else:
|
||||
sessions = sessions.order_by(cls.model.getter_by(orderby).asc())
|
||||
|
||||
sessions = sessions.paginate(page_number, items_per_page)
|
||||
|
||||
return list(sessions.dicts())
|
||||
|
||||
def message_fit_in(msg, max_length=4000):
|
||||
def count():
|
||||
|
||||
@ -49,6 +49,29 @@ from rag.utils.redis_conn import REDIS_CONN
|
||||
class DocumentService(CommonService):
|
||||
model = Document
|
||||
|
||||
@classmethod
|
||||
@DB.connection_context()
|
||||
def get_list(cls, kb_id, page_number, items_per_page,
|
||||
orderby, desc, keywords, id):
|
||||
docs =cls.model.select().where(cls.model.kb_id==kb_id)
|
||||
if id:
|
||||
docs = docs.where(
|
||||
cls.model.id== id )
|
||||
if keywords:
|
||||
docs = docs.where(
|
||||
fn.LOWER(cls.model.name).contains(keywords.lower())
|
||||
)
|
||||
count = docs.count()
|
||||
if desc:
|
||||
docs = docs.order_by(cls.model.getter_by(orderby).desc())
|
||||
else:
|
||||
docs = docs.order_by(cls.model.getter_by(orderby).asc())
|
||||
|
||||
docs = docs.paginate(page_number, items_per_page)
|
||||
|
||||
return list(docs.dicts()), count
|
||||
|
||||
|
||||
@classmethod
|
||||
@DB.connection_context()
|
||||
def get_by_kb_id(cls, kb_id, page_number, items_per_page,
|
||||
@ -268,7 +291,7 @@ class DocumentService(CommonService):
|
||||
@classmethod
|
||||
@DB.connection_context()
|
||||
def get_thumbnails(cls, docids):
|
||||
fields = [cls.model.id, cls.model.kb_id, cls.model.thumbnail]
|
||||
fields = [cls.model.id, cls.model.thumbnail]
|
||||
return list(cls.model.select(
|
||||
*fields).where(cls.model.id.in_(docids)).dicts())
|
||||
|
||||
|
||||
329
api/http_api.md
329
api/http_api.md
@ -1441,60 +1441,196 @@ Create a chat session
|
||||
### Request
|
||||
|
||||
- Method: POST
|
||||
- URL: `/api/v1/chat/{chat_id}/session`
|
||||
- URL: `http://{address}/api/v1/chat/{chat_id}/session`
|
||||
- Headers:
|
||||
- `content-Type: application/json`
|
||||
- 'Authorization: Bearer {YOUR_ACCESS_TOKEN}'
|
||||
- Body:
|
||||
- name: `string`
|
||||
|
||||
#### Request example
|
||||
```bash
|
||||
curl --request POST \
|
||||
--url http://{address}/api/v1/chat/{chat_id}/session \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer {YOUR_ACCESS_TOKEN}' \
|
||||
--data-binary '{
|
||||
--data '{
|
||||
"name": "new session"
|
||||
}'
|
||||
```
|
||||
#### Request parameters
|
||||
- `"id"`: (*Body parameter*)
|
||||
The ID of the created session used to identify different sessions.
|
||||
- `None`
|
||||
- `id` cannot be provided when creating.
|
||||
|
||||
- `"name"`: (*Body parameter*)
|
||||
The name of the created session.
|
||||
- `"New session"`
|
||||
|
||||
- `"messages"`: (*Body parameter*)
|
||||
The messages of the created session.
|
||||
- `[{"role": "assistant", "content": "Hi! I am your assistant, can I help you?"}]`
|
||||
- `messages` cannot be provided when creating.
|
||||
|
||||
- `"chat_id"`: (*Path parameter*)
|
||||
The ID of the associated chat.
|
||||
- `""`
|
||||
- `chat_id` cannot be changed.
|
||||
|
||||
### Response
|
||||
Success
|
||||
```json
|
||||
{
|
||||
"code": 0,
|
||||
"data": {
|
||||
"chat_id": "2ca4b22e878011ef88fe0242ac120005",
|
||||
"create_date": "Fri, 11 Oct 2024 08:46:14 GMT",
|
||||
"create_time": 1728636374571,
|
||||
"id": "4606b4ec87ad11efbc4f0242ac120006",
|
||||
"messages": [
|
||||
{
|
||||
"content": "Hi! I am your assistant，can I help you?",
|
||||
"role": "assistant"
|
||||
}
|
||||
],
|
||||
"name": "new session",
|
||||
"update_date": "Fri, 11 Oct 2024 08:46:14 GMT",
|
||||
"update_time": 1728636374571
|
||||
}
|
||||
}
|
||||
```
|
||||
Error
|
||||
```json
|
||||
{
|
||||
"code": 102,
|
||||
"message": "Name can not be empty."
|
||||
}
|
||||
```
|
||||
|
||||
## List the sessions of a chat
|
||||
|
||||
**GET** `/api/v1/chat/{chat_id}/session`
|
||||
**GET** `/api/v1/chat/{chat_id}/session?page={page}&page_size={page_size}&orderby={orderby}&desc={desc}&name={dataset_name}&id={dataset_id}`
|
||||
|
||||
List all the session of a chat
|
||||
List all sessions under the chat based on the filtering criteria.
|
||||
|
||||
### Request
|
||||
|
||||
- Method: GET
|
||||
- URL: `/api/v1/chat/{chat_id}/session`
|
||||
- URL: `http://{address}/api/v1/chat/{chat_id}/session?page={page}&page_size={page_size}&orderby={orderby}&desc={desc}&name={dataset_name}&id={dataset_id}`
|
||||
- Headers:
|
||||
- `content-Type: application/json`
|
||||
- 'Authorization: Bearer {YOUR_ACCESS_TOKEN}'
|
||||
|
||||
#### Request example
|
||||
```bash
|
||||
curl --request GET \
|
||||
--url http://{address}/api/v1/chat/554e96746aaa11efb06b0242ac120005/session \
|
||||
--header 'Content-Type: application/json' \
|
||||
--url http://{address}/api/v1/chat/{chat_id}/session?page={page}&page_size={page_size}&orderby={orderby}&desc={desc}&name={dataset_name}&id={dataset_id} \
|
||||
--header 'Authorization: Bearer {YOUR_ACCESS_TOKEN}'
|
||||
```
|
||||
|
||||
## Delete a chat session
|
||||
#### Request Parameters
|
||||
- `"page"`: (*Path parameter*)
|
||||
The current page number to retrieve from the paginated data. This parameter determines which set of records will be fetched.
|
||||
- `1`
|
||||
|
||||
**DELETE** `/api/v1/chat/{chat_id}/session/{session_id}`
|
||||
- `"page_size"`: (*Path parameter*)
|
||||
The number of records to retrieve per page. This controls how many records will be included in each page.
|
||||
- `1024`
|
||||
|
||||
Delete a chat session
|
||||
- `"orderby"`: (*Path parameter*)
|
||||
The field by which the records should be sorted. This specifies the attribute or column used to order the results.
|
||||
- `"create_time"`
|
||||
|
||||
- `"desc"`: (*Path parameter*)
|
||||
A boolean flag indicating whether the sorting should be in descending order.
|
||||
- `True`
|
||||
|
||||
- `"id"`: (*Path parameter*)
|
||||
The ID of the session to be retrieved.
|
||||
- `None`
|
||||
|
||||
- `"name"`: (*Path parameter*)
|
||||
The name of the session to be retrieved.
|
||||
- `None`
|
||||
### Response
|
||||
Success
|
||||
```json
|
||||
{
|
||||
"code": 0,
|
||||
"data": [
|
||||
{
|
||||
"chat": "2ca4b22e878011ef88fe0242ac120005",
|
||||
"create_date": "Fri, 11 Oct 2024 08:46:43 GMT",
|
||||
"create_time": 1728636403974,
|
||||
"id": "578d541e87ad11ef96b90242ac120006",
|
||||
"messages": [
|
||||
{
|
||||
"content": "Hi! I am your assistant，can I help you?",
|
||||
"role": "assistant"
|
||||
}
|
||||
],
|
||||
"name": "new session",
|
||||
"update_date": "Fri, 11 Oct 2024 08:46:43 GMT",
|
||||
"update_time": 1728636403974
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
Error
|
||||
```json
|
||||
{
|
||||
"code": 102,
|
||||
"message": "The session doesn't exist"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Delete chat sessions
|
||||
|
||||
**DELETE** `/api/v1/chat/{chat_id}/session`
|
||||
|
||||
Delete chat sessions
|
||||
|
||||
### Request
|
||||
|
||||
- Method: DELETE
|
||||
- URL: `/api/v1/chat/{chat_id}/session/{session_id}`
|
||||
- URL: `http://{address}/api/v1/chat/{chat_id}/session`
|
||||
- Headers:
|
||||
- `content-Type: application/json`
|
||||
- 'Authorization: Bearer {YOUR_ACCESS_TOKEN}'
|
||||
- Body:
|
||||
- `ids`: List[string]
|
||||
|
||||
#### Request example
|
||||
```bash
|
||||
# Either id or name must be provided, but not both.
|
||||
curl --request DELETE \
|
||||
--url http://{address}/api/v1/chat/554e96746aaa11efb06b0242ac120005/session/791aed9670ea11efbb7e0242ac120007 \
|
||||
--url http://{address}/api/v1/chat/{chat_id}/session \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer {YOUR_ACCESS_TOKEN}'
|
||||
--header 'Authorization: Bear {YOUR_ACCESS_TOKEN}' \
|
||||
--data '{
|
||||
"ids": ["test_1", "test_2"]
|
||||
}'
|
||||
```
|
||||
|
||||
#### Request Parameters
|
||||
- `ids`: (*Body Parameter*)
|
||||
IDs of the sessions to be deleted.
|
||||
- `None`
|
||||
### Response
|
||||
Success
|
||||
```json
|
||||
{
|
||||
"code": 0
|
||||
}
|
||||
```
|
||||
Error
|
||||
```json
|
||||
{
|
||||
"code": 102,
|
||||
"message": "The chat doesn't own the session"
|
||||
}
|
||||
```
|
||||
## Update a chat session
|
||||
|
||||
**PUT** `/api/v1/chat/{chat_id}/session/{session_id}`
|
||||
@ -1504,20 +1640,45 @@ Update a chat session
|
||||
### Request
|
||||
|
||||
- Method: PUT
|
||||
- URL: `/api/v1/chat/{chat_id}/session/{session_id}`
|
||||
- URL: `http://{address}/api/v1/chat/{chat_id}/session/{session_id}`
|
||||
- Headers:
|
||||
- `content-Type: application/json`
|
||||
- 'Authorization: Bearer {YOUR_ACCESS_TOKEN}'
|
||||
- Body:
|
||||
- `name`: string
|
||||
|
||||
#### Request example
|
||||
```bash
|
||||
curl --request PUT \
|
||||
--url http://{address}/api/v1/chat/554e96746aaa11efb06b0242ac120005/session/791aed9670ea11efbb7e0242ac120007 \
|
||||
--url http://{address}/api/v1/chat/{chat_id}/session/{session_id} \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer {YOUR_ACCESS_TOKEN}'
|
||||
--data-binary '{
|
||||
--header 'Authorization: Bearer {YOUR_ACCESS_TOKEN}' \
|
||||
--data '{
|
||||
"name": "Updated session"
|
||||
}'
|
||||
|
||||
```
|
||||
|
||||
#### Request Parameter
|
||||
- `name`:(*Body Parameter)
|
||||
The name of the created session.
|
||||
- `None`
|
||||
|
||||
### Response
|
||||
Success
|
||||
```json
|
||||
{
|
||||
"code": 0
|
||||
}
|
||||
```
|
||||
Error
|
||||
```json
|
||||
{
|
||||
"code": 102,
|
||||
"message": "Name can not be empty."
|
||||
}
|
||||
```
|
||||
|
||||
## Chat with a chat session
|
||||
|
||||
**POST** `/api/v1/chat/{chat_id}/session/{session_id}/completion`
|
||||
@ -1527,17 +1688,139 @@ Chat with a chat session
|
||||
### Request
|
||||
|
||||
- Method: POST
|
||||
- URL: `/api/v1/chat/{chat_id}/session/{session_id}/completion`
|
||||
- URL: `http://{address} /api/v1/chat/{chat_id}/session/{session_id}/completion`
|
||||
- Headers:
|
||||
- `content-Type: application/json`
|
||||
- 'Authorization: Bearer {YOUR_ACCESS_TOKEN}'
|
||||
- Body:
|
||||
- `question`: string
|
||||
- `stream`: bool
|
||||
|
||||
|
||||
#### Request example
|
||||
```bash
|
||||
curl --request POST \
|
||||
--url http://{address}/api/v1/chat/554e96746aaa11efb06b0242ac120005/session/791aed9670ea11efbb7e0242ac120007/completion \
|
||||
--url http://{address} /api/v1/chat/{chat_id}/session/{session_id}/completion \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer {YOUR_ACCESS_TOKEN}'
|
||||
--header 'Authorization: Bearer {YOUR_ACCESS_TOKEN}' \
|
||||
--data-binary '{
|
||||
"question": "Hello!",
|
||||
"stream": true,
|
||||
"question": "你好!",
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
#### Request Parameters
|
||||
- `question`:(*Body Parameter*)
|
||||
The question you want to ask.
|
||||
- question is required.
|
||||
`None`
|
||||
- `stream`: (*Body Parameter*)
|
||||
The approach of streaming text generation.
|
||||
`False`
|
||||
### Response
|
||||
Success
|
||||
```json
|
||||
data: {
|
||||
"code": 0,
|
||||
"data": {
|
||||
"answer": "您好！有什么具体的问题或者需要的帮助",
|
||||
"reference": {},
|
||||
"audio_binary": null,
|
||||
"id": "31153052-7bac-4741-a513-ed07d853f29e"
|
||||
}
|
||||
}
|
||||
|
||||
data: {
|
||||
"code": 0,
|
||||
"data": {
|
||||
"answer": "您好！有什么具体的问题或者需要的帮助可以告诉我吗？我在这里是为了帮助",
|
||||
"reference": {},
|
||||
"audio_binary": null,
|
||||
"id": "31153052-7bac-4741-a513-ed07d853f29e"
|
||||
}
|
||||
}
|
||||
|
||||
data: {
|
||||
"code": 0,
|
||||
"data": {
|
||||
"answer": "您好！有什么具体的问题或者需要的帮助可以告诉我吗？我在这里是为了帮助您的。如果您有任何疑问或是需要获取",
|
||||
"reference": {},
|
||||
"audio_binary": null,
|
||||
"id": "31153052-7bac-4741-a513-ed07d853f29e"
|
||||
}
|
||||
}
|
||||
|
||||
data: {
|
||||
"code": 0,
|
||||
"data": {
|
||||
"answer": "您好！有什么具体的问题或者需要的帮助可以告诉我吗？我在这里是为了帮助您的。如果您有任何疑问或是需要获取某些信息，请随时提出。",
|
||||
"reference": {},
|
||||
"audio_binary": null,
|
||||
"id": "31153052-7bac-4741-a513-ed07d853f29e"
|
||||
}
|
||||
}
|
||||
|
||||
data: {
|
||||
"code": 0,
|
||||
"data": {
|
||||
"answer": "您好！有什么具体的问题或者需要的帮助可以告诉我吗 ##0$$？我在这里是为了帮助您的。如果您有任何疑问或是需要获取某些信息，请随时提出。",
|
||||
"reference": {
|
||||
"total": 19,
|
||||
"chunks": [
|
||||
{
|
||||
"chunk_id": "9d87f9d70a0d8a7565694a81fd4c5d5f",
|
||||
"content_ltks": "当所有知识库内容都与问题无关时 ,你的回答必须包括“知识库中未找到您要的答案!”这句话。回答需要考虑聊天历史。\r\n以下是知识库:\r\n{knowledg}\r\n以上是知识库\r\n\"\"\"\r\n 1\r\n 2\r\n 3\r\n 4\r\n 5\r\n 6\r\n总结\r\n通过上面的介绍,可以对开源的 ragflow有了一个大致的了解,与前面的有道qanyth整体流程还是比较类似的。 ",
|
||||
"content_with_weight": "当所有知识库内容都与问题无关时，你的回答必须包括“知识库中未找到您要的答案！”这句话。回答需要考虑聊天历史。\r\n 以下是知识库：\r\n {knowledge}\r\n 以上是知识库\r\n\"\"\"\r\n1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n总结\r\n通过上面的介绍，可以对开源的 RagFlow 有了一个大致的了解，与前面的 有道 QAnything 整体流程还是比较类似的。",
|
||||
"doc_id": "5c5999ec7be811ef9cab0242ac120005",
|
||||
"docnm_kwd": "1.txt",
|
||||
"kb_id": "c7ee74067a2c11efb21c0242ac120006",
|
||||
"important_kwd": [],
|
||||
"img_id": "",
|
||||
"similarity": 0.38337178633282265,
|
||||
"vector_similarity": 0.3321336754679629,
|
||||
"term_similarity": 0.4053309767034769,
|
||||
"positions": [
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"chunk_id": "895d34de762e674b43e8613c6fb54c6d",
|
||||
"content_ltks": "\r\n\r\n实际内容可能会超过大模型的输入token数量,因此在调用大模型前会调用api/db/servic/dialog_service.py文件中 messag_fit_in ()根据大模型可用的 token数量进行过滤。这部分与有道的 qanyth的实现大同小异,就不额外展开了。\r\n\r\n将检索的内容,历史聊天记录以及问题构造为 prompt ,即可作为大模型的输入了 ,默认的英文prompt如下所示:\r\n\r\n\"\"\"\r\nyou are an intellig assistant. pleas summar the content of the knowledg base to answer the question. pleas list thedata in the knowledg base and answer in detail. when all knowledg base content is irrelev to the question , your answer must includ the sentenc\"the answer you are lookfor isnot found in the knowledg base!\" answer needto consid chat history.\r\n here is the knowledg base:\r\n{ knowledg}\r\nthe abov is the knowledg base.\r\n\"\"\"\r\n1\r\n 2\r\n 3\r\n 4\r\n 5\r\n 6\r\n对应的中文prompt如下所示:\r\n\r\n\"\"\"\r\n你是一个智能助手,请总结知识库的内容来回答问题,请列举知识库中的数据详细回答。 ",
|
||||
"content_with_weight": "\r\n\r\n实际内容可能会超过大模型的输入 token 数量，因此在调用大模型前会调用 api/db/services/dialog_service.py 文件中 message_fit_in() 根据大模型可用的 token 数量进行过滤。这部分与有道的 QAnything 的实现大同小异，就不额外展开了。\r\n\r\n将检索的内容，历史聊天记录以及问题构造为 prompt，即可作为大模型的输入了，默认的英文 prompt 如下所示：\r\n\r\n\"\"\"\r\nYou are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\" Answers need to consider chat history.\r\n Here is the knowledge base:\r\n {knowledge}\r\n The above is the knowledge base.\r\n\"\"\"\r\n1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n对应的中文 prompt 如下所示：\r\n\r\n\"\"\"\r\n你是一个智能助手，请总结知识库的内容来回答问题，请列举知识库中的数据详细回答。",
|
||||
"doc_id": "5c5999ec7be811ef9cab0242ac120005",
|
||||
"docnm_kwd": "1.txt",
|
||||
"kb_id": "c7ee74067a2c11efb21c0242ac120006",
|
||||
"important_kwd": [],
|
||||
"img_id": "",
|
||||
"similarity": 0.2788204323926715,
|
||||
"vector_similarity": 0.35489427679953667,
|
||||
"term_similarity": 0.2462173562183008,
|
||||
"positions": [
|
||||
""
|
||||
]
|
||||
}
|
||||
],
|
||||
"doc_aggs": [
|
||||
{
|
||||
"doc_name": "1.txt",
|
||||
"doc_id": "5c5999ec7be811ef9cab0242ac120005",
|
||||
"count": 2
|
||||
}
|
||||
]
|
||||
},
|
||||
"prompt": "你是一个智能助手，请总结知识库的内容来回答问题，请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时，你的回答必须包括“知识库中未找到您要的答案！”这句话。回答需要考虑聊天历史。\n 以下是知识库：\n 当所有知识库内容都与问题无关时，你的回答必须包括“知识库中未找到您要的答案！”这句话。回答需要考虑聊天历史。\r\n 以下是知识库：\r\n {knowledge}\r\n 以上是知识库\r\n\"\"\"\r\n1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n总结\r\n通过上面的介绍，可以对开源的 RagFlow 有了一个大致的了解，与前面的 有道 QAnything 整体流程还是比较类似的。\n\n------\n\n\r\n\r\n实际内容可能会超过大模型的输入 token 数量，因此在调用大模型前会调用 api/db/services/dialog_service.py 文件中 message_fit_in() 根据大模型可用的 token 数量进行过滤。这部分与有道的 QAnything 的实现大同小异，就不额外展开了。\r\n\r\n将检索的内容，历史聊天记录以及问题构造为 prompt，即可作为大模型的输入了，默认的英文 prompt 如下所示：\r\n\r\n\"\"\"\r\nYou are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\" Answers need to consider chat history.\r\n Here is the knowledge base:\r\n {knowledge}\r\n The above is the knowledge base.\r\n\"\"\"\r\n1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n对应的中文 prompt 如下所示：\r\n\r\n\"\"\"\r\n你是一个智能助手，请总结知识库的内容来回答问题，请列举知识库中的数据详细回答。\n 以上是知识库。\n\n### Query:\n你好，请问有什么问题需要我帮忙解答吗？\n\n### Elapsed\n - Retrieval: 9131.1 ms\n - LLM: 12802.6 ms",
|
||||
"id": "31153052-7bac-4741-a513-ed07d853f29e"
|
||||
}
|
||||
}
|
||||
|
||||
data:{
|
||||
"code": 0,
|
||||
"data": true
|
||||
}
|
||||
```
|
||||
Error
|
||||
```json
|
||||
{
|
||||
"code": 102,
|
||||
"message": "Please input your question."
|
||||
}
|
||||
```
|
||||
@ -906,7 +906,7 @@ Chat-session APIs
|
||||
## Create session
|
||||
|
||||
```python
|
||||
assistant_1.create_session(name: str = "New session") -> Session
|
||||
Chat.create_session(name: str = "New session") -> Session
|
||||
```
|
||||
|
||||
### Returns
|
||||
@ -916,8 +916,7 @@ A `session` object.
|
||||
#### id: `str`
|
||||
|
||||
The id of the created session is used to identify different sessions.
|
||||
- `id` cannot be provided in creating
|
||||
- `id` is required in updating
|
||||
- id can not be provided in creating
|
||||
|
||||
#### name: `str`
|
||||
|
||||
@ -936,10 +935,10 @@ Defaults:
|
||||
[{"role": "assistant", "content": "Hi! I am your assistant，can I help you?"}]
|
||||
```
|
||||
|
||||
#### assistant_id: `str`
|
||||
#### chat_id: `str`
|
||||
|
||||
The id of associated assistant. Defaults to `""`.
|
||||
- `assistant_id` is required in creating if you use HTTP API.
|
||||
The id of associated chat
|
||||
- `chat_id` can't be changed
|
||||
|
||||
### Examples
|
||||
|
||||
@ -947,58 +946,21 @@ The id of associated assistant. Defaults to `""`.
|
||||
from ragflow import RAGFlow
|
||||
|
||||
rag = RAGFlow(api_key="xxxxxx", base_url="http://xxx.xx.xx.xxx:9380")
|
||||
assi = rag.get_assistant(name="Miss R")
|
||||
assi = rag.list_chats(name="Miss R")
|
||||
assi = assi[0]
|
||||
sess = assi.create_session()
|
||||
```
|
||||
|
||||
## Retrieve session
|
||||
|
||||
## Update session
|
||||
|
||||
```python
|
||||
Assistant.get_session(id: str) -> Session
|
||||
Session.update(update_message:dict)
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
#### id: `str`, *Required*
|
||||
|
||||
???????????????????????????????
|
||||
|
||||
### Returns
|
||||
|
||||
### Returns
|
||||
|
||||
A `session` object.
|
||||
|
||||
#### id: `str`
|
||||
|
||||
The id of the created session is used to identify different sessions.
|
||||
- `id` cannot be provided in creating
|
||||
- `id` is required in updating
|
||||
|
||||
#### name: `str`
|
||||
|
||||
The name of the created session. Defaults to `"New session"`.
|
||||
|
||||
#### messages: `List[Message]`
|
||||
|
||||
The messages of the created session.
|
||||
- messages cannot be provided.
|
||||
|
||||
Defaults:
|
||||
|
||||
??????????????????????????????????????????????????????????????????????????????????????????????
|
||||
|
||||
```
|
||||
[{"role": "assistant", "content": "Hi! I am your assistant，can I help you?"}]
|
||||
```
|
||||
|
||||
#### assistant_id: `str`
|
||||
|
||||
|
||||
???????????????????????????????????????How to get
|
||||
|
||||
The id of associated assistant. Defaults to `""`.
|
||||
- `assistant_id` is required in creating if you use HTTP API.
|
||||
no return
|
||||
|
||||
### Examples
|
||||
|
||||
@ -1006,33 +968,10 @@ The id of associated assistant. Defaults to `""`.
|
||||
from ragflow import RAGFlow
|
||||
|
||||
rag = RAGFlow(api_key="xxxxxx", base_url="http://xxx.xx.xx.xxx:9380")
|
||||
assi = rag.get_assistant(name="Miss R")
|
||||
sess = assi.get_session(id="d5c55d2270dd11ef9bd90242ac120007")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Save session settings
|
||||
|
||||
```python
|
||||
Session.save() -> bool
|
||||
```
|
||||
|
||||
### Returns
|
||||
|
||||
bool
|
||||
description:the case of updating a session, True or False.
|
||||
|
||||
### Examples
|
||||
|
||||
```python
|
||||
from ragflow import RAGFlow
|
||||
|
||||
rag = RAGFlow(api_key="xxxxxx", base_url="http://xxx.xx.xx.xxx:9380")
|
||||
assi = rag.get_assistant(name="Miss R")
|
||||
sess = assi.get_session(id="d5c55d2270dd11ef9bd90242ac120007")
|
||||
sess.name = "Updated session"
|
||||
sess.save()
|
||||
assi = rag.list_chats(name="Miss R")
|
||||
assi = assi[0]
|
||||
sess = assi.create_session("new_session")
|
||||
sess.update({"name": "Updated session"...})
|
||||
```
|
||||
|
||||
---
|
||||
@ -1040,7 +979,7 @@ sess.save()
|
||||
## Chat
|
||||
|
||||
```python
|
||||
Session.chat(question: str, stream: bool = False) -> Optional[Message, iter[Message]]
|
||||
Session.ask(question: str, stream: bool = False) -> Optional[Message, iter[Message]]
|
||||
```
|
||||
|
||||
### Parameters
|
||||
@ -1053,7 +992,6 @@ The question to start an AI chat. Defaults to `None`. ???????????????????
|
||||
|
||||
The approach of streaming text generation. When stream is True, it outputs results in a streaming fashion; otherwise, it outputs the complete result after the model has finished generating.
|
||||
|
||||
#### session_id: `str` ??????????????????
|
||||
|
||||
### Returns
|
||||
|
||||
@ -1098,7 +1036,8 @@ The auto-generated reference of the message. Each `chunk` object includes the fo
|
||||
from ragflow import RAGFlow
|
||||
|
||||
rag = RAGFlow(api_key="xxxxxx", base_url="http://xxx.xx.xx.xxx:9380")
|
||||
assi = rag.get_assistant(name="Miss R")
|
||||
assi = rag.list_chats(name="Miss R")
|
||||
assi = assi[0]
|
||||
sess = assi.create_session()
|
||||
|
||||
print("\n==================== Miss R =====================\n")
|
||||
@ -1109,9 +1048,10 @@ while True:
|
||||
print("\n==================== Miss R =====================\n")
|
||||
|
||||
cont = ""
|
||||
for ans in sess.chat(question, stream=True):
|
||||
for ans in sess.ask(question, stream=True):
|
||||
print(ans.content[len(cont):], end='', flush=True)
|
||||
cont = ans.content
|
||||
|
||||
```
|
||||
|
||||
---
|
||||
@ -1119,7 +1059,14 @@ while True:
|
||||
## List sessions
|
||||
|
||||
```python
|
||||
Assistant.list_session() -> List[Session]
|
||||
Chat.list_sessions(
|
||||
page: int = 1,
|
||||
page_size: int = 1024,
|
||||
orderby: str = "create_time",
|
||||
desc: bool = True,
|
||||
id: str = None,
|
||||
name: str = None
|
||||
) -> List[Session]
|
||||
```
|
||||
|
||||
### Returns
|
||||
@ -1133,24 +1080,54 @@ description: the List contains information about multiple assistant object, with
|
||||
from ragflow import RAGFlow
|
||||
|
||||
rag = RAGFlow(api_key="xxxxxx", base_url="http://xxx.xx.xx.xxx:9380")
|
||||
assi = rag.get_assistant(name="Miss R")
|
||||
|
||||
for sess in assi.list_session():
|
||||
assi = rag.list_chats(name="Miss R")
|
||||
assi = assi[0]
|
||||
for sess in assi.list_sessions():
|
||||
print(sess)
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
#### page: `int`
|
||||
|
||||
The current page number to retrieve from the paginated data. This parameter determines which set of records will be fetched.
|
||||
- `1`
|
||||
|
||||
#### page_size: `int`
|
||||
|
||||
The number of records to retrieve per page. This controls how many records will be included in each page.
|
||||
- `1024`
|
||||
|
||||
#### orderby: `string`
|
||||
|
||||
The field by which the records should be sorted. This specifies the attribute or column used to order the results.
|
||||
- `"create_time"`
|
||||
|
||||
#### desc: `bool`
|
||||
|
||||
A boolean flag indicating whether the sorting should be in descending order.
|
||||
- `True`
|
||||
|
||||
#### id: `string`
|
||||
|
||||
The ID of the chat to be retrieved.
|
||||
- `None`
|
||||
|
||||
#### name: `string`
|
||||
|
||||
The name of the chat to be retrieved.
|
||||
- `None`
|
||||
---
|
||||
|
||||
## Delete session
|
||||
|
||||
```python
|
||||
Session.delete() -> bool
|
||||
Chat.delete_sessions(ids:List[str] = None)
|
||||
```
|
||||
|
||||
### Returns
|
||||
|
||||
bool
|
||||
description:the case of deleting a session, True or False.
|
||||
no return
|
||||
|
||||
### Examples
|
||||
|
||||
@ -1158,7 +1135,12 @@ description:the case of deleting a session, True or False.
|
||||
from ragflow import RAGFlow
|
||||
|
||||
rag = RAGFlow(api_key="xxxxxx", base_url="http://xxx.xx.xx.xxx:9380")
|
||||
assi = rag.get_assistant(name="Miss R")
|
||||
sess = assi.create_session()
|
||||
sess.delete()
|
||||
assi = rag.list_chats(name="Miss R")
|
||||
assi = assi[0]
|
||||
assi.delete_sessions(ids=["id_1","id_2"])
|
||||
```
|
||||
### Parameters
|
||||
#### ids: `List[string]`
|
||||
IDs of the sessions to be deleted.
|
||||
- `None`
|
||||
|
||||
|
||||
@ -51,28 +51,28 @@ class Chat(Base):
|
||||
|
||||
|
||||
def create_session(self, name: str = "New session") -> Session:
|
||||
res = self.post("/session/save", {"name": name, "assistant_id": self.id})
|
||||
res = self.post(f"/chat/{self.id}/session", {"name": name})
|
||||
res = res.json()
|
||||
if res.get("retmsg") == "success":
|
||||
if res.get("code") == 0:
|
||||
return Session(self.rag, res['data'])
|
||||
raise Exception(res["retmsg"])
|
||||
raise Exception(res["message"])
|
||||
|
||||
def list_session(self) -> List[Session]:
|
||||
res = self.get('/session/list', {"assistant_id": self.id})
|
||||
def list_sessions(self,page: int = 1, page_size: int = 1024, orderby: str = "create_time", desc: bool = True,
|
||||
id: str = None, name: str = None) -> List[Session]:
|
||||
res = self.get(f'/chat/{self.id}/session',{"page": page, "page_size": page_size, "orderby": orderby, "desc": desc, "id": id, "name": name} )
|
||||
res = res.json()
|
||||
if res.get("retmsg") == "success":
|
||||
if res.get("code") == 0:
|
||||
result_list = []
|
||||
for data in res["data"]:
|
||||
result_list.append(Session(self.rag, data))
|
||||
return result_list
|
||||
raise Exception(res["retmsg"])
|
||||
raise Exception(res["message"])
|
||||
|
||||
def get_session(self, id) -> Session:
|
||||
res = self.get("/session/get", {"id": id,"assistant_id":self.id})
|
||||
def delete_sessions(self,ids):
|
||||
res = self.rm(f"/chat/{self.id}/session", {"ids": ids})
|
||||
res = res.json()
|
||||
if res.get("retmsg") == "success":
|
||||
return Session(self.rag, res["data"])
|
||||
raise Exception(res["retmsg"])
|
||||
if res.get("code") != 0:
|
||||
raise Exception(res.get("message"))
|
||||
|
||||
def get_prologue(self):
|
||||
return self.prompt.opener
|
||||
|
||||
@ -8,20 +8,20 @@ class Session(Base):
|
||||
self.id = None
|
||||
self.name = "New session"
|
||||
self.messages = [{"role": "assistant", "content": "Hi! I am your assistant，can I help you?"}]
|
||||
self.assistant_id = None
|
||||
self.chat_id = None
|
||||
super().__init__(rag, res_dict)
|
||||
|
||||
def chat(self, question: str, stream: bool = False):
|
||||
def ask(self, question: str, stream: bool = False):
|
||||
for message in self.messages:
|
||||
if "reference" in message:
|
||||
message.pop("reference")
|
||||
res = self.post("/session/completion",
|
||||
{"session_id": self.id, "question": question, "stream": True}, stream=stream)
|
||||
res = self.post(f"/chat/{self.chat_id}/session/{self.id}/completion",
|
||||
{"question": question, "stream": True}, stream=stream)
|
||||
for line in res.iter_lines():
|
||||
line = line.decode("utf-8")
|
||||
if line.startswith("{"):
|
||||
json_data = json.loads(line)
|
||||
raise Exception(json_data["retmsg"])
|
||||
raise Exception(json_data["message"])
|
||||
if line.startswith("data:"):
|
||||
json_data = json.loads(line[5:])
|
||||
if json_data["data"] != True:
|
||||
@ -52,19 +52,12 @@ class Session(Base):
|
||||
message = Message(self.rag, temp_dict)
|
||||
yield message
|
||||
|
||||
def save(self):
|
||||
res = self.post("/session/save",
|
||||
{"id": self.id, "assistant_id": self.assistant_id, "name": self.name})
|
||||
def update(self,update_message):
|
||||
res = self.put(f"/chat/{self.chat_id}/session/{self.id}",
|
||||
update_message)
|
||||
res = res.json()
|
||||
if res.get("retmsg") == "success": return True
|
||||
raise Exception(res.get("retmsg"))
|
||||
|
||||
def delete(self):
|
||||
res = self.rm("/session/delete", {"id": self.id})
|
||||
res = res.json()
|
||||
if res.get("retmsg") == "success": return True
|
||||
raise Exception(res.get("retmsg"))
|
||||
|
||||
if res.get("code") != 0:
|
||||
raise Exception(res.get("message"))
|
||||
|
||||
class Message(Base):
|
||||
def __init__(self, rag, res_dict):
|
||||
|
||||
@ -7,52 +7,44 @@ class TestSession:
|
||||
def test_create_session(self):
|
||||
rag = RAGFlow(API_KEY, HOST_ADDRESS)
|
||||
kb = rag.create_dataset(name="test_create_session")
|
||||
assistant = rag.create_assistant(name="test_create_session", knowledgebases=[kb])
|
||||
assistant = rag.create_chat(name="test_create_session", knowledgebases=[kb])
|
||||
session = assistant.create_session()
|
||||
assert isinstance(session,Session), "Failed to create a session."
|
||||
|
||||
def test_create_chat_with_success(self):
|
||||
rag = RAGFlow(API_KEY, HOST_ADDRESS)
|
||||
kb = rag.create_dataset(name="test_create_chat")
|
||||
assistant = rag.create_assistant(name="test_create_chat", knowledgebases=[kb])
|
||||
assistant = rag.create_chat(name="test_create_chat", knowledgebases=[kb])
|
||||
session = assistant.create_session()
|
||||
question = "What is AI"
|
||||
for ans in session.chat(question, stream=True):
|
||||
for ans in session.ask(question, stream=True):
|
||||
pass
|
||||
assert not ans.content.startswith("**ERROR**"), "Please check this error."
|
||||
|
||||
def test_delete_session_with_success(self):
|
||||
def test_delete_sessions_with_success(self):
|
||||
rag = RAGFlow(API_KEY, HOST_ADDRESS)
|
||||
kb = rag.create_dataset(name="test_delete_session")
|
||||
assistant = rag.create_assistant(name="test_delete_session",knowledgebases=[kb])
|
||||
assistant = rag.create_chat(name="test_delete_session",knowledgebases=[kb])
|
||||
session=assistant.create_session()
|
||||
res=session.delete()
|
||||
assert res, "Failed to delete the dataset."
|
||||
res=assistant.delete_sessions(ids=[session.id])
|
||||
assert res is None, "Failed to delete the dataset."
|
||||
|
||||
def test_update_session_with_success(self):
|
||||
rag=RAGFlow(API_KEY,HOST_ADDRESS)
|
||||
kb=rag.create_dataset(name="test_update_session")
|
||||
assistant = rag.create_assistant(name="test_update_session",knowledgebases=[kb])
|
||||
assistant = rag.create_chat(name="test_update_session",knowledgebases=[kb])
|
||||
session=assistant.create_session(name="old session")
|
||||
session.name="new session"
|
||||
res=session.save()
|
||||
assert res,"Failed to update the session"
|
||||
res=session.update({"name":"new session"})
|
||||
assert res is None,"Failed to update the session"
|
||||
|
||||
def test_get_session_with_success(self):
|
||||
rag=RAGFlow(API_KEY,HOST_ADDRESS)
|
||||
kb=rag.create_dataset(name="test_get_session")
|
||||
assistant = rag.create_assistant(name="test_get_session",knowledgebases=[kb])
|
||||
session = assistant.create_session()
|
||||
session_2= assistant.get_session(id=session.id)
|
||||
assert session.to_json()==session_2.to_json(),"Failed to get the session"
|
||||
|
||||
def test_list_session_with_success(self):
|
||||
def test_list_sessions_with_success(self):
|
||||
rag=RAGFlow(API_KEY,HOST_ADDRESS)
|
||||
kb=rag.create_dataset(name="test_list_session")
|
||||
assistant=rag.create_assistant(name="test_list_session",knowledgebases=[kb])
|
||||
assistant=rag.create_chat(name="test_list_session",knowledgebases=[kb])
|
||||
assistant.create_session("test_1")
|
||||
assistant.create_session("test_2")
|
||||
sessions=assistant.list_session()
|
||||
sessions=assistant.list_sessions()
|
||||
if isinstance(sessions,list):
|
||||
for session in sessions:
|
||||
assert isinstance(session,Session),"Non-Session elements exist in the list"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user