From 301e0496ff553ea975b5233f01c53fc8f22689c0 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 25 Jan 2024 14:53:52 +0800 Subject: [PATCH] fix: chatbot support agent (#2201) --- web/app/components/base/voice-input/utils.ts | 8 +- web/app/components/share/chat/index.tsx | 2 - .../share/chatbot/hooks/use-conversation.ts | 4 +- web/app/components/share/chatbot/index.tsx | 219 +++++++++++++++--- 4 files changed, 195 insertions(+), 38 deletions(-) diff --git a/web/app/components/base/voice-input/utils.ts b/web/app/components/base/voice-input/utils.ts index cc0b25fa84..70133f459f 100644 --- a/web/app/components/base/voice-input/utils.ts +++ b/web/app/components/base/voice-input/utils.ts @@ -3,9 +3,11 @@ import MPEGMode from 'lamejs/src/js/MPEGMode' import Lame from 'lamejs/src/js/Lame' import BitStream from 'lamejs/src/js/BitStream' -(window as any).MPEGMode = MPEGMode -;(window as any).Lame = Lame -;(window as any).BitStream = BitStream +if (globalThis) { + (globalThis as any).MPEGMode = MPEGMode + ;(globalThis as any).Lame = Lame + ;(globalThis as any).BitStream = BitStream +} export const convertToMp3 = (recorder: any) => { const wav = lamejs.WavHeader.readHeader(recorder.getWAV()) diff --git a/web/app/components/share/chat/index.tsx b/web/app/components/share/chat/index.tsx index 3f7990c964..7cd4aaceaa 100644 --- a/web/app/components/share/chat/index.tsx +++ b/web/app/components/share/chat/index.tsx @@ -660,8 +660,6 @@ const Main: FC = ({ }) }, onThought(thought) { - // console.log(`${thought.id};${thought.thought};${thought.tool};${thought.tool_input}`) - isAgentMode = true const response = responseItem as any if (thought.message_id && !hasSetResponseId) { diff --git a/web/app/components/share/chatbot/hooks/use-conversation.ts b/web/app/components/share/chatbot/hooks/use-conversation.ts index 88a702f029..4e7f27cf50 100644 --- a/web/app/components/share/chatbot/hooks/use-conversation.ts +++ b/web/app/components/share/chatbot/hooks/use-conversation.ts @@ -1,5 +1,6 @@ import { useState } from 'react' import produce from 'immer' +import { useGetState } from 'ahooks' import type { ConversationItem } from '@/models/share' const storageConversationIdKey = 'conversationIdInfo' @@ -8,7 +9,7 @@ type ConversationInfoType = Omit function useConversation() { const [conversationList, setConversationList] = useState([]) const [pinnedConversationList, setPinnedConversationList] = useState([]) - const [currConversationId, doSetCurrConversationId] = useState('-1') + const [currConversationId, doSetCurrConversationId, getCurrConversationId] = useGetState('-1') // when set conversation id, we do not have set appId const setCurrConversationId = (id: string, appId: string, isSetToLocalStroge = true, newConversationName = '') => { doSetCurrConversationId(id) @@ -53,6 +54,7 @@ function useConversation() { pinnedConversationList, setPinnedConversationList, currConversationId, + getCurrConversationId, setCurrConversationId, getConversationIdFromStorage, isNewConversation, diff --git a/web/app/components/share/chatbot/index.tsx b/web/app/components/share/chatbot/index.tsx index 2c6d6f7c82..f3faa08ad0 100644 --- a/web/app/components/share/chatbot/index.tsx +++ b/web/app/components/share/chatbot/index.tsx @@ -5,7 +5,7 @@ import React, { useEffect, useRef, useState } from 'react' import cn from 'classnames' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' -import produce from 'immer' +import produce, { setAutoFreeze } from 'immer' import { useBoolean, useGetState } from 'ahooks' import { checkOrSetAccessToken } from '../utils' import AppUnavailable from '../../base/app-unavailable' @@ -13,8 +13,20 @@ import useConversation from './hooks/use-conversation' import { ToastContext } from '@/app/components/base/toast' import ConfigScene from '@/app/components/share/chatbot/config-scence' import Header from '@/app/components/share/header' -import { fetchAppInfo, fetchAppParams, fetchChatList, fetchConversations, fetchSuggestedQuestions, generationConversationName, sendChatMessage, stopChatMessageResponding, updateFeedback } from '@/service/share' -import type { ConversationItem, SiteInfo } from '@/models/share' +import { + fetchAppInfo, + fetchAppMeta, + fetchAppParams, + fetchChatList, + fetchConversations, + fetchSuggestedQuestions, + generationConversationName, + sendChatMessage, + stopChatMessageResponding, + updateFeedback, +} from '@/service/share' +import { addFileInfos, sortAgentSorts } from '@/app/components/tools/utils' +import type { AppMeta, ConversationItem, SiteInfo } from '@/models/share' import type { PromptConfig, SuggestedQuestionsAfterAnswerConfig } from '@/models/debug' import type { Feedbacktype, IChatItem } from '@/app/components/app/chat/type' import Chat from '@/app/components/app/chat' @@ -29,6 +41,7 @@ import LogoHeader from '@/app/components/base/logo/logo-embeded-chat-header' import LogoAvatar from '@/app/components/base/logo/logo-embeded-chat-avatar' import type { VisionFile, VisionSettings } from '@/types/app' import { Resolution, TransferMethod } from '@/types/app' +import type { Annotation as AnnotationType } from '@/models/log' export type IMainProps = { isInstalledApp?: boolean @@ -56,6 +69,8 @@ const Main: FC = ({ const [plan, setPlan] = useState('basic') // basic/plus/pro const [canReplaceLogo, setCanReplaceLogo] = useState(false) const [customConfig, setCustomConfig] = useState(null) + const [appMeta, setAppMeta] = useState(null) + // Can Use metadata(https://beta.nextjs.org/docs/api-reference/metadata) to set title. But it only works in server side client. useEffect(() => { if (siteInfo?.title) { @@ -66,6 +81,14 @@ const Main: FC = ({ } }, [siteInfo?.title, canReplaceLogo]) + // onData change thought (the produce obj). https://github.com/immerjs/immer/issues/576 + useEffect(() => { + setAutoFreeze(false) + return () => { + setAutoFreeze(true) + } + }, []) + /* * conversation info */ @@ -78,6 +101,7 @@ const Main: FC = ({ pinnedConversationList, setPinnedConversationList, currConversationId, + getCurrConversationId, setCurrConversationId, getConversationIdFromStorage, isNewConversation, @@ -189,14 +213,16 @@ const Main: FC = ({ id: `question-${item.id}`, content: item.query, isAnswer: false, - message_files: item.message_files, + message_files: item.message_files?.filter((file: any) => file.belongs_to === 'user') || [], }) newChatList.push({ id: item.id, content: item.answer, + agent_thoughts: addFileInfos(item.agent_thoughts ? sortAgentSorts(item.agent_thoughts) : item.agent_thoughts, item.message_files), feedback: item.feedback, isAnswer: true, citation: item.retriever_resources, + message_files: item.message_files?.filter((file: any) => file.belongs_to === 'assistant') || [], }) }) setChatList(newChatList) @@ -278,14 +304,15 @@ const Main: FC = ({ }, plan: 'basic', } - : fetchAppInfo(), fetchAllConversations(), fetchAppParams(isInstalledApp, installedAppInfo?.id)]) + : fetchAppInfo(), fetchAllConversations(), fetchAppParams(isInstalledApp, installedAppInfo?.id), fetchAppMeta(isInstalledApp, installedAppInfo?.id)]) } // init useEffect(() => { (async () => { try { - const [appData, conversationData, appParams]: any = await fetchInitData() + const [appData, conversationData, appParams, appMeta]: any = await fetchInitData() + setAppMeta(appMeta) const { app_id: appId, site: siteInfo, plan, can_replace_logo, custom_config }: any = appData setAppId(appId) setPlan(plan) @@ -384,6 +411,7 @@ const Main: FC = ({ const [suggestQuestions, setSuggestQuestions] = useState([]) const [messageTaskId, setMessageTaskId] = useState('') const [hasStopResponded, setHasStopResponded, getHasStopResponded] = useGetState(false) + const [isResponsingConIsCurrCon, setIsResponsingConCurrCon, getIsResponsingConIsCurrCon] = useGetState(true) const [shouldReload, setShouldReload] = useState(false) const [userQuery, setUserQuery] = useState('') const [visionConfig, setVisionConfig] = useState({ @@ -393,6 +421,29 @@ const Main: FC = ({ transfer_methods: [TransferMethod.local_file], }) + const updateCurrentQA = ({ + responseItem, + questionId, + placeholderAnswerId, + questionItem, + }: { + responseItem: IChatItem + questionId: string + placeholderAnswerId: string + questionItem: IChatItem + }) => { + // closesure new list is outdated. + const newListWithAnswer = produce( + getChatList().filter(item => item.id !== responseItem.id && item.id !== placeholderAnswerId), + (draft) => { + if (!draft.find(item => item.id === questionId)) + draft.push({ ...questionItem }) + + draft.push({ ...responseItem }) + }) + setChatList(newListWithAnswer) + } + const handleSend = async (message: string, files?: VisionFile[]) => { if (isResponsing) { notify({ type: 'info', message: t('appDebug.errorMessage.waitForResponse') }) @@ -440,14 +491,20 @@ const Main: FC = ({ const newList = [...getChatList(), questionItem, placeholderAnswerItem] setChatList(newList) + let isAgentMode = false + // answer const responseItem: IChatItem = { id: `${Date.now()}`, content: '', + agent_thoughts: [], + message_files: [], isAnswer: true, } + let hasSetResponseId = false - let tempNewConversationId = '' + const prevTempNewConversationId = getCurrConversationId() || '-1' + let tempNewConversationId = prevTempNewConversationId setHasStopResponded(false) setResponsingTrue() @@ -457,22 +514,34 @@ const Main: FC = ({ setAbortController(abortController) }, onData: (message: string, isFirstMessage: boolean, { conversationId: newConversationId, messageId, taskId }: any) => { - responseItem.content = responseItem.content + message - responseItem.id = messageId + if (!isAgentMode) { + responseItem.content = responseItem.content + message + } + else { + const lastThought = responseItem.agent_thoughts?.[responseItem.agent_thoughts?.length - 1] + if (lastThought) + lastThought.thought = lastThought.thought + message // need immer setAutoFreeze + } + if (messageId && !hasSetResponseId) { + responseItem.id = messageId + hasSetResponseId = true + } + if (isFirstMessage && newConversationId) tempNewConversationId = newConversationId setMessageTaskId(taskId) - // closesure new list is outdated. - const newListWithAnswer = produce( - getChatList().filter(item => item.id !== responseItem.id && item.id !== placeholderAnswerId), - (draft) => { - if (!draft.find(item => item.id === questionId)) - draft.push({ ...questionItem }) - - draft.push({ ...responseItem }) - }) - setChatList(newListWithAnswer) + // has switched to other conversation + if (prevTempNewConversationId !== getCurrConversationId()) { + setIsResponsingConCurrCon(false) + return + } + updateCurrentQA({ + responseItem, + questionId, + placeholderAnswerId, + questionItem, + }) }, async onCompleted(hasError?: boolean) { if (hasError) @@ -498,20 +567,105 @@ const Main: FC = ({ } setResponsingFalse() }, - onMessageReplace: (messageReplace) => { - setChatList(produce( - getChatList(), - (draft) => { - const current = draft.find(item => item.id === messageReplace.id) + onFile(file) { + const lastThought = responseItem.agent_thoughts?.[responseItem.agent_thoughts?.length - 1] + if (lastThought) + lastThought.message_files = [...(lastThought as any).message_files, { ...file }] - if (current) - current.content = messageReplace.answer - }, - )) + updateCurrentQA({ + responseItem, + questionId, + placeholderAnswerId, + questionItem, + }) }, - onError(errorMessage, errorCode) { - if (['provider_not_initialize', 'completion_request_error'].includes(errorCode as string)) - setShouldReload(true) + onThought(thought) { + isAgentMode = true + const response = responseItem as any + if (thought.message_id && !hasSetResponseId) { + response.id = thought.message_id + hasSetResponseId = true + } + // responseItem.id = thought.message_id; + if (response.agent_thoughts.length === 0) { + response.agent_thoughts.push(thought) + } + else { + const lastThought = response.agent_thoughts[response.agent_thoughts.length - 1] + // thought changed but still the same thought, so update. + if (lastThought.id === thought.id) { + thought.thought = lastThought.thought + thought.message_files = lastThought.message_files + responseItem.agent_thoughts![response.agent_thoughts.length - 1] = thought + } + else { + responseItem.agent_thoughts!.push(thought) + } + } + // has switched to other conversation + if (prevTempNewConversationId !== getCurrConversationId()) { + setIsResponsingConCurrCon(false) + return false + } + + updateCurrentQA({ + responseItem, + questionId, + placeholderAnswerId, + questionItem, + }) + }, + onMessageEnd: (messageEnd) => { + if (messageEnd.metadata?.annotation_reply) { + responseItem.id = messageEnd.id + responseItem.annotation = ({ + id: messageEnd.metadata.annotation_reply.id, + authorName: messageEnd.metadata.annotation_reply.account.name, + } as AnnotationType) + const newListWithAnswer = produce( + getChatList().filter(item => item.id !== responseItem.id && item.id !== placeholderAnswerId), + (draft) => { + if (!draft.find(item => item.id === questionId)) + draft.push({ ...questionItem }) + + draft.push({ + ...responseItem, + }) + }) + setChatList(newListWithAnswer) + return + } + // not support show citation + // responseItem.citation = messageEnd.retriever_resources + if (!isInstalledApp) + return + const newListWithAnswer = produce( + getChatList().filter(item => item.id !== responseItem.id && item.id !== placeholderAnswerId), + (draft) => { + if (!draft.find(item => item.id === questionId)) + draft.push({ ...questionItem }) + + draft.push({ ...responseItem }) + }) + setChatList(newListWithAnswer) + }, + onMessageReplace: (messageReplace) => { + if (isInstalledApp) { + responseItem.content = messageReplace.answer + } + else { + setChatList(produce( + getChatList(), + (draft) => { + const current = draft.find(item => item.id === messageReplace.id) + + if (current) + current.content = messageReplace.answer + }, + )) + } + }, + onError() { setResponsingFalse() // role back placeholder answer setChatList(produce(getChatList(), (draft) => { @@ -629,7 +783,7 @@ const Main: FC = ({ isHideFeedbackEdit onFeedback={handleFeedback} isResponsing={isResponsing} - canStopResponsing={!!messageTaskId} + canStopResponsing={!!messageTaskId && isResponsingConIsCurrCon} abortResponsing={async () => { await stopChatMessageResponding(appId, messageTaskId, isInstalledApp, installedAppInfo?.id) setHasStopResponded(true) @@ -645,6 +799,7 @@ const Main: FC = ({ isShowCitation={citationConfig?.enabled && isInstalledApp} answerIcon={} visionConfig={visionConfig} + allToolIcons={appMeta?.tool_icons || {}} /> )