fix: chatbot support agent (#2201)

web/app/components/base/voice-input/utils.ts

@@ -3,9 +3,11 @@ import MPEGMode from 'lamejs/src/js/MPEGMode'
 import Lame from 'lamejs/src/js/Lame'
 import BitStream from 'lamejs/src/js/BitStream'
 
-(window as any).MPEGMode = MPEGMode
+if (globalThis) {
-;(window as any).Lame = Lame
+  (globalThis as any).MPEGMode = MPEGMode
-;(window as any).BitStream = BitStream
+  ;(globalThis as any).Lame = Lame
+  ;(globalThis as any).BitStream = BitStream
+}
 
 export const convertToMp3 = (recorder: any) => {
   const wav = lamejs.WavHeader.readHeader(recorder.getWAV())

web/app/components/share/chat/index.tsx

@@ -660,8 +660,6 @@ const Main: FC<IMainProps> = ({
         })
       },
       onThought(thought) {
-        // console.log(`${thought.id};${thought.thought};${thought.tool};${thought.tool_input}`)
-
         isAgentMode = true
         const response = responseItem as any
         if (thought.message_id && !hasSetResponseId) {

web/app/components/share/chatbot/hooks/use-conversation.ts

@@ -1,5 +1,6 @@
 import { useState } from 'react'
 import produce from 'immer'
+import { useGetState } from 'ahooks'
 import type { ConversationItem } from '@/models/share'
 
 const storageConversationIdKey = 'conversationIdInfo'
@@ -8,7 +9,7 @@ type ConversationInfoType = Omit<ConversationItem, 'inputs' | 'id'>
 function useConversation() {
   const [conversationList, setConversationList] = useState<ConversationItem[]>([])
   const [pinnedConversationList, setPinnedConversationList] = useState<ConversationItem[]>([])
-  const [currConversationId, doSetCurrConversationId] = useState<string>('-1')
+  const [currConversationId, doSetCurrConversationId, getCurrConversationId] = useGetState<string>('-1')
   // when set conversation id, we do not have set appId
   const setCurrConversationId = (id: string, appId: string, isSetToLocalStroge = true, newConversationName = '') => {
     doSetCurrConversationId(id)
@@ -53,6 +54,7 @@ function useConversation() {
     pinnedConversationList,
     setPinnedConversationList,
     currConversationId,
+    getCurrConversationId,
     setCurrConversationId,
     getConversationIdFromStorage,
     isNewConversation,

web/app/components/share/chatbot/index.tsx

@@ -5,7 +5,7 @@ import React, { useEffect, useRef, useState } from 'react'
 import cn from 'classnames'
 import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
-import produce from 'immer'
+import produce, { setAutoFreeze } from 'immer'
 import { useBoolean, useGetState } from 'ahooks'
 import { checkOrSetAccessToken } from '../utils'
 import AppUnavailable from '../../base/app-unavailable'
@@ -13,8 +13,20 @@ import useConversation from './hooks/use-conversation'
 import { ToastContext } from '@/app/components/base/toast'
 import ConfigScene from '@/app/components/share/chatbot/config-scence'
 import Header from '@/app/components/share/header'
-import { fetchAppInfo, fetchAppParams, fetchChatList, fetchConversations, fetchSuggestedQuestions, generationConversationName, sendChatMessage, stopChatMessageResponding, updateFeedback } from '@/service/share'
+import {
-import type { ConversationItem, SiteInfo } from '@/models/share'
+  fetchAppInfo,
+  fetchAppMeta,
+  fetchAppParams,
+  fetchChatList,
+  fetchConversations,
+  fetchSuggestedQuestions,
+  generationConversationName,
+  sendChatMessage,
+  stopChatMessageResponding,
+  updateFeedback,
+} from '@/service/share'
+import { addFileInfos, sortAgentSorts } from '@/app/components/tools/utils'
+import type { AppMeta, ConversationItem, SiteInfo } from '@/models/share'
 import type { PromptConfig, SuggestedQuestionsAfterAnswerConfig } from '@/models/debug'
 import type { Feedbacktype, IChatItem } from '@/app/components/app/chat/type'
 import Chat from '@/app/components/app/chat'
@@ -29,6 +41,7 @@ import LogoHeader from '@/app/components/base/logo/logo-embeded-chat-header'
 import LogoAvatar from '@/app/components/base/logo/logo-embeded-chat-avatar'
 import type { VisionFile, VisionSettings } from '@/types/app'
 import { Resolution, TransferMethod } from '@/types/app'
+import type { Annotation as AnnotationType } from '@/models/log'
 
 export type IMainProps = {
   isInstalledApp?: boolean
@@ -56,6 +69,8 @@ const Main: FC<IMainProps> = ({
   const [plan, setPlan] = useState<string>('basic') // basic/plus/pro
   const [canReplaceLogo, setCanReplaceLogo] = useState<boolean>(false)
   const [customConfig, setCustomConfig] = useState<any>(null)
+  const [appMeta, setAppMeta] = useState<AppMeta | null>(null)
+
   // Can Use metadata(https://beta.nextjs.org/docs/api-reference/metadata) to set title. But it only works in server side client.
   useEffect(() => {
     if (siteInfo?.title) {
@@ -66,6 +81,14 @@ const Main: FC<IMainProps> = ({
     }
   }, [siteInfo?.title, canReplaceLogo])
 
+  // onData change thought (the produce obj). https://github.com/immerjs/immer/issues/576
+  useEffect(() => {
+    setAutoFreeze(false)
+    return () => {
+      setAutoFreeze(true)
+    }
+  }, [])
+
   /*
   * conversation info
   */
@@ -78,6 +101,7 @@ const Main: FC<IMainProps> = ({
     pinnedConversationList,
     setPinnedConversationList,
     currConversationId,
+    getCurrConversationId,
     setCurrConversationId,
     getConversationIdFromStorage,
     isNewConversation,
@@ -189,14 +213,16 @@ const Main: FC<IMainProps> = ({
             id: `question-${item.id}`,
             content: item.query,
             isAnswer: false,
-            message_files: item.message_files,
+            message_files: item.message_files?.filter((file: any) => file.belongs_to === 'user') || [],
           })
           newChatList.push({
             id: item.id,
             content: item.answer,
+            agent_thoughts: addFileInfos(item.agent_thoughts ? sortAgentSorts(item.agent_thoughts) : item.agent_thoughts, item.message_files),
             feedback: item.feedback,
             isAnswer: true,
             citation: item.retriever_resources,
+            message_files: item.message_files?.filter((file: any) => file.belongs_to === 'assistant') || [],
           })
         })
         setChatList(newChatList)
@@ -278,14 +304,15 @@ const Main: FC<IMainProps> = ({
         },
         plan: 'basic',
       }
-      : fetchAppInfo(), fetchAllConversations(), fetchAppParams(isInstalledApp, installedAppInfo?.id)])
+      : fetchAppInfo(), fetchAllConversations(), fetchAppParams(isInstalledApp, installedAppInfo?.id), fetchAppMeta(isInstalledApp, installedAppInfo?.id)])
   }
 
   // init
   useEffect(() => {
     (async () => {
       try {
-        const [appData, conversationData, appParams]: any = await fetchInitData()
+        const [appData, conversationData, appParams, appMeta]: any = await fetchInitData()
+        setAppMeta(appMeta)
         const { app_id: appId, site: siteInfo, plan, can_replace_logo, custom_config }: any = appData
         setAppId(appId)
         setPlan(plan)
@@ -384,6 +411,7 @@ const Main: FC<IMainProps> = ({
   const [suggestQuestions, setSuggestQuestions] = useState<string[]>([])
   const [messageTaskId, setMessageTaskId] = useState('')
   const [hasStopResponded, setHasStopResponded, getHasStopResponded] = useGetState(false)
+  const [isResponsingConIsCurrCon, setIsResponsingConCurrCon, getIsResponsingConIsCurrCon] = useGetState(true)
   const [shouldReload, setShouldReload] = useState(false)
   const [userQuery, setUserQuery] = useState('')
   const [visionConfig, setVisionConfig] = useState<VisionSettings>({
@@ -393,6 +421,29 @@ const Main: FC<IMainProps> = ({
     transfer_methods: [TransferMethod.local_file],
   })
 
+  const updateCurrentQA = ({
+    responseItem,
+    questionId,
+    placeholderAnswerId,
+    questionItem,
+  }: {
+    responseItem: IChatItem
+    questionId: string
+    placeholderAnswerId: string
+    questionItem: IChatItem
+  }) => {
+    // closesure new list is outdated.
+    const newListWithAnswer = produce(
+      getChatList().filter(item => item.id !== responseItem.id && item.id !== placeholderAnswerId),
+      (draft) => {
+        if (!draft.find(item => item.id === questionId))
+          draft.push({ ...questionItem })
+
+        draft.push({ ...responseItem })
+      })
+    setChatList(newListWithAnswer)
+  }
+
   const handleSend = async (message: string, files?: VisionFile[]) => {
     if (isResponsing) {
       notify({ type: 'info', message: t('appDebug.errorMessage.waitForResponse') })
@@ -440,14 +491,20 @@ const Main: FC<IMainProps> = ({
     const newList = [...getChatList(), questionItem, placeholderAnswerItem]
     setChatList(newList)
 
+    let isAgentMode = false
+
     // answer
     const responseItem: IChatItem = {
       id: `${Date.now()}`,
       content: '',
+      agent_thoughts: [],
+      message_files: [],
       isAnswer: true,
     }
+    let hasSetResponseId = false
 
-    let tempNewConversationId = ''
+    const prevTempNewConversationId = getCurrConversationId() || '-1'
+    let tempNewConversationId = prevTempNewConversationId
 
     setHasStopResponded(false)
     setResponsingTrue()
@@ -457,22 +514,34 @@ const Main: FC<IMainProps> = ({
         setAbortController(abortController)
       },
       onData: (message: string, isFirstMessage: boolean, { conversationId: newConversationId, messageId, taskId }: any) => {
-        responseItem.content = responseItem.content + message
+        if (!isAgentMode) {
-        responseItem.id = messageId
+          responseItem.content = responseItem.content + message
+        }
+        else {
+          const lastThought = responseItem.agent_thoughts?.[responseItem.agent_thoughts?.length - 1]
+          if (lastThought)
+            lastThought.thought = lastThought.thought + message // need immer setAutoFreeze
+        }
+        if (messageId && !hasSetResponseId) {
+          responseItem.id = messageId
+          hasSetResponseId = true
+        }
+
         if (isFirstMessage && newConversationId)
           tempNewConversationId = newConversationId
 
         setMessageTaskId(taskId)
-        // closesure new list is outdated.
+        // has switched to other conversation
-        const newListWithAnswer = produce(
+        if (prevTempNewConversationId !== getCurrConversationId()) {
-          getChatList().filter(item => item.id !== responseItem.id && item.id !== placeholderAnswerId),
+          setIsResponsingConCurrCon(false)
-          (draft) => {
+          return
-            if (!draft.find(item => item.id === questionId))
+        }
-              draft.push({ ...questionItem })
+        updateCurrentQA({
-
+          responseItem,
-            draft.push({ ...responseItem })
+          questionId,
-          })
+          placeholderAnswerId,
-        setChatList(newListWithAnswer)
+          questionItem,
+        })
       },
       async onCompleted(hasError?: boolean) {
         if (hasError)
@@ -498,20 +567,105 @@ const Main: FC<IMainProps> = ({
         }
         setResponsingFalse()
       },
-      onMessageReplace: (messageReplace) => {
+      onFile(file) {
-        setChatList(produce(
+        const lastThought = responseItem.agent_thoughts?.[responseItem.agent_thoughts?.length - 1]
-          getChatList(),
+        if (lastThought)
-          (draft) => {
+          lastThought.message_files = [...(lastThought as any).message_files, { ...file }]
-            const current = draft.find(item => item.id === messageReplace.id)
 
-            if (current)
+        updateCurrentQA({
-              current.content = messageReplace.answer
+          responseItem,
-          },
+          questionId,
-        ))
+          placeholderAnswerId,
+          questionItem,
+        })
       },
-      onError(errorMessage, errorCode) {
+      onThought(thought) {
-        if (['provider_not_initialize', 'completion_request_error'].includes(errorCode as string))
+        isAgentMode = true
-          setShouldReload(true)
+        const response = responseItem as any
+        if (thought.message_id && !hasSetResponseId) {
+          response.id = thought.message_id
+          hasSetResponseId = true
+        }
+        // responseItem.id = thought.message_id;
+        if (response.agent_thoughts.length === 0) {
+          response.agent_thoughts.push(thought)
+        }
+        else {
+          const lastThought = response.agent_thoughts[response.agent_thoughts.length - 1]
+          // thought changed but still the same thought, so update.
+          if (lastThought.id === thought.id) {
+            thought.thought = lastThought.thought
+            thought.message_files = lastThought.message_files
+            responseItem.agent_thoughts![response.agent_thoughts.length - 1] = thought
+          }
+          else {
+            responseItem.agent_thoughts!.push(thought)
+          }
+        }
+        // has switched to other conversation
+        if (prevTempNewConversationId !== getCurrConversationId()) {
+          setIsResponsingConCurrCon(false)
+          return false
+        }
+
+        updateCurrentQA({
+          responseItem,
+          questionId,
+          placeholderAnswerId,
+          questionItem,
+        })
+      },
+      onMessageEnd: (messageEnd) => {
+        if (messageEnd.metadata?.annotation_reply) {
+          responseItem.id = messageEnd.id
+          responseItem.annotation = ({
+            id: messageEnd.metadata.annotation_reply.id,
+            authorName: messageEnd.metadata.annotation_reply.account.name,
+          } as AnnotationType)
+          const newListWithAnswer = produce(
+            getChatList().filter(item => item.id !== responseItem.id && item.id !== placeholderAnswerId),
+            (draft) => {
+              if (!draft.find(item => item.id === questionId))
+                draft.push({ ...questionItem })
+
+              draft.push({
+                ...responseItem,
+              })
+            })
+          setChatList(newListWithAnswer)
+          return
+        }
+        // not support show citation
+        // responseItem.citation = messageEnd.retriever_resources
+        if (!isInstalledApp)
+          return
+        const newListWithAnswer = produce(
+          getChatList().filter(item => item.id !== responseItem.id && item.id !== placeholderAnswerId),
+          (draft) => {
+            if (!draft.find(item => item.id === questionId))
+              draft.push({ ...questionItem })
+
+            draft.push({ ...responseItem })
+          })
+        setChatList(newListWithAnswer)
+      },
+      onMessageReplace: (messageReplace) => {
+        if (isInstalledApp) {
+          responseItem.content = messageReplace.answer
+        }
+        else {
+          setChatList(produce(
+            getChatList(),
+            (draft) => {
+              const current = draft.find(item => item.id === messageReplace.id)
+
+              if (current)
+                current.content = messageReplace.answer
+            },
+          ))
+        }
+      },
+      onError() {
         setResponsingFalse()
         // role back placeholder answer
         setChatList(produce(getChatList(), (draft) => {
@@ -629,7 +783,7 @@ const Main: FC<IMainProps> = ({
                     isHideFeedbackEdit
                     onFeedback={handleFeedback}
                     isResponsing={isResponsing}
-                    canStopResponsing={!!messageTaskId}
+                    canStopResponsing={!!messageTaskId && isResponsingConIsCurrCon}
                     abortResponsing={async () => {
                       await stopChatMessageResponding(appId, messageTaskId, isInstalledApp, installedAppInfo?.id)
                       setHasStopResponded(true)
@@ -645,6 +799,7 @@ const Main: FC<IMainProps> = ({
                     isShowCitation={citationConfig?.enabled && isInstalledApp}
                     answerIcon={<LogoAvatar className='relative shrink-0' />}
                     visionConfig={visionConfig}
+                    allToolIcons={appMeta?.tool_icons || {}}
                   />
                 </div>
               </div>)