feat: Supports pronunciation while outputting text #2088 (#2227)

### What problem does this PR solve?

feat: Supports pronunciation while outputting text #2088

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

web/src/components/message-item/group-button.tsx

@@ -22,12 +22,14 @@ interface IProps {
   content: string;
   prompt?: string;
   showLikeButton: boolean;
+  audioBinary?: string;
 }
 
 export const AssistantGroupButton = ({
   messageId,
   content,
   prompt,
+  audioBinary,
   showLikeButton,
 }: IProps) => {
   const { visible, hideModal, showModal, onFeedbackOk, loading } =
@@ -38,7 +40,7 @@ export const AssistantGroupButton = ({
     showModal: showPromptModal,
   } = useSetModalState();
   const { t } = useTranslation();
-  const { handleRead, ref, isPlaying } = useSpeech(content);
+  const { handleRead, ref, isPlaying } = useSpeech(content, audioBinary);
 
   const handleLike = useCallback(() => {
     onFeedbackOk({ thumbup: true });

web/src/components/message-item/hooks.ts

@@ -52,7 +52,7 @@ export const useRemoveMessage = (
   return { onRemoveMessage, loading };
 };
 
-export const useSpeech = (content: string) => {
+export const useSpeech = (content: string, audioBinary?: string) => {
   const ref = useRef<HTMLAudioElement>(null);
   const { read } = useSpeechWithSse();
   const player = useRef<SpeechPlayer>();
@@ -94,6 +94,15 @@ export const useSpeech = (content: string) => {
     }
   }, [setIsPlaying, speech, isPlaying, pause]);
 
+  // useEffect(() => {
+  //   if (audioBinary) {
+  //     const units = hexStringToUint8Array(audioBinary);
+  //     if (units) {
+  //       player.current?.feed(units);
+  //     }
+  //   }
+  // }, [audioBinary]);
+
   useEffect(() => {
     initialize();
   }, [initialize]);

web/src/components/message-item/index.tsx

@@ -131,6 +131,7 @@ const MessageItem = ({
                     content={item.content}
                     prompt={item.prompt}
                     showLikeButton={showLikeButton}
+                    audioBinary={item.audio_binary}
                   ></AssistantGroupButton>
                 )
               ) : (

web/src/hooks/file-manager-hooks.ts

@@ -207,6 +207,7 @@ export const useUploadFile = () => {
         formData.append('file', file);
         formData.append('path', pathList[index]);
       });
+      try {
         const { data } = await fileManagerService.uploadFile(formData);
         if (data.retcode === 0) {
           message.success(t('message.uploaded'));
@@ -214,6 +215,9 @@ export const useUploadFile = () => {
           queryClient.invalidateQueries({ queryKey: ['fetchFileList'] });
         }
         return data.retcode;
+      } catch (error) {
+        console.log('🚀 ~ useUploadFile ~ error:', error);
+      }
     },
   });
 

web/src/hooks/logic-hooks.ts

@@ -433,6 +433,7 @@ export const useSelectDerivedMessages = () => {
             role: MessageType.Assistant,
           }),
           prompt: answer.prompt,
+          audio_binary: answer.audio_binary,
         },
       ];
     });

web/src/interfaces/database/chat.ts

@@ -70,6 +70,7 @@ export interface Message {
   doc_ids?: string[];
   prompt?: string;
   id?: string;
+  audio_binary?: string;
 }
 
 export interface IReference {
@@ -84,6 +85,7 @@ export interface IAnswer {
   conversationId?: string;
   prompt?: string;
   id?: string;
+  audio_binary?: string;
 }
 
 export interface Docagg {

web/src/utils/common-util.ts

@@ -72,3 +72,44 @@ export const toFixed = (value: unknown, fixed = 2) => {
   }
   return value;
 };
+
+export const stringToUint8Array = (str: string) => {
+  // const byteString = str.replace(/b'|'/g, '');
+  const byteString = str.slice(2, -1);
+
+  const uint8Array = new Uint8Array(byteString.length);
+  for (let i = 0; i < byteString.length; i++) {
+    uint8Array[i] = byteString.charCodeAt(i);
+  }
+
+  return uint8Array;
+};
+
+export const hexStringToUint8Array = (hex: string) => {
+  const arr = hex.match(/[\da-f]{2}/gi);
+  if (Array.isArray(arr)) {
+    return new Uint8Array(
+      arr.map(function (h) {
+        return parseInt(h, 16);
+      }),
+    );
+  }
+};
+
+export function hexToArrayBuffer(input: string) {
+  if (typeof input !== 'string') {
+    throw new TypeError('Expected input to be a string');
+  }
+
+  if (input.length % 2 !== 0) {
+    throw new RangeError('Expected string to be an even number of characters');
+  }
+
+  const view = new Uint8Array(input.length / 2);
+
+  for (let i = 0; i < input.length; i += 2) {
+    view[i / 2] = parseInt(input.substring(i, i + 2), 16);
+  }
+
+  return view.buffer;
+}