From 07de36ec86cafaca74d4804f7dd80d5985949224 Mon Sep 17 00:00:00 2001 From: balibabu Date: Tue, 3 Sep 2024 19:47:50 +0800 Subject: [PATCH] feat: Supports pronunciation while outputting text #2088 (#2227) ### What problem does this PR solve? feat: Supports pronunciation while outputting text #2088 ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- .../components/message-item/group-button.tsx | 4 +- web/src/components/message-item/hooks.ts | 11 ++++- web/src/components/message-item/index.tsx | 1 + web/src/hooks/file-manager-hooks.ts | 16 +++++--- web/src/hooks/logic-hooks.ts | 1 + web/src/interfaces/database/chat.ts | 2 + web/src/utils/common-util.ts | 41 +++++++++++++++++++ 7 files changed, 68 insertions(+), 8 deletions(-) diff --git a/web/src/components/message-item/group-button.tsx b/web/src/components/message-item/group-button.tsx index cbbb8abc2..2aac4f3b9 100644 --- a/web/src/components/message-item/group-button.tsx +++ b/web/src/components/message-item/group-button.tsx @@ -22,12 +22,14 @@ interface IProps { content: string; prompt?: string; showLikeButton: boolean; + audioBinary?: string; } export const AssistantGroupButton = ({ messageId, content, prompt, + audioBinary, showLikeButton, }: IProps) => { const { visible, hideModal, showModal, onFeedbackOk, loading } = @@ -38,7 +40,7 @@ export const AssistantGroupButton = ({ showModal: showPromptModal, } = useSetModalState(); const { t } = useTranslation(); - const { handleRead, ref, isPlaying } = useSpeech(content); + const { handleRead, ref, isPlaying } = useSpeech(content, audioBinary); const handleLike = useCallback(() => { onFeedbackOk({ thumbup: true }); diff --git a/web/src/components/message-item/hooks.ts b/web/src/components/message-item/hooks.ts index 1142509f4..adaab0887 100644 --- a/web/src/components/message-item/hooks.ts +++ b/web/src/components/message-item/hooks.ts @@ -52,7 +52,7 @@ export const useRemoveMessage = ( return { onRemoveMessage, loading }; }; -export const useSpeech = (content: string) => { +export const useSpeech = (content: string, audioBinary?: string) => { const ref = useRef(null); const { read } = useSpeechWithSse(); const player = useRef(); @@ -94,6 +94,15 @@ export const useSpeech = (content: string) => { } }, [setIsPlaying, speech, isPlaying, pause]); + // useEffect(() => { + // if (audioBinary) { + // const units = hexStringToUint8Array(audioBinary); + // if (units) { + // player.current?.feed(units); + // } + // } + // }, [audioBinary]); + useEffect(() => { initialize(); }, [initialize]); diff --git a/web/src/components/message-item/index.tsx b/web/src/components/message-item/index.tsx index c1974f85f..01be68ab4 100644 --- a/web/src/components/message-item/index.tsx +++ b/web/src/components/message-item/index.tsx @@ -131,6 +131,7 @@ const MessageItem = ({ content={item.content} prompt={item.prompt} showLikeButton={showLikeButton} + audioBinary={item.audio_binary} > ) ) : ( diff --git a/web/src/hooks/file-manager-hooks.ts b/web/src/hooks/file-manager-hooks.ts index 1918f0bd7..a3c0e7fe9 100644 --- a/web/src/hooks/file-manager-hooks.ts +++ b/web/src/hooks/file-manager-hooks.ts @@ -207,13 +207,17 @@ export const useUploadFile = () => { formData.append('file', file); formData.append('path', pathList[index]); }); - const { data } = await fileManagerService.uploadFile(formData); - if (data.retcode === 0) { - message.success(t('message.uploaded')); - setPaginationParams(1); - queryClient.invalidateQueries({ queryKey: ['fetchFileList'] }); + try { + const { data } = await fileManagerService.uploadFile(formData); + if (data.retcode === 0) { + message.success(t('message.uploaded')); + setPaginationParams(1); + queryClient.invalidateQueries({ queryKey: ['fetchFileList'] }); + } + return data.retcode; + } catch (error) { + console.log('🚀 ~ useUploadFile ~ error:', error); } - return data.retcode; }, }); diff --git a/web/src/hooks/logic-hooks.ts b/web/src/hooks/logic-hooks.ts index 80adf5188..88a3b59ad 100644 --- a/web/src/hooks/logic-hooks.ts +++ b/web/src/hooks/logic-hooks.ts @@ -433,6 +433,7 @@ export const useSelectDerivedMessages = () => { role: MessageType.Assistant, }), prompt: answer.prompt, + audio_binary: answer.audio_binary, }, ]; }); diff --git a/web/src/interfaces/database/chat.ts b/web/src/interfaces/database/chat.ts index 1ddcf8ca5..9ddebae90 100644 --- a/web/src/interfaces/database/chat.ts +++ b/web/src/interfaces/database/chat.ts @@ -70,6 +70,7 @@ export interface Message { doc_ids?: string[]; prompt?: string; id?: string; + audio_binary?: string; } export interface IReference { @@ -84,6 +85,7 @@ export interface IAnswer { conversationId?: string; prompt?: string; id?: string; + audio_binary?: string; } export interface Docagg { diff --git a/web/src/utils/common-util.ts b/web/src/utils/common-util.ts index f3d0aef63..fb41adead 100644 --- a/web/src/utils/common-util.ts +++ b/web/src/utils/common-util.ts @@ -72,3 +72,44 @@ export const toFixed = (value: unknown, fixed = 2) => { } return value; }; + +export const stringToUint8Array = (str: string) => { + // const byteString = str.replace(/b'|'/g, ''); + const byteString = str.slice(2, -1); + + const uint8Array = new Uint8Array(byteString.length); + for (let i = 0; i < byteString.length; i++) { + uint8Array[i] = byteString.charCodeAt(i); + } + + return uint8Array; +}; + +export const hexStringToUint8Array = (hex: string) => { + const arr = hex.match(/[\da-f]{2}/gi); + if (Array.isArray(arr)) { + return new Uint8Array( + arr.map(function (h) { + return parseInt(h, 16); + }), + ); + } +}; + +export function hexToArrayBuffer(input: string) { + if (typeof input !== 'string') { + throw new TypeError('Expected input to be a string'); + } + + if (input.length % 2 !== 0) { + throw new RangeError('Expected string to be an even number of characters'); + } + + const view = new Uint8Array(input.length / 2); + + for (let i = 0; i < input.length; i += 2) { + view[i / 2] = parseInt(input.substring(i, i + 2), 16); + } + + return view.buffer; +}