From 2f06f0c4339188e62d5566a8ffc20b769835a829 Mon Sep 17 00:00:00 2001 From: Li Xin Date: Sat, 19 Apr 2025 22:11:57 +0800 Subject: [PATCH] feat: enable podcast --- web/src/app/_components/message-list-view.tsx | 79 ++++++++++++++++++- web/src/app/_components/research-block.tsx | 8 +- .../app/_components/research-report-block.tsx | 36 ++++----- web/src/core/api/chat.ts | 4 +- web/src/core/api/podcast.ts | 19 +++++ web/src/core/messages/types.ts | 8 +- web/src/core/store/store.ts | 51 ++++++++++-- 7 files changed, 172 insertions(+), 33 deletions(-) create mode 100644 web/src/core/api/podcast.ts diff --git a/web/src/app/_components/message-list-view.tsx b/web/src/app/_components/message-list-view.tsx index 95edc71..21fecf8 100644 --- a/web/src/app/_components/message-list-view.tsx +++ b/web/src/app/_components/message-list-view.tsx @@ -1,9 +1,14 @@ // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates // SPDX-License-Identifier: MIT +import { + DownloadOutlined, + SoundOutlined, + LoadingOutlined, +} from "@ant-design/icons"; import { parse } from "best-effort-json-parser"; import { motion } from "framer-motion"; -import { useCallback, useMemo } from "react"; +import { useCallback, useMemo, useState } from "react"; import { Button } from "~/components/ui/button"; import { @@ -13,6 +18,11 @@ import { CardHeader, CardTitle, } from "~/components/ui/card"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "~/components/ui/tooltip"; import type { Message, Option } from "~/core/messages"; import { openResearch, @@ -114,6 +124,7 @@ function MessageListItem({ message.role === "user" || message.agent === "coordinator" || message.agent === "planner" || + message.agent === "podcast" || startOfResearch ) { let content: React.ReactNode; @@ -128,6 +139,12 @@ function MessageListItem({ /> ); + } else if (message.agent === "podcast") { + content = ( +
+ +
+ ); } else if (startOfResearch) { content = (
@@ -348,3 +365,63 @@ function PlanCard({ ); } + +function PodcastCard({ + className, + message, +}: { + className?: string; + message: Message; +}) { + const data = useMemo(() => { + return parse(message.content ?? ""); + }, [message.content]); + const title = useMemo(() => data?.title, [data]); + const audioUrl = useMemo(() => data?.audioUrl, [data]); + const isGenerating = useMemo(() => { + return message.isStreaming; + }, [message.isStreaming]); + const [isPlaying, setIsPlaying] = useState(false); + return ( + + +
+
+ {isGenerating ? : } + + {isGenerating + ? "Generating podcast..." + : isPlaying + ? "Now playing podcast..." + : "Podcast"} + +
+
+ + + + + Download podcast + +
+
+ +
+ {title} +
+ + + + + + ); +} diff --git a/web/src/app/_components/research-block.tsx b/web/src/app/_components/research-block.tsx index 2b742ec..d74e687 100644 --- a/web/src/app/_components/research-block.tsx +++ b/web/src/app/_components/research-block.tsx @@ -80,8 +80,12 @@ export function ResearchBlock({
- {reportId && ( - + {reportId && researchId && ( + )} diff --git a/web/src/app/_components/research-report-block.tsx b/web/src/app/_components/research-report-block.tsx index 7a99a7f..01b90f9 100644 --- a/web/src/app/_components/research-report-block.tsx +++ b/web/src/app/_components/research-report-block.tsx @@ -1,7 +1,7 @@ // Copyright (c) 2025 Bytedance Ltd. and/or its affiliates // SPDX-License-Identifier: MIT -import { PauseCircleOutlined, SoundOutlined } from "@ant-design/icons"; +import { SoundOutlined } from "@ant-design/icons"; import { useCallback, useRef, useState } from "react"; import { Button } from "~/components/ui/button"; @@ -10,7 +10,7 @@ import { TooltipTrigger, TooltipContent, } from "~/components/ui/tooltip"; -import { useMessage } from "~/core/store"; +import { listenToPodcast, useMessage } from "~/core/store"; import { cn } from "~/lib/utils"; import { LoadingAnimation } from "./loading-animation"; @@ -18,29 +18,23 @@ import { Markdown } from "./markdown"; export function ResearchReportBlock({ className, + researchId, messageId, }: { className?: string; + researchId: string; messageId: string; }) { const message = useMessage(messageId); const contentRef = useRef(null); - const [isTTS, setIsTTS] = useState(false); - const handleTTS = useCallback(() => { - if (contentRef.current) { - if (isTTS) { - window.speechSynthesis.cancel(); - setIsTTS(false); - } else { - const text = contentRef.current.textContent; - if (text) { - const utterance = new SpeechSynthesisUtterance(text); - setIsTTS(true); - window.speechSynthesis.speak(utterance); - } - } + const [isGenerated, setGenerated] = useState(false); + const handleListenToReport = useCallback(async () => { + if (isGenerated) { + return; } - }, [isTTS]); + setGenerated(true); + await listenToPodcast(researchId); + }, [isGenerated, researchId]); return (
{ - handleTTS(); + void handleListenToReport(); }} > - {isTTS ? : } + -

{isTTS ? "Pause" : "Listen to the report"}

+

+ {isGenerated ? "The podcast is generated" : "Generate podcast"} +

)} diff --git a/web/src/core/api/chat.ts b/web/src/core/api/chat.ts index 5438465..55941ec 100644 --- a/web/src/core/api/chat.ts +++ b/web/src/core/api/chat.ts @@ -64,9 +64,9 @@ async function* chatStreamMock( const [, event] = eventRaw.split("event: ", 2) as [string, string]; const [, data] = dataRaw.split("data: ", 2) as [string, string]; if (event === "message_chunk") { - await sleep(100); + await sleep(0); } else if (event === "tool_call_result") { - await sleep(2000); + await sleep(0); } try { yield { diff --git a/web/src/core/api/podcast.ts b/web/src/core/api/podcast.ts new file mode 100644 index 0000000..ee4d56d --- /dev/null +++ b/web/src/core/api/podcast.ts @@ -0,0 +1,19 @@ +// Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +// SPDX-License-Identifier: MIT + +export async function generatePodcast(content: string) { + const response = await fetch("/api/podcast/generate", { + method: "post", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ content }), + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + const arrayBuffer = await response.arrayBuffer(); + const blob = new Blob([arrayBuffer], { type: "audio/mp3" }); + const audioUrl = URL.createObjectURL(blob); + return audioUrl; +} diff --git a/web/src/core/messages/types.ts b/web/src/core/messages/types.ts index 4c260c2..1a2a74d 100644 --- a/web/src/core/messages/types.ts +++ b/web/src/core/messages/types.ts @@ -6,7 +6,13 @@ export type MessageRole = "user" | "assistant" | "tool"; export interface Message { id: string; threadId: string; - agent?: "coordinator" | "planner" | "researcher" | "coder" | "reporter"; + agent?: + | "coordinator" + | "planner" + | "researcher" + | "coder" + | "reporter" + | "podcast"; role: MessageRole; isStreaming?: boolean; content: string; diff --git a/web/src/core/store/store.ts b/web/src/core/store/store.ts index 19a90ec..d075eb9 100644 --- a/web/src/core/store/store.ts +++ b/web/src/core/store/store.ts @@ -6,6 +6,7 @@ import { nanoid } from "nanoid"; import { create } from "zustand"; import { chatStream } from "../api"; +import { generatePodcast } from "../api/podcast"; import type { Message } from "../messages"; import { mergeMessage } from "../messages"; @@ -88,7 +89,7 @@ export async function sendMessage( }; appendMessage(message); } - message ??= findMessage(messageId); + message ??= getMessage(messageId); if (message) { message = mergeMessage(message, event); updateMessage(message); @@ -107,7 +108,7 @@ function existsMessage(id: string) { return useStore.getState().messageIds.includes(id); } -function findMessage(id: string) { +function getMessage(id: string) { return useStore.getState().messages.get(id); } @@ -175,7 +176,7 @@ function appendResearch(researchId: string) { let planMessage: Message | undefined; const reversedMessageIds = [...useStore.getState().messageIds].reverse(); for (const messageId of reversedMessageIds) { - const message = findMessage(messageId); + const message = getMessage(messageId); if (message?.agent === "planner") { planMessage = message; break; @@ -224,6 +225,46 @@ export function openResearch(researchId: string | null) { }); } +export async function listenToPodcast(researchId: string) { + const planMessageId = useStore.getState().researchPlanIds.get(researchId); + const reportMessageId = useStore.getState().researchReportIds.get(researchId); + if (planMessageId && reportMessageId) { + const planMessage = getMessage(planMessageId)!; + const title = (JSON.parse(planMessage.content) as { title: string }).title; + const reportMessage = getMessage(reportMessageId); + if (reportMessage?.content) { + appendMessage({ + id: nanoid(), + threadId: THREAD_ID, + role: "user", + content: "Please generate a podcast for the above research.", + contentChunks: [], + }); + const podCastMessageId = nanoid(); + const podcastObject = { title, researchId }; + const podcastMessage: Message = { + id: podCastMessageId, + threadId: THREAD_ID, + role: "assistant", + agent: "podcast", + content: JSON.stringify(podcastObject), + contentChunks: [], + isStreaming: true, + }; + appendMessage(podcastMessage); + // Generating podcast... + const audioUrl = await generatePodcast(reportMessage.content); + useStore.setState((state) => ({ + messages: new Map(useStore.getState().messages).set(podCastMessageId, { + ...state.messages.get(podCastMessageId)!, + content: JSON.stringify({ ...podcastObject, audioUrl }), + isStreaming: false, + }), + })); + } + } +} + export function useResearchTitle(researchId: string) { const planMessage = useMessage( useStore.getState().researchPlanIds.get(researchId), @@ -236,7 +277,3 @@ export function useMessage(messageId: string | null | undefined) { messageId ? state.messages.get(messageId) : undefined, ); } - -// void sendMessage( -// "How many times taller is the Eiffel Tower than the tallest building in the world?", -// );