From a6dbd26f7593cabbcbdea6b015128299dfaa50a1 Mon Sep 17 00:00:00 2001 From: chenxu9741 Date: Wed, 17 Jul 2024 19:44:16 +0800 Subject: [PATCH] Add the API documentation for streaming TTS (Text-to-Speech) (#6382) --- .../develop/template/template.en.mdx | 24 ++++++++++++----- .../develop/template/template.zh.mdx | 25 +++++++++++++----- .../template/template_advanced_chat.en.mdx | 21 ++++++++++++--- .../template/template_advanced_chat.zh.mdx | 26 ++++++++++++++----- .../develop/template/template_chat.en.mdx | 26 ++++++++++++++----- .../develop/template/template_chat.zh.mdx | 26 ++++++++++++++----- .../develop/template/template_workflow.en.mdx | 12 +++++++++ .../develop/template/template_workflow.zh.mdx | 12 +++++++++ 8 files changed, 137 insertions(+), 35 deletions(-) diff --git a/web/app/components/develop/template/template.en.mdx b/web/app/components/develop/template/template.en.mdx index cedb26264d..a324bbefa8 100755 --- a/web/app/components/develop/template/template.en.mdx +++ b/web/app/components/develop/template/template.en.mdx @@ -103,6 +103,16 @@ The text generation application offers non-session support and is ideal for tran - `metadata` (object) Metadata - `usage` (Usage) Model usage information - `retriever_resources` (array[RetrieverResource]) Citation and Attribution List + - `event: tts_message` TTS audio stream event, that is, speech synthesis output. The content is an audio block in Mp3 format, encoded as a base64 string. When playing, simply decode the base64 and feed it into the player. (This message is available only when auto-play is enabled) + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The audio after speech synthesis, encoded in base64 text content, when playing, simply decode the base64 and feed it into the player + - `created_at` (int) Creation timestamp, e.g.: 1705395332 + - `event: tts_message_end` TTS audio stream end event, receiving this event indicates the end of the audio stream. + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The end event has no audio, so this is an empty string + - `created_at` (int) Creation timestamp, e.g.: 1705395332 - `event: message_replace` Message content replacement event. When output content moderation is enabled, if the content is flagged, then the message content will be replaced with a preset reply through this event. - `task_id` (string) Task ID, used for request tracking and the below Stop Generate API @@ -185,6 +195,8 @@ The text generation application offers non-session support and is ideal for tran data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -495,29 +507,29 @@ The text generation application offers non-session support and is ideal for tran ### Request Body + + For text messages generated by Dify, simply pass the generated message-id directly. The backend will use the message-id to look up the corresponding content and synthesize the voice information directly. If both message_id and text are provided simultaneously, the message_id is given priority. + Speech generated content。 The user identifier, defined by the developer, must ensure uniqueness within the app. - - Whether to enable streaming output, true、false。 - - + ```bash {{ title: 'cURL' }} curl -o text-to-audio.mp3 -X POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ + "message_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "text": "Hello Dify", - "user": "abc-123", - "streaming": false + "user": "abc-123" }' ``` diff --git a/web/app/components/develop/template/template.zh.mdx b/web/app/components/develop/template/template.zh.mdx index c996ed647c..e593f013c2 100755 --- a/web/app/components/develop/template/template.zh.mdx +++ b/web/app/components/develop/template/template.zh.mdx @@ -97,12 +97,22 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' - `message_id` (string) 消息唯一 ID - `answer` (string) LLM 返回文本块内容 - `created_at` (int) 创建时间戳，如：1705395332 - - `event: message_end` 消息结束事件，收到此事件则代表流式返回结束。 + - `event: message_end` 消息结束事件，收到此事件则代表文本流式返回结束。 - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 - `message_id` (string) 消息唯一 ID - `metadata` (object) 元数据 - `usage` (Usage) 模型用量信息 - `retriever_resources` (array[RetrieverResource]) 引用和归属分段列表 + - `event: tts_message` TTS 音频流事件，即：语音合成输出。内容是Mp3格式的音频块，使用 base64 编码后的字符串，播放的时候直接解码即可。(开启自动播放才有此消息) + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 语音合成之后的音频块使用 Base64 编码之后的文本内容，播放的时候直接 base64 解码送入播放器即可 + - `created_at` (int) 创建时间戳，如：1705395332 + - `event: tts_message_end` TTS 音频流结束事件，收到这个事件表示音频流返回结束。 + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 结束事件是没有音频的，所以这里是空字符串 + - `created_at` (int) 创建时间戳，如：1705395332 - `event: message_replace` 消息内容替换事件。 开启内容审查和审查输出内容时，若命中了审查条件，则会通过此事件替换消息内容为预设回复。 - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 @@ -162,6 +172,8 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ```streaming {{ title: 'Response' }} data: {"id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "answer": " I", "created_at": 1679586595} data: {"id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "answer": " I", "created_at": 1679586595} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -456,26 +468,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ### Request Body + + Dify 生成的文本消息，那么直接传递生成的message-id 即可，后台会通过 message_id 查找相应的内容直接合成语音信息。如果同时传 message_id 和 text，优先使用 message_id。 + - 语音生成内容。 + 语音生成内容。如果没有传 message-id的话，则会使用这个字段的内容 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - 是否启用流式输出true、false。 - - + ```bash {{ title: 'cURL' }} curl -o text-to-audio.mp3 -X POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ + "message_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "text": "你好Dify", "user": "abc-123", "streaming": false diff --git a/web/app/components/develop/template/template_advanced_chat.en.mdx b/web/app/components/develop/template/template_advanced_chat.en.mdx index c59f08f5d5..8481b4db43 100644 --- a/web/app/components/develop/template/template_advanced_chat.en.mdx +++ b/web/app/components/develop/template/template_advanced_chat.en.mdx @@ -118,6 +118,16 @@ Chat applications support session persistence, allowing previous chat history to - `metadata` (object) Metadata - `usage` (Usage) Model usage information - `retriever_resources` (array[RetrieverResource]) Citation and Attribution List + - `event: tts_message` TTS audio stream event, that is, speech synthesis output. The content is an audio block in Mp3 format, encoded as a base64 string. When playing, simply decode the base64 and feed it into the player. (This message is available only when auto-play is enabled) + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The audio after speech synthesis, encoded in base64 text content, when playing, simply decode the base64 and feed it into the player + - `created_at` (int) Creation timestamp, e.g.: 1705395332 + - `event: tts_message_end` TTS audio stream end event, receiving this event indicates the end of the audio stream. + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The end event has no audio, so this is an empty string + - `created_at` (int) Creation timestamp, e.g.: 1705395332 - `event: message_replace` Message content replacement event. When output content moderation is enabled, if the content is flagged, then the message content will be replaced with a preset reply through this event. - `task_id` (string) Task ID, used for request tracking and the below Stop Generate API @@ -276,6 +286,8 @@ Chat applications support session persistence, allowing previous chat history to data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -867,26 +879,27 @@ Chat applications support session persistence, allowing previous chat history to ### Request Body + + For text messages generated by Dify, simply pass the generated message-id directly. The backend will use the message-id to look up the corresponding content and synthesize the voice information directly. If both message_id and text are provided simultaneously, the message_id is given priority. + Speech generated content。 The user identifier, defined by the developer, must ensure uniqueness within the app. - - Whether to enable streaming output, true、false。 - - + ```bash {{ title: 'cURL' }} curl -o text-to-audio.mp3 -X POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ + "message_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "text": "Hello Dify", "user": "abc-123", "streaming": false diff --git a/web/app/components/develop/template/template_advanced_chat.zh.mdx b/web/app/components/develop/template/template_advanced_chat.zh.mdx index a07096ebd1..97a2df01a6 100755 --- a/web/app/components/develop/template/template_advanced_chat.zh.mdx +++ b/web/app/components/develop/template/template_advanced_chat.zh.mdx @@ -120,6 +120,16 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' - `metadata` (object) 元数据 - `usage` (Usage) 模型用量信息 - `retriever_resources` (array[RetrieverResource]) 引用和归属分段列表 + - `event: tts_message` TTS 音频流事件，即：语音合成输出。内容是Mp3格式的音频块，使用 base64 编码后的字符串，播放的时候直接解码即可。(开启自动播放才有此消息) + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 语音合成之后的音频块使用 Base64 编码之后的文本内容，播放的时候直接 base64 解码送入播放器即可 + - `created_at` (int) 创建时间戳，如：1705395332 + - `event: tts_message_end` TTS 音频流结束事件，收到这个事件表示音频流返回结束。 + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 结束事件是没有音频的，所以这里是空字符串 + - `created_at` (int) 创建时间戳，如：1705395332 - `event: message_replace` 消息内容替换事件。 开启内容审查和审查输出内容时，若命中了审查条件，则会通过此事件替换消息内容为预设回复。 - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 @@ -287,6 +297,8 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -898,29 +910,29 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ### Request Body + + Dify 生成的文本消息，那么直接传递生成的message-id 即可，后台会通过 message_id 查找相应的内容直接合成语音信息。如果同时传 message_id 和 text，优先使用 message_id。 + - 语音生成内容。 + 语音生成内容。如果没有传 message-id的话，则会使用这个字段的内容 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - 是否启用流式输出true、false。 - - + ```bash {{ title: 'cURL' }} curl -o text-to-audio.mp3 -X POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ + "message_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "text": "你好Dify", - "user": "abc-123", - "streaming": false + "user": "abc-123" }' ``` diff --git a/web/app/components/develop/template/template_chat.en.mdx b/web/app/components/develop/template/template_chat.en.mdx index 9febd1ca6b..00c02936a6 100644 --- a/web/app/components/develop/template/template_chat.en.mdx +++ b/web/app/components/develop/template/template_chat.en.mdx @@ -112,6 +112,16 @@ Chat applications support session persistence, allowing previous chat history to - `conversation_id` (string) Conversation ID - `answer` (string) LLM returned text chunk content - `created_at` (int) Creation timestamp, e.g., 1705395332 + - `event: tts_message` TTS audio stream event, that is, speech synthesis output. The content is an audio block in Mp3 format, encoded as a base64 string. When playing, simply decode the base64 and feed it into the player. (This message is available only when auto-play is enabled) + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The audio after speech synthesis, encoded in base64 text content, when playing, simply decode the base64 and feed it into the player + - `created_at` (int) Creation timestamp, e.g.: 1705395332 + - `event: tts_message_end` TTS audio stream end event, receiving this event indicates the end of the audio stream. + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The end event has no audio, so this is an empty string + - `created_at` (int) Creation timestamp, e.g.: 1705395332 - `event: agent_thought` thought of Agent, contains the thought of LLM, input and output of tool calls (Only supported in Agent mode) - `id` (string) Agent thought ID, every iteration has a unique agent thought ID - `task_id` (string) (string) Task ID, used for request tracking and the below Stop Generate API @@ -233,6 +243,8 @@ Chat applications support session persistence, allowing previous chat history to data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` ### Response Example(Agent Assistant) @@ -245,7 +257,9 @@ Chat applications support session persistence, allowing previous chat history to data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} - ``` + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} + ``` @@ -905,25 +919,25 @@ Chat applications support session persistence, allowing previous chat history to ### Request Body + + For text messages generated by Dify, simply pass the generated message-id directly. The backend will use the message-id to look up the corresponding content and synthesize the voice information directly. If both message_id and text are provided simultaneously, the message_id is given priority. + Speech generated content。 The user identifier, defined by the developer, must ensure uniqueness within the app. - - Whether to enable streaming output, true、false。 - - + ```bash {{ title: 'cURL' }} curl --location --request POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer ENTER-YOUR-SECRET-KEY' \ - --form 'file=Hello Dify;user=abc-123;streaming=false' + --form 'file=Hello Dify;user=abc-123;message_id=5ad4cb98-f0c7-4085-b384-88c403be6290' ``` diff --git a/web/app/components/develop/template/template_chat.zh.mdx b/web/app/components/develop/template/template_chat.zh.mdx index 36d47651b5..9323313d9f 100644 --- a/web/app/components/develop/template/template_chat.zh.mdx +++ b/web/app/components/develop/template/template_chat.zh.mdx @@ -140,6 +140,16 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' - `metadata` (object) 元数据 - `usage` (Usage) 模型用量信息 - `retriever_resources` (array[RetrieverResource]) 引用和归属分段列表 + - `event: tts_message` TTS 音频流事件，即：语音合成输出。内容是Mp3格式的音频块，使用 base64 编码后的字符串，播放的时候直接解码即可。(开启自动播放才有此消息) + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 语音合成之后的音频块使用 Base64 编码之后的文本内容，播放的时候直接 base64 解码送入播放器即可 + - `created_at` (int) 创建时间戳，如：1705395332 + - `event: tts_message_end` TTS 音频流结束事件，收到这个事件表示音频流返回结束。 + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 结束事件是没有音频的，所以这里是空字符串 + - `created_at` (int) 创建时间戳，如：1705395332 - `event: message_replace` 消息内容替换事件。 开启内容审查和审查输出内容时，若命中了审查条件，则会通过此事件替换消息内容为预设回复。 - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 @@ -246,6 +256,8 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -263,6 +275,8 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' data: {"event": "agent_message", "id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "task_id": "9cf1ddd7-f94b-459b-b942-b77b26c59e9b", "message_id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "answer": " suit .", "created_at": 1705639511, "conversation_id": "c216c595-2d89-438c-b33c-aae5ddddd142"} data: {"event": "agent_thought", "id": "67a99dc1-4f82-42d3-b354-18d4594840c8", "task_id": "9cf1ddd7-f94b-459b-b942-b77b26c59e9b", "message_id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "position": 2, "thought": "I have created an image of a cute Japanese anime girl with white hair and blue eyes wearing a bunny girl suit.", "observation": "", "tool": "", "tool_input": "", "created_at": 1705639511, "message_files": [], "conversation_id": "c216c595-2d89-438c-b33c-aae5ddddd142"} data: {"event": "message_end", "task_id": "9cf1ddd7-f94b-459b-b942-b77b26c59e9b", "id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "message_id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "conversation_id": "c216c595-2d89-438c-b33c-aae5ddddd142", "metadata": {"usage": {"prompt_tokens": 305, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0003050", "completion_tokens": 97, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0001940", "total_tokens": 184, "total_price": "0.0002290", "currency": "USD", "latency": 1.771092874929309}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -915,25 +929,25 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ### Request Body + + Dify 生成的文本消息，那么直接传递生成的message-id 即可，后台会通过 message_id 查找相应的内容直接合成语音信息。如果同时传 message_id 和 text，优先使用 message_id。 + - 语音生成内容。 + 语音生成内容。如果没有传 message-id的话，则会使用这个字段的内容 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - 是否启用流式输出true、false。 - - + ```bash {{ title: 'cURL' }} curl --location --request POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer ENTER-YOUR-SECRET-KEY' \ - --form 'file=你好Dify;user=abc-123;streaming=false' + --form 'file=你好Dify;user=abc-123;message_id=5ad4cb98-f0c7-4085-b384-88c403be6290' ``` diff --git a/web/app/components/develop/template/template_workflow.en.mdx b/web/app/components/develop/template/template_workflow.en.mdx index 9580c2aa0b..64a9092f59 100644 --- a/web/app/components/develop/template/template_workflow.en.mdx +++ b/web/app/components/develop/template/template_workflow.en.mdx @@ -147,6 +147,16 @@ Workflow applications offers non-session support and is ideal for translation, a - `total_steps` (int) default 0 - `created_at` (timestamp) start time - `finished_at` (timestamp) end time + - `event: tts_message` TTS audio stream event, that is, speech synthesis output. The content is an audio block in Mp3 format, encoded as a base64 string. When playing, simply decode the base64 and feed it into the player. (This message is available only when auto-play is enabled) + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The audio after speech synthesis, encoded in base64 text content, when playing, simply decode the base64 and feed it into the player + - `created_at` (int) Creation timestamp, e.g.: 1705395332 + - `event: tts_message_end` TTS audio stream end event, receiving this event indicates the end of the audio stream. + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The end event has no audio, so this is an empty string + - `created_at` (int) Creation timestamp, e.g.: 1705395332 - `event: ping` Ping event every 10 seconds to keep the connection alive. ### Errors @@ -204,6 +214,8 @@ Workflow applications offers non-session support and is ideal for translation, a data: {"event": "node_started", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "node_id": "dfjasklfjdslag", "node_type": "start", "title": "Start", "index": 0, "predecessor_node_id": "fdljewklfklgejlglsd", "inputs": {}, "created_at": 1679586595}} data: {"event": "node_finished", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "node_id": "dfjasklfjdslag", "node_type": "start", "title": "Start", "index": 0, "predecessor_node_id": "fdljewklfklgejlglsd", "inputs": {}, "outputs": {}, "status": "succeeded", "elapsed_time": 0.324, "execution_metadata": {"total_tokens": 63127864, "total_price": 2.378, "currency": "USD"}, "created_at": 1679586595}} data: {"event": "workflow_finished", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "workflow_id": "dfjasklfjdslag", "outputs": {}, "status": "succeeded", "elapsed_time": 0.324, "total_tokens": 63127864, "total_steps": "1", "created_at": 1679586595, "finished_at": 1679976595}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` diff --git a/web/app/components/develop/template/template_workflow.zh.mdx b/web/app/components/develop/template/template_workflow.zh.mdx index 6264eb57e5..c891982873 100644 --- a/web/app/components/develop/template/template_workflow.zh.mdx +++ b/web/app/components/develop/template/template_workflow.zh.mdx @@ -143,6 +143,16 @@ Workflow 应用无会话支持，适合用于翻译/文章写作/总结 AI 等 - `total_steps` (int) 总步数（冗余），默认 0 - `created_at` (timestamp) 开始时间 - `finished_at` (timestamp) 结束时间 + - `event: tts_message` TTS 音频流事件，即：语音合成输出。内容是Mp3格式的音频块，使用 base64 编码后的字符串，播放的时候直接解码即可。(开启自动播放才有此消息) + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 语音合成之后的音频块使用 Base64 编码之后的文本内容，播放的时候直接 base64 解码送入播放器即可 + - `created_at` (int) 创建时间戳，如：1705395332 + - `event: tts_message_end` TTS 音频流结束事件，收到这个事件表示音频流返回结束。 + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 结束事件是没有音频的，所以这里是空字符串 + - `created_at` (int) 创建时间戳，如：1705395332 - `event: ping` 每 10s 一次的 ping 事件，保持连接存活。 ### Errors @@ -200,6 +210,8 @@ Workflow 应用无会话支持，适合用于翻译/文章写作/总结 AI 等 data: {"event": "node_started", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "node_id": "dfjasklfjdslag", "node_type": "start", "title": "Start", "index": 0, "predecessor_node_id": "fdljewklfklgejlglsd", "inputs": {}, "created_at": 1679586595}} data: {"event": "node_finished", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "node_id": "dfjasklfjdslag", "node_type": "start", "title": "Start", "index": 0, "predecessor_node_id": "fdljewklfklgejlglsd", "inputs": {}, "outputs": {}, "status": "succeeded", "elapsed_time": 0.324, "execution_metadata": {"total_tokens": 63127864, "total_price": 2.378, "currency": "USD"}, "created_at": 1679586595}} data: {"event": "workflow_finished", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "workflow_id": "dfjasklfjdslag", "outputs": {}, "status": "succeeded", "elapsed_time": 0.324, "total_tokens": 63127864, "total_steps": "1", "created_at": 1679586595, "finished_at": 1679976595}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ```