AI/open-webui
1
0
Fork 1
mirror of https://git.mirrors.martin98.com/https://github.com/open-webui/open-webui synced 2025-08-15 07:55:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: Handle special cases of LLM output not following markdown syntax (20250525, formated) This commit introduces a utility function to address rendering issues in LLM output, particularly for Chinese characters and parentheses. The function ensures minimal modification of the original text while fixing markdown parsing problems. Changes include: - Added in for handling specific cases. - Updated in to incorporate the new utility. The fix ensures proper rendering of bold/italic text containing Chinese parentheses, improving readability for non-English content.

Browse Source
This commit is contained in:
YuQX 2025-05-25 19:37:43 +08:00
parent 49fe137553
commit 8ef7938c96
2 changed files with 90 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/lib/utils/index.ts
View File

@ -20,7 +20,7 @@ import markedExtension from '$lib/utils/marked/extension';
import markedKatexExtension from '$lib/utils/marked/katex-extension'; import markedKatexExtension from '$lib/utils/marked/katex-extension';
import hljs from 'highlight.js'; import hljs from 'highlight.js';
import { specialCases } from '$lib/utils/processResponseContent/special-cases' import { specialCases } from '$lib/utils/processResponseContent/special-cases';
////////////////////////// //////////////////////////
// Helper functions // Helper functions
@ -92,7 +92,7 @@ export const sanitizeResponseContent = (content: string) => {
}; };
export const processResponseContent = (content: string) => { export const processResponseContent = (content: string) => {
// This function is used to process the response content // This function is used to process the response content
// before the response content is rendered. // before the response content is rendered.
content = specialCases(content); content = specialCases(content);
return content.trim(); return content.trim();

169
src/lib/utils/processResponseContent/special-cases.ts
View File

@ -1,81 +1,88 @@
/* Expliantion: /* Expliantion:
* This file handles special cases of LLM output not following markdown syntax. * This file handles special cases of LLM output not following markdown syntax.
* It obeys the rule of modifying original text as **LITTLE** as possible. * It obeys the rule of modifying original text as **LITTLE** as possible.
* Detailed documentation of rendering problems is provided in comments. * Detailed documentation of rendering problems is provided in comments.
* More special cases can be added in future. * More special cases can be added in future.
* Note: * Note:
* It should NOT handle the case unless there is clear evidence that it occurs. * It should NOT handle the case unless there is clear evidence that it occurs.
* It only deals with special cases, especially with non-English characters, not general ones. * It only deals with special cases, especially with non-English characters, not general ones.
* Other general issues found, new files shall be added to folder `'$lib/utils/processResponseContent/`, * Other general issues found, new files shall be added to folder `'$lib/utils/processResponseContent/`,
* and function `processResponseContent` in `$lib/utils/index.ts` should be updated accordingly. * and function `processResponseContent` in `$lib/utils/index.ts` should be updated accordingly.
*/ */
export const specialCases = (src: string): string => { export const specialCases = (src: string): string => {
const lines = src.split('\n'); // Process from line to line. const lines = src.split('\n'); // Process from line to line.
const processedLines = lines.map(line => { const processedLines = lines.map((line) => {
// 1. 中文 (Chinese, CN)
// 1. 中文 (Chinese, CN) if (/[\u4e00-\u9fa5]/.test(line)) {
if (/[\u4e00-\u9fa5]/.test(line)) { // Only execute if there are Chinese characters. // Only execute if there are Chinese characters.
// 1.1. Problems caused by Chinese parentheses // 1.1. Problems caused by Chinese parentheses
/* Discription: /* Discription:
* When `*` has Chinese parentheses on the inside, markdown parser ignore bold or italic style. * When `*` has Chinese parentheses on the inside, markdown parser ignore bold or italic style.
* - e.g. `**中文名（English）**中文内容` will be parsed directly, * - e.g. `**中文名（English）**中文内容` will be parsed directly,
* instead of `<strong>中文名（English）</strong>中文内容`. * instead of `<strong>中文名（English）</strong>中文内容`.
* Solution: * Solution:
* Adding a `space` before and after the bold/italic part can solve the problem. * Adding a `space` before and after the bold/italic part can solve the problem.
* - e.g. `**中文名（English）**中文内容` -> ` **中文名（English）** 中文内容` * - e.g. `**中文名（English）**中文内容` -> ` **中文名（English）** 中文内容`
* Note: * Note:
* Similar problem was found with English parentheses and other full delimiters, * Similar problem was found with English parentheses and other full delimiters,
* but they are not handled here because they are less likely to appear in LLM output. * but they are not handled here because they are less likely to appear in LLM output.
* Change the behavior in future if needed. * Change the behavior in future if needed.
*/ */
if (line.includes('*')) { // Only execute if `*` is found in line. if (line.includes('*')) {
// 1.1.1. Handle **bold** with Chinese parentheses // Only execute if `*` is found in line.
line = processCN_01(line, '**', '（', '）');
// 1.1.2. Handle *italic* with Chinese parentheses // 1.1.1. Handle **bold** with Chinese parentheses
line = processCN_01(line, '*', '（', '）'); line = processCN_01(line, '**', '（', '）');
} // 1.1.2. Handle *italic* with Chinese parentheses
line = processCN_01(line, '*', '（', '）');
} }
return line; }
}); return line;
const result = processedLines.join('\n'); });
return result; const result = processedLines.join('\n');
} return result;
};
//////////////////////////
// Helper functions //////////////////////////
////////////////////////// // Helper functions
//////////////////////////
function isChineseChar(char: string): boolean {
return /\p{Script=Han}/u.test(char); function isChineseChar(char: string): boolean {
} return /\p{Script=Han}/u.test(char);
}
function escapeRegExp(string: string): string {
return string.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); function escapeRegExp(string: string): string {
} return string.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
}
////////////////////////// //////////////////////////
// Main functions // Main functions
////////////////////////// //////////////////////////
// Handle case `1.1.1` and `1.1.2` // Handle case `1.1.1` and `1.1.2`
function processCN_01(line: string, symbol: string, leftSymbol: string, rightSymbol: string): string { function processCN_01(
const escapedSymbol = escapeRegExp(symbol); line: string,
const regex = new RegExp(`(.*?)(?<!${escapedSymbol})(${escapedSymbol})([^${escapedSymbol}]+)(${escapedSymbol})(?!${escapedSymbol})(.*?)`, 'g'); symbol: string,
return line.replace(regex, (match, l, left, content, right, r) => { leftSymbol: string,
const result = ( rightSymbol: string
(content.startsWith(leftSymbol) || content.endsWith(rightSymbol)) && ): string {
(!l || (l && l.length > 0 && isChineseChar(l[l.length - 1]))) && const escapedSymbol = escapeRegExp(symbol);
(!r || (r && r.length > 0 && isChineseChar(r[0]))) const regex = new RegExp(
) `(.*?)(?<!${escapedSymbol})(${escapedSymbol})([^${escapedSymbol}]+)(${escapedSymbol})(?!${escapedSymbol})(.*?)`,
if (result) { 'g'
return ` ${left}${content}${right} `; );
} else { return line.replace(regex, (match, l, left, content, right, r) => {
return match; const result =
} (content.startsWith(leftSymbol) || content.endsWith(rightSymbol)) &&
}); (!l || (l && l.length > 0 && isChineseChar(l[l.length - 1]))) &&
} (!r || (r && r.length > 0 && isChineseChar(r[0])));
if (result) {
return ` ${left}${content}${right} `;
} else {
return match;
}
});
}