AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-16 14:25:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove tidyMarkdown at all

Browse Source
This commit is contained in:
Yanlong Wang 2024-05-09 11:33:56 +08:00
parent 59f807cb7c
commit 36bf5d96b5
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
2 changed files with 31 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/functions/src/cloud-functions/crawler.ts
View File

@ -16,7 +16,6 @@ import TurndownService from 'turndown';
import { parseString as parseSetCookieString } from 'set-cookie-parser';
import type { CookieParam } from 'puppeteer';
import { Crawled } from '../db/crawled';
import { tidyMarkdown } from '../utils/markdown';
import { cleanAttribute } from '../utils/misc';
import { randomUUID } from 'crypto';
import { JinaEmbeddingsAuthDTO } from '../shared/dto/jina-embeddings-auth';
@ -198,7 +197,7 @@ export class CrawlerHost extends RPCHost {
contentText = snapshot.text;
}
const cleanText = tidyMarkdown(contentText || '').trim();
const cleanText = (contentText || '').trim();
const formatted = {
title: (snapshot.parsed?.title || snapshot.title || '').trim(),

55
backend/functions/src/utils/markdown.ts
View File

@ -1,34 +1,39 @@
export function tidyMarkdown(markdown: string): string {
const lines = markdown.split('\n');
const processedLines = lines.map((line) => {
// Handle complex broken links with text and optional images
line = line.replace(/\[\s*([^\]\n!]*?)\s*(?:!\[([^\]]*)\]\((.*?)\))?\s*\]\s*\(\s*([^)\n]+)\s*\)/g, (match, text, alt, imgUrl, linkUrl) => {
text = text.replace(/\s+/g, ' ').trim();
alt = alt ? alt.replace(/\s+/g, ' ').trim() : '';
imgUrl = imgUrl ? imgUrl.replace(/\s+/g, '').trim() : '';
linkUrl = linkUrl.replace(/\s+/g, '').trim();
if (imgUrl) {
return `[${text} ![${alt}](${imgUrl})](${linkUrl})`;
} else {
return `[${text}](${linkUrl})`;
}
});
// Normalize regular links that may be broken across lines
line = line.replace(/\[\s*([^\]\n]+)\]\s*\(\s*([^)\n]+)\s*\)/g, (match, text, url) => {
text = text.replace(/\s+/g, ' ').trim();
url = url.replace(/\s+/g, '').trim();
return `[${text}](${url})`;
});
return line;
// Step 1: Handle complex broken links with text and optional images spread across multiple lines
let normalizedMarkdown = markdown.replace(/\[\s*([^\]\n]+?)\s*\]\s*\(\s*([^)]+)\s*\)/g, (match, text, url) => {
// Remove internal new lines and excessive spaces within the text
text = text.replace(/\s+/g, ' ').trim();
url = url.replace(/\s+/g, '').trim();
return `[${text}](${url})`;
});
// Join the processed lines back together
let normalizedMarkdown = processedLines.join('\n');
normalizedMarkdown = normalizedMarkdown.replace(/\[\s*([^\]\n!]*?)\s*\n*(?:!\[([^\]]*)\]\((.*?)\))?\s*\n*\]\s*\(\s*([^)]+)\s*\)/g, (match, text, alt, imgUrl, linkUrl) => {
// Normalize by removing excessive spaces and new lines
text = text.replace(/\s+/g, ' ').trim();
alt = alt ? alt.replace(/\s+/g, ' ').trim() : '';
imgUrl = imgUrl ? imgUrl.replace(/\s+/g, '').trim() : '';
linkUrl = linkUrl.replace(/\s+/g, '').trim();
if (imgUrl) {
return `[${text} ![${alt}](${imgUrl})](${linkUrl})`;
} else {
return `[${text}](${linkUrl})`;
}
});
// Replace more than two consecutive empty lines with exactly two empty lines
// Step 2: Normalize regular links that may be broken across lines
normalizedMarkdown = normalizedMarkdown.replace(/\[\s*([^\]]+)\]\s*\(\s*([^)]+)\)/g, (match, text, url) => {
text = text.replace(/\s+/g, ' ').trim();
url = url.replace(/\s+/g, '').trim();
return `[${text}](${url})`;
});
// Step 3: Replace more than two consecutive empty lines with exactly two empty lines
normalizedMarkdown = normalizedMarkdown.replace(/\n{3,}/g, '\n\n');
// Step 4: Remove leading spaces from each line
normalizedMarkdown = normalizedMarkdown.replace(/^[ \t]+/gm, '');
return normalizedMarkdown.trim();
}