AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-18 02:25:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove tidyMarkdown at all

Browse Source
This commit is contained in:
Yanlong Wang 2024-05-09 11:33:56 +08:00
parent 59f807cb7c
commit 36bf5d96b5
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
2 changed files with 31 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/functions/src/cloud-functions/crawler.ts
View File

@ -16,7 +16,6 @@ import TurndownService from 'turndown';
import { parseString as parseSetCookieString } from 'set-cookie-parser'; import { parseString as parseSetCookieString } from 'set-cookie-parser';
import type { CookieParam } from 'puppeteer'; import type { CookieParam } from 'puppeteer';
import { Crawled } from '../db/crawled'; import { Crawled } from '../db/crawled';
import { tidyMarkdown } from '../utils/markdown';
import { cleanAttribute } from '../utils/misc'; import { cleanAttribute } from '../utils/misc';
import { randomUUID } from 'crypto'; import { randomUUID } from 'crypto';
import { JinaEmbeddingsAuthDTO } from '../shared/dto/jina-embeddings-auth'; import { JinaEmbeddingsAuthDTO } from '../shared/dto/jina-embeddings-auth';
@ -198,7 +197,7 @@ export class CrawlerHost extends RPCHost {
contentText = snapshot.text; contentText = snapshot.text;
} }
const cleanText = tidyMarkdown(contentText || '').trim(); const cleanText = (contentText || '').trim();
const formatted = { const formatted = {
title: (snapshot.parsed?.title || snapshot.title || '').trim(), title: (snapshot.parsed?.title || snapshot.title || '').trim(),

55
backend/functions/src/utils/markdown.ts
View File

@ -1,34 +1,39 @@
export function tidyMarkdown(markdown: string): string { export function tidyMarkdown(markdown: string): string {
const lines = markdown.split('\n');
const processedLines = lines.map((line) => {
// Handle complex broken links with text and optional images
line = line.replace(/\[\s*([^\]\n!]*?)\s*(?:!\[([^\]]*)\]\((.*?)\))?\s*\]\s*\(\s*([^)\n]+)\s*\)/g, (match, text, alt, imgUrl, linkUrl) => {
text = text.replace(/\s+/g, ' ').trim();
alt = alt ? alt.replace(/\s+/g, ' ').trim() : '';
imgUrl = imgUrl ? imgUrl.replace(/\s+/g, '').trim() : '';
linkUrl = linkUrl.replace(/\s+/g, '').trim();
if (imgUrl) {
return `[${text} ![${alt}](${imgUrl})](${linkUrl})`;
} else {
return `[${text}](${linkUrl})`;
}
});
// Normalize regular links that may be broken across lines // Step 1: Handle complex broken links with text and optional images spread across multiple lines
line = line.replace(/\[\s*([^\]\n]+)\]\s*\(\s*([^)\n]+)\s*\)/g, (match, text, url) => { let normalizedMarkdown = markdown.replace(/\[\s*([^\]\n]+?)\s*\]\s*\(\s*([^)]+)\s*\)/g, (match, text, url) => {
text = text.replace(/\s+/g, ' ').trim(); // Remove internal new lines and excessive spaces within the text
url = url.replace(/\s+/g, '').trim(); text = text.replace(/\s+/g, ' ').trim();
return `[${text}](${url})`; url = url.replace(/\s+/g, '').trim();
}); return `[${text}](${url})`;
return line;
}); });
// Join the processed lines back together normalizedMarkdown = normalizedMarkdown.replace(/\[\s*([^\]\n!]*?)\s*\n*(?:!\[([^\]]*)\]\((.*?)\))?\s*\n*\]\s*\(\s*([^)]+)\s*\)/g, (match, text, alt, imgUrl, linkUrl) => {
let normalizedMarkdown = processedLines.join('\n'); // Normalize by removing excessive spaces and new lines
text = text.replace(/\s+/g, ' ').trim();
alt = alt ? alt.replace(/\s+/g, ' ').trim() : '';
imgUrl = imgUrl ? imgUrl.replace(/\s+/g, '').trim() : '';
linkUrl = linkUrl.replace(/\s+/g, '').trim();
if (imgUrl) {
return `[${text} ![${alt}](${imgUrl})](${linkUrl})`;
} else {
return `[${text}](${linkUrl})`;
}
});
// Replace more than two consecutive empty lines with exactly two empty lines // Step 2: Normalize regular links that may be broken across lines
normalizedMarkdown = normalizedMarkdown.replace(/\[\s*([^\]]+)\]\s*\(\s*([^)]+)\)/g, (match, text, url) => {
text = text.replace(/\s+/g, ' ').trim();
url = url.replace(/\s+/g, '').trim();
return `[${text}](${url})`;
});
// Step 3: Replace more than two consecutive empty lines with exactly two empty lines
normalizedMarkdown = normalizedMarkdown.replace(/\n{3,}/g, '\n\n'); normalizedMarkdown = normalizedMarkdown.replace(/\n{3,}/g, '\n\n');
// Step 4: Remove leading spaces from each line
normalizedMarkdown = normalizedMarkdown.replace(/^[ \t]+/gm, '');
return normalizedMarkdown.trim(); return normalizedMarkdown.trim();
} }