GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 01:45:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: poorly transformed detection

Browse Source
This commit is contained in:
Yanlong Wang 2024-11-16 13:59:31 +08:00
parent 706de20e5c
commit 6fa8ce309e
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/functions/src/services/snapshot-formatter.ts
View File

@ -578,12 +578,20 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
} }
if (content.includes('<table') && content.includes('</table>')) { if (content.includes('<table') && content.includes('</table>')) {
if (node?.textContent && content.length > node.textContent.length * 0.8) {
return true;
}
const tableElms = node?.querySelectorAll('table') || []; const tableElms = node?.querySelectorAll('table') || [];
const deepTableElms = node?.querySelectorAll('table table'); const deepTableElms = node?.querySelectorAll('table table');
if (node && tableElms.length) {
const wrappingTables = _.without(tableElms, ...Array.from(deepTableElms || []));
const tableTextsLength = _.sum(wrappingTables.map((x) => (x.innerHTML?.length || 0)));
if ((deepTableElms?.length || 0) / tableElms.length > 0.6) { if (tableTextsLength / (content.length) > 0.6) {
return true; return true;
} }
}
const tbodyElms = node?.querySelectorAll('tbody') || []; const tbodyElms = node?.querySelectorAll('tbody') || [];
const deepTbodyElms = node?.querySelectorAll('tbody tbody'); const deepTbodyElms = node?.querySelectorAll('tbody tbody');