AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-06 08:36:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: poorly transformed detection

Browse Source
This commit is contained in:
Yanlong Wang 2024-11-16 13:59:31 +08:00
parent 706de20e5c
commit 6fa8ce309e
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
backend/functions/src/services/snapshot-formatter.ts
View File

@ -578,11 +578,19 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
}
if (content.includes('<table') && content.includes('</table>')) {
if (node?.textContent && content.length > node.textContent.length * 0.8) {
return true;
}
const tableElms = node?.querySelectorAll('table') || [];
const deepTableElms = node?.querySelectorAll('table table');
if (node && tableElms.length) {
const wrappingTables = _.without(tableElms, ...Array.from(deepTableElms || []));
const tableTextsLength = _.sum(wrappingTables.map((x) => (x.innerHTML?.length || 0)));
if ((deepTableElms?.length || 0) / tableElms.length > 0.6) {
return true;
if (tableTextsLength / (content.length) > 0.6) {
return true;
}
}
const tbodyElms = node?.querySelectorAll('tbody') || [];