AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-06 08:36:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove more attrs in readerlm preprocessing

Browse Source
This commit is contained in:
yanlong.wang 2025-01-20 11:54:31 +08:00
parent 140a6f86ae
commit 234f61d066
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 18 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
backend/functions/src/services/jsdom.ts
View File

@ -273,6 +273,24 @@ export class JSDomControl extends AsyncService {
}
x.removeAttribute('style');
});
const treeWalker = jsdom.window.document.createTreeWalker(
jsdom.window.document, // Start from the root document
0x80 // Only show comment nodes
);
let currentNode;
while ((currentNode = treeWalker.nextNode())) {
currentNode.parentNode?.removeChild(currentNode); // Remove each comment node
}
jsdom.window.document.querySelectorAll('*').forEach((x)=> {
const attrs = x.getAttributeNames();
for (const attr of attrs) {
if (attr.startsWith('data-') || attr.startsWith('aria-')) {
x.removeAttribute(attr);
}
}
});
const dt = Date.now() - t0;
if (dt > 1000) {