GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 05:15:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: curl cookie mimicking

Browse Source
This commit is contained in:
Yanlong Wang 2025-03-08 18:14:43 +08:00
parent 89e5dbbe9c
commit 26f6202f79
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/services/curl.ts
View File

@ -115,15 +115,33 @@ export class CurlControl extends AsyncService {
const headersToSet = { ...crawlOpts?.extraHeaders };
if (crawlOpts?.cookies?.length) {
const cookieChunks = crawlOpts.cookies.filter((x)=> {
if (x.maxAge && x.maxAge < 0) {
return false;
const cookieKv: Record<string, string> = {};
for (const cookie of crawlOpts.cookies) {
cookieKv[cookie.name] = cookie.value;
}
for (const cookie of crawlOpts.cookies) {
if (cookie.maxAge && cookie.maxAge < 0) {
delete cookieKv[cookie.name];
continue;
}
if (x.expires && x.expires < new Date()) {
return false;
if (cookie.expires && cookie.expires < new Date()) {
delete cookieKv[cookie.name];
continue;
}
return true;
}).map((cookie) => `${cookie.name}=${encodeURIComponent(cookie.value)}`);
if (cookie.secure && urlToCrawl.protocol !== 'https:') {
delete cookieKv[cookie.name];
continue;
}
if (cookie.domain && !urlToCrawl.hostname.endsWith(cookie.domain)) {
delete cookieKv[cookie.name];
continue;
}
if (cookie.path && !urlToCrawl.pathname.startsWith(cookie.path)) {
delete cookieKv[cookie.name];
continue;
}
}
const cookieChunks = Object.entries(cookieKv).map(([k, v]) => `${k}=${encodeURIComponent(v)}`);
headersToSet.cookie ??= cookieChunks.join('; ');
}
if (crawlOpts?.referer) {