GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 06:55:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: curl implicit redirect

Browse Source
This commit is contained in:
Yanlong Wang 2025-03-08 17:18:53 +08:00
parent 8597daa96b
commit fb43578fdd
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/services/curl.ts
View File

@ -10,7 +10,7 @@ import { AssertionFailureError, FancyFile } from 'civkit';
import { ServiceBadAttemptError, TempFileManager } from '../shared';
import { createBrotliDecompress, createInflate, createGunzip } from 'zlib';
import { ZSTDDecompress } from 'simple-zstd';
import _ from 'lodash';
import _, { set } from 'lodash';
import { Readable } from 'stream';
import { AsyncLocalContext } from './async-context';
@ -278,9 +278,6 @@ export class CurlControl extends AsyncService {
if ([301, 302, 307, 308].includes(r.statusCode)) {
const headers = r.headers[r.headers.length - 1];
const location = headers.Location || headers.location;
if (!location) {
throw new AssertionFailureError(`Failed to access ${urlToCrawl}: Bad redirection from ${nextHopUrl}`);
}
const setCookieHeader = headers['Set-Cookie'] || headers['set-cookie'];
if (setCookieHeader) {
@ -291,6 +288,10 @@ export class CurlControl extends AsyncService {
}
}
if (!location && !setCookieHeader) {
throw new AssertionFailureError(`Failed to access ${urlToCrawl}: Bad redirection from ${nextHopUrl}`);
}
nextHopUrl = new URL(location, nextHopUrl);
fakeHeaderInfos.push(...r.headers);
leftRedirection -= 1;