AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-14 11:45:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: url check

Browse Source
This commit is contained in:
yanlong.wang 2025-03-10 18:23:30 +08:00
parent 5bbd75a6d6
commit dceb361a7a
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/api/crawler.ts
View File

@ -46,6 +46,8 @@ import { RobotsTxtService } from '../services/robots-text';
import { lookup } from 'dns/promises';
import { isIP } from 'net';
const normalizeUrl = require('@esm2cjs/normalize-url').default;
export interface ExtraScrappingOptions extends ScrappingOptions {
withIframe?: boolean | 'quoted';
withShadowDom?: boolean;
@ -474,8 +476,7 @@ export class CrawlerHost extends RPCHost {
const targetUrlFromGet = originPath.slice(1);
if (crawlerOptions.pdf) {
const pdfBuf = crawlerOptions.pdf instanceof Blob ? await crawlerOptions.pdf.arrayBuffer().then((x) => Buffer.from(x)) : Buffer.from(crawlerOptions.pdf, 'base64');
url = `blob://pdf/${md5Hasher.hash(pdfBuf)}`;
url = `blob://pdf/${randomUUID()}`;
} else if (targetUrlFromGet) {
url = targetUrlFromGet.trim();
} else if (crawlerOptions.url) {
@ -485,7 +486,6 @@ export class CrawlerHost extends RPCHost {
}
let result: URL;
const normalizeUrl = require('@esm2cjs/normalize-url').default;
try {
result = new URL(
normalizeUrl(