GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 01:55:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(adaptive): url hash

Browse Source
This commit is contained in:
Zhaofeng Miao 2024-09-23 16:21:46 +08:00
parent 39e49cac63
commit 3f88f8d2f7
1 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
backend/functions/src/cloud-functions/adaptive-crawler.ts
View File

@ -21,9 +21,13 @@ import { Timestamp } from 'firebase-admin/firestore';
const md5Hasher = new HashManager('md5', 'hex'); const md5Hasher = new HashManager('md5', 'hex');
const removeURLHash = (url: string) => { const removeURLHash = (url: string) => {
const o = new URL(url); try {
o.hash = ''; const o = new URL(url);
return o.toString(); o.hash = '';
return o.toString();
} catch (e) {
return url;
}
} }
@singleton() @singleton()
@ -440,7 +444,7 @@ export class AdaptiveCrawlerHost extends RPCHost {
}[]; }[];
}; };
return json.results.filter(r => r.relevance_score > 0.3).map(r => r.document.text); return json.results.filter(r => r.relevance_score > 0.3).map(r => removeURLHash(r.document.text));
} }
getIndex(user?: JinaEmbeddingsTokenAccount) { getIndex(user?: JinaEmbeddingsTokenAccount) {