GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 06:35:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: uri encoding

Browse Source
This commit is contained in:
yanlong.wang 2024-10-14 17:27:29 +08:00
parent f82504540b
commit 6a97f0bfa6
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
5 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backend/functions/package-lock.json generated
View File

@ -21,7 +21,7 @@
"dayjs": "^1.11.9",
"express": "^4.19.2",
"firebase-admin": "^12.1.0",
"firebase-functions": "^4.9.0",
"firebase-functions": "^6.0.1",
"htmlparser2": "^9.0.0",
"jose": "^5.1.0",
"langdetect": "^0.2.1",
@ -5503,9 +5503,10 @@
}
},
"node_modules/firebase-functions": {
"version": "4.9.0",
"resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-4.9.0.tgz",
"integrity": "sha512-IqxOEsVAWGcRv9KRGzWQR5mOFuNsil3vsfkRPPiaV1U/ATC27/jbahh4z8I4rW8Xqa6cQE5xqnw0ueyMH7i7Ag==",
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-6.0.1.tgz",
"integrity": "sha512-0rIpTU6dnLRvP3IK+okn1FDjoqjzShm0/S+i4OMY7JFu/HJoyJ1JNkrT4KjECy1/mCHK49KsmH8iYE0rzrglHg==",
"license": "MIT",
"dependencies": {
"@types/cors": "^2.8.5",
"@types/express": "4.17.3",
@ -5520,7 +5521,7 @@
"node": ">=14.10.0"
},
"peerDependencies": {
"firebase-admin": "^10.0.0 || ^11.0.0 || ^12.0.0"
"firebase-admin": "^11.10.0 || ^12.0.0"
}
},
"node_modules/firebase-functions-test": {

2
backend/functions/package.json
View File

@ -41,7 +41,7 @@
"dayjs": "^1.11.9",
"express": "^4.19.2",
"firebase-admin": "^12.1.0",
"firebase-functions": "^4.9.0",
"firebase-functions": "^6.0.1",
"htmlparser2": "^9.0.0",
"jose": "^5.1.0",
"langdetect": "^0.2.1",

6
backend/functions/src/cloud-functions/crawler.ts
View File

@ -156,7 +156,7 @@ export class CrawlerHost extends RPCHost {
let chargeAmount = 0;
const crawlerOptions = ctx.req.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed;
const targetUrl = await this.getTargetUrl(ctx.req.url, crawlerOptions);
const targetUrl = await this.getTargetUrl(decodeURIComponent(ctx.req.url), crawlerOptions);
if (!targetUrl) {
const latestUser = uid ? await auth.assertUser() : undefined;
if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {
@ -592,11 +592,11 @@ export class CrawlerHost extends RPCHost {
assignChargeAmount(formatted: FormattedPage) {
if (!formatted) {
return undefined;
return 0;
}
const textContent = formatted?.content || formatted?.description || formatted?.text || formatted?.html;
let amount;
let amount = 0;
do {
if (typeof textContent === 'string') {
amount = estimateToken(textContent);

4
backend/functions/src/cloud-functions/searcher.ts
View File

@ -90,7 +90,7 @@ export class SearcherHost extends RPCHost {
) {
const uid = await auth.solveUID();
let chargeAmount = 0;
const noSlashPath = ctx.req.url.slice(1);
const noSlashPath = decodeURIComponent(ctx.req.url).slice(1);
if (!noSlashPath && !q) {
const latestUser = uid ? await auth.assertUser() : undefined;
if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {
@ -152,7 +152,7 @@ export class SearcherHost extends RPCHost {
delete crawlerOptions.html;
const crawlOpts = this.crawler.configure(crawlerOptions);
const searchQuery = braveSearchExplicitOperators.addTo(q || ctx.req.path.slice(1));
const searchQuery = braveSearchExplicitOperators.addTo(q || noSlashPath);
const r = await this.cachedWebSearch({
q: searchQuery,
count: Math.floor(count + 2)

2
thinapps-shared

@ -1 +1 @@
Subproject commit d91b3aab5924c4090cff0d9d6cde9c5c0a4dc8bd
Subproject commit da6e55d2d5ea310ebc443dab6cdca9f7a9d8e67d