GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 14:19:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: uri encoding

Browse Source
This commit is contained in:
yanlong.wang 2024-10-14 17:27:29 +08:00
parent f82504540b
commit 6a97f0bfa6
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
5 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backend/functions/package-lock.json generated
View File

@ -21,7 +21,7 @@
"dayjs": "^1.11.9", "dayjs": "^1.11.9",
"express": "^4.19.2", "express": "^4.19.2",
"firebase-admin": "^12.1.0", "firebase-admin": "^12.1.0",
"firebase-functions": "^4.9.0", "firebase-functions": "^6.0.1",
"htmlparser2": "^9.0.0", "htmlparser2": "^9.0.0",
"jose": "^5.1.0", "jose": "^5.1.0",
"langdetect": "^0.2.1", "langdetect": "^0.2.1",
@ -5503,9 +5503,10 @@
} }
}, },
"node_modules/firebase-functions": { "node_modules/firebase-functions": {
"version": "4.9.0", "version": "6.0.1",
"resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-4.9.0.tgz", "resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-6.0.1.tgz",
"integrity": "sha512-IqxOEsVAWGcRv9KRGzWQR5mOFuNsil3vsfkRPPiaV1U/ATC27/jbahh4z8I4rW8Xqa6cQE5xqnw0ueyMH7i7Ag==", "integrity": "sha512-0rIpTU6dnLRvP3IK+okn1FDjoqjzShm0/S+i4OMY7JFu/HJoyJ1JNkrT4KjECy1/mCHK49KsmH8iYE0rzrglHg==",
"license": "MIT",
"dependencies": { "dependencies": {
"@types/cors": "^2.8.5", "@types/cors": "^2.8.5",
"@types/express": "4.17.3", "@types/express": "4.17.3",
@ -5520,7 +5521,7 @@
"node": ">=14.10.0" "node": ">=14.10.0"
}, },
"peerDependencies": { "peerDependencies": {
"firebase-admin": "^10.0.0 || ^11.0.0 || ^12.0.0" "firebase-admin": "^11.10.0 || ^12.0.0"
} }
}, },
"node_modules/firebase-functions-test": { "node_modules/firebase-functions-test": {

2
backend/functions/package.json
View File

@ -41,7 +41,7 @@
"dayjs": "^1.11.9", "dayjs": "^1.11.9",
"express": "^4.19.2", "express": "^4.19.2",
"firebase-admin": "^12.1.0", "firebase-admin": "^12.1.0",
"firebase-functions": "^4.9.0", "firebase-functions": "^6.0.1",
"htmlparser2": "^9.0.0", "htmlparser2": "^9.0.0",
"jose": "^5.1.0", "jose": "^5.1.0",
"langdetect": "^0.2.1", "langdetect": "^0.2.1",

6
backend/functions/src/cloud-functions/crawler.ts
View File

@ -156,7 +156,7 @@ export class CrawlerHost extends RPCHost {
let chargeAmount = 0; let chargeAmount = 0;
const crawlerOptions = ctx.req.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed; const crawlerOptions = ctx.req.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed;
const targetUrl = await this.getTargetUrl(ctx.req.url, crawlerOptions); const targetUrl = await this.getTargetUrl(decodeURIComponent(ctx.req.url), crawlerOptions);
if (!targetUrl) { if (!targetUrl) {
const latestUser = uid ? await auth.assertUser() : undefined; const latestUser = uid ? await auth.assertUser() : undefined;
if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) { if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {
@ -592,11 +592,11 @@ export class CrawlerHost extends RPCHost {
assignChargeAmount(formatted: FormattedPage) { assignChargeAmount(formatted: FormattedPage) {
if (!formatted) { if (!formatted) {
return undefined; return 0;
} }
const textContent = formatted?.content || formatted?.description || formatted?.text || formatted?.html; const textContent = formatted?.content || formatted?.description || formatted?.text || formatted?.html;
let amount; let amount = 0;
do { do {
if (typeof textContent === 'string') { if (typeof textContent === 'string') {
amount = estimateToken(textContent); amount = estimateToken(textContent);

4
backend/functions/src/cloud-functions/searcher.ts
View File

@ -90,7 +90,7 @@ export class SearcherHost extends RPCHost {
) { ) {
const uid = await auth.solveUID(); const uid = await auth.solveUID();
let chargeAmount = 0; let chargeAmount = 0;
const noSlashPath = ctx.req.url.slice(1); const noSlashPath = decodeURIComponent(ctx.req.url).slice(1);
if (!noSlashPath && !q) { if (!noSlashPath && !q) {
const latestUser = uid ? await auth.assertUser() : undefined; const latestUser = uid ? await auth.assertUser() : undefined;
if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) { if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {
@ -152,7 +152,7 @@ export class SearcherHost extends RPCHost {
delete crawlerOptions.html; delete crawlerOptions.html;
const crawlOpts = this.crawler.configure(crawlerOptions); const crawlOpts = this.crawler.configure(crawlerOptions);
const searchQuery = braveSearchExplicitOperators.addTo(q || ctx.req.path.slice(1)); const searchQuery = braveSearchExplicitOperators.addTo(q || noSlashPath);
const r = await this.cachedWebSearch({ const r = await this.cachedWebSearch({
q: searchQuery, q: searchQuery,
count: Math.floor(count + 2) count: Math.floor(count + 2)

2
thinapps-shared

@ -1 +1 @@
Subproject commit d91b3aab5924c4090cff0d9d6cde9c5c0a4dc8bd Subproject commit da6e55d2d5ea310ebc443dab6cdca9f7a9d8e67d