GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-18 18:05:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: deps and bugs

Browse Source
This commit is contained in:
Yanlong Wang 2024-11-13 00:27:39 +08:00
parent 7ae2545a30
commit 68c4df2df3
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
5 changed files with 14 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
backend/functions/package-lock.json generated
View File

@ -15,7 +15,7 @@
"archiver": "^6.0.1",
"axios": "^1.3.3",
"bcrypt": "^5.1.0",
"civkit": "^0.8.2-c9ca977",
"civkit": "^0.8.2-2eddf1b",
"core-js": "^3.37.1",
"cors": "^2.8.5",
"dayjs": "^1.11.9",
@ -3729,10 +3729,9 @@
}
},
"node_modules/civkit": {
"version": "0.8.2-c9ca977",
"resolved": "https://registry.npmjs.org/civkit/-/civkit-0.8.2-c9ca977.tgz",
"integrity": "sha512-wuJ6zs88qx5WiIxSBErH3f8wBArSkT2goHHvfm5ZLqL17v6rkS4iQWx2+YhJfhmfBzqU8oAZI1QD9v2LY1awBg==",
"license": "AGPL",
"version": "0.8.2-2eddf1b",
"resolved": "https://registry.npmjs.org/civkit/-/civkit-0.8.2-2eddf1b.tgz",
"integrity": "sha512-iRYQKasePTQYIajPZpTh+uQn09XF7e6+tBaFwxs7mlUIHoU8ci8CT307ITYMnppDLzCh7BRpSgt53mz4Jwg78w==",
"dependencies": {
"lodash": "^4.17.21",
"tslib": "^2.5.0"

2
backend/functions/package.json
View File

@ -35,7 +35,7 @@
"archiver": "^6.0.1",
"axios": "^1.3.3",
"bcrypt": "^5.1.0",
"civkit": "^0.8.2-c9ca977",
"civkit": "^0.8.2-2eddf1b",
"core-js": "^3.37.1",
"cors": "^2.8.5",
"dayjs": "^1.11.9",

2
backend/functions/src/services/jsdom.ts
View File

@ -113,7 +113,7 @@ export class JSDomControl extends AsyncService {
let rootDoc: Document;
if (allNodes.length === 1 && allNodes[0].nodeName === '#document') {
rootDoc = allNodes[0] as any;
if (rootDoc.body.innerText) {
if (rootDoc.documentElement && rootDoc.body?.innerText) {
textNodes.push(rootDoc.body);
}
} else {

10
backend/functions/src/services/snapshot-formatter.ts
View File

@ -76,7 +76,10 @@ export class SnapshotFormatter extends AsyncService {
const f = {
...this.getGeneralSnapshotMixins(snapshot),
};
let modeOK = false;
if (mode.includes('screenshot')) {
modeOK = true;
if (snapshot.screenshot && !snapshot.screenshotUrl) {
const fid = `instant-screenshots/${randomUUID()}`;
await this.firebaseObjectStorage.saveFile(fid, snapshot.screenshot, {
@ -86,7 +89,6 @@ export class SnapshotFormatter extends AsyncService {
});
snapshot.screenshotUrl = await this.firebaseObjectStorage.signDownloadUrl(fid, Date.now() + urlValidMs);
}
Object.assign(f, {
screenshotUrl: snapshot.screenshotUrl,
});
@ -94,6 +96,7 @@ export class SnapshotFormatter extends AsyncService {
Object.defineProperty(f, 'textRepresentation', { value: `${f.screenshotUrl}\n`, enumerable: false, configurable: true });
}
if (mode.includes('pageshot')) {
modeOK = true;
if (snapshot.pageshot && !snapshot.pageshotUrl) {
const fid = `instant-screenshots/${randomUUID()}`;
await this.firebaseObjectStorage.saveFile(fid, snapshot.pageshot, {
@ -103,7 +106,6 @@ export class SnapshotFormatter extends AsyncService {
});
snapshot.pageshotUrl = await this.firebaseObjectStorage.signDownloadUrl(fid, Date.now() + urlValidMs);
}
Object.assign(f, {
html: snapshot.html,
pageshotUrl: snapshot.pageshotUrl,
@ -111,6 +113,7 @@ export class SnapshotFormatter extends AsyncService {
Object.defineProperty(f, 'textRepresentation', { value: `${f.pageshotUrl}\n`, enumerable: false, configurable: true });
}
if (mode.includes('html')) {
modeOK = true;
Object.assign(f, {
html: snapshot.html,
});
@ -141,13 +144,14 @@ export class SnapshotFormatter extends AsyncService {
}
if (mode.includes('text')) {
modeOK = true;
Object.assign(f, {
text: snapshot.text,
});
Object.defineProperty(f, 'textRepresentation', { value: snapshot.text, enumerable: false, configurable: true });
}
if (!mode.includes('markdown') && !mode.includes('content')) {
if (modeOK && !mode.includes('markdown') && !mode.includes('content')) {
const dt = Date.now() - t0;
this.logger.info(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt });

2
thinapps-shared

@ -1 +1 @@
Subproject commit 06cc23b16cafdd17c7e7db996f2167a39cc6d1eb
Subproject commit 081595c106d3b777a8bc4a6c3b129645ca1c1e21