AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-15 05:55:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: respond with markdown

Browse Source
This commit is contained in:
yanlong.wang 2024-04-25 18:58:42 +08:00
parent 69231ad59e
commit a6a5b7c530
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 7 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backend/functions/src/cloud-functions/crawler.ts
View File

@ -74,7 +74,7 @@ export class CrawlerHost extends RPCHost {
return turnDownService;
}
async formatSnapshot(mode: string | 'markdown' | 'full-markdown' | 'html' | 'text' | 'screenshot', snapshot: PageSnapshot & {
async formatSnapshot(mode: string | 'markdown' | 'html' | 'text' | 'screenshot', snapshot: PageSnapshot & {
screenshotUrl?: string;
}, nominalUrl?: URL) {
if (mode === 'screenshot') {
@ -112,8 +112,8 @@ export class CrawlerHost extends RPCHost {
};
}
const toBeTurnedToMd = mode === 'full-markdown' ? snapshot.html : snapshot.parsed?.content;
let turnDownService = mode === 'markdown' ? this.getTurndown('without any rule') : this.getTurndown();
const toBeTurnedToMd = mode === 'markdown' ? snapshot.html : snapshot.parsed?.content;
let turnDownService = mode === 'markdown' ? this.getTurndown() : this.getTurndown('without any rule');
for (const plugin of this.turnDownPlugins) {
turnDownService = turnDownService.use(plugin);
}
@ -198,7 +198,7 @@ export class CrawlerHost extends RPCHost {
mixins.push(`Published Time: ${this.publishedTime}`);
}
if (mode === 'full-markdown') {
if (mode === 'markdown') {
return this.content;
}
@ -253,14 +253,12 @@ ${this.content}
schema: { type: 'string' }
},
'X-Respond-With': {
description: `Specifies the form factor of the crawled data you prefer. \n\n` +
description: `Specifies the (non-default) form factor of the crawled data you prefer. \n\n` +
`Supported formats:\n` +
`- markdown\n` +
`- full-markdown\n` +
`- html\n` +
`- text\n` +
`- screenshot\n\n` +
`Defaults to: markdown`
`- screenshot\n`
,
in: 'header',
schema: { type: 'string' }
@ -322,7 +320,7 @@ ${this.content}
});
}
const customMode = ctx.req.get('x-respond-with') || 'markdown';
const customMode = ctx.req.get('x-respond-with') || 'default';
const noCache = Boolean(ctx.req.get('x-no-cache'));
const cookies: CookieParam[] = [];
const setCookieHeaders = ctx.req.headers['x-set-cookie'];