AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-14 05:35:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: provide option to discard links

Browse Source
This commit is contained in:
Yanlong Wang 2025-03-12 10:46:01 +08:00
parent 7ded2de54c
commit cadb2fd94c
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
2 changed files with 67 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/dto/turndown-tweakable-options.ts
View File

@ -36,15 +36,15 @@ export class TurnDownTweakableOptions extends AutoCastable {
@Prop({
desc: 'Turndown options > linkStyle',
type: new Set(['inlined', 'referenced']),
type: new Set(['inlined', 'referenced', 'discarded']),
})
linkStyle?: 'inlined' | 'referenced';
linkStyle?: 'inlined' | 'referenced' | 'discarded';
@Prop({
desc: 'Turndown options > linkReferenceStyle',
type: new Set(['full', 'collapsed', 'shortcut']),
type: new Set(['full', 'collapsed', 'shortcut', 'discarded']),
})
linkReferenceStyle?: 'full' | 'collapsed' | 'shortcut';
linkReferenceStyle?: 'full' | 'collapsed' | 'shortcut' | 'discarded';
static fromCtx(ctx: Context, prefix= 'x-md-') {
const draft: Record<string, string> = {};

72
src/services/snapshot-formatter.ts
View File

@ -655,22 +655,37 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
return `${trimmed.replace(/\n{3,}/g, '\n\n')}\n\n`;
}
});
turnDownService.addRule('improved-inline-link', {
filter: function (node, options) {
let realLinkStyle: 'inlined' | 'collapsed' | 'shortcut' | 'referenced' | 'discarded' = 'inlined';
if (turndownOpts?.linkStyle === 'referenced' || turndownOpts?.linkReferenceStyle) {
realLinkStyle = 'referenced';
if (turndownOpts?.linkReferenceStyle === 'collapsed') {
realLinkStyle = 'collapsed';
} else if (turndownOpts?.linkReferenceStyle === 'shortcut') {
realLinkStyle = 'shortcut';
} else if (turndownOpts?.linkReferenceStyle === 'discarded') {
realLinkStyle = 'discarded';
}
} else if (turndownOpts?.linkStyle === 'discarded') {
realLinkStyle = 'discarded';
}
turnDownService.addRule('improved-link', {
filter: function (node, _options) {
return Boolean(
options.linkStyle === 'inlined' &&
node.nodeName === 'A' &&
node.getAttribute('href')
);
},
replacement: function (content, node: any) {
const href = node.getAttribute('href');
replacement: function (this: { references: string[]; }, content, node: any) {
var href = node.getAttribute('href');
let title = cleanAttribute(node.getAttribute('title'));
if (title) title = ' "' + title.replace(/"/g, '\\"') + '"';
if (title) title = ` "${title.replace(/"/g, '\\"')}"`;
let replacement;
let reference;
const fixedContent = content.replace(/\s+/g, ' ').trim();
let fixedHref = href.replace(/\s+/g, '').trim();
let fixedHref = href;
if (options?.url) {
try {
fixedHref = new URL(fixedHref, options.url).toString();
@ -679,7 +694,46 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
}
}
return `[${fixedContent}](${fixedHref}${title || ''})`;
switch (realLinkStyle) {
case 'inlined':
replacement = `[${fixedContent}](${fixedHref}${title || ''})`;
reference = undefined;
break;
case 'collapsed':
replacement = `[${fixedContent}][]`;
reference = `[${fixedContent}]: ${fixedHref}${title}`;
break;
case 'shortcut':
replacement = `[${fixedContent}]`;
reference = `[${fixedContent}]: ${fixedHref}${title}`;
break;
case 'discarded':
replacement = content;
reference = undefined;
break;
default:
const id = this.references.length + 1;
replacement = `[${fixedContent}][${id}]`;
reference = `[${id}]${fixedHref}${title}`;
}
if (reference) {
this.references.push(reference);
}
return replacement;
},
// @ts-ignore
references: [],
append: function (this: { references: string[]; }) {
let references = '';
if (this.references.length) {
references = `\n\n${this.references.join('\n')}\n\n`;
this.references = []; // Reset references
}
return references;
}
});
turnDownService.addRule('improved-code', {