AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-15 00:06:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: provide option to discard links

Browse Source
This commit is contained in:
Yanlong Wang 2025-03-12 10:46:01 +08:00
parent 7ded2de54c
commit cadb2fd94c
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
2 changed files with 67 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/dto/turndown-tweakable-options.ts
View File

@ -36,15 +36,15 @@ export class TurnDownTweakableOptions extends AutoCastable {
@Prop({ @Prop({
desc: 'Turndown options > linkStyle', desc: 'Turndown options > linkStyle',
type: new Set(['inlined', 'referenced']), type: new Set(['inlined', 'referenced', 'discarded']),
}) })
linkStyle?: 'inlined' | 'referenced'; linkStyle?: 'inlined' | 'referenced' | 'discarded';
@Prop({ @Prop({
desc: 'Turndown options > linkReferenceStyle', desc: 'Turndown options > linkReferenceStyle',
type: new Set(['full', 'collapsed', 'shortcut']), type: new Set(['full', 'collapsed', 'shortcut', 'discarded']),
}) })
linkReferenceStyle?: 'full' | 'collapsed' | 'shortcut'; linkReferenceStyle?: 'full' | 'collapsed' | 'shortcut' | 'discarded';
static fromCtx(ctx: Context, prefix= 'x-md-') { static fromCtx(ctx: Context, prefix= 'x-md-') {
const draft: Record<string, string> = {}; const draft: Record<string, string> = {};

72
src/services/snapshot-formatter.ts
View File

@ -655,22 +655,37 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
return `${trimmed.replace(/\n{3,}/g, '\n\n')}\n\n`; return `${trimmed.replace(/\n{3,}/g, '\n\n')}\n\n`;
} }
}); });
turnDownService.addRule('improved-inline-link', {
filter: function (node, options) { let realLinkStyle: 'inlined' | 'collapsed' | 'shortcut' | 'referenced' | 'discarded' = 'inlined';
if (turndownOpts?.linkStyle === 'referenced' || turndownOpts?.linkReferenceStyle) {
realLinkStyle = 'referenced';
if (turndownOpts?.linkReferenceStyle === 'collapsed') {
realLinkStyle = 'collapsed';
} else if (turndownOpts?.linkReferenceStyle === 'shortcut') {
realLinkStyle = 'shortcut';
} else if (turndownOpts?.linkReferenceStyle === 'discarded') {
realLinkStyle = 'discarded';
}
} else if (turndownOpts?.linkStyle === 'discarded') {
realLinkStyle = 'discarded';
}
turnDownService.addRule('improved-link', {
filter: function (node, _options) {
return Boolean( return Boolean(
options.linkStyle === 'inlined' &&
node.nodeName === 'A' && node.nodeName === 'A' &&
node.getAttribute('href') node.getAttribute('href')
); );
}, },
replacement: function (content, node: any) { replacement: function (this: { references: string[]; }, content, node: any) {
const href = node.getAttribute('href'); var href = node.getAttribute('href');
let title = cleanAttribute(node.getAttribute('title')); let title = cleanAttribute(node.getAttribute('title'));
if (title) title = ' "' + title.replace(/"/g, '\\"') + '"'; if (title) title = ` "${title.replace(/"/g, '\\"')}"`;
let replacement;
let reference;
const fixedContent = content.replace(/\s+/g, ' ').trim(); const fixedContent = content.replace(/\s+/g, ' ').trim();
let fixedHref = href.replace(/\s+/g, '').trim(); let fixedHref = href;
if (options?.url) { if (options?.url) {
try { try {
fixedHref = new URL(fixedHref, options.url).toString(); fixedHref = new URL(fixedHref, options.url).toString();
@ -679,7 +694,46 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
} }
} }
return `[${fixedContent}](${fixedHref}${title || ''})`; switch (realLinkStyle) {
case 'inlined':
replacement = `[${fixedContent}](${fixedHref}${title || ''})`;
reference = undefined;
break;
case 'collapsed':
replacement = `[${fixedContent}][]`;
reference = `[${fixedContent}]: ${fixedHref}${title}`;
break;
case 'shortcut':
replacement = `[${fixedContent}]`;
reference = `[${fixedContent}]: ${fixedHref}${title}`;
break;
case 'discarded':
replacement = content;
reference = undefined;
break;
default:
const id = this.references.length + 1;
replacement = `[${fixedContent}][${id}]`;
reference = `[${id}]${fixedHref}${title}`;
}
if (reference) {
this.references.push(reference);
}
return replacement;
},
// @ts-ignore
references: [],
append: function (this: { references: string[]; }) {
let references = '';
if (this.references.length) {
references = `\n\n${this.references.join('\n')}\n\n`;
this.references = []; // Reset references
}
return references;
} }
}); });
turnDownService.addRule('improved-code', { turnDownService.addRule('improved-code', {