From e23d9f30a6572e7e811155d373d31e376ddf19c1 Mon Sep 17 00:00:00 2001 From: "yanlong.wang" Date: Thu, 16 Jan 2025 15:37:16 +0800 Subject: [PATCH] fix: base parameter --- backend/functions/src/cloud-functions/crawler.ts | 2 +- backend/functions/src/dto/scrapping-options.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/functions/src/cloud-functions/crawler.ts b/backend/functions/src/cloud-functions/crawler.ts index 839e76e..412e37a 100644 --- a/backend/functions/src/cloud-functions/crawler.ts +++ b/backend/functions/src/cloud-functions/crawler.ts @@ -832,7 +832,7 @@ export class CrawlerHost extends RPCHost { nominalUrl?: URL, urlValidMs?: number ) { - const presumedURL = crawlerOptions.base === 'eventual' ? new URL(snapshot.href) : nominalUrl; + const presumedURL = crawlerOptions.base === 'final' ? new URL(snapshot.href) : nominalUrl; const respondWith = crawlerOptions.respondWith; if (respondWith === CONTENT_FORMAT.READER_LM || respondWith === CONTENT_FORMAT.VLM) { diff --git a/backend/functions/src/dto/scrapping-options.ts b/backend/functions/src/dto/scrapping-options.ts index bb3521e..d7059ee 100644 --- a/backend/functions/src/dto/scrapping-options.ts +++ b/backend/functions/src/dto/scrapping-options.ts @@ -25,7 +25,7 @@ const CONTENT_FORMAT_VALUES = new Set(Object.values(CONTENT_FORMAT)); export const IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p'] as const; const IMAGE_RETENTION_MODE_VALUES = new Set(IMAGE_RETENTION_MODES); -export const BASE_URL_MODES = ['initial', 'eventual'] as const; +export const BASE_URL_MODES = ['initial', 'final'] as const; const BASE_URL_MODE_VALUES = new Set(BASE_URL_MODES); class Viewport extends AutoCastable { @@ -199,7 +199,7 @@ class Viewport extends AutoCastable { schema: { type: 'string' } }, 'X-Base': { - description: 'Select base modes of relative URLs.\n\nSupported: initial, eventual', + description: 'Select base modes of relative URLs.\n\nSupported: initial, final', in: 'header', schema: { type: 'string' } }, @@ -490,7 +490,9 @@ export class CrawlerOptions extends AutoCastable { instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined; const baseMode = ctx?.req.get('x-base') || undefined; - instance.base ??= baseMode as any; + if (baseMode) { + instance.base = baseMode as any; + } if (instance.cacheTolerance) { instance.cacheTolerance = instance.cacheTolerance * 1000;