diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index dd0faff7..a9449431 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -454,6 +454,7 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) { allowBackwardCrawling: x.allowBackwardLinks, allowExternalContentLinks: x.allowExternalLinks, ignoreSitemap: x.ignoreSitemap, + deduplicateSimilarURLs: x.deduplicateSimilarURLs, }; } @@ -467,7 +468,7 @@ export function fromLegacyCrawlerOptions(x: any): { crawlOptions: CrawlerOptions allowBackwardLinks: x.allowBackwardCrawling, allowExternalLinks: x.allowExternalContentLinks, ignoreSitemap: x.ignoreSitemap, - // TODO: returnOnlyUrls support + deduplicateSimilarURLs: x.deduplicateSimilarURLs, }), internalOptions: { v0CrawlOnlyUrls: x.returnOnlyUrls,