diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts index 449f9106..ffa04b82 100644 --- a/apps/api/src/controllers/v1/scrape.ts +++ b/apps/api/src/controllers/v1/scrape.ts @@ -9,13 +9,12 @@ import { scrapeQueueEvents } from '../../services/queue-service'; import { logJob } from "../../services/logging/log_job"; export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>, res: Response) { - req.body = scrapeRequestSchema.parse(req.body); + req.body = scrapeRequestSchema.parse(req.body); let earlyReturn = false; const origin = req.body.origin; const timeout = req.body.timeout; const pageOptions = legacyScrapeOptions(req.body); - const jobId = uuidv4(); const startTime = new Date().getTime(); diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index ef04785a..45a36bb1 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -293,7 +293,10 @@ export class WebScraperDataProvider { documents = await this.getSitemapData(this.urls[0], documents); } - documents = this.applyPathReplacements(documents); + if (this.pageOptions.includeMarkdown) { + documents = this.applyPathReplacements(documents); + } + // documents = await this.applyImgAltText(documents); if ( (this.extractorOptions.mode === "llm-extraction" || diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.ts index f0a0303d..971a0b0d 100644 --- a/apps/api/src/search/fireEngine.ts +++ b/apps/api/src/search/fireEngine.ts @@ -14,12 +14,12 @@ export async function fireEngineSearch(q: string, options: { page?: number; }): Promise { let data = JSON.stringify({ - q: q, + query: q, lang: options.lang, country: options.country, location: options.location, tbs: options.tbs, - num: options.numResults, + numResults: options.numResults, page: options.page ?? 1, }); @@ -36,7 +36,7 @@ export async function fireEngineSearch(q: string, options: { data: data, }; const response = await axios(config); - if (response && response.data) { + if (response && response) { return response.data } else { return [];