AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 18:55:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed markdown format

Browse Source
This commit is contained in:
rafaelsideguide 2024-08-16 18:39:13 -03:00
parent aabfaf0ac5
commit 086ba6280b
3 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/controllers/v1/scrape.ts
View File

@ -9,13 +9,12 @@ import { scrapeQueueEvents } from '../../services/queue-service';
import { logJob } from "../../services/logging/log_job"; import { logJob } from "../../services/logging/log_job";
export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>, res: Response<ScrapeResponse>) { export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>, res: Response<ScrapeResponse>) {
req.body = scrapeRequestSchema.parse(req.body); req.body = scrapeRequestSchema.parse(req.body);
let earlyReturn = false; let earlyReturn = false;
const origin = req.body.origin; const origin = req.body.origin;
const timeout = req.body.timeout; const timeout = req.body.timeout;
const pageOptions = legacyScrapeOptions(req.body); const pageOptions = legacyScrapeOptions(req.body);
const jobId = uuidv4(); const jobId = uuidv4();
const startTime = new Date().getTime(); const startTime = new Date().getTime();

5
apps/api/src/scraper/WebScraper/index.ts
View File

@ -293,7 +293,10 @@ export class WebScraperDataProvider {
documents = await this.getSitemapData(this.urls[0], documents); documents = await this.getSitemapData(this.urls[0], documents);
} }
documents = this.applyPathReplacements(documents); if (this.pageOptions.includeMarkdown) {
documents = this.applyPathReplacements(documents);
}
// documents = await this.applyImgAltText(documents); // documents = await this.applyImgAltText(documents);
if ( if (
(this.extractorOptions.mode === "llm-extraction" || (this.extractorOptions.mode === "llm-extraction" ||

6
apps/api/src/search/fireEngine.ts
View File

@ -14,12 +14,12 @@ export async function fireEngineSearch(q: string, options: {
page?: number; page?: number;
}): Promise<SearchResult[]> { }): Promise<SearchResult[]> {
let data = JSON.stringify({ let data = JSON.stringify({
q: q, query: q,
lang: options.lang, lang: options.lang,
country: options.country, country: options.country,
location: options.location, location: options.location,
tbs: options.tbs, tbs: options.tbs,
num: options.numResults, numResults: options.numResults,
page: options.page ?? 1, page: options.page ?? 1,
}); });
@ -36,7 +36,7 @@ export async function fireEngineSearch(q: string, options: {
data: data, data: data,
}; };
const response = await axios(config); const response = await axios(config);
if (response && response.data) { if (response && response) {
return response.data return response.data
} else { } else {
return []; return [];