AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 04:55:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

includehtml fix

Browse Source
This commit is contained in:
rafaelsideguide 2024-08-21 15:54:00 -03:00
parent 5818236659
commit fe2e8c0b7a
5 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/controllers/v0/crawlPreview.ts
View File

@ -44,7 +44,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
} }
const crawlerOptions = req.body.crawlerOptions ?? {}; const crawlerOptions = req.body.crawlerOptions ?? {};
const pageOptions = req.body.pageOptions ?? { onlyMainContent: false, includeHtml: false, removeTags: [] }; const pageOptions = req.body.pageOptions ?? { onlyMainContent: false, includeHtml: true, removeTags: [] };
// if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this? // if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?
// try { // try {

2
apps/api/src/controllers/v0/search.ts
View File

@ -132,7 +132,7 @@ export async function searchController(req: Request, res: Response) {
} }
const crawlerOptions = req.body.crawlerOptions ?? {}; const crawlerOptions = req.body.crawlerOptions ?? {};
const pageOptions = req.body.pageOptions ?? { const pageOptions = req.body.pageOptions ?? {
includeHtml: false, includeHtml: true,
onlyMainContent: true, onlyMainContent: true,
fetchPageContent: true, fetchPageContent: true,
removeTags: [], removeTags: [],

4
apps/api/src/lib/default-values.ts
View File

@ -4,7 +4,7 @@ export const defaultTimeout = 45000; // 45 seconds
export const defaultPageOptions = { export const defaultPageOptions = {
onlyMainContent: false, onlyMainContent: false,
includeHtml: false, includeHtml: true,
waitFor: 0, waitFor: 0,
screenshot: false, screenshot: false,
fullPageScreenshot: false, fullPageScreenshot: false,
@ -17,7 +17,7 @@ export const defaultCrawlerOptions = {
export const defaultCrawlPageOptions = { export const defaultCrawlPageOptions = {
onlyMainContent: false, onlyMainContent: false,
includeHtml: false, includeHtml: true,
removeTags: [], removeTags: [],
parsePDF: true parsePDF: true
} }

2
apps/api/src/scraper/WebScraper/index.ts
View File

@ -574,7 +574,7 @@ export class WebScraperDataProvider {
options.crawlerOptions?.generateImgAltText ?? false; options.crawlerOptions?.generateImgAltText ?? false;
this.pageOptions = options.pageOptions ?? { this.pageOptions = options.pageOptions ?? {
onlyMainContent: false, onlyMainContent: false,
includeHtml: false, includeHtml: true,
replaceAllPathsWithAbsolutePaths: false, replaceAllPathsWithAbsolutePaths: false,
parsePDF: true, parsePDF: true,
removeTags: [], removeTags: [],

2
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -125,7 +125,7 @@ export async function scrapSingleUrl(
pageOptions: PageOptions = { pageOptions: PageOptions = {
includeMarkdown: true, includeMarkdown: true,
onlyMainContent: true, onlyMainContent: true,
includeHtml: false, includeHtml: true,
includeRawHtml: false, includeRawHtml: false,
waitFor: 0, waitFor: 0,
screenshot: false, screenshot: false,