diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts index 01b480c9..f135c0b5 100644 --- a/apps/api/src/__tests__/snips/scrape.test.ts +++ b/apps/api/src/__tests__/snips/scrape.test.ts @@ -70,8 +70,6 @@ describe("Scrape tests", () => { formats: ["rawHtml"], }); - console.log(response.rawHtml, response.metadata.contentType); - const obj = JSON.parse(response.rawHtml!); expect(obj.id).toBe(1); }, 30000); diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts index df061b69..57ae8f6b 100644 --- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts @@ -3,6 +3,7 @@ import { EngineScrapeResult } from ".."; import { Meta } from "../.."; import { TimeoutError } from "../../error"; import { robustFetch } from "../../lib/fetch"; +import { getInnerJSON } from "../../../../lib/html-transformer"; export async function scrapeURLWithPlaywright( meta: Meta, @@ -42,6 +43,10 @@ export async function scrapeURLWithPlaywright( })(), ]); + if (response.contentType?.includes("application/json")) { + response.content = await getInnerJSON(response.content); + } + return { url: meta.url, // TODO: impove redirect following html: response.content,