mirror of
https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl
synced 2025-08-12 19:08:58 +08:00
FIR-1951: Add automatic URL encoding in preprocessing for special characters in query parameters (#1547)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Nicolas Camara <nicolascamara29@gmail.com>
This commit is contained in:
parent
7ccbbec488
commit
8eeb3c5cd4
@ -0,0 +1,34 @@
|
|||||||
|
import { url } from "../../../controllers/v1/types";
|
||||||
|
import { describe, it, expect } from "@jest/globals";
|
||||||
|
|
||||||
|
describe("URL Schema Validation with Special Characters", () => {
|
||||||
|
it("should handle URLs with special characters in query parameters", () => {
|
||||||
|
const testUrl = "https://www.boulanger.com/c/nav-filtre/televiseur?_merchant_des~boulanger|brand~lg";
|
||||||
|
|
||||||
|
expect(() => url.parse(testUrl)).not.toThrow();
|
||||||
|
|
||||||
|
const parsedUrl = url.parse(testUrl);
|
||||||
|
expect(parsedUrl).toContain("_merchant_des%7Eboulanger%7Cbrand%7Elg");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should preserve URL structure when encoding special characters", () => {
|
||||||
|
const testUrl = "https://example.com/path?param1=value1¶m2=value~with|special¶m3=normal";
|
||||||
|
|
||||||
|
expect(() => url.parse(testUrl)).not.toThrow();
|
||||||
|
|
||||||
|
const parsedUrl = url.parse(testUrl);
|
||||||
|
expect(parsedUrl).toContain("example.com/path?");
|
||||||
|
expect(parsedUrl).toContain("param1=value1");
|
||||||
|
expect(parsedUrl).toContain("param2=value%7Ewith%7Cspecial");
|
||||||
|
expect(parsedUrl).toContain("param3=normal");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle URLs with already encoded special characters", () => {
|
||||||
|
const testUrl = "https://example.com/path?param=value%7Eencoded";
|
||||||
|
|
||||||
|
expect(() => url.parse(testUrl)).not.toThrow();
|
||||||
|
|
||||||
|
const parsedUrl = url.parse(testUrl);
|
||||||
|
expect(parsedUrl).toContain("param=value%7Eencoded");
|
||||||
|
});
|
||||||
|
});
|
@ -24,8 +24,18 @@ export type Format =
|
|||||||
export const url = z.preprocess(
|
export const url = z.preprocess(
|
||||||
(x) => {
|
(x) => {
|
||||||
if (!protocolIncluded(x as string)) {
|
if (!protocolIncluded(x as string)) {
|
||||||
return `http://${x}`;
|
x = `http://${x}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const urlObj = new URL(x as string);
|
||||||
|
if (urlObj.search) {
|
||||||
|
const searchParams = new URLSearchParams(urlObj.search.substring(1));
|
||||||
|
return `${urlObj.origin}${urlObj.pathname}?${searchParams.toString()}`;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
}
|
||||||
|
|
||||||
return x;
|
return x;
|
||||||
},
|
},
|
||||||
z
|
z
|
||||||
|
Loading…
x
Reference in New Issue
Block a user