mirror of
https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl
synced 2025-08-12 16:49:00 +08:00
FIR-1951: Add automatic URL encoding in preprocessing for special characters in query parameters (#1547)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Nicolas Camara <nicolascamara29@gmail.com>
This commit is contained in:
parent
7ccbbec488
commit
8eeb3c5cd4
@ -0,0 +1,34 @@
|
||||
import { url } from "../../../controllers/v1/types";
|
||||
import { describe, it, expect } from "@jest/globals";
|
||||
|
||||
describe("URL Schema Validation with Special Characters", () => {
|
||||
it("should handle URLs with special characters in query parameters", () => {
|
||||
const testUrl = "https://www.boulanger.com/c/nav-filtre/televiseur?_merchant_des~boulanger|brand~lg";
|
||||
|
||||
expect(() => url.parse(testUrl)).not.toThrow();
|
||||
|
||||
const parsedUrl = url.parse(testUrl);
|
||||
expect(parsedUrl).toContain("_merchant_des%7Eboulanger%7Cbrand%7Elg");
|
||||
});
|
||||
|
||||
it("should preserve URL structure when encoding special characters", () => {
|
||||
const testUrl = "https://example.com/path?param1=value1¶m2=value~with|special¶m3=normal";
|
||||
|
||||
expect(() => url.parse(testUrl)).not.toThrow();
|
||||
|
||||
const parsedUrl = url.parse(testUrl);
|
||||
expect(parsedUrl).toContain("example.com/path?");
|
||||
expect(parsedUrl).toContain("param1=value1");
|
||||
expect(parsedUrl).toContain("param2=value%7Ewith%7Cspecial");
|
||||
expect(parsedUrl).toContain("param3=normal");
|
||||
});
|
||||
|
||||
it("should handle URLs with already encoded special characters", () => {
|
||||
const testUrl = "https://example.com/path?param=value%7Eencoded";
|
||||
|
||||
expect(() => url.parse(testUrl)).not.toThrow();
|
||||
|
||||
const parsedUrl = url.parse(testUrl);
|
||||
expect(parsedUrl).toContain("param=value%7Eencoded");
|
||||
});
|
||||
});
|
@ -24,8 +24,18 @@ export type Format =
|
||||
export const url = z.preprocess(
|
||||
(x) => {
|
||||
if (!protocolIncluded(x as string)) {
|
||||
return `http://${x}`;
|
||||
x = `http://${x}`;
|
||||
}
|
||||
|
||||
try {
|
||||
const urlObj = new URL(x as string);
|
||||
if (urlObj.search) {
|
||||
const searchParams = new URLSearchParams(urlObj.search.substring(1));
|
||||
return `${urlObj.origin}${urlObj.pathname}?${searchParams.toString()}`;
|
||||
}
|
||||
} catch (e) {
|
||||
}
|
||||
|
||||
return x;
|
||||
},
|
||||
z
|
||||
|
Loading…
x
Reference in New Issue
Block a user