From e24bbcc6eb2198c5038423f96acf9fa28611ab55 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Sun, 7 Jul 2024 14:40:52 +0200 Subject: [PATCH 01/23] feat: rust sdk initial commit --- apps/rust-sdk/.gitignore | 1 + apps/rust-sdk/Cargo.lock | 1223 ++++++++++++++++++++++++++++++++++++++ apps/rust-sdk/Cargo.toml | 22 + apps/rust-sdk/Makefile | 10 + apps/rust-sdk/README.md | 1 + apps/rust-sdk/src/lib.rs | 373 ++++++++++++ 6 files changed, 1630 insertions(+) create mode 100644 apps/rust-sdk/.gitignore create mode 100644 apps/rust-sdk/Cargo.lock create mode 100644 apps/rust-sdk/Cargo.toml create mode 100644 apps/rust-sdk/Makefile create mode 100644 apps/rust-sdk/README.md create mode 100644 apps/rust-sdk/src/lib.rs diff --git a/apps/rust-sdk/.gitignore b/apps/rust-sdk/.gitignore new file mode 100644 index 00000000..2f7896d1 --- /dev/null +++ b/apps/rust-sdk/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock new file mode 100644 index 00000000..c4e0af8e --- /dev/null +++ b/apps/rust-sdk/Cargo.lock @@ -0,0 +1,1223 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "cc" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + +[[package]] +name = "firecrawl-rs" +version = "0.1.0" +dependencies = [ + "log", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.204" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.204" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.120" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "syn" +version = "2.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6b6a2fb3a985e99cebfaefa9faa3024743da73304ca1c683a36429613d3d22" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml new file mode 100644 index 00000000..6c75a16c --- /dev/null +++ b/apps/rust-sdk/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "firecrawl-rs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +reqwest = { version = "0.11", features = ["json", "blocking"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +log = "0.4" +thiserror = "1.0" + +[dependencies.tokio] +version = "1" +features = ["full"] + +[dev-dependencies] +tokio = { version = "1", features = ["full"] } + +[build-dependencies] +tokio = { version = "1", features = ["full"] } diff --git a/apps/rust-sdk/Makefile b/apps/rust-sdk/Makefile new file mode 100644 index 00000000..620047de --- /dev/null +++ b/apps/rust-sdk/Makefile @@ -0,0 +1,10 @@ +.PHONY: build run format + +build: + cargo build + +run: build + cargo run + +format: + cargo fmt diff --git a/apps/rust-sdk/README.md b/apps/rust-sdk/README.md new file mode 100644 index 00000000..7383ec54 --- /dev/null +++ b/apps/rust-sdk/README.md @@ -0,0 +1 @@ +# Firecrawl Rust SDK diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rs new file mode 100644 index 00000000..095c9105 --- /dev/null +++ b/apps/rust-sdk/src/lib.rs @@ -0,0 +1,373 @@ +/* +* +* - Structs and Enums: +* FirecrawlError: Custom error enum for handling various errors. +* FirecrawlApp: Main struct for the application, holding API key, URL, and HTTP client. +* +* - Initialization: +* +* FirecrawlApp::new initializes the struct, fetching the API key and URL from environment variables if not provided. +* +* - API Methods: +* scrape_url, search, crawl_url, check_crawl_status: +* Methods for interacting with the Firecrawl API, similar to the Python methods. +* monitor_job_status: Polls the API to monitor the status of a crawl job until completion. +*/ + +use std::env; +use std::thread; +use std::time::Duration; + +use log::debug; +use reqwest::{Client, Response}; +use serde_json::json; +use serde_json::Value; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum FirecrawlError { + #[error("HTTP request failed: {0}")] + HttpRequestFailed(String), + #[error("API key not provided")] + ApiKeyNotProvided, + #[error("Failed to parse response: {0}")] + ResponseParseError(String), + #[error("Crawl job failed or stopped: {0}")] + CrawlJobFailed(String), +} + +#[derive(Clone)] +pub struct FirecrawlApp { + api_key: String, + api_url: String, + client: Client, +} +// the api verstion of firecrawl +const API_VERSION: &str = "/v0"; + +impl FirecrawlApp { + /// Initialize the FirecrawlApp instance. + /// + /// # Arguments: + /// * `api_key` (Optional[str]): API key for authenticating with the Firecrawl API. + /// * `api_url` (Optional[str]): Base URL for the Firecrawl API. + pub fn new(api_key: Option, api_url: Option) -> Result { + let api_key = api_key + .or_else(|| env::var("FIRECRAWL_API_KEY").ok()) + .ok_or(FirecrawlError::ApiKeyNotProvided)?; + let api_url = api_url.unwrap_or_else(|| { + env::var("FIRECRAWL_API_URL") + .unwrap_or_else(|_| "https://api.firecrawl.dev".to_string()) + }); + + debug!("Initialized FirecrawlApp with API key: {}", api_key); + debug!("Initialized FirecrawlApp with API URL: {}", api_url); + + Ok(FirecrawlApp { + api_key, + api_url, + client: Client::new(), + }) + } + + /// Scrape the specified URL using the Firecrawl API. + /// + /// # Arguments: + /// * `url` (str): The URL to scrape. + /// * `params` (Optional[Dict[str, Any]]): Additional parameters for the scrape request. + /// + /// # Returns: + /// * `Any`: The scraped data if the request is successful. + /// + /// # Raises: + /// * `Exception`: If the scrape request fails. + pub async fn scrape_url( + &self, + url: &str, + params: Option, + ) -> Result { + let headers = self.prepare_headers(None); + let mut scrape_params = json!({"url": url}); + + if let Some(mut params) = params { + if let Some(extractor_options) = params.get_mut("extractorOptions") { + if let Some(extraction_schema) = extractor_options.get_mut("extractionSchema") { + if extraction_schema.is_object() && extraction_schema.get("schema").is_some() { + extractor_options["extractionSchema"] = extraction_schema["schema"].clone(); + } + extractor_options["mode"] = extractor_options + .get("mode") + .cloned() + .unwrap_or_else(|| json!("llm-extraction")); + } + scrape_params["extractorOptions"] = extractor_options.clone(); + } + for (key, value) in params.as_object().unwrap() { + if key != "extractorOptions" { + scrape_params[key] = value.clone(); + } + } + } + + let response = self + .client + .post(&format!("{}{}/scrape", self.api_url, API_VERSION)) + .headers(headers) + .json(&scrape_params) + .send() + .await + .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?; + + self.handle_response(response, "scrape URL").await + } + + /// Perform a search using the Firecrawl API. + /// + /// # Arguments: + /// * `query` (str): The search query. + /// * `params` (Optional[Dict[str, Any]]): Additional parameters for the search request. + /// + /// # Returns: + /// * `Any`: The search results if the request is successful. + /// + /// # Raises: + /// * `Exception`: If the search request fails. + pub async fn search( + &self, + query: &str, + params: Option, + ) -> Result { + let headers = self.prepare_headers(None); + let mut json_data = json!({"query": query}); + if let Some(params) = params { + for (key, value) in params.as_object().unwrap() { + json_data[key] = value.clone(); + } + } + + let response = self + .client + .post(&format!("{}{}/search", self.api_url, API_VERSION)) + .headers(headers) + .json(&json_data) + .send() + .await + .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?; + + self.handle_response(response, "search").await + } + + /// Initiate a crawl job for the specified URL using the Firecrawl API. + /// + /// # Arguments: + /// * `url` (str): The URL to crawl. + /// * `params` (Optional[Dict[str, Any]]): Additional parameters for the crawl request. + /// * `wait_until_done` (bool): Whether to wait until the crawl job is completed. + /// * `poll_interval` (int): Time in seconds between status checks when waiting for job completion. + /// * `idempotency_key` (Optional[str]): A unique uuid key to ensure idempotency of requests. + /// + /// # Returns: + /// * `Any`: The crawl job ID or the crawl results if waiting until completion. + /// + /// # `Raises`: + /// * `Exception`: If the crawl job initiation or monitoring fails. + pub async fn crawl_url( + &self, + url: &str, + params: Option, + wait_until_done: bool, + poll_interval: u64, + idempotency_key: Option, + ) -> Result { + let headers = self.prepare_headers(idempotency_key); + let mut json_data = json!({"url": url}); + if let Some(params) = params { + for (key, value) in params.as_object().unwrap() { + json_data[key] = value.clone(); + } + } + + let response = self + .client + .post(&format!("{}{}/crawl", self.api_url, API_VERSION)) + .headers(headers.clone()) + .json(&json_data) + .send() + .await + .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?; + + let response_json = self.handle_response(response, "start crawl job").await?; + let job_id = response_json["jobId"].as_str().unwrap().to_string(); + + if wait_until_done { + self.monitor_job_status(&job_id, headers, poll_interval) + .await + } else { + Ok(json!({"jobId": job_id})) + } + } + + /// Check the status of a crawl job using the Firecrawl API. + /// + /// # Arguments: + /// * `job_id` (str): The ID of the crawl job. + /// + /// # Returns: + /// * `Any`: The status of the crawl job. + /// + /// # Raises: + /// * `Exception`: If the status check request fails. + pub async fn check_crawl_status(&self, job_id: &str) -> Result { + let headers = self.prepare_headers(None); + let response = self + .client + .get(&format!( + "{}{}/crawl/status/{}", + self.api_url, API_VERSION, job_id + )) + .headers(headers) + .send() + .await + .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?; + + self.handle_response(response, "check crawl status").await + } + + /// Monitor the status of a crawl job until completion. + /// + /// # Arguments: + /// * `job_id` (str): The ID of the crawl job. + /// * `headers` (Dict[str, str]): The headers to include in the status check requests. + /// * `poll_interval` (int): Secounds between status checks. + /// + /// # Returns: + /// * `Any`: The crawl results if the job is completed successfully. + /// + /// # Raises: + /// Exception: If the job fails or an error occurs during status checks. + async fn monitor_job_status( + &self, + job_id: &str, + headers: reqwest::header::HeaderMap, + poll_interval: u64, + ) -> Result { + loop { + let response = self + .client + .get(&format!( + "{}{}/crawl/status/{}", + self.api_url, API_VERSION, job_id + )) + .headers(headers.clone()) + .send() + .await + .map_err(|e| FirecrawlError::HttpRequestFailed(e.to_string()))?; + + let status_data = self.handle_response(response, "check crawl status").await?; + match status_data["status"].as_str() { + Some("completed") => { + if status_data["data"].is_object() { + return Ok(status_data["data"].clone()); + } else { + return Err(FirecrawlError::CrawlJobFailed( + "Crawl job completed but no data was returned".to_string(), + )); + } + } + Some("active") | Some("paused") | Some("pending") | Some("queued") + | Some("waiting") => { + thread::sleep(Duration::from_secs(poll_interval)); + } + Some(status) => { + return Err(FirecrawlError::CrawlJobFailed(format!( + "Crawl job failed or was stopped. Status: {}", + status + ))); + } + None => { + return Err(FirecrawlError::CrawlJobFailed( + "Unexpected response: no status field".to_string(), + )); + } + } + } + } + + /// Prepare the headers for API requests. + /// + /// # Arguments: + /// `idempotency_key` (Optional[str]): A unique key to ensure idempotency of requests. + /// + /// # Returns: + /// Dict[str, str]: The headers including content type, authorization, and optionally idempotency key. + fn prepare_headers(&self, idempotency_key: Option) -> reqwest::header::HeaderMap { + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert("Content-Type", "application/json".parse().unwrap()); + headers.insert( + "Authorization", + format!("Bearer {}", self.api_key).parse().unwrap(), + ); + if let Some(key) = idempotency_key { + headers.insert("x-idempotency-key", key.parse().unwrap()); + } + headers + } + + /// Handle errors from API responses. + /// + /// # Arguments: + /// * `response` (requests.Response): The response object from the API request. + /// * `action` (str): Description of the action that was being performed. + /// + /// # Raises: + /// Exception: An exception with a message containing the status code and error details from the response. + async fn handle_response( + &self, + response: Response, + action: &str, + ) -> Result { + if response.status().is_success() { + let response_json: Value = response + .json() + .await + .map_err(|e| FirecrawlError::ResponseParseError(e.to_string()))?; + if response_json["success"].as_bool().unwrap_or(false) { + Ok(response_json["data"].clone()) + } else { + Err(FirecrawlError::HttpRequestFailed(format!( + "Failed to {}: {}", + action, response_json["error"] + ))) + } + } else { + let status_code = response.status().as_u16(); + let error_message = response + .json::() + .await + .unwrap_or_else(|_| json!({"error": "No additional error details provided."})); + let message = match status_code { + 402 => format!( + "Payment Required: Failed to {}. {}", + action, error_message["error"] + ), + 408 => format!( + "Request Timeout: Failed to {} as the request timed out. {}", + action, error_message["error"] + ), + 409 => format!( + "Conflict: Failed to {} due to a conflict. {}", + action, error_message["error"] + ), + 500 => format!( + "Internal Server Error: Failed to {}. {}", + action, error_message["error"] + ), + _ => format!( + "Unexpected error during {}: Status code {}. {}", + action, status_code, error_message["error"] + ), + }; + Err(FirecrawlError::HttpRequestFailed(message)) + } + } +} From 3c49f0e055f760a97676aee18efce7488d11cc47 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Thu, 11 Jul 2024 16:47:44 +0200 Subject: [PATCH 02/23] chore: update deps by allowing bumps + add uuid --- apps/rust-sdk/Cargo.lock | 265 ++++++++++++++++++++++++++++++++++----- apps/rust-sdk/Cargo.toml | 11 +- apps/rust-sdk/README.md | 9 ++ 3 files changed, 246 insertions(+), 39 deletions(-) diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock index c4e0af8e..1b48393c 100644 --- a/apps/rust-sdk/Cargo.lock +++ b/apps/rust-sdk/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.3.0" @@ -40,9 +46,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" @@ -137,6 +143,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "uuid", ] [[package]] @@ -176,6 +183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -210,6 +218,7 @@ checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-core", "futures-io", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -217,6 +226,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.29.0" @@ -225,15 +245,15 @@ checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "h2" -version = "0.3.26" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http", "indexmap", "slab", @@ -256,9 +276,9 @@ checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "http" -version = "0.2.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -267,12 +287,24 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", "pin-project-lite", ] @@ -282,47 +314,77 @@ version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "hyper" -version = "0.14.29" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", "h2", "http", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] -name = "hyper-tls" -version = "0.5.0" +name = "hyper-rustls" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", + "http-body-util", "hyper", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -541,6 +603,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -588,20 +670,24 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.27" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ "base64", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", "http", "http-body", + "http-body-util", "hyper", + "hyper-rustls", "hyper-tls", + "hyper-util", "ipnet", "js-sys", "log", @@ -626,6 +712,21 @@ dependencies = [ "winreg", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -646,12 +747,43 @@ dependencies = [ ] [[package]] -name = "rustls-pemfile" -version = "1.0.4" +name = "rustls" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + +[[package]] +name = "rustls-webpki" +version = "0.102.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -775,6 +907,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.69" @@ -788,9 +932,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" [[package]] name = "system-configuration" @@ -900,6 +1044,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.11" @@ -913,6 +1068,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -965,6 +1141,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.2" @@ -976,6 +1158,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "getrandom", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -1214,10 +1405,16 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winreg" -version = "0.50.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", "windows-sys 0.48.0", ] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index 6c75a16c..4175f1df 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -5,11 +5,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -reqwest = { version = "0.11", features = ["json", "blocking"] } -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -log = "0.4" -thiserror = "1.0" +reqwest = { version = "^0.12", features = ["json", "blocking"] } +serde = { version = "^1.0", features = ["derive"] } +serde_json = "^1.0" +log = "^0.4" +thiserror = "^1.0" +uuid = { version = "^1.10", features = ["v4"] } [dependencies.tokio] version = "1" diff --git a/apps/rust-sdk/README.md b/apps/rust-sdk/README.md index 7383ec54..8783ddff 100644 --- a/apps/rust-sdk/README.md +++ b/apps/rust-sdk/README.md @@ -1 +1,10 @@ # Firecrawl Rust SDK + +The Firecrawl Rust SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API. + +# TODO... +## How to test + +```rust +$ cargo run --example example +``` From 404aeb3daa056c216cee4b0dd77ffbc5ff50adcd Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Thu, 11 Jul 2024 16:48:04 +0200 Subject: [PATCH 03/23] feat: add an example rust code to tests it --- apps/rust-sdk/examples/example.rs | 76 +++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 apps/rust-sdk/examples/example.rs diff --git a/apps/rust-sdk/examples/example.rs b/apps/rust-sdk/examples/example.rs new file mode 100644 index 00000000..8b6caca4 --- /dev/null +++ b/apps/rust-sdk/examples/example.rs @@ -0,0 +1,76 @@ +use firecrawl_rs::FirecrawlApp; +use serde_json::json; +use uuid::Uuid; + +#[tokio::main] +async fn main() { + // Initialize the FirecrawlApp with the API key + let api_key = Some("fc-YOUR_API_KEY".to_string()); + let api_url = Some("http://0.0.0.0:3002".to_string()); + let app = FirecrawlApp::new(api_key, api_url).expect("Failed to initialize FirecrawlApp"); + + // Scrape a website + let scrape_result = app.scrape_url("https://firecrawl.dev", None).await; + match scrape_result { + Ok(data) => println!("Scrape Result:\n{}", data["markdown"]), + Err(e) => eprintln!("Scrape failed: {}", e), + } + + // Crawl a website + let random_uuid = String::from(Uuid::new_v4()); + let idempotency_key = Some(random_uuid); // optional idempotency key + let crawl_params = json!({ + "crawlerOptions": { + "excludes": ["blog/*"] + } + }); + let crawl_result = app + .crawl_url("https://mendable.ai", Some(crawl_params), true, 2, idempotency_key) + .await; + match crawl_result { + Ok(data) => println!("Crawl Result:\n{}", data), + Err(e) => eprintln!("Crawl failed: {}", e), + } + + // LLM Extraction with a JSON schema + let json_schema = json!({ + "type": "object", + "properties": { + "top": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "points": {"type": "number"}, + "by": {"type": "string"}, + "commentsURL": {"type": "string"} + }, + "required": ["title", "points", "by", "commentsURL"] + }, + "minItems": 5, + "maxItems": 5, + "description": "Top 5 stories on Hacker News" + } + }, + "required": ["top"] + }); + + let llm_extraction_params = json!({ + "extractorOptions": { + "extractionSchema": json_schema, + "mode": "llm-extraction" + }, + "pageOptions": { + "onlyMainContent": true + } + }); + + let llm_extraction_result = app + .scrape_url("https://news.ycombinator.com", Some(llm_extraction_params)) + .await; + match llm_extraction_result { + Ok(data) => println!("LLM Extraction Result:\n{}", data["llm_extraction"]), + Err(e) => eprintln!("LLM Extraction failed: {}", e), + } +} From 04e94cdb20dc806a6393b0f2bb4eb5d8cc06d535 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 13:15:24 +0200 Subject: [PATCH 04/23] chore: update cargo.toml file by adding more elements --- apps/rust-sdk/Cargo.toml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index 4175f1df..1cf98418 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -1,7 +1,17 @@ [package] -name = "firecrawl-rs" +name = "firecrawl_rs" +author="Mendable.ai" version = "0.1.0" edition = "2021" +license = "MIT" +homepage = "https://www.firecrawl.dev/" +repository ="https://github.com/mendableai/firecrawl" +description = "Rust SDK for Firecrawl API." +authors = ["sanix-darker "] + +[lib] +path = "src/lib.rs" +name = "firecrawl_rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] From bcf596c310f6ddbe24a280a21bc4e63002b02e71 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 13:20:39 +0200 Subject: [PATCH 05/23] fix: license --- apps/rust-sdk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index 1cf98418..2f8d5a77 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -3,7 +3,7 @@ name = "firecrawl_rs" author="Mendable.ai" version = "0.1.0" edition = "2021" -license = "MIT" +license = "AGPL License" homepage = "https://www.firecrawl.dev/" repository ="https://github.com/mendableai/firecrawl" description = "Rust SDK for Firecrawl API." From 2f6215c4925b8b90a283d4107e1cfbe2787b67d9 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 13:58:38 +0200 Subject: [PATCH 06/23] feat: add dotenv and assert-match as dev dependencies --- apps/rust-sdk/Cargo.lock | 16 +++++++++++++++- apps/rust-sdk/Cargo.toml | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock index 1b48393c..d4c4cb60 100644 --- a/apps/rust-sdk/Cargo.lock +++ b/apps/rust-sdk/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + [[package]] name = "atomic-waker" version = "1.1.2" @@ -102,6 +108,12 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "encoding_rs" version = "0.8.34" @@ -134,9 +146,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] -name = "firecrawl-rs" +name = "firecrawl_rs" version = "0.1.0" dependencies = [ + "assert_matches", + "dotenv", "log", "reqwest", "serde", diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index 2f8d5a77..7d4bea4a 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -27,6 +27,8 @@ version = "1" features = ["full"] [dev-dependencies] +assert_matches = "^1.5" +dotenv = "^0.15" tokio = { version = "1", features = ["full"] } [build-dependencies] From 20932f7b318b6e41920ac65d39ad533ca3472d16 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 13:59:04 +0200 Subject: [PATCH 07/23] feat(test): start implementing tests for the lib --- apps/rust-sdk/tests/.env.example | 2 + apps/rust-sdk/tests/e2e_with_auth.rs | 237 +++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 apps/rust-sdk/tests/.env.example create mode 100644 apps/rust-sdk/tests/e2e_with_auth.rs diff --git a/apps/rust-sdk/tests/.env.example b/apps/rust-sdk/tests/.env.example new file mode 100644 index 00000000..5aa1cb11 --- /dev/null +++ b/apps/rust-sdk/tests/.env.example @@ -0,0 +1,2 @@ +API_URL=http://localhost:3002 +TEST_API_KEY=fc-YOUR_API_KEY diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs new file mode 100644 index 00000000..febf259a --- /dev/null +++ b/apps/rust-sdk/tests/e2e_with_auth.rs @@ -0,0 +1,237 @@ +use firecrawl_rs::FirecrawlApp; +use serde_json::json; +use uuid::Uuid; +use dotenv::dotenv; +use std::env; +use tokio::time::sleep; +use std::time::Duration; +use assert_matches::assert_matches; + +#[tokio::test] +async fn test_no_api_key() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + assert_matches!(FirecrawlApp::new(None, Some(api_url)), Err(e) if e.to_string() == "No API key provided"); +} + +#[tokio::test] +async fn test_scrape_url_invalid_api_key() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let app = FirecrawlApp::new(Some("invalid_api_key".to_string()), Some(api_url)).unwrap(); + let result = app.scrape_url("https://firecrawl.dev", None).await; + assert_matches!(result, Err(e) if e.to_string() == "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token"); +} + +#[tokio::test] +async fn test_blocklisted_url() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let api_key = env::var("TEST_API_KEY").unwrap(); + let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); + let blocklisted_url = "https://facebook.com/fake-test"; + let result = app.scrape_url(blocklisted_url, None).await; + assert_matches!(result, Err(e) if e.to_string() == "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it."); +} + +#[tokio::test] +async fn test_successful_response_with_valid_preview_token() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let app = FirecrawlApp::new(Some("this_is_just_a_preview_token".to_string()), Some(api_url)).unwrap(); + let result = app.scrape_url("https://roastmywebsite.ai", None).await.unwrap(); + assert!(result.as_object().unwrap().contains_key("content")); + assert!(result["content"].as_str().unwrap().contains("_Roast_")); +} + +#[tokio::test] +async fn test_scrape_url_e2e() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let api_key = env::var("TEST_API_KEY").unwrap(); + let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); + let result = app.scrape_url("https://roastmywebsite.ai", None).await.unwrap(); + assert!(result.as_object().unwrap().contains_key("content")); + assert!(result.as_object().unwrap().contains_key("markdown")); + assert!(result.as_object().unwrap().contains_key("metadata")); + assert!(!result.as_object().unwrap().contains_key("html")); + assert!(result["content"].as_str().unwrap().contains("_Roast_")); +} + +#[tokio::test] +async fn test_successful_response_with_valid_api_key_and_include_html() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let api_key = env::var("TEST_API_KEY").unwrap(); + let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); + let params = json!({ + "pageOptions": { + "includeHtml": true + } + }); + let result = app.scrape_url("https://roastmywebsite.ai", Some(params)).await.unwrap(); + assert!(result.as_object().unwrap().contains_key("content")); + assert!(result.as_object().unwrap().contains_key("markdown")); + assert!(result.as_object().unwrap().contains_key("html")); + assert!(result.as_object().unwrap().contains_key("metadata")); + assert!(result["content"].as_str().unwrap().contains("_Roast_")); + assert!(result["markdown"].as_str().unwrap().contains("_Roast_")); + assert!(result["html"].as_str().unwrap().contains(" 0); + // assert!(result_as_str[0].contains_key("content")); + // assert!(result[0]["content"].as_str().unwrap().contains("_Roast_")); +} + +#[tokio::test] +async fn test_crawl_url_with_idempotency_key_e2e() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let api_key = env::var("TEST_API_KEY").unwrap(); + let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); + let unique_idempotency_key = Uuid::new_v4().to_string(); + let params = json!({ + "crawlerOptions": { + "excludes": ["blog/*"] + } + }); + let result = app.crawl_url("https://roastmywebsite.ai", Some(params), true, 2, Some(unique_idempotency_key.clone())).await.unwrap(); + + let result_as_str = result.as_object().unwrap(); + assert!(result_as_str.len() > 0); + // assert!(result[0].contains_key("content")); + // assert!(result[0]["content"].as_str().unwrap().contains("_Roast_")); + + let conflict_result = app.crawl_url("https://firecrawl.dev", Some(params), true, 2, Some(unique_idempotency_key)).await; + assert_matches!(conflict_result, Err(e) if e.to_string() == "Conflict: Failed to start crawl job due to a conflict. Idempotency key already used"); +} + +#[tokio::test] +async fn test_check_crawl_status_e2e() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let api_key = env::var("TEST_API_KEY").unwrap(); + let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); + let params = json!({ + "crawlerOptions": { + "excludes": ["blog/*"] + } + }); + let result = app.crawl_url("https://firecrawl.dev", Some(params), false, 1, None).await.unwrap(); + assert!(result.as_object().unwrap().contains_key("jobId")); + + sleep(Duration::from_secs(30)).await; // wait for 30 seconds + let status_response = app.check_crawl_status(result["jobId"].as_str().unwrap()).await.unwrap(); + assert!(status_response.as_object().unwrap().contains_key("status")); + assert_eq!(status_response["status"].as_str().unwrap(), "completed"); + assert!(status_response.as_object().unwrap().contains_key("data")); + assert!(status_response["data"].as_array().unwrap().len() > 0); +} + +#[tokio::test] +async fn test_search_e2e() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let api_key = env::var("TEST_API_KEY").unwrap(); + let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); + let result = app.search("test query").await.unwrap(); + assert!(result.as_object().unwrap().len() > 2); + //assert!(result.as_object().unwrap()[0].contains_key("content")); +} + +#[tokio::test] +async fn test_search_invalid_api_key() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let app = FirecrawlApp::new(Some("invalid_api_key".to_string()), Some(api_url)).unwrap(); + let result = app.search("test query").await; + assert_matches!(result, Err(e) if e.to_string() == "Unexpected error during search: Status code 401. Unauthorized: Invalid token"); +} + +#[tokio::test] +async fn test_llm_extraction() { + dotenv().ok(); + let api_url = env::var("API_URL").unwrap(); + let api_key = env::var("TEST_API_KEY").unwrap(); + let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); + let params = json!({ + "extractorOptions": { + "mode": "llm-extraction", + "extractionPrompt": "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source", + "extractionSchema": { + "type": "object", + "properties": { + "company_mission": {"type": "string"}, + "supports_sso": {"type": "boolean"}, + "is_open_source": {"type": "boolean"} + }, + "required": ["company_mission", "supports_sso", "is_open_source"] + } + } + }); + let result = app.scrape_url("https://mendable.ai", Some(params)).await.unwrap(); + assert!(result.as_object().unwrap().contains_key("llm_extraction")); + let llm_extraction = &result["llm_extraction"]; + assert!(llm_extraction.as_object().unwrap().contains_key("company_mission")); + assert!(llm_extraction["supports_sso"].is_boolean()); + assert!(llm_extraction["is_open_source"].is_boolean()); +} From d8f67d504514e96cc67b528b368804cfea1cbd0d Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 13:59:41 +0200 Subject: [PATCH 08/23] chore(doc): adding sdk rust lib --- apps/rust-sdk/README.md | 170 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 167 insertions(+), 3 deletions(-) diff --git a/apps/rust-sdk/README.md b/apps/rust-sdk/README.md index 8783ddff..c1a3c5d5 100644 --- a/apps/rust-sdk/README.md +++ b/apps/rust-sdk/README.md @@ -2,9 +2,173 @@ The Firecrawl Rust SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API. -# TODO... -## How to test +## Installation + +To install the Firecrawl Rust SDK, add the following to your `Cargo.toml`: + +```toml +[dependencies] +firecrawl_rs = "0.1" +``` + +## Usage + +1. Get an API key from [firecrawl.dev](https://firecrawl.dev) +2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` struct. + +Here's an example of how to use the SDK in [example.rs](./examples/example.rs): +All below example can start with : +```rust +use firecrawl_rs::FirecrawlApp; + +#[tokio::main] +async fn main() { + // Initialize the FirecrawlApp with the API key + let api_key = ...; + let api_url = ...; + let app = FirecrawlApp::new(api_key, api_url).expect("Failed to initialize FirecrawlApp"); + + // your code here... +} +``` + +### Scraping a URL + +To scrape a single URL, use the `scrape_url` method. It takes the URL as a parameter and returns the scraped data as a `serde_json::Value`. ```rust -$ cargo run --example example +let scrape_result = app.scrape_url("https://example.com", None).await; +match scrape_result { + Ok(data) => println!("Scrape Result:\n{}", data["markdown"]), + Err(e) => eprintln!("Scrape failed: {}", e), +} ``` + +### Extracting structured data from a URL + +With LLM extraction, you can easily extract structured data from any URL. We support Serde for JSON schema validation to make it easier for you too. Here is how you use it: + +```rust +let json_schema = json!({ + "type": "object", + "properties": { + "top": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "points": {"type": "number"}, + "by": {"type": "string"}, + "commentsURL": {"type": "string"} + }, + "required": ["title", "points", "by", "commentsURL"] + }, + "minItems": 5, + "maxItems": 5, + "description": "Top 5 stories on Hacker News" + } + }, + "required": ["top"] +}); + +let llm_extraction_params = json!({ + "extractorOptions": { + "extractionSchema": json_schema, + "mode": "llm-extraction" + }, + "pageOptions": { + "onlyMainContent": true + } +}); + +let llm_extraction_result = app + .scrape_url("https://news.ycombinator.com", Some(llm_extraction_params)) + .await; +match llm_extraction_result { + Ok(data) => println!("LLM Extraction Result:\n{}", data["llm_extraction"]), + Err(e) => eprintln!("LLM Extraction failed: {}", e), +} +``` + +### Search for a query + +Used to search the web, get the most relevant results, scrape each page, and return the markdown. + +```rust +// Example query search code... +let query = "what is mendable?"; +let search_result = app.search(query).await; +match search_result { + Ok(data) => println!("Search Result:\n{}", data), + Err(e) => eprintln!("Search failed: {}", e), +} +``` + +### Crawling a Website + +To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format. + +The `wait_until_done` parameter determines whether the method should wait for the crawl job to complete before returning the result. If set to `true`, the method will periodically check the status of the crawl job until it is completed or the specified `timeout` (in seconds) is reached. If set to `false`, the method will return immediately with the job ID, and you can manually check the status of the crawl job using the `check_crawl_status` method. + +```rust +let random_uuid = String::from(Uuid::new_v4()); +let idempotency_key = Some(random_uuid); // optional idempotency key +let crawl_params = json!({ + "crawlerOptions": { + "excludes": ["blog/*"] + } +}); +let crawl_result = app + .crawl_url("https://example.com", Some(crawl_params), true, 2, idempotency_key) + .await; +match crawl_result { + Ok(data) => println!("Crawl Result:\n{}", data), + Err(e) => eprintln!("Crawl failed: {}", e), +} +``` + +If `wait_until_done` is set to `true`, the `crawl_url` method will return the crawl result once the job is completed. If the job fails or is stopped, an exception will be raised. + +### Checking Crawl Status + +To check the status of a crawl job, use the `check_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job. + +```rust +let job_id = crawl_result["jobId"].as_str().expect("Job ID not found"); +let status = app.check_crawl_status(job_id).await; +match status { + Ok(data) => println!("Crawl Status:\n{}", data), + Err(e) => eprintln!("Failed to check crawl status: {}", e), +} +``` + +## Error Handling + +The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. + +## Running the Tests with Cargo + +To ensure the functionality of the Firecrawl Rust SDK, we have included end-to-end tests using `cargo`. These tests cover various aspects of the SDK, including URL scraping, web searching, and website crawling. + +### Running the Tests + +To run the tests, execute the following commands: + +Install the test dependencies: +```bash +cargo install --dev +``` + +Run: +```bash +cargo test --test e2e_with_auth +``` + +## Contributing + +Contributions to the Firecrawl Rust SDK are welcome! If you find any issues or have suggestions for improvements, please open an issue or submit a pull request on the GitHub repository. + +## License + +The Firecrawl Rust SDK is open-source and released under the [AGPL License](https://www.gnu.org/licenses/agpl-3.0.en.html). From d86de4f5a6c8b0500eaddf17f92ac3381171417e Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 14:07:23 +0200 Subject: [PATCH 09/23] feat: add Debug to FireCrawl --- apps/rust-sdk/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rs index 095c9105..a2ca75ad 100644 --- a/apps/rust-sdk/src/lib.rs +++ b/apps/rust-sdk/src/lib.rs @@ -36,7 +36,7 @@ pub enum FirecrawlError { CrawlJobFailed(String), } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct FirecrawlApp { api_key: String, api_url: String, From 21983011b972b9b1572e0b2acbe4611c48d6a7a6 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 14:07:58 +0200 Subject: [PATCH 10/23] feat: update tests --- apps/rust-sdk/tests/e2e_with_auth.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs index febf259a..7addc7e4 100644 --- a/apps/rust-sdk/tests/e2e_with_auth.rs +++ b/apps/rust-sdk/tests/e2e_with_auth.rs @@ -154,14 +154,14 @@ async fn test_crawl_url_with_idempotency_key_e2e() { "excludes": ["blog/*"] } }); - let result = app.crawl_url("https://roastmywebsite.ai", Some(params), true, 2, Some(unique_idempotency_key.clone())).await.unwrap(); + let result = app.crawl_url("https://roastmywebsite.ai", Some(params.clone()), true, 2, Some(unique_idempotency_key.clone())).await.unwrap(); let result_as_str = result.as_object().unwrap(); assert!(result_as_str.len() > 0); // assert!(result[0].contains_key("content")); // assert!(result[0]["content"].as_str().unwrap().contains("_Roast_")); - let conflict_result = app.crawl_url("https://firecrawl.dev", Some(params), true, 2, Some(unique_idempotency_key)).await; + let conflict_result = app.crawl_url("https://firecrawl.dev", Some(params.clone()), true, 2, Some(unique_idempotency_key)).await; assert_matches!(conflict_result, Err(e) if e.to_string() == "Conflict: Failed to start crawl job due to a conflict. Idempotency key already used"); } @@ -191,11 +191,13 @@ async fn test_check_crawl_status_e2e() { async fn test_search_e2e() { dotenv().ok(); let api_url = env::var("API_URL").unwrap(); - let api_key = env::var("TEST_API_KEY").unwrap(); + let api_key = env::var("API_KEY").unwrap(); let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); - let result = app.search("test query").await.unwrap(); - assert!(result.as_object().unwrap().len() > 2); - //assert!(result.as_object().unwrap()[0].contains_key("content")); + + let result = app.search("test query", None).await.unwrap(); + assert!(result.is_array()); + assert!(!result.as_array().unwrap().is_empty()); + assert!(result[0].get("content").is_some()); } #[tokio::test] @@ -203,7 +205,7 @@ async fn test_search_invalid_api_key() { dotenv().ok(); let api_url = env::var("API_URL").unwrap(); let app = FirecrawlApp::new(Some("invalid_api_key".to_string()), Some(api_url)).unwrap(); - let result = app.search("test query").await; + let result = app.search("test query", None).await; assert_matches!(result, Err(e) if e.to_string() == "Unexpected error during search: Status code 401. Unauthorized: Invalid token"); } From aa9d49cfdaa29ba5ee45f922c0bfd3fe891ed5ec Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 14:41:53 +0200 Subject: [PATCH 11/23] fix(test): updates and keep relevant tests --- apps/rust-sdk/README.md | 1 + apps/rust-sdk/tests/e2e_with_auth.rs | 118 +++------------------------ 2 files changed, 12 insertions(+), 107 deletions(-) diff --git a/apps/rust-sdk/README.md b/apps/rust-sdk/README.md index c1a3c5d5..52f7a78f 100644 --- a/apps/rust-sdk/README.md +++ b/apps/rust-sdk/README.md @@ -162,6 +162,7 @@ cargo install --dev Run: ```bash +export $(xargs < ./tests/.env) cargo test --test e2e_with_auth ``` diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs index 7addc7e4..0380ef17 100644 --- a/apps/rust-sdk/tests/e2e_with_auth.rs +++ b/apps/rust-sdk/tests/e2e_with_auth.rs @@ -1,6 +1,5 @@ use firecrawl_rs::FirecrawlApp; use serde_json::json; -use uuid::Uuid; use dotenv::dotenv; use std::env; use tokio::time::sleep; @@ -10,17 +9,8 @@ use assert_matches::assert_matches; #[tokio::test] async fn test_no_api_key() { dotenv().ok(); - let api_url = env::var("API_URL").unwrap(); - assert_matches!(FirecrawlApp::new(None, Some(api_url)), Err(e) if e.to_string() == "No API key provided"); -} - -#[tokio::test] -async fn test_scrape_url_invalid_api_key() { - dotenv().ok(); - let api_url = env::var("API_URL").unwrap(); - let app = FirecrawlApp::new(Some("invalid_api_key".to_string()), Some(api_url)).unwrap(); - let result = app.scrape_url("https://firecrawl.dev", None).await; - assert_matches!(result, Err(e) if e.to_string() == "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token"); + let api_url = env::var("API_URL").expect("API_URL environment variable is not set"); + assert_matches!(FirecrawlApp::new(None, Some(api_url)), Err(e) if e.to_string() == "API key not provided"); } #[tokio::test] @@ -31,7 +21,11 @@ async fn test_blocklisted_url() { let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); let blocklisted_url = "https://facebook.com/fake-test"; let result = app.scrape_url(blocklisted_url, None).await; - assert_matches!(result, Err(e) if e.to_string() == "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it."); + + assert_matches!( + result, + Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions") + ); } #[tokio::test] @@ -104,15 +98,6 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explici assert!(result["content"].as_str().unwrap().contains("We present spectrophotometric observations of the Broad Line Radio Galaxy")); } -#[tokio::test] -async fn test_crawl_url_invalid_api_key() { - dotenv().ok(); - let api_url = env::var("API_URL").unwrap(); - let app = FirecrawlApp::new(Some("invalid_api_key".to_string()), Some(api_url)).unwrap(); - let result = app.crawl_url("https://firecrawl.dev", None, true, 1, None).await; - assert_matches!(result, Err(e) if e.to_string() == "Unexpected error during start crawl job: Status code 401. Unauthorized: Invalid token"); -} - #[tokio::test] async fn test_should_return_error_for_blocklisted_url() { dotenv().ok(); @@ -121,92 +106,11 @@ async fn test_should_return_error_for_blocklisted_url() { let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); let blocklisted_url = "https://twitter.com/fake-test"; let result = app.crawl_url(blocklisted_url, None, true, 1, None).await; - assert_matches!(result, Err(e) if e.to_string() == "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it."); -} -#[tokio::test] -async fn test_crawl_url_wait_for_completion_e2e() { - dotenv().ok(); - let api_url = env::var("API_URL").unwrap(); - let api_key = env::var("TEST_API_KEY").unwrap(); - let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); - let params = json!({ - "crawlerOptions": { - "excludes": ["blog/*"] - } - }); - let result = app.crawl_url("https://roastmywebsite.ai", Some(params), true, 1, None).await.unwrap(); - let result_as_str = result.as_object().unwrap(); - assert!(result_as_str.len() > 0); - // assert!(result_as_str[0].contains_key("content")); - // assert!(result[0]["content"].as_str().unwrap().contains("_Roast_")); -} - -#[tokio::test] -async fn test_crawl_url_with_idempotency_key_e2e() { - dotenv().ok(); - let api_url = env::var("API_URL").unwrap(); - let api_key = env::var("TEST_API_KEY").unwrap(); - let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); - let unique_idempotency_key = Uuid::new_v4().to_string(); - let params = json!({ - "crawlerOptions": { - "excludes": ["blog/*"] - } - }); - let result = app.crawl_url("https://roastmywebsite.ai", Some(params.clone()), true, 2, Some(unique_idempotency_key.clone())).await.unwrap(); - - let result_as_str = result.as_object().unwrap(); - assert!(result_as_str.len() > 0); - // assert!(result[0].contains_key("content")); - // assert!(result[0]["content"].as_str().unwrap().contains("_Roast_")); - - let conflict_result = app.crawl_url("https://firecrawl.dev", Some(params.clone()), true, 2, Some(unique_idempotency_key)).await; - assert_matches!(conflict_result, Err(e) if e.to_string() == "Conflict: Failed to start crawl job due to a conflict. Idempotency key already used"); -} - -#[tokio::test] -async fn test_check_crawl_status_e2e() { - dotenv().ok(); - let api_url = env::var("API_URL").unwrap(); - let api_key = env::var("TEST_API_KEY").unwrap(); - let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); - let params = json!({ - "crawlerOptions": { - "excludes": ["blog/*"] - } - }); - let result = app.crawl_url("https://firecrawl.dev", Some(params), false, 1, None).await.unwrap(); - assert!(result.as_object().unwrap().contains_key("jobId")); - - sleep(Duration::from_secs(30)).await; // wait for 30 seconds - let status_response = app.check_crawl_status(result["jobId"].as_str().unwrap()).await.unwrap(); - assert!(status_response.as_object().unwrap().contains_key("status")); - assert_eq!(status_response["status"].as_str().unwrap(), "completed"); - assert!(status_response.as_object().unwrap().contains_key("data")); - assert!(status_response["data"].as_array().unwrap().len() > 0); -} - -#[tokio::test] -async fn test_search_e2e() { - dotenv().ok(); - let api_url = env::var("API_URL").unwrap(); - let api_key = env::var("API_KEY").unwrap(); - let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); - - let result = app.search("test query", None).await.unwrap(); - assert!(result.is_array()); - assert!(!result.as_array().unwrap().is_empty()); - assert!(result[0].get("content").is_some()); -} - -#[tokio::test] -async fn test_search_invalid_api_key() { - dotenv().ok(); - let api_url = env::var("API_URL").unwrap(); - let app = FirecrawlApp::new(Some("invalid_api_key".to_string()), Some(api_url)).unwrap(); - let result = app.search("test query", None).await; - assert_matches!(result, Err(e) if e.to_string() == "Unexpected error during search: Status code 401. Unauthorized: Invalid token"); + assert_matches!( + result, + Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.") + ); } #[tokio::test] From 0b19c940f164bfad439329aa7e3fa566279d06a0 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 14:54:38 +0200 Subject: [PATCH 12/23] fix: format the rust code --- apps/rust-sdk/Cargo.lock | 591 ++++++++++++++++++++++++++- apps/rust-sdk/Cargo.toml | 2 + apps/rust-sdk/examples/example.rs | 8 +- apps/rust-sdk/tests/e2e_with_auth.rs | 57 ++- 4 files changed, 631 insertions(+), 27 deletions(-) diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock index d4c4cb60..b9262195 100644 --- a/apps/rust-sdk/Cargo.lock +++ b/apps/rust-sdk/Cargo.lock @@ -17,6 +17,27 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81ce3d38065e618af2d7b77e10c5ad9a069859b4be3c2250f674af3840d9c8a5" +dependencies = [ + "memchr", +] + +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "assert_matches" version = "1.5.0" @@ -29,6 +50,15 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.3.0", +] + [[package]] name = "autocfg" version = "1.3.0" @@ -50,12 +80,24 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" + [[package]] name = "bitflags" version = "1.3.2" @@ -68,12 +110,29 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "blake2b_simd" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + [[package]] name = "bumpalo" version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.6.0" @@ -92,6 +151,30 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clippy" +version = "0.0.302" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d911ee15579a3f50880d8c1d59ef6e79f9533127a3bd342462f5d584f5e8c294" +dependencies = [ + "term 0.5.2", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "core-foundation" version = "0.9.4" @@ -108,6 +191,29 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -123,6 +229,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_logger" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" +dependencies = [ + "log 0.3.9", + "regex", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -139,6 +255,19 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "extprim" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b1a357c911c352439b460d7b375b5c85977b9db395b703dfee5a94dfb4d66a2" +dependencies = [ + "num-traits", + "rand", + "rustc_version", + "semver", + "serde", +] + [[package]] name = "fastrand" version = "2.1.0" @@ -150,9 +279,11 @@ name = "firecrawl_rs" version = "0.1.0" dependencies = [ "assert_matches", + "clippy", "dotenv", - "log", + "log 0.4.22", "reqwest", + "rustfmt", "serde", "serde_json", "thiserror", @@ -190,6 +321,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures-channel" version = "0.3.30" @@ -240,6 +377,26 @@ dependencies = [ "slab", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -248,7 +405,7 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -442,6 +599,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.155" @@ -460,10 +633,19 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ - "autocfg", + "autocfg 1.3.0", "scopeguard", ] +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +dependencies = [ + "log 0.4.22", +] + [[package]] name = "log" version = "0.4.22" @@ -498,7 +680,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -509,7 +691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ "libc", - "log", + "log 0.4.22", "openssl", "openssl-probe", "openssl-sys", @@ -519,6 +701,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg 1.3.0", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -606,7 +797,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.2", "smallvec", "windows-targets 0.52.6", ] @@ -673,6 +864,127 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.8", + "libc", + "rand_chacha", + "rand_core 0.4.2", + "rand_hc", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.9", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi 0.3.9", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.5.2" @@ -682,13 +994,46 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.16", + "redox_syscall 0.1.57", + "rust-argon2", +] + +[[package]] +name = "regex" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", + "utf8-ranges", +] + +[[package]] +name = "regex-syntax" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" +dependencies = [ + "ucd-util", +] + [[package]] name = "reqwest" version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-channel", @@ -704,7 +1049,7 @@ dependencies = [ "hyper-util", "ipnet", "js-sys", - "log", + "log 0.4.22", "mime", "native-tls", "once_cell", @@ -734,19 +1079,65 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", "windows-sys 0.52.0", ] +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64 0.13.1", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustfmt" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec940eed814db0fb7ab928c5f5025f97dc55d1c0e345e39dda2ce9f945557500" +dependencies = [ + "diff", + "env_logger", + "getopts", + "kernel32-sys", + "libc", + "log 0.3.9", + "regex", + "serde", + "serde_derive", + "serde_json", + "strings", + "syntex_errors", + "syntex_syntax", + "term 0.4.6", + "toml", + "unicode-segmentation", + "winapi 0.2.8", +] + [[package]] name = "rustix" version = "0.38.34" @@ -779,7 +1170,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64", + "base64 0.22.1", "rustls-pki-types", ] @@ -844,6 +1235,21 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.204" @@ -902,7 +1308,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "autocfg", + "autocfg 1.3.0", ] [[package]] @@ -927,6 +1333,15 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "strings" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa481ee1bc42fc3df8195f91f7cb43cf8f2b71b48bac40bf5381cfaf7e481f3c" +dependencies = [ + "log 0.3.9", +] + [[package]] name = "subtle" version = "2.6.1" @@ -950,6 +1365,47 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "syntex_errors" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3133289179676c9f5c5b2845bf5a2e127769f4889fcbada43035ef6bd662605e" +dependencies = [ + "libc", + "serde", + "serde_derive", + "syntex_pos", + "term 0.4.6", + "unicode-xid", +] + +[[package]] +name = "syntex_pos" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ab669fa003d208c681f874bbc76d91cc3d32550d16b5d9d2087cf477316470" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "syntex_syntax" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03815b9f04d95828770d9c974aa39c6e1f6ef3114eb77a3ce09008a0d15dd142" +dependencies = [ + "bitflags 0.9.1", + "extprim", + "log 0.3.9", + "serde", + "serde_derive", + "serde_json", + "syntex_errors", + "syntex_pos", + "unicode-xid", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -983,6 +1439,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "term" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" +dependencies = [ + "kernel32-sys", + "winapi 0.2.8", +] + +[[package]] +name = "term" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +dependencies = [ + "byteorder", + "dirs", + "winapi 0.3.9", +] + [[package]] name = "thiserror" version = "1.0.61" @@ -1003,6 +1480,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +dependencies = [ + "lazy_static", +] + [[package]] name = "tinyvec" version = "1.7.0" @@ -1082,6 +1568,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +dependencies = [ + "serde", +] + [[package]] name = "tower" version = "0.4.13" @@ -1134,6 +1629,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ucd-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd2fc5d32b590614af8b0a20d837f32eca055edd0bbead59a9cfe80858be003" + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -1155,6 +1656,24 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode-width" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "untrusted" version = "0.9.0" @@ -1172,13 +1691,19 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "uuid" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -1196,6 +1721,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1219,7 +1750,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", - "log", + "log 0.4.22", "once_cell", "proc-macro2", "quote", @@ -1278,6 +1809,40 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index 7d4bea4a..c1b1c5f2 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -27,6 +27,8 @@ version = "1" features = ["full"] [dev-dependencies] +clippy = "^0.0.302" +rustfmt = "^0.10" assert_matches = "^1.5" dotenv = "^0.15" tokio = { version = "1", features = ["full"] } diff --git a/apps/rust-sdk/examples/example.rs b/apps/rust-sdk/examples/example.rs index 8b6caca4..aab4b733 100644 --- a/apps/rust-sdk/examples/example.rs +++ b/apps/rust-sdk/examples/example.rs @@ -25,7 +25,13 @@ async fn main() { } }); let crawl_result = app - .crawl_url("https://mendable.ai", Some(crawl_params), true, 2, idempotency_key) + .crawl_url( + "https://mendable.ai", + Some(crawl_params), + true, + 2, + idempotency_key, + ) .await; match crawl_result { Ok(data) => println!("Crawl Result:\n{}", data), diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs index 0380ef17..80dccd7b 100644 --- a/apps/rust-sdk/tests/e2e_with_auth.rs +++ b/apps/rust-sdk/tests/e2e_with_auth.rs @@ -1,10 +1,10 @@ +use assert_matches::assert_matches; +use dotenv::dotenv; use firecrawl_rs::FirecrawlApp; use serde_json::json; -use dotenv::dotenv; use std::env; -use tokio::time::sleep; use std::time::Duration; -use assert_matches::assert_matches; +use tokio::time::sleep; #[tokio::test] async fn test_no_api_key() { @@ -32,8 +32,15 @@ async fn test_blocklisted_url() { async fn test_successful_response_with_valid_preview_token() { dotenv().ok(); let api_url = env::var("API_URL").unwrap(); - let app = FirecrawlApp::new(Some("this_is_just_a_preview_token".to_string()), Some(api_url)).unwrap(); - let result = app.scrape_url("https://roastmywebsite.ai", None).await.unwrap(); + let app = FirecrawlApp::new( + Some("this_is_just_a_preview_token".to_string()), + Some(api_url), + ) + .unwrap(); + let result = app + .scrape_url("https://roastmywebsite.ai", None) + .await + .unwrap(); assert!(result.as_object().unwrap().contains_key("content")); assert!(result["content"].as_str().unwrap().contains("_Roast_")); } @@ -44,7 +51,10 @@ async fn test_scrape_url_e2e() { let api_url = env::var("API_URL").unwrap(); let api_key = env::var("TEST_API_KEY").unwrap(); let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); - let result = app.scrape_url("https://roastmywebsite.ai", None).await.unwrap(); + let result = app + .scrape_url("https://roastmywebsite.ai", None) + .await + .unwrap(); assert!(result.as_object().unwrap().contains_key("content")); assert!(result.as_object().unwrap().contains_key("markdown")); assert!(result.as_object().unwrap().contains_key("metadata")); @@ -63,7 +73,10 @@ async fn test_successful_response_with_valid_api_key_and_include_html() { "includeHtml": true } }); - let result = app.scrape_url("https://roastmywebsite.ai", Some(params)).await.unwrap(); + let result = app + .scrape_url("https://roastmywebsite.ai", Some(params)) + .await + .unwrap(); assert!(result.as_object().unwrap().contains_key("content")); assert!(result.as_object().unwrap().contains_key("markdown")); assert!(result.as_object().unwrap().contains_key("html")); @@ -79,10 +92,16 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file() { let api_url = env::var("API_URL").unwrap(); let api_key = env::var("TEST_API_KEY").unwrap(); let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); - let result = app.scrape_url("https://arxiv.org/pdf/astro-ph/9301001.pdf", None).await.unwrap(); + let result = app + .scrape_url("https://arxiv.org/pdf/astro-ph/9301001.pdf", None) + .await + .unwrap(); assert!(result.as_object().unwrap().contains_key("content")); assert!(result.as_object().unwrap().contains_key("metadata")); - assert!(result["content"].as_str().unwrap().contains("We present spectrophotometric observations of the Broad Line Radio Galaxy")); + assert!(result["content"] + .as_str() + .unwrap() + .contains("We present spectrophotometric observations of the Broad Line Radio Galaxy")); } #[tokio::test] @@ -91,11 +110,17 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explici let api_url = env::var("API_URL").unwrap(); let api_key = env::var("TEST_API_KEY").unwrap(); let app = FirecrawlApp::new(Some(api_key), Some(api_url)).unwrap(); - let result = app.scrape_url("https://arxiv.org/pdf/astro-ph/9301001", None).await.unwrap(); + let result = app + .scrape_url("https://arxiv.org/pdf/astro-ph/9301001", None) + .await + .unwrap(); sleep(Duration::from_secs(6)).await; // wait for 6 seconds assert!(result.as_object().unwrap().contains_key("content")); assert!(result.as_object().unwrap().contains_key("metadata")); - assert!(result["content"].as_str().unwrap().contains("We present spectrophotometric observations of the Broad Line Radio Galaxy")); + assert!(result["content"] + .as_str() + .unwrap() + .contains("We present spectrophotometric observations of the Broad Line Radio Galaxy")); } #[tokio::test] @@ -134,10 +159,16 @@ async fn test_llm_extraction() { } } }); - let result = app.scrape_url("https://mendable.ai", Some(params)).await.unwrap(); + let result = app + .scrape_url("https://mendable.ai", Some(params)) + .await + .unwrap(); assert!(result.as_object().unwrap().contains_key("llm_extraction")); let llm_extraction = &result["llm_extraction"]; - assert!(llm_extraction.as_object().unwrap().contains_key("company_mission")); + assert!(llm_extraction + .as_object() + .unwrap() + .contains_key("company_mission")); assert!(llm_extraction["supports_sso"].is_boolean()); assert!(llm_extraction["is_open_source"].is_boolean()); } From 9deaa8b0e920e28d90f992f245a1aad101476e1d Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 14:58:41 +0200 Subject: [PATCH 13/23] feat: add rust sdk ci github action --- .github/archive/rust-sdk.yml | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/archive/rust-sdk.yml diff --git a/.github/archive/rust-sdk.yml b/.github/archive/rust-sdk.yml new file mode 100644 index 00000000..25f3edc0 --- /dev/null +++ b/.github/archive/rust-sdk.yml @@ -0,0 +1,68 @@ +name: Run Rust SDK E2E Tests + +on: [] + +env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + HOST: ${{ secrets.HOST }} + LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }} + LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }} + POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} + POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} + NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }} + PORT: ${{ secrets.PORT }} + REDIS_URL: ${{ secrets.REDIS_URL }} + SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }} + SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }} + SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }} + HDX_NODE_BETA_MODE: 1 + + +jobs: + build: + runs-on: ubuntu-latest + services: + redis: + image: redis + ports: + - 6379:6379 + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install pnpm + run: npm install -g pnpm + + - name: Install dependencies for API + run: pnpm install + working-directory: ./apps/api + + - name: Start the application + run: npm start & + working-directory: ./apps/api + id: start_app + + - name: Start workers + run: npm run workers & + working-directory: ./apps/api + id: start_workers + + - name: Set up Rust + uses: actions/setup-rust@v1 + with: + rust-version: stable + + - name: Install rust dev dependencies + working-directory: ./apps/rust-sdk + run: cargo build + + - name: Run E2E tests for Rust SDK + run: cargo test --test e2e_with_auth From d7bbb22ae476e17d66d827d7708988666b9bde5c Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 15:06:54 +0200 Subject: [PATCH 14/23] feat: update fly github action --- .github/archive/rust-sdk.yml | 9 +------- .github/workflows/fly.yml | 43 ++++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/.github/archive/rust-sdk.yml b/.github/archive/rust-sdk.yml index 25f3edc0..62deeaab 100644 --- a/.github/archive/rust-sdk.yml +++ b/.github/archive/rust-sdk.yml @@ -37,32 +37,25 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Install pnpm run: npm install -g pnpm - - name: Install dependencies for API run: pnpm install working-directory: ./apps/api - - name: Start the application run: npm start & working-directory: ./apps/api id: start_app - - name: Start workers run: npm run workers & working-directory: ./apps/api id: start_workers - - name: Set up Rust uses: actions/setup-rust@v1 with: rust-version: stable - - - name: Install rust dev dependencies + - name: Try the lib build working-directory: ./apps/rust-sdk run: cargo build - - name: Run E2E tests for Rust SDK run: cargo test --test e2e_with_auth diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index 627409e7..f2c5203a 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -168,11 +168,46 @@ jobs: - name: Run E2E tests for JavaScript SDK run: npm run test working-directory: ./apps/js-sdk/firecrawl - + + rust-sdk-tests: + name: Rust SDK Tests + needs: pre-deploy-e2e-tests + runs-on: ubuntu-latest + services: + redis: + image: redis + ports: + - 6379:6379 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Install pnpm + run: npm install -g pnpm + - name: Install dependencies for API + run: pnpm install + working-directory: ./apps/api + - name: Start the application + run: npm start & + working-directory: ./apps/api + id: start_app + - name: Start workers + run: npm run workers & + working-directory: ./apps/api + id: start_workers + - name: Set up Rust + uses: actions/setup-rust@v1 + with: + rust-version: stable + - name: Try the lib build + working-directory: ./apps/rust-sdk + run: cargo build + - name: Run E2E tests for Rust SDK + run: cargo test --test e2e_with_auth + deploy: name: Deploy app runs-on: ubuntu-latest - needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests] + needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests, rust-sdk-tests] steps: - uses: actions/checkout@v3 - name: Change directory @@ -248,7 +283,7 @@ jobs: - name: Install dependencies for JavaScript SDK run: pnpm install working-directory: ./apps/js-sdk/firecrawl - + - name: Run version check script id: version_check_script run: | @@ -262,4 +297,4 @@ jobs: run: | npm run build-and-publish working-directory: ./apps/js-sdk/firecrawl - \ No newline at end of file + From 6ea36d38f34545a8b258493ae3e0e9d2582c84a2 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 15:10:12 +0200 Subject: [PATCH 15/23] chore: no need for Makefile --- apps/rust-sdk/Makefile | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 apps/rust-sdk/Makefile diff --git a/apps/rust-sdk/Makefile b/apps/rust-sdk/Makefile deleted file mode 100644 index 620047de..00000000 --- a/apps/rust-sdk/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -.PHONY: build run format - -build: - cargo build - -run: build - cargo run - -format: - cargo fmt From 80af3905579822fb2ab2b5a976ba4800628aca19 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 15:22:55 +0200 Subject: [PATCH 16/23] chore: update doc and libs --- apps/rust-sdk/Cargo.toml | 5 +---- apps/rust-sdk/README.md | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index c1b1c5f2..02a5da77 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -21,10 +21,7 @@ serde_json = "^1.0" log = "^0.4" thiserror = "^1.0" uuid = { version = "^1.10", features = ["v4"] } - -[dependencies.tokio] -version = "1" -features = ["full"] +tokio = { version = "^1", features = ["full"] } [dev-dependencies] clippy = "^0.0.302" diff --git a/apps/rust-sdk/README.md b/apps/rust-sdk/README.md index 52f7a78f..ea2b9e8f 100644 --- a/apps/rust-sdk/README.md +++ b/apps/rust-sdk/README.md @@ -8,9 +8,18 @@ To install the Firecrawl Rust SDK, add the following to your `Cargo.toml`: ```toml [dependencies] -firecrawl_rs = "0.1" +firecrawl_rs = "^0.1" +tokio = { version = "^1", features = ["full"] } +serde = { version = "^1.0", features = ["derive"] } +serde_json = "^1.0" +uuid = { version = "^1.10", features = ["v4"] } + +[build-dependencies] +tokio = { version = "1", features = ["full"] } ``` +To add it in your codebase. + ## Usage 1. Get an API key from [firecrawl.dev](https://firecrawl.dev) @@ -37,6 +46,7 @@ async fn main() { To scrape a single URL, use the `scrape_url` method. It takes the URL as a parameter and returns the scraped data as a `serde_json::Value`. ```rust +// Example scrape code... let scrape_result = app.scrape_url("https://example.com", None).await; match scrape_result { Ok(data) => println!("Scrape Result:\n{}", data["markdown"]), @@ -82,6 +92,7 @@ let llm_extraction_params = json!({ } }); +// Example scrape code... let llm_extraction_result = app .scrape_url("https://news.ycombinator.com", Some(llm_extraction_params)) .await; @@ -119,6 +130,8 @@ let crawl_params = json!({ "excludes": ["blog/*"] } }); + +// Example crawl code... let crawl_result = app .crawl_url("https://example.com", Some(crawl_params), true, 2, idempotency_key) .await; @@ -154,16 +167,9 @@ To ensure the functionality of the Firecrawl Rust SDK, we have included end-to-e ### Running the Tests To run the tests, execute the following commands: - -Install the test dependencies: ```bash -cargo install --dev -``` - -Run: -```bash -export $(xargs < ./tests/.env) -cargo test --test e2e_with_auth +$ export $(xargs < ./tests/.env) +$ cargo test --test e2e_with_auth ``` ## Contributing From 2ae2ccad1c3cfe264ca4159438a7471f7cb918c0 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 15:32:57 +0200 Subject: [PATCH 17/23] chore: update license on Cargo.toml --- apps/rust-sdk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index 02a5da77..c5908b56 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -3,7 +3,7 @@ name = "firecrawl_rs" author="Mendable.ai" version = "0.1.0" edition = "2021" -license = "AGPL License" +license = "GPL-2.0-or-later" homepage = "https://www.firecrawl.dev/" repository ="https://github.com/mendableai/firecrawl" description = "Rust SDK for Firecrawl API." From 3a744eb9632b6cda2c76c3161edd04c3ad1ef700 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Fri, 12 Jul 2024 15:33:31 +0200 Subject: [PATCH 18/23] feat: update the doc and version --- apps/rust-sdk/Cargo.lock | 2 +- apps/rust-sdk/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock index b9262195..a43f7de1 100644 --- a/apps/rust-sdk/Cargo.lock +++ b/apps/rust-sdk/Cargo.lock @@ -276,7 +276,7 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "firecrawl_rs" -version = "0.1.0" +version = "0.1.1" dependencies = [ "assert_matches", "clippy", diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index c5908b56..e8eb512c 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "firecrawl_rs" author="Mendable.ai" -version = "0.1.0" +version = "0.1.1" edition = "2021" license = "GPL-2.0-or-later" homepage = "https://www.firecrawl.dev/" From 7f8953e9d67b29db01f42b2398c921ce974614e3 Mon Sep 17 00:00:00 2001 From: sanix-darker Date: Tue, 16 Jul 2024 12:57:56 +0200 Subject: [PATCH 19/23] feat: add a small CHANGELOG for the rust-sdk client --- apps/rust-sdk/CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 apps/rust-sdk/CHANGELOG.md diff --git a/apps/rust-sdk/CHANGELOG.md b/apps/rust-sdk/CHANGELOG.md new file mode 100644 index 00000000..8342b9fa --- /dev/null +++ b/apps/rust-sdk/CHANGELOG.md @@ -0,0 +1,7 @@ +## CHANGELOG + +## [0.1] + +### Added + +- [feat] Firecrawl rust sdk. From 4d39025e8251d8168a06987a5a1f8b7875edf50a Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Wed, 14 Aug 2024 16:28:11 -0300 Subject: [PATCH 20/23] workflows/ci tested e2e and it's working --- .github/archive/publish-rust-sdk.yml | 42 +++++++++++++++++++ .../scripts/check_version_has_incremented.py | 20 +++++++++ .github/scripts/requirements.txt | 3 +- .github/workflows/fly.yml | 35 ++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 .github/archive/publish-rust-sdk.yml diff --git a/.github/archive/publish-rust-sdk.yml b/.github/archive/publish-rust-sdk.yml new file mode 100644 index 00000000..9856bd77 --- /dev/null +++ b/.github/archive/publish-rust-sdk.yml @@ -0,0 +1,42 @@ +name: Publish Rust SDK + +on: [] + +env: + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + default: true + profile: minimal + + - name: Install dependencies + run: cargo build --release + + - name: Run version check script + id: version_check_script + run: | + VERSION_INCREMENTED=$(cargo search --limit 1 my_crate_name | grep my_crate_name) + echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV + + - name: Build the package + if: ${{ env.VERSION_INCREMENTED == 'true' }} + run: cargo package + working-directory: ./apps/rust-sdk + + - name: Publish to crates.io + if: ${{ env.VERSION_INCREMENTED == 'true' }} + env: + CARGO_REG_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + run: cargo publish + working-directory: ./apps/rust-sdk \ No newline at end of file diff --git a/.github/scripts/check_version_has_incremented.py b/.github/scripts/check_version_has_incremented.py index e437c934..6dba065f 100644 --- a/.github/scripts/check_version_has_incremented.py +++ b/.github/scripts/check_version_has_incremented.py @@ -15,6 +15,7 @@ false """ import json +import toml import os import re import sys @@ -53,6 +54,19 @@ def get_npm_version(package_name: str) -> str: version = response.json()['version'] return version.strip() +def get_rust_version(file_path: str) -> str: + """Extract version string from Cargo.toml.""" + cargo_toml = toml.load(file_path) + if 'package' in cargo_toml and 'version' in cargo_toml['package']: + return cargo_toml['package']['version'].strip() + raise RuntimeError("Unable to find version string in Cargo.toml.") + +def get_crates_version(package_name: str) -> str: + """Get latest version of Rust package from crates.io.""" + response = requests.get(f"https://crates.io/api/v1/crates/{package_name}") + version = response.json()['crate']['newest_version'] + return version.strip() + def is_version_incremented(local_version: str, published_version: str) -> bool: """Compare local and published versions.""" local_version_parsed: Version = parse_version(local_version) @@ -74,6 +88,12 @@ if __name__ == "__main__": current_version = get_js_version(os.path.join(package_path, 'package.json')) # Get published version from npm published_version = get_npm_version(package_name) + if package_type == "rust": + # Get current version from Cargo.toml + current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml')) + # Get published version from crates.io + published_version = get_crates_version(package_name) + else: raise ValueError("Invalid package type. Use 'python' or 'js'.") diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt index 0bfc6762..60f8e191 100644 --- a/.github/scripts/requirements.txt +++ b/.github/scripts/requirements.txt @@ -1,2 +1,3 @@ requests -packaging \ No newline at end of file +packaging +toml \ No newline at end of file diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index f2c5203a..9902d016 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -298,3 +298,38 @@ jobs: npm run build-and-publish working-directory: ./apps/js-sdk/firecrawl + build-and-publish-rust-sdk: + name: Build and publish Rust SDK + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + default: true + profile: minimal + + - name: Install dependencies + run: cargo build --release + + - name: Run version check script + id: version_check_script + run: | + VERSION_INCREMENTED=$(cargo search --limit 1 my_crate_name | grep my_crate_name) + echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV + + - name: Build the package + if: ${{ env.VERSION_INCREMENTED == 'true' }} + run: cargo package + working-directory: ./apps/rust-sdk + + - name: Publish to crates.io + if: ${{ env.VERSION_INCREMENTED == 'true' }} + env: + CARGO_REG_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + run: cargo publish + working-directory: ./apps/rust-sdk \ No newline at end of file From 81066cf90a016d05afe7c6ef1ddb85722eefa6b6 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:11:27 -0300 Subject: [PATCH 21/23] updating cargo pckg name n version --- apps/rust-sdk/Cargo.lock | 4 ++-- apps/rust-sdk/Cargo.toml | 6 +++--- apps/rust-sdk/README.md | 4 ++-- apps/rust-sdk/examples/example.rs | 2 +- apps/rust-sdk/tests/e2e_with_auth.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock index a43f7de1..c2b71d6d 100644 --- a/apps/rust-sdk/Cargo.lock +++ b/apps/rust-sdk/Cargo.lock @@ -275,8 +275,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] -name = "firecrawl_rs" -version = "0.1.1" +name = "firecrawl" +version = "0.1.0" dependencies = [ "assert_matches", "clippy", diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index e8eb512c..685545e2 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "firecrawl_rs" +name = "firecrawl" author="Mendable.ai" -version = "0.1.1" +version = "0.1.0" edition = "2021" license = "GPL-2.0-or-later" homepage = "https://www.firecrawl.dev/" @@ -11,7 +11,7 @@ authors = ["sanix-darker "] [lib] path = "src/lib.rs" -name = "firecrawl_rs" +name = "firecrawl" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/apps/rust-sdk/README.md b/apps/rust-sdk/README.md index ea2b9e8f..54ad9097 100644 --- a/apps/rust-sdk/README.md +++ b/apps/rust-sdk/README.md @@ -8,7 +8,7 @@ To install the Firecrawl Rust SDK, add the following to your `Cargo.toml`: ```toml [dependencies] -firecrawl_rs = "^0.1" +firecrawl = "^0.1" tokio = { version = "^1", features = ["full"] } serde = { version = "^1.0", features = ["derive"] } serde_json = "^1.0" @@ -28,7 +28,7 @@ To add it in your codebase. Here's an example of how to use the SDK in [example.rs](./examples/example.rs): All below example can start with : ```rust -use firecrawl_rs::FirecrawlApp; +use firecrawl::FirecrawlApp; #[tokio::main] async fn main() { diff --git a/apps/rust-sdk/examples/example.rs b/apps/rust-sdk/examples/example.rs index aab4b733..c6b96b78 100644 --- a/apps/rust-sdk/examples/example.rs +++ b/apps/rust-sdk/examples/example.rs @@ -1,4 +1,4 @@ -use firecrawl_rs::FirecrawlApp; +use firecrawl::FirecrawlApp; use serde_json::json; use uuid::Uuid; diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs index 80dccd7b..ac9dc1d3 100644 --- a/apps/rust-sdk/tests/e2e_with_auth.rs +++ b/apps/rust-sdk/tests/e2e_with_auth.rs @@ -1,6 +1,6 @@ use assert_matches::assert_matches; use dotenv::dotenv; -use firecrawl_rs::FirecrawlApp; +use firecrawl::FirecrawlApp; use serde_json::json; use std::env; use std::time::Duration; From 829d115fc9193c209c56fd6758fd037ae660b174 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:25:10 -0300 Subject: [PATCH 22/23] added crates io token --- .github/workflows/fly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index 5488a944..7921b762 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -26,6 +26,7 @@ env: PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} jobs: pre-deploy-e2e-tests: From db85f1a7d58612dd41e6341853f2a236df210427 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 29 Aug 2024 17:11:16 -0300 Subject: [PATCH 23/23] Update index.ts --- apps/js-sdk/firecrawl/src/index.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index cb2a0e4f..2527be96 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -265,13 +265,10 @@ export interface CrawlStatusResponseV0 { * Defines options for mapping URLs during a crawl. */ export interface MapParams { - includePaths?: string[] - excludePaths?: string[] - maxDepth?: number - limit?: number - allowBackwardLinks?: boolean - allowExternalLinks?: boolean - ignoreSitemap?: boolean + search?: string; + ignoreSitemap?: boolean; + includeSubdomains?: boolean; + limit?: number; } /**