AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-11 16:08:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

small improvements

Browse Source 
- wait for getting results on crawl: sometimes crawl takes some a second to save the data on the db and this causes response.data to be empty
- added timeout value to test script
- increased http client timeout (llm extract was failing on e2e tests)
- fixed env path on test script
This commit is contained in:
rafaelsideguide 2024-08-09 11:13:14 -03:00
parent b2e1b2ca68
commit b802ea02a1
4 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/fly.yml vendored
View File

@ -201,7 +201,7 @@ jobs:
run: go mod tidy run: go mod tidy
working-directory: ./apps/go-sdk working-directory: ./apps/go-sdk
- name: Run tests for Go SDK - name: Run tests for Go SDK
run: go test -v ./... run: go test -v ./... -timeout 180s
working-directory: ./apps/go-sdk/firecrawl working-directory: ./apps/go-sdk/firecrawl
deploy: deploy:

6
apps/go-sdk/examples/example.go
View File

@ -33,7 +33,11 @@ func main() {
if err != nil { if err != nil {
log.Fatalf("Failed to crawl URL: %v", err) log.Fatalf("Failed to crawl URL: %v", err)
} }
fmt.Println(crawlResult) jsonCrawlResult, err := json.MarshalIndent(crawlResult, "", " ")
if err != nil {
log.Fatalf("Failed to marshal crawl result: %v", err)
}
fmt.Println(string(jsonCrawlResult))
// LLM Extraction using JSON schema // LLM Extraction using JSON schema
jsonSchema := map[string]any{ jsonSchema := map[string]any{

6
apps/go-sdk/firecrawl/firecrawl.go
View File

@ -195,7 +195,7 @@ func NewFirecrawlApp(apiKey, apiURL string) (*FirecrawlApp, error) {
} }
client := &http.Client{ client := &http.Client{
Timeout: 30 * time.Second, Timeout: 60 * time.Second,
} }
return &FirecrawlApp{ return &FirecrawlApp{
@ -502,6 +502,7 @@ func (app *FirecrawlApp) makeRequest(method, url string, data map[string]any, he
// - []*FirecrawlDocument: The crawl result if the job is completed. // - []*FirecrawlDocument: The crawl result if the job is completed.
// - error: An error if the crawl status check request fails. // - error: An error if the crawl status check request fails.
func (app *FirecrawlApp) monitorJobStatus(jobID string, headers map[string]string, pollInterval int) ([]*FirecrawlDocument, error) { func (app *FirecrawlApp) monitorJobStatus(jobID string, headers map[string]string, pollInterval int) ([]*FirecrawlDocument, error) {
attempts := 0
for { for {
resp, err := app.makeRequest( resp, err := app.makeRequest(
http.MethodGet, http.MethodGet,
@ -531,7 +532,10 @@ func (app *FirecrawlApp) monitorJobStatus(jobID string, headers map[string]strin
if statusData.Data != nil { if statusData.Data != nil {
return statusData.Data, nil return statusData.Data, nil
} }
attempts++
if attempts > 3 {
return nil, fmt.Errorf("crawl job completed but no data was returned") return nil, fmt.Errorf("crawl job completed but no data was returned")
}
} else if status == "active" || status == "paused" || status == "pending" || status == "queued" || status == "waiting" { } else if status == "active" || status == "paused" || status == "pending" || status == "queued" || status == "waiting" {
pollInterval = max(pollInterval, 2) pollInterval = max(pollInterval, 2)
time.Sleep(time.Duration(pollInterval) * time.Second) time.Sleep(time.Duration(pollInterval) * time.Second)

2
apps/go-sdk/firecrawl/firecrawl_test.go
View File

@ -16,7 +16,7 @@ var API_URL string
var TEST_API_KEY string var TEST_API_KEY string
func init() { func init() {
err := godotenv.Load() err := godotenv.Load("../.env")
if err != nil { if err != nil {
log.Fatalf("Error loading .env file: %v", err) log.Fatalf("Error loading .env file: %v", err)
} }