From 0566e54d859330b8942ce0f1c6341760e30a0ded Mon Sep 17 00:00:00 2001 From: Eric Ciarla Date: Mon, 26 Aug 2024 15:16:50 -0400 Subject: [PATCH 001/102] init --- .../turning_docs_into_api_specs/api_spec.json | 771 ++++++++++++++++++ .../dify_api_spec.json | 164 ++++ .../docs.firecrawl.dev/api_spec_0.json | 211 +++++ .../docs.firecrawl.dev/api_spec_1.json | 165 ++++ .../docs.firecrawl.dev/api_spec_10.json | 93 +++ .../docs.firecrawl.dev/api_spec_11.json | 131 +++ .../docs.firecrawl.dev/api_spec_13.json | 87 ++ .../docs.firecrawl.dev/api_spec_15.json | 83 ++ .../docs.firecrawl.dev/api_spec_16.json | 200 +++++ .../docs.firecrawl.dev/api_spec_2.json | 54 ++ .../docs.firecrawl.dev/api_spec_22.json | 166 ++++ .../docs.firecrawl.dev/api_spec_25.json | 229 ++++++ .../docs.firecrawl.dev/api_spec_26.json | 115 +++ .../docs.firecrawl.dev/api_spec_3.json | 185 +++++ .../docs.firecrawl.dev/api_spec_30.json | 212 +++++ .../docs.firecrawl.dev/api_spec_31.json | 199 +++++ .../docs.firecrawl.dev/api_spec_33.json | 202 +++++ .../docs.firecrawl.dev/api_spec_34.json | 201 +++++ .../docs.firecrawl.dev/api_spec_35.json | 245 ++++++ .../docs.firecrawl.dev/api_spec_4.json | 129 +++ .../docs.firecrawl.dev/api_spec_5.json | 186 +++++ .../docs.firecrawl.dev/api_spec_7.json | 86 ++ .../docs.firecrawl.dev/api_spec_8.json | 59 ++ .../docs.firecrawl.dev/combined_api_spec.json | 738 +++++++++++++++++ .../turning_docs_into_api_specs.ipynb | 287 +++++++ 25 files changed, 5198 insertions(+) create mode 100644 examples/turning_docs_into_api_specs/api_spec.json create mode 100644 examples/turning_docs_into_api_specs/dify_api_spec.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_0.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_1.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_10.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_11.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_13.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_15.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_16.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_2.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_22.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_25.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_26.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_3.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_30.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_31.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_33.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_34.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_35.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_4.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_5.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_7.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_8.json create mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/combined_api_spec.json create mode 100644 examples/turning_docs_into_api_specs/turning_docs_into_api_specs.ipynb diff --git a/examples/turning_docs_into_api_specs/api_spec.json b/examples/turning_docs_into_api_specs/api_spec.json new file mode 100644 index 00000000..d866efd3 --- /dev/null +++ b/examples/turning_docs_into_api_specs/api_spec.json @@ -0,0 +1,771 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/crawl": { + "post": { + "/crawl/cancel/{jobId}": { + "/crawl/status/{jobId}": { + "get": { + "/scrape": { + "/search": { + "post": { + "components": { + "securitySchemes": { + "Authorization": { + "bearerFormat": "JWT", + "scheme": "bearer", + "type": "http" + } + } + }, + "description": "Send a request to perform a web search and get scraped results from the top pages.", + "operationId": "searchWeb", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "pageOptions": { + "description": "Options for controlling the scraping behavior of search result pages.", + "properties": { + "fetchPageContent": { + "default": true, + "description": "Fetch the content of each page. If false, defaults to a basic fast serp API.", + "type": "boolean" + }, + "includeHtml": { + "default": false, + "description": "Include the HTML version of the content on page. Will output a html key in the response.", + "type": "boolean" + }, + "includeRawHtml": { + "default": false, + "description": "Include the raw HTML content of the page. Will output a rawHtml key in the response.", + "type": "boolean" + }, + "onlyMainContent": { + "default": false, + "description": "Only return the main content of the page excluding headers, navs, footers, etc.", + "type": "boolean" + } + }, + "type": "object" + }, + "query": { + "description": "The search query.", + "required": true, + "type": "string" + }, + "searchOptions": { + "description": "Options for controlling the search.", + "properties": { + "limit": { + "description": "Maximum number of search results to return.", + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" + } + } + }, + "responses": { + "200": { + "402": { + "description": "Payment required." + }, + "429": { + "description": "Rate limit exceeded." + }, + "500": { + "description": "Internal server error." + }, + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "description": "An array of search results.", + "items": { + "properties": { + "content": { + "description": "Raw content of the search result page.", + "type": "string" + }, + "markdown": { + "description": "Markdown content of the search result page.", + "type": "string" + }, + "metadata": { + "description": "Metadata extracted from the search result page.", + "properties": { + "description": { + "description": "Page description.", + "type": "string" + }, + "language": { + "description": "Page language.", + "nullable": true, + "type": "string" + }, + "sourceURL": { + "description": "Source URL of the search result page.", + "type": "string" + }, + "title": { + "description": "Page title.", + "type": "string" + } + }, + "type": "object" + }, + "url": { + "description": "URL of the search result.", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "success": { + "description": "Indicates if the search was successful.", + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Web search completed successfully." + } + } + }, + "summary": "Search the Web" + } + }, + "post": { + "description": "Send a request to scrape a single URL and get its content.", + "operationId": "scrapeURL", + "parameters": [], + "requestBody": { + "402": { + "description": "Payment required." + }, + "429": { + "description": "Rate limit exceeded." + }, + "500": { + "description": "Internal server error." + }, + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "description": "Options for extraction of structured information from the page content. Note: LLM-based extraction is not performed by default and only occurs when explicitly configured. The 'markdown' mode simply returns the scraped markdown and is the default mode for scraping.", + "properties": { + "extractionPrompt": { + "description": "A prompt describing what information to extract from the page, applicable for LLM extraction modes.", + "type": "string" + }, + "extractionSchema": { + "description": "The schema for the data to be extracted, required only for LLM extraction modes.", + "type": "object" + }, + "mode": { + "default": "markdown", + "description": "The extraction mode to use. 'markdown': Returns the scraped markdown content, does not perform LLM extraction. 'llm-extraction': Extracts information from the cleaned and parsed content using LLM. 'llm-extraction-from-raw-html': Extracts information directly from the raw HTML using LLM. 'llm-extraction-from-markdown': Extracts information from the markdown content using LLM.", + "enum": [ + "markdown", + "llm-extraction", + "llm-extraction-from-raw-html", + "llm-extraction-from-markdown" + ], + "type": "string" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Options for controlling the scraping behavior.", + "properties": { + "fullPageScreenshot": { + "default": false, + "description": "Include a full page screenshot of the page that you are scraping.", + "type": "boolean" + }, + "headers": { + "description": "Headers to send with the request. Can be used to send cookies, user-agent, etc.", + "type": "object" + }, + "includeHtml": { + "default": false, + "description": "Include the HTML version of the content on page. Will output a html key in the response.", + "type": "boolean" + }, + "includeRawHtml": { + "default": false, + "description": "Include the raw HTML content of the page. Will output a rawHtml key in the response.", + "type": "boolean" + }, + "onlyIncludeTags": { + "description": "Only include tags, classes and ids from the page in the final output. Use comma separated values. Example: 'script, .ad, #footer'", + "items": { + "type": "string" + }, + "type": "array" + }, + "onlyMainContent": { + "default": false, + "description": "Only return the main content of the page excluding headers, navs, footers, etc.", + "type": "boolean" + }, + "removeTags": { + "description": "Tags, classes and ids to remove from the page. Use comma separated values. Example: 'script, .ad, #footer'", + "items": { + "type": "string" + }, + "type": "array" + }, + "replaceAllPathsWithAbsolutePaths": { + "default": false, + "description": "Replace all relative paths with absolute paths for images and links", + "type": "boolean" + }, + "screenshot": { + "default": false, + "description": "Include a screenshot of the top of the page that you are scraping.", + "type": "boolean" + }, + "waitFor": { + "default": 0, + "description": "Wait x amount of milliseconds for the page to load to fetch content", + "type": "integer" + } + }, + "type": "object" + }, + "timeout": { + "default": 30000, + "description": "Timeout in milliseconds for the request", + "type": "integer" + }, + "url": { + "description": "The URL to scrape.", + "required": true, + "type": "string" + } + }, + "type": "object" + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": { + "content": { + "description": "Raw content of the page.", + "type": "string" + }, + "html": { + "description": "HTML version of the page content, only present if `includeHtml` was set to `true` in the request.", + "nullable": true, + "type": "string" + }, + "llm_extraction": { + "description": "Extracted data from the page using the specified schema, only present if an LLM extraction mode was used.", + "nullable": true, + "type": "object" + }, + "markdown": { + "description": "Markdown version of the page content.", + "type": "string" + }, + "metadata": { + "properties": { + " ": { + "description": "Any other extracted metadata.", + "type": "string" + }, + "description": { + "description": "Page description.", + "type": "string" + }, + "language": { + "description": "Page language.", + "nullable": true, + "type": "string" + }, + "pageError": { + "description": "Error message if there was an error scraping the page.", + "nullable": true, + "type": "string" + }, + "pageStatusCode": { + "description": "HTTP status code of the page.", + "type": "integer" + }, + "sourceURL": { + "description": "Source URL of the page.", + "type": "string" + }, + "title": { + "description": "Page title.", + "type": "string" + } + }, + "type": "object" + }, + "rawHtml": { + "description": "Raw HTML content of the page, only present if `includeRawHtml` was set to `true` in the request.", + "nullable": true, + "type": "string" + }, + "warning": { + "description": "Warning message from the LLM extraction process, if any.", + "nullable": true, + "type": "string" + } + }, + "type": "object" + }, + "success": { + "description": "Indicates whether the scraping was successful.", + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "URL scraped successfully." + } + } + }, + "summary": "Scrape a URL" + } + }, + "description": "Send a request to get the status and results of a crawl job.", + "operationId": "getCrawlJobStatus", + "parameters": [ + { + "description": "ID of the crawl job to check.", + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": {} + }, + "responses": { + "200": { + "402": { + "description": "Payment required." + }, + "429": { + "description": "Rate limit exceeded." + }, + "500": { + "description": "Internal server error." + }, + "content": { + "application/json": { + "schema": { + "properties": { + "current": { + "description": "The number of pages crawled so far.", + "type": "integer" + }, + "data": { + "description": "The crawl results. Only available when the crawl job is completed.", + "items": { + "properties": { + "content": { + "description": "Raw content of the page.", + "type": "string" + }, + "html": { + "description": "HTML version of the page content, only present if `includeHtml` was set to `true` in the crawl request.", + "type": "string" + }, + "index": { + "description": "The index of the crawled page in the results.", + "type": "integer" + }, + "markdown": { + "description": "Markdown content of the page.", + "type": "string" + }, + "metadata": { + "description": "Metadata extracted from the page.", + "properties": { + " ": { + "description": "Any other extracted metadata.", + "type": "string" + }, + "description": { + "description": "Page description.", + "type": "string" + }, + "language": { + "description": "Page language.", + "type": "string" + }, + "pageError": { + "description": "Error message if there was an error scraping the page.", + "type": "string" + }, + "pageStatusCode": { + "description": "HTTP status code of the page.", + "type": "integer" + }, + "sourceURL": { + "description": "Source URL of the page.", + "type": "string" + }, + "title": { + "description": "Page title.", + "type": "string" + } + }, + "type": "object" + }, + "rawHtml": { + "description": "Raw HTML content of the page, only present if `includeRawHtml` was set to `true` in the crawl request.", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "partial_data": { + "description": "Partial results streamed as the crawl progresses. This feature is in alpha and may change.", + "items": { + "properties": { + "content": { + "description": "Raw content of the page.", + "type": "string" + }, + "html": { + "description": "HTML version of the page content, only present if `includeHtml` was set to `true` in the crawl request.", + "type": "string" + }, + "index": { + "description": "The index of the crawled page in the results.", + "type": "integer" + }, + "markdown": { + "description": "Markdown content of the page.", + "type": "string" + }, + "metadata": { + "description": "Metadata extracted from the page.", + "properties": { + " ": { + "description": "Any other extracted metadata.", + "type": "string" + }, + "description": { + "description": "Page description.", + "type": "string" + }, + "language": { + "description": "Page language.", + "type": "string" + }, + "pageError": { + "description": "Error message if there was an error scraping the page.", + "type": "string" + }, + "pageStatusCode": { + "description": "HTTP status code of the page.", + "type": "integer" + }, + "sourceURL": { + "description": "Source URL of the page.", + "type": "string" + }, + "title": { + "description": "Page title.", + "type": "string" + } + }, + "type": "object" + }, + "rawHtml": { + "description": "Raw HTML content of the page, only present if `includeRawHtml` was set to `true` in the crawl request.", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "status": { + "description": "Status of the crawl job. Can be 'completed', 'active', 'failed', or 'paused'.", + "enum": [ + "completed", + "active", + "failed", + "paused" + ], + "type": "string" + }, + "total": { + "description": "The total estimated number of pages to crawl.", + "type": "integer" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job status retrieved." + } + }, + "summary": "Get Crawl Job Status" + } + }, + "delete": { + "description": "Send a request to cancel a running crawl job.", + "operationId": "cancelCrawlJob", + "parameters": [ + { + "description": "ID of the crawl job to cancel.", + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": {} + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "status": { + "description": "The status of the crawl job cancellation request, usually 'cancelled'.", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job cancellation request submitted." + }, + "402": { + "description": "Payment required." + }, + "429": { + "description": "Rate limit exceeded." + }, + "500": { + "description": "Internal server error." + } + }, + "summary": "Cancel a Crawl Job" + } + }, + "description": "Send a request to crawl a URL and all accessible subpages. This submits a crawl job and returns a job ID to check the status of the crawl.", + "operationId": "crawlWebsite", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "description": "Options for controlling the crawling behavior.", + "properties": { + "allowBackwardCrawling": { + "default": false, + "description": "Enables the crawler to navigate from a specific URL to previously linked pages. For instance, from 'example.com/product/123' back to 'example.com/product'", + "type": "boolean" + }, + "allowExternalContentLinks": { + "default": false, + "description": "Allows the crawler to follow links to external websites.", + "type": "boolean" + }, + "excludes": { + "description": "URL patterns to exclude", + "items": { + "type": "string" + }, + "type": "array" + }, + "generateImgAltText": { + "default": false, + "description": "Generate alt text for images using LLMs (must have a paid plan)", + "type": "boolean" + }, + "ignoreSitemap": { + "default": false, + "description": "Ignore the website sitemap when crawling", + "type": "boolean" + }, + "includes": { + "description": "URL patterns to include", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "default": 10000, + "description": "Maximum number of pages to crawl", + "type": "integer" + }, + "maxDepth": { + "description": "Maximum depth to crawl relative to the entered URL. A maxDepth of 0 scrapes only the entered URL. A maxDepth of 1 scrapes the entered URL and all pages one level deep. A maxDepth of 2 scrapes the entered URL and all pages up to two levels deep. Higher values follow the same pattern.", + "type": "integer" + }, + "mode": { + "default": "default", + "description": "The crawling mode to use. Fast mode crawls 4x faster websites without sitemap, but may not be as accurate and shouldn't be used in heavy js-rendered websites.", + "enum": [ + "default", + "fast" + ], + "type": "string" + }, + "returnOnlyUrls": { + "default": false, + "description": "If true, returns only the URLs as a list on the crawl status. Attention: the return response will be a list of URLs inside the data, not a list of documents.", + "type": "boolean" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Options for controlling the scraping behavior of individual pages.", + "properties": { + "fullPageScreenshot": { + "default": false, + "description": "Include a full page screenshot of the page that you are scraping.", + "type": "boolean" + }, + "headers": { + "description": "Headers to send with the request. Can be used to send cookies, user-agent, etc.", + "type": "object" + }, + "includeHtml": { + "default": false, + "description": "Include the HTML version of the content on page. Will output a html key in the response.", + "type": "boolean" + }, + "includeRawHtml": { + "default": false, + "description": "Include the raw HTML content of the page. Will output a rawHtml key in the response.", + "type": "boolean" + }, + "onlyIncludeTags": { + "description": "Only include tags, classes and ids from the page in the final output. Use comma separated values. Example: 'script, .ad, #footer'", + "items": { + "type": "string" + }, + "type": "array" + }, + "onlyMainContent": { + "default": false, + "description": "Only return the main content of the page excluding headers, navs, footers, etc.", + "type": "boolean" + }, + "removeTags": { + "description": "Tags, classes and ids to remove from the page. Use comma separated values. Example: 'script, .ad, #footer'", + "items": { + "type": "string" + }, + "type": "array" + }, + "replaceAllPathsWithAbsolutePaths": { + "default": false, + "description": "Replace all relative paths with absolute paths for images and links", + "type": "boolean" + }, + "screenshot": { + "default": false, + "description": "Include a screenshot of the top of the page that you are scraping.", + "type": "boolean" + }, + "waitFor": { + "default": 0, + "description": "Wait x amount of milliseconds for the page to load to fetch content", + "type": "integer" + } + }, + "type": "object" + }, + "url": { + "description": "The base URL to start crawling from", + "required": true, + "type": "string" + } + }, + "type": "object" + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "The ID of the submitted crawl job.", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job submitted successfully." + }, + "402": { + "description": "Payment required." + }, + "429": { + "description": "Rate limit exceeded." + }, + "500": { + "description": "Internal server error." + } + } + }, + "summary": "Crawl a Website" + } + } + }, + "servers": [ + { + "url": "https://api.firecrawl.dev/v0" + } + ] +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/dify_api_spec.json b/examples/turning_docs_into_api_specs/dify_api_spec.json new file mode 100644 index 00000000..e6eec457 --- /dev/null +++ b/examples/turning_docs_into_api_specs/dify_api_spec.json @@ -0,0 +1,164 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Knowledge Base API", + "description": "API for managing knowledge bases and documents." + }, + "paths": { + "/datasets": { + "post": { + "summary": "Create an Empty Dataset", + "description": "Only used to create an empty dataset", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + } + } + }, + "responses": {} + }, + "get": { + "summary": "Dataset List", + "parameters": [ + { + "name": "page", + "in": "query", + "schema": { + "type": "integer" + } + }, + { + "name": "limit", + "in": "query", + "schema": { + "type": "integer" + } + } + ], + "responses": {} + } + }, + "/datasets/{dataset_id}/document/create_by_text": { + "post": { + "summary": "Create Document by Text", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "text": { + "type": "string" + }, + "indexing_technique": { + "type": "string" + }, + "process_rule": { + "type": "object" + } + } + } + } + } + }, + "responses": {} + } + }, + "/datasets/{dataset_id}/document/create_by_file": { + "post": { + "summary": "Create Document by File", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "string" + }, + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": {} + } + }, + "/datasets/{dataset_id}/documents/{batch}/indexing-status": { + "get": { + "summary": "Get Document Embedding Status (Progress)", + "responses": {} + } + }, + "/datasets/{dataset_id}/documents/{document_id}": { + "delete": { + "summary": "Delete Document", + "responses": {} + } + }, + "/datasets/{dataset_id}/documents": { + "get": { + "summary": "Dataset Document List", + "responses": {} + } + }, + "/datasets/{dataset_id}/documents/{document_id}/segments": { + "post": { + "summary": "Add Segments", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "segments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "answer": { + "type": "string" + }, + "keywords": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "responses": {} + } + }, + "/datasets/{dataset_id}/segments/{segment_id}": { + "delete": { + "summary": "Delete Document Segment", + "responses": {} + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_0.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_0.json new file mode 100644 index 00000000..84bce02c --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_0.json @@ -0,0 +1,211 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/v0/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "description": "Crawling options.", + "properties": { + "excludes": { + "description": "URL patterns to exclude.", + "items": { + "type": "string" + }, + "type": "array" + }, + "includes": { + "description": "URL patterns to include.", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Maximum pages to crawl.", + "type": "integer" + }, + "maxDepth": { + "description": "Maximum crawl depth.", + "type": "integer" + }, + "mode": { + "description": "Crawling mode.", + "enum": [ + "default", + "fast" + ], + "type": "string" + }, + "returnOnlyUrls": { + "description": "Return only URLs.", + "type": "boolean" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Page scraping options.", + "properties": { + "includeHtml": { + "description": "Include HTML content.", + "type": "boolean" + }, + "includeRawHtml": { + "description": "Include raw HTML content.", + "type": "boolean" + }, + "onlyMainContent": { + "description": "Only main content.", + "type": "boolean" + }, + "screenshot": { + "description": "Include page screenshot.", + "type": "boolean" + }, + "waitFor": { + "description": "Wait time in milliseconds.", + "type": "integer" + } + }, + "type": "object" + }, + "url": { + "description": "Base URL to crawl.", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Crawl job ID.", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job initiated." + } + }, + "summary": "Crawl multiple pages." + } + }, + "/v0/crawl/status/{jobId}": { + "get": { + "parameters": [ + { + "description": "Crawl job ID.", + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Crawl job status." + } + }, + "summary": "Check crawl job status." + } + }, + "/v0/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "description": "Data extraction options.", + "properties": { + "extractionPrompt": { + "description": "Prompt for data extraction.", + "type": "string" + }, + "extractionSchema": { + "description": "Schema for data extraction.", + "type": "object" + }, + "mode": { + "description": "Extraction mode.", + "enum": [ + "llm-extraction", + "llm-extraction-from-raw-html" + ], + "type": "string" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Page scraping options.", + "properties": { + "includeHtml": { + "description": "Include HTML content.", + "type": "boolean" + }, + "includeRawHtml": { + "description": "Include raw HTML content.", + "type": "boolean" + }, + "onlyMainContent": { + "description": "Only main content.", + "type": "boolean" + }, + "screenshot": { + "description": "Include page screenshot.", + "type": "boolean" + }, + "waitFor": { + "description": "Wait time in milliseconds.", + "type": "integer" + } + }, + "type": "object" + }, + "timeout": { + "description": "Timeout in milliseconds.", + "type": "integer" + }, + "url": { + "description": "URL to scrape.", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "description": "Successful scraping." + } + }, + "summary": "Scrape a single page." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_1.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_1.json new file mode 100644 index 00000000..8656c978 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_1.json @@ -0,0 +1,165 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "properties": { + "allowBackwardCrawling": { + "description": "Allow backward crawling", + "type": "boolean" + }, + "allowExternalContentLinks": { + "description": "Allow external links", + "type": "boolean" + }, + "excludes": { + "description": "URL patterns to exclude", + "items": { + "type": "string" + }, + "type": "array" + }, + "generateImgAltText": { + "description": "Generate alt text for images", + "type": "boolean" + }, + "ignoreSitemap": { + "description": "Ignore website sitemap", + "type": "boolean" + }, + "includes": { + "description": "URL patterns to include", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Maximum pages to crawl", + "type": "integer" + }, + "maxDepth": { + "description": "Maximum crawl depth", + "type": "integer" + }, + "mode": { + "description": "Crawling mode", + "enum": [ + "default", + "fast" + ], + "type": "string" + }, + "returnOnlyUrls": { + "description": "Return only crawled URLs", + "type": "boolean" + } + }, + "type": "object" + }, + "pageOptions": { + "properties": { + "fullPageScreenshot": { + "description": "Include full page screenshot", + "type": "boolean" + }, + "headers": { + "description": "Headers for requests", + "type": "object" + }, + "includeHtml": { + "description": "Include HTML content", + "type": "boolean" + }, + "includeRawHtml": { + "description": "Include raw HTML content", + "type": "boolean" + }, + "onlyIncludeTags": { + "description": "Include only specific tags", + "items": { + "type": "string" + }, + "type": "array" + }, + "onlyMainContent": { + "description": "Return only main content", + "type": "boolean" + }, + "removeTags": { + "description": "Remove specific tags", + "items": { + "type": "string" + }, + "type": "array" + }, + "replaceAllPathsWithAbsolutePaths": { + "description": "Use absolute paths", + "type": "boolean" + }, + "screenshot": { + "description": "Include page screenshot", + "type": "boolean" + }, + "waitFor": { + "description": "Wait for page load (ms)", + "type": "integer" + } + }, + "type": "object" + }, + "url": { + "description": "Base URL to crawl", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Job ID of the crawl", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl request successful" + } + }, + "security": [ + { + "Bearer": [] + } + ], + "summary": "Crawl a website" + } + } + }, + "securitySchemes": { + "Bearer": { + "bearerFormat": "JWT", + "scheme": "bearer", + "type": "http" + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_10.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_10.json new file mode 100644 index 00000000..55f73a32 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_10.json @@ -0,0 +1,93 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/check_crawl_status": { + "post": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "current": { + "type": "integer" + }, + "data": { + "items": { + "properties": { + "content": { + "type": "string" + }, + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "provider": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "status": { + "type": "string" + }, + "total": { + "type": "integer" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job status" + } + }, + "summary": "Check crawl job status" + } + }, + "/crawl": { + "post": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Job ID" + } + }, + "summary": "Crawl URL and subpages" + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_11.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_11.json new file mode 100644 index 00000000..e19ed056 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_11.json @@ -0,0 +1,131 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "example": { + "extractorOptions": { + "extractionPrompt": "Based on the information on the page, extract the information from the schema. ", + "extractionSchema": { + "properties": { + "company_mission": { + "type": "string" + }, + "is_in_yc": { + "type": "boolean" + }, + "is_open_source": { + "type": "boolean" + }, + "supports_sso": { + "type": "boolean" + } + }, + "required": [ + "company_mission", + "supports_sso", + "is_open_source", + "is_in_yc" + ], + "type": "object" + }, + "mode": "llm-extraction" + }, + "url": "https://docs.firecrawl.dev/" + }, + "schema": { + "properties": { + "extractorOptions": { + "properties": { + "extractionPrompt": { + "description": "Prompt for extraction", + "type": "string" + }, + "extractionSchema": { + "description": "Schema for data extraction", + "type": "object" + }, + "mode": { + "description": "Extraction mode", + "type": "string" + } + }, + "type": "object" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "example": { + "data": { + "content": "Raw Content", + "llm_extraction": { + "company_mission": "Train a secure AI on your technical resources that answers customer and employee questions so your team doesn't have to", + "is_in_yc": true, + "is_open_source": false, + "supports_sso": true + }, + "metadata": { + "description": "Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide", + "ogDescription": "Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide", + "ogImage": "https://docs.firecrawl.dev/mendable_new_og1.png", + "ogLocaleAlternate": [], + "ogSiteName": "Mendable", + "ogTitle": "Mendable", + "ogUrl": "https://docs.firecrawl.dev/", + "robots": "follow, index", + "sourceURL": "https://docs.firecrawl.dev/", + "title": "Mendable" + } + }, + "success": true + }, + "schema": { + "properties": { + "data": { + "properties": { + "content": { + "type": "string" + }, + "llm_extraction": { + "type": "object" + }, + "metadata": { + "type": "object" + } + }, + "type": "object" + }, + "success": { + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Successful scrape" + } + }, + "summary": "Extract data from pages." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_13.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_13.json new file mode 100644 index 00000000..0352c66f --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_13.json @@ -0,0 +1,87 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/search": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "pageOptions": { + "properties": { + "fetchPageContent": { + "type": "boolean" + } + }, + "type": "object" + }, + "query": { + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "provider": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "success": { + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Successful search and scrape." + } + }, + "summary": "Search web, scrape, return markdown." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_15.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_15.json new file mode 100644 index 00000000..e7384f8e --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_15.json @@ -0,0 +1,83 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "url": { + "description": "Website URL to crawl.", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "properties": { + "markdown": { + "description": "Markdown content.", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + } + }, + "description": "Website crawled successfully." + } + }, + "summary": "Crawl a website." + } + }, + "/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "url": { + "description": "Page URL to scrape.", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "text/plain": { + "schema": { + "description": "Scraped content.", + "type": "string" + } + } + }, + "description": "Page scraped successfully." + } + }, + "summary": "Scrape a single page." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_16.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_16.json new file mode 100644 index 00000000..ed6fb9d6 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_16.json @@ -0,0 +1,200 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawler_options": { + "properties": { + "exclude": { + "description": "URL patterns to exclude", + "items": { + "type": "string" + }, + "type": "array" + }, + "generateImgAltText": { + "description": "Generate alt text for images", + "type": "boolean" + }, + "includes": { + "description": "URL patterns to include", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Max pages to crawl", + "type": "integer" + }, + "maxDepth": { + "description": "Maximum crawl depth", + "type": "integer" + }, + "mode": { + "description": "Crawling mode", + "type": "string" + }, + "returnOnlyUrls": { + "description": "Return only URLs", + "type": "boolean" + }, + "timeout": { + "description": "Timeout in milliseconds", + "type": "integer" + } + }, + "type": "object" + }, + "page_options": { + "properties": { + "includeHtml": { + "description": "Include raw HTML", + "type": "boolean" + }, + "onlyMainContent": { + "description": "Only main content", + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "Base URL to crawl", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "description": "Crawl successful." + } + }, + "summary": "Crawl a website." + } + }, + "/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractor_options": { + "properties": { + "extractionPrompt": { + "description": "Prompt for extraction", + "type": "string" + }, + "extractionSchema": { + "description": "Schema for extraction", + "type": "string" + }, + "mode": { + "description": "Extraction mode", + "type": "string" + } + }, + "type": "object" + }, + "page_options": { + "properties": { + "includeHtml": { + "description": "Include raw HTML", + "type": "boolean" + }, + "onlyMainContent": { + "description": "Only main content", + "type": "boolean" + } + }, + "type": "object" + }, + "timeout": { + "description": "Timeout in milliseconds", + "type": "integer" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "description": "Scrape successful." + } + }, + "summary": "Scrape a website." + } + }, + "/search": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "page_options": { + "properties": { + "fetchPageContent": { + "description": "Fetch full content", + "type": "boolean" + }, + "includeHtml": { + "description": "Include raw HTML", + "type": "boolean" + }, + "onlyMainContent": { + "description": "Only main content", + "type": "boolean" + } + }, + "type": "object" + }, + "query": { + "description": "Search query string", + "type": "string" + }, + "search_options": { + "properties": { + "limit": { + "description": "Max results", + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "description": "Search successful." + } + }, + "summary": "Search Firecrawl index." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_2.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_2.json new file mode 100644 index 00000000..25cf6c05 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_2.json @@ -0,0 +1,54 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/crawl/cancel/{jobId}": { + "delete": { + "parameters": [ + { + "description": "ID of crawl job", + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "status": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Returns cancelled." + } + }, + "security": [ + { + "Bearer": [] + } + ], + "summary": "Cancel crawl job" + } + } + }, + "securitySchemes": { + "Bearer": { + "bearerFormat": "Bearer ", + "scheme": "bearer", + "type": "http" + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_22.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_22.json new file mode 100644 index 00000000..ac146a63 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_22.json @@ -0,0 +1,166 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/check-crawl-status/{jobId}": { + "get": { + "parameters": [ + { + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "current": { + "description": "Current progress", + "type": "integer" + }, + "data": { + "items": { + "properties": { + "content": { + "description": "Raw content", + "type": "string" + }, + "markdown": { + "description": "Markdown content", + "type": "string" + }, + "metadata": { + "description": "Page metadata", + "type": "object" + }, + "provider": { + "description": "Data provider", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "status": { + "description": "Job status", + "type": "string" + }, + "total": { + "description": "Total pages", + "type": "integer" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job status." + } + }, + "summary": "Check crawl job status." + } + }, + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "description": "Crawler options", + "type": "object" + }, + "url": { + "description": "URL to crawl", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Job ID", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job submitted." + } + }, + "summary": "Crawl a URL." + } + }, + "/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "description": "Extractor options", + "type": "object" + }, + "pageOptions": { + "description": "Page options", + "type": "object" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "description": "Scraped data", + "type": "object" + }, + "success": { + "description": "Success flag", + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Scraped data." + } + }, + "summary": "Scrape a single URL." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_25.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_25.json new file mode 100644 index 00000000..9701a462 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_25.json @@ -0,0 +1,229 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/v0/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "properties": { + "excludes": { + "description": "Paths to exclude", + "items": { + "type": "string" + }, + "type": "array" + }, + "includes": { + "description": "Paths to include", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Maximum pages to crawl", + "type": "integer" + }, + "maxDepth": { + "description": "Maximum crawl depth", + "type": "integer" + }, + "returnOnlyUrls": { + "description": "Only return URLs", + "type": "boolean" + } + }, + "type": "object" + }, + "pageOptions": { + "properties": { + "onlyMainContent": { + "description": "Extract main content", + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "URL to crawl", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Job ID", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job created" + } + }, + "summary": "Crawl a website" + } + }, + "/v0/crawl/status/{jobId}": { + "get": { + "parameters": [ + { + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "current": { + "type": "integer" + }, + "data": { + "items": { + "properties": { + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "status": { + "description": "Job status", + "type": "string" + }, + "total": { + "type": "integer" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job status" + } + }, + "summary": "Get crawl job status" + } + }, + "/v0/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "pageOptions": { + "properties": { + "onlyMainContent": { + "description": "Extract main content", + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": { + "content": { + "type": "string" + }, + "html": { + "type": "string" + }, + "llm_extraction": { + "type": "object" + }, + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "pageError": { + "type": "string" + }, + "pageStatusCode": { + "type": "integer" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "rawHtml": { + "type": "string" + }, + "warning": { + "type": "string" + } + }, + "type": "object" + }, + "success": { + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Scrape results" + } + }, + "summary": "Scrape a webpage" + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_26.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_26.json new file mode 100644 index 00000000..b642e9c0 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_26.json @@ -0,0 +1,115 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "example": { + "extractorOptions": { + "extractionPrompt": "Extract company info.", + "extractionSchema": { + "properties": { + "company_description": { + "type": "string" + }, + "company_industry": { + "type": "string" + }, + "who_they_serve": { + "type": "string" + } + }, + "required": [ + "company_description", + "company_industry", + "who_they_serve" + ], + "type": "object" + }, + "mode": "llm-extraction" + }, + "pageOptions": { + "onlyMainContent": true + }, + "url": "https://example.com" + }, + "schema": { + "properties": { + "extractorOptions": { + "properties": { + "extractionPrompt": { + "description": "Prompt for LLM extraction.", + "type": "string" + }, + "extractionSchema": { + "properties": { + "properties": { + "company_description": { + "type": "string" + }, + "company_industry": { + "type": "string" + }, + "who_they_serve": { + "type": "string" + } + }, + "required": [ + "company_description", + "company_industry", + "who_they_serve" + ], + "type": { + "type": "string" + } + }, + "type": "object" + }, + "mode": { + "description": "Extraction mode.", + "type": "string" + } + }, + "type": "object" + }, + "pageOptions": { + "properties": { + "onlyMainContent": { + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "URL to scrape.", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "Successful scrape." + } + }, + "summary": "Scrape data from URL." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_3.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_3.json new file mode 100644 index 00000000..bcf94159 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_3.json @@ -0,0 +1,185 @@ +{ + "components": { + "securitySchemes": { + "bearerAuth": { + "scheme": "bearer", + "type": "http" + } + } + }, + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/v0/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "description": "Options for extraction", + "properties": { + "extractionPrompt": { + "description": "Prompt for LLM extraction", + "type": "string" + }, + "extractionSchema": { + "description": "Schema for LLM extraction", + "type": "object" + }, + "mode": { + "description": "Extraction mode", + "enum": [ + "markdown", + "llm-extraction", + "llm-extraction-from-raw-html", + "llm-extraction-from-markdown" + ], + "type": "string" + } + }, + "type": "object" + }, + "pageOptions": { + "properties": { + "fullPageScreenshot": { + "description": "Include full page screenshot", + "type": "boolean" + }, + "headers": { + "description": "Headers for request", + "type": "object" + }, + "includeHtml": { + "description": "Include HTML content", + "type": "boolean" + }, + "includeRawHtml": { + "description": "Include raw HTML content", + "type": "boolean" + }, + "onlyIncludeTags": { + "description": "Include only these tags", + "items": { + "type": "string" + }, + "type": "array" + }, + "onlyMainContent": { + "description": "Only return main content", + "type": "boolean" + }, + "removeTags": { + "description": "Remove these tags", + "items": { + "type": "string" + }, + "type": "array" + }, + "replaceAllPathsWithAbsolutePaths": { + "description": "Replace relative paths", + "type": "boolean" + }, + "screenshot": { + "description": "Include screenshot", + "type": "boolean" + }, + "waitFor": { + "description": "Wait time in ms", + "type": "integer" + } + }, + "type": "object" + }, + "timeout": { + "description": "Timeout in ms", + "type": "integer" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": { + "content": { + "type": "string" + }, + "html": { + "type": "string" + }, + "llm_extraction": { + "type": "object" + }, + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "pageError": { + "type": "string" + }, + "pageStatusCode": { + "type": "integer" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "rawHtml": { + "type": "string" + }, + "warning": { + "type": "string" + } + }, + "type": "object" + }, + "success": { + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Successful scrape" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "Scrape a webpage" + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_30.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_30.json new file mode 100644 index 00000000..bc542e2a --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_30.json @@ -0,0 +1,212 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "description": "Crawl job options", + "properties": { + "excludes": { + "description": "Pages to exclude", + "items": { + "type": "string" + }, + "type": "array" + }, + "includes": { + "description": "Pages to include", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Max pages to crawl", + "type": "integer" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Page scraping options", + "properties": { + "onlyMainContent": { + "description": "Only scrape main content", + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "URL to crawl", + "type": "string" + } + }, + "required": [ + "url" + ], + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "description": "Crawl job result", + "type": "object" + } + } + }, + "description": "Crawl job result" + } + }, + "summary": "Crawl a website" + } + }, + "/crawl/{jobId}/cancel": { + "post": { + "parameters": [ + { + "description": "Crawl job ID", + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "description": "Cancellation status", + "type": "object" + } + } + }, + "description": "Cancellation status" + } + }, + "summary": "Cancel crawl job" + } + }, + "/crawl/{jobId}/status": { + "get": { + "parameters": [ + { + "description": "Crawl job ID", + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "description": "Crawl status", + "type": "object" + } + } + }, + "description": "Crawl status" + } + }, + "summary": "Check crawl status" + } + }, + "/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "description": "LLM extraction options", + "properties": { + "extractionSchema": { + "description": "JSON schema for extraction", + "type": "object" + } + }, + "type": "object" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "required": [ + "url" + ], + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "description": "Scraped data", + "type": "object" + } + } + }, + "description": "Scraped data" + } + }, + "summary": "Scrape a single URL" + } + }, + "/search": { + "get": { + "parameters": [ + { + "description": "Search query", + "in": "query", + "name": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "description": "Search results", + "type": "object" + } + } + }, + "description": "Search results" + } + }, + "summary": "Search and scrape" + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_31.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_31.json new file mode 100644 index 00000000..07f71759 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_31.json @@ -0,0 +1,199 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "properties": { + "excludes": { + "description": "Paths to exclude", + "items": { + "type": "string" + }, + "type": "array" + }, + "includes": { + "description": "Paths to include", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Maximum pages to crawl", + "type": "integer" + } + }, + "type": "object" + }, + "pageOptions": { + "properties": { + "onlyMainContent": { + "description": "Extract only main content", + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "Starting URL for crawl", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Unique job identifier", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job started" + } + }, + "summary": "Crawl a website" + } + }, + "/crawl/{jobId}/status": { + "get": { + "parameters": [ + { + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "status": { + "description": "Current job status", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job status" + } + }, + "summary": "Check crawl status" + } + }, + "/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "properties": { + "extractionSchema": { + "description": "Zod schema for extraction", + "type": "object" + } + }, + "type": "object" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "description": "Extracted data", + "type": "object" + } + }, + "type": "object" + } + } + }, + "description": "Scraped data" + } + }, + "summary": "Scrape a single URL" + } + }, + "/search": { + "get": { + "parameters": [ + { + "in": "query", + "name": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "properties": { + "content": { + "description": "Page content (optional)", + "type": "string" + }, + "url": { + "description": "Result URL", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + } + }, + "description": "Search results" + } + }, + "summary": "Search for a query" + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_33.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_33.json new file mode 100644 index 00000000..b45ae841 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_33.json @@ -0,0 +1,202 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "description": "Options for crawling", + "properties": { + "excludes": { + "description": "URLs to exclude", + "items": { + "type": "string" + }, + "type": "array" + }, + "includes": { + "description": "URLs to include", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Maximum pages to crawl", + "type": "integer" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Options for page content", + "properties": { + "onlyMainContent": { + "description": "Extract only main content", + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "URL to crawl", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Unique crawl job ID", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job started." + } + }, + "summary": "Crawl a website." + } + }, + "/crawl/{jobId}": { + "get": { + "parameters": [ + { + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "status": { + "description": "Current job status", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job status." + } + }, + "summary": "Check crawl job status." + } + }, + "/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "description": "Options for data extraction", + "properties": { + "extractionSchema": { + "description": "Pydantic schema", + "type": "object" + }, + "mode": { + "description": "Extraction mode", + "type": "string" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Options for page content", + "properties": { + "onlyMainContent": { + "description": "Extract only main content", + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "Scraped data." + } + }, + "summary": "Scrape a single URL." + } + }, + "/search": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "query": { + "description": "Search query", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "Search results." + } + }, + "summary": "Search the web." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_34.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_34.json new file mode 100644 index 00000000..3bafda42 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_34.json @@ -0,0 +1,201 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "0.1" + }, + "openapi": "3.0.0", + "paths": { + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "description": "Crawl job options", + "properties": { + "excludes": { + "description": "URLs to exclude", + "items": { + "type": "string" + }, + "type": "array" + }, + "includes": { + "description": "URLs to include", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Maximum pages to crawl", + "type": "integer" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Page scraping options", + "properties": { + "onlyMainContent": { + "description": "Only scrape main content", + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "URL to crawl", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "Crawl job started" + } + }, + "summary": "Crawl a website." + } + }, + "/crawl/{job_id}/cancel": { + "post": { + "parameters": [ + { + "description": "Crawl job ID", + "in": "path", + "name": "job_id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "Cancellation status" + } + }, + "summary": "Cancel crawl job." + } + }, + "/crawl/{job_id}/status": { + "get": { + "parameters": [ + { + "description": "Crawl job ID", + "in": "path", + "name": "job_id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "Crawl status" + } + }, + "summary": "Check crawl status." + } + }, + "/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "description": "LLM extraction options", + "properties": { + "extractionSchema": { + "description": "JSON schema for extraction", + "type": "object" + } + }, + "type": "object" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "Scraped data" + } + }, + "summary": "Scrape a single URL." + } + }, + "/search": { + "get": { + "parameters": [ + { + "description": "Search query", + "in": "query", + "name": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "Search results" + } + }, + "summary": "Search and scrape results." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_35.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_35.json new file mode 100644 index 00000000..890d31b1 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_35.json @@ -0,0 +1,245 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/check-crawl-status": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Crawl job ID", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "current": { + "description": "Current page count", + "type": "integer" + }, + "data": { + "description": "Crawl data", + "items": { + "properties": { + "content": { + "description": "Raw content", + "type": "string" + }, + "markdown": { + "description": "Markdown content", + "type": "string" + }, + "metadata": { + "description": "Page metadata", + "properties": { + "description": { + "description": "Page description", + "type": "string" + }, + "language": { + "description": "Page language", + "type": "string" + }, + "sourceURL": { + "description": "Page URL", + "type": "string" + }, + "title": { + "description": "Page title", + "type": "string" + } + }, + "type": "object" + }, + "provider": { + "description": "Content provider", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "status": { + "description": "Crawl status", + "type": "string" + }, + "total": { + "description": "Total page count", + "type": "integer" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job status." + } + }, + "summary": "Check crawl job status." + } + }, + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "description": "Crawler options", + "properties": { + "excludes": { + "description": "URLs to exclude", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "url": { + "description": "URL to crawl", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Job ID", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job submitted." + } + }, + "summary": "Crawl a URL." + } + }, + "/scrape-url": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "description": "Extractor options", + "properties": { + "extractionSchema": { + "description": "Extraction schema", + "type": "string" + }, + "mode": { + "description": "Extraction mode", + "type": "string" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Page options", + "properties": { + "onlyMainContent": { + "description": "Only main content", + "type": "boolean" + } + }, + "type": "object" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "description": "Scraped data", + "properties": { + "content": { + "description": "Raw content", + "type": "string" + }, + "html": { + "description": "HTML content", + "type": "string" + }, + "llm_extraction": { + "description": "LLM extraction results", + "type": "object" + }, + "markdown": { + "description": "Markdown content", + "type": "string" + }, + "metadata": { + "description": "Page metadata", + "type": "object" + }, + "rawHtml": { + "description": "Raw HTML content", + "type": "string" + }, + "warning": { + "description": "Warning message", + "type": "string" + } + }, + "type": "object" + }, + "success": { + "description": "Request success", + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Scraped data." + } + }, + "summary": "Scrape a single URL." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_4.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_4.json new file mode 100644 index 00000000..daf53932 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_4.json @@ -0,0 +1,129 @@ +{ + "components": { + "securitySchemes": { + "Bearer": { + "scheme": "bearer", + "type": "http" + } + } + }, + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/search": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "pageOptions": { + "properties": { + "fetchPageContent": { + "description": "Fetch content of each page.", + "type": "boolean" + }, + "includeHtml": { + "description": "Include HTML content.", + "type": "boolean" + }, + "includeRawHtml": { + "description": "Include raw HTML content.", + "type": "boolean" + }, + "onlyMainContent": { + "description": "Only return main content.", + "type": "boolean" + } + }, + "type": "object" + }, + "query": { + "description": "The query to search for", + "type": "string" + }, + "searchOptions": { + "properties": { + "limit": { + "description": "Maximum number of results.", + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "content": { + "type": "string" + }, + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "success": { + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Successful search." + } + }, + "security": [ + { + "Bearer": [] + } + ], + "summary": "Search the web." + } + } + }, + "servers": [ + { + "url": "https://api.firecrawl.dev/v0" + } + ] +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_5.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_5.json new file mode 100644 index 00000000..4fae28c0 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_5.json @@ -0,0 +1,186 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/crawl/status/{jobId}": { + "get": { + "parameters": [ + { + "description": "ID of crawl job", + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "current": { + "description": "Current page number", + "type": "integer" + }, + "data": { + "description": "Data from the job", + "items": { + "properties": { + "content": { + "type": "string" + }, + "html": { + "description": "HTML content", + "nullable": true, + "type": "string" + }, + "index": { + "description": "Page number crawled", + "type": "integer" + }, + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "nullable": true, + "type": "string" + }, + "pageError": { + "description": "Error message of page", + "nullable": true, + "type": "string" + }, + "pageStatusCode": { + "description": "Status code of page", + "type": "integer" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + }, + "{any other metadata}": { + "type": "string" + } + }, + "type": "object" + }, + "rawHtml": { + "description": "Raw HTML content", + "nullable": true, + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "partial_data": { + "description": "Partial documents (streaming)", + "items": { + "properties": { + "content": { + "type": "string" + }, + "html": { + "description": "HTML content", + "nullable": true, + "type": "string" + }, + "index": { + "description": "Page number crawled", + "type": "integer" + }, + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "nullable": true, + "type": "string" + }, + "pageError": { + "description": "Error message of page", + "nullable": true, + "type": "string" + }, + "pageStatusCode": { + "description": "Status code of page", + "type": "integer" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + }, + "{any other metadata}": { + "type": "string" + } + }, + "type": "object" + }, + "rawHtml": { + "description": "Raw HTML content", + "nullable": true, + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "status": { + "description": "Status of the job", + "type": "string" + }, + "total": { + "description": "Total number of pages", + "type": "integer" + } + }, + "type": "object" + } + } + }, + "description": "Successful operation" + } + }, + "security": [ + { + "Authorization": [] + } + ], + "summary": "Get crawl job status" + } + } + }, + "securitySchemes": { + "Authorization": { + "bearerFormat": "Bearer ", + "scheme": "bearer", + "type": "http" + } + }, + "servers": [ + { + "url": "https://api.firecrawl.dev/v0" + } + ] +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_7.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_7.json new file mode 100644 index 00000000..b74b9886 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_7.json @@ -0,0 +1,86 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/v0/search": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "pageOptions": { + "properties": { + "fetchPageContent": { + "description": "Fetch page content", + "type": "boolean" + } + }, + "type": "object" + }, + "query": { + "description": "Search term", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "success": { + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Successful search" + } + }, + "summary": "Search and extract content" + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_8.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_8.json new file mode 100644 index 00000000..2d5f40e2 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_8.json @@ -0,0 +1,59 @@ +{ + "info": { + "title": "Firecrawl API", + "version": "v0" + }, + "openapi": "3.0.0", + "paths": { + "/test": { + "get": { + "description": "Returns a test message.", + "responses": { + "200": { + "content": { + "text/plain": { + "schema": { + "example": "Hello, world!", + "type": "string" + } + } + }, + "description": "Successful operation" + } + }, + "summary": "Test endpoint" + } + }, + "/v0/crawl": { + "post": { + "description": "Processes crawl job for URL.", + "requestBody": { + "content": { + "application/json": { + "example": { + "url": "https://docs.firecrawl.dev" + }, + "schema": { + "properties": { + "url": { + "description": "Website URL", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "URL to crawl", + "required": true + }, + "responses": { + "200": { + "description": "Crawl initiated." + } + }, + "summary": "Crawl a given URL." + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/combined_api_spec.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/combined_api_spec.json new file mode 100644 index 00000000..77d67234 --- /dev/null +++ b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/combined_api_spec.json @@ -0,0 +1,738 @@ +{ + "components": { + "schemas": {} + }, + "info": { + "title": "https://docs.firecrawl.dev API Specification", + "version": "1.0.0" + }, + "openapi": "3.0.0", + "paths": { + "/check_crawl_status": { + "post": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "current": { + "type": "integer" + }, + "data": { + "items": { + "properties": { + "content": { + "type": "string" + }, + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "provider": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "status": { + "type": "string" + }, + "total": { + "type": "integer" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job status" + } + }, + "summary": "Check crawl job status" + } + }, + "/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "properties": { + "allowBackwardCrawling": { + "description": "Allow backward crawling", + "type": "boolean" + }, + "allowExternalContentLinks": { + "description": "Allow external links", + "type": "boolean" + }, + "excludes": { + "description": "URL patterns to exclude", + "items": { + "type": "string" + }, + "type": "array" + }, + "generateImgAltText": { + "description": "Generate alt text for images", + "type": "boolean" + }, + "ignoreSitemap": { + "description": "Ignore website sitemap", + "type": "boolean" + }, + "includes": { + "description": "URL patterns to include", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Maximum pages to crawl", + "type": "integer" + }, + "maxDepth": { + "description": "Maximum crawl depth", + "type": "integer" + }, + "mode": { + "description": "Crawling mode", + "enum": [ + "default", + "fast" + ], + "type": "string" + }, + "returnOnlyUrls": { + "description": "Return only crawled URLs", + "type": "boolean" + } + }, + "type": "object" + }, + "pageOptions": { + "properties": { + "fullPageScreenshot": { + "description": "Include full page screenshot", + "type": "boolean" + }, + "headers": { + "description": "Headers for requests", + "type": "object" + }, + "includeHtml": { + "description": "Include HTML content", + "type": "boolean" + }, + "includeRawHtml": { + "description": "Include raw HTML content", + "type": "boolean" + }, + "onlyIncludeTags": { + "description": "Include only specific tags", + "items": { + "type": "string" + }, + "type": "array" + }, + "onlyMainContent": { + "description": "Return only main content", + "type": "boolean" + }, + "removeTags": { + "description": "Remove specific tags", + "items": { + "type": "string" + }, + "type": "array" + }, + "replaceAllPathsWithAbsolutePaths": { + "description": "Use absolute paths", + "type": "boolean" + }, + "screenshot": { + "description": "Include page screenshot", + "type": "boolean" + }, + "waitFor": { + "description": "Wait for page load (ms)", + "type": "integer" + } + }, + "type": "object" + }, + "url": { + "description": "Base URL to crawl", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Job ID of the crawl", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl request successful" + } + }, + "security": [ + { + "Bearer": [] + } + ], + "summary": "Crawl a website" + } + }, + "/crawl/cancel/{jobId}": { + "delete": { + "parameters": [ + { + "description": "ID of crawl job", + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "status": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Returns cancelled." + } + }, + "security": [ + { + "Bearer": [] + } + ], + "summary": "Cancel crawl job" + } + }, + "/search": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "pageOptions": { + "properties": { + "fetchPageContent": { + "description": "Fetch content of each page.", + "type": "boolean" + }, + "includeHtml": { + "description": "Include HTML content.", + "type": "boolean" + }, + "includeRawHtml": { + "description": "Include raw HTML content.", + "type": "boolean" + }, + "onlyMainContent": { + "description": "Only return main content.", + "type": "boolean" + } + }, + "type": "object" + }, + "query": { + "description": "The query to search for", + "type": "string" + }, + "searchOptions": { + "properties": { + "limit": { + "description": "Maximum number of results.", + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "content": { + "type": "string" + }, + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "success": { + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Successful search." + } + }, + "security": [ + { + "Bearer": [] + } + ], + "summary": "Search the web." + } + }, + "/v0/crawl": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "crawlerOptions": { + "description": "Crawling options.", + "properties": { + "excludes": { + "description": "URL patterns to exclude.", + "items": { + "type": "string" + }, + "type": "array" + }, + "includes": { + "description": "URL patterns to include.", + "items": { + "type": "string" + }, + "type": "array" + }, + "limit": { + "description": "Maximum pages to crawl.", + "type": "integer" + }, + "maxDepth": { + "description": "Maximum crawl depth.", + "type": "integer" + }, + "mode": { + "description": "Crawling mode.", + "enum": [ + "default", + "fast" + ], + "type": "string" + }, + "returnOnlyUrls": { + "description": "Return only URLs.", + "type": "boolean" + } + }, + "type": "object" + }, + "pageOptions": { + "description": "Page scraping options.", + "properties": { + "includeHtml": { + "description": "Include HTML content.", + "type": "boolean" + }, + "includeRawHtml": { + "description": "Include raw HTML content.", + "type": "boolean" + }, + "onlyMainContent": { + "description": "Only main content.", + "type": "boolean" + }, + "screenshot": { + "description": "Include page screenshot.", + "type": "boolean" + }, + "waitFor": { + "description": "Wait time in milliseconds.", + "type": "integer" + } + }, + "type": "object" + }, + "url": { + "description": "Base URL to crawl.", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "jobId": { + "description": "Crawl job ID.", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Crawl job initiated." + } + }, + "summary": "Crawl multiple pages." + } + }, + "/v0/crawl/status/{jobId}": { + "get": { + "parameters": [ + { + "description": "Crawl job ID.", + "in": "path", + "name": "jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Crawl job status." + } + }, + "summary": "Check crawl job status." + } + }, + "/v0/scrape": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "extractorOptions": { + "description": "Options for extraction", + "properties": { + "extractionPrompt": { + "description": "Prompt for LLM extraction", + "type": "string" + }, + "extractionSchema": { + "description": "Schema for LLM extraction", + "type": "object" + }, + "mode": { + "description": "Extraction mode", + "enum": [ + "markdown", + "llm-extraction", + "llm-extraction-from-raw-html", + "llm-extraction-from-markdown" + ], + "type": "string" + } + }, + "type": "object" + }, + "pageOptions": { + "properties": { + "fullPageScreenshot": { + "description": "Include full page screenshot", + "type": "boolean" + }, + "headers": { + "description": "Headers for request", + "type": "object" + }, + "includeHtml": { + "description": "Include HTML content", + "type": "boolean" + }, + "includeRawHtml": { + "description": "Include raw HTML content", + "type": "boolean" + }, + "onlyIncludeTags": { + "description": "Include only these tags", + "items": { + "type": "string" + }, + "type": "array" + }, + "onlyMainContent": { + "description": "Only return main content", + "type": "boolean" + }, + "removeTags": { + "description": "Remove these tags", + "items": { + "type": "string" + }, + "type": "array" + }, + "replaceAllPathsWithAbsolutePaths": { + "description": "Replace relative paths", + "type": "boolean" + }, + "screenshot": { + "description": "Include screenshot", + "type": "boolean" + }, + "waitFor": { + "description": "Wait time in ms", + "type": "integer" + } + }, + "type": "object" + }, + "timeout": { + "description": "Timeout in ms", + "type": "integer" + }, + "url": { + "description": "URL to scrape", + "type": "string" + } + }, + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": { + "content": { + "type": "string" + }, + "html": { + "type": "string" + }, + "llm_extraction": { + "type": "object" + }, + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "pageError": { + "type": "string" + }, + "pageStatusCode": { + "type": "integer" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "rawHtml": { + "type": "string" + }, + "warning": { + "type": "string" + } + }, + "type": "object" + }, + "success": { + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Successful scrape" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "Scrape a webpage" + } + }, + "/v0/search": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "pageOptions": { + "properties": { + "fetchPageContent": { + "description": "Fetch page content", + "type": "boolean" + } + }, + "type": "object" + }, + "query": { + "description": "Search term", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "markdown": { + "type": "string" + }, + "metadata": { + "properties": { + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "sourceURL": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "success": { + "type": "boolean" + } + }, + "type": "object" + } + } + }, + "description": "Successful search" + } + }, + "summary": "Search and extract content" + } + } + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/turning_docs_into_api_specs.ipynb b/examples/turning_docs_into_api_specs/turning_docs_into_api_specs.ipynb new file mode 100644 index 00000000..1b97f67b --- /dev/null +++ b/examples/turning_docs_into_api_specs/turning_docs_into_api_specs.ipynb @@ -0,0 +1,287 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/ericciarla/projects/python_projects/agents_testing/.conda/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import os\n", + "import datetime\n", + "import time\n", + "from firecrawl import FirecrawlApp\n", + "import json\n", + "import google.generativeai as genai\n", + "from dotenv import load_dotenv\n", + "\n", + "# Load environment variables\n", + "load_dotenv()\n", + "\n", + "# Retrieve API keys from environment variables\n", + "google_api_key = os.getenv(\"GOOGLE_API_KEY\")\n", + "firecrawl_api_key = os.getenv(\"FIRECRAWL_API_KEY\")\n", + "\n", + "# Configure the Google Generative AI module with the API key\n", + "genai.configure(api_key=google_api_key)\n", + "model = genai.GenerativeModel(\"gemini-1.5-pro-001\")\n", + "\n", + "# Set the docs URL\n", + "docs_url=\"https://docs.firecrawl.dev\"\n", + "\n", + "# Initialize the FirecrawlApp with your API key\n", + "app = FirecrawlApp(api_key=firecrawl_api_key)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "36\n" + ] + } + ], + "source": [ + "# Crawl all pages on docs\n", + "params = {\n", + " \"pageOptions\": {\n", + " \"onlyMainContent\": True\n", + " },\n", + "}\n", + "crawl_result = app.crawl_url(docs_url, params=params)\n", + "\n", + "print(len(crawl_result))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "prompt_instructions = f\"\"\"Given the following API documentation content, generate an OpenAPI 3.0 specification in JSON format ONLY if you are 100% confident and clear about all details. Focus on extracting the main endpoints, their HTTP methods, parameters, request bodies, and responses. The specification should follow OpenAPI 3.0 structure and conventions. Include only the 200 response for each endpoint. Limit all descriptions to 5 words or less.\n", + "\n", + "If there is ANY uncertainty, lack of complete information, or if you are not 100% confident about ANY part of the specification, return an empty JSON object {{}}.\n", + "\n", + "Do not make anything up. Only include information that is explicitly provided in the documentation. If any detail is unclear or missing, do not attempt to fill it in.\n", + "\n", + "API Documentation Content:\n", + "{{content}}\n", + "\n", + "Generate the OpenAPI 3.0 specification in JSON format ONLY if you are 100% confident about every single detail. Include only the JSON object, no additional text, and ensure it has no errors in the JSON format so it can be parsed. Remember to include only the 200 response for each endpoint and keep all descriptions to 5 words maximum.\n", + "\n", + "Once again, if there is ANY doubt, uncertainty, or lack of complete information, return an empty JSON object {{}}.\n", + "\n", + "To reiterate: accuracy is paramount. Do not make anything up. If you are not 100% clear or confident about the entire OpenAPI spec, return an empty JSON object {{}}.\n", + "\"\"\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "API specification saved to docs.firecrawl.dev/api_spec_0.json\n", + "API specification saved to docs.firecrawl.dev/api_spec_1.json\n", + "API specification saved to docs.firecrawl.dev/api_spec_2.json\n", + "API specification saved to docs.firecrawl.dev/api_spec_3.json\n", + "API specification saved to docs.firecrawl.dev/api_spec_4.json\n", + "An error occurred for page 5: 'content'\n", + "No API specification found for page 6\n", + "API specification saved to docs.firecrawl.dev/api_spec_7.json\n", + "No API specification found for page 8\n", + "No API specification found for page 9\n", + "API specification saved to docs.firecrawl.dev/api_spec_10.json\n", + "No API specification found for page 11\n", + "No API specification found for page 12\n", + "API specification saved to docs.firecrawl.dev/api_spec_13.json\n", + "No API specification found for page 14\n", + "No API specification found for page 15\n", + "No API specification found for page 16\n", + "No API specification found for page 17\n", + "No API specification found for page 18\n", + "No API specification found for page 19\n", + "No API specification found for page 20\n", + "No API specification found for page 21\n", + "No API specification found for page 22\n", + "No API specification found for page 23\n", + "No API specification found for page 24\n", + "No API specification found for page 25\n", + "No API specification found for page 26\n", + "No API specification found for page 27\n", + "No API specification found for page 28\n", + "No API specification found for page 29\n", + "No API specification found for page 30\n", + "No API specification found for page 31\n", + "No API specification found for page 32\n", + "No API specification found for page 33\n", + "No API specification found for page 34\n", + "No API specification found for page 35\n", + "Total API specifications collected: 8\n" + ] + } + ], + "source": [ + "# Create a folder for storing API specs\n", + "import os\n", + "import urllib.parse\n", + "\n", + "folder_name = urllib.parse.urlparse(docs_url).netloc\n", + "os.makedirs(folder_name, exist_ok=True)\n", + "\n", + "# Initialize a list to store all API specs\n", + "all_api_specs = []\n", + "\n", + "# Process each page in crawl_result\n", + "for index, result in enumerate(crawl_result):\n", + " if 'content' in result:\n", + " # Update prompt_instructions with the current page's content\n", + " current_prompt = prompt_instructions.replace(\"{content}\", result['content'])\n", + " try:\n", + " # Query the model\n", + " response = model.generate_content([current_prompt])\n", + " response_dict = response.to_dict()\n", + " response_text = response_dict['candidates'][0]['content']['parts'][0]['text']\n", + " \n", + " # Remove the ```json code wrap if present\n", + " response_text = response_text.strip().removeprefix('```json').removesuffix('```').strip()\n", + " \n", + " # Parse JSON\n", + " json_data = json.loads(response_text)\n", + " \n", + " # Save non-empty API specs\n", + " if json_data != {}:\n", + " output_file = os.path.join(folder_name, f'api_spec_{index}.json')\n", + " with open(output_file, 'w') as f:\n", + " json.dump(json_data, f, indent=2, sort_keys=True)\n", + " print(f\"API specification saved to {output_file}\")\n", + " \n", + " # Add the API spec to the list\n", + " all_api_specs.append(json_data)\n", + " else:\n", + " print(f\"No API specification found for page {index}\")\n", + " \n", + " except json.JSONDecodeError:\n", + " print(f\"Error parsing JSON response for page {index}\")\n", + " except Exception as e:\n", + " print(f\"An error occurred for page {index}: {str(e)}\")\n", + "\n", + "# Print the total number of API specs collected\n", + "print(f\"Total API specifications collected: {len(all_api_specs)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Combined API specification saved to docs.firecrawl.dev/combined_api_spec.json\n", + "Total paths in combined spec: 8\n", + "Total schemas in combined spec: 0\n" + ] + } + ], + "source": [ + "# Combine all API specs and keep the most filled out spec for each path and method\n", + "combined_spec = {\n", + " \"openapi\": \"3.0.0\",\n", + " \"info\": {\n", + " \"title\": f\"{docs_url} API Specification\",\n", + " \"version\": \"1.0.0\"\n", + " },\n", + " \"paths\": {},\n", + " \"components\": {\n", + " \"schemas\": {}\n", + " }\n", + "}\n", + "\n", + "def count_properties(obj):\n", + " if isinstance(obj, dict):\n", + " return sum(count_properties(v) for v in obj.values()) + len(obj)\n", + " elif isinstance(obj, list):\n", + " return sum(count_properties(item) for item in obj)\n", + " else:\n", + " return 1\n", + "\n", + "for spec in all_api_specs:\n", + " if \"paths\" in spec:\n", + " for path, methods in spec[\"paths\"].items():\n", + " if path not in combined_spec[\"paths\"]:\n", + " combined_spec[\"paths\"][path] = {}\n", + " for method, details in methods.items():\n", + " if method not in combined_spec[\"paths\"][path] or count_properties(details) > count_properties(combined_spec[\"paths\"][path][method]):\n", + " combined_spec[\"paths\"][path][method] = details\n", + "\n", + " if \"components\" in spec and \"schemas\" in spec[\"components\"]:\n", + " for schema_name, schema in spec[\"components\"][\"schemas\"].items():\n", + " if schema_name not in combined_spec[\"components\"][\"schemas\"] or count_properties(schema) > count_properties(combined_spec[\"components\"][\"schemas\"][schema_name]):\n", + " combined_spec[\"components\"][\"schemas\"][schema_name] = schema\n", + "\n", + "# Save the combined API spec\n", + "output_file = os.path.join(folder_name, 'combined_api_spec.json')\n", + "with open(output_file, 'w') as f:\n", + " json.dump(combined_spec, f, indent=2, sort_keys=True)\n", + "\n", + "print(f\"Combined API specification saved to {output_file}\")\n", + "print(f\"Total paths in combined spec: {len(combined_spec['paths'])}\")\n", + "print(f\"Total schemas in combined spec: {len(combined_spec['components']['schemas'])}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# note: turn this into a simple web app like roast my site\n", + "- select which methods you want to add\n", + "- generate a UI for each method\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 48056ea1bd731e3023fdb840d1867b5128fe5da3 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Mon, 2 Sep 2024 14:15:56 -0300 Subject: [PATCH 002/102] feat: added go html to md parser --- .../lib/__tests__/html-to-markdown.test.ts | 29 ++++ apps/api/src/lib/go-html-to-md/go.mod | 11 ++ apps/api/src/lib/go-html-to-md/go.sum | 81 +++++++++ .../src/lib/go-html-to-md/html-to-markdown.go | 41 +++++ apps/api/src/lib/html-to-markdown.ts | 162 +++++++++++------- 5 files changed, 260 insertions(+), 64 deletions(-) create mode 100644 apps/api/src/lib/__tests__/html-to-markdown.test.ts create mode 100644 apps/api/src/lib/go-html-to-md/go.mod create mode 100644 apps/api/src/lib/go-html-to-md/go.sum create mode 100644 apps/api/src/lib/go-html-to-md/html-to-markdown.go diff --git a/apps/api/src/lib/__tests__/html-to-markdown.test.ts b/apps/api/src/lib/__tests__/html-to-markdown.test.ts new file mode 100644 index 00000000..00db7758 --- /dev/null +++ b/apps/api/src/lib/__tests__/html-to-markdown.test.ts @@ -0,0 +1,29 @@ +import { parseMarkdown } from '../html-to-markdown'; + +describe('parseMarkdown', () => { + it('should correctly convert simple HTML to Markdown', async () => { + const html = '

Hello, world!

'; + const expectedMarkdown = 'Hello, world!'; + await expect(parseMarkdown(html)).resolves.toBe(expectedMarkdown); + }); + + it('should convert complex HTML with nested elements to Markdown', async () => { + const html = '

Hello bold world!

  • List item
'; + const expectedMarkdown = 'Hello **bold** world!\n\n- List item'; + await expect(parseMarkdown(html)).resolves.toBe(expectedMarkdown); + }); + + it('should return empty string when input is empty', async () => { + const html = ''; + const expectedMarkdown = ''; + await expect(parseMarkdown(html)).resolves.toBe(expectedMarkdown); + }); + + it('should handle null input gracefully', async () => { + const html = null; + const expectedMarkdown = ''; + await expect(parseMarkdown(html)).resolves.toBe(expectedMarkdown); + }); + + +}); diff --git a/apps/api/src/lib/go-html-to-md/go.mod b/apps/api/src/lib/go-html-to-md/go.mod new file mode 100644 index 00000000..40cce17d --- /dev/null +++ b/apps/api/src/lib/go-html-to-md/go.mod @@ -0,0 +1,11 @@ +module html-to-markdown.go + +go 1.22.6 + +require github.com/JohannesKaufmann/html-to-markdown v1.6.0 + +require ( + github.com/PuerkitoBio/goquery v1.9.2 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect + golang.org/x/net v0.25.0 // indirect +) diff --git a/apps/api/src/lib/go-html-to-md/go.sum b/apps/api/src/lib/go-html-to-md/go.sum new file mode 100644 index 00000000..59bcf2f9 --- /dev/null +++ b/apps/api/src/lib/go-html-to-md/go.sum @@ -0,0 +1,81 @@ +github.com/JohannesKaufmann/html-to-markdown v1.6.0 h1:04VXMiE50YYfCfLboJCLcgqF5x+rHJnb1ssNmqpLH/k= +github.com/JohannesKaufmann/html-to-markdown v1.6.0/go.mod h1:NUI78lGg/a7vpEJTz/0uOcYMaibytE4BUOQS8k78yPQ= +github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U= +github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/apps/api/src/lib/go-html-to-md/html-to-markdown.go b/apps/api/src/lib/go-html-to-md/html-to-markdown.go new file mode 100644 index 00000000..474c8324 --- /dev/null +++ b/apps/api/src/lib/go-html-to-md/html-to-markdown.go @@ -0,0 +1,41 @@ +package main + +import ( + "flag" + "fmt" + "log" + "sync" + + md "github.com/JohannesKaufmann/html-to-markdown" + "github.com/JohannesKaufmann/html-to-markdown/plugin" +) + +func convertHTMLToMarkdown(html string, wg *sync.WaitGroup, results chan<- string) { + defer wg.Done() + converter := md.NewConverter("", true, nil) + converter.Use(plugin.GitHubFlavored()) + + markdown, err := converter.ConvertString(html) + if err != nil { + log.Fatal(err) + } + results <- markdown +} + +func main() { + html := flag.String("html", "", "") + flag.Parse() + + var wg sync.WaitGroup + results := make(chan string, 1) + + wg.Add(1) + go convertHTMLToMarkdown(*html, &wg, results) + + wg.Wait() + close(results) + + for markdown := range results { + fmt.Println(markdown) + } +} diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts index 002cb7be..04fec4c6 100644 --- a/apps/api/src/lib/html-to-markdown.ts +++ b/apps/api/src/lib/html-to-markdown.ts @@ -1,74 +1,108 @@ -export async function parseMarkdown(html: string) { - var TurndownService = require("turndown"); - var turndownPluginGfm = require('joplin-turndown-plugin-gfm') +import { spawn } from 'node:child_process'; +import { join } from 'node:path'; +export async function parseMarkdown(html: string): Promise { + if (!html) { + return ''; + } + + if (process.env.USE_GO_MARKDOWN_PARSER == "true") { + const goScriptPath = join(__dirname, 'go-html-to-md/html-to-markdown.go'); + const goModDir = join(__dirname, 'go-html-to-md'); + const child = spawn('go', ['run', goScriptPath, '--html', html], { + cwd: goModDir, + }); + + return new Promise((resolve, reject) => { + let data = ''; + + child.stdout.on('data', (chunk) => { + data += chunk.toString(); // Convert Buffer to string + }); + + child.stderr.on('data', (chunk) => { + reject(chunk.toString()); // Convert Buffer to string before rejecting + }); + + child.on('close', (code) => { + if (code === 0) { + resolve(data.trim()); + } else { + reject(new Error(`Process exited with code ${code}`)); + } + }); + }); + } else { + var TurndownService = require("turndown"); + var turndownPluginGfm = require('joplin-turndown-plugin-gfm') + + const turndownService = new TurndownService(); + turndownService.addRule("inlineLink", { + filter: function (node, options) { + return ( + options.linkStyle === "inlined" && + node.nodeName === "A" && + node.getAttribute("href") + ); + }, + replacement: function (content, node) { + var href = node.getAttribute("href").trim(); + var title = node.title ? ' "' + node.title + '"' : ""; + return "[" + content.trim() + "](" + href + title + ")\n"; + }, + }); + var gfm = turndownPluginGfm.gfm; + turndownService.use(gfm); + let markdownContent = ""; + const turndownPromise = new Promise((resolve, reject) => { + try { + const result = turndownService.turndown(html); + resolve(result); + } catch (error) { + reject("Error converting HTML to Markdown: " + error); + } + }); + + const timeoutPromise = new Promise((resolve, reject) => { + const timeout = 5000; // Timeout in milliseconds + setTimeout(() => reject("Conversion timed out after " + timeout + "ms"), timeout); + }); - const turndownService = new TurndownService(); - turndownService.addRule("inlineLink", { - filter: function (node, options) { - return ( - options.linkStyle === "inlined" && - node.nodeName === "A" && - node.getAttribute("href") - ); - }, - replacement: function (content, node) { - var href = node.getAttribute("href").trim(); - var title = node.title ? ' "' + node.title + '"' : ""; - return "[" + content.trim() + "](" + href + title + ")\n"; - }, - }); - var gfm = turndownPluginGfm.gfm; - turndownService.use(gfm); - let markdownContent = ""; - const turndownPromise = new Promise((resolve, reject) => { try { - const result = turndownService.turndown(html); - resolve(result); + markdownContent = await Promise.race([turndownPromise, timeoutPromise]); } catch (error) { - reject("Error converting HTML to Markdown: " + error); + console.error(error); + return ""; // Optionally return an empty string or handle the error as needed } - }); - const timeoutPromise = new Promise((resolve, reject) => { - const timeout = 5000; // Timeout in milliseconds - setTimeout(() => reject("Conversion timed out after " + timeout + "ms"), timeout); - }); + // multiple line links + let insideLinkContent = false; + let newMarkdownContent = ""; + let linkOpenCount = 0; + for (let i = 0; i < markdownContent.length; i++) { + const char = markdownContent[i]; - try { - markdownContent = await Promise.race([turndownPromise, timeoutPromise]); - } catch (error) { - console.error(error); - return ""; // Optionally return an empty string or handle the error as needed + if (char == "[") { + linkOpenCount++; + } else if (char == "]") { + linkOpenCount = Math.max(0, linkOpenCount - 1); + } + insideLinkContent = linkOpenCount > 0; + + if (insideLinkContent && char == "\n") { + newMarkdownContent += "\\" + "\n"; + } else { + newMarkdownContent += char; + } + } + markdownContent = newMarkdownContent; + + // Remove [Skip to Content](#page) and [Skip to content](#skip) + markdownContent = markdownContent.replace( + /\[Skip to Content\]\(#[^\)]*\)/gi, + "" + ); + return markdownContent; } - - // multiple line links - let insideLinkContent = false; - let newMarkdownContent = ""; - let linkOpenCount = 0; - for (let i = 0; i < markdownContent.length; i++) { - const char = markdownContent[i]; - - if (char == "[") { - linkOpenCount++; - } else if (char == "]") { - linkOpenCount = Math.max(0, linkOpenCount - 1); - } - insideLinkContent = linkOpenCount > 0; - - if (insideLinkContent && char == "\n") { - newMarkdownContent += "\\" + "\n"; - } else { - newMarkdownContent += char; - } - } - markdownContent = newMarkdownContent; - - // Remove [Skip to Content](#page) and [Skip to content](#skip) - markdownContent = markdownContent.replace( - /\[Skip to Content\]\(#[^\)]*\)/gi, - "" - ); - return markdownContent; } From 995a3ff5bb9b7091ba5cb40e7656d10062dad87e Mon Sep 17 00:00:00 2001 From: Andrei Bobkov Date: Tue, 3 Sep 2024 10:49:59 +0200 Subject: [PATCH 003/102] chore(tsconfig): modernize and remove commonjs --- apps/js-sdk/firecrawl/tsconfig.json | 124 ++++++---------------------- 1 file changed, 23 insertions(+), 101 deletions(-) diff --git a/apps/js-sdk/firecrawl/tsconfig.json b/apps/js-sdk/firecrawl/tsconfig.json index 56f13ced..071b13ce 100644 --- a/apps/js-sdk/firecrawl/tsconfig.json +++ b/apps/js-sdk/firecrawl/tsconfig.json @@ -1,110 +1,32 @@ { "compilerOptions": { - /* Visit https://aka.ms/tsconfig to read more about this file */ + // See https://www.totaltypescript.com/tsconfig-cheat-sheet + /* Base Options: */ + "esModuleInterop": true, + "skipLibCheck": true, + "target": "es2022", + "allowJs": true, + "resolveJsonModule": true, + "moduleDetection": "force", + "isolatedModules": true, + "verbatimModuleSyntax": true, - /* Projects */ - // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ - // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ - // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ - // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ - // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ - // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + /* Strictness */ + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, - /* Language and Environment */ - "target": "es2020", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ - // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ - // "jsx": "preserve", /* Specify what JSX code is generated. */ - // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ - // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ - // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ - // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ - // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ - // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ - // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ - // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ - // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + /* If transpiling with TypeScript: */ + "module": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "sourceMap": true, - /* Modules */ - "module": "commonjs", /* Specify what module code is generated. */ - "rootDir": "./src", /* Specify the root folder within your source files. */ - "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ - // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ - // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ - // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ - // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ - // "types": [], /* Specify type package names to be included without being referenced in a source file. */ - // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ - // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ - // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ - // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ - // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ - // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ - // "resolveJsonModule": true, /* Enable importing .json files. */ - // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ - // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + /* AND if you're building for a library: */ + "declaration": true, - /* JavaScript Support */ - // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ - // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ - // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ - - /* Emit */ - "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ - // "declarationMap": true, /* Create sourcemaps for d.ts files. */ - // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ - // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ - // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ - // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./build", /* Specify an output folder for all emitted files. */ - // "removeComments": true, /* Disable emitting comments. */ - // "noEmit": true, /* Disable emitting files from a compilation. */ - // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ - // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ - // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ - // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ - // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ - // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ - // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ - // "newLine": "crlf", /* Set the newline character for emitting files. */ - // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ - // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ - // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ - // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ - "declarationDir": "./types", /* Specify the output directory for generated declaration files. */ - // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ - - /* Interop Constraints */ - // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ - // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ - // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ - "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ - // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ - "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ - - /* Type Checking */ - "strict": true, /* Enable all strict type-checking options. */ - // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ - // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ - // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ - // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ - // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ - // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ - // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ - // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ - // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ - // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ - // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ - // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ - // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ - // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ - // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ - // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ - // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ - // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ - - /* Completeness */ - // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ - "skipLibCheck": true /* Skip type checking all .d.ts files. */ + /* AND if you're building for a library in a monorepo: */ + "declarationMap": true /* Skip type checking all .d.ts files. */ }, "include": ["src/**/*"], "exclude": ["node_modules", "dist", "**/__tests__/*"] From 41241f4d652f52669ca27e24969969e68f536468 Mon Sep 17 00:00:00 2001 From: Andrei Bobkov Date: Tue, 3 Sep 2024 10:50:19 +0200 Subject: [PATCH 004/102] chore(.gitignore): add `apps/js-sdk/firecrawl/dist` --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 9eb551a9..45f0d802 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ apps/playwright-service-ts/package-lock.json *.pyc .rdb + +apps/js-sdk/firecrawl/dist \ No newline at end of file From fe8f9d4b2ff2cb822be1a34eabd2125d4fc5db5c Mon Sep 17 00:00:00 2001 From: Andrei Bobkov Date: Tue, 3 Sep 2024 10:50:52 +0200 Subject: [PATCH 005/102] feat(js-sdk): drop `commonjs` outputs and simplify build process --- apps/js-sdk/firecrawl/build/cjs/index.js | 347 ------------------- apps/js-sdk/firecrawl/build/cjs/package.json | 1 - apps/js-sdk/firecrawl/build/esm/index.js | 339 ------------------ apps/js-sdk/firecrawl/build/esm/package.json | 1 - apps/js-sdk/firecrawl/package.json | 16 +- apps/js-sdk/firecrawl/src/index.ts | 2 +- apps/js-sdk/firecrawl/types/index.d.ts | 260 -------------- 7 files changed, 4 insertions(+), 962 deletions(-) delete mode 100644 apps/js-sdk/firecrawl/build/cjs/index.js delete mode 100644 apps/js-sdk/firecrawl/build/cjs/package.json delete mode 100644 apps/js-sdk/firecrawl/build/esm/index.js delete mode 100644 apps/js-sdk/firecrawl/build/esm/package.json delete mode 100644 apps/js-sdk/firecrawl/types/index.d.ts diff --git a/apps/js-sdk/firecrawl/build/cjs/index.js b/apps/js-sdk/firecrawl/build/cjs/index.js deleted file mode 100644 index 2908b09d..00000000 --- a/apps/js-sdk/firecrawl/build/cjs/index.js +++ /dev/null @@ -1,347 +0,0 @@ -"use strict"; -var __importDefault = (this && this.__importDefault) || function (mod) { - return (mod && mod.__esModule) ? mod : { "default": mod }; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.CrawlWatcher = void 0; -const axios_1 = __importDefault(require("axios")); -const zod_to_json_schema_1 = require("zod-to-json-schema"); -const isows_1 = require("isows"); -const typescript_event_target_1 = require("typescript-event-target"); -/** - * Main class for interacting with the Firecrawl API. - * Provides methods for scraping, searching, crawling, and mapping web content. - */ -class FirecrawlApp { - /** - * Initializes a new instance of the FirecrawlApp class. - * @param config - Configuration options for the FirecrawlApp instance. - */ - constructor({ apiKey = null, apiUrl = null }) { - this.apiKey = apiKey || ""; - this.apiUrl = apiUrl || "https://api.firecrawl.dev"; - } - /** - * Scrapes a URL using the Firecrawl API. - * @param url - The URL to scrape. - * @param params - Additional parameters for the scrape request. - * @returns The response from the scrape operation. - */ - async scrapeUrl(url, params) { - const headers = { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - }; - let jsonData = { url, ...params }; - if (jsonData?.extract?.schema) { - let schema = jsonData.extract.schema; - // Try parsing the schema as a Zod schema - try { - schema = (0, zod_to_json_schema_1.zodToJsonSchema)(schema); - } - catch (error) { - } - jsonData = { - ...jsonData, - extract: { - ...jsonData.extract, - schema: schema, - }, - }; - } - try { - const response = await axios_1.default.post(this.apiUrl + `/v1/scrape`, jsonData, { headers }); - if (response.status === 200) { - const responseData = response.data; - if (responseData.success) { - return { - success: true, - warning: responseData.warning, - error: responseData.error, - ...responseData.data - }; - } - else { - throw new Error(`Failed to scrape URL. Error: ${responseData.error}`); - } - } - else { - this.handleError(response, "scrape URL"); - } - } - catch (error) { - throw new Error(error.message); - } - return { success: false, error: "Internal server error." }; - } - /** - * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API. - * @param query - The search query string. - * @param params - Additional parameters for the search. - * @returns Throws an error advising to use version 0 of the API. - */ - async search(query, params) { - throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0."); - } - /** - * Initiates a crawl job for a URL using the Firecrawl API. - * @param url - The URL to crawl. - * @param params - Additional parameters for the crawl request. - * @param pollInterval - Time in seconds for job status checks. - * @param idempotencyKey - Optional idempotency key for the request. - * @returns The response from the crawl operation. - */ - async crawlUrl(url, params, pollInterval = 2, idempotencyKey) { - const headers = this.prepareHeaders(idempotencyKey); - let jsonData = { url, ...params }; - try { - const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers); - if (response.status === 200) { - const id = response.data.id; - return this.monitorJobStatus(id, headers, pollInterval); - } - else { - this.handleError(response, "start crawl job"); - } - } - catch (error) { - if (error.response?.data?.error) { - throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`); - } - else { - throw new Error(error.message); - } - } - return { success: false, error: "Internal server error." }; - } - async asyncCrawlUrl(url, params, idempotencyKey) { - const headers = this.prepareHeaders(idempotencyKey); - let jsonData = { url, ...params }; - try { - const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers); - if (response.status === 200) { - return response.data; - } - else { - this.handleError(response, "start crawl job"); - } - } - catch (error) { - if (error.response?.data?.error) { - throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`); - } - else { - throw new Error(error.message); - } - } - return { success: false, error: "Internal server error." }; - } - /** - * Checks the status of a crawl job using the Firecrawl API. - * @param id - The ID of the crawl operation. - * @returns The response containing the job status. - */ - async checkCrawlStatus(id) { - if (!id) { - throw new Error("No crawl ID provided"); - } - const headers = this.prepareHeaders(); - try { - const response = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers); - if (response.status === 200) { - return ({ - success: true, - status: response.data.status, - total: response.data.total, - completed: response.data.completed, - creditsUsed: response.data.creditsUsed, - expiresAt: new Date(response.data.expiresAt), - next: response.data.next, - data: response.data.data, - error: response.data.error - }); - } - else { - this.handleError(response, "check crawl status"); - } - } - catch (error) { - throw new Error(error.message); - } - return { success: false, error: "Internal server error." }; - } - async crawlUrlAndWatch(url, params, idempotencyKey) { - const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey); - if (crawl.success && crawl.id) { - const id = crawl.id; - return new CrawlWatcher(id, this); - } - throw new Error("Crawl job failed to start"); - } - async mapUrl(url, params) { - const headers = this.prepareHeaders(); - let jsonData = { url, ...params }; - try { - const response = await this.postRequest(this.apiUrl + `/v1/map`, jsonData, headers); - if (response.status === 200) { - return response.data; - } - else { - this.handleError(response, "map"); - } - } - catch (error) { - throw new Error(error.message); - } - return { success: false, error: "Internal server error." }; - } - /** - * Prepares the headers for an API request. - * @param idempotencyKey - Optional key to ensure idempotency. - * @returns The prepared headers. - */ - prepareHeaders(idempotencyKey) { - return { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}), - }; - } - /** - * Sends a POST request to the specified URL. - * @param url - The URL to send the request to. - * @param data - The data to send in the request. - * @param headers - The headers for the request. - * @returns The response from the POST request. - */ - postRequest(url, data, headers) { - return axios_1.default.post(url, data, { headers }); - } - /** - * Sends a GET request to the specified URL. - * @param url - The URL to send the request to. - * @param headers - The headers for the request. - * @returns The response from the GET request. - */ - getRequest(url, headers) { - return axios_1.default.get(url, { headers }); - } - /** - * Monitors the status of a crawl job until completion or failure. - * @param id - The ID of the crawl operation. - * @param headers - The headers for the request. - * @param checkInterval - Interval in seconds for job status checks. - * @param checkUrl - Optional URL to check the status (used for v1 API) - * @returns The final job status or data. - */ - async monitorJobStatus(id, headers, checkInterval) { - while (true) { - const statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers); - if (statusResponse.status === 200) { - const statusData = statusResponse.data; - if (statusData.status === "completed") { - if ("data" in statusData) { - return statusData; - } - else { - throw new Error("Crawl job completed but no data was returned"); - } - } - else if (["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)) { - checkInterval = Math.max(checkInterval, 2); - await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); - } - else { - throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`); - } - } - else { - this.handleError(statusResponse, "check crawl status"); - } - } - } - /** - * Handles errors from API responses. - * @param {AxiosResponse} response - The response from the API. - * @param {string} action - The action being performed when the error occurred. - */ - handleError(response, action) { - if ([402, 408, 409, 500].includes(response.status)) { - const errorMessage = response.data.error || "Unknown error occurred"; - throw new Error(`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`); - } - else { - throw new Error(`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`); - } - } -} -exports.default = FirecrawlApp; -class CrawlWatcher extends typescript_event_target_1.TypedEventTarget { - constructor(id, app) { - super(); - this.ws = new isows_1.WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); - this.status = "scraping"; - this.data = []; - const messageHandler = (msg) => { - if (msg.type === "done") { - this.status = "completed"; - this.dispatchTypedEvent("done", new CustomEvent("done", { - detail: { - status: this.status, - data: this.data, - }, - })); - } - else if (msg.type === "error") { - this.status = "failed"; - this.dispatchTypedEvent("error", new CustomEvent("error", { - detail: { - status: this.status, - data: this.data, - error: msg.error, - }, - })); - } - else if (msg.type === "catchup") { - this.status = msg.data.status; - this.data.push(...(msg.data.data ?? [])); - for (const doc of this.data) { - this.dispatchTypedEvent("document", new CustomEvent("document", { - detail: doc, - })); - } - } - else if (msg.type === "document") { - this.dispatchTypedEvent("document", new CustomEvent("document", { - detail: msg.data, - })); - } - }; - this.ws.onmessage = ((ev) => { - if (typeof ev.data !== "string") { - this.ws.close(); - return; - } - const msg = JSON.parse(ev.data); - messageHandler(msg); - }).bind(this); - this.ws.onclose = ((ev) => { - const msg = JSON.parse(ev.reason); - messageHandler(msg); - }).bind(this); - this.ws.onerror = ((_) => { - this.status = "failed"; - this.dispatchTypedEvent("error", new CustomEvent("error", { - detail: { - status: this.status, - data: this.data, - error: "WebSocket error", - }, - })); - }).bind(this); - } - close() { - this.ws.close(); - } -} -exports.CrawlWatcher = CrawlWatcher; diff --git a/apps/js-sdk/firecrawl/build/cjs/package.json b/apps/js-sdk/firecrawl/build/cjs/package.json deleted file mode 100644 index b731bd61..00000000 --- a/apps/js-sdk/firecrawl/build/cjs/package.json +++ /dev/null @@ -1 +0,0 @@ -{"type": "commonjs"} diff --git a/apps/js-sdk/firecrawl/build/esm/index.js b/apps/js-sdk/firecrawl/build/esm/index.js deleted file mode 100644 index 4245cc37..00000000 --- a/apps/js-sdk/firecrawl/build/esm/index.js +++ /dev/null @@ -1,339 +0,0 @@ -import axios from "axios"; -import { zodToJsonSchema } from "zod-to-json-schema"; -import { WebSocket } from "isows"; -import { TypedEventTarget } from "typescript-event-target"; -/** - * Main class for interacting with the Firecrawl API. - * Provides methods for scraping, searching, crawling, and mapping web content. - */ -export default class FirecrawlApp { - /** - * Initializes a new instance of the FirecrawlApp class. - * @param config - Configuration options for the FirecrawlApp instance. - */ - constructor({ apiKey = null, apiUrl = null }) { - this.apiKey = apiKey || ""; - this.apiUrl = apiUrl || "https://api.firecrawl.dev"; - } - /** - * Scrapes a URL using the Firecrawl API. - * @param url - The URL to scrape. - * @param params - Additional parameters for the scrape request. - * @returns The response from the scrape operation. - */ - async scrapeUrl(url, params) { - const headers = { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - }; - let jsonData = { url, ...params }; - if (jsonData?.extract?.schema) { - let schema = jsonData.extract.schema; - // Try parsing the schema as a Zod schema - try { - schema = zodToJsonSchema(schema); - } - catch (error) { - } - jsonData = { - ...jsonData, - extract: { - ...jsonData.extract, - schema: schema, - }, - }; - } - try { - const response = await axios.post(this.apiUrl + `/v1/scrape`, jsonData, { headers }); - if (response.status === 200) { - const responseData = response.data; - if (responseData.success) { - return { - success: true, - warning: responseData.warning, - error: responseData.error, - ...responseData.data - }; - } - else { - throw new Error(`Failed to scrape URL. Error: ${responseData.error}`); - } - } - else { - this.handleError(response, "scrape URL"); - } - } - catch (error) { - throw new Error(error.message); - } - return { success: false, error: "Internal server error." }; - } - /** - * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API. - * @param query - The search query string. - * @param params - Additional parameters for the search. - * @returns Throws an error advising to use version 0 of the API. - */ - async search(query, params) { - throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0."); - } - /** - * Initiates a crawl job for a URL using the Firecrawl API. - * @param url - The URL to crawl. - * @param params - Additional parameters for the crawl request. - * @param pollInterval - Time in seconds for job status checks. - * @param idempotencyKey - Optional idempotency key for the request. - * @returns The response from the crawl operation. - */ - async crawlUrl(url, params, pollInterval = 2, idempotencyKey) { - const headers = this.prepareHeaders(idempotencyKey); - let jsonData = { url, ...params }; - try { - const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers); - if (response.status === 200) { - const id = response.data.id; - return this.monitorJobStatus(id, headers, pollInterval); - } - else { - this.handleError(response, "start crawl job"); - } - } - catch (error) { - if (error.response?.data?.error) { - throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`); - } - else { - throw new Error(error.message); - } - } - return { success: false, error: "Internal server error." }; - } - async asyncCrawlUrl(url, params, idempotencyKey) { - const headers = this.prepareHeaders(idempotencyKey); - let jsonData = { url, ...params }; - try { - const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers); - if (response.status === 200) { - return response.data; - } - else { - this.handleError(response, "start crawl job"); - } - } - catch (error) { - if (error.response?.data?.error) { - throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`); - } - else { - throw new Error(error.message); - } - } - return { success: false, error: "Internal server error." }; - } - /** - * Checks the status of a crawl job using the Firecrawl API. - * @param id - The ID of the crawl operation. - * @returns The response containing the job status. - */ - async checkCrawlStatus(id) { - if (!id) { - throw new Error("No crawl ID provided"); - } - const headers = this.prepareHeaders(); - try { - const response = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers); - if (response.status === 200) { - return ({ - success: true, - status: response.data.status, - total: response.data.total, - completed: response.data.completed, - creditsUsed: response.data.creditsUsed, - expiresAt: new Date(response.data.expiresAt), - next: response.data.next, - data: response.data.data, - error: response.data.error - }); - } - else { - this.handleError(response, "check crawl status"); - } - } - catch (error) { - throw new Error(error.message); - } - return { success: false, error: "Internal server error." }; - } - async crawlUrlAndWatch(url, params, idempotencyKey) { - const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey); - if (crawl.success && crawl.id) { - const id = crawl.id; - return new CrawlWatcher(id, this); - } - throw new Error("Crawl job failed to start"); - } - async mapUrl(url, params) { - const headers = this.prepareHeaders(); - let jsonData = { url, ...params }; - try { - const response = await this.postRequest(this.apiUrl + `/v1/map`, jsonData, headers); - if (response.status === 200) { - return response.data; - } - else { - this.handleError(response, "map"); - } - } - catch (error) { - throw new Error(error.message); - } - return { success: false, error: "Internal server error." }; - } - /** - * Prepares the headers for an API request. - * @param idempotencyKey - Optional key to ensure idempotency. - * @returns The prepared headers. - */ - prepareHeaders(idempotencyKey) { - return { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}), - }; - } - /** - * Sends a POST request to the specified URL. - * @param url - The URL to send the request to. - * @param data - The data to send in the request. - * @param headers - The headers for the request. - * @returns The response from the POST request. - */ - postRequest(url, data, headers) { - return axios.post(url, data, { headers }); - } - /** - * Sends a GET request to the specified URL. - * @param url - The URL to send the request to. - * @param headers - The headers for the request. - * @returns The response from the GET request. - */ - getRequest(url, headers) { - return axios.get(url, { headers }); - } - /** - * Monitors the status of a crawl job until completion or failure. - * @param id - The ID of the crawl operation. - * @param headers - The headers for the request. - * @param checkInterval - Interval in seconds for job status checks. - * @param checkUrl - Optional URL to check the status (used for v1 API) - * @returns The final job status or data. - */ - async monitorJobStatus(id, headers, checkInterval) { - while (true) { - const statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers); - if (statusResponse.status === 200) { - const statusData = statusResponse.data; - if (statusData.status === "completed") { - if ("data" in statusData) { - return statusData; - } - else { - throw new Error("Crawl job completed but no data was returned"); - } - } - else if (["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)) { - checkInterval = Math.max(checkInterval, 2); - await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); - } - else { - throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`); - } - } - else { - this.handleError(statusResponse, "check crawl status"); - } - } - } - /** - * Handles errors from API responses. - * @param {AxiosResponse} response - The response from the API. - * @param {string} action - The action being performed when the error occurred. - */ - handleError(response, action) { - if ([402, 408, 409, 500].includes(response.status)) { - const errorMessage = response.data.error || "Unknown error occurred"; - throw new Error(`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`); - } - else { - throw new Error(`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`); - } - } -} -export class CrawlWatcher extends TypedEventTarget { - constructor(id, app) { - super(); - this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); - this.status = "scraping"; - this.data = []; - const messageHandler = (msg) => { - if (msg.type === "done") { - this.status = "completed"; - this.dispatchTypedEvent("done", new CustomEvent("done", { - detail: { - status: this.status, - data: this.data, - }, - })); - } - else if (msg.type === "error") { - this.status = "failed"; - this.dispatchTypedEvent("error", new CustomEvent("error", { - detail: { - status: this.status, - data: this.data, - error: msg.error, - }, - })); - } - else if (msg.type === "catchup") { - this.status = msg.data.status; - this.data.push(...(msg.data.data ?? [])); - for (const doc of this.data) { - this.dispatchTypedEvent("document", new CustomEvent("document", { - detail: doc, - })); - } - } - else if (msg.type === "document") { - this.dispatchTypedEvent("document", new CustomEvent("document", { - detail: msg.data, - })); - } - }; - this.ws.onmessage = ((ev) => { - if (typeof ev.data !== "string") { - this.ws.close(); - return; - } - const msg = JSON.parse(ev.data); - messageHandler(msg); - }).bind(this); - this.ws.onclose = ((ev) => { - const msg = JSON.parse(ev.reason); - messageHandler(msg); - }).bind(this); - this.ws.onerror = ((_) => { - this.status = "failed"; - this.dispatchTypedEvent("error", new CustomEvent("error", { - detail: { - status: this.status, - data: this.data, - error: "WebSocket error", - }, - })); - }).bind(this); - } - close() { - this.ws.close(); - } -} diff --git a/apps/js-sdk/firecrawl/build/esm/package.json b/apps/js-sdk/firecrawl/build/esm/package.json deleted file mode 100644 index 6990891f..00000000 --- a/apps/js-sdk/firecrawl/build/esm/package.json +++ /dev/null @@ -1 +0,0 @@ -{"type": "module"} diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index e68b3014..430cffff 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -2,21 +2,11 @@ "name": "@mendable/firecrawl-js", "version": "1.2.1", "description": "JavaScript SDK for Firecrawl API", - "main": "build/cjs/index.js", - "types": "types/index.d.ts", + "main": "dist/index.js", + "types": "dist/index.d.ts", "type": "module", - "exports": { - "require": { - "types": "./types/index.d.ts", - "default": "./build/cjs/index.js" - }, - "import": { - "types": "./types/index.d.ts", - "default": "./build/esm/index.js" - } - }, "scripts": { - "build": "tsc --module commonjs --moduleResolution node10 --outDir build/cjs/ && echo '{\"type\": \"commonjs\"}' > build/cjs/package.json && npx tsc --module NodeNext --moduleResolution NodeNext --outDir build/esm/ && echo '{\"type\": \"module\"}' > build/esm/package.json", + "build": "tsc", "build-and-publish": "npm run build && npm publish --access public", "publish-beta": "npm run build && npm publish --access public --tag beta", "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts" diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 1d1bb4ee..e9411527 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -1,4 +1,4 @@ -import axios, { AxiosResponse, AxiosRequestHeaders } from "axios"; +import axios, { type AxiosResponse, type AxiosRequestHeaders } from "axios"; import { z } from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import { WebSocket } from "isows"; diff --git a/apps/js-sdk/firecrawl/types/index.d.ts b/apps/js-sdk/firecrawl/types/index.d.ts deleted file mode 100644 index 36356c4e..00000000 --- a/apps/js-sdk/firecrawl/types/index.d.ts +++ /dev/null @@ -1,260 +0,0 @@ -import { AxiosResponse, AxiosRequestHeaders } from "axios"; -import { z } from "zod"; -import { TypedEventTarget } from "typescript-event-target"; -/** - * Configuration interface for FirecrawlApp. - * @param apiKey - Optional API key for authentication. - * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'. - */ -export interface FirecrawlAppConfig { - apiKey?: string | null; - apiUrl?: string | null; -} -/** - * Metadata for a Firecrawl document. - * Includes various optional properties for document metadata. - */ -export interface FirecrawlDocumentMetadata { - title?: string; - description?: string; - language?: string; - keywords?: string; - robots?: string; - ogTitle?: string; - ogDescription?: string; - ogUrl?: string; - ogImage?: string; - ogAudio?: string; - ogDeterminer?: string; - ogLocale?: string; - ogLocaleAlternate?: string[]; - ogSiteName?: string; - ogVideo?: string; - dctermsCreated?: string; - dcDateCreated?: string; - dcDate?: string; - dctermsType?: string; - dcType?: string; - dctermsAudience?: string; - dctermsSubject?: string; - dcSubject?: string; - dcDescription?: string; - dctermsKeywords?: string; - modifiedTime?: string; - publishedTime?: string; - articleTag?: string; - articleSection?: string; - sourceURL?: string; - statusCode?: number; - error?: string; - [key: string]: any; -} -/** - * Document interface for Firecrawl. - * Represents a document retrieved or processed by Firecrawl. - */ -export interface FirecrawlDocument { - url?: string; - markdown?: string; - html?: string; - rawHtml?: string; - links?: string[]; - extract?: Record; - screenshot?: string; - metadata?: FirecrawlDocumentMetadata; -} -/** - * Parameters for scraping operations. - * Defines the options and configurations available for scraping web content. - */ -export interface ScrapeParams { - formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract" | "full@scrennshot")[]; - headers?: Record; - includeTags?: string[]; - excludeTags?: string[]; - onlyMainContent?: boolean; - extract?: { - prompt?: string; - schema?: z.ZodSchema | any; - systemPrompt?: string; - }; - waitFor?: number; - timeout?: number; -} -/** - * Response interface for scraping operations. - * Defines the structure of the response received after a scraping operation. - */ -export interface ScrapeResponse extends FirecrawlDocument { - success: true; - warning?: string; - error?: string; -} -/** - * Parameters for crawling operations. - * Includes options for both scraping and mapping during a crawl. - */ -export interface CrawlParams { - includePaths?: string[]; - excludePaths?: string[]; - maxDepth?: number; - limit?: number; - allowBackwardLinks?: boolean; - allowExternalLinks?: boolean; - ignoreSitemap?: boolean; - scrapeOptions?: ScrapeParams; - webhook?: string; -} -/** - * Response interface for crawling operations. - * Defines the structure of the response received after initiating a crawl. - */ -export interface CrawlResponse { - id?: string; - url?: string; - success: true; - error?: string; -} -/** - * Response interface for job status checks. - * Provides detailed status of a crawl job including progress and results. - */ -export interface CrawlStatusResponse { - success: true; - total: number; - completed: number; - creditsUsed: number; - expiresAt: Date; - status: "scraping" | "completed" | "failed"; - next: string; - data?: FirecrawlDocument[]; - error?: string; -} -/** - * Parameters for mapping operations. - * Defines options for mapping URLs during a crawl. - */ -export interface MapParams { - search?: string; - ignoreSitemap?: boolean; - includeSubdomains?: boolean; - limit?: number; -} -/** - * Response interface for mapping operations. - * Defines the structure of the response received after a mapping operation. - */ -export interface MapResponse { - success: true; - links?: string[]; - error?: string; -} -/** - * Error response interface. - * Defines the structure of the response received when an error occurs. - */ -export interface ErrorResponse { - success: false; - error: string; -} -/** - * Main class for interacting with the Firecrawl API. - * Provides methods for scraping, searching, crawling, and mapping web content. - */ -export default class FirecrawlApp { - apiKey: string; - apiUrl: string; - /** - * Initializes a new instance of the FirecrawlApp class. - * @param config - Configuration options for the FirecrawlApp instance. - */ - constructor({ apiKey, apiUrl }: FirecrawlAppConfig); - /** - * Scrapes a URL using the Firecrawl API. - * @param url - The URL to scrape. - * @param params - Additional parameters for the scrape request. - * @returns The response from the scrape operation. - */ - scrapeUrl(url: string, params?: ScrapeParams): Promise; - /** - * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API. - * @param query - The search query string. - * @param params - Additional parameters for the search. - * @returns Throws an error advising to use version 0 of the API. - */ - search(query: string, params?: any): Promise; - /** - * Initiates a crawl job for a URL using the Firecrawl API. - * @param url - The URL to crawl. - * @param params - Additional parameters for the crawl request. - * @param pollInterval - Time in seconds for job status checks. - * @param idempotencyKey - Optional idempotency key for the request. - * @returns The response from the crawl operation. - */ - crawlUrl(url: string, params?: CrawlParams, pollInterval?: number, idempotencyKey?: string): Promise; - asyncCrawlUrl(url: string, params?: CrawlParams, idempotencyKey?: string): Promise; - /** - * Checks the status of a crawl job using the Firecrawl API. - * @param id - The ID of the crawl operation. - * @returns The response containing the job status. - */ - checkCrawlStatus(id?: string): Promise; - crawlUrlAndWatch(url: string, params?: CrawlParams, idempotencyKey?: string): Promise; - mapUrl(url: string, params?: MapParams): Promise; - /** - * Prepares the headers for an API request. - * @param idempotencyKey - Optional key to ensure idempotency. - * @returns The prepared headers. - */ - prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders; - /** - * Sends a POST request to the specified URL. - * @param url - The URL to send the request to. - * @param data - The data to send in the request. - * @param headers - The headers for the request. - * @returns The response from the POST request. - */ - postRequest(url: string, data: any, headers: AxiosRequestHeaders): Promise; - /** - * Sends a GET request to the specified URL. - * @param url - The URL to send the request to. - * @param headers - The headers for the request. - * @returns The response from the GET request. - */ - getRequest(url: string, headers: AxiosRequestHeaders): Promise; - /** - * Monitors the status of a crawl job until completion or failure. - * @param id - The ID of the crawl operation. - * @param headers - The headers for the request. - * @param checkInterval - Interval in seconds for job status checks. - * @param checkUrl - Optional URL to check the status (used for v1 API) - * @returns The final job status or data. - */ - monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise; - /** - * Handles errors from API responses. - * @param {AxiosResponse} response - The response from the API. - * @param {string} action - The action being performed when the error occurred. - */ - handleError(response: AxiosResponse, action: string): void; -} -interface CrawlWatcherEvents { - document: CustomEvent; - done: CustomEvent<{ - status: CrawlStatusResponse["status"]; - data: FirecrawlDocument[]; - }>; - error: CustomEvent<{ - status: CrawlStatusResponse["status"]; - data: FirecrawlDocument[]; - error: string; - }>; -} -export declare class CrawlWatcher extends TypedEventTarget { - private ws; - data: FirecrawlDocument[]; - status: CrawlStatusResponse["status"]; - constructor(id: string, app: FirecrawlApp); - close(): void; -} -export {}; From 2a8f55e533175d75381c699c68526763dfe5892a Mon Sep 17 00:00:00 2001 From: Andrei Bobkov Date: Tue, 3 Sep 2024 11:12:28 +0200 Subject: [PATCH 006/102] perf(js-sdk): remove whole `z` import and instead use type-only import --- apps/js-sdk/firecrawl/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 1d1bb4ee..95b4eebd 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -1,5 +1,5 @@ import axios, { AxiosResponse, AxiosRequestHeaders } from "axios"; -import { z } from "zod"; +import type { ZodSchema } from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import { WebSocket } from "isows"; import { TypedEventTarget } from "typescript-event-target"; @@ -81,7 +81,7 @@ export interface ScrapeParams { onlyMainContent?: boolean; extract?: { prompt?: string; - schema?: z.ZodSchema | any; + schema?: ZodSchema | any; systemPrompt?: string; }; waitFor?: number; From 2b0e447bc26ec94f930af68de4d0ad4e6d6fb08f Mon Sep 17 00:00:00 2001 From: Andrei Bobkov Date: Tue, 3 Sep 2024 11:13:48 +0200 Subject: [PATCH 007/102] perf(js-sdk): move `dotenv` and `uuid` to `devDependencies` --- apps/js-sdk/firecrawl/package-lock.json | 12 +++++++----- apps/js-sdk/firecrawl/package.json | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json index ce6a1a4a..7c2ecbfd 100644 --- a/apps/js-sdk/firecrawl/package-lock.json +++ b/apps/js-sdk/firecrawl/package-lock.json @@ -1,19 +1,17 @@ { "name": "@mendable/firecrawl-js", - "version": "1.1.0", + "version": "1.2.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mendable/firecrawl-js", - "version": "1.1.0", + "version": "1.2.1", "license": "MIT", "dependencies": { "axios": "^1.6.8", - "dotenv": "^16.4.5", "isows": "^1.0.4", "typescript-event-target": "^1.1.1", - "uuid": "^9.0.1", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.0" }, @@ -25,9 +23,11 @@ "@types/mocha": "^10.0.6", "@types/node": "^20.12.12", "@types/uuid": "^9.0.8", + "dotenv": "^16.4.5", "jest": "^29.7.0", "ts-jest": "^29.2.2", - "typescript": "^5.4.5" + "typescript": "^5.4.5", + "uuid": "^9.0.1" } }, "node_modules/@ampproject/remapping": { @@ -1657,6 +1657,7 @@ "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "dev": true, "engines": { "node": ">=12" }, @@ -3794,6 +3795,7 @@ "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "dev": true, "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index e68b3014..62120b35 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -29,10 +29,8 @@ "license": "MIT", "dependencies": { "axios": "^1.6.8", - "dotenv": "^16.4.5", "isows": "^1.0.4", "typescript-event-target": "^1.1.1", - "uuid": "^9.0.1", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.0" }, @@ -41,6 +39,8 @@ }, "homepage": "https://github.com/mendableai/firecrawl#readme", "devDependencies": { + "uuid": "^9.0.1", + "dotenv": "^16.4.5", "@jest/globals": "^29.7.0", "@types/axios": "^0.14.0", "@types/dotenv": "^8.2.0", From 291d9e375b27e442e5928e6e5a62e1a96e35d674 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:56:07 -0300 Subject: [PATCH 008/102] now using compiled go/C lib with koffi --- apps/api/Dockerfile | 12 +- apps/api/package.json | 1 + apps/api/pnpm-lock.yaml | 8 + apps/api/src/lib/go-html-to-md/README.md | 7 + apps/api/src/lib/go-html-to-md/go.mod | 5 +- apps/api/src/lib/go-html-to-md/go.sum | 12 ++ .../src/lib/go-html-to-md/html-to-markdown.go | 28 +-- apps/api/src/lib/html-to-markdown.ts | 178 +++++++++--------- apps/test-suite/tests/scrape.test.ts | 11 +- 9 files changed, 144 insertions(+), 118 deletions(-) create mode 100644 apps/api/src/lib/go-html-to-md/README.md diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 3ffede0d..a4a2c76b 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -17,8 +17,15 @@ RUN pnpm install RUN --mount=type=secret,id=SENTRY_AUTH_TOKEN \ bash -c 'export SENTRY_AUTH_TOKEN="$(cat /run/secrets/SENTRY_AUTH_TOKEN)"; if [ -z $SENTRY_AUTH_TOKEN ]; then pnpm run build:nosentry; else pnpm run build; fi' -# Install packages needed for deployment +# Install Go 1.19 +FROM golang:1.19 AS go-base +COPY src/lib/go-html-to-md /app/src/lib/go-html-to-md +# Install Go dependencies and build +RUN cd /app/src/lib/go-html-to-md && \ + go mod tidy && \ + go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go && \ + chmod +x html-to-markdown.so FROM base RUN apt-get update -qq && \ @@ -26,8 +33,7 @@ RUN apt-get update -qq && \ rm -rf /var/lib/apt/lists /var/cache/apt/archives COPY --from=prod-deps /app/node_modules /app/node_modules COPY --from=build /app /app - - +COPY --from=go-base /app/src/lib/go-html-to-md/html-to-markdown.so /app/src/lib/go-html-to-md/html-to-markdown.so # Start the server by default, this can be overwritten at runtime diff --git a/apps/api/package.json b/apps/api/package.json index bac13e79..dc26b34b 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -86,6 +86,7 @@ "joplin-turndown-plugin-gfm": "^1.0.12", "json-schema-to-zod": "^2.3.0", "keyword-extractor": "^0.0.28", + "koffi": "^2.9.0", "langchain": "^0.2.8", "languagedetect": "^2.0.0", "logsnag": "^1.0.0", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 2762a84c..b8f876a8 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -122,6 +122,9 @@ importers: keyword-extractor: specifier: ^0.0.28 version: 0.0.28 + koffi: + specifier: ^2.9.0 + version: 2.9.0 langchain: specifier: ^0.2.8 version: 0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0) @@ -3170,6 +3173,9 @@ packages: resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} engines: {node: '>=6'} + koffi@2.9.0: + resolution: {integrity: sha512-KCsuJ2gM58n6bNdR2Z7gqsh/3TchxxQFbVgax2/UvAjRTgwNSYAJDx9E3jrkBP4jEDHWRCfE47Y2OG+/fiSvEw==} + langchain@0.2.8: resolution: {integrity: sha512-kb2IOMA71xH8e6EXFg0l4S+QSMC/c796pj1+7mPBkR91HHwoyHZhFRrBaZv4tV+Td+Ba91J2uEDBmySklZLpNQ==} engines: {node: '>=18'} @@ -8492,6 +8498,8 @@ snapshots: kleur@3.0.3: {} + koffi@2.9.0: {} + langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0): dependencies: '@langchain/core': 0.2.12(langchain@0.2.8(@supabase/supabase-js@2.44.2)(axios@1.7.2)(cheerio@1.0.0-rc.12)(handlebars@4.7.8)(html-to-text@9.0.5)(ioredis@5.4.1)(mammoth@1.7.2)(mongodb@6.6.2(socks@2.8.3))(openai@4.57.0(zod@3.23.8))(pdf-parse@1.1.1)(puppeteer@22.12.1(typescript@5.4.5))(redis@4.6.14)(ws@8.18.0))(openai@4.57.0(zod@3.23.8)) diff --git a/apps/api/src/lib/go-html-to-md/README.md b/apps/api/src/lib/go-html-to-md/README.md new file mode 100644 index 00000000..4ad510c3 --- /dev/null +++ b/apps/api/src/lib/go-html-to-md/README.md @@ -0,0 +1,7 @@ +To build the go-html-to-md library, run the following command: + +```bash +cd apps/api/src/lib/go-html-to-md +go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go +chmod +x html-to-markdown.so +``` \ No newline at end of file diff --git a/apps/api/src/lib/go-html-to-md/go.mod b/apps/api/src/lib/go-html-to-md/go.mod index 40cce17d..0836f441 100644 --- a/apps/api/src/lib/go-html-to-md/go.mod +++ b/apps/api/src/lib/go-html-to-md/go.mod @@ -1,11 +1,14 @@ module html-to-markdown.go -go 1.22.6 +go 1.19 require github.com/JohannesKaufmann/html-to-markdown v1.6.0 require ( github.com/PuerkitoBio/goquery v1.9.2 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/kr/pretty v0.3.0 // indirect golang.org/x/net v0.25.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/apps/api/src/lib/go-html-to-md/go.sum b/apps/api/src/lib/go-html-to-md/go.sum index 59bcf2f9..7961629d 100644 --- a/apps/api/src/lib/go-html-to-md/go.sum +++ b/apps/api/src/lib/go-html-to-md/go.sum @@ -4,14 +4,22 @@ github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4 github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -75,7 +83,11 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/apps/api/src/lib/go-html-to-md/html-to-markdown.go b/apps/api/src/lib/go-html-to-md/html-to-markdown.go index 474c8324..9905a69a 100644 --- a/apps/api/src/lib/go-html-to-md/html-to-markdown.go +++ b/apps/api/src/lib/go-html-to-md/html-to-markdown.go @@ -1,41 +1,25 @@ package main import ( - "flag" - "fmt" + "C" "log" - "sync" md "github.com/JohannesKaufmann/html-to-markdown" "github.com/JohannesKaufmann/html-to-markdown/plugin" ) -func convertHTMLToMarkdown(html string, wg *sync.WaitGroup, results chan<- string) { - defer wg.Done() +//export ConvertHTMLToMarkdown +func ConvertHTMLToMarkdown(html *C.char) *C.char { converter := md.NewConverter("", true, nil) converter.Use(plugin.GitHubFlavored()) - markdown, err := converter.ConvertString(html) + markdown, err := converter.ConvertString(C.GoString(html)) if err != nil { log.Fatal(err) } - results <- markdown + return C.CString(markdown) } func main() { - html := flag.String("html", "", "") - flag.Parse() - - var wg sync.WaitGroup - results := make(chan string, 1) - - wg.Add(1) - go convertHTMLToMarkdown(*html, &wg, results) - - wg.Wait() - close(results) - - for markdown := range results { - fmt.Println(markdown) - } + // This function is required for the main package } diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts index 04fec4c6..4c7cffdd 100644 --- a/apps/api/src/lib/html-to-markdown.ts +++ b/apps/api/src/lib/html-to-markdown.ts @@ -1,108 +1,106 @@ -import { spawn } from 'node:child_process'; -import { join } from 'node:path'; +import koffi from 'koffi'; +import { join } from 'path'; +import "../services/sentry" +import * as Sentry from "@sentry/node"; + +import dotenv from 'dotenv'; +import { Logger } from './logger'; +dotenv.config(); export async function parseMarkdown(html: string): Promise { if (!html) { return ''; } - if (process.env.USE_GO_MARKDOWN_PARSER == "true") { - const goScriptPath = join(__dirname, 'go-html-to-md/html-to-markdown.go'); - const goModDir = join(__dirname, 'go-html-to-md'); - const child = spawn('go', ['run', goScriptPath, '--html', html], { - cwd: goModDir, - }); + try { + if (process.env.USE_GO_MARKDOWN_PARSER == "true") { + const goExecutablePath = join(__dirname, 'go-html-to-md/html-to-markdown.so'); + const lib = koffi.load(goExecutablePath); + + const convert = lib.func('Convert', 'string', ['string']); - return new Promise((resolve, reject) => { - let data = ''; - - child.stdout.on('data', (chunk) => { - data += chunk.toString(); // Convert Buffer to string + let markdownContent = await new Promise((resolve, reject) => { + convert.async(html, (err: Error, res: string) => { + if (err) { + reject(err); + } else { + resolve(res); + } + }); }); - child.stderr.on('data', (chunk) => { - reject(chunk.toString()); // Convert Buffer to string before rejecting - }); - - child.on('close', (code) => { - if (code === 0) { - resolve(data.trim()); - } else { - reject(new Error(`Process exited with code ${code}`)); - } - }); - }); - } else { - var TurndownService = require("turndown"); - var turndownPluginGfm = require('joplin-turndown-plugin-gfm') - - const turndownService = new TurndownService(); - turndownService.addRule("inlineLink", { - filter: function (node, options) { - return ( - options.linkStyle === "inlined" && - node.nodeName === "A" && - node.getAttribute("href") - ); - }, - replacement: function (content, node) { - var href = node.getAttribute("href").trim(); - var title = node.title ? ' "' + node.title + '"' : ""; - return "[" + content.trim() + "](" + href + title + ")\n"; - }, - }); - var gfm = turndownPluginGfm.gfm; - turndownService.use(gfm); - let markdownContent = ""; - const turndownPromise = new Promise((resolve, reject) => { - try { - const result = turndownService.turndown(html); - resolve(result); - } catch (error) { - reject("Error converting HTML to Markdown: " + error); - } - }); - - const timeoutPromise = new Promise((resolve, reject) => { - const timeout = 5000; // Timeout in milliseconds - setTimeout(() => reject("Conversion timed out after " + timeout + "ms"), timeout); - }); - - try { - markdownContent = await Promise.race([turndownPromise, timeoutPromise]); - } catch (error) { - console.error(error); - return ""; // Optionally return an empty string or handle the error as needed + markdownContent = processMultiLineLinks(markdownContent); + markdownContent = removeSkipToContentLinks(markdownContent); + return markdownContent; } + } catch (error) { + Sentry.captureException(error); + Logger.error(`Error converting HTML to Markdown with Go parser: ${error}`); + } - // multiple line links - let insideLinkContent = false; - let newMarkdownContent = ""; - let linkOpenCount = 0; - for (let i = 0; i < markdownContent.length; i++) { - const char = markdownContent[i]; + // Fallback to TurndownService if Go parser fails or is not enabled + var TurndownService = require("turndown"); + var turndownPluginGfm = require('joplin-turndown-plugin-gfm'); - if (char == "[") { - linkOpenCount++; - } else if (char == "]") { - linkOpenCount = Math.max(0, linkOpenCount - 1); - } - insideLinkContent = linkOpenCount > 0; + const turndownService = new TurndownService(); + turndownService.addRule("inlineLink", { + filter: function (node, options) { + return ( + options.linkStyle === "inlined" && + node.nodeName === "A" && + node.getAttribute("href") + ); + }, + replacement: function (content, node) { + var href = node.getAttribute("href").trim(); + var title = node.title ? ' "' + node.title + '"' : ""; + return "[" + content.trim() + "](" + href + title + ")\n"; + }, + }); + var gfm = turndownPluginGfm.gfm; + turndownService.use(gfm); - if (insideLinkContent && char == "\n") { - newMarkdownContent += "\\" + "\n"; - } else { - newMarkdownContent += char; - } - } - markdownContent = newMarkdownContent; + try { + let markdownContent = await turndownService.turndown(html); + markdownContent = processMultiLineLinks(markdownContent); + markdownContent = removeSkipToContentLinks(markdownContent); - // Remove [Skip to Content](#page) and [Skip to content](#skip) - markdownContent = markdownContent.replace( - /\[Skip to Content\]\(#[^\)]*\)/gi, - "" - ); return markdownContent; + } catch (error) { + console.error("Error converting HTML to Markdown: ", error); + return ""; // Optionally return an empty string or handle the error as needed } } + +function processMultiLineLinks(markdownContent: string): string { + let insideLinkContent = false; + let newMarkdownContent = ""; + let linkOpenCount = 0; + for (let i = 0; i < markdownContent.length; i++) { + const char = markdownContent[i]; + + if (char == "[") { + linkOpenCount++; + } else if (char == "]") { + linkOpenCount = Math.max(0, linkOpenCount - 1); + } + insideLinkContent = linkOpenCount > 0; + + if (insideLinkContent && char == "\n") { + newMarkdownContent += "\\" + "\n"; + } else { + newMarkdownContent += char; + } + } + return newMarkdownContent; +} + +function removeSkipToContentLinks(markdownContent: string): string { + // Remove [Skip to Content](#page) and [Skip to content](#skip) + const newMarkdownContent = markdownContent.replace( + /\[Skip to Content\]\(#[^\)]*\)/gi, + "" + ); + return newMarkdownContent; +} \ No newline at end of file diff --git a/apps/test-suite/tests/scrape.test.ts b/apps/test-suite/tests/scrape.test.ts index ec7b7202..8b2e15d1 100644 --- a/apps/test-suite/tests/scrape.test.ts +++ b/apps/test-suite/tests/scrape.test.ts @@ -31,6 +31,7 @@ describe("Scraping Checkup (E2E)", () => { describe("Scraping website tests with a dataset", () => { it("Should scrape the website and prompt it against OpenAI", async () => { + let totalTimeTaken = 0; let passedTests = 0; const batchSize = 15; // Adjusted to comply with the rate limit of 15 per minute const batchPromises = []; @@ -51,11 +52,16 @@ describe("Scraping Checkup (E2E)", () => { const batchPromise = Promise.all( batch.map(async (websiteData: WebsiteData) => { try { + const startTime = new Date().getTime(); const scrapedContent = await request(TEST_URL || "") - .post("/v0/scrape") + .post("/v1/scrape") .set("Content-Type", "application/json") .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .send({ url: websiteData.website, pageOptions: { onlyMainContent: true } }); + .send({ url: websiteData.website }); + + const endTime = new Date().getTime(); + const timeTaken = endTime - startTime; + totalTimeTaken += timeTaken; if (scrapedContent.statusCode !== 200) { console.error(`Failed to scrape ${websiteData.website} ${scrapedContent.statusCode}`); @@ -165,6 +171,7 @@ describe("Scraping Checkup (E2E)", () => { const timeTaken = (endTime - startTime) / 1000; console.log(`Score: ${score}%`); console.log(`Total tokens: ${totalTokens}`); + console.log(`Total time taken: ${totalTimeTaken} miliseconds`); await logErrors(errorLog, timeTaken, totalTokens, score, websitesData.length); From ebf403548487249bdb2d8ca2bfe086f950736622 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:15:21 -0300 Subject: [PATCH 009/102] added log so we can check --- apps/api/src/lib/html-to-markdown.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts index 4c7cffdd..e4f8f692 100644 --- a/apps/api/src/lib/html-to-markdown.ts +++ b/apps/api/src/lib/html-to-markdown.ts @@ -32,6 +32,7 @@ export async function parseMarkdown(html: string): Promise { markdownContent = processMultiLineLinks(markdownContent); markdownContent = removeSkipToContentLinks(markdownContent); + Logger.info(`HTML to Markdown conversion using Go parser successful`); return markdownContent; } } catch (error) { From d60fa6e0849fe13538cffaf36bafec55bc1c6ff6 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:08:07 -0300 Subject: [PATCH 010/102] fixed dockerfile and function name. it's working --- apps/api/Dockerfile | 9 ++++----- apps/api/src/lib/html-to-markdown.ts | 6 +++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index a4a2c76b..527a6dc7 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -17,11 +17,11 @@ RUN pnpm install RUN --mount=type=secret,id=SENTRY_AUTH_TOKEN \ bash -c 'export SENTRY_AUTH_TOKEN="$(cat /run/secrets/SENTRY_AUTH_TOKEN)"; if [ -z $SENTRY_AUTH_TOKEN ]; then pnpm run build:nosentry; else pnpm run build; fi' -# Install Go 1.19 +# Install Go FROM golang:1.19 AS go-base COPY src/lib/go-html-to-md /app/src/lib/go-html-to-md -# Install Go dependencies and build +# Install Go dependencies and build parser lib RUN cd /app/src/lib/go-html-to-md && \ go mod tidy && \ go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go && \ @@ -33,9 +33,8 @@ RUN apt-get update -qq && \ rm -rf /var/lib/apt/lists /var/cache/apt/archives COPY --from=prod-deps /app/node_modules /app/node_modules COPY --from=build /app /app -COPY --from=go-base /app/src/lib/go-html-to-md/html-to-markdown.so /app/src/lib/go-html-to-md/html-to-markdown.so - +COPY --from=go-base /app/src/lib/go-html-to-md/html-to-markdown.so /app/dist/src/lib/go-html-to-md/html-to-markdown.so # Start the server by default, this can be overwritten at runtime EXPOSE 8080 -ENV PUPPETEER_EXECUTABLE_PATH="/usr/bin/chromium" +ENV PUPPETEER_EXECUTABLE_PATH="/usr/bin/chromium" \ No newline at end of file diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts index e4f8f692..a5f69962 100644 --- a/apps/api/src/lib/html-to-markdown.ts +++ b/apps/api/src/lib/html-to-markdown.ts @@ -8,6 +8,10 @@ import dotenv from 'dotenv'; import { Logger } from './logger'; dotenv.config(); +// TODO: test with invalid html +// TODO: create a singleton for the converter +// TODO: add a timeout to the Go parser + export async function parseMarkdown(html: string): Promise { if (!html) { return ''; @@ -18,7 +22,7 @@ export async function parseMarkdown(html: string): Promise { const goExecutablePath = join(__dirname, 'go-html-to-md/html-to-markdown.so'); const lib = koffi.load(goExecutablePath); - const convert = lib.func('Convert', 'string', ['string']); + const convert = lib.func('ConvertHTMLToMarkdown', 'string', ['string']); let markdownContent = await new Promise((resolve, reject) => { convert.async(html, (err: Error, res: string) => { From c5e1d77a8253e471a7b021229ac6c85743ba8343 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 3 Sep 2024 15:21:45 -0300 Subject: [PATCH 011/102] added invalid html tests --- apps/api/src/lib/__tests__/html-to-markdown.test.ts | 13 ++++++++++++- apps/api/src/lib/html-to-markdown.ts | 1 - 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/api/src/lib/__tests__/html-to-markdown.test.ts b/apps/api/src/lib/__tests__/html-to-markdown.test.ts index 00db7758..3c68c959 100644 --- a/apps/api/src/lib/__tests__/html-to-markdown.test.ts +++ b/apps/api/src/lib/__tests__/html-to-markdown.test.ts @@ -25,5 +25,16 @@ describe('parseMarkdown', () => { await expect(parseMarkdown(html)).resolves.toBe(expectedMarkdown); }); - + it('should handle various types of invalid HTML gracefully', async () => { + const invalidHtmls = [ + { html: '

Unclosed tag', expected: 'Unclosed tag' }, + { html: '

Missing closing div', expected: 'Missing closing div' }, + { html: '

Wrong nesting

', expected: '**Wrong nesting**' }, + { html: 'Link without closing tag', expected: '[Link without closing tag](http://example.com)' } + ]; + + for (const { html, expected } of invalidHtmls) { + await expect(parseMarkdown(html)).resolves.toBe(expected); + } + }); }); diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts index a5f69962..103948f4 100644 --- a/apps/api/src/lib/html-to-markdown.ts +++ b/apps/api/src/lib/html-to-markdown.ts @@ -8,7 +8,6 @@ import dotenv from 'dotenv'; import { Logger } from './logger'; dotenv.config(); -// TODO: test with invalid html // TODO: create a singleton for the converter // TODO: add a timeout to the Go parser From 5ecb2436932f00529a0f4116a388f1d874dff8d7 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 4 Sep 2024 15:19:45 -0300 Subject: [PATCH 012/102] Nick: --- .../src/services/billing/credit_billing.ts | 143 +++++++++++++----- apps/api/src/services/queue-jobs.ts | 2 +- 2 files changed, 103 insertions(+), 42 deletions(-) diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts index ab00eab9..9ea0435e 100644 --- a/apps/api/src/services/billing/credit_billing.ts +++ b/apps/api/src/services/billing/credit_billing.ts @@ -5,7 +5,7 @@ import { supabase_service } from "../supabase"; import { Logger } from "../../lib/logger"; import { getValue, setValue } from "../redis"; import { redlock } from "../redlock"; - +import * as Sentry from "@sentry/node"; const FREE_CREDITS = 500; @@ -176,9 +176,24 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { return { success: true, message: "Preview team, no credits used", remainingCredits: Infinity }; } - // Retrieve the team's active subscription and check for available coupons concurrently - const [{ data: subscription, error: subscriptionError }, { data: coupons }] = - await Promise.all([ + + let cacheKeySubscription = `subscription_${team_id}`; + let cacheKeyCoupons = `coupons_${team_id}`; + + // Try to get data from cache first + const [cachedSubscription, cachedCoupons] = await Promise.all([ + getValue(cacheKeySubscription), + getValue(cacheKeyCoupons) + ]); + + let subscription, subscriptionError, coupons; + + if (cachedSubscription && cachedCoupons) { + subscription = JSON.parse(cachedSubscription); + coupons = JSON.parse(cachedCoupons); + } else { + // If not in cache, retrieve from database + const [subscriptionResult, couponsResult] = await Promise.all([ supabase_service .from("subscriptions") .select("id, price_id, current_period_start, current_period_end") @@ -192,6 +207,18 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { .eq("status", "active"), ]); + subscription = subscriptionResult.data; + subscriptionError = subscriptionResult.error; + coupons = couponsResult.data; + + // Cache the results for a minute, sub can be null and that's fine + await setValue(cacheKeySubscription, JSON.stringify(subscription), 60); // Cache for 1 minute, even if null + + if (coupons) { + await setValue(cacheKeyCoupons, JSON.stringify(coupons), 60); // Cache for 1 minute + } + } + let couponCredits = 0; if (coupons && coupons.length > 0) { couponCredits = coupons.reduce( @@ -212,41 +239,54 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { let creditUsages; let creditUsageError; - let retries = 0; - const maxRetries = 3; - const retryInterval = 2000; // 2 seconds + let totalCreditsUsed = 0; + const cacheKeyCreditUsage = `credit_usage_${team_id}`; - while (retries < maxRetries) { - const result = await supabase_service - .from("credit_usage") - .select("credits_used") - .is("subscription_id", null) - .eq("team_id", team_id); + // Try to get credit usage from cache + const cachedCreditUsage = await getValue(cacheKeyCreditUsage); - creditUsages = result.data; - creditUsageError = result.error; + if (cachedCreditUsage) { + totalCreditsUsed = parseInt(cachedCreditUsage); + } else { + let retries = 0; + const maxRetries = 3; + const retryInterval = 2000; // 2 seconds - if (!creditUsageError) { - break; + while (retries < maxRetries) { + const result = await supabase_service + .from("credit_usage") + .select("credits_used") + .is("subscription_id", null) + .eq("team_id", team_id); + + creditUsages = result.data; + creditUsageError = result.error; + + if (!creditUsageError) { + break; + } + + retries++; + if (retries < maxRetries) { + await new Promise(resolve => setTimeout(resolve, retryInterval)); + } } - retries++; - if (retries < maxRetries) { - await new Promise(resolve => setTimeout(resolve, retryInterval)); + if (creditUsageError) { + Logger.error(`Credit usage error after ${maxRetries} attempts: ${creditUsageError}`); + throw new Error( + `Failed to retrieve credit usage for team_id: ${team_id}` + ); } - } - if (creditUsageError) { - Logger.error(`Credit usage error after ${maxRetries} attempts: ${creditUsageError}`); - throw new Error( - `Failed to retrieve credit usage for team_id: ${team_id}` + totalCreditsUsed = creditUsages.reduce( + (acc, usage) => acc + usage.credits_used, + 0 ); - } - const totalCreditsUsed = creditUsages.reduce( - (acc, usage) => acc + usage.credits_used, - 0 - ); + // Cache the result for 30 seconds + await setValue(cacheKeyCreditUsage, totalCreditsUsed.toString(), 30); + } Logger.info(`totalCreditsUsed: ${totalCreditsUsed}`); @@ -312,7 +352,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { if (creditUsages && creditUsages.length > 0) { totalCreditsUsed = creditUsages[0].total_credits_used; - await setValue(cacheKey, totalCreditsUsed.toString(), 1800); // Cache for 30 minutes + await setValue(cacheKey, totalCreditsUsed.toString(), 500); // Cache for 8 minutes // Logger.info(`Cache set for credit usage: ${totalCreditsUsed}`); } } @@ -325,17 +365,38 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { // Adjust total credits used by subtracting coupon value const adjustedCreditsUsed = Math.max(0, totalCreditsUsed - couponCredits); - // Get the price details - const { data: price, error: priceError } = await supabase_service - .from("prices") - .select("credits") - .eq("id", subscription.price_id) - .single(); - if (priceError) { - throw new Error( - `Failed to retrieve price for price_id: ${subscription.price_id}` - ); + // Get the price details from cache or database + const priceCacheKey = `price_${subscription.price_id}`; + let price; + + try { + const cachedPrice = await getValue(priceCacheKey); + if (cachedPrice) { + price = JSON.parse(cachedPrice); + } else { + const { data, error: priceError } = await supabase_service + .from("prices") + .select("credits") + .eq("id", subscription.price_id) + .single(); + + if (priceError) { + throw new Error( + `Failed to retrieve price for price_id: ${subscription.price_id}` + ); + } + + price = data; + // There are only 21 records, so this is super fine + // Cache the price for a long time (e.g., 1 day) + await setValue(priceCacheKey, JSON.stringify(price), 86400); + } + } catch (error) { + Logger.error(`Error retrieving or caching price: ${error}`); + Sentry.captureException(error); + // If errors, just assume it's a big number so user don't get an error + price = { credits: 1000000 }; } const creditLimit = price.credits; diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts index 941b571d..7a698772 100644 --- a/apps/api/src/services/queue-jobs.ts +++ b/apps/api/src/services/queue-jobs.ts @@ -67,6 +67,6 @@ export function waitForJob(jobId: string, timeout: number) { reject((await getScrapeQueue().getJob(jobId)).failedReason); } } - }, 1000); + }, 500); }) } From cb8571abad6d0388daf9b66e7db76a22116df6df Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:57:57 -0300 Subject: [PATCH 013/102] fix: enforced dotenv config --- apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts | 4 ++-- apps/api/src/controllers/v0/crawl-cancel.ts | 2 ++ apps/api/src/controllers/v0/crawl-status.ts | 2 ++ apps/api/src/controllers/v1/crawl-cancel.ts | 2 ++ apps/api/src/controllers/v1/crawl-status.ts | 2 ++ apps/api/src/lib/logger.ts | 3 +++ apps/api/src/lib/scrape-events.ts | 2 ++ apps/api/src/lib/withAuth.ts | 2 ++ apps/api/src/main/runWebScraper.ts | 2 ++ apps/api/src/services/logging/crawl_log.ts | 3 ++- apps/api/src/services/logging/log_job.ts | 2 ++ apps/api/src/services/logging/scrape_log.ts | 2 ++ apps/api/src/services/queue-worker.ts | 2 ++ apps/api/src/services/supabase.ts | 2 ++ apps/api/src/services/webhook.ts | 2 ++ apps/test-suite/utils/supabase.ts | 3 ++- 16 files changed, 33 insertions(+), 4 deletions(-) diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts index 880d34a1..913f9408 100644 --- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts @@ -1,11 +1,11 @@ import request from "supertest"; -import dotenv from "dotenv"; +import { configDotenv } from "dotenv"; import { ScrapeRequest, ScrapeResponseRequestTest, } from "../../controllers/v1/types"; -dotenv.config(); +configDotenv(); const TEST_URL = "http://127.0.0.1:3002"; describe("E2E Tests for v1 API Routes", () => { diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts index bf1c2d0a..efcd454a 100644 --- a/apps/api/src/controllers/v0/crawl-cancel.ts +++ b/apps/api/src/controllers/v0/crawl-cancel.ts @@ -5,6 +5,8 @@ import { supabase_service } from "../../../src/services/supabase"; import { Logger } from "../../../src/lib/logger"; import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis"; import * as Sentry from "@sentry/node"; +import { configDotenv } from "dotenv"; +configDotenv(); export async function crawlCancelController(req: Request, res: Response) { try { diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts index b0649cd0..a3f3f16f 100644 --- a/apps/api/src/controllers/v0/crawl-status.ts +++ b/apps/api/src/controllers/v0/crawl-status.ts @@ -6,6 +6,8 @@ import { Logger } from "../../../src/lib/logger"; import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis"; import { supabaseGetJobsById } from "../../../src/lib/supabase-jobs"; import * as Sentry from "@sentry/node"; +import { configDotenv } from "dotenv"; +configDotenv(); export async function getJobs(ids: string[]) { const jobs = (await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))).filter(x => x); diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts index 06a5b26e..21fc7cf9 100644 --- a/apps/api/src/controllers/v1/crawl-cancel.ts +++ b/apps/api/src/controllers/v1/crawl-cancel.ts @@ -5,6 +5,8 @@ import { supabase_service } from "../../services/supabase"; import { Logger } from "../../lib/logger"; import { getCrawl, saveCrawl } from "../../lib/crawl-redis"; import * as Sentry from "@sentry/node"; +import { configDotenv } from "dotenv"; +configDotenv(); export async function crawlCancelController(req: Request, res: Response) { try { diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts index 845f616c..05144a9b 100644 --- a/apps/api/src/controllers/v1/crawl-status.ts +++ b/apps/api/src/controllers/v1/crawl-status.ts @@ -3,6 +3,8 @@ import { CrawlStatusParams, CrawlStatusResponse, ErrorResponse, legacyDocumentCo import { getCrawl, getCrawlExpiry, getCrawlJobs, getDoneJobsOrdered, getDoneJobsOrderedLength } from "../../lib/crawl-redis"; import { getScrapeQueue } from "../../services/queue-service"; import { supabaseGetJobById, supabaseGetJobsById } from "../../lib/supabase-jobs"; +import { configDotenv } from "dotenv"; +configDotenv(); export async function getJob(id: string) { const job = await getScrapeQueue().getJob(id); diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts index fb0468c2..cb8b4119 100644 --- a/apps/api/src/lib/logger.ts +++ b/apps/api/src/lib/logger.ts @@ -1,3 +1,6 @@ +import { configDotenv } from "dotenv"; +configDotenv(); + enum LogLevel { NONE = 'NONE', // No logs will be output. ERROR = 'ERROR', // For logging error messages that indicate a failure in a specific operation. diff --git a/apps/api/src/lib/scrape-events.ts b/apps/api/src/lib/scrape-events.ts index ed011b78..ad70dfef 100644 --- a/apps/api/src/lib/scrape-events.ts +++ b/apps/api/src/lib/scrape-events.ts @@ -2,6 +2,8 @@ import { Job } from "bullmq"; import type { baseScrapers } from "../scraper/WebScraper/single_url"; import { supabase_service as supabase } from "../services/supabase"; import { Logger } from "./logger"; +import { configDotenv } from "dotenv"; +configDotenv(); export type ScrapeErrorEvent = { type: "error", diff --git a/apps/api/src/lib/withAuth.ts b/apps/api/src/lib/withAuth.ts index 90cfb449..b45b8973 100644 --- a/apps/api/src/lib/withAuth.ts +++ b/apps/api/src/lib/withAuth.ts @@ -1,6 +1,8 @@ import { AuthResponse } from "../../src/types"; import { Logger } from "./logger"; import * as Sentry from "@sentry/node"; +import { configDotenv } from "dotenv"; +configDotenv(); let warningCount = 0; diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index cd199fa1..f67a1cd0 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -12,6 +12,8 @@ import { Document } from "../lib/entities"; import { supabase_service } from "../services/supabase"; import { Logger } from "../lib/logger"; import { ScrapeEvents } from "../lib/scrape-events"; +import { configDotenv } from "dotenv"; +configDotenv(); export async function startWebScraperPipeline({ job, diff --git a/apps/api/src/services/logging/crawl_log.ts b/apps/api/src/services/logging/crawl_log.ts index f19b0297..3850e05b 100644 --- a/apps/api/src/services/logging/crawl_log.ts +++ b/apps/api/src/services/logging/crawl_log.ts @@ -1,6 +1,7 @@ import { supabase_service } from "../supabase"; import { Logger } from "../../../src/lib/logger"; -import "dotenv/config"; +import { configDotenv } from "dotenv"; +configDotenv(); export async function logCrawl(job_id: string, team_id: string) { const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true'; diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index d4494f09..4d8ee014 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -4,6 +4,8 @@ import { FirecrawlJob } from "../../types"; import { posthog } from "../posthog"; import "dotenv/config"; import { Logger } from "../../lib/logger"; +import { configDotenv } from "dotenv"; +configDotenv(); export async function logJob(job: FirecrawlJob) { try { diff --git a/apps/api/src/services/logging/scrape_log.ts b/apps/api/src/services/logging/scrape_log.ts index 30d8fd1e..fbe41653 100644 --- a/apps/api/src/services/logging/scrape_log.ts +++ b/apps/api/src/services/logging/scrape_log.ts @@ -3,6 +3,8 @@ import { ScrapeLog } from "../../types"; import { supabase_service } from "../supabase"; import { PageOptions } from "../../lib/entities"; import { Logger } from "../../lib/logger"; +import { configDotenv } from "dotenv"; +configDotenv(); export async function logScrape( scrapeLog: ScrapeLog, diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index 6488759f..ad0e4ad5 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -36,6 +36,8 @@ import { } from "../../src/lib/job-priority"; import { PlanType } from "../types"; import { getJobs } from "../../src/controllers/v1/crawl-status"; +import { configDotenv } from "dotenv"; +configDotenv(); if (process.env.ENV === "production") { initSDK({ diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.ts index 414d1925..7636717e 100644 --- a/apps/api/src/services/supabase.ts +++ b/apps/api/src/services/supabase.ts @@ -1,5 +1,7 @@ import { createClient, SupabaseClient } from "@supabase/supabase-js"; import { Logger } from "../lib/logger"; +import { configDotenv } from "dotenv"; +configDotenv(); // SupabaseService class initializes the Supabase client conditionally based on environment variables. class SupabaseService { diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.ts index 56dd5c58..06e5649d 100644 --- a/apps/api/src/services/webhook.ts +++ b/apps/api/src/services/webhook.ts @@ -3,6 +3,8 @@ import { legacyDocumentConverter } from "../../src/controllers/v1/types"; import { Logger } from "../../src/lib/logger"; import { supabase_service } from "./supabase"; import { WebhookEventType } from "../types"; +import { configDotenv } from "dotenv"; +configDotenv(); export const callWebhook = async ( teamId: string, diff --git a/apps/test-suite/utils/supabase.ts b/apps/test-suite/utils/supabase.ts index 3e66a991..a1549e24 100644 --- a/apps/test-suite/utils/supabase.ts +++ b/apps/test-suite/utils/supabase.ts @@ -1,5 +1,6 @@ import { createClient, SupabaseClient } from "@supabase/supabase-js"; -import "dotenv/config"; +import { configDotenv } from "dotenv"; +configDotenv(); // SupabaseService class initializes the Supabase client conditionally based on environment variables. class SupabaseService { From 78edf13ec6f52c12956b576712c4ca663a5d16ad Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:31:42 -0300 Subject: [PATCH 014/102] test: usedbauth envs wth --- apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts index 913f9408..5631adf0 100644 --- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts @@ -22,6 +22,13 @@ describe("E2E Tests for v1 API Routes", () => { const response: ScrapeResponseRequestTest = await request(TEST_URL).get( "/is-production" ); + + console.log('process.env.USE_DB_AUTHENTICATION', process.env.USE_DB_AUTHENTICATION); + console.log('?', process.env.USE_DB_AUTHENTICATION === 'true'); + const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true'; + console.log('useDbAuthentication', useDbAuthentication); + console.log('!useDbAuthentication', !useDbAuthentication); + expect(response.statusCode).toBe(200); expect(response.body).toHaveProperty("isProduction"); }); From 85b824e122a095595b9f188902eb771590392f06 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:35:32 -0300 Subject: [PATCH 015/102] test: what about false false? --- apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts index 5631adf0..8aabf748 100644 --- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts @@ -26,7 +26,7 @@ describe("E2E Tests for v1 API Routes", () => { console.log('process.env.USE_DB_AUTHENTICATION', process.env.USE_DB_AUTHENTICATION); console.log('?', process.env.USE_DB_AUTHENTICATION === 'true'); const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true'; - console.log('useDbAuthentication', useDbAuthentication); + console.log('!!useDbAuthentication', !!useDbAuthentication); console.log('!useDbAuthentication', !useDbAuthentication); expect(response.statusCode).toBe(200); From 28c5635502ebfdc98852ecf576cf7b9aa27f48e8 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 4 Sep 2024 16:45:56 -0300 Subject: [PATCH 016/102] Update ci.yml --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2e42e4a..ff22858b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,7 @@ env: HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }} HDX_NODE_BETA_MODE: 1 FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }} + USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }} jobs: From a0113dac3753725500d659f632bfe77a67e8e191 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 4 Sep 2024 16:54:20 -0300 Subject: [PATCH 017/102] Update credit_billing.ts --- apps/api/src/services/billing/credit_billing.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts index 9ea0435e..53031de9 100644 --- a/apps/api/src/services/billing/credit_billing.ts +++ b/apps/api/src/services/billing/credit_billing.ts @@ -213,10 +213,8 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { // Cache the results for a minute, sub can be null and that's fine await setValue(cacheKeySubscription, JSON.stringify(subscription), 60); // Cache for 1 minute, even if null - - if (coupons) { - await setValue(cacheKeyCoupons, JSON.stringify(coupons), 60); // Cache for 1 minute - } + await setValue(cacheKeyCoupons, JSON.stringify(coupons), 60); // Cache for 1 minute + } let couponCredits = 0; From 82cb80c8170b299c83cf954b94c6d9c30c2166c0 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 4 Sep 2024 23:46:18 -0300 Subject: [PATCH 018/102] Update map.ts --- apps/api/src/controllers/v1/map.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts index 4c94f041..e6abd9ae 100644 --- a/apps/api/src/controllers/v1/map.ts +++ b/apps/api/src/controllers/v1/map.ts @@ -62,8 +62,8 @@ export async function mapController( : `site:${req.body.url}`; // www. seems to exclude subdomains in some cases const mapResults = await fireEngineMap(mapUrl, { - // limit to 50 results (beta) - numResults: Math.min(limit, 50), + // limit to 100 results (beta) + numResults: Math.min(limit, 100), }); if (mapResults.length > 0) { From eb03a81152883b5371b2a4519f98bae1d71065dd Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 5 Sep 2024 12:55:04 -0300 Subject: [PATCH 019/102] Update crawl-status.ts --- apps/api/src/controllers/v1/crawl-status.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts index 05144a9b..ad4d21d2 100644 --- a/apps/api/src/controllers/v1/crawl-status.ts +++ b/apps/api/src/controllers/v1/crawl-status.ts @@ -94,7 +94,8 @@ export async function crawlStatusController(req: RequestWithAuth x.returnvalue); - const nextURL = new URL(`${req.protocol}://${req.get("host")}/v1/crawl/${req.params.jobId}`); + const protocol = process.env.ENV === "local" ? req.protocol : "https"; + const nextURL = new URL(`${protocol}://${req.get("host")}/v1/crawl/${req.params.jobId}`); nextURL.searchParams.set("skip", (start + data.length).toString()); From 7561bfe173c2a7934ef1ab7b64d1a79f3177833b Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:59:32 -0300 Subject: [PATCH 020/102] added envs to github action workflows --- .github/workflows/ci.yml | 2 +- .github/workflows/fly-direct.yml | 6 ++++++ .github/workflows/fly.yml | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff22858b..8a9a74cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ env: HDX_NODE_BETA_MODE: 1 FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }} USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }} - + ENV: ${{ secrets.ENV }} jobs: pre-deploy: diff --git a/.github/workflows/fly-direct.yml b/.github/workflows/fly-direct.yml index 8ec675fa..2473642c 100644 --- a/.github/workflows/fly-direct.yml +++ b/.github/workflows/fly-direct.yml @@ -22,7 +22,13 @@ env: SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} SUPABASE_URL: ${{ secrets.SUPABASE_URL }} TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }} + ENV: ${{ secrets.ENV }} jobs: deploy: diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index 9209309f..7b45921a 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -29,6 +29,7 @@ env: CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }} + ENV: ${{ secrets.ENV }} jobs: pre-deploy-e2e-tests: From c6f1d8099296496bcd650dac6f8cc52643e34aaa Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 5 Sep 2024 13:03:43 -0300 Subject: [PATCH 021/102] Update crawl.ts --- apps/api/src/controllers/v1/crawl.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts index c2d5bdca..e0883fa8 100644 --- a/apps/api/src/controllers/v1/crawl.ts +++ b/apps/api/src/controllers/v1/crawl.ts @@ -155,10 +155,12 @@ export async function crawlController( await callWebhook(req.auth.team_id, id, null, req.body.webhook, true, "crawl.started"); } + const protocol = process.env.ENV === "local" ? req.protocol : "https"; + return res.status(200).json({ success: true, id, - url: `${req.protocol}://${req.get("host")}/v1/crawl/${id}`, + url: `${protocol}://${req.get("host")}/v1/crawl/${id}`, }); } From b301ffc922561fd363c5207d44e33c6b96e69c9a Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:57:26 -0300 Subject: [PATCH 022/102] added missing variables --- apps/api/src/scraper/WebScraper/index.ts | 3 +++ apps/api/src/scraper/WebScraper/single_url.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index fc828224..8bd7d493 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -589,6 +589,9 @@ export class WebScraperDataProvider { includeLinks: options.pageOptions?.includeLinks ?? true, fullPageScreenshot: options.pageOptions?.fullPageScreenshot ?? false, screenshot: options.pageOptions?.screenshot ?? false, + useFastMode: options.pageOptions?.useFastMode ?? false, + disableJSDom: options.pageOptions?.disableJSDom ?? false, + atsv: options.pageOptions?.atsv ?? false }; this.extractorOptions = options.extractorOptions ?? { mode: "markdown" }; this.replaceAllPathsWithAbsolutePaths = diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 11e1fe37..f39f045f 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -146,6 +146,9 @@ export async function scrapSingleUrl( parsePDF: pageOptions.parsePDF ?? true, removeTags: pageOptions.removeTags ?? [], onlyIncludeTags: pageOptions.onlyIncludeTags ?? [], + useFastMode: pageOptions.useFastMode ?? false, + disableJSDom: pageOptions.disableJSDom ?? false, + atsv: pageOptions.atsv ?? false } if (extractorOptions) { From 8c1097e9e19f60213cc66299b6452c9141ba9365 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:16:31 -0300 Subject: [PATCH 023/102] fix: pageOptions --- apps/api/src/lib/entities.ts | 2 +- apps/api/src/scraper/WebScraper/index.ts | 2 +- apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts | 3 ++- apps/api/src/scraper/WebScraper/single_url.ts | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts index dfd17c63..d7ec2a83 100644 --- a/apps/api/src/lib/entities.ts +++ b/apps/api/src/lib/entities.ts @@ -28,7 +28,7 @@ export type PageOptions = { onlyIncludeTags?: string | string[]; includeLinks?: boolean; useFastMode?: boolean; // beta - disableJSDom?: boolean; // beta + disableJsDom?: boolean; // beta atsv?: boolean; // beta }; diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index 8bd7d493..2f7efa47 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -590,7 +590,7 @@ export class WebScraperDataProvider { fullPageScreenshot: options.pageOptions?.fullPageScreenshot ?? false, screenshot: options.pageOptions?.screenshot ?? false, useFastMode: options.pageOptions?.useFastMode ?? false, - disableJSDom: options.pageOptions?.disableJSDom ?? false, + disableJsDom: options.pageOptions?.disableJsDom ?? false, atsv: options.pageOptions?.atsv ?? false }; this.extractorOptions = options.extractorOptions ?? { mode: "markdown" }; diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index aa86ad5e..d5f764b5 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -104,12 +104,13 @@ export async function scrapWithFireEngine({ screenshot: screenshotParam, fullPageScreenshot: fullPageScreenshotParam, headers: headers, - pageOptions: pageOptions, disableJsDom: pageOptions?.disableJsDom ?? false, priority, engine, instantReturn: true, ...fireEngineOptionsParam, + atsv: pageOptions?.atsv ?? false, + scrollXPaths: pageOptions?.scrollXPaths ?? [], }, { headers: { diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index f39f045f..8bafd203 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -147,7 +147,7 @@ export async function scrapSingleUrl( removeTags: pageOptions.removeTags ?? [], onlyIncludeTags: pageOptions.onlyIncludeTags ?? [], useFastMode: pageOptions.useFastMode ?? false, - disableJSDom: pageOptions.disableJSDom ?? false, + disableJsDom: pageOptions.disableJsDom ?? false, atsv: pageOptions.atsv ?? false } @@ -203,6 +203,7 @@ export async function scrapSingleUrl( fireEngineOptions: { engine: engine, atsv: pageOptions.atsv, + disableJsDom: pageOptions.disableJsDom, }, priority, teamId, From cb630bfc341725f46ceb12c0169c7dd7d0ebdb6c Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 5 Sep 2024 14:24:10 -0300 Subject: [PATCH 024/102] Update fireEngine.ts --- apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index d5f764b5..e7361c5c 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -69,15 +69,15 @@ export async function scrapWithFireEngine({ let engine = engineParam; // do we want fireEngineOptions as first choice? - Logger.info( - `⛏️ Fire-Engine (${engine}): Scraping ${url} | params: { wait: ${waitParam}, screenshot: ${screenshotParam}, fullPageScreenshot: ${fullPageScreenshot}, method: ${fireEngineOptionsParam?.method ?? "null"} }` - ); - if (pageOptions?.useFastMode) { fireEngineOptionsParam.engine = "tlsclient"; engine = "tlsclient"; } + Logger.info( + `⛏️ Fire-Engine (${engine}): Scraping ${url} | params: { wait: ${waitParam}, screenshot: ${screenshotParam}, fullPageScreenshot: ${fullPageScreenshot}, method: ${fireEngineOptionsParam?.method ?? "null"} }` + ); + // atsv is only available for beta customers const betaCustomersString = process.env.BETA_CUSTOMERS; const betaCustomers = betaCustomersString ? betaCustomersString.split(",") : []; @@ -96,6 +96,7 @@ export async function scrapWithFireEngine({ const _response = await Sentry.startSpan({ name: "Call to fire-engine" }, async span => { + return await axiosInstance.post( process.env.FIRE_ENGINE_BETA_URL + endpoint, { From b3f21d437b8035304b2b9f075e39cabd097a5e3a Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 5 Sep 2024 15:30:10 -0300 Subject: [PATCH 025/102] Update README.md --- README.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 63dd6ea5..c624543b 100644 --- a/README.md +++ b/README.md @@ -402,15 +402,11 @@ class TopArticlesSchema(BaseModel): top: List[ArticleSchema] = Field(..., max_items=5, description="Top 5 stories") data = app.scrape_url('https://news.ycombinator.com', { - 'extractorOptions': { - 'extractionSchema': TopArticlesSchema.model_json_schema(), - 'mode': 'llm-extraction' - }, - 'pageOptions':{ - 'onlyMainContent': True + 'extract': { + 'schema': TopArticlesSchema.model_json_schema() } }) -print(data["llm_extraction"]) +print(data["extract"]) ``` ## Using the Node SDK From 82d6bf4ec8e8d62102436f28b2705e2c532bedc7 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:14:21 -0300 Subject: [PATCH 026/102] feat(go-parser): singleton --- apps/api/src/lib/html-to-markdown.ts | 47 +++++++++++++++++++--------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts index 103948f4..a542a434 100644 --- a/apps/api/src/lib/html-to-markdown.ts +++ b/apps/api/src/lib/html-to-markdown.ts @@ -8,9 +8,38 @@ import dotenv from 'dotenv'; import { Logger } from './logger'; dotenv.config(); -// TODO: create a singleton for the converter // TODO: add a timeout to the Go parser +class GoMarkdownConverter { + private static instance: GoMarkdownConverter; + private convert: any; + + private constructor() { + const goExecutablePath = join(__dirname, 'go-html-to-md/html-to-markdown.so'); + const lib = koffi.load(goExecutablePath); + this.convert = lib.func('ConvertHTMLToMarkdown', 'string', ['string']); + } + + public static getInstance(): GoMarkdownConverter { + if (!GoMarkdownConverter.instance) { + GoMarkdownConverter.instance = new GoMarkdownConverter(); + } + return GoMarkdownConverter.instance; + } + + public async convertHTMLToMarkdown(html: string): Promise { + return new Promise((resolve, reject) => { + this.convert.async(html, (err: Error, res: string) => { + if (err) { + reject(err); + } else { + resolve(res); + } + }); + }); + } +} + export async function parseMarkdown(html: string): Promise { if (!html) { return ''; @@ -18,20 +47,8 @@ export async function parseMarkdown(html: string): Promise { try { if (process.env.USE_GO_MARKDOWN_PARSER == "true") { - const goExecutablePath = join(__dirname, 'go-html-to-md/html-to-markdown.so'); - const lib = koffi.load(goExecutablePath); - - const convert = lib.func('ConvertHTMLToMarkdown', 'string', ['string']); - - let markdownContent = await new Promise((resolve, reject) => { - convert.async(html, (err: Error, res: string) => { - if (err) { - reject(err); - } else { - resolve(res); - } - }); - }); + const converter = GoMarkdownConverter.getInstance(); + let markdownContent = await converter.convertHTMLToMarkdown(html); markdownContent = processMultiLineLinks(markdownContent); markdownContent = removeSkipToContentLinks(markdownContent); From 4fa917f2b3c14b7c7f59a79b509debd298c19da3 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 5 Sep 2024 16:45:23 -0300 Subject: [PATCH 027/102] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c624543b..96878ea2 100644 --- a/README.md +++ b/README.md @@ -402,6 +402,7 @@ class TopArticlesSchema(BaseModel): top: List[ArticleSchema] = Field(..., max_items=5, description="Top 5 stories") data = app.scrape_url('https://news.ycombinator.com', { + 'formats': ['extract'], 'extract': { 'schema': TopArticlesSchema.model_json_schema() } From f5b84e15e1d8fe2e22c5d44bce685c28a5e4d752 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 5 Sep 2024 17:52:27 -0300 Subject: [PATCH 028/102] Update sitemap.ts --- apps/api/src/scraper/WebScraper/sitemap.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts index b1a6a6ff..13dfc26e 100644 --- a/apps/api/src/scraper/WebScraper/sitemap.ts +++ b/apps/api/src/scraper/WebScraper/sitemap.ts @@ -36,17 +36,15 @@ export async function getLinksFromSitemap( const root = parsed.urlset || parsed.sitemapindex; if (root && root.sitemap) { - for (const sitemap of root.sitemap) { - if (sitemap.loc && sitemap.loc.length > 0) { - await getLinksFromSitemap({ sitemapUrl: sitemap.loc[0], allUrls, mode }); - } - } + const sitemapPromises = root.sitemap + .filter(sitemap => sitemap.loc && sitemap.loc.length > 0) + .map(sitemap => getLinksFromSitemap({ sitemapUrl: sitemap.loc[0], allUrls, mode })); + await Promise.all(sitemapPromises); } else if (root && root.url) { - for (const url of root.url) { - if (url.loc && url.loc.length > 0 && !WebCrawler.prototype.isFile(url.loc[0])) { - allUrls.push(url.loc[0]); - } - } + const validUrls = root.url + .filter(url => url.loc && url.loc.length > 0 && !WebCrawler.prototype.isFile(url.loc[0])) + .map(url => url.loc[0]); + allUrls.push(...validUrls); } } catch (error) { Logger.debug(`Error processing sitemapUrl: ${sitemapUrl} | Error: ${error.message}`); From aa2cf686f4c891e5fe5b8be8eb15050bff01d261 Mon Sep 17 00:00:00 2001 From: Tadashi Shigeoka Date: Fri, 6 Sep 2024 21:41:31 +0900 Subject: [PATCH 029/102] [Docs] upgraded the path of the self-hosted documentation URL to `/v1`. --- SELF_HOST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SELF_HOST.md b/SELF_HOST.md index f631cf18..2fa87776 100644 --- a/SELF_HOST.md +++ b/SELF_HOST.md @@ -106,7 +106,7 @@ You should be able to see the Bull Queue Manager UI on `http://localhost:3002/ad If you’d like to test the crawl endpoint, you can run this: ```bash - curl -X POST http://localhost:3002/v0/crawl \ + curl -X POST http://localhost:3002/v1/crawl \ -H 'Content-Type: application/json' \ -d '{ "url": "https://mendable.ai" From 2044e71fcf1fb811a94f8aae1b87acdfaaaac2be Mon Sep 17 00:00:00 2001 From: Eric Ciarla Date: Fri, 6 Sep 2024 15:26:33 -0400 Subject: [PATCH 030/102] Docs to API Spec --- .../turning_docs_into_api_specs/api_spec.json | 771 ------------------ .../combined_api_spec.json | 510 ++++++++++++ .../dify_api_spec.json | 164 ---- .../docs.firecrawl.dev/api_spec_0.json | 211 ----- .../docs.firecrawl.dev/api_spec_1.json | 165 ---- .../docs.firecrawl.dev/api_spec_10.json | 93 --- .../docs.firecrawl.dev/api_spec_11.json | 131 --- .../docs.firecrawl.dev/api_spec_13.json | 87 -- .../docs.firecrawl.dev/api_spec_15.json | 83 -- .../docs.firecrawl.dev/api_spec_16.json | 200 ----- .../docs.firecrawl.dev/api_spec_2.json | 54 -- .../docs.firecrawl.dev/api_spec_22.json | 166 ---- .../docs.firecrawl.dev/api_spec_25.json | 229 ------ .../docs.firecrawl.dev/api_spec_26.json | 115 --- .../docs.firecrawl.dev/api_spec_3.json | 185 ----- .../docs.firecrawl.dev/api_spec_30.json | 212 ----- .../docs.firecrawl.dev/api_spec_31.json | 199 ----- .../docs.firecrawl.dev/api_spec_33.json | 202 ----- .../docs.firecrawl.dev/api_spec_34.json | 201 ----- .../docs.firecrawl.dev/api_spec_35.json | 245 ------ .../docs.firecrawl.dev/api_spec_4.json | 129 --- .../docs.firecrawl.dev/api_spec_5.json | 186 ----- .../docs.firecrawl.dev/api_spec_7.json | 86 -- .../docs.firecrawl.dev/api_spec_8.json | 59 -- .../docs.firecrawl.dev/combined_api_spec.json | 738 ----------------- .../turning_docs_into_api_specs.ipynb | 287 ------- .../turning_docs_into_api_specs.py | 137 ++++ 27 files changed, 647 insertions(+), 5198 deletions(-) delete mode 100644 examples/turning_docs_into_api_specs/api_spec.json create mode 100644 examples/turning_docs_into_api_specs/combined_api_spec.json delete mode 100644 examples/turning_docs_into_api_specs/dify_api_spec.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_0.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_1.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_10.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_11.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_13.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_15.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_16.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_2.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_22.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_25.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_26.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_3.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_30.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_31.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_33.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_34.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_35.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_4.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_5.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_7.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_8.json delete mode 100644 examples/turning_docs_into_api_specs/docs.firecrawl.dev/combined_api_spec.json delete mode 100644 examples/turning_docs_into_api_specs/turning_docs_into_api_specs.ipynb create mode 100644 examples/turning_docs_into_api_specs/turning_docs_into_api_specs.py diff --git a/examples/turning_docs_into_api_specs/api_spec.json b/examples/turning_docs_into_api_specs/api_spec.json deleted file mode 100644 index d866efd3..00000000 --- a/examples/turning_docs_into_api_specs/api_spec.json +++ /dev/null @@ -1,771 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/crawl": { - "post": { - "/crawl/cancel/{jobId}": { - "/crawl/status/{jobId}": { - "get": { - "/scrape": { - "/search": { - "post": { - "components": { - "securitySchemes": { - "Authorization": { - "bearerFormat": "JWT", - "scheme": "bearer", - "type": "http" - } - } - }, - "description": "Send a request to perform a web search and get scraped results from the top pages.", - "operationId": "searchWeb", - "parameters": [], - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "pageOptions": { - "description": "Options for controlling the scraping behavior of search result pages.", - "properties": { - "fetchPageContent": { - "default": true, - "description": "Fetch the content of each page. If false, defaults to a basic fast serp API.", - "type": "boolean" - }, - "includeHtml": { - "default": false, - "description": "Include the HTML version of the content on page. Will output a html key in the response.", - "type": "boolean" - }, - "includeRawHtml": { - "default": false, - "description": "Include the raw HTML content of the page. Will output a rawHtml key in the response.", - "type": "boolean" - }, - "onlyMainContent": { - "default": false, - "description": "Only return the main content of the page excluding headers, navs, footers, etc.", - "type": "boolean" - } - }, - "type": "object" - }, - "query": { - "description": "The search query.", - "required": true, - "type": "string" - }, - "searchOptions": { - "description": "Options for controlling the search.", - "properties": { - "limit": { - "description": "Maximum number of search results to return.", - "type": "integer" - } - }, - "type": "object" - } - }, - "type": "object" - } - } - }, - "responses": { - "200": { - "402": { - "description": "Payment required." - }, - "429": { - "description": "Rate limit exceeded." - }, - "500": { - "description": "Internal server error." - }, - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "description": "An array of search results.", - "items": { - "properties": { - "content": { - "description": "Raw content of the search result page.", - "type": "string" - }, - "markdown": { - "description": "Markdown content of the search result page.", - "type": "string" - }, - "metadata": { - "description": "Metadata extracted from the search result page.", - "properties": { - "description": { - "description": "Page description.", - "type": "string" - }, - "language": { - "description": "Page language.", - "nullable": true, - "type": "string" - }, - "sourceURL": { - "description": "Source URL of the search result page.", - "type": "string" - }, - "title": { - "description": "Page title.", - "type": "string" - } - }, - "type": "object" - }, - "url": { - "description": "URL of the search result.", - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "success": { - "description": "Indicates if the search was successful.", - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Web search completed successfully." - } - } - }, - "summary": "Search the Web" - } - }, - "post": { - "description": "Send a request to scrape a single URL and get its content.", - "operationId": "scrapeURL", - "parameters": [], - "requestBody": { - "402": { - "description": "Payment required." - }, - "429": { - "description": "Rate limit exceeded." - }, - "500": { - "description": "Internal server error." - }, - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "description": "Options for extraction of structured information from the page content. Note: LLM-based extraction is not performed by default and only occurs when explicitly configured. The 'markdown' mode simply returns the scraped markdown and is the default mode for scraping.", - "properties": { - "extractionPrompt": { - "description": "A prompt describing what information to extract from the page, applicable for LLM extraction modes.", - "type": "string" - }, - "extractionSchema": { - "description": "The schema for the data to be extracted, required only for LLM extraction modes.", - "type": "object" - }, - "mode": { - "default": "markdown", - "description": "The extraction mode to use. 'markdown': Returns the scraped markdown content, does not perform LLM extraction. 'llm-extraction': Extracts information from the cleaned and parsed content using LLM. 'llm-extraction-from-raw-html': Extracts information directly from the raw HTML using LLM. 'llm-extraction-from-markdown': Extracts information from the markdown content using LLM.", - "enum": [ - "markdown", - "llm-extraction", - "llm-extraction-from-raw-html", - "llm-extraction-from-markdown" - ], - "type": "string" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Options for controlling the scraping behavior.", - "properties": { - "fullPageScreenshot": { - "default": false, - "description": "Include a full page screenshot of the page that you are scraping.", - "type": "boolean" - }, - "headers": { - "description": "Headers to send with the request. Can be used to send cookies, user-agent, etc.", - "type": "object" - }, - "includeHtml": { - "default": false, - "description": "Include the HTML version of the content on page. Will output a html key in the response.", - "type": "boolean" - }, - "includeRawHtml": { - "default": false, - "description": "Include the raw HTML content of the page. Will output a rawHtml key in the response.", - "type": "boolean" - }, - "onlyIncludeTags": { - "description": "Only include tags, classes and ids from the page in the final output. Use comma separated values. Example: 'script, .ad, #footer'", - "items": { - "type": "string" - }, - "type": "array" - }, - "onlyMainContent": { - "default": false, - "description": "Only return the main content of the page excluding headers, navs, footers, etc.", - "type": "boolean" - }, - "removeTags": { - "description": "Tags, classes and ids to remove from the page. Use comma separated values. Example: 'script, .ad, #footer'", - "items": { - "type": "string" - }, - "type": "array" - }, - "replaceAllPathsWithAbsolutePaths": { - "default": false, - "description": "Replace all relative paths with absolute paths for images and links", - "type": "boolean" - }, - "screenshot": { - "default": false, - "description": "Include a screenshot of the top of the page that you are scraping.", - "type": "boolean" - }, - "waitFor": { - "default": 0, - "description": "Wait x amount of milliseconds for the page to load to fetch content", - "type": "integer" - } - }, - "type": "object" - }, - "timeout": { - "default": 30000, - "description": "Timeout in milliseconds for the request", - "type": "integer" - }, - "url": { - "description": "The URL to scrape.", - "required": true, - "type": "string" - } - }, - "type": "object" - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "properties": { - "content": { - "description": "Raw content of the page.", - "type": "string" - }, - "html": { - "description": "HTML version of the page content, only present if `includeHtml` was set to `true` in the request.", - "nullable": true, - "type": "string" - }, - "llm_extraction": { - "description": "Extracted data from the page using the specified schema, only present if an LLM extraction mode was used.", - "nullable": true, - "type": "object" - }, - "markdown": { - "description": "Markdown version of the page content.", - "type": "string" - }, - "metadata": { - "properties": { - " ": { - "description": "Any other extracted metadata.", - "type": "string" - }, - "description": { - "description": "Page description.", - "type": "string" - }, - "language": { - "description": "Page language.", - "nullable": true, - "type": "string" - }, - "pageError": { - "description": "Error message if there was an error scraping the page.", - "nullable": true, - "type": "string" - }, - "pageStatusCode": { - "description": "HTTP status code of the page.", - "type": "integer" - }, - "sourceURL": { - "description": "Source URL of the page.", - "type": "string" - }, - "title": { - "description": "Page title.", - "type": "string" - } - }, - "type": "object" - }, - "rawHtml": { - "description": "Raw HTML content of the page, only present if `includeRawHtml` was set to `true` in the request.", - "nullable": true, - "type": "string" - }, - "warning": { - "description": "Warning message from the LLM extraction process, if any.", - "nullable": true, - "type": "string" - } - }, - "type": "object" - }, - "success": { - "description": "Indicates whether the scraping was successful.", - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "URL scraped successfully." - } - } - }, - "summary": "Scrape a URL" - } - }, - "description": "Send a request to get the status and results of a crawl job.", - "operationId": "getCrawlJobStatus", - "parameters": [ - { - "description": "ID of the crawl job to check.", - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": {} - }, - "responses": { - "200": { - "402": { - "description": "Payment required." - }, - "429": { - "description": "Rate limit exceeded." - }, - "500": { - "description": "Internal server error." - }, - "content": { - "application/json": { - "schema": { - "properties": { - "current": { - "description": "The number of pages crawled so far.", - "type": "integer" - }, - "data": { - "description": "The crawl results. Only available when the crawl job is completed.", - "items": { - "properties": { - "content": { - "description": "Raw content of the page.", - "type": "string" - }, - "html": { - "description": "HTML version of the page content, only present if `includeHtml` was set to `true` in the crawl request.", - "type": "string" - }, - "index": { - "description": "The index of the crawled page in the results.", - "type": "integer" - }, - "markdown": { - "description": "Markdown content of the page.", - "type": "string" - }, - "metadata": { - "description": "Metadata extracted from the page.", - "properties": { - " ": { - "description": "Any other extracted metadata.", - "type": "string" - }, - "description": { - "description": "Page description.", - "type": "string" - }, - "language": { - "description": "Page language.", - "type": "string" - }, - "pageError": { - "description": "Error message if there was an error scraping the page.", - "type": "string" - }, - "pageStatusCode": { - "description": "HTTP status code of the page.", - "type": "integer" - }, - "sourceURL": { - "description": "Source URL of the page.", - "type": "string" - }, - "title": { - "description": "Page title.", - "type": "string" - } - }, - "type": "object" - }, - "rawHtml": { - "description": "Raw HTML content of the page, only present if `includeRawHtml` was set to `true` in the crawl request.", - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "partial_data": { - "description": "Partial results streamed as the crawl progresses. This feature is in alpha and may change.", - "items": { - "properties": { - "content": { - "description": "Raw content of the page.", - "type": "string" - }, - "html": { - "description": "HTML version of the page content, only present if `includeHtml` was set to `true` in the crawl request.", - "type": "string" - }, - "index": { - "description": "The index of the crawled page in the results.", - "type": "integer" - }, - "markdown": { - "description": "Markdown content of the page.", - "type": "string" - }, - "metadata": { - "description": "Metadata extracted from the page.", - "properties": { - " ": { - "description": "Any other extracted metadata.", - "type": "string" - }, - "description": { - "description": "Page description.", - "type": "string" - }, - "language": { - "description": "Page language.", - "type": "string" - }, - "pageError": { - "description": "Error message if there was an error scraping the page.", - "type": "string" - }, - "pageStatusCode": { - "description": "HTTP status code of the page.", - "type": "integer" - }, - "sourceURL": { - "description": "Source URL of the page.", - "type": "string" - }, - "title": { - "description": "Page title.", - "type": "string" - } - }, - "type": "object" - }, - "rawHtml": { - "description": "Raw HTML content of the page, only present if `includeRawHtml` was set to `true` in the crawl request.", - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "status": { - "description": "Status of the crawl job. Can be 'completed', 'active', 'failed', or 'paused'.", - "enum": [ - "completed", - "active", - "failed", - "paused" - ], - "type": "string" - }, - "total": { - "description": "The total estimated number of pages to crawl.", - "type": "integer" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job status retrieved." - } - }, - "summary": "Get Crawl Job Status" - } - }, - "delete": { - "description": "Send a request to cancel a running crawl job.", - "operationId": "cancelCrawlJob", - "parameters": [ - { - "description": "ID of the crawl job to cancel.", - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": {} - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "status": { - "description": "The status of the crawl job cancellation request, usually 'cancelled'.", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job cancellation request submitted." - }, - "402": { - "description": "Payment required." - }, - "429": { - "description": "Rate limit exceeded." - }, - "500": { - "description": "Internal server error." - } - }, - "summary": "Cancel a Crawl Job" - } - }, - "description": "Send a request to crawl a URL and all accessible subpages. This submits a crawl job and returns a job ID to check the status of the crawl.", - "operationId": "crawlWebsite", - "parameters": [], - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "description": "Options for controlling the crawling behavior.", - "properties": { - "allowBackwardCrawling": { - "default": false, - "description": "Enables the crawler to navigate from a specific URL to previously linked pages. For instance, from 'example.com/product/123' back to 'example.com/product'", - "type": "boolean" - }, - "allowExternalContentLinks": { - "default": false, - "description": "Allows the crawler to follow links to external websites.", - "type": "boolean" - }, - "excludes": { - "description": "URL patterns to exclude", - "items": { - "type": "string" - }, - "type": "array" - }, - "generateImgAltText": { - "default": false, - "description": "Generate alt text for images using LLMs (must have a paid plan)", - "type": "boolean" - }, - "ignoreSitemap": { - "default": false, - "description": "Ignore the website sitemap when crawling", - "type": "boolean" - }, - "includes": { - "description": "URL patterns to include", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "default": 10000, - "description": "Maximum number of pages to crawl", - "type": "integer" - }, - "maxDepth": { - "description": "Maximum depth to crawl relative to the entered URL. A maxDepth of 0 scrapes only the entered URL. A maxDepth of 1 scrapes the entered URL and all pages one level deep. A maxDepth of 2 scrapes the entered URL and all pages up to two levels deep. Higher values follow the same pattern.", - "type": "integer" - }, - "mode": { - "default": "default", - "description": "The crawling mode to use. Fast mode crawls 4x faster websites without sitemap, but may not be as accurate and shouldn't be used in heavy js-rendered websites.", - "enum": [ - "default", - "fast" - ], - "type": "string" - }, - "returnOnlyUrls": { - "default": false, - "description": "If true, returns only the URLs as a list on the crawl status. Attention: the return response will be a list of URLs inside the data, not a list of documents.", - "type": "boolean" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Options for controlling the scraping behavior of individual pages.", - "properties": { - "fullPageScreenshot": { - "default": false, - "description": "Include a full page screenshot of the page that you are scraping.", - "type": "boolean" - }, - "headers": { - "description": "Headers to send with the request. Can be used to send cookies, user-agent, etc.", - "type": "object" - }, - "includeHtml": { - "default": false, - "description": "Include the HTML version of the content on page. Will output a html key in the response.", - "type": "boolean" - }, - "includeRawHtml": { - "default": false, - "description": "Include the raw HTML content of the page. Will output a rawHtml key in the response.", - "type": "boolean" - }, - "onlyIncludeTags": { - "description": "Only include tags, classes and ids from the page in the final output. Use comma separated values. Example: 'script, .ad, #footer'", - "items": { - "type": "string" - }, - "type": "array" - }, - "onlyMainContent": { - "default": false, - "description": "Only return the main content of the page excluding headers, navs, footers, etc.", - "type": "boolean" - }, - "removeTags": { - "description": "Tags, classes and ids to remove from the page. Use comma separated values. Example: 'script, .ad, #footer'", - "items": { - "type": "string" - }, - "type": "array" - }, - "replaceAllPathsWithAbsolutePaths": { - "default": false, - "description": "Replace all relative paths with absolute paths for images and links", - "type": "boolean" - }, - "screenshot": { - "default": false, - "description": "Include a screenshot of the top of the page that you are scraping.", - "type": "boolean" - }, - "waitFor": { - "default": 0, - "description": "Wait x amount of milliseconds for the page to load to fetch content", - "type": "integer" - } - }, - "type": "object" - }, - "url": { - "description": "The base URL to start crawling from", - "required": true, - "type": "string" - } - }, - "type": "object" - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "The ID of the submitted crawl job.", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job submitted successfully." - }, - "402": { - "description": "Payment required." - }, - "429": { - "description": "Rate limit exceeded." - }, - "500": { - "description": "Internal server error." - } - } - }, - "summary": "Crawl a Website" - } - } - }, - "servers": [ - { - "url": "https://api.firecrawl.dev/v0" - } - ] -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/combined_api_spec.json b/examples/turning_docs_into_api_specs/combined_api_spec.json new file mode 100644 index 00000000..526dec8b --- /dev/null +++ b/examples/turning_docs_into_api_specs/combined_api_spec.json @@ -0,0 +1,510 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "https://docs.firecrawl.dev/api-reference API Specification", + "version": "1.0.0" + }, + "paths": { + "/crawl": { + "post": { + "summary": "Crawl a website", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "Base URL to crawl" + }, + "excludePaths": { + "type": "array", + "items": { + "type": "string" + }, + "description": "URL patterns to exclude" + }, + "includePaths": { + "type": "array", + "items": { + "type": "string" + }, + "description": "URL patterns to include" + }, + "maxDepth": { + "type": "integer", + "description": "Maximum crawl depth" + }, + "ignoreSitemap": { + "type": "boolean", + "description": "Ignore sitemap?" + }, + "limit": { + "type": "integer", + "description": "Maximum pages to crawl" + }, + "allowBackwardLinks": { + "type": "boolean", + "description": "Allow backward links?" + }, + "allowExternalLinks": { + "type": "boolean", + "description": "Allow external links?" + }, + "webhook": { + "type": "string", + "description": "Webhook URL" + }, + "scrapeOptions": { + "type": "object", + "properties": { + "formats": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Formats to include" + }, + "headers": { + "type": "object", + "description": "Headers to send" + }, + "includeTags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tags to include" + }, + "excludeTags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tags to exclude" + }, + "onlyMainContent": { + "type": "boolean", + "description": "Only main content?" + }, + "waitFor": { + "type": "integer", + "description": "Wait time in ms" + } + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Crawl started", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + } + } + } + }, + "security": [ + { + "Authorization": [] + } + ] + } + }, + "/scrape": { + "post": { + "summary": "Scrape a webpage", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "URL to scrape" + }, + "formats": { + "type": "array", + "description": "Output formats", + "items": { + "type": "string", + "enum": [ + "markdown", + "html", + "rawHtml", + "links", + "screenshot", + "extract", + "screenshot@fullPage" + ] + } + }, + "onlyMainContent": { + "type": "boolean", + "description": "Only main content" + }, + "includeTags": { + "type": "array", + "description": "Tags to include", + "items": { + "type": "string" + } + }, + "excludeTags": { + "type": "array", + "description": "Tags to exclude", + "items": { + "type": "string" + } + }, + "headers": { + "type": "object", + "description": "Request headers" + }, + "waitFor": { + "type": "integer", + "description": "Delay in ms" + }, + "timeout": { + "type": "integer", + "description": "Timeout in ms" + }, + "extract": { + "type": "object", + "description": "Extract object", + "properties": { + "schema": { + "type": "object", + "description": "Extraction schema" + }, + "systemPrompt": { + "type": "string", + "description": "System prompt" + }, + "prompt": { + "type": "string", + "description": "Extraction prompt" + } + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Successful scrape", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "markdown": { + "type": "string" + }, + "html": { + "type": "string" + }, + "rawHtml": { + "type": "string" + }, + "screenshot": { + "type": "string" + }, + "links": { + "type": "array", + "items": { + "type": "string" + } + }, + "metadata": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "sourceURL": { + "type": "string" + }, + "statusCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "llm_extraction": { + "type": "object" + }, + "warning": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/v1/crawl/{id}": { + "get": { + "summary": "Get crawl status", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "ID of crawl job", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Crawl status", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "status": { + "type": "string", + "description": "Current status of crawl" + }, + "total": { + "type": "integer", + "description": "Total pages crawled" + }, + "completed": { + "type": "integer", + "description": "Number of pages crawled" + }, + "creditsUsed": { + "type": "integer", + "description": "Credits used" + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "description": "Crawl expiry" + }, + "next": { + "type": "string", + "nullable": true, + "description": "URL for next data" + }, + "data": { + "type": "array", + "description": "Data of the crawl", + "items": { + "type": "object", + "properties": { + "markdown": { + "type": "string" + }, + "html": { + "type": "string" + }, + "rawHtml": { + "type": "string" + }, + "links": { + "type": "array", + "items": { + "type": "string" + } + }, + "screenshot": { + "type": "string" + }, + "metadata": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "language": { + "type": "string" + }, + "sourceURL": { + "type": "string" + }, + "statusCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + }, + "security": [ + { + "Bearer": [] + } + ] + } + }, + "/crawl/{id}": { + "delete": { + "summary": "Cancel crawl job", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "description": "ID of crawl job", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Crawl job cancelled", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/map": { + "post": { + "summary": "Map website and return links", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "Base URL to crawl" + }, + "search": { + "type": "string", + "description": "Search query for mapping" + }, + "ignoreSitemap": { + "type": "boolean", + "description": "Ignore sitemap?" + }, + "includeSubdomains": { + "type": "boolean", + "description": "Include subdomains?" + }, + "limit": { + "type": "integer", + "description": "Max links to return" + } + }, + "required": [ + "url" + ] + } + } + } + }, + "responses": { + "200": { + "description": "Successful mapping", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "links": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + }, + "components": { + "schemas": {} + } +} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/dify_api_spec.json b/examples/turning_docs_into_api_specs/dify_api_spec.json deleted file mode 100644 index e6eec457..00000000 --- a/examples/turning_docs_into_api_specs/dify_api_spec.json +++ /dev/null @@ -1,164 +0,0 @@ -{ - "openapi": "3.0.0", - "info": { - "title": "Knowledge Base API", - "description": "API for managing knowledge bases and documents." - }, - "paths": { - "/datasets": { - "post": { - "summary": "Create an Empty Dataset", - "description": "Only used to create an empty dataset", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - } - } - } - } - } - }, - "responses": {} - }, - "get": { - "summary": "Dataset List", - "parameters": [ - { - "name": "page", - "in": "query", - "schema": { - "type": "integer" - } - }, - { - "name": "limit", - "in": "query", - "schema": { - "type": "integer" - } - } - ], - "responses": {} - } - }, - "/datasets/{dataset_id}/document/create_by_text": { - "post": { - "summary": "Create Document by Text", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "text": { - "type": "string" - }, - "indexing_technique": { - "type": "string" - }, - "process_rule": { - "type": "object" - } - } - } - } - } - }, - "responses": {} - } - }, - "/datasets/{dataset_id}/document/create_by_file": { - "post": { - "summary": "Create Document by File", - "requestBody": { - "content": { - "multipart/form-data": { - "schema": { - "type": "object", - "properties": { - "data": { - "type": "string" - }, - "file": { - "type": "string", - "format": "binary" - } - } - } - } - } - }, - "responses": {} - } - }, - "/datasets/{dataset_id}/documents/{batch}/indexing-status": { - "get": { - "summary": "Get Document Embedding Status (Progress)", - "responses": {} - } - }, - "/datasets/{dataset_id}/documents/{document_id}": { - "delete": { - "summary": "Delete Document", - "responses": {} - } - }, - "/datasets/{dataset_id}/documents": { - "get": { - "summary": "Dataset Document List", - "responses": {} - } - }, - "/datasets/{dataset_id}/documents/{document_id}/segments": { - "post": { - "summary": "Add Segments", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "segments": { - "type": "array", - "items": { - "type": "object", - "properties": { - "content": { - "type": "string" - }, - "answer": { - "type": "string" - }, - "keywords": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - } - }, - "responses": {} - } - }, - "/datasets/{dataset_id}/segments/{segment_id}": { - "delete": { - "summary": "Delete Document Segment", - "responses": {} - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_0.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_0.json deleted file mode 100644 index 84bce02c..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_0.json +++ /dev/null @@ -1,211 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/v0/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "description": "Crawling options.", - "properties": { - "excludes": { - "description": "URL patterns to exclude.", - "items": { - "type": "string" - }, - "type": "array" - }, - "includes": { - "description": "URL patterns to include.", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Maximum pages to crawl.", - "type": "integer" - }, - "maxDepth": { - "description": "Maximum crawl depth.", - "type": "integer" - }, - "mode": { - "description": "Crawling mode.", - "enum": [ - "default", - "fast" - ], - "type": "string" - }, - "returnOnlyUrls": { - "description": "Return only URLs.", - "type": "boolean" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Page scraping options.", - "properties": { - "includeHtml": { - "description": "Include HTML content.", - "type": "boolean" - }, - "includeRawHtml": { - "description": "Include raw HTML content.", - "type": "boolean" - }, - "onlyMainContent": { - "description": "Only main content.", - "type": "boolean" - }, - "screenshot": { - "description": "Include page screenshot.", - "type": "boolean" - }, - "waitFor": { - "description": "Wait time in milliseconds.", - "type": "integer" - } - }, - "type": "object" - }, - "url": { - "description": "Base URL to crawl.", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Crawl job ID.", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job initiated." - } - }, - "summary": "Crawl multiple pages." - } - }, - "/v0/crawl/status/{jobId}": { - "get": { - "parameters": [ - { - "description": "Crawl job ID.", - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Crawl job status." - } - }, - "summary": "Check crawl job status." - } - }, - "/v0/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "description": "Data extraction options.", - "properties": { - "extractionPrompt": { - "description": "Prompt for data extraction.", - "type": "string" - }, - "extractionSchema": { - "description": "Schema for data extraction.", - "type": "object" - }, - "mode": { - "description": "Extraction mode.", - "enum": [ - "llm-extraction", - "llm-extraction-from-raw-html" - ], - "type": "string" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Page scraping options.", - "properties": { - "includeHtml": { - "description": "Include HTML content.", - "type": "boolean" - }, - "includeRawHtml": { - "description": "Include raw HTML content.", - "type": "boolean" - }, - "onlyMainContent": { - "description": "Only main content.", - "type": "boolean" - }, - "screenshot": { - "description": "Include page screenshot.", - "type": "boolean" - }, - "waitFor": { - "description": "Wait time in milliseconds.", - "type": "integer" - } - }, - "type": "object" - }, - "timeout": { - "description": "Timeout in milliseconds.", - "type": "integer" - }, - "url": { - "description": "URL to scrape.", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "description": "Successful scraping." - } - }, - "summary": "Scrape a single page." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_1.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_1.json deleted file mode 100644 index 8656c978..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_1.json +++ /dev/null @@ -1,165 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "properties": { - "allowBackwardCrawling": { - "description": "Allow backward crawling", - "type": "boolean" - }, - "allowExternalContentLinks": { - "description": "Allow external links", - "type": "boolean" - }, - "excludes": { - "description": "URL patterns to exclude", - "items": { - "type": "string" - }, - "type": "array" - }, - "generateImgAltText": { - "description": "Generate alt text for images", - "type": "boolean" - }, - "ignoreSitemap": { - "description": "Ignore website sitemap", - "type": "boolean" - }, - "includes": { - "description": "URL patterns to include", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Maximum pages to crawl", - "type": "integer" - }, - "maxDepth": { - "description": "Maximum crawl depth", - "type": "integer" - }, - "mode": { - "description": "Crawling mode", - "enum": [ - "default", - "fast" - ], - "type": "string" - }, - "returnOnlyUrls": { - "description": "Return only crawled URLs", - "type": "boolean" - } - }, - "type": "object" - }, - "pageOptions": { - "properties": { - "fullPageScreenshot": { - "description": "Include full page screenshot", - "type": "boolean" - }, - "headers": { - "description": "Headers for requests", - "type": "object" - }, - "includeHtml": { - "description": "Include HTML content", - "type": "boolean" - }, - "includeRawHtml": { - "description": "Include raw HTML content", - "type": "boolean" - }, - "onlyIncludeTags": { - "description": "Include only specific tags", - "items": { - "type": "string" - }, - "type": "array" - }, - "onlyMainContent": { - "description": "Return only main content", - "type": "boolean" - }, - "removeTags": { - "description": "Remove specific tags", - "items": { - "type": "string" - }, - "type": "array" - }, - "replaceAllPathsWithAbsolutePaths": { - "description": "Use absolute paths", - "type": "boolean" - }, - "screenshot": { - "description": "Include page screenshot", - "type": "boolean" - }, - "waitFor": { - "description": "Wait for page load (ms)", - "type": "integer" - } - }, - "type": "object" - }, - "url": { - "description": "Base URL to crawl", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Job ID of the crawl", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl request successful" - } - }, - "security": [ - { - "Bearer": [] - } - ], - "summary": "Crawl a website" - } - } - }, - "securitySchemes": { - "Bearer": { - "bearerFormat": "JWT", - "scheme": "bearer", - "type": "http" - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_10.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_10.json deleted file mode 100644 index 55f73a32..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_10.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/check_crawl_status": { - "post": { - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "current": { - "type": "integer" - }, - "data": { - "items": { - "properties": { - "content": { - "type": "string" - }, - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "provider": { - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "status": { - "type": "string" - }, - "total": { - "type": "integer" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job status" - } - }, - "summary": "Check crawl job status" - } - }, - "/crawl": { - "post": { - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Job ID" - } - }, - "summary": "Crawl URL and subpages" - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_11.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_11.json deleted file mode 100644 index e19ed056..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_11.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "example": { - "extractorOptions": { - "extractionPrompt": "Based on the information on the page, extract the information from the schema. ", - "extractionSchema": { - "properties": { - "company_mission": { - "type": "string" - }, - "is_in_yc": { - "type": "boolean" - }, - "is_open_source": { - "type": "boolean" - }, - "supports_sso": { - "type": "boolean" - } - }, - "required": [ - "company_mission", - "supports_sso", - "is_open_source", - "is_in_yc" - ], - "type": "object" - }, - "mode": "llm-extraction" - }, - "url": "https://docs.firecrawl.dev/" - }, - "schema": { - "properties": { - "extractorOptions": { - "properties": { - "extractionPrompt": { - "description": "Prompt for extraction", - "type": "string" - }, - "extractionSchema": { - "description": "Schema for data extraction", - "type": "object" - }, - "mode": { - "description": "Extraction mode", - "type": "string" - } - }, - "type": "object" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "example": { - "data": { - "content": "Raw Content", - "llm_extraction": { - "company_mission": "Train a secure AI on your technical resources that answers customer and employee questions so your team doesn't have to", - "is_in_yc": true, - "is_open_source": false, - "supports_sso": true - }, - "metadata": { - "description": "Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide", - "ogDescription": "Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide", - "ogImage": "https://docs.firecrawl.dev/mendable_new_og1.png", - "ogLocaleAlternate": [], - "ogSiteName": "Mendable", - "ogTitle": "Mendable", - "ogUrl": "https://docs.firecrawl.dev/", - "robots": "follow, index", - "sourceURL": "https://docs.firecrawl.dev/", - "title": "Mendable" - } - }, - "success": true - }, - "schema": { - "properties": { - "data": { - "properties": { - "content": { - "type": "string" - }, - "llm_extraction": { - "type": "object" - }, - "metadata": { - "type": "object" - } - }, - "type": "object" - }, - "success": { - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Successful scrape" - } - }, - "summary": "Extract data from pages." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_13.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_13.json deleted file mode 100644 index 0352c66f..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_13.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/search": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "pageOptions": { - "properties": { - "fetchPageContent": { - "type": "boolean" - } - }, - "type": "object" - }, - "query": { - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "items": { - "properties": { - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "provider": { - "type": "string" - }, - "url": { - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "success": { - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Successful search and scrape." - } - }, - "summary": "Search web, scrape, return markdown." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_15.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_15.json deleted file mode 100644 index e7384f8e..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_15.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "url": { - "description": "Website URL to crawl.", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "items": { - "properties": { - "markdown": { - "description": "Markdown content.", - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - } - } - }, - "description": "Website crawled successfully." - } - }, - "summary": "Crawl a website." - } - }, - "/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "url": { - "description": "Page URL to scrape.", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "text/plain": { - "schema": { - "description": "Scraped content.", - "type": "string" - } - } - }, - "description": "Page scraped successfully." - } - }, - "summary": "Scrape a single page." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_16.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_16.json deleted file mode 100644 index ed6fb9d6..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_16.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawler_options": { - "properties": { - "exclude": { - "description": "URL patterns to exclude", - "items": { - "type": "string" - }, - "type": "array" - }, - "generateImgAltText": { - "description": "Generate alt text for images", - "type": "boolean" - }, - "includes": { - "description": "URL patterns to include", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Max pages to crawl", - "type": "integer" - }, - "maxDepth": { - "description": "Maximum crawl depth", - "type": "integer" - }, - "mode": { - "description": "Crawling mode", - "type": "string" - }, - "returnOnlyUrls": { - "description": "Return only URLs", - "type": "boolean" - }, - "timeout": { - "description": "Timeout in milliseconds", - "type": "integer" - } - }, - "type": "object" - }, - "page_options": { - "properties": { - "includeHtml": { - "description": "Include raw HTML", - "type": "boolean" - }, - "onlyMainContent": { - "description": "Only main content", - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "Base URL to crawl", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "description": "Crawl successful." - } - }, - "summary": "Crawl a website." - } - }, - "/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractor_options": { - "properties": { - "extractionPrompt": { - "description": "Prompt for extraction", - "type": "string" - }, - "extractionSchema": { - "description": "Schema for extraction", - "type": "string" - }, - "mode": { - "description": "Extraction mode", - "type": "string" - } - }, - "type": "object" - }, - "page_options": { - "properties": { - "includeHtml": { - "description": "Include raw HTML", - "type": "boolean" - }, - "onlyMainContent": { - "description": "Only main content", - "type": "boolean" - } - }, - "type": "object" - }, - "timeout": { - "description": "Timeout in milliseconds", - "type": "integer" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "description": "Scrape successful." - } - }, - "summary": "Scrape a website." - } - }, - "/search": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "page_options": { - "properties": { - "fetchPageContent": { - "description": "Fetch full content", - "type": "boolean" - }, - "includeHtml": { - "description": "Include raw HTML", - "type": "boolean" - }, - "onlyMainContent": { - "description": "Only main content", - "type": "boolean" - } - }, - "type": "object" - }, - "query": { - "description": "Search query string", - "type": "string" - }, - "search_options": { - "properties": { - "limit": { - "description": "Max results", - "type": "integer" - } - }, - "type": "object" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "description": "Search successful." - } - }, - "summary": "Search Firecrawl index." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_2.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_2.json deleted file mode 100644 index 25cf6c05..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_2.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/crawl/cancel/{jobId}": { - "delete": { - "parameters": [ - { - "description": "ID of crawl job", - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "status": { - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Returns cancelled." - } - }, - "security": [ - { - "Bearer": [] - } - ], - "summary": "Cancel crawl job" - } - } - }, - "securitySchemes": { - "Bearer": { - "bearerFormat": "Bearer ", - "scheme": "bearer", - "type": "http" - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_22.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_22.json deleted file mode 100644 index ac146a63..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_22.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/check-crawl-status/{jobId}": { - "get": { - "parameters": [ - { - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "current": { - "description": "Current progress", - "type": "integer" - }, - "data": { - "items": { - "properties": { - "content": { - "description": "Raw content", - "type": "string" - }, - "markdown": { - "description": "Markdown content", - "type": "string" - }, - "metadata": { - "description": "Page metadata", - "type": "object" - }, - "provider": { - "description": "Data provider", - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "status": { - "description": "Job status", - "type": "string" - }, - "total": { - "description": "Total pages", - "type": "integer" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job status." - } - }, - "summary": "Check crawl job status." - } - }, - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "description": "Crawler options", - "type": "object" - }, - "url": { - "description": "URL to crawl", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Job ID", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job submitted." - } - }, - "summary": "Crawl a URL." - } - }, - "/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "description": "Extractor options", - "type": "object" - }, - "pageOptions": { - "description": "Page options", - "type": "object" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "description": "Scraped data", - "type": "object" - }, - "success": { - "description": "Success flag", - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Scraped data." - } - }, - "summary": "Scrape a single URL." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_25.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_25.json deleted file mode 100644 index 9701a462..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_25.json +++ /dev/null @@ -1,229 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/v0/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "properties": { - "excludes": { - "description": "Paths to exclude", - "items": { - "type": "string" - }, - "type": "array" - }, - "includes": { - "description": "Paths to include", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Maximum pages to crawl", - "type": "integer" - }, - "maxDepth": { - "description": "Maximum crawl depth", - "type": "integer" - }, - "returnOnlyUrls": { - "description": "Only return URLs", - "type": "boolean" - } - }, - "type": "object" - }, - "pageOptions": { - "properties": { - "onlyMainContent": { - "description": "Extract main content", - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "URL to crawl", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Job ID", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job created" - } - }, - "summary": "Crawl a website" - } - }, - "/v0/crawl/status/{jobId}": { - "get": { - "parameters": [ - { - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "current": { - "type": "integer" - }, - "data": { - "items": { - "properties": { - "url": { - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "status": { - "description": "Job status", - "type": "string" - }, - "total": { - "type": "integer" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job status" - } - }, - "summary": "Get crawl job status" - } - }, - "/v0/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "pageOptions": { - "properties": { - "onlyMainContent": { - "description": "Extract main content", - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "properties": { - "content": { - "type": "string" - }, - "html": { - "type": "string" - }, - "llm_extraction": { - "type": "object" - }, - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "pageError": { - "type": "string" - }, - "pageStatusCode": { - "type": "integer" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "rawHtml": { - "type": "string" - }, - "warning": { - "type": "string" - } - }, - "type": "object" - }, - "success": { - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Scrape results" - } - }, - "summary": "Scrape a webpage" - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_26.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_26.json deleted file mode 100644 index b642e9c0..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_26.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "example": { - "extractorOptions": { - "extractionPrompt": "Extract company info.", - "extractionSchema": { - "properties": { - "company_description": { - "type": "string" - }, - "company_industry": { - "type": "string" - }, - "who_they_serve": { - "type": "string" - } - }, - "required": [ - "company_description", - "company_industry", - "who_they_serve" - ], - "type": "object" - }, - "mode": "llm-extraction" - }, - "pageOptions": { - "onlyMainContent": true - }, - "url": "https://example.com" - }, - "schema": { - "properties": { - "extractorOptions": { - "properties": { - "extractionPrompt": { - "description": "Prompt for LLM extraction.", - "type": "string" - }, - "extractionSchema": { - "properties": { - "properties": { - "company_description": { - "type": "string" - }, - "company_industry": { - "type": "string" - }, - "who_they_serve": { - "type": "string" - } - }, - "required": [ - "company_description", - "company_industry", - "who_they_serve" - ], - "type": { - "type": "string" - } - }, - "type": "object" - }, - "mode": { - "description": "Extraction mode.", - "type": "string" - } - }, - "type": "object" - }, - "pageOptions": { - "properties": { - "onlyMainContent": { - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "URL to scrape.", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - }, - "description": "Successful scrape." - } - }, - "summary": "Scrape data from URL." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_3.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_3.json deleted file mode 100644 index bcf94159..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_3.json +++ /dev/null @@ -1,185 +0,0 @@ -{ - "components": { - "securitySchemes": { - "bearerAuth": { - "scheme": "bearer", - "type": "http" - } - } - }, - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/v0/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "description": "Options for extraction", - "properties": { - "extractionPrompt": { - "description": "Prompt for LLM extraction", - "type": "string" - }, - "extractionSchema": { - "description": "Schema for LLM extraction", - "type": "object" - }, - "mode": { - "description": "Extraction mode", - "enum": [ - "markdown", - "llm-extraction", - "llm-extraction-from-raw-html", - "llm-extraction-from-markdown" - ], - "type": "string" - } - }, - "type": "object" - }, - "pageOptions": { - "properties": { - "fullPageScreenshot": { - "description": "Include full page screenshot", - "type": "boolean" - }, - "headers": { - "description": "Headers for request", - "type": "object" - }, - "includeHtml": { - "description": "Include HTML content", - "type": "boolean" - }, - "includeRawHtml": { - "description": "Include raw HTML content", - "type": "boolean" - }, - "onlyIncludeTags": { - "description": "Include only these tags", - "items": { - "type": "string" - }, - "type": "array" - }, - "onlyMainContent": { - "description": "Only return main content", - "type": "boolean" - }, - "removeTags": { - "description": "Remove these tags", - "items": { - "type": "string" - }, - "type": "array" - }, - "replaceAllPathsWithAbsolutePaths": { - "description": "Replace relative paths", - "type": "boolean" - }, - "screenshot": { - "description": "Include screenshot", - "type": "boolean" - }, - "waitFor": { - "description": "Wait time in ms", - "type": "integer" - } - }, - "type": "object" - }, - "timeout": { - "description": "Timeout in ms", - "type": "integer" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "properties": { - "content": { - "type": "string" - }, - "html": { - "type": "string" - }, - "llm_extraction": { - "type": "object" - }, - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "pageError": { - "type": "string" - }, - "pageStatusCode": { - "type": "integer" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "rawHtml": { - "type": "string" - }, - "warning": { - "type": "string" - } - }, - "type": "object" - }, - "success": { - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Successful scrape" - } - }, - "security": [ - { - "bearerAuth": [] - } - ], - "summary": "Scrape a webpage" - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_30.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_30.json deleted file mode 100644 index bc542e2a..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_30.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "description": "Crawl job options", - "properties": { - "excludes": { - "description": "Pages to exclude", - "items": { - "type": "string" - }, - "type": "array" - }, - "includes": { - "description": "Pages to include", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Max pages to crawl", - "type": "integer" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Page scraping options", - "properties": { - "onlyMainContent": { - "description": "Only scrape main content", - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "URL to crawl", - "type": "string" - } - }, - "required": [ - "url" - ], - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "description": "Crawl job result", - "type": "object" - } - } - }, - "description": "Crawl job result" - } - }, - "summary": "Crawl a website" - } - }, - "/crawl/{jobId}/cancel": { - "post": { - "parameters": [ - { - "description": "Crawl job ID", - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "description": "Cancellation status", - "type": "object" - } - } - }, - "description": "Cancellation status" - } - }, - "summary": "Cancel crawl job" - } - }, - "/crawl/{jobId}/status": { - "get": { - "parameters": [ - { - "description": "Crawl job ID", - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "description": "Crawl status", - "type": "object" - } - } - }, - "description": "Crawl status" - } - }, - "summary": "Check crawl status" - } - }, - "/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "description": "LLM extraction options", - "properties": { - "extractionSchema": { - "description": "JSON schema for extraction", - "type": "object" - } - }, - "type": "object" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "required": [ - "url" - ], - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "description": "Scraped data", - "type": "object" - } - } - }, - "description": "Scraped data" - } - }, - "summary": "Scrape a single URL" - } - }, - "/search": { - "get": { - "parameters": [ - { - "description": "Search query", - "in": "query", - "name": "query", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "description": "Search results", - "type": "object" - } - } - }, - "description": "Search results" - } - }, - "summary": "Search and scrape" - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_31.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_31.json deleted file mode 100644 index 07f71759..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_31.json +++ /dev/null @@ -1,199 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "properties": { - "excludes": { - "description": "Paths to exclude", - "items": { - "type": "string" - }, - "type": "array" - }, - "includes": { - "description": "Paths to include", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Maximum pages to crawl", - "type": "integer" - } - }, - "type": "object" - }, - "pageOptions": { - "properties": { - "onlyMainContent": { - "description": "Extract only main content", - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "Starting URL for crawl", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Unique job identifier", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job started" - } - }, - "summary": "Crawl a website" - } - }, - "/crawl/{jobId}/status": { - "get": { - "parameters": [ - { - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "status": { - "description": "Current job status", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job status" - } - }, - "summary": "Check crawl status" - } - }, - "/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "properties": { - "extractionSchema": { - "description": "Zod schema for extraction", - "type": "object" - } - }, - "type": "object" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "description": "Extracted data", - "type": "object" - } - }, - "type": "object" - } - } - }, - "description": "Scraped data" - } - }, - "summary": "Scrape a single URL" - } - }, - "/search": { - "get": { - "parameters": [ - { - "in": "query", - "name": "query", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "items": { - "properties": { - "content": { - "description": "Page content (optional)", - "type": "string" - }, - "url": { - "description": "Result URL", - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - } - } - }, - "description": "Search results" - } - }, - "summary": "Search for a query" - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_33.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_33.json deleted file mode 100644 index b45ae841..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_33.json +++ /dev/null @@ -1,202 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "description": "Options for crawling", - "properties": { - "excludes": { - "description": "URLs to exclude", - "items": { - "type": "string" - }, - "type": "array" - }, - "includes": { - "description": "URLs to include", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Maximum pages to crawl", - "type": "integer" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Options for page content", - "properties": { - "onlyMainContent": { - "description": "Extract only main content", - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "URL to crawl", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Unique crawl job ID", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job started." - } - }, - "summary": "Crawl a website." - } - }, - "/crawl/{jobId}": { - "get": { - "parameters": [ - { - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "status": { - "description": "Current job status", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job status." - } - }, - "summary": "Check crawl job status." - } - }, - "/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "description": "Options for data extraction", - "properties": { - "extractionSchema": { - "description": "Pydantic schema", - "type": "object" - }, - "mode": { - "description": "Extraction mode", - "type": "string" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Options for page content", - "properties": { - "onlyMainContent": { - "description": "Extract only main content", - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - }, - "description": "Scraped data." - } - }, - "summary": "Scrape a single URL." - } - }, - "/search": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "query": { - "description": "Search query", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - }, - "description": "Search results." - } - }, - "summary": "Search the web." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_34.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_34.json deleted file mode 100644 index 3bafda42..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_34.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "0.1" - }, - "openapi": "3.0.0", - "paths": { - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "description": "Crawl job options", - "properties": { - "excludes": { - "description": "URLs to exclude", - "items": { - "type": "string" - }, - "type": "array" - }, - "includes": { - "description": "URLs to include", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Maximum pages to crawl", - "type": "integer" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Page scraping options", - "properties": { - "onlyMainContent": { - "description": "Only scrape main content", - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "URL to crawl", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - }, - "description": "Crawl job started" - } - }, - "summary": "Crawl a website." - } - }, - "/crawl/{job_id}/cancel": { - "post": { - "parameters": [ - { - "description": "Crawl job ID", - "in": "path", - "name": "job_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - }, - "description": "Cancellation status" - } - }, - "summary": "Cancel crawl job." - } - }, - "/crawl/{job_id}/status": { - "get": { - "parameters": [ - { - "description": "Crawl job ID", - "in": "path", - "name": "job_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - }, - "description": "Crawl status" - } - }, - "summary": "Check crawl status." - } - }, - "/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "description": "LLM extraction options", - "properties": { - "extractionSchema": { - "description": "JSON schema for extraction", - "type": "object" - } - }, - "type": "object" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - }, - "description": "Scraped data" - } - }, - "summary": "Scrape a single URL." - } - }, - "/search": { - "get": { - "parameters": [ - { - "description": "Search query", - "in": "query", - "name": "query", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - }, - "description": "Search results" - } - }, - "summary": "Search and scrape results." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_35.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_35.json deleted file mode 100644 index 890d31b1..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_35.json +++ /dev/null @@ -1,245 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/check-crawl-status": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Crawl job ID", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "current": { - "description": "Current page count", - "type": "integer" - }, - "data": { - "description": "Crawl data", - "items": { - "properties": { - "content": { - "description": "Raw content", - "type": "string" - }, - "markdown": { - "description": "Markdown content", - "type": "string" - }, - "metadata": { - "description": "Page metadata", - "properties": { - "description": { - "description": "Page description", - "type": "string" - }, - "language": { - "description": "Page language", - "type": "string" - }, - "sourceURL": { - "description": "Page URL", - "type": "string" - }, - "title": { - "description": "Page title", - "type": "string" - } - }, - "type": "object" - }, - "provider": { - "description": "Content provider", - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "status": { - "description": "Crawl status", - "type": "string" - }, - "total": { - "description": "Total page count", - "type": "integer" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job status." - } - }, - "summary": "Check crawl job status." - } - }, - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "description": "Crawler options", - "properties": { - "excludes": { - "description": "URLs to exclude", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "url": { - "description": "URL to crawl", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Job ID", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job submitted." - } - }, - "summary": "Crawl a URL." - } - }, - "/scrape-url": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "description": "Extractor options", - "properties": { - "extractionSchema": { - "description": "Extraction schema", - "type": "string" - }, - "mode": { - "description": "Extraction mode", - "type": "string" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Page options", - "properties": { - "onlyMainContent": { - "description": "Only main content", - "type": "boolean" - } - }, - "type": "object" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "description": "Scraped data", - "properties": { - "content": { - "description": "Raw content", - "type": "string" - }, - "html": { - "description": "HTML content", - "type": "string" - }, - "llm_extraction": { - "description": "LLM extraction results", - "type": "object" - }, - "markdown": { - "description": "Markdown content", - "type": "string" - }, - "metadata": { - "description": "Page metadata", - "type": "object" - }, - "rawHtml": { - "description": "Raw HTML content", - "type": "string" - }, - "warning": { - "description": "Warning message", - "type": "string" - } - }, - "type": "object" - }, - "success": { - "description": "Request success", - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Scraped data." - } - }, - "summary": "Scrape a single URL." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_4.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_4.json deleted file mode 100644 index daf53932..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_4.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "components": { - "securitySchemes": { - "Bearer": { - "scheme": "bearer", - "type": "http" - } - } - }, - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/search": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "pageOptions": { - "properties": { - "fetchPageContent": { - "description": "Fetch content of each page.", - "type": "boolean" - }, - "includeHtml": { - "description": "Include HTML content.", - "type": "boolean" - }, - "includeRawHtml": { - "description": "Include raw HTML content.", - "type": "boolean" - }, - "onlyMainContent": { - "description": "Only return main content.", - "type": "boolean" - } - }, - "type": "object" - }, - "query": { - "description": "The query to search for", - "type": "string" - }, - "searchOptions": { - "properties": { - "limit": { - "description": "Maximum number of results.", - "type": "integer" - } - }, - "type": "object" - } - }, - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "items": { - "properties": { - "content": { - "type": "string" - }, - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "url": { - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "success": { - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Successful search." - } - }, - "security": [ - { - "Bearer": [] - } - ], - "summary": "Search the web." - } - } - }, - "servers": [ - { - "url": "https://api.firecrawl.dev/v0" - } - ] -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_5.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_5.json deleted file mode 100644 index 4fae28c0..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_5.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/crawl/status/{jobId}": { - "get": { - "parameters": [ - { - "description": "ID of crawl job", - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "current": { - "description": "Current page number", - "type": "integer" - }, - "data": { - "description": "Data from the job", - "items": { - "properties": { - "content": { - "type": "string" - }, - "html": { - "description": "HTML content", - "nullable": true, - "type": "string" - }, - "index": { - "description": "Page number crawled", - "type": "integer" - }, - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "nullable": true, - "type": "string" - }, - "pageError": { - "description": "Error message of page", - "nullable": true, - "type": "string" - }, - "pageStatusCode": { - "description": "Status code of page", - "type": "integer" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - }, - "{any other metadata}": { - "type": "string" - } - }, - "type": "object" - }, - "rawHtml": { - "description": "Raw HTML content", - "nullable": true, - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "partial_data": { - "description": "Partial documents (streaming)", - "items": { - "properties": { - "content": { - "type": "string" - }, - "html": { - "description": "HTML content", - "nullable": true, - "type": "string" - }, - "index": { - "description": "Page number crawled", - "type": "integer" - }, - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "nullable": true, - "type": "string" - }, - "pageError": { - "description": "Error message of page", - "nullable": true, - "type": "string" - }, - "pageStatusCode": { - "description": "Status code of page", - "type": "integer" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - }, - "{any other metadata}": { - "type": "string" - } - }, - "type": "object" - }, - "rawHtml": { - "description": "Raw HTML content", - "nullable": true, - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "status": { - "description": "Status of the job", - "type": "string" - }, - "total": { - "description": "Total number of pages", - "type": "integer" - } - }, - "type": "object" - } - } - }, - "description": "Successful operation" - } - }, - "security": [ - { - "Authorization": [] - } - ], - "summary": "Get crawl job status" - } - } - }, - "securitySchemes": { - "Authorization": { - "bearerFormat": "Bearer ", - "scheme": "bearer", - "type": "http" - } - }, - "servers": [ - { - "url": "https://api.firecrawl.dev/v0" - } - ] -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_7.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_7.json deleted file mode 100644 index b74b9886..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_7.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/v0/search": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "pageOptions": { - "properties": { - "fetchPageContent": { - "description": "Fetch page content", - "type": "boolean" - } - }, - "type": "object" - }, - "query": { - "description": "Search term", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "items": { - "properties": { - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "url": { - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "success": { - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Successful search" - } - }, - "summary": "Search and extract content" - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_8.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_8.json deleted file mode 100644 index 2d5f40e2..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/api_spec_8.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "info": { - "title": "Firecrawl API", - "version": "v0" - }, - "openapi": "3.0.0", - "paths": { - "/test": { - "get": { - "description": "Returns a test message.", - "responses": { - "200": { - "content": { - "text/plain": { - "schema": { - "example": "Hello, world!", - "type": "string" - } - } - }, - "description": "Successful operation" - } - }, - "summary": "Test endpoint" - } - }, - "/v0/crawl": { - "post": { - "description": "Processes crawl job for URL.", - "requestBody": { - "content": { - "application/json": { - "example": { - "url": "https://docs.firecrawl.dev" - }, - "schema": { - "properties": { - "url": { - "description": "Website URL", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "URL to crawl", - "required": true - }, - "responses": { - "200": { - "description": "Crawl initiated." - } - }, - "summary": "Crawl a given URL." - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/combined_api_spec.json b/examples/turning_docs_into_api_specs/docs.firecrawl.dev/combined_api_spec.json deleted file mode 100644 index 77d67234..00000000 --- a/examples/turning_docs_into_api_specs/docs.firecrawl.dev/combined_api_spec.json +++ /dev/null @@ -1,738 +0,0 @@ -{ - "components": { - "schemas": {} - }, - "info": { - "title": "https://docs.firecrawl.dev API Specification", - "version": "1.0.0" - }, - "openapi": "3.0.0", - "paths": { - "/check_crawl_status": { - "post": { - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "current": { - "type": "integer" - }, - "data": { - "items": { - "properties": { - "content": { - "type": "string" - }, - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "provider": { - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "status": { - "type": "string" - }, - "total": { - "type": "integer" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job status" - } - }, - "summary": "Check crawl job status" - } - }, - "/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "properties": { - "allowBackwardCrawling": { - "description": "Allow backward crawling", - "type": "boolean" - }, - "allowExternalContentLinks": { - "description": "Allow external links", - "type": "boolean" - }, - "excludes": { - "description": "URL patterns to exclude", - "items": { - "type": "string" - }, - "type": "array" - }, - "generateImgAltText": { - "description": "Generate alt text for images", - "type": "boolean" - }, - "ignoreSitemap": { - "description": "Ignore website sitemap", - "type": "boolean" - }, - "includes": { - "description": "URL patterns to include", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Maximum pages to crawl", - "type": "integer" - }, - "maxDepth": { - "description": "Maximum crawl depth", - "type": "integer" - }, - "mode": { - "description": "Crawling mode", - "enum": [ - "default", - "fast" - ], - "type": "string" - }, - "returnOnlyUrls": { - "description": "Return only crawled URLs", - "type": "boolean" - } - }, - "type": "object" - }, - "pageOptions": { - "properties": { - "fullPageScreenshot": { - "description": "Include full page screenshot", - "type": "boolean" - }, - "headers": { - "description": "Headers for requests", - "type": "object" - }, - "includeHtml": { - "description": "Include HTML content", - "type": "boolean" - }, - "includeRawHtml": { - "description": "Include raw HTML content", - "type": "boolean" - }, - "onlyIncludeTags": { - "description": "Include only specific tags", - "items": { - "type": "string" - }, - "type": "array" - }, - "onlyMainContent": { - "description": "Return only main content", - "type": "boolean" - }, - "removeTags": { - "description": "Remove specific tags", - "items": { - "type": "string" - }, - "type": "array" - }, - "replaceAllPathsWithAbsolutePaths": { - "description": "Use absolute paths", - "type": "boolean" - }, - "screenshot": { - "description": "Include page screenshot", - "type": "boolean" - }, - "waitFor": { - "description": "Wait for page load (ms)", - "type": "integer" - } - }, - "type": "object" - }, - "url": { - "description": "Base URL to crawl", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Job ID of the crawl", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl request successful" - } - }, - "security": [ - { - "Bearer": [] - } - ], - "summary": "Crawl a website" - } - }, - "/crawl/cancel/{jobId}": { - "delete": { - "parameters": [ - { - "description": "ID of crawl job", - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "status": { - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Returns cancelled." - } - }, - "security": [ - { - "Bearer": [] - } - ], - "summary": "Cancel crawl job" - } - }, - "/search": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "pageOptions": { - "properties": { - "fetchPageContent": { - "description": "Fetch content of each page.", - "type": "boolean" - }, - "includeHtml": { - "description": "Include HTML content.", - "type": "boolean" - }, - "includeRawHtml": { - "description": "Include raw HTML content.", - "type": "boolean" - }, - "onlyMainContent": { - "description": "Only return main content.", - "type": "boolean" - } - }, - "type": "object" - }, - "query": { - "description": "The query to search for", - "type": "string" - }, - "searchOptions": { - "properties": { - "limit": { - "description": "Maximum number of results.", - "type": "integer" - } - }, - "type": "object" - } - }, - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "items": { - "properties": { - "content": { - "type": "string" - }, - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "url": { - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "success": { - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Successful search." - } - }, - "security": [ - { - "Bearer": [] - } - ], - "summary": "Search the web." - } - }, - "/v0/crawl": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "crawlerOptions": { - "description": "Crawling options.", - "properties": { - "excludes": { - "description": "URL patterns to exclude.", - "items": { - "type": "string" - }, - "type": "array" - }, - "includes": { - "description": "URL patterns to include.", - "items": { - "type": "string" - }, - "type": "array" - }, - "limit": { - "description": "Maximum pages to crawl.", - "type": "integer" - }, - "maxDepth": { - "description": "Maximum crawl depth.", - "type": "integer" - }, - "mode": { - "description": "Crawling mode.", - "enum": [ - "default", - "fast" - ], - "type": "string" - }, - "returnOnlyUrls": { - "description": "Return only URLs.", - "type": "boolean" - } - }, - "type": "object" - }, - "pageOptions": { - "description": "Page scraping options.", - "properties": { - "includeHtml": { - "description": "Include HTML content.", - "type": "boolean" - }, - "includeRawHtml": { - "description": "Include raw HTML content.", - "type": "boolean" - }, - "onlyMainContent": { - "description": "Only main content.", - "type": "boolean" - }, - "screenshot": { - "description": "Include page screenshot.", - "type": "boolean" - }, - "waitFor": { - "description": "Wait time in milliseconds.", - "type": "integer" - } - }, - "type": "object" - }, - "url": { - "description": "Base URL to crawl.", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "jobId": { - "description": "Crawl job ID.", - "type": "string" - } - }, - "type": "object" - } - } - }, - "description": "Crawl job initiated." - } - }, - "summary": "Crawl multiple pages." - } - }, - "/v0/crawl/status/{jobId}": { - "get": { - "parameters": [ - { - "description": "Crawl job ID.", - "in": "path", - "name": "jobId", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Crawl job status." - } - }, - "summary": "Check crawl job status." - } - }, - "/v0/scrape": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "extractorOptions": { - "description": "Options for extraction", - "properties": { - "extractionPrompt": { - "description": "Prompt for LLM extraction", - "type": "string" - }, - "extractionSchema": { - "description": "Schema for LLM extraction", - "type": "object" - }, - "mode": { - "description": "Extraction mode", - "enum": [ - "markdown", - "llm-extraction", - "llm-extraction-from-raw-html", - "llm-extraction-from-markdown" - ], - "type": "string" - } - }, - "type": "object" - }, - "pageOptions": { - "properties": { - "fullPageScreenshot": { - "description": "Include full page screenshot", - "type": "boolean" - }, - "headers": { - "description": "Headers for request", - "type": "object" - }, - "includeHtml": { - "description": "Include HTML content", - "type": "boolean" - }, - "includeRawHtml": { - "description": "Include raw HTML content", - "type": "boolean" - }, - "onlyIncludeTags": { - "description": "Include only these tags", - "items": { - "type": "string" - }, - "type": "array" - }, - "onlyMainContent": { - "description": "Only return main content", - "type": "boolean" - }, - "removeTags": { - "description": "Remove these tags", - "items": { - "type": "string" - }, - "type": "array" - }, - "replaceAllPathsWithAbsolutePaths": { - "description": "Replace relative paths", - "type": "boolean" - }, - "screenshot": { - "description": "Include screenshot", - "type": "boolean" - }, - "waitFor": { - "description": "Wait time in ms", - "type": "integer" - } - }, - "type": "object" - }, - "timeout": { - "description": "Timeout in ms", - "type": "integer" - }, - "url": { - "description": "URL to scrape", - "type": "string" - } - }, - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "properties": { - "content": { - "type": "string" - }, - "html": { - "type": "string" - }, - "llm_extraction": { - "type": "object" - }, - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "pageError": { - "type": "string" - }, - "pageStatusCode": { - "type": "integer" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "rawHtml": { - "type": "string" - }, - "warning": { - "type": "string" - } - }, - "type": "object" - }, - "success": { - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Successful scrape" - } - }, - "security": [ - { - "bearerAuth": [] - } - ], - "summary": "Scrape a webpage" - } - }, - "/v0/search": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "pageOptions": { - "properties": { - "fetchPageContent": { - "description": "Fetch page content", - "type": "boolean" - } - }, - "type": "object" - }, - "query": { - "description": "Search term", - "type": "string" - } - }, - "type": "object" - } - } - } - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "data": { - "items": { - "properties": { - "markdown": { - "type": "string" - }, - "metadata": { - "properties": { - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "sourceURL": { - "type": "string" - }, - "title": { - "type": "string" - } - }, - "type": "object" - }, - "url": { - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "success": { - "type": "boolean" - } - }, - "type": "object" - } - } - }, - "description": "Successful search" - } - }, - "summary": "Search and extract content" - } - } - } -} \ No newline at end of file diff --git a/examples/turning_docs_into_api_specs/turning_docs_into_api_specs.ipynb b/examples/turning_docs_into_api_specs/turning_docs_into_api_specs.ipynb deleted file mode 100644 index 1b97f67b..00000000 --- a/examples/turning_docs_into_api_specs/turning_docs_into_api_specs.ipynb +++ /dev/null @@ -1,287 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/ericciarla/projects/python_projects/agents_testing/.conda/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "import os\n", - "import datetime\n", - "import time\n", - "from firecrawl import FirecrawlApp\n", - "import json\n", - "import google.generativeai as genai\n", - "from dotenv import load_dotenv\n", - "\n", - "# Load environment variables\n", - "load_dotenv()\n", - "\n", - "# Retrieve API keys from environment variables\n", - "google_api_key = os.getenv(\"GOOGLE_API_KEY\")\n", - "firecrawl_api_key = os.getenv(\"FIRECRAWL_API_KEY\")\n", - "\n", - "# Configure the Google Generative AI module with the API key\n", - "genai.configure(api_key=google_api_key)\n", - "model = genai.GenerativeModel(\"gemini-1.5-pro-001\")\n", - "\n", - "# Set the docs URL\n", - "docs_url=\"https://docs.firecrawl.dev\"\n", - "\n", - "# Initialize the FirecrawlApp with your API key\n", - "app = FirecrawlApp(api_key=firecrawl_api_key)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "36\n" - ] - } - ], - "source": [ - "# Crawl all pages on docs\n", - "params = {\n", - " \"pageOptions\": {\n", - " \"onlyMainContent\": True\n", - " },\n", - "}\n", - "crawl_result = app.crawl_url(docs_url, params=params)\n", - "\n", - "print(len(crawl_result))" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "prompt_instructions = f\"\"\"Given the following API documentation content, generate an OpenAPI 3.0 specification in JSON format ONLY if you are 100% confident and clear about all details. Focus on extracting the main endpoints, their HTTP methods, parameters, request bodies, and responses. The specification should follow OpenAPI 3.0 structure and conventions. Include only the 200 response for each endpoint. Limit all descriptions to 5 words or less.\n", - "\n", - "If there is ANY uncertainty, lack of complete information, or if you are not 100% confident about ANY part of the specification, return an empty JSON object {{}}.\n", - "\n", - "Do not make anything up. Only include information that is explicitly provided in the documentation. If any detail is unclear or missing, do not attempt to fill it in.\n", - "\n", - "API Documentation Content:\n", - "{{content}}\n", - "\n", - "Generate the OpenAPI 3.0 specification in JSON format ONLY if you are 100% confident about every single detail. Include only the JSON object, no additional text, and ensure it has no errors in the JSON format so it can be parsed. Remember to include only the 200 response for each endpoint and keep all descriptions to 5 words maximum.\n", - "\n", - "Once again, if there is ANY doubt, uncertainty, or lack of complete information, return an empty JSON object {{}}.\n", - "\n", - "To reiterate: accuracy is paramount. Do not make anything up. If you are not 100% clear or confident about the entire OpenAPI spec, return an empty JSON object {{}}.\n", - "\"\"\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "API specification saved to docs.firecrawl.dev/api_spec_0.json\n", - "API specification saved to docs.firecrawl.dev/api_spec_1.json\n", - "API specification saved to docs.firecrawl.dev/api_spec_2.json\n", - "API specification saved to docs.firecrawl.dev/api_spec_3.json\n", - "API specification saved to docs.firecrawl.dev/api_spec_4.json\n", - "An error occurred for page 5: 'content'\n", - "No API specification found for page 6\n", - "API specification saved to docs.firecrawl.dev/api_spec_7.json\n", - "No API specification found for page 8\n", - "No API specification found for page 9\n", - "API specification saved to docs.firecrawl.dev/api_spec_10.json\n", - "No API specification found for page 11\n", - "No API specification found for page 12\n", - "API specification saved to docs.firecrawl.dev/api_spec_13.json\n", - "No API specification found for page 14\n", - "No API specification found for page 15\n", - "No API specification found for page 16\n", - "No API specification found for page 17\n", - "No API specification found for page 18\n", - "No API specification found for page 19\n", - "No API specification found for page 20\n", - "No API specification found for page 21\n", - "No API specification found for page 22\n", - "No API specification found for page 23\n", - "No API specification found for page 24\n", - "No API specification found for page 25\n", - "No API specification found for page 26\n", - "No API specification found for page 27\n", - "No API specification found for page 28\n", - "No API specification found for page 29\n", - "No API specification found for page 30\n", - "No API specification found for page 31\n", - "No API specification found for page 32\n", - "No API specification found for page 33\n", - "No API specification found for page 34\n", - "No API specification found for page 35\n", - "Total API specifications collected: 8\n" - ] - } - ], - "source": [ - "# Create a folder for storing API specs\n", - "import os\n", - "import urllib.parse\n", - "\n", - "folder_name = urllib.parse.urlparse(docs_url).netloc\n", - "os.makedirs(folder_name, exist_ok=True)\n", - "\n", - "# Initialize a list to store all API specs\n", - "all_api_specs = []\n", - "\n", - "# Process each page in crawl_result\n", - "for index, result in enumerate(crawl_result):\n", - " if 'content' in result:\n", - " # Update prompt_instructions with the current page's content\n", - " current_prompt = prompt_instructions.replace(\"{content}\", result['content'])\n", - " try:\n", - " # Query the model\n", - " response = model.generate_content([current_prompt])\n", - " response_dict = response.to_dict()\n", - " response_text = response_dict['candidates'][0]['content']['parts'][0]['text']\n", - " \n", - " # Remove the ```json code wrap if present\n", - " response_text = response_text.strip().removeprefix('```json').removesuffix('```').strip()\n", - " \n", - " # Parse JSON\n", - " json_data = json.loads(response_text)\n", - " \n", - " # Save non-empty API specs\n", - " if json_data != {}:\n", - " output_file = os.path.join(folder_name, f'api_spec_{index}.json')\n", - " with open(output_file, 'w') as f:\n", - " json.dump(json_data, f, indent=2, sort_keys=True)\n", - " print(f\"API specification saved to {output_file}\")\n", - " \n", - " # Add the API spec to the list\n", - " all_api_specs.append(json_data)\n", - " else:\n", - " print(f\"No API specification found for page {index}\")\n", - " \n", - " except json.JSONDecodeError:\n", - " print(f\"Error parsing JSON response for page {index}\")\n", - " except Exception as e:\n", - " print(f\"An error occurred for page {index}: {str(e)}\")\n", - "\n", - "# Print the total number of API specs collected\n", - "print(f\"Total API specifications collected: {len(all_api_specs)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Combined API specification saved to docs.firecrawl.dev/combined_api_spec.json\n", - "Total paths in combined spec: 8\n", - "Total schemas in combined spec: 0\n" - ] - } - ], - "source": [ - "# Combine all API specs and keep the most filled out spec for each path and method\n", - "combined_spec = {\n", - " \"openapi\": \"3.0.0\",\n", - " \"info\": {\n", - " \"title\": f\"{docs_url} API Specification\",\n", - " \"version\": \"1.0.0\"\n", - " },\n", - " \"paths\": {},\n", - " \"components\": {\n", - " \"schemas\": {}\n", - " }\n", - "}\n", - "\n", - "def count_properties(obj):\n", - " if isinstance(obj, dict):\n", - " return sum(count_properties(v) for v in obj.values()) + len(obj)\n", - " elif isinstance(obj, list):\n", - " return sum(count_properties(item) for item in obj)\n", - " else:\n", - " return 1\n", - "\n", - "for spec in all_api_specs:\n", - " if \"paths\" in spec:\n", - " for path, methods in spec[\"paths\"].items():\n", - " if path not in combined_spec[\"paths\"]:\n", - " combined_spec[\"paths\"][path] = {}\n", - " for method, details in methods.items():\n", - " if method not in combined_spec[\"paths\"][path] or count_properties(details) > count_properties(combined_spec[\"paths\"][path][method]):\n", - " combined_spec[\"paths\"][path][method] = details\n", - "\n", - " if \"components\" in spec and \"schemas\" in spec[\"components\"]:\n", - " for schema_name, schema in spec[\"components\"][\"schemas\"].items():\n", - " if schema_name not in combined_spec[\"components\"][\"schemas\"] or count_properties(schema) > count_properties(combined_spec[\"components\"][\"schemas\"][schema_name]):\n", - " combined_spec[\"components\"][\"schemas\"][schema_name] = schema\n", - "\n", - "# Save the combined API spec\n", - "output_file = os.path.join(folder_name, 'combined_api_spec.json')\n", - "with open(output_file, 'w') as f:\n", - " json.dump(combined_spec, f, indent=2, sort_keys=True)\n", - "\n", - "print(f\"Combined API specification saved to {output_file}\")\n", - "print(f\"Total paths in combined spec: {len(combined_spec['paths'])}\")\n", - "print(f\"Total schemas in combined spec: {len(combined_spec['components']['schemas'])}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# note: turn this into a simple web app like roast my site\n", - "- select which methods you want to add\n", - "- generate a UI for each method\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/turning_docs_into_api_specs/turning_docs_into_api_specs.py b/examples/turning_docs_into_api_specs/turning_docs_into_api_specs.py new file mode 100644 index 00000000..47b54ede --- /dev/null +++ b/examples/turning_docs_into_api_specs/turning_docs_into_api_specs.py @@ -0,0 +1,137 @@ +# %% +import os +import datetime +import time +from firecrawl import FirecrawlApp +import json +import google.generativeai as genai +from dotenv import load_dotenv + +# Load environment variables +load_dotenv() + +# Retrieve API keys from environment variables +google_api_key = os.getenv("GOOGLE_API_KEY") +firecrawl_api_key = os.getenv("FIRECRAWL_API_KEY") + +# Configure the Google Generative AI module with the API key +genai.configure(api_key=google_api_key) +model = genai.GenerativeModel("gemini-1.5-pro-001") + +# Set the docs URL +docs_url = "https://docs.firecrawl.dev/api-reference" + +# Initialize the FirecrawlApp with your API key +app = FirecrawlApp(api_key=firecrawl_api_key) + +# %% +# Crawl all pages on docs +crawl_result = app.crawl_url(docs_url) +print(f"Total pages crawled: {len(crawl_result['data'])}") + +# %% +# Define the prompt instructions for generating OpenAPI specs +prompt_instructions = """ +Given the following API documentation content, generate an OpenAPI 3.0 specification in JSON format ONLY if you are 100% confident and clear about all details. Focus on extracting the main endpoints, their HTTP methods, parameters, request bodies, and responses. The specification should follow OpenAPI 3.0 structure and conventions. Include only the 200 response for each endpoint. Limit all descriptions to 5 words or less. + +If there is ANY uncertainty, lack of complete information, or if you are not 100% confident about ANY part of the specification, return an empty JSON object {{}}. + +Do not make anything up. Only include information that is explicitly provided in the documentation. If any detail is unclear or missing, do not attempt to fill it in. + +API Documentation Content: +{{content}} + +Generate the OpenAPI 3.0 specification in JSON format ONLY if you are 100% confident about every single detail. Include only the JSON object, no additional text, and ensure it has no errors in the JSON format so it can be parsed. Remember to include only the 200 response for each endpoint and keep all descriptions to 5 words maximum. + +Once again, if there is ANY doubt, uncertainty, or lack of complete information, return an empty JSON object {{}}. + +To reiterate: accuracy is paramount. Do not make anything up. If you are not 100% clear or confident about the entire OpenAPI spec, return an empty JSON object {{}}. +""" + +# %% +# Initialize a list to store all API specs +all_api_specs = [] + +# Process each page in crawl_result +for index, page in enumerate(crawl_result['data']): + if 'markdown' in page: + # Update prompt_instructions with the current page's content + current_prompt = prompt_instructions.replace("{content}", page['markdown']) + try: + # Query the model + response = model.generate_content([current_prompt]) + response_dict = response.to_dict() + response_text = response_dict['candidates'][0]['content']['parts'][0]['text'] + + # Remove the ```json code wrap if present + response_text = response_text.strip().removeprefix('```json').removesuffix('```').strip() + + # Parse JSON + json_data = json.loads(response_text) + + # Add non-empty API specs to the list + if json_data != {}: + all_api_specs.append(json_data) + print(f"API specification generated for page {index}") + else: + print(f"No API specification found for page {index}") + + except json.JSONDecodeError: + print(f"Error parsing JSON response for page {index}") + except Exception as e: + print(f"An error occurred for page {index}: {str(e)}") + +# Print the total number of API specs collected +print(f"Total API specifications collected: {len(all_api_specs)}") + +# %% +# Combine all API specs and keep the most filled out spec for each path and method +combined_spec = { + "openapi": "3.0.0", + "info": { + "title": f"{docs_url} API Specification", + "version": "1.0.0" + }, + "paths": {}, + "components": { + "schemas": {} + } +} + +# Helper function to count properties in an object +def count_properties(obj): + if isinstance(obj, dict): + return sum(count_properties(v) for v in obj.values()) + len(obj) + elif isinstance(obj, list): + return sum(count_properties(item) for item in obj) + else: + return 1 + +# Combine specs, keeping the most detailed version of each path and schema +for spec in all_api_specs: + # Combine paths + if "paths" in spec: + for path, methods in spec["paths"].items(): + if path not in combined_spec["paths"]: + combined_spec["paths"][path] = {} + for method, details in methods.items(): + if method not in combined_spec["paths"][path] or count_properties(details) > count_properties(combined_spec["paths"][path][method]): + combined_spec["paths"][path][method] = details + + # Combine schemas + if "components" in spec and "schemas" in spec["components"]: + for schema_name, schema in spec["components"]["schemas"].items(): + if schema_name not in combined_spec["components"]["schemas"] or count_properties(schema) > count_properties(combined_spec["components"]["schemas"][schema_name]): + combined_spec["components"]["schemas"][schema_name] = schema + +# Print summary of combined spec +print(f"Combined API specification generated") +print(f"Total paths in combined spec: {len(combined_spec['paths'])}") +print(f"Total schemas in combined spec: {len(combined_spec['components']['schemas'])}") + +# Save the combined spec to a JSON file in the same directory as the Python file +output_file = os.path.join(os.path.dirname(__file__), "combined_api_spec.json") +with open(output_file, "w") as f: + json.dump(combined_spec, f, indent=2) + +print(f"Combined API specification saved to {output_file}") From 2d245a35f2131cb2f00b759b92adb75d306a4447 Mon Sep 17 00:00:00 2001 From: Eric Ciarla Date: Fri, 6 Sep 2024 15:27:58 -0400 Subject: [PATCH 031/102] Delete combined_api_spec.json --- .../combined_api_spec.json | 510 ------------------ 1 file changed, 510 deletions(-) delete mode 100644 examples/turning_docs_into_api_specs/combined_api_spec.json diff --git a/examples/turning_docs_into_api_specs/combined_api_spec.json b/examples/turning_docs_into_api_specs/combined_api_spec.json deleted file mode 100644 index 526dec8b..00000000 --- a/examples/turning_docs_into_api_specs/combined_api_spec.json +++ /dev/null @@ -1,510 +0,0 @@ -{ - "openapi": "3.0.0", - "info": { - "title": "https://docs.firecrawl.dev/api-reference API Specification", - "version": "1.0.0" - }, - "paths": { - "/crawl": { - "post": { - "summary": "Crawl a website", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "Base URL to crawl" - }, - "excludePaths": { - "type": "array", - "items": { - "type": "string" - }, - "description": "URL patterns to exclude" - }, - "includePaths": { - "type": "array", - "items": { - "type": "string" - }, - "description": "URL patterns to include" - }, - "maxDepth": { - "type": "integer", - "description": "Maximum crawl depth" - }, - "ignoreSitemap": { - "type": "boolean", - "description": "Ignore sitemap?" - }, - "limit": { - "type": "integer", - "description": "Maximum pages to crawl" - }, - "allowBackwardLinks": { - "type": "boolean", - "description": "Allow backward links?" - }, - "allowExternalLinks": { - "type": "boolean", - "description": "Allow external links?" - }, - "webhook": { - "type": "string", - "description": "Webhook URL" - }, - "scrapeOptions": { - "type": "object", - "properties": { - "formats": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Formats to include" - }, - "headers": { - "type": "object", - "description": "Headers to send" - }, - "includeTags": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Tags to include" - }, - "excludeTags": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Tags to exclude" - }, - "onlyMainContent": { - "type": "boolean", - "description": "Only main content?" - }, - "waitFor": { - "type": "integer", - "description": "Wait time in ms" - } - } - } - } - } - } - } - }, - "responses": { - "200": { - "description": "Crawl started", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "success": { - "type": "boolean" - }, - "id": { - "type": "string" - }, - "url": { - "type": "string" - } - } - } - } - } - } - }, - "security": [ - { - "Authorization": [] - } - ] - } - }, - "/scrape": { - "post": { - "summary": "Scrape a webpage", - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "URL to scrape" - }, - "formats": { - "type": "array", - "description": "Output formats", - "items": { - "type": "string", - "enum": [ - "markdown", - "html", - "rawHtml", - "links", - "screenshot", - "extract", - "screenshot@fullPage" - ] - } - }, - "onlyMainContent": { - "type": "boolean", - "description": "Only main content" - }, - "includeTags": { - "type": "array", - "description": "Tags to include", - "items": { - "type": "string" - } - }, - "excludeTags": { - "type": "array", - "description": "Tags to exclude", - "items": { - "type": "string" - } - }, - "headers": { - "type": "object", - "description": "Request headers" - }, - "waitFor": { - "type": "integer", - "description": "Delay in ms" - }, - "timeout": { - "type": "integer", - "description": "Timeout in ms" - }, - "extract": { - "type": "object", - "description": "Extract object", - "properties": { - "schema": { - "type": "object", - "description": "Extraction schema" - }, - "systemPrompt": { - "type": "string", - "description": "System prompt" - }, - "prompt": { - "type": "string", - "description": "Extraction prompt" - } - } - } - } - } - } - } - }, - "responses": { - "200": { - "description": "Successful scrape", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "success": { - "type": "boolean" - }, - "data": { - "type": "object", - "properties": { - "markdown": { - "type": "string" - }, - "html": { - "type": "string" - }, - "rawHtml": { - "type": "string" - }, - "screenshot": { - "type": "string" - }, - "links": { - "type": "array", - "items": { - "type": "string" - } - }, - "metadata": { - "type": "object", - "properties": { - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "sourceURL": { - "type": "string" - }, - "statusCode": { - "type": "integer" - }, - "error": { - "type": "string" - } - } - }, - "llm_extraction": { - "type": "object" - }, - "warning": { - "type": "string" - } - } - } - } - } - } - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/v1/crawl/{id}": { - "get": { - "summary": "Get crawl status", - "parameters": [ - { - "name": "id", - "in": "path", - "description": "ID of crawl job", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Crawl status", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "status": { - "type": "string", - "description": "Current status of crawl" - }, - "total": { - "type": "integer", - "description": "Total pages crawled" - }, - "completed": { - "type": "integer", - "description": "Number of pages crawled" - }, - "creditsUsed": { - "type": "integer", - "description": "Credits used" - }, - "expiresAt": { - "type": "string", - "format": "date-time", - "description": "Crawl expiry" - }, - "next": { - "type": "string", - "nullable": true, - "description": "URL for next data" - }, - "data": { - "type": "array", - "description": "Data of the crawl", - "items": { - "type": "object", - "properties": { - "markdown": { - "type": "string" - }, - "html": { - "type": "string" - }, - "rawHtml": { - "type": "string" - }, - "links": { - "type": "array", - "items": { - "type": "string" - } - }, - "screenshot": { - "type": "string" - }, - "metadata": { - "type": "object", - "properties": { - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "language": { - "type": "string" - }, - "sourceURL": { - "type": "string" - }, - "statusCode": { - "type": "integer" - }, - "error": { - "type": "string" - } - } - } - } - } - } - } - } - } - } - } - }, - "security": [ - { - "Bearer": [] - } - ] - } - }, - "/crawl/{id}": { - "delete": { - "summary": "Cancel crawl job", - "security": [ - { - "bearerAuth": [] - } - ], - "parameters": [ - { - "name": "id", - "in": "path", - "description": "ID of crawl job", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Crawl job cancelled", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "success": { - "type": "boolean" - }, - "message": { - "type": "string" - } - } - } - } - } - } - } - } - }, - "/map": { - "post": { - "summary": "Map website and return links", - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "Base URL to crawl" - }, - "search": { - "type": "string", - "description": "Search query for mapping" - }, - "ignoreSitemap": { - "type": "boolean", - "description": "Ignore sitemap?" - }, - "includeSubdomains": { - "type": "boolean", - "description": "Include subdomains?" - }, - "limit": { - "type": "integer", - "description": "Max links to return" - } - }, - "required": [ - "url" - ] - } - } - } - }, - "responses": { - "200": { - "description": "Successful mapping", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "success": { - "type": "boolean" - }, - "links": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - } - } - }, - "components": { - "schemas": {} - } -} \ No newline at end of file From a0f9ab2be74b53fa0f5af8632c344900245a2b2b Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 6 Sep 2024 20:14:47 -0300 Subject: [PATCH 032/102] Update map.ts --- apps/api/src/controllers/v1/map.ts | 44 +++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts index e6abd9ae..9142f5c7 100644 --- a/apps/api/src/controllers/v1/map.ts +++ b/apps/api/src/controllers/v1/map.ts @@ -47,24 +47,42 @@ export async function mapController( const crawler = crawlToCrawler(id, sc); - const sitemap = req.body.ignoreSitemap ? null : await crawler.tryGetSitemap(); - - if (sitemap !== null) { - sitemap.map((x) => { - links.push(x.url); - }); - } - let urlWithoutWww = req.body.url.replace("www.", ""); let mapUrl = req.body.search ? `"${req.body.search}" site:${urlWithoutWww}` : `site:${req.body.url}`; - // www. seems to exclude subdomains in some cases - const mapResults = await fireEngineMap(mapUrl, { - // limit to 100 results (beta) - numResults: Math.min(limit, 100), - }); + + const maxResults = 5000; + const resultsPerPage = 100; + const maxPages = Math.ceil(maxResults / resultsPerPage); + + const fetchPage = async (page: number) => { + return fireEngineMap(mapUrl, { + numResults: resultsPerPage, + page: page + }); + }; + + const pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1)); + + // Parallelize sitemap fetch with serper search + const [sitemap, ...allResults] = await Promise.all([ + req.body.ignoreSitemap ? null : crawler.tryGetSitemap(), + ...pagePromises + ]); + + if (sitemap !== null) { + sitemap.forEach((x) => { + links.push(x.url); + }); + } + + let mapResults = allResults.flat().filter(result => result !== null && result !== undefined); + + if (mapResults.length > maxResults) { + mapResults = mapResults.slice(0, maxResults); + } if (mapResults.length > 0) { if (req.body.search) { From 79870e73053ef2a960112f2fa2227b39d0512bdb Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 6 Sep 2024 20:15:26 -0300 Subject: [PATCH 033/102] Update excludeTags.ts --- apps/api/src/scraper/WebScraper/utils/excludeTags.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/utils/excludeTags.ts b/apps/api/src/scraper/WebScraper/utils/excludeTags.ts index bb9c5194..400ef84f 100644 --- a/apps/api/src/scraper/WebScraper/utils/excludeTags.ts +++ b/apps/api/src/scraper/WebScraper/utils/excludeTags.ts @@ -39,16 +39,8 @@ export const excludeNonMainTags = [ "#search", ".share", "#share", - ".pagination", - "#pagination", ".widget", "#widget", - ".related", - "#related", - ".tag", - "#tag", - ".category", - "#category", ".cookie", "#cookie" ]; From 5758af3291aaebbdc13c3e7c469b3406f730476a Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sat, 7 Sep 2024 13:12:46 -0300 Subject: [PATCH 034/102] Update website_params.ts --- .../src/scraper/WebScraper/utils/custom/website_params.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts index af8d1f34..8169d9d3 100644 --- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts +++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts @@ -242,5 +242,13 @@ export const urlSpecificParams = { engine: "chrome-cdp", }, }, + }, + "lorealparis.hu":{ + defaultScraper: "fire-engine", + params:{ + fireEngineOptions:{ + engine: "tlsclient", + }, + }, } }; From 48c665519ebff263316601a33620115d68d00c41 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sat, 7 Sep 2024 13:42:45 -0300 Subject: [PATCH 035/102] Update credit_billing.ts --- .../src/services/billing/credit_billing.ts | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts index 53031de9..d22f0372 100644 --- a/apps/api/src/services/billing/credit_billing.ts +++ b/apps/api/src/services/billing/credit_billing.ts @@ -186,7 +186,8 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { getValue(cacheKeyCoupons) ]); - let subscription, subscriptionError, coupons; + let subscription, subscriptionError; + let coupons : {credits: number}[]; if (cachedSubscription && cachedCoupons) { subscription = JSON.parse(cachedSubscription); @@ -225,16 +226,16 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { ); } + + // If there are available coupons and they are enough for the operation + if (couponCredits >= credits) { + return { success: true, message: "Sufficient credits available", remainingCredits: couponCredits }; + } // Free credits, no coupons if (!subscription || subscriptionError) { - // If there is no active subscription but there are available coupons - if (couponCredits >= credits) { - return { success: true, message: "Sufficient credits available", remainingCredits: couponCredits }; - } - let creditUsages; let creditUsageError; let totalCreditsUsed = 0; @@ -251,6 +252,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { const retryInterval = 2000; // 2 seconds while (retries < maxRetries) { + // Reminder, this has an 1000 limit. const result = await supabase_service .from("credit_usage") .select("credits_used") @@ -292,7 +294,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { end.setDate(end.getDate() + 30); // check if usage is within 80% of the limit const creditLimit = FREE_CREDITS; - const creditUsagePercentage = (totalCreditsUsed + credits) / creditLimit; + const creditUsagePercentage = totalCreditsUsed / creditLimit; // Add a check to ensure totalCreditsUsed is greater than 0 if (totalCreditsUsed > 0 && creditUsagePercentage >= 0.8 && creditUsagePercentage < 1) { @@ -306,7 +308,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { } // 5. Compare the total credits used with the credits allowed by the plan. - if (totalCreditsUsed + credits > FREE_CREDITS) { + if (totalCreditsUsed > FREE_CREDITS) { // Send email notification for insufficient credits await sendNotification( team_id, @@ -366,7 +368,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { // Get the price details from cache or database const priceCacheKey = `price_${subscription.price_id}`; - let price; + let price : {credits: number}; try { const cachedPrice = await getValue(priceCacheKey); @@ -394,29 +396,31 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { Logger.error(`Error retrieving or caching price: ${error}`); Sentry.captureException(error); // If errors, just assume it's a big number so user don't get an error - price = { credits: 1000000 }; + price = { credits: 10000000 }; } const creditLimit = price.credits; - const creditUsagePercentage = (adjustedCreditsUsed + credits) / creditLimit; + + // Removal of + credits + const creditUsagePercentage = adjustedCreditsUsed / creditLimit; // Compare the adjusted total credits used with the credits allowed by the plan - if (adjustedCreditsUsed + credits > price.credits) { - // await sendNotification( - // team_id, - // NotificationType.LIMIT_REACHED, - // subscription.current_period_start, - // subscription.current_period_end - // ); + if (adjustedCreditsUsed > price.credits) { + await sendNotification( + team_id, + NotificationType.LIMIT_REACHED, + subscription.current_period_start, + subscription.current_period_end + ); return { success: false, message: "Insufficient credits, please upgrade!", remainingCredits: creditLimit - adjustedCreditsUsed }; - } else if (creditUsagePercentage >= 0.8) { + } else if (creditUsagePercentage >= 0.8 && creditUsagePercentage < 1) { // Send email notification for approaching credit limit - // await sendNotification( - // team_id, - // NotificationType.APPROACHING_LIMIT, - // subscription.current_period_start, - // subscription.current_period_end - // ); + await sendNotification( + team_id, + NotificationType.APPROACHING_LIMIT, + subscription.current_period_start, + subscription.current_period_end + ); } return { success: true, message: "Sufficient credits available", remainingCredits: creditLimit - adjustedCreditsUsed }; From fbdfa1256bb6095a08434b356fb51688d5337780 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 8 Sep 2024 13:07:10 -0300 Subject: [PATCH 036/102] Update credit_billing.ts --- apps/api/src/services/billing/credit_billing.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts index d22f0372..6a71b40a 100644 --- a/apps/api/src/services/billing/credit_billing.ts +++ b/apps/api/src/services/billing/credit_billing.ts @@ -308,7 +308,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { } // 5. Compare the total credits used with the credits allowed by the plan. - if (totalCreditsUsed > FREE_CREDITS) { + if (totalCreditsUsed >= FREE_CREDITS) { // Send email notification for insufficient credits await sendNotification( team_id, @@ -405,7 +405,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) { const creditUsagePercentage = adjustedCreditsUsed / creditLimit; // Compare the adjusted total credits used with the credits allowed by the plan - if (adjustedCreditsUsed > price.credits) { + if (adjustedCreditsUsed >= price.credits) { await sendNotification( team_id, NotificationType.LIMIT_REACHED, From 60a15d00eb73244257b99dfd05a2d55b0aab9dd4 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 8 Sep 2024 16:39:12 -0300 Subject: [PATCH 037/102] Update types.ts --- apps/api/src/controllers/v1/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index c4e0cf84..63ec1dd4 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -322,6 +322,7 @@ export function legacyScrapeOptions(x: ScrapeOptions): PageOptions { removeTags: x.excludeTags, onlyMainContent: x.onlyMainContent, waitFor: x.waitFor, + headers: x.headers, includeLinks: x.formats.includes("links"), screenshot: x.formats.includes("screenshot"), fullPageScreenshot: x.formats.includes("screenshot@fullPage"), From 22a5e85899eb893c9a68f53201e13f5fb569bc46 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 9 Sep 2024 12:26:55 -0300 Subject: [PATCH 038/102] Update index.ts --- apps/api/src/index.ts | 102 ++++++++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 34 deletions(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 58370158..1edf3759 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1,5 +1,5 @@ import "dotenv/config"; -import "./services/sentry" +import "./services/sentry"; import * as Sentry from "@sentry/node"; import express, { NextFunction, Request, Response } from "express"; import bodyParser from "body-parser"; @@ -12,9 +12,9 @@ import os from "os"; import { Logger } from "./lib/logger"; import { adminRouter } from "./routes/admin"; import { ScrapeEvents } from "./lib/scrape-events"; -import http from 'node:http'; -import https from 'node:https'; -import CacheableLookup from 'cacheable-lookup'; +import http from "node:http"; +import https from "node:https"; +import CacheableLookup from "cacheable-lookup"; import { v1Router } from "./routes/v1"; import expressWs from "express-ws"; import { crawlStatusWSController } from "./controllers/v1/crawl-status-ws"; @@ -31,11 +31,11 @@ Logger.info(`Number of CPUs: ${numCPUs} available`); const cacheable = new CacheableLookup({ // this is important to avoid querying local hostnames see https://github.com/szmarczak/cacheable-lookup readme - lookup:false + lookup: false, }); cacheable.install(http.globalAgent); -cacheable.install(https.globalAgent) +cacheable.install(https.globalAgent); if (cluster.isMaster) { Logger.info(`Master ${process.pid} is running`); @@ -115,9 +115,7 @@ if (cluster.isMaster) { app.get(`/serverHealthCheck`, async (req, res) => { try { const scrapeQueue = getScrapeQueue(); - const [waitingJobs] = await Promise.all([ - scrapeQueue.getWaitingCount(), - ]); + const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]); const noWaitingJobs = waitingJobs === 0; // 200 if no active jobs, 503 if there are active jobs @@ -190,38 +188,77 @@ if (cluster.isMaster) { res.send({ isProduction: global.isProduction }); }); - app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response, next: NextFunction) => { - if (err instanceof ZodError) { - res.status(400).json({ success: false, error: "Bad Request", details: err.errors }); - } else { + app.use( + ( + err: unknown, + req: Request<{}, ErrorResponse, undefined>, + res: Response, + next: NextFunction + ) => { + if (err instanceof ZodError) { + res + .status(400) + .json({ success: false, error: "Bad Request", details: err.errors }); + } else { next(err); + } } - }); + ); Sentry.setupExpressErrorHandler(app); - app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry, next: NextFunction) => { - const id = res.sentry ?? uuidv4(); - let verbose = JSON.stringify(err); - if (verbose === "{}") { - if (err instanceof Error) { - verbose = JSON.stringify({ - message: err.message, - name: err.name, - stack: err.stack, - }); - } - } + app.use( + ( + err: unknown, + req: Request<{}, ErrorResponse, undefined>, + res: ResponseWithSentry, + next: NextFunction + ) => { + if ( + err instanceof SyntaxError && + "status" in err && + err.status === 400 && + "body" in err + ) { + return res + .status(400) + .json({ success: false, error: "Bad request, malformed JSON" }); + } - Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose); - res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + id }); - }); + const id = res.sentry ?? uuidv4(); + let verbose = JSON.stringify(err); + if (verbose === "{}") { + if (err instanceof Error) { + verbose = JSON.stringify({ + message: err.message, + name: err.name, + stack: err.stack, + }); + } + } + + Logger.error( + "Error occurred in request! (" + + req.path + + ") -- ID " + + id + + " -- " + + verbose + ); + res + .status(500) + .json({ + success: false, + error: + "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + + id, + }); + } + ); Logger.info(`Worker ${process.pid} started`); } - - // const sq = getScrapeQueue(); // sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting")); @@ -230,6 +267,3 @@ if (cluster.isMaster) { // sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused")); // sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed")); // sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed")); - - - From ca9a781eb7fbadf7aee7dd6926aea3a0b1ca5e07 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 9 Sep 2024 12:27:55 -0300 Subject: [PATCH 039/102] Update index.ts --- apps/api/src/index.ts | 106 +++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 68 deletions(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 1edf3759..7d8817af 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1,5 +1,5 @@ import "dotenv/config"; -import "./services/sentry"; +import "./services/sentry" import * as Sentry from "@sentry/node"; import express, { NextFunction, Request, Response } from "express"; import bodyParser from "body-parser"; @@ -12,9 +12,9 @@ import os from "os"; import { Logger } from "./lib/logger"; import { adminRouter } from "./routes/admin"; import { ScrapeEvents } from "./lib/scrape-events"; -import http from "node:http"; -import https from "node:https"; -import CacheableLookup from "cacheable-lookup"; +import http from 'node:http'; +import https from 'node:https'; +import CacheableLookup from 'cacheable-lookup'; import { v1Router } from "./routes/v1"; import expressWs from "express-ws"; import { crawlStatusWSController } from "./controllers/v1/crawl-status-ws"; @@ -31,11 +31,11 @@ Logger.info(`Number of CPUs: ${numCPUs} available`); const cacheable = new CacheableLookup({ // this is important to avoid querying local hostnames see https://github.com/szmarczak/cacheable-lookup readme - lookup: false, + lookup:false }); cacheable.install(http.globalAgent); -cacheable.install(https.globalAgent); +cacheable.install(https.globalAgent) if (cluster.isMaster) { Logger.info(`Master ${process.pid} is running`); @@ -115,7 +115,9 @@ if (cluster.isMaster) { app.get(`/serverHealthCheck`, async (req, res) => { try { const scrapeQueue = getScrapeQueue(); - const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]); + const [waitingJobs] = await Promise.all([ + scrapeQueue.getWaitingCount(), + ]); const noWaitingJobs = waitingJobs === 0; // 200 if no active jobs, 503 if there are active jobs @@ -188,77 +190,42 @@ if (cluster.isMaster) { res.send({ isProduction: global.isProduction }); }); - app.use( - ( - err: unknown, - req: Request<{}, ErrorResponse, undefined>, - res: Response, - next: NextFunction - ) => { - if (err instanceof ZodError) { - res - .status(400) - .json({ success: false, error: "Bad Request", details: err.errors }); - } else { + app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response, next: NextFunction) => { + if (err instanceof ZodError) { + res.status(400).json({ success: false, error: "Bad Request", details: err.errors }); + } else { next(err); - } } - ); + }); Sentry.setupExpressErrorHandler(app); - app.use( - ( - err: unknown, - req: Request<{}, ErrorResponse, undefined>, - res: ResponseWithSentry, - next: NextFunction - ) => { - if ( - err instanceof SyntaxError && - "status" in err && - err.status === 400 && - "body" in err - ) { - return res - .status(400) - .json({ success: false, error: "Bad request, malformed JSON" }); - } - - const id = res.sentry ?? uuidv4(); - let verbose = JSON.stringify(err); - if (verbose === "{}") { - if (err instanceof Error) { - verbose = JSON.stringify({ - message: err.message, - name: err.name, - stack: err.stack, - }); - } - } - - Logger.error( - "Error occurred in request! (" + - req.path + - ") -- ID " + - id + - " -- " + - verbose - ); - res - .status(500) - .json({ - success: false, - error: - "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + - id, - }); + app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry, next: NextFunction) => { + if (err instanceof SyntaxError && 'status' in err && err.status === 400 && 'body' in err) { + return res.status(400).json({ success: false, error: 'Bad request, malformed JSON' }); } - ); + + const id = res.sentry ?? uuidv4(); + let verbose = JSON.stringify(err); + if (verbose === "{}") { + if (err instanceof Error) { + verbose = JSON.stringify({ + message: err.message, + name: err.name, + stack: err.stack, + }); + } + } + + Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose); + res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + id }); + }); Logger.info(`Worker ${process.pid} started`); } + + // const sq = getScrapeQueue(); // sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting")); @@ -267,3 +234,6 @@ if (cluster.isMaster) { // sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused")); // sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed")); // sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed")); + + + From 17e419a7fb82dacba45692ea676f0487e66d5f70 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 9 Sep 2024 21:06:23 -0300 Subject: [PATCH 040/102] Nick: --- .../scraper/WebScraper/scrapers/fireEngine.ts | 2 +- apps/api/src/scraper/WebScraper/single_url.ts | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index e7361c5c..a3f393c8 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -55,7 +55,7 @@ export async function scrapWithFireEngine({ try { const reqParams = await generateRequestParams(url); let waitParam = reqParams["params"]?.wait ?? waitFor; - let engineParam = reqParams["params"]?.engine ?? reqParams["params"]?.fireEngineOptions?.engine ?? fireEngineOptions?.engine ?? "playwright"; + let engineParam = reqParams["params"]?.engine ?? reqParams["params"]?.fireEngineOptions?.engine ?? fireEngineOptions?.engine ?? "chrome-cdp"; let screenshotParam = reqParams["params"]?.screenshot ?? screenshot; let fullPageScreenshotParam = reqParams["params"]?.fullPageScreenshot ?? fullPageScreenshot; let fireEngineOptionsParam : FireEngineOptions = reqParams["params"]?.fireEngineOptions ?? fireEngineOptions; diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 8bafd203..2be65899 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -96,15 +96,15 @@ function getScrapingFallbackOrder( "fetch", ].filter(Boolean); - if (isWaitPresent || isScreenshotPresent || isHeadersPresent) { - defaultOrder = [ - "fire-engine", - useFireEngine ? undefined : "playwright", - ...defaultOrder.filter( - (scraper) => scraper !== "fire-engine" && scraper !== "playwright" - ), - ].filter(Boolean); - } + // if (isWaitPresent || isScreenshotPresent || isHeadersPresent) { + // defaultOrder = [ + // "fire-engine", + // useFireEngine ? undefined : "playwright", + // ...defaultOrder.filter( + // (scraper) => scraper !== "fire-engine" && scraper !== "playwright" + // ), + // ].filter(Boolean); + // } const filteredDefaultOrder = defaultOrder.filter( (scraper: (typeof baseScrapers)[number]) => From a6bcf7b4389409f9972a69b3d48e1ecd084e3121 Mon Sep 17 00:00:00 2001 From: Gergo Moricz Date: Tue, 10 Sep 2024 08:51:58 +0200 Subject: [PATCH 041/102] fix(v0/crawl-status): don't crash on big crawls when requesting jobs from supabase --- apps/api/src/controllers/v0/crawl-status.ts | 8 ++--- apps/api/src/controllers/v0/status.ts | 2 +- apps/api/src/lib/supabase-jobs.ts | 34 +++++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts index a3f3f16f..41491f86 100644 --- a/apps/api/src/controllers/v0/crawl-status.ts +++ b/apps/api/src/controllers/v0/crawl-status.ts @@ -4,16 +4,16 @@ import { RateLimiterMode } from "../../../src/types"; import { getScrapeQueue } from "../../../src/services/queue-service"; import { Logger } from "../../../src/lib/logger"; import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis"; -import { supabaseGetJobsById } from "../../../src/lib/supabase-jobs"; +import { supabaseGetJobsByCrawlId } from "../../../src/lib/supabase-jobs"; import * as Sentry from "@sentry/node"; import { configDotenv } from "dotenv"; configDotenv(); -export async function getJobs(ids: string[]) { +export async function getJobs(crawlId: string, ids: string[]) { const jobs = (await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))).filter(x => x); if (process.env.USE_DB_AUTHENTICATION === "true") { - const supabaseData = await supabaseGetJobsById(ids); + const supabaseData = await supabaseGetJobsByCrawlId(crawlId); supabaseData.forEach(x => { const job = jobs.find(y => y.id === x.job_id); @@ -52,7 +52,7 @@ export async function crawlStatusController(req: Request, res: Response) { const jobIDs = await getCrawlJobs(req.params.jobId); - const jobs = (await getJobs(jobIDs)).sort((a, b) => a.timestamp - b.timestamp); + const jobs = (await getJobs(req.params.jobId, jobIDs)).sort((a, b) => a.timestamp - b.timestamp); const jobStatuses = await Promise.all(jobs.map(x => x.getState())); const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : jobStatuses.some(x => x === "failed") ? "failed" : "active"; diff --git a/apps/api/src/controllers/v0/status.ts b/apps/api/src/controllers/v0/status.ts index 34ebb3c6..bf8d2834 100644 --- a/apps/api/src/controllers/v0/status.ts +++ b/apps/api/src/controllers/v0/status.ts @@ -22,7 +22,7 @@ export async function crawlJobStatusPreviewController(req: Request, res: Respons // } // } - const jobs = (await getJobs(jobIDs)).sort((a, b) => a.timestamp - b.timestamp); + const jobs = (await getJobs(req.params.jobId, jobIDs)).sort((a, b) => a.timestamp - b.timestamp); const jobStatuses = await Promise.all(jobs.map(x => x.getState())); const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : jobStatuses.some(x => x === "failed") ? "failed" : "active"; diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.ts index cda6fd46..52e594c4 100644 --- a/apps/api/src/lib/supabase-jobs.ts +++ b/apps/api/src/lib/supabase-jobs.ts @@ -2,6 +2,11 @@ import { supabase_service } from "../services/supabase"; import { Logger } from "./logger"; import * as Sentry from "@sentry/node"; +/** + * Get a single firecrawl_job by ID + * @param jobId ID of Job + * @returns {any | null} Job + */ export const supabaseGetJobById = async (jobId: string) => { const { data, error } = await supabase_service .from("firecrawl_jobs") @@ -20,6 +25,11 @@ export const supabaseGetJobById = async (jobId: string) => { return data; }; +/** + * Get multiple firecrawl_jobs by ID. Use this if you're not requesting a lot (50+) of jobs at once. + * @param jobIds IDs of Jobs + * @returns {any[]} Jobs + */ export const supabaseGetJobsById = async (jobIds: string[]) => { const { data, error } = await supabase_service.rpc("get_jobs_by_ids", { job_ids: jobIds, @@ -38,6 +48,30 @@ export const supabaseGetJobsById = async (jobIds: string[]) => { return data; }; +/** + * Get multiple firecrawl_jobs by crawl ID. Use this if you need a lot of jobs at once. + * @param crawlId ID of crawl + * @returns {any[]} Jobs + */ +export const supabaseGetJobsByCrawlId = async (crawlId: string) => { + const { data, error } = await supabase_service + .from("firecrawl_jobs") + .select() + .eq("crawl_id", crawlId) + + if (error) { + Logger.error(`Error in supabaseGetJobsByCrawlId: ${error}`); + Sentry.captureException(error); + return []; + } + + if (!data) { + return []; + } + + return data; +}; + export const supabaseGetJobByIdOnlyData = async (jobId: string) => { const { data, error } = await supabase_service From f8fbc71f91a842c86fba9b02ceca4bff4e74d7d6 Mon Sep 17 00:00:00 2001 From: Gergo Moricz Date: Tue, 10 Sep 2024 09:20:18 +0200 Subject: [PATCH 042/102] fix(supabase-jobs): do not use RPCs RPCs are more failure-prone for this use case than regular queries are. --- apps/api/src/lib/supabase-jobs.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.ts index 52e594c4..c418a6e0 100644 --- a/apps/api/src/lib/supabase-jobs.ts +++ b/apps/api/src/lib/supabase-jobs.ts @@ -31,12 +31,13 @@ export const supabaseGetJobById = async (jobId: string) => { * @returns {any[]} Jobs */ export const supabaseGetJobsById = async (jobIds: string[]) => { - const { data, error } = await supabase_service.rpc("get_jobs_by_ids", { - job_ids: jobIds, - }); + const { data, error } = await supabase_service + .from("firecrawl_jobs") + .select() + .in("job_id", jobIds); if (error) { - Logger.error(`Error in get_jobs_by_ids: ${error}`); + Logger.error(`Error in supabaseGetJobsById: ${error}`); Sentry.captureException(error); return []; } From 26f2095de61103e854ef95326b6e0570b2494879 Mon Sep 17 00:00:00 2001 From: Gergo Moricz Date: Tue, 10 Sep 2024 09:24:23 +0200 Subject: [PATCH 043/102] fix(v1): proper Invalid URL handling --- apps/api/src/controllers/v1/types.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 63ec1dd4..f812f981 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -30,7 +30,14 @@ export const url = z.preprocess( "URL must have a valid top-level domain or be a valid path" ) .refine( - (x) => checkUrl(x as string), + (x) => { + try { + checkUrl(x as string) + return true; + } catch (_) { + return false; + } + }, "Invalid URL" ) .refine( From b4dbf7553750a54040ff47fea9042d2858aaa9cd Mon Sep 17 00:00:00 2001 From: Gergo Moricz Date: Tue, 10 Sep 2024 10:25:14 +0200 Subject: [PATCH 044/102] fix(v1): check if url is string in blocklistMiddleware Fixes FIRECRAWL-SCRAPER-JS-9Z --- apps/api/src/routes/v1.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts index daa9bf43..484ab5dc 100644 --- a/apps/api/src/routes/v1.ts +++ b/apps/api/src/routes/v1.ts @@ -83,7 +83,7 @@ function idempotencyMiddleware(req: Request, res: Response, next: NextFunction) } function blocklistMiddleware(req: Request, res: Response, next: NextFunction) { - if (req.body.url && isUrlBlocked(req.body.url)) { + if (typeof req.body.url === "string" && isUrlBlocked(req.body.url)) { if (!res.headersSent) { return res.status(403).json({ success: false, error: "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." }); } From a17e1cac929ace616e371b4df4100a1029300609 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 10 Sep 2024 06:53:24 -0300 Subject: [PATCH 045/102] Rate bump --- apps/api/src/services/rate-limiter.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index dade8493..7cfff35b 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -6,7 +6,7 @@ const RATE_LIMITS = { crawl: { default: 3, free: 2, - starter: 3, + starter: 10, standard: 5, standardOld: 40, scale: 50, @@ -19,9 +19,9 @@ const RATE_LIMITS = { scrape: { default: 20, free: 10, - starter: 20, + starter: 100, standard: 100, - standardOld: 40, + standardOld: 100, scale: 500, hobby: 20, standardNew: 100, @@ -32,8 +32,8 @@ const RATE_LIMITS = { search: { default: 20, free: 5, - starter: 20, - standard: 40, + starter: 50, + standard: 50, standardOld: 40, scale: 500, hobby: 10, @@ -45,9 +45,9 @@ const RATE_LIMITS = { map:{ default: 20, free: 5, - starter: 20, - standard: 40, - standardOld: 40, + starter: 50, + standard: 50, + standardOld: 50, scale: 500, hobby: 10, standardNew: 50, From 45237a29dde6f38af4a1a9b7c3d203fbb6c38795 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 10 Sep 2024 09:09:39 -0300 Subject: [PATCH 046/102] updated js-sdk examples --- apps/js-sdk/example.js | 2 +- apps/js-sdk/example.ts | 2 +- apps/js-sdk/package-lock.json | 76 +++++++++++++++++++++++++++++++++-- apps/js-sdk/package.json | 1 + 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/apps/js-sdk/example.js b/apps/js-sdk/example.js index eb4bc489..c4b21d5f 100644 --- a/apps/js-sdk/example.js +++ b/apps/js-sdk/example.js @@ -1,4 +1,4 @@ -import FirecrawlApp from '@mendable/firecrawl-js'; +import FirecrawlApp from 'firecrawl'; const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); diff --git a/apps/js-sdk/example.ts b/apps/js-sdk/example.ts index 4142416f..7412e479 100644 --- a/apps/js-sdk/example.ts +++ b/apps/js-sdk/example.ts @@ -1,4 +1,4 @@ -import FirecrawlApp, { CrawlStatusResponse, ErrorResponse } from '@mendable/firecrawl-js'; +import FirecrawlApp, { CrawlStatusResponse, ErrorResponse } from 'firecrawl'; const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); diff --git a/apps/js-sdk/package-lock.json b/apps/js-sdk/package-lock.json index 95dd7d27..b0f358cb 100644 --- a/apps/js-sdk/package-lock.json +++ b/apps/js-sdk/package-lock.json @@ -9,8 +9,8 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@mendable/firecrawl-js": "^0.0.36", "axios": "^1.6.8", + "firecrawl": "^1.2.0", "ts-node": "^10.9.2", "typescript": "^5.4.5", "uuid": "^10.0.0", @@ -422,12 +422,14 @@ } }, "node_modules/@mendable/firecrawl-js": { - "version": "0.0.36", - "resolved": "https://registry.npmjs.org/@mendable/firecrawl-js/-/firecrawl-js-0.0.36.tgz", - "integrity": "sha512-5zQMWUD49r6Q7cxj+QBthQ964Bm9fMooW4E8E4nIca3BMXCeEuQFVf5C3OEWwZf0SjJvR+5Yx2wUbXJWd1wCOA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@mendable/firecrawl-js/-/firecrawl-js-1.2.2.tgz", + "integrity": "sha512-2A1GzLD0bczlFIlcjxHcm/x8i76ndtV4EUzOfc81oOJ/HbycE2mbT6EUthoL+r4s5A8yO3bKr9o/GxmEn456VA==", "dependencies": { "axios": "^1.6.8", "dotenv": "^16.4.5", + "isows": "^1.0.4", + "typescript-event-target": "^1.1.1", "uuid": "^9.0.1", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.0" @@ -594,6 +596,32 @@ "@esbuild/win32-x64": "0.20.2" } }, + "node_modules/firecrawl": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/firecrawl/-/firecrawl-1.2.0.tgz", + "integrity": "sha512-Sy1BCCvs5FhGc4yxPP7NG9iWnK8RXdvA1ZS/K1Gj+LrEN3iAT2WRzhYET7x8G2bif25F6rHJg57vdVb5sr6RyQ==", + "dependencies": { + "axios": "^1.6.8", + "dotenv": "^16.4.5", + "isows": "^1.0.4", + "typescript-event-target": "^1.1.1", + "uuid": "^9.0.1", + "zod": "^3.23.8", + "zod-to-json-schema": "^3.23.0" + } + }, + "node_modules/firecrawl/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/follow-redirects": { "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", @@ -652,6 +680,20 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/isows": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/isows/-/isows-1.0.4.tgz", + "integrity": "sha512-hEzjY+x9u9hPmBom9IIAqdJCwNLax+xrPb51vEPpERoFlIxgmZcHzsT5jKG06nvInKOBGvReAVz80Umed5CczQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wagmi-dev" + } + ], + "peerDependencies": { + "ws": "*" + } + }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -763,6 +805,11 @@ "node": ">=14.17" } }, + "node_modules/typescript-event-target": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/typescript-event-target/-/typescript-event-target-1.1.1.tgz", + "integrity": "sha512-dFSOFBKV6uwaloBCCUhxlD3Pr/P1a/tJdcmPrTXCHlEFD3faj0mztjcGn6VBAhQ0/Bdy8K3VWrrqwbt/ffsYsg==" + }, "node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", @@ -786,6 +833,27 @@ "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==" }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "peer": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/apps/js-sdk/package.json b/apps/js-sdk/package.json index b5d919f4..ac3ef038 100644 --- a/apps/js-sdk/package.json +++ b/apps/js-sdk/package.json @@ -13,6 +13,7 @@ "dependencies": { "@mendable/firecrawl-js": "^1.0.3", "axios": "^1.6.8", + "firecrawl": "^1.2.0", "ts-node": "^10.9.2", "typescript": "^5.4.5", "uuid": "^10.0.0", From ee8a54213c50ae88720ce5a03f76a65d270e81d0 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 10 Sep 2024 10:25:27 -0300 Subject: [PATCH 047/102] fix(py-sdk): removed asyncio package tested websocket with example.py without asyncio and it works with no problem. --- apps/python-sdk/firecrawl/firecrawl.py | 1 - apps/python-sdk/pyproject.toml | 3 +-- apps/python-sdk/requirements.txt | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index 254f4c70..3961631e 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -13,7 +13,6 @@ import logging import os import time from typing import Any, Dict, Optional, List -import asyncio import json import requests diff --git a/apps/python-sdk/pyproject.toml b/apps/python-sdk/pyproject.toml index 969fb051..87cb91f1 100644 --- a/apps/python-sdk/pyproject.toml +++ b/apps/python-sdk/pyproject.toml @@ -12,8 +12,7 @@ dependencies = [ "requests", "python-dotenv", "websockets", - "asyncio", -"nest-asyncio" + "nest-asyncio" ] authors = [{name = "Mendable.ai",email = "nick@mendable.ai"}] maintainers = [{name = "Mendable.ai",email = "nick@mendable.ai"}] diff --git a/apps/python-sdk/requirements.txt b/apps/python-sdk/requirements.txt index 94971fde..db67ceeb 100644 --- a/apps/python-sdk/requirements.txt +++ b/apps/python-sdk/requirements.txt @@ -2,5 +2,4 @@ requests pytest python-dotenv websockets -asyncio nest-asyncio \ No newline at end of file From f855ad3436f97972383193980f1fb9f775636a0f Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 10 Sep 2024 10:29:44 -0300 Subject: [PATCH 048/102] bumping py-sdk version --- apps/python-sdk/firecrawl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index f178cd61..540ce67e 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -13,7 +13,7 @@ import os from .firecrawl import FirecrawlApp -__version__ = "1.2.3" +__version__ = "1.2.4" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") From 4ebc35c9dde46e1fd2e38364000aa493287b9650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Tue, 10 Sep 2024 18:59:09 +0200 Subject: [PATCH 049/102] fix(crawl-status): add success: true --- apps/api/src/controllers/v1/crawl-status-ws.ts | 1 + apps/api/src/controllers/v1/crawl-status.ts | 1 + apps/api/src/controllers/v1/types.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts index 8d823096..16a67682 100644 --- a/apps/api/src/controllers/v1/crawl-status-ws.ts +++ b/apps/api/src/controllers/v1/crawl-status-ws.ts @@ -103,6 +103,7 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth Date: Tue, 10 Sep 2024 19:29:38 +0200 Subject: [PATCH 050/102] feat(js-sdk): paginate next on checkCrawlStatus + better types for CSR --- apps/js-sdk/firecrawl/package.json | 2 +- apps/js-sdk/firecrawl/src/index.ts | 62 ++++++++++++++++++------------ 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 7114a625..75ebe390 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.2.2", + "version": "1.2.3", "description": "JavaScript SDK for Firecrawl API", "main": "build/cjs/index.js", "types": "types/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 8b16adfb..55c5be0b 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -131,15 +131,14 @@ export interface CrawlResponse { */ export interface CrawlStatusResponse { success: true; - total: number; + status: "scraping" | "completed" | "failed" | "cancelled"; completed: number; + total: number; creditsUsed: number; expiresAt: Date; - status: "scraping" | "completed" | "failed"; - next: string; - data?: FirecrawlDocument[]; - error?: string; -} + next?: string; + data: FirecrawlDocument[]; +}; /** * Parameters for mapping operations. @@ -329,9 +328,10 @@ export default class FirecrawlApp { /** * Checks the status of a crawl job using the Firecrawl API. * @param id - The ID of the crawl operation. + * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`) * @returns The response containing the job status. */ - async checkCrawlStatus(id?: string): Promise { + async checkCrawlStatus(id?: string, getAllData = false): Promise { if (!id) { throw new Error("No crawl ID provided"); } @@ -342,17 +342,29 @@ export default class FirecrawlApp { `${this.apiUrl}/v1/crawl/${id}`, headers ); - if (response.status === 200) { + if (response.status === 200 && getAllData) { + let allData = response.data.data; + if (response.data.status === "completed") { + let statusData = response.data + if ("data" in statusData) { + let data = statusData.data; + while ('next' in statusData) { + statusData = (await this.getRequest(statusData.next, headers)).data; + data = data.concat(statusData.data); + } + allData = data; + } + } return ({ - success: true, + success: response.data.success, status: response.data.status, total: response.data.total, completed: response.data.completed, creditsUsed: response.data.creditsUsed, expiresAt: new Date(response.data.expiresAt), next: response.data.next, - data: response.data.data, - error: response.data.error + data: allData, + error: response.data.error, }) } else { this.handleError(response, "check crawl status"); @@ -452,7 +464,7 @@ export default class FirecrawlApp { id: string, headers: AxiosRequestHeaders, checkInterval: number - ): Promise { + ): Promise { while (true) { let statusResponse: AxiosResponse = await this.getRequest( `${this.apiUrl}/v1/crawl/${id}`, @@ -460,20 +472,20 @@ export default class FirecrawlApp { ); if (statusResponse.status === 200) { let statusData = statusResponse.data; - if (statusData.status === "completed") { - if ("data" in statusData) { - let data = statusData.data; - while ('next' in statusData) { - statusResponse = await this.getRequest(statusData.next, headers); - statusData = statusResponse.data; - data = data.concat(statusData.data); + if (statusData.status === "completed") { + if ("data" in statusData) { + let data = statusData.data; + while ('next' in statusData) { + statusResponse = await this.getRequest(statusData.next, headers); + statusData = statusResponse.data; + data = data.concat(statusData.data); + } + statusData.data = data; + return statusData; + } else { + throw new Error("Crawl job completed but no data was returned"); } - statusData.data = data; - return statusData; - } else { - throw new Error("Crawl job completed but no data was returned"); - } - } else if ( + } else if ( ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status) ) { checkInterval = Math.max(checkInterval, 2); From ad1a6fbc74eeb51c8ac2be870c4535382c8e0428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Tue, 10 Sep 2024 19:41:01 +0200 Subject: [PATCH 051/102] fix(v1/map): handle invalid URLs gracefully --- apps/api/src/controllers/v1/map.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts index e6abd9ae..a9c61d04 100644 --- a/apps/api/src/controllers/v1/map.ts +++ b/apps/api/src/controllers/v1/map.ts @@ -88,7 +88,13 @@ export async function mapController( links = performCosineSimilarity(links, searchQuery); } - links = links.map((x) => checkAndUpdateURLForMap(x).url.trim()); + links = links.map((x) => { + try { + return checkAndUpdateURLForMap(x).url.trim() + } catch (_) { + return null; + } + }).filter(x => x !== null); // allows for subdomains to be included links = links.filter((x) => isSameDomain(x, req.body.url)); From 83a165db0fd0e680f4dfb1c41cbcb20901d5e8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Tue, 10 Sep 2024 21:18:53 +0200 Subject: [PATCH 052/102] fix(v0/scrape): ensure url is string --- apps/api/src/controllers/v0/scrape.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts index bc91da18..2a5f1d4f 100644 --- a/apps/api/src/controllers/v0/scrape.ts +++ b/apps/api/src/controllers/v0/scrape.ts @@ -39,7 +39,7 @@ export async function scrapeHelper( returnCode: number; }> { const url = req.body.url; - if (!url) { + if (typeof url !== "string") { return { success: false, error: "Url is required", returnCode: 400 }; } From 97ffabff3a6b6bd1c7455b85ce794949f540469b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Tue, 10 Sep 2024 21:21:20 +0200 Subject: [PATCH 053/102] fix(v1): converting bad docs always gives null --- apps/api/src/controllers/v1/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 6b2db308..c44c1cc5 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -348,7 +348,7 @@ export function legacyExtractorOptions(x: ExtractOptions): ExtractorOptions { } export function legacyDocumentConverter(doc: any): Document { - if (doc === null || doc === undefined) return doc; + if (doc === null || doc === undefined) return null; if (doc.metadata) { if (doc.metadata.screenshot) { From f6fc71b46a54f16b34cdada9be0c6fb53810582d Mon Sep 17 00:00:00 2001 From: Andrei Bobkov Date: Wed, 11 Sep 2024 17:53:17 +0300 Subject: [PATCH 054/102] fix(js-sdk): bring back cjs exports --- apps/js-sdk/firecrawl/package-lock.json | 1549 ++++++++++++++++++++++- apps/js-sdk/firecrawl/package.json | 10 +- apps/js-sdk/firecrawl/tsconfig.json | 12 +- apps/js-sdk/firecrawl/tsup.config.ts | 9 + 4 files changed, 1560 insertions(+), 20 deletions(-) create mode 100644 apps/js-sdk/firecrawl/tsup.config.ts diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json index ce6a1a4a..ee1baba3 100644 --- a/apps/js-sdk/firecrawl/package-lock.json +++ b/apps/js-sdk/firecrawl/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mendable/firecrawl-js", - "version": "1.1.0", + "version": "1.2.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mendable/firecrawl-js", - "version": "1.1.0", + "version": "1.2.1", "license": "MIT", "dependencies": { "axios": "^1.6.8", @@ -27,6 +27,7 @@ "@types/uuid": "^9.0.8", "jest": "^29.7.0", "ts-jest": "^29.2.2", + "tsup": "^8.2.4", "typescript": "^5.4.5" } }, @@ -600,6 +601,486 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz", + "integrity": "sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.23.1.tgz", + "integrity": "sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.23.1.tgz", + "integrity": "sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.23.1.tgz", + "integrity": "sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.23.1.tgz", + "integrity": "sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.23.1.tgz", + "integrity": "sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.23.1.tgz", + "integrity": "sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.23.1.tgz", + "integrity": "sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.23.1.tgz", + "integrity": "sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.23.1.tgz", + "integrity": "sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.23.1.tgz", + "integrity": "sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.23.1.tgz", + "integrity": "sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.23.1.tgz", + "integrity": "sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.23.1.tgz", + "integrity": "sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.23.1.tgz", + "integrity": "sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.23.1.tgz", + "integrity": "sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.23.1.tgz", + "integrity": "sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.23.1.tgz", + "integrity": "sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.23.1.tgz", + "integrity": "sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.23.1.tgz", + "integrity": "sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.23.1.tgz", + "integrity": "sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.23.1.tgz", + "integrity": "sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.23.1.tgz", + "integrity": "sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.23.1.tgz", + "integrity": "sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -951,6 +1432,259 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.21.2.tgz", + "integrity": "sha512-fSuPrt0ZO8uXeS+xP3b+yYTCBUd05MoSp2N/MFOgjhhUhMmchXlpTQrTpI8T+YAwAQuK7MafsCOxW7VrPMrJcg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.21.2.tgz", + "integrity": "sha512-xGU5ZQmPlsjQS6tzTTGwMsnKUtu0WVbl0hYpTPauvbRAnmIvpInhJtgjj3mcuJpEiuUw4v1s4BimkdfDWlh7gA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.21.2.tgz", + "integrity": "sha512-99AhQ3/ZMxU7jw34Sq8brzXqWH/bMnf7ZVhvLk9QU2cOepbQSVTns6qoErJmSiAvU3InRqC2RRZ5ovh1KN0d0Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.21.2.tgz", + "integrity": "sha512-ZbRaUvw2iN/y37x6dY50D8m2BnDbBjlnMPotDi/qITMJ4sIxNY33HArjikDyakhSv0+ybdUxhWxE6kTI4oX26w==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.21.2.tgz", + "integrity": "sha512-ztRJJMiE8nnU1YFcdbd9BcH6bGWG1z+jP+IPW2oDUAPxPjo9dverIOyXz76m6IPA6udEL12reYeLojzW2cYL7w==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.21.2.tgz", + "integrity": "sha512-flOcGHDZajGKYpLV0JNc0VFH361M7rnV1ee+NTeC/BQQ1/0pllYcFmxpagltANYt8FYf9+kL6RSk80Ziwyhr7w==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.21.2.tgz", + "integrity": "sha512-69CF19Kp3TdMopyteO/LJbWufOzqqXzkrv4L2sP8kfMaAQ6iwky7NoXTp7bD6/irKgknDKM0P9E/1l5XxVQAhw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.21.2.tgz", + "integrity": "sha512-48pD/fJkTiHAZTnZwR0VzHrao70/4MlzJrq0ZsILjLW/Ab/1XlVUStYyGt7tdyIiVSlGZbnliqmult/QGA2O2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.21.2.tgz", + "integrity": "sha512-cZdyuInj0ofc7mAQpKcPR2a2iu4YM4FQfuUzCVA2u4HI95lCwzjoPtdWjdpDKyHxI0UO82bLDoOaLfpZ/wviyQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.21.2.tgz", + "integrity": "sha512-RL56JMT6NwQ0lXIQmMIWr1SW28z4E4pOhRRNqwWZeXpRlykRIlEpSWdsgNWJbYBEWD84eocjSGDu/XxbYeCmwg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.21.2.tgz", + "integrity": "sha512-PMxkrWS9z38bCr3rWvDFVGD6sFeZJw4iQlhrup7ReGmfn7Oukrr/zweLhYX6v2/8J6Cep9IEA/SmjXjCmSbrMQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.21.2.tgz", + "integrity": "sha512-B90tYAUoLhU22olrafY3JQCFLnT3NglazdwkHyxNDYF/zAxJt5fJUB/yBoWFoIQ7SQj+KLe3iL4BhOMa9fzgpw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.21.2.tgz", + "integrity": "sha512-7twFizNXudESmC9oneLGIUmoHiiLppz/Xs5uJQ4ShvE6234K0VB1/aJYU3f/4g7PhssLGKBVCC37uRkkOi8wjg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.21.2.tgz", + "integrity": "sha512-9rRero0E7qTeYf6+rFh3AErTNU1VCQg2mn7CQcI44vNUWM9Ze7MSRS/9RFuSsox+vstRt97+x3sOhEey024FRQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.21.2.tgz", + "integrity": "sha512-5rA4vjlqgrpbFVVHX3qkrCo/fZTj1q0Xxpg+Z7yIo3J2AilW7t2+n6Q8Jrx+4MrYpAnjttTYF8rr7bP46BPzRw==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.21.2.tgz", + "integrity": "sha512-6UUxd0+SKomjdzuAcp+HAmxw1FlGBnl1v2yEPSabtx4lBfdXHDVsW7+lQkgz9cNFJGY3AWR7+V8P5BqkD9L9nA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -1036,6 +1770,12 @@ "dotenv": "*" } }, + "node_modules/@types/estree": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", + "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", + "dev": true + }, "node_modules/@types/graceful-fs": { "version": "4.1.9", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", @@ -1160,6 +1900,12 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true + }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -1182,6 +1928,15 @@ "sprintf-js": "~1.0.2" } }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/async": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", @@ -1316,6 +2071,18 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1397,6 +2164,30 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/bundle-require": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-5.0.0.tgz", + "integrity": "sha512-GuziW3fSSmopcx4KRymQEJVbZUfqlCqcq7dvs6TYwKRZiegK/2buMxQTPs6MGlNv50wms1699qYO54R8XfRX4w==", + "dev": true, + "dependencies": { + "load-tsconfig": "^0.2.3" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "peerDependencies": { + "esbuild": ">=0.18" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1460,6 +2251,30 @@ "node": ">=10" } }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, "node_modules/ci-info": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", @@ -1540,12 +2355,30 @@ "node": ">= 0.8" } }, + "node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "node_modules/consola": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.2.3.tgz", + "integrity": "sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ==", + "dev": true, + "engines": { + "node": "^14.18.0 || >=16.10.0" + } + }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", @@ -1588,12 +2421,12 @@ } }, "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", "dev": true, "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -1653,6 +2486,18 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/dotenv": { "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", @@ -1664,6 +2509,12 @@ "url": "https://dotenvx.com" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true + }, "node_modules/ejs": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", @@ -1712,6 +2563,45 @@ "is-arrayish": "^0.2.1" } }, + "node_modules/esbuild": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.23.1.tgz", + "integrity": "sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.23.1", + "@esbuild/android-arm": "0.23.1", + "@esbuild/android-arm64": "0.23.1", + "@esbuild/android-x64": "0.23.1", + "@esbuild/darwin-arm64": "0.23.1", + "@esbuild/darwin-x64": "0.23.1", + "@esbuild/freebsd-arm64": "0.23.1", + "@esbuild/freebsd-x64": "0.23.1", + "@esbuild/linux-arm": "0.23.1", + "@esbuild/linux-arm64": "0.23.1", + "@esbuild/linux-ia32": "0.23.1", + "@esbuild/linux-loong64": "0.23.1", + "@esbuild/linux-mips64el": "0.23.1", + "@esbuild/linux-ppc64": "0.23.1", + "@esbuild/linux-riscv64": "0.23.1", + "@esbuild/linux-s390x": "0.23.1", + "@esbuild/linux-x64": "0.23.1", + "@esbuild/netbsd-x64": "0.23.1", + "@esbuild/openbsd-arm64": "0.23.1", + "@esbuild/openbsd-x64": "0.23.1", + "@esbuild/sunos-x64": "0.23.1", + "@esbuild/win32-arm64": "0.23.1", + "@esbuild/win32-ia32": "0.23.1", + "@esbuild/win32-x64": "0.23.1" + } + }, "node_modules/escalade": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", @@ -1791,12 +2681,37 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/fast-glob": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", + "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true }, + "node_modules/fastq": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", + "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, "node_modules/fb-watchman": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", @@ -1880,6 +2795,34 @@ } } }, + "node_modules/foreground-child": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", + "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", @@ -1981,6 +2924,18 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", @@ -1990,6 +2945,26 @@ "node": ">=4" } }, + "node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -2032,6 +3007,15 @@ "node": ">=10.17.0" } }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/import-local": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", @@ -2082,6 +3066,18 @@ "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/is-core-module": { "version": "2.13.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", @@ -2094,6 +3090,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -2112,6 +3117,18 @@ "node": ">=6" } }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -2252,6 +3269,21 @@ "node": ">=8" } }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jake": { "version": "10.9.1", "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.1.tgz", @@ -2858,6 +3890,15 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -2925,12 +3966,33 @@ "node": ">=6" } }, + "node_modules/lilconfig": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.2.tgz", + "integrity": "sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "dev": true }, + "node_modules/load-tsconfig": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/load-tsconfig/-/load-tsconfig-0.2.5.tgz", + "integrity": "sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -2949,6 +4011,12 @@ "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", "dev": true }, + "node_modules/lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==", + "dev": true + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -3027,6 +4095,15 @@ "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, "node_modules/micromatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", @@ -3080,12 +4157,32 @@ "node": "*" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "dev": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -3125,6 +4222,15 @@ "node": ">=8" } }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -3200,6 +4306,12 @@ "node": ">=6" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz", + "integrity": "sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==", + "dev": true + }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -3251,6 +4363,37 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/picocolors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", @@ -3290,6 +4433,48 @@ "node": ">=8" } }, + "node_modules/postcss-load-config": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", + "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "lilconfig": "^3.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "jiti": ">=1.21.0", + "postcss": ">=8.0.9", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + }, + "postcss": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, "node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", @@ -3334,6 +4519,15 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", @@ -3350,12 +4544,44 @@ } ] }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/react-is": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", "dev": true }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3412,6 +4638,74 @@ "node": ">=10" } }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.21.2.tgz", + "integrity": "sha512-e3TapAgYf9xjdLvKQCkQTnbTKd4a6jwlpQSJJFokHGaX2IVjoEqkIIhiQfqsi0cdwlOD+tQGuOd5AJkc5RngBw==", + "dev": true, + "dependencies": { + "@types/estree": "1.0.5" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.21.2", + "@rollup/rollup-android-arm64": "4.21.2", + "@rollup/rollup-darwin-arm64": "4.21.2", + "@rollup/rollup-darwin-x64": "4.21.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.21.2", + "@rollup/rollup-linux-arm-musleabihf": "4.21.2", + "@rollup/rollup-linux-arm64-gnu": "4.21.2", + "@rollup/rollup-linux-arm64-musl": "4.21.2", + "@rollup/rollup-linux-powerpc64le-gnu": "4.21.2", + "@rollup/rollup-linux-riscv64-gnu": "4.21.2", + "@rollup/rollup-linux-s390x-gnu": "4.21.2", + "@rollup/rollup-linux-x64-gnu": "4.21.2", + "@rollup/rollup-linux-x64-musl": "4.21.2", + "@rollup/rollup-win32-arm64-msvc": "4.21.2", + "@rollup/rollup-win32-ia32-msvc": "4.21.2", + "@rollup/rollup-win32-x64-msvc": "4.21.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -3527,6 +4821,21 @@ "node": ">=8" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -3539,6 +4848,19 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-bom": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", @@ -3569,6 +4891,72 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/sucrase": { + "version": "3.35.0", + "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz", + "integrity": "sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.2", + "commander": "^4.0.0", + "glob": "^10.3.10", + "lines-and-columns": "^1.1.6", + "mz": "^2.7.0", + "pirates": "^4.0.1", + "ts-interface-checker": "^0.1.9" + }, + "bin": { + "sucrase": "bin/sucrase", + "sucrase-node": "bin/sucrase-node" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/sucrase/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/sucrase/node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "dev": true, + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/sucrase/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -3607,6 +4995,27 @@ "node": ">=8" } }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -3634,6 +5043,30 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "dev": true, + "bin": { + "tree-kill": "cli.js" + } + }, + "node_modules/ts-interface-checker": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", + "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", + "dev": true + }, "node_modules/ts-jest": { "version": "29.2.2", "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.2.2.tgz", @@ -3715,6 +5148,69 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true }, + "node_modules/tsup": { + "version": "8.2.4", + "resolved": "https://registry.npmjs.org/tsup/-/tsup-8.2.4.tgz", + "integrity": "sha512-akpCPePnBnC/CXgRrcy72ZSntgIEUa1jN0oJbbvpALWKNOz1B7aM+UVDWGRGIO/T/PZugAESWDJUAb5FD48o8Q==", + "dev": true, + "dependencies": { + "bundle-require": "^5.0.0", + "cac": "^6.7.14", + "chokidar": "^3.6.0", + "consola": "^3.2.3", + "debug": "^4.3.5", + "esbuild": "^0.23.0", + "execa": "^5.1.1", + "globby": "^11.1.0", + "joycon": "^3.1.1", + "picocolors": "^1.0.1", + "postcss-load-config": "^6.0.1", + "resolve-from": "^5.0.0", + "rollup": "^4.19.0", + "source-map": "0.8.0-beta.0", + "sucrase": "^3.35.0", + "tree-kill": "^1.2.2" + }, + "bin": { + "tsup": "dist/cli-default.js", + "tsup-node": "dist/cli-node.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@microsoft/api-extractor": "^7.36.0", + "@swc/core": "^1", + "postcss": "^8.4.12", + "typescript": ">=4.5.0" + }, + "peerDependenciesMeta": { + "@microsoft/api-extractor": { + "optional": true + }, + "@swc/core": { + "optional": true + }, + "postcss": { + "optional": true + }, + "typescript": { + "optional": true + } + } + }, + "node_modules/tsup/node_modules/source-map": { + "version": "0.8.0-beta.0", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.8.0-beta.0.tgz", + "integrity": "sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==", + "dev": true, + "dependencies": { + "whatwg-url": "^7.0.0" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -3825,6 +5321,23 @@ "makeerror": "1.0.12" } }, + "node_modules/webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==", + "dev": true + }, + "node_modules/whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "dev": true, + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -3857,6 +5370,24 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 430cffff..f717365e 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -4,9 +4,16 @@ "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", + "exports": { + "./package.json": "./package.json", + ".": { + "import": "./dist/index.js", + "default": "./dist/index.cjs" + } + }, "type": "module", "scripts": { - "build": "tsc", + "build": "tsup", "build-and-publish": "npm run build && npm publish --access public", "publish-beta": "npm run build && npm publish --access public --tag beta", "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts" @@ -40,6 +47,7 @@ "@types/uuid": "^9.0.8", "jest": "^29.7.0", "ts-jest": "^29.2.2", + "tsup": "^8.2.4", "typescript": "^5.4.5" }, "keywords": [ diff --git a/apps/js-sdk/firecrawl/tsconfig.json b/apps/js-sdk/firecrawl/tsconfig.json index 071b13ce..1297aed9 100644 --- a/apps/js-sdk/firecrawl/tsconfig.json +++ b/apps/js-sdk/firecrawl/tsconfig.json @@ -16,17 +16,9 @@ "noUncheckedIndexedAccess": true, "noImplicitOverride": true, - /* If transpiling with TypeScript: */ + /* If NOT transpiling with TypeScript: */ "module": "NodeNext", - "outDir": "dist", - "rootDir": "src", - "sourceMap": true, - - /* AND if you're building for a library: */ - "declaration": true, - - /* AND if you're building for a library in a monorepo: */ - "declarationMap": true /* Skip type checking all .d.ts files. */ + "noEmit": true, }, "include": ["src/**/*"], "exclude": ["node_modules", "dist", "**/__tests__/*"] diff --git a/apps/js-sdk/firecrawl/tsup.config.ts b/apps/js-sdk/firecrawl/tsup.config.ts new file mode 100644 index 00000000..b3b7e42d --- /dev/null +++ b/apps/js-sdk/firecrawl/tsup.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entryPoints: ["src/index.ts"], + format: ["cjs", "esm"], + dts: true, + outDir: "dist", + clean: true, +}); \ No newline at end of file From 4cd1065ae22ff24df26a495f277193330beb30a6 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 11 Sep 2024 14:03:34 -0400 Subject: [PATCH 055/102] Update rate-limiter.ts --- apps/api/src/services/rate-limiter.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index 7cfff35b..1a40671a 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -112,14 +112,16 @@ export const scrapeStatusRateLimiter = new RateLimiterRedis({ duration: 60, // Duration in seconds }); +const testSuiteTokens = ["a01ccae", "6254cf9", "0f96e673", "23befa1b", "69141c4"]; + export function getRateLimiter( mode: RateLimiterMode, token: string, plan?: string, teamId?: string ) { - - if (token.includes("a01ccae") || token.includes("6254cf9") || token.includes("0f96e673") || token.includes("23befa1b")) { + + if (testSuiteTokens.some(testToken => token.includes(testToken))) { return testSuiteRateLimiter; } From 6e1cf2f40d6877de1d0f1cb666cfaa9403d14338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 11 Sep 2024 20:15:34 +0200 Subject: [PATCH 056/102] feat(js-sdk): fixes, update tests --- apps/js-sdk/firecrawl/package-lock.json | 4 +- .../__tests__/v1/e2e_withAuth/index.test.ts | 199 ++++++++++-------- apps/js-sdk/firecrawl/src/index.ts | 6 +- 3 files changed, 118 insertions(+), 91 deletions(-) diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json index 83745a5b..641f55fc 100644 --- a/apps/js-sdk/firecrawl/package-lock.json +++ b/apps/js-sdk/firecrawl/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mendable/firecrawl-js", - "version": "1.2.1", + "version": "1.2.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mendable/firecrawl-js", - "version": "1.2.1", + "version": "1.2.3", "license": "MIT", "dependencies": { "axios": "^1.6.8", diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts index 9f6c6462..5eadd92e 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts @@ -1,4 +1,4 @@ -import FirecrawlApp, { CrawlParams, CrawlResponse, CrawlStatusResponse, MapResponse, ScrapeParams, ScrapeResponse } from '../../../index'; +import FirecrawlApp, { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, type MapResponse, type ScrapeResponse } from '../../../index'; import { v4 as uuidv4 } from 'uuid'; import dotenv from 'dotenv'; import { describe, test, expect } from '@jest/globals'; @@ -6,7 +6,7 @@ import { describe, test, expect } from '@jest/globals'; dotenv.config(); const TEST_API_KEY = process.env.TEST_API_KEY; -const API_URL = "http://127.0.0.1:3002"; +const API_URL = "https://api.firecrawl.dev"; describe('FirecrawlApp E2E Tests', () => { test.concurrent('should throw error for no API key', async () => { @@ -71,6 +71,7 @@ describe('FirecrawlApp E2E Tests', () => { expect(response.links?.length).toBeGreaterThan(0); expect(response.links?.[0]).toContain("https://"); expect(response.metadata).not.toBeNull(); + expect(response.metadata).not.toBeUndefined(); expect(response.metadata).toHaveProperty("title"); expect(response.metadata).toHaveProperty("description"); expect(response.metadata).toHaveProperty("keywords"); @@ -85,19 +86,21 @@ describe('FirecrawlApp E2E Tests', () => { expect(response.metadata).not.toHaveProperty("pageStatusCode"); expect(response.metadata).toHaveProperty("statusCode"); expect(response.metadata).not.toHaveProperty("pageError"); - expect(response.metadata.error).toBeUndefined(); - expect(response.metadata.title).toBe("Roast My Website"); - expect(response.metadata.description).toBe("Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"); - expect(response.metadata.keywords).toBe("Roast My Website,Roast,Website,GitHub,Firecrawl"); - expect(response.metadata.robots).toBe("follow, index"); - expect(response.metadata.ogTitle).toBe("Roast My Website"); - expect(response.metadata.ogDescription).toBe("Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"); - expect(response.metadata.ogUrl).toBe("https://www.roastmywebsite.ai"); - expect(response.metadata.ogImage).toBe("https://www.roastmywebsite.ai/og.png"); - expect(response.metadata.ogLocaleAlternate).toStrictEqual([]); - expect(response.metadata.ogSiteName).toBe("Roast My Website"); - expect(response.metadata.sourceURL).toBe("https://roastmywebsite.ai"); - expect(response.metadata.statusCode).toBe(200); + if (response.metadata !== undefined) { + expect(response.metadata.error).toBeUndefined(); + expect(response.metadata.title).toBe("Roast My Website"); + expect(response.metadata.description).toBe("Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"); + expect(response.metadata.keywords).toBe("Roast My Website,Roast,Website,GitHub,Firecrawl"); + expect(response.metadata.robots).toBe("follow, index"); + expect(response.metadata.ogTitle).toBe("Roast My Website"); + expect(response.metadata.ogDescription).toBe("Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️"); + expect(response.metadata.ogUrl).toBe("https://www.roastmywebsite.ai"); + expect(response.metadata.ogImage).toBe("https://www.roastmywebsite.ai/og.png"); + expect(response.metadata.ogLocaleAlternate).toStrictEqual([]); + expect(response.metadata.ogSiteName).toBe("Roast My Website"); + expect(response.metadata.sourceURL).toBe("https://roastmywebsite.ai"); + expect(response.metadata.statusCode).toBe(200); + } }, 30000); // 30 seconds timeout test.concurrent('should return successful response for valid scrape with PDF file', async () => { @@ -127,7 +130,7 @@ describe('FirecrawlApp E2E Tests', () => { test.concurrent('should return successful response for crawl and wait for completion', async () => { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); - const response = await app.crawlUrl('https://roastmywebsite.ai', {}, true, 30) as CrawlStatusResponse; + const response = await app.crawlUrl('https://roastmywebsite.ai', {}, 30) as CrawlStatusResponse; expect(response).not.toBeNull(); expect(response).toHaveProperty("total"); expect(response.total).toBeGreaterThan(0); @@ -138,21 +141,25 @@ describe('FirecrawlApp E2E Tests', () => { expect(response).toHaveProperty("status"); expect(response.status).toBe("completed"); expect(response).not.toHaveProperty("next"); // wait until done - expect(response.data?.length).toBeGreaterThan(0); - expect(response.data?.[0]).toHaveProperty("markdown"); - expect(response.data?.[0].markdown).toContain("_Roast_"); - expect(response.data?.[0]).not.toHaveProperty('content'); // v0 - expect(response.data?.[0]).not.toHaveProperty("html"); - expect(response.data?.[0]).not.toHaveProperty("rawHtml"); - expect(response.data?.[0]).not.toHaveProperty("screenshot"); - expect(response.data?.[0]).not.toHaveProperty("links"); - expect(response.data?.[0]).toHaveProperty("metadata"); - expect(response.data?.[0].metadata).toHaveProperty("title"); - expect(response.data?.[0].metadata).toHaveProperty("description"); - expect(response.data?.[0].metadata).toHaveProperty("language"); - expect(response.data?.[0].metadata).toHaveProperty("sourceURL"); - expect(response.data?.[0].metadata).toHaveProperty("statusCode"); - expect(response.data?.[0].metadata).not.toHaveProperty("error"); + expect(response.data.length).toBeGreaterThan(0); + expect(response.data[0]).not.toBeNull(); + expect(response.data[0]).not.toBeUndefined(); + if (response.data[0]) { + expect(response.data[0]).toHaveProperty("markdown"); + expect(response.data[0].markdown).toContain("_Roast_"); + expect(response.data[0]).not.toHaveProperty('content'); // v0 + expect(response.data[0]).not.toHaveProperty("html"); + expect(response.data[0]).not.toHaveProperty("rawHtml"); + expect(response.data[0]).not.toHaveProperty("screenshot"); + expect(response.data[0]).not.toHaveProperty("links"); + expect(response.data[0]).toHaveProperty("metadata"); + expect(response.data[0].metadata).toHaveProperty("title"); + expect(response.data[0].metadata).toHaveProperty("description"); + expect(response.data[0].metadata).toHaveProperty("language"); + expect(response.data[0].metadata).toHaveProperty("sourceURL"); + expect(response.data[0].metadata).toHaveProperty("statusCode"); + expect(response.data[0].metadata).not.toHaveProperty("error"); + } }, 60000); // 60 seconds timeout test.concurrent('should return successful response for crawl with options and wait for completion', async () => { @@ -173,7 +180,7 @@ describe('FirecrawlApp E2E Tests', () => { onlyMainContent: true, waitFor: 1000 } - } as CrawlParams, true, 30) as CrawlStatusResponse; + } as CrawlParams, 30) as CrawlStatusResponse; expect(response).not.toBeNull(); expect(response).toHaveProperty("total"); expect(response.total).toBeGreaterThan(0); @@ -184,41 +191,45 @@ describe('FirecrawlApp E2E Tests', () => { expect(response).toHaveProperty("status"); expect(response.status).toBe("completed"); expect(response).not.toHaveProperty("next"); - expect(response.data?.length).toBeGreaterThan(0); - expect(response.data?.[0]).toHaveProperty("markdown"); - expect(response.data?.[0].markdown).toContain("_Roast_"); - expect(response.data?.[0]).not.toHaveProperty('content'); // v0 - expect(response.data?.[0]).toHaveProperty("html"); - expect(response.data?.[0].html).toContain(" { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const uniqueIdempotencyKey = uuidv4(); - const response = await app.crawlUrl('https://roastmywebsite.ai', {}, false, 2, uniqueIdempotencyKey) as CrawlResponse; + const response = await app.asyncCrawlUrl('https://roastmywebsite.ai', {}, uniqueIdempotencyKey) as CrawlResponse; expect(response).not.toBeNull(); expect(response.id).toBeDefined(); - await expect(app.crawlUrl('https://roastmywebsite.ai', {}, true, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409"); + await expect(app.crawlUrl('https://roastmywebsite.ai', {}, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409"); }); test.concurrent('should check crawl status', async () => { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); - const response = await app.crawlUrl('https://firecrawl.dev', { scrapeOptions: { formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links']}} as CrawlParams, false) as CrawlResponse; + const response = await app.asyncCrawlUrl('https://firecrawl.dev', { scrapeOptions: { formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links']}} as CrawlParams) as CrawlResponse; expect(response).not.toBeNull(); expect(response.id).toBeDefined(); @@ -226,7 +237,8 @@ describe('FirecrawlApp E2E Tests', () => { const maxChecks = 15; let checks = 0; - while (statusResponse.status === 'scraping' && checks < maxChecks) { + expect(statusResponse.success).toBe(true); + while ((statusResponse as any).status === 'scraping' && checks < maxChecks) { await new Promise(resolve => setTimeout(resolve, 5000)); expect(statusResponse).not.toHaveProperty("partial_data"); // v0 expect(statusResponse).not.toHaveProperty("current"); // v0 @@ -236,44 +248,55 @@ describe('FirecrawlApp E2E Tests', () => { expect(statusResponse).toHaveProperty("expiresAt"); expect(statusResponse).toHaveProperty("status"); expect(statusResponse).toHaveProperty("next"); - expect(statusResponse.total).toBeGreaterThan(0); - expect(statusResponse.creditsUsed).toBeGreaterThan(0); - expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now()); - expect(statusResponse.status).toBe("scraping"); - expect(statusResponse.next).toContain("/v1/crawl/"); + expect(statusResponse.success).toBe(true); + if (statusResponse.success === true) { + expect(statusResponse.total).toBeGreaterThan(0); + expect(statusResponse.creditsUsed).toBeGreaterThan(0); + expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now()); + expect(statusResponse.status).toBe("scraping"); + expect(statusResponse.next).toContain("/v1/crawl/"); + } statusResponse = await app.checkCrawlStatus(response.id) as CrawlStatusResponse; + expect(statusResponse.success).toBe(true); checks++; } expect(statusResponse).not.toBeNull(); expect(statusResponse).toHaveProperty("total"); - expect(statusResponse.total).toBeGreaterThan(0); - expect(statusResponse).toHaveProperty("creditsUsed"); - expect(statusResponse.creditsUsed).toBeGreaterThan(0); - expect(statusResponse).toHaveProperty("expiresAt"); - expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now()); - expect(statusResponse).toHaveProperty("status"); - expect(statusResponse.status).toBe("completed"); - expect(statusResponse.data?.length).toBeGreaterThan(0); - expect(statusResponse.data?.[0]).toHaveProperty("markdown"); - expect(statusResponse.data?.[0].markdown?.length).toBeGreaterThan(10); - expect(statusResponse.data?.[0]).not.toHaveProperty('content'); // v0 - expect(statusResponse.data?.[0]).toHaveProperty("html"); - expect(statusResponse.data?.[0].html).toContain(" { diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 6c61c628..6248789b 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -183,7 +183,11 @@ export default class FirecrawlApp { * @param config - Configuration options for the FirecrawlApp instance. */ constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) { - this.apiKey = apiKey || ""; + if (typeof apiKey !== "string") { + throw new Error("No API key provided"); + } + + this.apiKey = apiKey; this.apiUrl = apiUrl || "https://api.firecrawl.dev"; } From ad70c30be537fa4aa8283ae98331eb828c0a276b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 11 Sep 2024 20:31:58 +0200 Subject: [PATCH 057/102] fix(js-sdk): check at bad if --- apps/js-sdk/firecrawl/src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 6248789b..115e62e9 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -186,7 +186,7 @@ export default class FirecrawlApp { if (typeof apiKey !== "string") { throw new Error("No API key provided"); } - + this.apiKey = apiKey; this.apiUrl = apiUrl || "https://api.firecrawl.dev"; } @@ -346,9 +346,9 @@ export default class FirecrawlApp { `${this.apiUrl}/v1/crawl/${id}`, headers ); - if (response.status === 200 && getAllData) { + if (response.status === 200) { let allData = response.data.data; - if (response.data.status === "completed") { + if (getAllData && response.data.status === "completed") { let statusData = response.data if ("data" in statusData) { let data = statusData.data; From 5adfd74cc5a6e251d0ad6fc4bc4be7cc7cd6d04c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 11 Sep 2024 20:32:34 +0200 Subject: [PATCH 058/102] feat(js-sdk/test): add API_URL env var --- .../firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts index 5eadd92e..98a52538 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts @@ -6,7 +6,7 @@ import { describe, test, expect } from '@jest/globals'; dotenv.config(); const TEST_API_KEY = process.env.TEST_API_KEY; -const API_URL = "https://api.firecrawl.dev"; +const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev"; describe('FirecrawlApp E2E Tests', () => { test.concurrent('should throw error for no API key', async () => { From 99c1af0a9f6d2cf00e3ffce33ad7984aca345548 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 11 Sep 2024 14:59:36 -0400 Subject: [PATCH 059/102] Update package.json --- apps/js-sdk/firecrawl/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index f5de3159..c3135aca 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.2.3", + "version": "1.3.0", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", From 503c8b3efa46697258e2decfd0892bf103679afd Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 12 Sep 2024 11:35:26 -0400 Subject: [PATCH 060/102] Update package-lock.json --- apps/js-sdk/firecrawl/package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json index 641f55fc..2dcca44d 100644 --- a/apps/js-sdk/firecrawl/package-lock.json +++ b/apps/js-sdk/firecrawl/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mendable/firecrawl-js", - "version": "1.2.3", + "version": "1.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mendable/firecrawl-js", - "version": "1.2.3", + "version": "1.3.0", "license": "MIT", "dependencies": { "axios": "^1.6.8", From eec22a56d3dce2253da390afd17753850e70ae31 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 12 Sep 2024 11:43:40 -0400 Subject: [PATCH 061/102] Nick: self host issue template --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .github/ISSUE_TEMPLATE/self_host_issue.md | 40 +++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/self_host_issue.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index bb47b47f..bbc1e098 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a report to help us improve -title: "[BUG]" +title: "[Bug] " labels: bug assignees: '' diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index b01699b7..6760afa8 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,7 +1,7 @@ --- name: Feature request about: Suggest an idea for this project -title: "[Feat]" +title: "[Feat] " labels: '' assignees: '' diff --git a/.github/ISSUE_TEMPLATE/self_host_issue.md b/.github/ISSUE_TEMPLATE/self_host_issue.md new file mode 100644 index 00000000..73a0ef9d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/self_host_issue.md @@ -0,0 +1,40 @@ +--- +name: Self-host issue +about: Report an issue with self-hosting Firecrawl +title: "[Self-Host] " +labels: self-host +assignees: '' + +--- + +**Describe the Issue** +Provide a clear and concise description of the self-hosting issue you're experiencing. + +**To Reproduce** +Steps to reproduce the issue: +1. Configure the environment or settings with '...' +2. Run the command '...' +3. Observe the error or unexpected output at '...' +4. Log output/error message + +**Expected Behavior** +A clear and concise description of what you expected to happen when self-hosting. + +**Screenshots** +If applicable, add screenshots or copies of the command line output to help explain the self-hosting issue. + +**Environment (please complete the following information):** +- OS: [e.g. macOS, Linux, Windows] +- Firecrawl Version: [e.g. 1.2.3] +- Node.js Version: [e.g. 14.x] +- Docker Version (if applicable): [e.g. 20.10.14] +- Database Type and Version: [e.g. PostgreSQL 13.4] + +**Logs** +If applicable, include detailed logs to help understand the self-hosting problem. + +**Configuration** +Provide relevant parts of your configuration files (with sensitive information redacted). + +**Additional Context** +Add any other context about the self-hosting issue here, such as specific infrastructure details, network setup, or any modifications made to the original Firecrawl setup. From a2903e75cfb06fa7cbbb31e663908c70c42a544d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 12 Sep 2024 18:48:19 +0200 Subject: [PATCH 062/102] feat(js-sdk): type-safe LLM extract --- apps/js-sdk/firecrawl/src/index.ts | 43 ++++++++++++++++-------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 115e62e9..124a84e8 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -1,5 +1,5 @@ import axios, { type AxiosResponse, type AxiosRequestHeaders } from "axios"; -import type { ZodSchema } from "zod"; +import type { infer as ZodInfer, ZodSchema } from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import { WebSocket } from "isows"; import { TypedEventTarget } from "typescript-event-target"; @@ -58,13 +58,13 @@ export interface FirecrawlDocumentMetadata { * Document interface for Firecrawl. * Represents a document retrieved or processed by Firecrawl. */ -export interface FirecrawlDocument { +export interface FirecrawlDocument { url?: string; markdown?: string; html?: string; rawHtml?: string; links?: string[]; - extract?: Record; + extract?: T; screenshot?: string; metadata?: FirecrawlDocumentMetadata; } @@ -73,26 +73,29 @@ export interface FirecrawlDocument { * Parameters for scraping operations. * Defines the options and configurations available for scraping web content. */ -export interface ScrapeParams { +export interface CrawlScrapeOptions { formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract" | "full@scrennshot")[]; headers?: Record; includeTags?: string[]; excludeTags?: string[]; onlyMainContent?: boolean; - extract?: { - prompt?: string; - schema?: ZodSchema | any; - systemPrompt?: string; - }; waitFor?: number; timeout?: number; } +export interface ScrapeParams extends CrawlScrapeOptions { + extract?: { + prompt?: string; + schema?: LLMSchema; + systemPrompt?: string; + }; +} + /** * Response interface for scraping operations. * Defines the structure of the response received after a scraping operation. */ -export interface ScrapeResponse extends FirecrawlDocument { +export interface ScrapeResponse extends FirecrawlDocument { success: true; warning?: string; error?: string; @@ -110,7 +113,7 @@ export interface CrawlParams { allowBackwardLinks?: boolean; allowExternalLinks?: boolean; ignoreSitemap?: boolean; - scrapeOptions?: ScrapeParams; + scrapeOptions?: CrawlScrapeOptions; webhook?: string; } @@ -137,7 +140,7 @@ export interface CrawlStatusResponse { creditsUsed: number; expiresAt: Date; next?: string; - data: FirecrawlDocument[]; + data: FirecrawlDocument[]; }; /** @@ -197,10 +200,10 @@ export default class FirecrawlApp { * @param params - Additional parameters for the scrape request. * @returns The response from the scrape operation. */ - async scrapeUrl( + async scrapeUrl( url: string, - params?: ScrapeParams - ): Promise { + params?: ScrapeParams + ): Promise> | ErrorResponse> { const headers: AxiosRequestHeaders = { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}`, @@ -528,21 +531,21 @@ export default class FirecrawlApp { } interface CrawlWatcherEvents { - document: CustomEvent, + document: CustomEvent>, done: CustomEvent<{ status: CrawlStatusResponse["status"]; - data: FirecrawlDocument[]; + data: FirecrawlDocument[]; }>, error: CustomEvent<{ status: CrawlStatusResponse["status"], - data: FirecrawlDocument[], + data: FirecrawlDocument[], error: string, }>, } export class CrawlWatcher extends TypedEventTarget { private ws: WebSocket; - public data: FirecrawlDocument[]; + public data: FirecrawlDocument[]; public status: CrawlStatusResponse["status"]; constructor(id: string, app: FirecrawlApp) { @@ -563,7 +566,7 @@ export class CrawlWatcher extends TypedEventTarget { type DocumentMessage = { type: "document", - data: FirecrawlDocument, + data: FirecrawlDocument, } type DoneMessage = { type: "done" } From 620b02f9ca7de0436b1ea0499d7c4684090d351c Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 12 Sep 2024 12:51:14 -0400 Subject: [PATCH 063/102] Nick: --- apps/api/src/controllers/v0/scrape.ts | 2 +- apps/api/src/controllers/v1/scrape.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts index 2a5f1d4f..c46ebc62 100644 --- a/apps/api/src/controllers/v0/scrape.ts +++ b/apps/api/src/controllers/v0/scrape.ts @@ -229,7 +229,7 @@ export async function scrapeController(req: Request, res: Response) { if (result.success) { let creditsToBeBilled = 1; - const creditsPerLLMExtract = 49; + const creditsPerLLMExtract = 4; if (extractorOptions.mode.includes("llm-extraction")) { // creditsToBeBilled = creditsToBeBilled + (creditsPerLLMExtract * filteredDocs.length); diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts index 0835cc2a..f0744c22 100644 --- a/apps/api/src/controllers/v1/scrape.ts +++ b/apps/api/src/controllers/v1/scrape.ts @@ -103,7 +103,7 @@ export async function scrapeController( return; } if(req.body.extract && req.body.formats.includes("extract")) { - creditsToBeBilled = 50; + creditsToBeBilled = 5; } billTeam(req.auth.team_id, creditsToBeBilled).catch(error => { From d497284b40dede0c4aae4fe68d46166207195d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 12 Sep 2024 19:47:15 +0200 Subject: [PATCH 064/102] feat(api/queue): auto-remove completed jobs after 25 hours --- apps/api/src/services/queue-service.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts index 113b3fa3..14dddebe 100644 --- a/apps/api/src/services/queue-service.ts +++ b/apps/api/src/services/queue-service.ts @@ -16,6 +16,14 @@ export function getScrapeQueue() { scrapeQueueName, { connection: redisConnection, + defaultJobOptions: { + removeOnComplete: { + age: 90000, // 25 hours + }, + removeOnFail: { + age: 90000, // 25 hours + }, + }, } // { // settings: { From d30356a22c559408547f88979a5a19ca238ff0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 12 Sep 2024 19:57:33 +0200 Subject: [PATCH 065/102] fix(js-sdk): infer keyword collision --- apps/js-sdk/firecrawl/src/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 124a84e8..949cfe98 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -1,5 +1,5 @@ import axios, { type AxiosResponse, type AxiosRequestHeaders } from "axios"; -import type { infer as ZodInfer, ZodSchema } from "zod"; +import type * as zt from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import { WebSocket } from "isows"; import { TypedEventTarget } from "typescript-event-target"; @@ -83,7 +83,7 @@ export interface CrawlScrapeOptions { timeout?: number; } -export interface ScrapeParams extends CrawlScrapeOptions { +export interface ScrapeParams extends CrawlScrapeOptions { extract?: { prompt?: string; schema?: LLMSchema; @@ -200,10 +200,10 @@ export default class FirecrawlApp { * @param params - Additional parameters for the scrape request. * @returns The response from the scrape operation. */ - async scrapeUrl( + async scrapeUrl( url: string, params?: ScrapeParams - ): Promise> | ErrorResponse> { + ): Promise> | ErrorResponse> { const headers: AxiosRequestHeaders = { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}`, From e6ac90c1a0ec04739ae6a8e9485d3449c4981c28 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 12 Sep 2024 14:01:47 -0400 Subject: [PATCH 066/102] Update package.json --- apps/js-sdk/firecrawl/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index c3135aca..f6f14fb2 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.3.0", + "version": "1.4.2", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", From 0d1b46d4763013c2f7950dc71c66d6d30429e0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 12 Sep 2024 21:30:17 +0200 Subject: [PATCH 067/102] fix(js-sdk): improve error logging --- apps/js-sdk/firecrawl/src/index.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 949cfe98..661ce34b 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -1,4 +1,4 @@ -import axios, { type AxiosResponse, type AxiosRequestHeaders } from "axios"; +import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios"; import type * as zt from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import { WebSocket } from "isows"; @@ -452,11 +452,19 @@ export default class FirecrawlApp { * @param headers - The headers for the request. * @returns The response from the GET request. */ - getRequest( + async getRequest( url: string, headers: AxiosRequestHeaders ): Promise { - return axios.get(url, { headers }); + try { + return await axios.get(url, { headers }); + } catch (error) { + if (error instanceof AxiosError && error.response) { + return error.response as AxiosResponse; + } else { + throw error; + } + } } /** From f7c4cee404e17b3ed201e005185a5041009d0e6f Mon Sep 17 00:00:00 2001 From: Gergo Moricz Date: Fri, 13 Sep 2024 14:02:49 +0200 Subject: [PATCH 068/102] fix(queue-worker): don't send LLM extract hallucination error to Sentry --- apps/api/src/services/queue-worker.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index ad0e4ad5..37e14baf 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -448,11 +448,13 @@ async function processJob(job: Job, token: string) { } catch (error) { Logger.error(`🐂 Job errored ${job.id} - ${error}`); - Sentry.captureException(error, { - data: { - job: job.id, - }, - }); + if (!(error instanceof Error && error.message.includes("JSON parsing error(s): "))) { + Sentry.captureException(error, { + data: { + job: job.id, + }, + }); + } if (error instanceof CustomError) { // Here we handle the error, then save the failed job From 000a316cc362b935976ac47b73ec02923f4175c5 Mon Sep 17 00:00:00 2001 From: Gergo Moricz Date: Fri, 13 Sep 2024 16:41:27 +0200 Subject: [PATCH 069/102] fix(fire-engine): poll more frequently --- apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index a3f393c8..80ac7924 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -127,7 +127,7 @@ export async function scrapWithFireEngine({ let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`); while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitParam) { - await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second + await new Promise(resolve => setTimeout(resolve, 250)); // wait 0.25 seconds checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`); } From 2ee7d1d0e4a99008b9356cd08dca50d7cb25d975 Mon Sep 17 00:00:00 2001 From: Eric Ciarla Date: Fri, 13 Sep 2024 15:08:23 -0400 Subject: [PATCH 070/102] init --- .gitignore | 1 + examples/o1_web_crawler /o1_web_crawler.py | 132 +++++++++++++++++++++ examples/o1_web_crawler /requirements.txt | 3 + 3 files changed, 136 insertions(+) create mode 100644 examples/o1_web_crawler /o1_web_crawler.py create mode 100644 examples/o1_web_crawler /requirements.txt diff --git a/.gitignore b/.gitignore index 91b7ef48..367f28a7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ apps/test-suite/load-test-results/test-run-report.json apps/playwright-service-ts/node_modules/ apps/playwright-service-ts/package-lock.json +/examples/o1_web_crawler /venv diff --git a/examples/o1_web_crawler /o1_web_crawler.py b/examples/o1_web_crawler /o1_web_crawler.py new file mode 100644 index 00000000..497dd771 --- /dev/null +++ b/examples/o1_web_crawler /o1_web_crawler.py @@ -0,0 +1,132 @@ +import os +from firecrawl import FirecrawlApp +import json +from dotenv import load_dotenv +from openai import OpenAI + +# Load environment variables +load_dotenv() + +# Retrieve API keys from environment variables +firecrawl_api_key = os.getenv("FIRECRAWL_API_KEY") +openai_api_key = os.getenv("OPENAI_API_KEY") + +# Initialize the FirecrawlApp and OpenAI client +app = FirecrawlApp(api_key=firecrawl_api_key) +client = OpenAI(api_key=openai_api_key) + +# Find the page that most likely contains the objective +def find_relevant_page_via_map(objective, url, app, client): + try: + print(f"Okay, the objective is: {objective}") + print(f"I am going to search the website: {url}") + + map_prompt = f""" + The map function generates a list of URLs from a website and it accepts a search parameter. Based on the objective of: {objective}, come up with a 1-2 word search parameter that will help us find the information we need. Only respond with 1-2 words nothing else. + """ + + print("I'm asking the AI to suggest a search parameter...") + completion = client.chat.completions.create( + model="o1-preview", + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": map_prompt + } + ] + } + ] + ) + + map_search_parameter = completion.choices[0].message.content + print(f"I think the search parameter should be: {map_search_parameter}") + + print(f"Now I'm going to map the website using this search parameter...") + map_website = app.map_url(url, params=map_search_parameter) + print("I've successfully mapped the website!") + return map_website + except Exception as e: + print(f"Oops! An error occurred while finding the relevant page: {str(e)}") + return None + +# Scrape the top 3 pages and see if the objective is met, if so return in json format else return None +def find_objective_in_top_pages(map_website, objective, app, client): + try: + # Get top 3 links from the map result + top_links = map_website['links'][:3] + print(f"I'm going to check the top 3 links: {top_links}") + + for link in top_links: + print(f"Now I'm scraping this page: {link}") + # Scrape the page + scrape_result = app.scrape_url(link, params={'formats': ['markdown']}) + print("I've successfully scraped the page!") + + # Check if objective is met + check_prompt = f""" + Given the following scraped content and objective, determine if the objective is met. + If it is, extract the relevant information in JSON format. + If not, respond with 'Objective not met'. + + Objective: {objective} + Scraped content: {scrape_result['data']['markdown']} + """ + + print("I'm asking the AI to check if the objective is met...") + completion = client.chat.completions.create( + model="o1-preview", + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": check_prompt + } + ] + } + ] + ) + + result = completion.choices[0].message.content + + if result != "Objective not met": + print("Great news! I think I've found what we're looking for!") + return json.loads(result) + else: + print("This page doesn't seem to have what we need. Moving to the next one...") + + print("I've checked all 3 pages, but couldn't find what we're looking for.") + return None + except Exception as e: + print(f"Oh no! An error occurred while scraping top pages: {str(e)}") + return None + +# Main function to execute the process +def main(): + # Get user input + url = input("Enter the website to crawl: ") + objective = input("Enter your objective: ") + + print("Alright, let's get started!") + # Find the relevant page + map_website = find_relevant_page_via_map(objective, url, app, client) + + if map_website: + print("Great! I've found some relevant pages. Now let's see if we can find what we're looking for...") + # Find objective in top pages + result = find_objective_in_top_pages(map_website, objective, app, client) + + if result: + print("Success! I've found what you're looking for. Here's the extracted information:") + print(json.dumps(result, indent=2)) + else: + print("I'm sorry, but I couldn't find what you're looking for in the top 3 pages.") + else: + print("I'm afraid I couldn't find any relevant pages. Maybe we could try a different website or rephrase the objective?") + +if __name__ == "__main__": + main() diff --git a/examples/o1_web_crawler /requirements.txt b/examples/o1_web_crawler /requirements.txt new file mode 100644 index 00000000..249f8beb --- /dev/null +++ b/examples/o1_web_crawler /requirements.txt @@ -0,0 +1,3 @@ +firecrawl-py +python-dotenv +openai \ No newline at end of file From 030ecab6eebd4ad73945e5faf315f1cc547a3277 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 13 Sep 2024 18:09:59 -0400 Subject: [PATCH 071/102] Update rate-limiter.ts --- apps/api/src/services/rate-limiter.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index 1a40671a..51a0ecfa 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -104,6 +104,13 @@ export const devBRateLimiter = new RateLimiterRedis({ duration: 60, // Duration in seconds }); +export const manualRateLimiter = new RateLimiterRedis({ + storeClient: redisRateLimitClient, + keyPrefix: "manual", + points: 2000, + duration: 60, // Duration in seconds +}); + export const scrapeStatusRateLimiter = new RateLimiterRedis({ storeClient: redisRateLimitClient, @@ -114,6 +121,8 @@ export const scrapeStatusRateLimiter = new RateLimiterRedis({ const testSuiteTokens = ["a01ccae", "6254cf9", "0f96e673", "23befa1b", "69141c4"]; +const manual = ["69be9e74-7624-4990-b20d-08e0acc70cf6"]; + export function getRateLimiter( mode: RateLimiterMode, token: string, @@ -129,6 +138,10 @@ export function getRateLimiter( return devBRateLimiter; } + if(teamId && manual.includes(teamId)) { + return manualRateLimiter; + } + const rateLimitConfig = RATE_LIMITS[mode]; // {default : 5} if (!rateLimitConfig) return serverRateLimiter; From 3900603a28910af1b6fed23835774196b2da45c6 Mon Sep 17 00:00:00 2001 From: Eric Ciarla Date: Mon, 16 Sep 2024 11:18:57 -0400 Subject: [PATCH 072/102] Almost done --- examples/o1_web_crawler /o1_web_crawler.py | 65 +++++++++++++--------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/examples/o1_web_crawler /o1_web_crawler.py b/examples/o1_web_crawler /o1_web_crawler.py index 497dd771..a0f98034 100644 --- a/examples/o1_web_crawler /o1_web_crawler.py +++ b/examples/o1_web_crawler /o1_web_crawler.py @@ -18,14 +18,14 @@ client = OpenAI(api_key=openai_api_key) # Find the page that most likely contains the objective def find_relevant_page_via_map(objective, url, app, client): try: - print(f"Okay, the objective is: {objective}") - print(f"I am going to search the website: {url}") + print(f"Understood. The objective is: {objective}") + print(f"Initiating search on the website: {url}") map_prompt = f""" The map function generates a list of URLs from a website and it accepts a search parameter. Based on the objective of: {objective}, come up with a 1-2 word search parameter that will help us find the information we need. Only respond with 1-2 words nothing else. """ - print("I'm asking the AI to suggest a search parameter...") + print("Analyzing objective to determine optimal search parameter...") completion = client.chat.completions.create( model="o1-preview", messages=[ @@ -42,40 +42,47 @@ def find_relevant_page_via_map(objective, url, app, client): ) map_search_parameter = completion.choices[0].message.content - print(f"I think the search parameter should be: {map_search_parameter}") + print(f"Optimal search parameter identified: {map_search_parameter}") - print(f"Now I'm going to map the website using this search parameter...") - map_website = app.map_url(url, params=map_search_parameter) - print("I've successfully mapped the website!") + print(f"Mapping website using the identified search parameter...") + map_website = app.map_url(url, params={"search": map_search_parameter}) + print("Website mapping completed successfully.") + print(f"Located {len(map_website)} relevant links.") return map_website except Exception as e: - print(f"Oops! An error occurred while finding the relevant page: {str(e)}") + print(f"Error encountered during relevant page identification: {str(e)}") return None # Scrape the top 3 pages and see if the objective is met, if so return in json format else return None def find_objective_in_top_pages(map_website, objective, app, client): try: # Get top 3 links from the map result - top_links = map_website['links'][:3] - print(f"I'm going to check the top 3 links: {top_links}") + top_links = map_website[:3] if isinstance(map_website, list) else [] + print(f"Proceeding to analyze top {len(top_links)} links: {top_links}") for link in top_links: - print(f"Now I'm scraping this page: {link}") + print(f"Initiating scrape of page: {link}") # Scrape the page scrape_result = app.scrape_url(link, params={'formats': ['markdown']}) - print("I've successfully scraped the page!") + print("Page scraping completed successfully.") + # Check if objective is met check_prompt = f""" - Given the following scraped content and objective, determine if the objective is met. - If it is, extract the relevant information in JSON format. - If not, respond with 'Objective not met'. + Given the following scraped content and objective, determine if the objective is met with high confidence. + If it is, extract the relevant information in a simple and concise JSON format. Use only the necessary fields and avoid nested structures if possible. + If the objective is not met with high confidence, respond with 'Objective not met'. Objective: {objective} - Scraped content: {scrape_result['data']['markdown']} + Scraped content: {scrape_result['markdown']} + + Remember: + 1. Only return JSON if you are highly confident the objective is fully met. + 2. Keep the JSON structure as simple and flat as possible. + 3. Do not include any explanations or markdown formatting in your response. """ - print("I'm asking the AI to check if the objective is met...") + print("Analyzing scraped content to determine objective fulfillment...") completion = client.chat.completions.create( model="o1-preview", messages=[ @@ -94,15 +101,19 @@ def find_objective_in_top_pages(map_website, objective, app, client): result = completion.choices[0].message.content if result != "Objective not met": - print("Great news! I think I've found what we're looking for!") - return json.loads(result) + print("Objective potentially fulfilled. Relevant information identified.") + try: + print(result) + return json.loads(result) + except json.JSONDecodeError: + print("Error in parsing response. Proceeding to next page...") else: - print("This page doesn't seem to have what we need. Moving to the next one...") + print("Objective not met on this page. Proceeding to next link...") - print("I've checked all 3 pages, but couldn't find what we're looking for.") + print("All available pages analyzed. Objective not fulfilled in examined content.") return None except Exception as e: - print(f"Oh no! An error occurred while scraping top pages: {str(e)}") + print(f"Error encountered during page analysis: {str(e)}") return None # Main function to execute the process @@ -111,22 +122,22 @@ def main(): url = input("Enter the website to crawl: ") objective = input("Enter your objective: ") - print("Alright, let's get started!") + print("Initiating web crawling process.") # Find the relevant page map_website = find_relevant_page_via_map(objective, url, app, client) if map_website: - print("Great! I've found some relevant pages. Now let's see if we can find what we're looking for...") + print("Relevant pages identified. Proceeding with detailed analysis...") # Find objective in top pages result = find_objective_in_top_pages(map_website, objective, app, client) if result: - print("Success! I've found what you're looking for. Here's the extracted information:") + print("Objective successfully fulfilled. Extracted information:") print(json.dumps(result, indent=2)) else: - print("I'm sorry, but I couldn't find what you're looking for in the top 3 pages.") + print("Unable to fulfill the objective with the available content.") else: - print("I'm afraid I couldn't find any relevant pages. Maybe we could try a different website or rephrase the objective?") + print("No relevant pages identified. Consider refining the search parameters or trying a different website.") if __name__ == "__main__": main() From 8c05aed6e9998889802dc03c3e696211f87f2ebe Mon Sep 17 00:00:00 2001 From: Eric Ciarla Date: Mon, 16 Sep 2024 11:30:25 -0400 Subject: [PATCH 073/102] Finishing o1 crawler example --- examples/o1_web_crawler /o1_web_crawler.py | 61 +++++++++++++--------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/examples/o1_web_crawler /o1_web_crawler.py b/examples/o1_web_crawler /o1_web_crawler.py index a0f98034..e4fee3e8 100644 --- a/examples/o1_web_crawler /o1_web_crawler.py +++ b/examples/o1_web_crawler /o1_web_crawler.py @@ -4,6 +4,16 @@ import json from dotenv import load_dotenv from openai import OpenAI +# ANSI color codes +class Colors: + CYAN = '\033[96m' + YELLOW = '\033[93m' + GREEN = '\033[92m' + RED = '\033[91m' + MAGENTA = '\033[95m' + BLUE = '\033[94m' + RESET = '\033[0m' + # Load environment variables load_dotenv() @@ -18,14 +28,14 @@ client = OpenAI(api_key=openai_api_key) # Find the page that most likely contains the objective def find_relevant_page_via_map(objective, url, app, client): try: - print(f"Understood. The objective is: {objective}") - print(f"Initiating search on the website: {url}") + print(f"{Colors.CYAN}Understood. The objective is: {objective}{Colors.RESET}") + print(f"{Colors.CYAN}Initiating search on the website: {url}{Colors.RESET}") map_prompt = f""" The map function generates a list of URLs from a website and it accepts a search parameter. Based on the objective of: {objective}, come up with a 1-2 word search parameter that will help us find the information we need. Only respond with 1-2 words nothing else. """ - print("Analyzing objective to determine optimal search parameter...") + print(f"{Colors.YELLOW}Analyzing objective to determine optimal search parameter...{Colors.RESET}") completion = client.chat.completions.create( model="o1-preview", messages=[ @@ -42,15 +52,15 @@ def find_relevant_page_via_map(objective, url, app, client): ) map_search_parameter = completion.choices[0].message.content - print(f"Optimal search parameter identified: {map_search_parameter}") + print(f"{Colors.GREEN}Optimal search parameter identified: {map_search_parameter}{Colors.RESET}") - print(f"Mapping website using the identified search parameter...") + print(f"{Colors.YELLOW}Mapping website using the identified search parameter...{Colors.RESET}") map_website = app.map_url(url, params={"search": map_search_parameter}) - print("Website mapping completed successfully.") - print(f"Located {len(map_website)} relevant links.") + print(f"{Colors.GREEN}Website mapping completed successfully.{Colors.RESET}") + print(f"{Colors.GREEN}Located {len(map_website)} relevant links.{Colors.RESET}") return map_website except Exception as e: - print(f"Error encountered during relevant page identification: {str(e)}") + print(f"{Colors.RED}Error encountered during relevant page identification: {str(e)}{Colors.RESET}") return None # Scrape the top 3 pages and see if the objective is met, if so return in json format else return None @@ -58,13 +68,13 @@ def find_objective_in_top_pages(map_website, objective, app, client): try: # Get top 3 links from the map result top_links = map_website[:3] if isinstance(map_website, list) else [] - print(f"Proceeding to analyze top {len(top_links)} links: {top_links}") + print(f"{Colors.CYAN}Proceeding to analyze top {len(top_links)} links: {top_links}{Colors.RESET}") for link in top_links: - print(f"Initiating scrape of page: {link}") + print(f"{Colors.YELLOW}Initiating scrape of page: {link}{Colors.RESET}") # Scrape the page scrape_result = app.scrape_url(link, params={'formats': ['markdown']}) - print("Page scraping completed successfully.") + print(f"{Colors.GREEN}Page scraping completed successfully.{Colors.RESET}") # Check if objective is met @@ -82,7 +92,7 @@ def find_objective_in_top_pages(map_website, objective, app, client): 3. Do not include any explanations or markdown formatting in your response. """ - print("Analyzing scraped content to determine objective fulfillment...") + print(f"{Colors.YELLOW}Analyzing scraped content to determine objective fulfillment...{Colors.RESET}") completion = client.chat.completions.create( model="o1-preview", messages=[ @@ -101,43 +111,42 @@ def find_objective_in_top_pages(map_website, objective, app, client): result = completion.choices[0].message.content if result != "Objective not met": - print("Objective potentially fulfilled. Relevant information identified.") + print(f"{Colors.GREEN}Objective potentially fulfilled. Relevant information identified.{Colors.RESET}") try: - print(result) return json.loads(result) except json.JSONDecodeError: - print("Error in parsing response. Proceeding to next page...") + print(f"{Colors.RED}Error in parsing response. Proceeding to next page...{Colors.RESET}") else: - print("Objective not met on this page. Proceeding to next link...") + print(f"{Colors.YELLOW}Objective not met on this page. Proceeding to next link...{Colors.RESET}") - print("All available pages analyzed. Objective not fulfilled in examined content.") + print(f"{Colors.RED}All available pages analyzed. Objective not fulfilled in examined content.{Colors.RESET}") return None except Exception as e: - print(f"Error encountered during page analysis: {str(e)}") + print(f"{Colors.RED}Error encountered during page analysis: {str(e)}{Colors.RESET}") return None # Main function to execute the process def main(): # Get user input - url = input("Enter the website to crawl: ") - objective = input("Enter your objective: ") + url = input(f"{Colors.BLUE}Enter the website to crawl: {Colors.RESET}") + objective = input(f"{Colors.BLUE}Enter your objective: {Colors.RESET}") - print("Initiating web crawling process.") + print(f"{Colors.YELLOW}Initiating web crawling process...{Colors.RESET}") # Find the relevant page map_website = find_relevant_page_via_map(objective, url, app, client) if map_website: - print("Relevant pages identified. Proceeding with detailed analysis...") + print(f"{Colors.GREEN}Relevant pages identified. Proceeding with detailed analysis...{Colors.RESET}") # Find objective in top pages result = find_objective_in_top_pages(map_website, objective, app, client) if result: - print("Objective successfully fulfilled. Extracted information:") - print(json.dumps(result, indent=2)) + print(f"{Colors.GREEN}Objective successfully fulfilled. Extracted information:{Colors.RESET}") + print(f"{Colors.MAGENTA}{json.dumps(result, indent=2)}{Colors.RESET}") else: - print("Unable to fulfill the objective with the available content.") + print(f"{Colors.RED}Unable to fulfill the objective with the available content.{Colors.RESET}") else: - print("No relevant pages identified. Consider refining the search parameters or trying a different website.") + print(f"{Colors.RED}No relevant pages identified. Consider refining the search parameters or trying a different website.{Colors.RESET}") if __name__ == "__main__": main() From 9da3432596eb78dd55b7b94ff1a67518d1aca6c9 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Sep 2024 12:03:14 -0400 Subject: [PATCH 074/102] Update map.ts --- apps/api/src/controllers/v1/map.ts | 133 +++++++++++++++++++++++------ 1 file changed, 105 insertions(+), 28 deletions(-) diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts index aeedc792..6b13f762 100644 --- a/apps/api/src/controllers/v1/map.ts +++ b/apps/api/src/controllers/v1/map.ts @@ -19,8 +19,15 @@ import { billTeam } from "../../services/billing/credit_billing"; import { logJob } from "../../services/logging/log_job"; import { performCosineSimilarity } from "../../lib/map-cosine"; import { Logger } from "../../lib/logger"; +import Redis from "ioredis"; configDotenv(); +const redis = new Redis(process.env.REDIS_URL); + +// Max Links that /map can return +const MAX_MAP_LIMIT = 5000; +// Max Links that "Smart /map" can return +const MAX_FIRE_ENGINE_RESULTS = 1000; export async function mapController( req: RequestWithAuth<{}, MapResponse, MapRequest>, @@ -30,8 +37,7 @@ export async function mapController( req.body = mapRequestSchema.parse(req.body); - - const limit : number = req.body.limit ?? 5000; + const limit: number = req.body.limit ?? MAX_MAP_LIMIT; const id = uuidv4(); let links: string[] = [req.body.url]; @@ -53,35 +59,54 @@ export async function mapController( ? `"${req.body.search}" site:${urlWithoutWww}` : `site:${req.body.url}`; - const maxResults = 5000; const resultsPerPage = 100; - const maxPages = Math.ceil(maxResults / resultsPerPage); + const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage); - const fetchPage = async (page: number) => { - return fireEngineMap(mapUrl, { - numResults: resultsPerPage, - page: page - }); - }; + const cacheKey = `fireEngineMap:${mapUrl}`; + const cachedResult = await redis.get(cacheKey); + + let allResults: any[]; + let pagePromises: Promise[]; + + if (cachedResult) { + allResults = JSON.parse(cachedResult); + } else { + const fetchPage = async (page: number) => { + return fireEngineMap(mapUrl, { + numResults: resultsPerPage, + page: page, + }); + }; + + pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1)); + allResults = await Promise.all(pagePromises); + + await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours + } - const pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1)); - // Parallelize sitemap fetch with serper search - const [sitemap, ...allResults] = await Promise.all([ + const [sitemap, ...searchResults] = await Promise.all([ req.body.ignoreSitemap ? null : crawler.tryGetSitemap(), - ...pagePromises + ...(cachedResult ? [] : pagePromises), ]); + if (!cachedResult) { + allResults = searchResults; + } + if (sitemap !== null) { sitemap.forEach((x) => { links.push(x.url); }); } - let mapResults = allResults.flat().filter(result => result !== null && result !== undefined); + let mapResults = allResults + .flat() + .filter((result) => result !== null && result !== undefined); - if (mapResults.length > maxResults) { - mapResults = mapResults.slice(0, maxResults); + const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit); + if (mapResults.length > minumumCutoff) { + mapResults = mapResults.slice(0, minumumCutoff); } if (mapResults.length > 0) { @@ -102,17 +127,19 @@ export async function mapController( // Perform cosine similarity between the search query and the list of links if (req.body.search) { const searchQuery = req.body.search.toLowerCase(); - + links = performCosineSimilarity(links, searchQuery); } - links = links.map((x) => { - try { - return checkAndUpdateURLForMap(x).url.trim() - } catch (_) { - return null; - } - }).filter(x => x !== null); + links = links + .map((x) => { + try { + return checkAndUpdateURLForMap(x).url.trim(); + } catch (_) { + return null; + } + }) + .filter((x) => x !== null); // allows for subdomains to be included links = links.filter((x) => isSameDomain(x, req.body.url)); @@ -125,8 +152,10 @@ export async function mapController( // remove duplicates that could be due to http/https or www links = removeDuplicateUrls(links); - billTeam(req.auth.team_id, 1).catch(error => { - Logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`); + billTeam(req.auth.team_id, 1).catch((error) => { + Logger.error( + `Failed to bill team ${req.auth.team_id} for 1 credit: ${error}` + ); // Optionally, you could notify an admin or add to a retry queue here }); @@ -134,7 +163,7 @@ export async function mapController( const timeTakenInSeconds = (endTime - startTime) / 1000; const linksToReturn = links.slice(0, limit); - + logJob({ job_id: id, success: links.length > 0, @@ -158,3 +187,51 @@ export async function mapController( scrape_id: req.body.origin?.includes("website") ? id : undefined, }); } + +// Subdomain sitemap url checking + +// // For each result, check for subdomains, get their sitemaps and add them to the links +// const processedUrls = new Set(); +// const processedSubdomains = new Set(); + +// for (const result of links) { +// let url; +// let hostParts; +// try { +// url = new URL(result); +// hostParts = url.hostname.split('.'); +// } catch (e) { +// continue; +// } + +// console.log("hostParts", hostParts); +// // Check if it's a subdomain (more than 2 parts, and not 'www') +// if (hostParts.length > 2 && hostParts[0] !== 'www') { +// const subdomain = hostParts[0]; +// console.log("subdomain", subdomain); +// const subdomainUrl = `${url.protocol}//${subdomain}.${hostParts.slice(-2).join('.')}`; +// console.log("subdomainUrl", subdomainUrl); + +// if (!processedSubdomains.has(subdomainUrl)) { +// processedSubdomains.add(subdomainUrl); + +// const subdomainCrawl = crawlToCrawler(id, { +// originUrl: subdomainUrl, +// crawlerOptions: legacyCrawlerOptions(req.body), +// pageOptions: {}, +// team_id: req.auth.team_id, +// createdAt: Date.now(), +// plan: req.auth.plan, +// }); +// const subdomainSitemap = await subdomainCrawl.tryGetSitemap(); +// if (subdomainSitemap) { +// subdomainSitemap.forEach((x) => { +// if (!processedUrls.has(x.url)) { +// processedUrls.add(x.url); +// links.push(x.url); +// } +// }); +// } +// } +// } +// } From e58144798f124629543ac1de1a3cae798228cf4d Mon Sep 17 00:00:00 2001 From: Eric Ciarla Date: Mon, 16 Sep 2024 16:04:32 -0400 Subject: [PATCH 075/102] Update o1_web_crawler.py --- examples/o1_web_crawler /o1_web_crawler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/o1_web_crawler /o1_web_crawler.py b/examples/o1_web_crawler /o1_web_crawler.py index e4fee3e8..45bbd1ee 100644 --- a/examples/o1_web_crawler /o1_web_crawler.py +++ b/examples/o1_web_crawler /o1_web_crawler.py @@ -79,20 +79,19 @@ def find_objective_in_top_pages(map_website, objective, app, client): # Check if objective is met check_prompt = f""" - Given the following scraped content and objective, determine if the objective is met with high confidence. + Given the following scraped content and objective, determine if the objective is met. If it is, extract the relevant information in a simple and concise JSON format. Use only the necessary fields and avoid nested structures if possible. - If the objective is not met with high confidence, respond with 'Objective not met'. + If the objective is not met with confidence, respond with 'Objective not met'. Objective: {objective} Scraped content: {scrape_result['markdown']} Remember: - 1. Only return JSON if you are highly confident the objective is fully met. + 1. Only return JSON if you are confident the objective is fully met. 2. Keep the JSON structure as simple and flat as possible. 3. Do not include any explanations or markdown formatting in your response. """ - - print(f"{Colors.YELLOW}Analyzing scraped content to determine objective fulfillment...{Colors.RESET}") + completion = client.chat.completions.create( model="o1-preview", messages=[ @@ -121,6 +120,7 @@ def find_objective_in_top_pages(map_website, objective, app, client): print(f"{Colors.RED}All available pages analyzed. Objective not fulfilled in examined content.{Colors.RESET}") return None + except Exception as e: print(f"{Colors.RED}Error encountered during page analysis: {str(e)}{Colors.RESET}") return None From 0f8c0a570dca877d14d590e6002eaffd345a3927 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Sep 2024 21:44:56 -0400 Subject: [PATCH 076/102] Update single_url.ts --- apps/api/src/scraper/WebScraper/single_url.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 2be65899..79876d1e 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -27,9 +27,9 @@ const useScrapingBee = process.env.SCRAPING_BEE_API_KEY !== '' && process.env.SC const useFireEngine = process.env.FIRE_ENGINE_BETA_URL !== '' && process.env.FIRE_ENGINE_BETA_URL !== undefined; export const baseScrapers = [ +useScrapingBee ? "scrapingBee" : undefined, useFireEngine ? "fire-engine;chrome-cdp" : undefined, useFireEngine ? "fire-engine" : undefined, - useScrapingBee ? "scrapingBee" : undefined, useFireEngine ? undefined : "playwright", useScrapingBee ? "scrapingBeeLoad" : undefined, "fetch", @@ -88,9 +88,10 @@ function getScrapingFallbackOrder( }); let defaultOrder = [ +useScrapingBee ? "scrapingBee" : undefined, useFireEngine ? "fire-engine;chrome-cdp" : undefined, useFireEngine ? "fire-engine" : undefined, - useScrapingBee ? "scrapingBee" : undefined, + useScrapingBee ? "scrapingBeeLoad" : undefined, useFireEngine ? undefined : "playwright", "fetch", From d21a797ef92e6eabd605884d8d32defe34b14240 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Sep 2024 21:46:40 -0400 Subject: [PATCH 077/102] Update fly.yml --- .github/workflows/fly.yml | 213 -------------------------------------- 1 file changed, 213 deletions(-) diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index 7b45921a..31a09f08 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -32,219 +32,6 @@ env: ENV: ${{ secrets.ENV }} jobs: - pre-deploy-e2e-tests: - name: Pre-deploy checks - runs-on: ubuntu-latest - services: - redis: - image: redis - ports: - - 6379:6379 - steps: - - uses: actions/checkout@v3 - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: "20" - - name: Install pnpm - run: npm install -g pnpm - - name: Install dependencies - run: pnpm install - working-directory: ./apps/api - - name: Start the application - run: npm start & - working-directory: ./apps/api - id: start_app - - name: Start workers - run: npm run workers & - working-directory: ./apps/api - id: start_workers - - name: Wait for the application to be ready - run: | - sleep 10 - - name: Run E2E tests - run: | - npm run test:prod - working-directory: ./apps/api - - pre-deploy-test-suite: - name: Test Suite - needs: pre-deploy-e2e-tests - runs-on: ubuntu-latest - services: - redis: - image: redis - ports: - - 6379:6379 - steps: - - uses: actions/checkout@v3 - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: "20" - - name: Install pnpm - run: npm install -g pnpm - - name: Install dependencies - run: pnpm install - working-directory: ./apps/api - - name: Start the application - run: npm start & - working-directory: ./apps/api - id: start_app - - name: Start workers - run: npm run workers & - working-directory: ./apps/api - id: start_workers - - name: Install dependencies - run: pnpm install - working-directory: ./apps/test-suite - - name: Run E2E tests - run: | - npm run test:suite - working-directory: ./apps/test-suite - - python-sdk-tests: - name: Python SDK Tests - needs: pre-deploy-e2e-tests - runs-on: ubuntu-latest - services: - redis: - image: redis - ports: - - 6379:6379 - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - name: Install pnpm - run: npm install -g pnpm - - name: Install dependencies - run: pnpm install - working-directory: ./apps/api - - name: Start the application - run: npm start & - working-directory: ./apps/api - id: start_app - - name: Start workers - run: npm run workers & - working-directory: ./apps/api - id: start_workers - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - working-directory: ./apps/python-sdk - - name: Run E2E tests for Python SDK - run: | - pytest firecrawl/__tests__/v1/e2e_withAuth/test.py - working-directory: ./apps/python-sdk - - js-sdk-tests: - name: JavaScript SDK Tests - needs: pre-deploy-e2e-tests - runs-on: ubuntu-latest - services: - redis: - image: redis - ports: - - 6379:6379 - steps: - - uses: actions/checkout@v3 - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: "20" - - name: Install pnpm - run: npm install -g pnpm - - name: Install dependencies - run: pnpm install - working-directory: ./apps/api - - name: Start the application - run: npm start & - working-directory: ./apps/api - id: start_app - - name: Start workers - run: npm run workers & - working-directory: ./apps/api - id: start_workers - - name: Install dependencies for JavaScript SDK - run: pnpm install - working-directory: ./apps/js-sdk/firecrawl - - name: Run E2E tests for JavaScript SDK - run: npm run test - working-directory: ./apps/js-sdk/firecrawl - - go-sdk-tests: - name: Go SDK Tests - needs: pre-deploy-e2e-tests - runs-on: ubuntu-latest - services: - redis: - image: redis - ports: - - 6379:6379 - steps: - - uses: actions/checkout@v3 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: "go.mod" - - name: Install pnpm - run: npm install -g pnpm - - name: Install dependencies - run: pnpm install - working-directory: ./apps/api - - name: Start the application - run: npm start & - working-directory: ./apps/api - id: start_app - - name: Start workers - run: npm run workers & - working-directory: ./apps/api - id: start_workers - - name: Install dependencies for Go SDK - run: go mod tidy - working-directory: ./apps/go-sdk - - name: Run tests for Go SDK - run: go test -v ./... -timeout 180s - working-directory: ./apps/go-sdk/firecrawl - - rust-sdk-tests: - name: Rust SDK Tests - needs: pre-deploy-e2e-tests - runs-on: ubuntu-latest - services: - redis: - image: redis - ports: - - 6379:6379 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Install pnpm - run: npm install -g pnpm - - name: Install dependencies for API - run: pnpm install - working-directory: ./apps/api - - name: Start the application - run: npm start & - working-directory: ./apps/api - id: start_app - - name: Start workers - run: npm run workers & - working-directory: ./apps/api - id: start_workers - - name: Set up Rust - uses: actions/setup-rust@v1 - with: - rust-version: stable - - name: Try the lib build - working-directory: ./apps/rust-sdk - run: cargo build - - name: Run E2E tests for Rust SDK - run: cargo test --test e2e_with_auth deploy: name: Deploy app From 473e8491b528e399602266229172e02b71714b41 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Sep 2024 21:46:54 -0400 Subject: [PATCH 078/102] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 96878ea2..40a76a6e 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Crawl and convert any website into LLM-ready markdown or structured data. Built _This repository is in its early development stages. We are still merging custom modules in the mono repo. It's not completely yet ready for full self-host deployment, but you can already run it locally._ + ## What is Firecrawl? [Firecrawl](https://firecrawl.dev?ref=github) is an API service that takes a URL, crawls it, and converts it into clean markdown or structured data. We crawl all accessible subpages and give you clean data for each. No sitemap required. Check out our [documentation](https://docs.firecrawl.dev). From 4884439fd9491170184ee3af3b2e226476248e94 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Sep 2024 21:48:12 -0400 Subject: [PATCH 079/102] Update fly.yml --- .github/workflows/fly.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index 31a09f08..aa666712 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -32,7 +32,6 @@ env: ENV: ${{ secrets.ENV }} jobs: - deploy: name: Deploy app runs-on: ubuntu-latest From 358f8f9defb90eeb77bd91f7595eb99eb5fba54c Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Sep 2024 21:50:56 -0400 Subject: [PATCH 080/102] Update fly-direct.yml --- .github/workflows/fly-direct.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fly-direct.yml b/.github/workflows/fly-direct.yml index 2473642c..4713ecb2 100644 --- a/.github/workflows/fly-direct.yml +++ b/.github/workflows/fly-direct.yml @@ -1,7 +1,7 @@ name: Fly Deploy Direct on: schedule: - - cron: '0 */2 * * *' + - cron: '*/5 * * * *' env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} From 66577d1d03727ae49489de06a34890b868bfbc4c Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Sep 2024 21:53:35 -0400 Subject: [PATCH 081/102] Update fly.yml --- .github/workflows/fly.yml | 118 +------------------------------------- 1 file changed, 1 insertion(+), 117 deletions(-) diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index aa666712..34073519 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -35,7 +35,6 @@ jobs: deploy: name: Deploy app runs-on: ubuntu-latest - needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests, rust-sdk-tests] steps: - uses: actions/checkout@v3 - uses: superfly/flyctl-actions/setup-flyctl@master @@ -46,119 +45,4 @@ jobs: BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - build-and-publish-python-sdk: - name: Build and publish Python SDK - runs-on: ubuntu-latest - needs: deploy - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine build requests packaging - - - name: Run version check script - id: version_check_script - run: | - PYTHON_SDK_VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py) - echo "PYTHON_SDK_VERSION_INCREMENTED=$PYTHON_SDK_VERSION_INCREMENTED" >> $GITHUB_ENV - - - name: Build the package - if: ${{ env.PYTHON_SDK_VERSION_INCREMENTED == 'true' }} - run: | - python -m build - working-directory: ./apps/python-sdk - - - name: Publish to PyPI - if: ${{ env.PYTHON_SDK_VERSION_INCREMENTED == 'true' }} - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - twine upload dist/* - working-directory: ./apps/python-sdk - - build-and-publish-js-sdk: - name: Build and publish JavaScript SDK - runs-on: ubuntu-latest - needs: deploy - - steps: - - uses: actions/checkout@v3 - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '20' - registry-url: 'https://registry.npmjs.org/' - scope: '@mendable' - always-auth: true - - - name: Install pnpm - run: npm install -g pnpm - - - name: Install python for running version check script - run: | - python -m pip install --upgrade pip - pip install setuptools wheel requests packaging - - - name: Install dependencies for JavaScript SDK - run: pnpm install - working-directory: ./apps/js-sdk/firecrawl - - - name: Run version check script - id: version_check_script - run: | - VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py js ./apps/js-sdk/firecrawl @mendable/firecrawl-js) - echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV - - - name: Build and publish to npm - if: ${{ env.VERSION_INCREMENTED == 'true' }} - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - npm run build-and-publish - working-directory: ./apps/js-sdk/firecrawl - build-and-publish-rust-sdk: - name: Build and publish Rust SDK - runs-on: ubuntu-latest - needs: deploy - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true - profile: minimal - - - name: Install dependencies - run: cargo build --release - - - name: Run version check script - id: version_check_script - run: | - VERSION_INCREMENTED=$(cargo search --limit 1 my_crate_name | grep my_crate_name) - echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV - - - name: Build the package - if: ${{ env.VERSION_INCREMENTED == 'true' }} - run: cargo package - working-directory: ./apps/rust-sdk - - - name: Publish to crates.io - if: ${{ env.VERSION_INCREMENTED == 'true' }} - env: - CARGO_REG_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} - run: cargo publish - working-directory: ./apps/rust-sdk \ No newline at end of file + \ No newline at end of file From a6e1b173860a134288447004f328332c627753ff Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Sep 2024 21:53:51 -0400 Subject: [PATCH 082/102] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 40a76a6e..96878ea2 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,6 @@ Crawl and convert any website into LLM-ready markdown or structured data. Built _This repository is in its early development stages. We are still merging custom modules in the mono repo. It's not completely yet ready for full self-host deployment, but you can already run it locally._ - ## What is Firecrawl? [Firecrawl](https://firecrawl.dev?ref=github) is an API service that takes a URL, crawls it, and converts it into clean markdown or structured data. We crawl all accessible subpages and give you clean data for each. No sitemap required. Check out our [documentation](https://docs.firecrawl.dev). From a4039bd0089580e0f79a39bd69be6ae256453158 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Sep 2024 23:36:38 -0400 Subject: [PATCH 083/102] Revert "Update single_url.ts" This reverts commit 0f8c0a570dca877d14d590e6002eaffd345a3927. --- apps/api/src/scraper/WebScraper/single_url.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 79876d1e..2be65899 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -27,9 +27,9 @@ const useScrapingBee = process.env.SCRAPING_BEE_API_KEY !== '' && process.env.SC const useFireEngine = process.env.FIRE_ENGINE_BETA_URL !== '' && process.env.FIRE_ENGINE_BETA_URL !== undefined; export const baseScrapers = [ -useScrapingBee ? "scrapingBee" : undefined, useFireEngine ? "fire-engine;chrome-cdp" : undefined, useFireEngine ? "fire-engine" : undefined, + useScrapingBee ? "scrapingBee" : undefined, useFireEngine ? undefined : "playwright", useScrapingBee ? "scrapingBeeLoad" : undefined, "fetch", @@ -88,10 +88,9 @@ function getScrapingFallbackOrder( }); let defaultOrder = [ -useScrapingBee ? "scrapingBee" : undefined, useFireEngine ? "fire-engine;chrome-cdp" : undefined, useFireEngine ? "fire-engine" : undefined, - + useScrapingBee ? "scrapingBee" : undefined, useScrapingBee ? "scrapingBeeLoad" : undefined, useFireEngine ? undefined : "playwright", "fetch", From 18b024c238ffed4dd19f8f3928d8b41b9d0a49f3 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 17 Sep 2024 01:41:46 -0400 Subject: [PATCH 084/102] Update single_url.ts --- apps/api/src/scraper/WebScraper/single_url.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 2be65899..ce534b00 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -89,8 +89,8 @@ function getScrapingFallbackOrder( let defaultOrder = [ useFireEngine ? "fire-engine;chrome-cdp" : undefined, - useFireEngine ? "fire-engine" : undefined, useScrapingBee ? "scrapingBee" : undefined, + useFireEngine ? "fire-engine" : undefined, useScrapingBee ? "scrapingBeeLoad" : undefined, useFireEngine ? undefined : "playwright", "fetch", From 3c2bfe2da2d85a84c90f9564b0a85c0a48a0c231 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 17 Sep 2024 01:58:47 -0400 Subject: [PATCH 085/102] Update single_url.ts --- apps/api/src/scraper/WebScraper/single_url.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index ce534b00..8143bab0 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -28,8 +28,8 @@ const useFireEngine = process.env.FIRE_ENGINE_BETA_URL !== '' && process.env.FIR export const baseScrapers = [ useFireEngine ? "fire-engine;chrome-cdp" : undefined, - useFireEngine ? "fire-engine" : undefined, useScrapingBee ? "scrapingBee" : undefined, + useFireEngine ? "fire-engine" : undefined, useFireEngine ? undefined : "playwright", useScrapingBee ? "scrapingBeeLoad" : undefined, "fetch", From fb8a2c75497c0221abde4b14e3c73a7926317c15 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 17 Sep 2024 11:03:33 -0400 Subject: [PATCH 086/102] fixed screenshot typo and added test for fullpage screenshot --- apps/js-sdk/firecrawl/package-lock.json | 4 +- apps/js-sdk/firecrawl/package.json | 2 +- .../__tests__/v1/e2e_withAuth/index.test.ts | 47 +++++++++++++++++-- apps/js-sdk/firecrawl/src/index.ts | 2 +- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json index 2dcca44d..e27e259d 100644 --- a/apps/js-sdk/firecrawl/package-lock.json +++ b/apps/js-sdk/firecrawl/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mendable/firecrawl-js", - "version": "1.3.0", + "version": "1.4.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mendable/firecrawl-js", - "version": "1.3.0", + "version": "1.4.3", "license": "MIT", "dependencies": { "axios": "^1.6.8", diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index f6f14fb2..45fbcee9 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.4.2", + "version": "1.4.3", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts index 98a52538..dea55846 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts @@ -28,14 +28,22 @@ describe('FirecrawlApp E2E Tests', () => { test.concurrent('should return successful response with valid preview token', async () => { const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL }); - const response = await app.scrapeUrl('https://roastmywebsite.ai') as ScrapeResponse; + const response = await app.scrapeUrl('https://roastmywebsite.ai'); + if (!response.success) { + throw new Error(response.error); + } + expect(response).not.toBeNull(); expect(response?.markdown).toContain("_Roast_"); }, 30000); // 30 seconds timeout test.concurrent('should return successful response for valid scrape', async () => { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); - const response = await app.scrapeUrl('https://roastmywebsite.ai') as ScrapeResponse; + const response = await app.scrapeUrl('https://roastmywebsite.ai'); + if (!response.success) { + throw new Error(response.error); + } + expect(response).not.toBeNull(); expect(response).not.toHaveProperty('content'); // v0 expect(response).not.toHaveProperty('html'); @@ -58,7 +66,11 @@ describe('FirecrawlApp E2E Tests', () => { onlyMainContent: true, timeout: 30000, waitFor: 1000 - }) as ScrapeResponse; + }); + if (!response.success) { + throw new Error(response.error); + } + expect(response).not.toBeNull(); expect(response).not.toHaveProperty('content'); // v0 expect(response.markdown).toContain("_Roast_"); @@ -86,6 +98,7 @@ describe('FirecrawlApp E2E Tests', () => { expect(response.metadata).not.toHaveProperty("pageStatusCode"); expect(response.metadata).toHaveProperty("statusCode"); expect(response.metadata).not.toHaveProperty("pageError"); + if (response.metadata !== undefined) { expect(response.metadata.error).toBeUndefined(); expect(response.metadata.title).toBe("Roast My Website"); @@ -103,16 +116,40 @@ describe('FirecrawlApp E2E Tests', () => { } }, 30000); // 30 seconds timeout + test.concurrent('should return successful response with valid API key and screenshot fullPage', async () => { + const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); + const response = await app.scrapeUrl( + 'https://roastmywebsite.ai', { + formats: ['screenshot@fullPage'], + }); + if (!response.success) { + throw new Error(response.error); + } + + expect(response).not.toBeNull(); + expect(response.screenshot).not.toBeUndefined(); + expect(response.screenshot).not.toBeNull(); + expect(response.screenshot).toContain("https://"); + }, 30000); // 30 seconds timeout + test.concurrent('should return successful response for valid scrape with PDF file', async () => { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); - const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf') as ScrapeResponse; + const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf'); + if (!response.success) { + throw new Error(response.error); + } + expect(response).not.toBeNull(); expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy'); }, 30000); // 30 seconds timeout test.concurrent('should return successful response for valid scrape with PDF file without explicit extension', async () => { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); - const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001') as ScrapeResponse; + const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001'); + if (!response.success) { + throw new Error(response.error); + } + expect(response).not.toBeNull(); expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy'); }, 30000); // 30 seconds timeout diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 661ce34b..b06a037d 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -74,7 +74,7 @@ export interface FirecrawlDocument { * Defines the options and configurations available for scraping web content. */ export interface CrawlScrapeOptions { - formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract" | "full@scrennshot")[]; + formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract")[]; headers?: Record; includeTags?: string[]; excludeTags?: string[]; From 31633317dd63a73733544de29b4021417e79fc3b Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 17 Sep 2024 12:33:07 -0400 Subject: [PATCH 087/102] Update package.json --- apps/js-sdk/firecrawl/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 45fbcee9..abda7f69 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.4.3", + "version": "1.4.4", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", From a0189acbecd5166f2de6c0fed8800e17bfc7397a Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 17 Sep 2024 12:58:49 -0400 Subject: [PATCH 088/102] Update fly.yml --- .github/workflows/fly.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index 34073519..ed8dade1 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -32,8 +32,41 @@ env: ENV: ${{ secrets.ENV }} jobs: + pre-deploy: + name: Pre-deploy checks + runs-on: ubuntu-latest + services: + redis: + image: redis + ports: + - 6379:6379 + steps: + - uses: actions/checkout@v3 + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: "20" + - name: Install pnpm + run: npm install -g pnpm + - name: Install dependencies + run: pnpm install + working-directory: ./apps/api + - name: Start the application + run: npm start & + working-directory: ./apps/api + id: start_app + - name: Start workers + run: npm run workers & + working-directory: ./apps/api + id: start_workers + - name: Run E2E tests + run: | + npm run test:prod + working-directory: ./apps/api + deploy: name: Deploy app + needs: pre-deploy runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 43d8563bb1245b0fcff6bb6eab8f0635314d2e20 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 17 Sep 2024 14:15:34 -0400 Subject: [PATCH 089/102] Update fly-direct.yml --- .github/workflows/fly-direct.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fly-direct.yml b/.github/workflows/fly-direct.yml index 4713ecb2..3ee460e6 100644 --- a/.github/workflows/fly-direct.yml +++ b/.github/workflows/fly-direct.yml @@ -1,7 +1,7 @@ name: Fly Deploy Direct on: schedule: - - cron: '*/5 * * * *' + - cron: '0 * * * *' env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} From b2b7f8d87495847335321a5e63e2186c13dc6cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Tue, 17 Sep 2024 20:49:01 +0200 Subject: [PATCH 090/102] fix(js-sdk): default type for LLM extract --- apps/js-sdk/firecrawl/package-lock.json | 4 ++-- apps/js-sdk/firecrawl/src/index.ts | 6 +++--- apps/js-sdk/package-lock.json | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json index e27e259d..81a4a146 100644 --- a/apps/js-sdk/firecrawl/package-lock.json +++ b/apps/js-sdk/firecrawl/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mendable/firecrawl-js", - "version": "1.4.3", + "version": "1.4.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mendable/firecrawl-js", - "version": "1.4.3", + "version": "1.4.4", "license": "MIT", "dependencies": { "axios": "^1.6.8", diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index b06a037d..6c859bee 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -58,7 +58,7 @@ export interface FirecrawlDocumentMetadata { * Document interface for Firecrawl. * Represents a document retrieved or processed by Firecrawl. */ -export interface FirecrawlDocument { +export interface FirecrawlDocument { url?: string; markdown?: string; html?: string; @@ -83,7 +83,7 @@ export interface CrawlScrapeOptions { timeout?: number; } -export interface ScrapeParams extends CrawlScrapeOptions { +export interface ScrapeParams extends CrawlScrapeOptions { extract?: { prompt?: string; schema?: LLMSchema; @@ -95,7 +95,7 @@ export interface ScrapeParams extends CrawlScrap * Response interface for scraping operations. * Defines the structure of the response received after a scraping operation. */ -export interface ScrapeResponse extends FirecrawlDocument { +export interface ScrapeResponse extends FirecrawlDocument { success: true; warning?: string; error?: string; diff --git a/apps/js-sdk/package-lock.json b/apps/js-sdk/package-lock.json index b0f358cb..975b14e8 100644 --- a/apps/js-sdk/package-lock.json +++ b/apps/js-sdk/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "@mendable/firecrawl-js": "^1.0.3", "axios": "^1.6.8", "firecrawl": "^1.2.0", "ts-node": "^10.9.2", From 255db84879fb432408eebd0feac23128b3372b05 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 17 Sep 2024 15:06:53 -0400 Subject: [PATCH 091/102] Update package.json --- apps/js-sdk/firecrawl/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index abda7f69..4b93536f 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.4.4", + "version": "1.4.5", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", From c28e1e29594985d0dc267c783f1519f7be35be59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 18 Sep 2024 19:35:27 +0200 Subject: [PATCH 092/102] fix(v1/zod): formats: add error message if both sc and sc@fP --- apps/api/src/controllers/v1/types.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index c44c1cc5..ab811067 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -70,7 +70,8 @@ export const scrapeOptions = z.object({ ]) .array() .optional() - .default(["markdown"]), + .default(["markdown"]) + .refine(x => !(x.includes("screenshot") && x.includes("screenshot@fullPage")), "You may only specify either screenshot or screenshot@fullPage"), headers: z.record(z.string(), z.string()).optional(), includeTags: z.string().array().optional(), excludeTags: z.string().array().optional(), From d338c3402bf34a88e23986535980d2a6cb241f0a Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 18 Sep 2024 15:18:44 -0400 Subject: [PATCH 093/102] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d0145a6b..2a843aa8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,7 +103,7 @@ This should return the response Hello, world! If you’d like to test the crawl endpoint, you can run this ```curl -curl -X POST http://localhost:3002/v0/crawl \ +curl -X POST http://localhost:3002/v1/crawl \ -H 'Content-Type: application/json' \ -d '{ "url": "https://mendable.ai" From 3a4dd8fc7e94bc076b94fde4ed0303eb61926c2e Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 18 Sep 2024 15:58:53 -0400 Subject: [PATCH 094/102] Nick: v1 openapi spec --- apps/api/v1-openapi.json | 823 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 823 insertions(+) create mode 100644 apps/api/v1-openapi.json diff --git a/apps/api/v1-openapi.json b/apps/api/v1-openapi.json new file mode 100644 index 00000000..e61eb7cf --- /dev/null +++ b/apps/api/v1-openapi.json @@ -0,0 +1,823 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Firecrawl API", + "version": "v1", + "description": "API for interacting with Firecrawl services to perform web scraping and crawling tasks.", + "contact": { + "name": "Firecrawl Support", + "url": "https://firecrawl.dev/support", + "email": "support@firecrawl.dev" + } + }, + "servers": [ + { + "url": "https://api.firecrawl.dev/v1" + } + ], + "paths": { + "/scrape": { + "post": { + "summary": "Scrape a single URL and optionally extract information using an LLM", + "operationId": "scrapeAndExtractFromUrl", + "tags": ["Scraping"], + "security": [ + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri", + "description": "The URL to scrape" + }, + "formats": { + "type": "array", + "items": { + "type": "string", + "enum": ["markdown", "html", "rawHtml", "links", "screenshot", "extract", "screenshot@fullPage"] + }, + "description": "Formats to include in the output.", + "default": ["markdown"] + }, + "onlyMainContent": { + "type": "boolean", + "description": "Only return the main content of the page excluding headers, navs, footers, etc.", + "default": true + }, + "includeTags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tags to include in the output." + }, + "excludeTags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tags to exclude from the output." + }, + "headers": { + "type": "object", + "description": "Headers to send with the request. Can be used to send cookies, user-agent, etc." + }, + "waitFor": { + "type": "integer", + "description": "Specify a delay in milliseconds before fetching the content, allowing the page sufficient time to load.", + "default": 0 + }, + "timeout": { + "type": "integer", + "description": "Timeout in milliseconds for the request", + "default": 30000 + }, + "extract": { + "type": "object", + "description": "Extract object", + "properties": { + "schema": { + "type": "object", + "description": "The schema to use for the extraction (Optional)" + }, + "systemPrompt": { + "type": "string", + "description": "The system prompt to use for the extraction (Optional)" + }, + "prompt": { + "type": "string", + "description": "The prompt to use for the extraction without a schema (Optional)" + } + } + } + }, + "required": ["url"] + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScrapeResponse" + } + } + } + }, + "402": { + "description": "Payment required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Payment required to access this resource." + } + } + } + } + } + }, + "429": { + "description": "Too many requests", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Request rate limit exceeded. Please wait and try again later." + } + } + } + } + } + }, + "500": { + "description": "Server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "An unexpected error occurred on the server." + } + } + } + } + } + } + } + } + }, + "/crawl/{id}": { + "parameters": [ + { + "name": "id", + "in": "path", + "description": "The ID of the crawl job", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "get": { + "summary": "Get the status of a crawl job", + "operationId": "getCrawlStatus", + "tags": ["Crawling"], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CrawlStatusResponseObj" + } + } + } + }, + "402": { + "description": "Payment required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Payment required to access this resource." + } + } + } + } + } + }, + "429": { + "description": "Too many requests", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Request rate limit exceeded. Please wait and try again later." + } + } + } + } + } + }, + "500": { + "description": "Server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "An unexpected error occurred on the server." + } + } + } + } + } + } + } + }, + "delete": { + "summary": "Cancel a crawl job", + "operationId": "cancelCrawl", + "tags": ["Crawling"], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "description": "Successful cancellation", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean", + "example": true + }, + "message": { + "type": "string", + "example": "Crawl job successfully cancelled." + } + } + } + } + } + }, + "404": { + "description": "Crawl job not found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Crawl job not found." + } + } + } + } + } + }, + "500": { + "description": "Server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "An unexpected error occurred on the server." + } + } + } + } + } + } + } + } + }, + "/crawl": { + "post": { + "summary": "Crawl multiple URLs based on options", + "operationId": "crawlUrls", + "tags": ["Crawling"], + "security": [ + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri", + "description": "The base URL to start crawling from" + }, + "excludePaths": { + "type": "array", + "items": { + "type": "string" + }, + "description": "URL patterns to exclude" + }, + "includePaths": { + "type": "array", + "items": { + "type": "string" + }, + "description": "URL patterns to include" + }, + "maxDepth": { + "type": "integer", + "description": "Maximum depth to crawl relative to the entered URL.", + "default": 2 + }, + "ignoreSitemap": { + "type": "boolean", + "description": "Ignore the website sitemap when crawling", + "default": true + }, + "limit": { + "type": "integer", + "description": "Maximum number of pages to crawl", + "default": 10 + }, + "allowBackwardLinks": { + "type": "boolean", + "description": "Enables the crawler to navigate from a specific URL to previously linked pages.", + "default": false + }, + "allowExternalLinks": { + "type": "boolean", + "description": "Allows the crawler to follow links to external websites.", + "default": false + }, + "webhook": { + "type": "string", + "description": "The URL to send the webhook to. This will trigger for crawl started (crawl.started) ,every page crawled (crawl.page) and when the crawl is completed (crawl.completed or crawl.failed). The response will be the same as the `/scrape` endpoint." + }, + "scrapeOptions": { + "type": "object", + "properties": { + "formats": { + "type": "array", + "items": { + "type": "string", + "enum": ["markdown", "html", "rawHtml", "links", "screenshot"] + }, + "description": "Formats to include in the output.", + "default": ["markdown"] + }, + "headers": { + "type": "object", + "description": "Headers to send with the request. Can be used to send cookies, user-agent, etc." + }, + "includeTags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tags to include in the output." + }, + "excludeTags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tags to exclude from the output." + }, + "onlyMainContent": { + "type": "boolean", + "description": "Only return the main content of the page excluding headers, navs, footers, etc.", + "default": true + }, + "waitFor": { + "type": "integer", + "description": "Wait x amount of milliseconds for the page to load to fetch content", + "default": 123 + } + } + } + }, + "required": ["url"] + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CrawlResponse" + } + } + } + }, + "402": { + "description": "Payment required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Payment required to access this resource." + } + } + } + } + } + }, + "429": { + "description": "Too many requests", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Request rate limit exceeded. Please wait and try again later." + } + } + } + } + } + }, + "500": { + "description": "Server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "An unexpected error occurred on the server." + } + } + } + } + } + } + } + } + }, + "/map": { + "post": { + "summary": "Map multiple URLs based on options", + "operationId": "mapUrls", + "tags": ["Mapping"], + "security": [ + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri", + "description": "The base URL to start crawling from" + }, + "search": { + "type": "string", + "description": "Search query to use for mapping. During the Alpha phase, the 'smart' part of the search functionality is limited to 100 search results. However, if map finds more results, there is no limit applied." + }, + "ignoreSitemap": { + "type": "boolean", + "description": "Ignore the website sitemap when crawling", + "default": true + }, + "includeSubdomains": { + "type": "boolean", + "description": "Include subdomains of the website", + "default": false + }, + "limit": { + "type": "integer", + "description": "Maximum number of links to return", + "default": 5000, + "maximum": 5000 + } + }, + "required": ["url"] + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MapResponse" + } + } + } + }, + "402": { + "description": "Payment required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Payment required to access this resource." + } + } + } + } + } + }, + "429": { + "description": "Too many requests", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Request rate limit exceeded. Please wait and try again later." + } + } + } + } + } + }, + "500": { + "description": "Server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "An unexpected error occurred on the server." + } + } + } + } + } + } + } + } + } + }, + "components": { + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer" + } + }, + "schemas": { + "ScrapeResponse": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "markdown": { + "type": "string" + }, + "html": { + "type": "string", + "nullable": true, + "description": "HTML version of the content on page if `html` is in `formats`" + }, + "rawHtml": { + "type": "string", + "nullable": true, + "description": "Raw HTML content of the page if `rawHtml` is in `formats`" + }, + "screenshot": { + "type": "string", + "nullable": true, + "description": "Screenshot of the page if `screenshot` is in `formats`" + }, + "links": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of links on the page if `links` is in `formats`" + }, + "metadata": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "language": { + "type": "string", + "nullable": true + }, + "sourceURL": { + "type": "string", + "format": "uri" + }, + " ": { + "type": "string" + }, + "statusCode": { + "type": "integer", + "description": "The status code of the page" + }, + "error": { + "type": "string", + "nullable": true, + "description": "The error message of the page" + } + + } + }, + "llm_extraction": { + "type": "object", + "description": "Displayed when using LLM Extraction. Extracted data from the page following the schema defined.", + "nullable": true + }, + "warning": { + "type": "string", + "nullable": true, + "description": "Can be displayed when using LLM Extraction. Warning message will let you know any issues with the extraction." + } + } + } + } + }, + "CrawlStatusResponseObj": { + "type": "object", + "properties": { + "status": { + "type": "string", + "description": "The current status of the crawl. Can be `scraping`, `completed`, or `failed`." + }, + "total": { + "type": "integer", + "description": "The total number of pages that were attempted to be crawled." + }, + "completed": { + "type": "integer", + "description": "The number of pages that have been successfully crawled." + }, + "creditsUsed": { + "type": "integer", + "description": "The number of credits used for the crawl." + }, + "expiresAt": { + "type": "string", + "format": "date-time", + "description": "The date and time when the crawl will expire." + }, + "next": { + "type": "string", + "nullable": true, + "description": "The URL to retrieve the next 10MB of data. Returned if the crawl is not completed or if the response is larger than 10MB." + }, + "data": { + "type": "array", + "description": "The data of the crawl.", + "items": { + "type": "object", + "properties": { + "markdown": { + "type": "string" + }, + "html": { + "type": "string", + "nullable": true, + "description": "HTML version of the content on page if `includeHtml` is true" + }, + "rawHtml": { + "type": "string", + "nullable": true, + "description": "Raw HTML content of the page if `includeRawHtml` is true" + }, + "links": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of links on the page if `includeLinks` is true" + }, + "screenshot": { + "type": "string", + "nullable": true, + "description": "Screenshot of the page if `includeScreenshot` is true" + }, + "metadata": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "language": { + "type": "string", + "nullable": true + }, + "sourceURL": { + "type": "string", + "format": "uri" + }, + " ": { + "type": "string" + }, + "statusCode": { + "type": "integer", + "description": "The status code of the page" + }, + "error": { + "type": "string", + "nullable": true, + "description": "The error message of the page" + } + } + } + } + } + } + } + }, + "CrawlResponse": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + } + } + }, + "MapResponse": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "links": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "security": [ + { + "bearerAuth": [] + } + ] +} \ No newline at end of file From 00bb958b32e28d4f6af1cb8e6333a3cb02a8a7f7 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 18 Sep 2024 15:59:11 -0400 Subject: [PATCH 095/102] Update v1-openapi.json --- apps/api/v1-openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/v1-openapi.json b/apps/api/v1-openapi.json index e61eb7cf..1ff0fb9b 100644 --- a/apps/api/v1-openapi.json +++ b/apps/api/v1-openapi.json @@ -6,7 +6,7 @@ "description": "API for interacting with Firecrawl services to perform web scraping and crawling tasks.", "contact": { "name": "Firecrawl Support", - "url": "https://firecrawl.dev/support", + "url": "https://firecrawl.dev", "email": "support@firecrawl.dev" } }, From a5322322f02bc48dbf7b692f3e187a78f645a1bb Mon Sep 17 00:00:00 2001 From: Eric Ciarla <43451761+ericciarla@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:40:22 -0400 Subject: [PATCH 096/102] Update README.md --- README.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 96878ea2..279a1d82 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ # 🔥 Firecrawl -Crawl and convert any website into LLM-ready markdown or structured data. Built by [Mendable.ai](https://mendable.ai?ref=gfirecrawl) and the Firecrawl community. Includes powerful scraping, crawling and data extraction capabilities. +Empower your AI apps with clean data from any website. Featuring advanced scraping, crawling, and data extraction capabilities. -_This repository is in its early development stages. We are still merging custom modules in the mono repo. It's not completely yet ready for full self-host deployment, but you can already run it locally._ +_This repository is in early development, and we’re still integrating custom modules into the mono repo. It's not fully ready for self-hosted deployment yet, but you can run it locally._ ## What is Firecrawl? @@ -52,9 +52,12 @@ _Pst. hey, you, join our stargazers :)_ We provide an easy to use API with our hosted version. You can find the playground and documentation [here](https://firecrawl.dev/playground). You can also self host the backend if you'd like. -- [x] [API](https://firecrawl.dev/playground) -- [x] [Python SDK](https://github.com/mendableai/firecrawl/tree/main/apps/python-sdk) -- [x] [Node SDK](https://github.com/mendableai/firecrawl/tree/main/apps/js-sdk) +Check out the following resources to get started: +- [x] [API](https://docs.firecrawl.dev/api-reference/introduction) +- [x] [Python SDK](https://docs.firecrawl.dev/sdks/python) +- [x] [Node SDK](https://docs.firecrawl.dev/sdks/node) +- [x] [Go SDK](https://docs.firecrawl.dev/sdks/go) +- [x] [Rust SDK](https://docs.firecrawl.dev/sdks/rust) - [x] [Langchain Integration 🦜🔗](https://python.langchain.com/docs/integrations/document_loaders/firecrawl/) - [x] [Langchain JS Integration 🦜🔗](https://js.langchain.com/docs/integrations/document_loaders/web_loaders/firecrawl) - [x] [Llama Index Integration 🦙](https://docs.llamaindex.ai/en/latest/examples/data_connectors/WebPageDemo/#using-firecrawl-reader) @@ -62,8 +65,12 @@ We provide an easy to use API with our hosted version. You can find the playgrou - [x] [Langflow Integration](https://docs.langflow.org/) - [x] [Crew.ai Integration](https://docs.crewai.com/) - [x] [Flowise AI Integration](https://docs.flowiseai.com/integrations/langchain/document-loaders/firecrawl) +- [x] [Composio Integration](https://composio.dev/tools/firecrawl/all) - [x] [PraisonAI Integration](https://docs.praison.ai/firecrawl/) - [x] [Zapier Integration](https://zapier.com/apps/firecrawl/integrations) +- [x] [Cargo Integration](https://docs.getcargo.io/integration/firecrawl) +- [x] [Pipedream Integration](https://pipedream.com/apps/firecrawl/) +- [x] [Pabbly Integration](https://www.pabbly.com/connect/integrations/firecrawl/) - [ ] Want an SDK or Integration? Let us know by opening an issue. To run locally, refer to guide [here](https://github.com/mendableai/firecrawl/blob/main/CONTRIBUTING.md). From 80b4d7dcf21d6be913f91d0a4f8f5b82719d6417 Mon Sep 17 00:00:00 2001 From: Eric Ciarla <43451761+ericciarla@users.noreply.github.com> Date: Thu, 19 Sep 2024 11:17:55 -0400 Subject: [PATCH 097/102] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 279a1d82..03fa019a 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Empower your AI apps with clean data from any website. Featuring advanced scraping, crawling, and data extraction capabilities. -_This repository is in early development, and we’re still integrating custom modules into the mono repo. It's not fully ready for self-hosted deployment yet, but you can run it locally._ +_This repository is in development, and we’re still integrating custom modules into the mono repo. It's not fully ready for self-hosted deployment yet, but you can run it locally._ ## What is Firecrawl? From b8b8522b3357f4fdad022736c58cdc37328560ab Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 19 Sep 2024 12:49:33 -0400 Subject: [PATCH 098/102] Nick: fixed map exception --- apps/api/src/search/fireEngine.ts | 67 ++++++++++++++++++------------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.ts index 7c6d8a4d..d5e15656 100644 --- a/apps/api/src/search/fireEngine.ts +++ b/apps/api/src/search/fireEngine.ts @@ -1,10 +1,14 @@ import axios from "axios"; import dotenv from "dotenv"; import { SearchResult } from "../../src/lib/entities"; +import * as Sentry from "@sentry/node"; +import { Logger } from "../lib/logger"; dotenv.config(); -export async function fireEngineMap(q: string, options: { +export async function fireEngineMap( + q: string, + options: { tbs?: string; filter?: string; lang?: string; @@ -12,34 +16,43 @@ export async function fireEngineMap(q: string, options: { location?: string; numResults: number; page?: number; -}): Promise { - let data = JSON.stringify({ - query: q, - lang: options.lang, - country: options.country, - location: options.location, - tbs: options.tbs, - numResults: options.numResults, - page: options.page ?? 1, - }); - - if (!process.env.FIRE_ENGINE_BETA_URL) { - console.warn("(v1/map Beta) Results might differ from cloud offering currently."); - return []; } +): Promise { + try { + let data = JSON.stringify({ + query: q, + lang: options.lang, + country: options.country, + location: options.location, + tbs: options.tbs, + numResults: options.numResults, + page: options.page ?? 1, + }); - let config = { - method: "POST", - url: `${process.env.FIRE_ENGINE_BETA_URL}/search`, - headers: { - "Content-Type": "application/json", - }, - data: data, - }; - const response = await axios(config); - if (response && response) { - return response.data - } else { + if (!process.env.FIRE_ENGINE_BETA_URL) { + console.warn( + "(v1/map Beta) Results might differ from cloud offering currently." + ); + return []; + } + + let config = { + method: "POST", + url: `${process.env.FIRE_ENGINE_BETA_URL}/search`, + headers: { + "Content-Type": "application/json", + }, + data: data, + }; + const response = await axios(config); + if (response && response) { + return response.data; + } else { + return []; + } + } catch (error) { + Logger.error(error); + Sentry.captureException(error); return []; } } From c45a132cd55c486790de9a536bc61686bbba13c8 Mon Sep 17 00:00:00 2001 From: anjor Date: Tue, 3 Sep 2024 10:09:52 +0100 Subject: [PATCH 099/102] Remove print statement in map --- apps/python-sdk/firecrawl/firecrawl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index 3961631e..97f4e04f 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -228,7 +228,7 @@ class FirecrawlApp: json_data = {'url': url} if params: json_data.update(params) - + # Make the POST request with the prepared headers and JSON data response = requests.post( f'{self.api_url}{endpoint}', @@ -238,7 +238,7 @@ class FirecrawlApp: if response.status_code == 200: response = response.json() if response['success'] and 'links' in response: - return response['links'] + return response else: raise Exception(f'Failed to map URL. Error: {response["error"]}') else: @@ -434,4 +434,4 @@ class CrawlWatcher: self.dispatch_event('document', doc) elif msg['type'] == 'document': self.data.append(msg['data']) - self.dispatch_event('document', msg['data']) \ No newline at end of file + self.dispatch_event('document', msg['data']) From 2d597672bec308d25e75520cf55f4cc3a12a3d2b Mon Sep 17 00:00:00 2001 From: Anjor Kanekar Date: Tue, 3 Sep 2024 22:36:29 +0100 Subject: [PATCH 100/102] return links --- apps/python-sdk/firecrawl/firecrawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index 97f4e04f..7482e63e 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -238,7 +238,7 @@ class FirecrawlApp: if response.status_code == 200: response = response.json() if response['success'] and 'links' in response: - return response + return response['links'] else: raise Exception(f'Failed to map URL. Error: {response["error"]}') else: From 506d5c2716c8a4c6f7029cd36f091e5d426fece2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 19 Sep 2024 20:07:06 +0200 Subject: [PATCH 101/102] Revert "return links" This reverts commit 2d597672bec308d25e75520cf55f4cc3a12a3d2b. --- apps/python-sdk/firecrawl/firecrawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index 7482e63e..97f4e04f 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -238,7 +238,7 @@ class FirecrawlApp: if response.status_code == 200: response = response.json() if response['success'] and 'links' in response: - return response['links'] + return response else: raise Exception(f'Failed to map URL. Error: {response["error"]}') else: From b2f61da7c673eee6b806a3c236607614fd5b63a9 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 19 Sep 2024 17:56:26 -0400 Subject: [PATCH 102/102] Nick: clarification on open source vs cloud --- README.md | 11 +++++++++++ img/open-source-cloud.png | Bin 0 -> 198162 bytes 2 files changed, 11 insertions(+) create mode 100644 img/open-source-cloud.png diff --git a/README.md b/README.md index 03fa019a..3358f9cb 100644 --- a/README.md +++ b/README.md @@ -494,6 +494,17 @@ const scrapeResult = await app.scrapeUrl("https://news.ycombinator.com", { console.log(scrapeResult.data["llm_extraction"]); ``` +## Open Source vs Cloud Offering + +Firecrawl is open source available under the AGPL-3.0 license. + +To deliver the best possible product, we offer a hosted version of Firecrawl alongside our open-source offering. The cloud solution allows us to continuously innovate and maintain a high-quality, sustainable service for all users. + +Firecrawl Cloud is available at [firecrawl.dev](https://firecrawl.dev) and offers a range of features that are not available in the open source version: + +![Open Source vs Cloud Offering](https://raw.githubusercontent.com/mendableai/firecrawl/main/img/open-source-cloud.png) + + ## Contributing We love contributions! Please read our [contributing guide](CONTRIBUTING.md) before submitting a pull request. diff --git a/img/open-source-cloud.png b/img/open-source-cloud.png new file mode 100644 index 0000000000000000000000000000000000000000..acc15859c5dafca231e3080a6c83361b69b3d599 GIT binary patch literal 198162 zcmeFZXIN9));0_%77)-)Q>lugbP?%Q5D}3gy(7|Vq=g!ah$vM-igZPKuc3t?0#Q0r zLJ!hQAk+ks5Z=Ysz4v*)=bU|>^Zxn%9Ih*7X00{XnstnO+~Z!p*3wX+p}I^(Mn*=X z`rw{685z|+8QBTLGpB$f)Db>JGO}}z9F&x_RF#xCwcJ6r4$d}YWDi~^7@RiL`NouC zqJ8D$nR^Nkm(|a|{Y~NF8G+#4tFInDpm_0{m7-Qfb|h7)*}d|QMVx{S4HT85<$RG( zF_Dy%HfH>z3F23R1qq$FMZ#il(2?X$pX5^Sr2lC$KWPizp1Y#tUlZoPJ$-&?L?0Jt z^hra3d|ZM4BDq1cg5bhE<**ZmrzYS@P$cCKmm$d@Fn$@71`I-fM)&^ zWYO$jtF|dGX@%2fQ)>M7!H}YYqfM^9UMK8)U-;;$@fOLIhY;|%x74Me!AC6L=XXS2 zmh&c(6lu?QI_FgWo_M$lpi#8ve#VrT7Szf`0SR1uLeITIQGK!iB?Xr7ZPT1^cUow=p+l$YLB-N zKiJI5n6tL|oi}(rA5w8b3}LNfb!rKgAvouLYG0$d7_WS|U|QOSvQu#50Hfls!& zYioM7P50G_?e?=5&O0fvUg?XWVJTPT`8t39-1t`p;Z8$-&cxT(K9hJM#dpk@7(|}H zr48gRN`v`zx+5>PB<$l=4lgGrH|D@yc=g zhK#Xmsyjq0T`MaE5EhPA87xs zlOurR%xTl}WQGdlPM4FZDue548kwI2=RPb(yT+Tx=9|||YGbbcB%?HY^UE1a(mw6er&%wTB z$=Gty{fQp-$UpGSVaGF0dbyUHr|hf;P7AyaX(M}jM);L|B+HCk9Wy&^Szx);ob#mB ztr!sY#f=wYtxBz;iqbzu7~DQMCnwF_l^&wI{8e)tcksC+E0=F0)?djhKT5>y#7uBw zi?2kTP}&${;g!N`(kkYsWaCHsL~I)a`uCn{_IvGdE=q4L3JY}$_(@dRRifI3U3 zxOcwYK5-ILDfGf+^qHFTSumM&C>x^JQr%GYiKR-#sbWg$;x;9|4Y#8RmNbD}RHk#j zNI*}3e`V*R&UDALhBUb%{PsPO?RTkfrS+tAxO=~LTe;0JE49t4o{Y&h?A7QMB)bWh zFGNnPEt4=*t
0>l$!m~03H0*A~g#!_rXN(WqL#dnOA|HxCO39br1NoCX=@g;$o zBEX~n$Hw64juzI@7N4{bmQ9L4&hsyxKRACnuEly8!fm>A`xaaGDQblaH_7=vOKH*8 z2CrvP5zk>m{rXSFgbvP7)SWC1k$$%48N8Hrx`{IL{K+dcH5=!qj6Em%!XZT8M?)78zK3` zIVw{5={G|u?k|f^+YNd2Uk^Wx-juZXJYM07KB3WeNrKutm`~zlEP`_XPJm$ZIUz*R zrWEm7AZ1W6ZpPy5o#$6RXUwS1T;M)``^?~3Y}H9rR8&NiuEt)pVmzScb`8YQZkRnEqJ`9l>MRxJJhmK+y?Pwk_J_ZJ8u?gddvl0j$ABxP-0qQW87HN>)_=WUtLmtq9$xS zvKs1?*&o=}a&hsTpi+zB*U`@s?aTdFRt?uSJu5vQdIpX-46h6fk9ZBJ_osg_`QUE| zO@A$1k}P9O>p|%uzazCXcP9Hn+J%o7)KrFI>SSiqryt15%f2U7l`HKOYlqKmsRJWEVUQ0moKqU zvZzLjT};2VDrS3`aLMjcnW)^;Nt@PNTf!Goi$x_Jx1(~pX3O0TZHnWI*>#i#&+CM( zhz;rwYTAz2ZrO&}`uA+S*XbEsTv>GLN$v6Hf5hKm(p{ThQRkvsVQpe-Tnd%3I_T{j z=N)jY(1Ypa)nthFHMBHz;7e(GI*_(0>N!1nCLaYR#3sP;>(`}+dWQuEOItTS$~t0f zg=$gOD6xoJomTIxbr>(I0yD;irDd8_6L`)~zkWL?> z++^!L>JSv=!85_L!6ORmE&DCoErh$acQ4%)y({~<^mEno*s$}8t%~|#lQWmLPBuB3 zYv%her}f8Mu3A=DE?CliS#P!Z5j{7s^IJ{D)^Djzbb zN5w>xz6$(JgxeO}{oe3C`rYVM1!#^qr{6a98v5FxP5eebz4mW~^wZZ*T`Sda&b;N< zns$~s?p$1(D$Tb~^DpMpzrpBF+$c>QeLI_?{mvZ{xA*B%e6PA#a+=7&CIg;<^zKB# zi=nomse;Udx5S5mpvyuoR>rZxe1<|kg1hvkjNsQz%0$;8*ia{|`DpR$IlX>6yg@ zZyVnU8p!5&9+c1X{bn5Flp(str|Y+X*FLIi-z(I_`45fx`QxQJR`-dH41ead)be6|KOqHg3}I+Wwt<+tkB zj>lGRR=%WRV$pZqH_b2#ENO*5#oYfAttQ?)ac$Dmw+gj5E^y!EVQs)9@3fzX$!2ZC z!=ldb>uu`?6Pf4-qi%~gu$sftG~h%uu~MzF2*#VcYF*>+wdeX}iqEnCva&idD2ILMRCr>H6m?V$Q*dFP-Va%{-Bg+S1h&l|V$I@*F%Bcx8I5o-F) zP(S$Qu9io(!Q5vQYtY27^|-E^A#O8*Yltf{W|vJsV%-0cUopX_V@st+Ck2z1{A_mh z+j4wKUaLmBESo33?NaaiY&%$=So)5P_p%Hj;E)nZ?Ilnua5~8Q@bF2IiPMxaE|Mlm zMo_di+DY1d47a_-M{b~Dv0w&!CNOI}vA4e_4*E1{59etb^W|F_>6$oz%1^xyN_qy} z&scM4@?<8)Q{j=vA}yF$SyJ_I4f@1a6nH+B<%@FK#- zP}Nplos1VKpCLO*ewmCCD3JpnIdYai%lFA|lAZYdI0YG5xC7bAU)N{=-^YJ%fX{KA zpWi3qUXq;#e*Fe~e6lG1xSDD|>%<@B6NbP!vb#D;s;a=Zj+MKOjjM+}$P+#IqX;-~ z_Spj?4>B^Q>&G8*Rqbn=!2OX9x`v*H>W`$XKrVuo)}W_0f<7+Kj_*Sz<0AzWU2Hrp zIelE5T|K0HWUu~yg%nUeJ}h*V^Y=?Uon)^Xs%vp7f!u94#RP8&-nuGB#mUJj<8Ey$ zrG4-Iuj;@r*{k-Rp3kI&guK1I1-(TCLGE@!!jh7bLbq-U-M%dVTp{4$>*{IgBjD=6 z_46iw+~=N+hn2g-GfxMQE9dcjEuVtCJY}z5J$}%ie?Qx4nC3t*Yl?D9G2FYf(z>7K^b)F-#NmsrzbflY)98IWl+|8(HDeapOc9B@VTSEJt^-c-A(GA*^9 z{$bAE_Kt|-`?b++&-6kmFMQ#us) z(tjuEmofP7PC9e`zdPv<%ky7z_g|m%$67dZ{=Ysc_`ks6PjK;H-~eFM|3A@PRt{xu zprmYbnlN82i?B@X<$ib$9meP1!c&Dy08qSrv*9oJAHaMvS3UCrPG-}7-9+{Y95!1X ziY+t4{(HbbDnmihoQjxweD+T>;S3H*LTE-A+<@ubinTZ*7TQlWc-_sU{I~H&mJQ{} z0F#l4fxjB28{Ja(x(rcwwv4%ZDbK)yHC@_^ce(1{F8+@ro>HlHzgzz#M*7$h^Y_BkH zxnV*0;;{;DO5e2sJZ_zSDeWM1dGv#lY;En?kkt!5t7FVo+x{F(kCxxj$40Z&ay;(0 z3;DOE$t+OLuPdO0R`SPorW^ ziiNjiYsxWm1nfF|{-7zpuzyiF!0`9(73y&6r5puxRwv(H?=q~kHp0r*@{2hea`sXHb$ zj*N%+-I&MHF3gP^O*)<{lMaJ7EMEv1Kd}K%HDMPgwqm{IUM27u46nh`{I=aHre6Kn z8y~s<*GBj!8&Nk2Fe_f^bX8Y=_D1Y6rye{28&4vj43&rBXO|~cYesXWoG{Htk7W+fkv2uUV!IdDC!n!9nj~{D{L=S+GZ%Q++3sXf z=?*yF_R*m`p4&`1&xSETrEsP(EFCg<{m+&1KMR|h7tzT+NeYL1>t~YvFmrI8%`1DtAu7hsALL`TBBgzv0Sv$`G6@4ab&1Fz!IKr z5Gg5j2zz~Hj{SC7Pwc2EDM0Fh@);1)!t>@_!v~B;aPJ|@ULBn$`ePU4QHPuzngO!S z<_%4EMbuDMTu`>lgLb_UVUdaV{0wb&%^N%3hO6Qqyt=t1q8s2H)I0VMSP1Y0GD;jTB60u_)OU4{8z~DK^q^{~E6;!A*TG zmVX6#H!J02C&ZRCmxRK*l}K}fmrYMx>#i^dS63wVHi_$}-y5%2pAd6vY4&fg{Zjjy z1R-?H0;XuAiPGr$$a~=fSnSg;cp65;%6N^3sqZ)6fl`-=hzY~TFbcIKfmob8UP6wz zS(PjT3Jm3L=y~&5c(MP`s=tMtX1uTC?>i`Y+`-=Z+d9AIhKqvFpd`i#R3|%9YGzWo zz}^_$8TdW$k?K({n79NwVnqe)sE=7nd7{rJS@?iGdw5+OPZmC&X-M7pN|B@Hlm}F6 z+=phGnm3}AzfgXMENhBSGOdl2PAvCML|;`@3u>M7=T?| zKiuf8{)x{*wkcNZdN~5-3!W#G7}rDu9kmZrLey(EHwYZ<^nFcQZgoGDpsz(VWh-X= z_|7BQN;L^ri_Q=M%Y?vt zt#!FJ!VWM@m#+ITW0wYvfUu`>J zL7{hjbBP5Md)-2)C|xIZ!)$J~OEBLI`_b}va|>nkY*2dD)UJiF2$w_GoW`SB+06mS?e*Dzhrue2x!Q{octLVp?! zH8}UJlsWqEfIY$-v?=V0Vl9v}(tqD2`B=BkhXI9w{Ts zvfj~3bkp+VP!B=zIR|(9pKJY}m@1%asb+xRsLG_E|HE%+XPLH$M;~#(n{s*R2Is9x zjA2r_8tLqjNrKS}T}cT1-deU=lHfp*O6>A*k&3fuR!}^LhZ@@(y}3Tq%2Z)1aJ3`b z08}%xZY735#oJ1IS%OngVri&@V@A4Z=@-~fV!U|U{z;lg2i-@nwR3^=6KVB3vx(9i zH>Ep2oL*-+y&>L!mP#MyDz69z^|4RQ&hGR@(XLMz^nHTPymuZgzmltw%1&KHb<~w8 z6gfNwDc8x<7^w2cEss_CpSvt^YrNK_iW07*bK`$Hm_Lr}8vr2t&_K3on)&AfjG=ON zAV(=jCbcQW%3RuE;xF8Zs&PSzfiKp(uzI%04K&)tN_Qgbflz zTU1FurYN1KLAbr)zx;ZCF%{&F$vQT$?!&9k9m~IjyVBcb_{}omX~j|1YlYLPt+1u^ zy)WG9lfWK6f9!g{CBrJqXA_dr48B{?unD-K3SQ!jG9@RyP+Y>0#cnBa7t{v`YtT^g zs9X43FIPJon-H(H5yY2RSvHGDhn3i71J^bE661CN6C%h7>zBUNjx|hjdtr0Pwu;LZ z+lJ86plM>*fH1YKcc0?;igY6=bJ}ZTA~|6tP{hAC&9Ls78-oU)t_^4`GywH3;_eLlhS^bmool5EOIunk^SZwLs#d( z62bK){FP>#4?_XZY`3&rkrW4D9Mc(Mzu9%8BELXOAs(RV&v>;+)9cG@RY;n&Cq!B|& z&9umV%P^CYlWx?ztgzrkv)^7y9VxNV>L7wqbg*P!T$a^sCXmp^-ncPzjWMf>ZMmqJ zcXuKl)kl%7Rw;a zjOkRDp$yQJ+e%)32LX$ZBnH6lum`P%NqrgT$%nKKgd=DtVn;`xv6yADr>UaKFpzyW z=dX&*3&sR~(}3mAg!W}wumIGU<6k4r7A9`aV=#B?4=}gNl+UnvxRt;xdNb{v;xiuB z#KJiNqirvKv)bF#L&bfCTYKGyX}H{%UWv8|9!GJMO(P{{1~`_&@>+j=$X@YakXD&v zGtZ0N8B|6T5TEE?EUETCg5aWrjb7C?ySUkUEL_4b`l2`vDR21hxpWwbFoMipn_Q&D$pUKS0lSW2bbGX)tkhP+T;~?qL-Jy*XuyQW6NS3LEFAfx4;4X2YzGsq3SHV8^c`S zF<%6~OoIuGB6|jjW*h5Lq+CJV1F*`LJGS#*5#-bggE6ZTq9$kz) z_x+fQ`fj+S4%lUS3;)Jh{465@U1P#3xi~Ys;U|oqDd3Snw-OWMKwYhwM^RyodZ}GuIcxP9tqY1%q>gLb)Bg{O1n5 z^wmY-g%0(QV=^FZAI8;HGH!2C$nP`N-j(w6S&+FF2#!=+MP8N>w;9)g21>*k+}sNqu!=f+uF zyQYe^5o&r_nz9FV#L~$%hSkG0E=|#85h>oH#k38#5)<*Jk06gJZ&JYAdX44K?j@P^ z(}?z0KfX{+gQ&p0Cgu5AWwsx@^walKx&-a}zp#rE#1JSNBby)02#r_b(}B>txMf|3 z!*V5TSC|cpNZBq~7gww-y~kn?sC(lrMY#$)_}MYBAcfOJWe0VZ4$hvI2aj7?rvpz68Gu*L8Q$B_MD5&Q^}pv*vy;Nc;k#YGznQJ+ z#I$zky_?b4@z}_x`HBE+sj)KL&X$>SWKHD0->bPowZ2He!;RQ^O#Gb__y9No#ta6X zKb(LN-~=Yb39q*T4++Z1;%!SRSaFYHtp!9>$QB|h4zNjMGbg~SL&8Pxd{sknYSJ=? zxTIMQOhA7=q-(vJP?|kkmyLK=?02~CJ$t~6NZ1ws#6`)VqgA1unxQw?v{V*5(pdOp zOM>eqUG$))BajqEGY>b z5XwwzcwJ#Tq;V~OQUktO!*a7qQ@)@%b>bJx^M{-N$H~v?JeGT)UnJB1bS32u6bS&G zEatc}JweNI+0I%XXFmA7uT}nNn@`wjByD-B(V)FEsjG?;$#LZF)LuR{5(1DFnMC)!Vv5?4bz{`^4w~c zG@(Lib_T`-{%F1a_Id33I5fI9T}nBYpJ6o5u5WU?RoTLB*a;r9&;8LONn@Zu331F% zdiK?hIVv16rIYqmL2Z0<7JQP6eWmqxq`c5t-%$)~JIwr1ETBfHb^a_tq4E*Z*er7^ z?rww>fMPfd%-mf)d)Z?~4V`)k8ujJ&NxkWFHw?_)rH;HS7<9*FNB0Pqz=#LTU}X2ukpQni=VS-NWdOlzIEQhg>CV%qOr`-2&Ib=G zSc)12$XRM*`LJA9p)&aeXxp%%sbO1o^rnq(FZq9iNs_S_z##SZ`8>MyollD`7 zgG%4`-hV+^1YDo?Axu42*IC|3l5Z}~D&t|+W*lLl5O8fDzIzOZI@En&K2@iLkcjHZ zy>~%b>M+bF$6R2NB&niVb0o zUR${}T`rZp4a{i}$8*a_Xsr~XX(JLQ-(i2LF?7Rnx4 zd-=S_QeSMW9O)xkiS@+k-N?vZSr{1pP{}6l| zJxF+M^MzRO97Gl3^f!!>$#M9f=-u0${1Xi2D`oMVXkN*yhq5)R9|xkRa)UK-u8gtA zJ5V-L1fc0OI=OL+W4tVQ0c`tGW#u%*_lIJfvhVjl7ULYY6;3PiI=!0L1{d!Tc6WHTX1FvHqP#{NP_dRqqjhznP`#Y3K^bi?1Y1?s8kp;)&G0+IJzS}o zQxFviik#N{4XpN|F4KMyw&YkBD!j3yWpC4gTNfKn{M!#lIXd;*qxVno zFn=uBJwScc+A~g*TBN_CVch)ORQRF$k0@jWoFQpYzA(sEhh%)n7g7Afa^=~G^Kv#iy1~lnB%%tfF>)EoTe8V`bHWE_1}}oWSqBmfT^|o%9M{zu#7&!j@8WCj&mT?{guJA3e%K7=@9;oEfj>pW zQ(Zkpk|iA)$?u9{-pj|PAbHTq6!tD%FPwGiw&8(CNh#{oq?l~K(loE)6|N;Gs9KzYT6hmDIcbj3ve{`6xjN2aPbD@ZpW zavTv%@_)--DFMKI&~D0MK1nj*XpqG#V)7m8t=5|C=A1 z1t5qV!iG31K&QNGrr3Dnz-VD61&dV4;5T_q`ZN~{WWQVus*GSqxrGBO1Ux8KHhrqB z>okfEcu7|yTzk|OTR5!WSak5bD)WF?;&KVxB%5ozB*@;s(hW?81dan1s2?iBPNJtd* zO1saOk6s}d?KJT2x;i&6JQrP{ux@9(Ianuc^Eo@ditKxCtu9O`tS&-H%VAO@U25@p zm;Xks$GK%x8CHwu(H+a#wff9ZA?D|5AIfws#o@Bs^B!A2R5BO67tHf`Ujx|3+;jbV zy|tM-LK*|Ps3}Ky+Vk{NIg*h%GoJFDTG0iuBt0s4?s?~WeFH;2m(0MAhi`9W2tb`a z@ixmh;^UUz=-rw&^yj38Kr+JM5ffVoR`%m8Td8z9LRWaFgkPb-b}z-$Vq5As%zwtT zD}V$z1O29VhmZW-)CgvAdapK;DBB#zoj~d_%;uLZs*(x*2g-ITnbWXiyZ58(lE@su zxR1?$*;^fx6_mm4`suAr&WGBZ_*952zZ4kMXs2{KMfA9laejA(_S}gv18Sr||Ot3J-a1g{;w{UoG|w)cBQU zHdE4FZ|JLZT#l7Fz(sssyt9=}D}U`irrv4YfB&d57>!4hhq(*5B-NII)Q~^a$CRk} z3`+S&uTR;L1d2>8=oJLK=}?Xe@X4jBCx~c%;$geNY%~-lFt$oSurW$wsv#~%G|*`_ zY9_%_GYz2z@Oqja1_nf_SEyH{x4n&qvFluIjborK(ms&{=Iocq!CdnZPS6$_dR;LTJ5v7pbKfNHg+Eul2y=M`H)&|{+IUQ z561S-03+WVNQAG-ire1*xjyeGX33GGbZs_U4sj7zjB2KH9W`w}41Y875rwD!)Lnp1 z1QuML!IZ)zgW@OGtYT#DV8ksa$aV_sAs+c4m(y!S1ja9KaC?mg_dTco-l4XfQb@ z^xDItlxWArX~#jHszHH>Lps?DJdgAW7!7)htIcwPG4P|=Rnuk3bzJM{*48*nyxPm< zILiDW_XXwahhBR6f(E``t)IjJn2Dt7I2d3oi1R#H?V@B68*Jq(Yi6Ri2x@dbFHE`J z=)L|Mgs}TM#z=Jbn5}5?-I-gn>2fME%gvg-5YG0_Y+|pBwxIxI^W`PY))8r`Ay)zX zd(Se*-IQT6Tn3oE{a^@nYyAO%->~kcYu#Q3u{5G=P>8m=OR?DrSyBWc`!w0%&}czg zu5>Tj{TsoBGx>3@utoK@@~>nl={R=f(Rx>i%1KldR-fvy(R=2kC?jgcY!I3*9(1>K z)v^bl0$0Cq$0Q|;RmY|;*Li{BQ9?j!A(dMXF|}(o9=+7X(HYp$RF_bsYueQFv0ZIe zNF1d%X;z-bYe-s-pk6C}ng+hfyeUAtp-*)+@38M{;g{SjKXtV$b?`}d@3EOZ5@F+# z6|G#B?cg|EC^qvV0CS4KlCXU@Cf3p-3XjhnJ#}w*U-V|m%Zpj-MNQZ-zV*kUFQrR+ z7y7g@a6c-!9tt~%uiwa(e!3Nxb6{4V1>Qz$Bbune`Bs)a<27!BF1WV1vZMs(1PX#Y zY;`s+qFvn9a^^sJ_|cB{wCg>6)!q5dgnRO&`5hmC^Vs(O1}iOyN@X%$ZH|x8%ZzAq zXq+u9Ex=Udk=QXo`>K9M6TA>XNAXcbb3kFBwuPOi>+BViFSih!Ni9vcuwnFOA=KAP z7R^eF6U@6n1r)x8_cBVyP{bcEiF)-&!UoJ=@P$R?M@5yw)$~GtW~dy@zhi+pU8moy z^x;nT#}U3Iw62&W_`^E=8dP*chuDO)T% z;(BT@{P!KT)(jnZ8vAif7k5P`i;uD~rBF0!curHE0|VJ~`+P^I91DacJ-a5D@&7oK+>@C z3!rD2_xZ7~MY$#jo9PC^CC$5;VbRnw#8Hv0t-Q=6=j0#_HT{t)E7J3@w~8+p3N`Ys zbg=p>4CAkj=@wu2ZfpmoOw04h^es-}N>SCVtWFD7^zcQx_F*<@Xje1x%FUx6f zUDq#uB+Wu%MrrKc+6XiHI457V=2jNL7!ZE$^2Ouy)pH21fUX}WmkUb1QPV7Ls%(4W z=&+~eAhI5rdU+ED3qI`YyR0xb()Hc7AlfT)oD z*J%lbowH~5#WG9Y{>p~;a02Y2|1a&6_(W7ErG<`Ho5c~jaNH_z+d*FzTk(Ch0>LiI zjh!sn$TC~!WHf(AJ?phP#M3KJ2zNt5=#prH^e^)I;Z`r$rGwo)F(WnB@$akP5z-sJ z`JdoCik%0c0NSXJh-c35H$KjWl76U}!%0}_En?){Lp-0C>9gJfVf&fjhedPE{L8Y} zsRNdvDVe1f749}f!2Vge=#y7zvos9{qbD;i1Jlyk3Ij5{aDXN3RUBu&mx`9+NB%)i zQx@s^^UvhtTHBDQh$Qp(hF}5&J1zhuiJy&mv8<)-cbIH#^TqSNB)z1Y;_j8%Ym2kg zey}MYL|BR=5o1o^P|~!r&)~$;IAnq2W)o#372c_IaA*v$=$^&yKZrvXgMe1=vVpm~ z`IuuhU@jw%NQZl(sf(mU>Q_w8{cY51m7M4v?@1FDCUq3u>2-N2z{Txyb2#9;jm97( z(Dgaiz)|uF)b!E0pd{-3kv%enZ%>rjeisXEbyF+DE-a-VgkHSuqg5(z7TO*%jMt~p z(EC=_#hc4~`jr!{A*)8IXAj7+1v=?{9|$OBnUSW(A_?3F(w5&@guz6o>sC3Oq*fsF z`WfLUjMmVw9=17X};#&5kEt9`%emhA2u>N<7&wC z(y6re*;ZVg95ojsl(=c1qo51nbzq83g*O3loe}45%yG!U5qRWZaR{DRG!FtS-_~}W zffZmwuTKnn0wX1OA5-JR3hpO~nq3dl5glN-Dd6gW;uDm+aV$S*Dtz44W3$&Nk75Nd z7jN=lM@aXJKvqu+I~>< z&^(W-D=%Xg<}jgWAkus5846f->B}}JAd`fHECa`zC7^4{HWYx{5-NCeboPkJljT+L>INVl7&-uWN` zedW2q+##Jj&sp^)4v{Hl4at={^xgD~gDLYr6cGPJ73loo7z!-E7IFCn1)3Cco|sT+ z9V4b~Y0tPT2d6peG+CRDy zbinF|bLn)Qr5ycbpfvvxo&h;(d1k1)Wjn*B8QfCo6X#5B?XlwQI46|;rD=0}-)Ft2 zweU?ido_r4^dzJ5n#?AAxzn8-Mues)ZmMtuJ_9n`8V6G~WjYCCbn^H+)4rH5l_9O; zG*!$zlgX{Hh0>DhsU#8Gjwzgc99(S-S?|`{*WqB{7p?JLN?;>s|6AOxALjmORs^e? zVf8%ZhB^trKbKZW=B2}VRDOlla*##z^C_Xm&En;WJny>}9s-YmHeHl<)gJpXc<2{v z<{fbY$HoT08Qw;9H#;iBphSR#^x5HHG!uj`qn8I84P23_m8V4(ZRqS7pdvv&isv}Q zs_PnOw^=}N7NICM4wlL%Zf+Ca82(+qQRoLGbJaSpUPjN6Yd6OY*{E$mO*mroxyhDb zak$%QpDK%fV@%aT5^jVuLqiS;9m+t~dD0BC(7>yLfnzd+7iUha%Icf3kbX3v8y%D_ z$q414xX2UOsm3Qz2I~y$LR4~s-e9JPw`H}%gcKgPMz3@VfE77U9+aqj*7!Rmc&#B2 z;|O4f-b?&4Um983El@dm6${s zH9zi8ZGG`ObjGJR{%AhU5y>-1Rn#{UmCoD6o_2^Pk_c$M+DiK#FbiSAR6jv6*@TXr zRfR3^2@`gWzN#6|JMDio=W~N_7Vpl;J2)ASxxSS7w&;zCLwB_qkZGuTHs)YtGgs{ zl=p+rn^Tp%-wyHEZ;<GjP3;q!u_pwxv5oOU6kyJ^wc$~|&6~=jY ze;^=8**IPxFBfJ|6^!X(q-Z`sJhDWYPcS=VJpwFkA6<-_oiCJlC(Ktw6*xpt+!2;i8#-am)VRbo;Ajpz_| z3BH^z%O6*mxCG7VN^GwYpM1O}C-qDL&i_v8CbMG3-tZ*aGNQmsf4zr88rB;mk*lq| z0?;4reWlHHK1e?y1N;$#=!s9bH@s!YlLbzA7Ci3gaHAov&xA@X?}+Zh-$W|`-asA% z-ncX^{ZsRF6i1QJD#63x?Z@=gu@^hx1d13o`YHrxOVOK=-VEZ~RQKCi2f}WH9fKuv zqgF0h;3gXZS-o!B4_xLqEi2~06CJe^v?YcP z)!#V*EA-9I0?KMcx{u^Vr|$bjrEdCt&V~5lqPm2n^0N9A4=H)|U?oRqyg)`MsxBUx z48?gGhXM-kSpHYd)&Nf#uXna&NU-<$dV!Re8p@lrMu$wI(X;^aE0mcf4|hmc^{xdj zzkWVku)5-FcUYcPR?ox>5f?>l0lLIE>WNyXiCXZ!?iDFL3n2qvahD;XBI~BP7_Mwk znR)%g*&SxVC`xK(#78vhd3&T4L_eolTkXoYnvjG@19-)>@>|#3jSfH@&$|M8pi;!Y zU@AzQZcty_KM+k?Y_{M%$b}pU6xmwntU#|IO2)9*MviWFVzN$^Ar^!}PuDByi$aoJ zukHTve*xsJi+s7S8%6>+{?~w5ohd54qpAC_uBGi?zuPg>2#9Y8X3}%hLcbe_eP%#j zvF-3+v!dc266J27>h*;_M)oRr$`>*D2H*<#%>8Vhj{OMQn(O+gM8Mi0U5a;pF;qNb z#geEUlXz65T_%T#d-p>&9~~_{HsAI!Ok!mr`n#QRP2NtwQ3NM zAvl<_v7noFR7eZL7T6#Y)*e{_$r=&LqVHSZ`=+7ao_iS2owV&vWAr}S8;H2u*%38f zvRv6ONHz z+#GdDO0UFBw~xv0Z^Z2taOqvw9R9{K@%`|yws9l2T`h6M=@o;_#$c6-bIowe1Kh%u zQ0TaQ7>cDT{NAFSj=JauVDg3Ar_>vWztr-NZ!M1HW$F?Jm*^`q?Kw<-+!zl!i0@iE*Ti(r zxLS*6C{0XyZ+`A-*i1?`1b^_(RCLlkfpN{FDWxAHW z%FSH^kHz7jJSvk7AlG@(<7>#<(r|n&km1Xu+_VrJ4cPB@nO)|h?bSn>H-cS0H{kgh z4DNzf2)#oN799{H%IOq>`u6^}1m*;qN_{_hDC!Vd{M*{(^Wo#jgFjdjAS@-Br;oEf zs~Dl-gE0_#>sXwtA-%BwwQSNvMjui*z=Y9pkd$q5pB%7S^+NfQa@@A?CK%nzh0Z}# zIhk&&ssqy0U4o`f*Id6+G>I<3ZNt+mXEweW1$%#PRW`c5=kM4+O>%W?5vQpYEP4_E zB)vNfJeHWF%urzr_;tXOo$D>F`7v?s^@lNU^E@;mD>occdMp)4 zmVCd5f*fU=el7jpH@pOYf2RzF8@(PQMSMfts4QEuS(>T!XD_}ut!8yd5Gw6mC!nc? z5GF@HaI^%^IdJdF4aA$`?{1DfQT|H$SL#d`exz4shROdJ)q; z>g4rV3fd^Eo1Zidt}Z=H@tb-{!!kSAm)j;i23hX9RP|9fWQ{196MXbGoI{szOO@0x zi(Es3zJ~P;p12(Sf7tu-cqrGf|A?|=3l*{*vSiH~5>Zqrl65kJh)hf+`(A0aX3Z{3 zWgELJGeq`%-;M0dShDYa_jIb`oH*xw|N8y&{^RqQ<$3Pwy1&u*T)hMrF)G3y&Y!xui{e)&(v{3mvq#W%msH0}DS$TEC4%=}G__7gwWwJr1jo(inqQFHb zzlnhUTNxT70G!3yjN&kfV_i6rL7&*4B;x-I6uE*Upwr|Gt-Vx-e-zRF3L)!2Mq&R(n)nos+V*82rW zudU_qvOM0d)L&0etEd2#aNj{EXR7EF+q)-av$*cp; zhl93zz9?H%uqJ)?O*M%eP=#X!e(p`wGQ7bXA7$g~hkdfQ*D^;zM>pQ1RI~MVhTP4! z?3K`?&bAah(t~!{z=t^TO}0s{P@VFDs|(QirIt|8feeQElx9Aa2R&F=QRytx-r6X) zD#X#n+4i6{Dw7$6h`qQ`-?W|I5JwyrtU+sbY%YBDyP0bDC0EgU++ZYj0vG5T@rqcsl^7r)NO**Ryf|fE4 zWix{HHkPW=+GsQOv4;YlN!K?%fAcPw(Mg3ew4d`T^S%|^y-?U?L+bww5*wipKey$t$TH6Y*l?d zMNg>-1_u^JG1gr3XeYtcw}MT1 zYVsI&vx>F}^T*Pmu5SUEN$KQ_%+Bl0^vA~ted_N9$*s`2hMlptJ+>Hm+_E>jTH&_r zJGUM5TTS21dd5v@&pM3rM*l6pEaerJjK+b&(WJ~Vs%(5W(1ibc;|Ir1s2DX~cyj1Y zyFC@cU)Zt!9F%dzqR7AA+Ths0wXZ`4%9gB^EK-Sga*81_@&(H+Pbk+?`^F_)DiRwb z-X@RrCfa}=Toin9svQpW*ye-eujl}|==CCKsBDU#LYp^fMTgm@fMxh%q!efa%y900 zr5mkvJdJe2wyty>W!+hp*iQ~=8J-nF9Yqu^S^mRny^*``uP+Q;F#l7Y%`?- z-?oux99n+O3OmM}8T0919FTubbgmw!zs%m#4XgL=w_|AM{5tR@u+*`*aApkn=ATlM zZ5myV6~QoQz0gPU+6h?@!8U2T&yPc;tgqG8o!(4{F*6X5^Sjw;J1Ohx@0ns|Vl*Pi~|SkH2D{b5C&ZuTk`^w09S zIkuPUjm}+T>TfvoFHITYI`}Efbs6a&De9Oj#5d9>8&VYR=hqaf*~CmM9RXvK=hHry zSyr7^Ogx>Rb*Ir=svEPYiG3aH97)cZ2ge9f%994>oCRY}KUi_rTk6qHm)j!LySx4Tzpzt)pKY+WbdXZ{ha+sx>yd|9V}m-I zDx@jQB(h{_zW2%45R|C^vvS8Rv!dxc!P!1g56~Ab#!|1ay=IA&we`8}bC$n2Sr0Uh zi;*W^D1Jitk#LR!G&{l?Li4L<(OSvc-`i|KH~8&v_9v1K=MMhVUwf#wY%jo?+Osl0 zk1sRKB8^YS@X>OWzbl7zI#Du9+g5qHD@_ZFJhKw*y6(^kX?R8?QCu>vnOJd`#J!ikmyIB4ULd1fif=_G6h_ZXNhz4TRf z-*NGsd5~G%nSbBK#Q9g5f3T1<-LXDwb($5yZ(j?Bp_t90*0=uqCEKE>~3Rp+&P zG{O7mY9nS_N+-7P&NvuSs%Q@}P|uq9GjjgbQt;cw&3yc@QZ!rZPZ_|2LSB*B)oEHS z!?As;r(YiopcikiY58wk{0oHfZ>ewL*X8C@=eK)Z6i4w>)kDcr3uMbuAQ&?)q$wj` z=l>V*eP4nN82Hp4cR0hhd#X=Ne-QYeuc~KV)zf!gd{rC<27u?kkI`u71$c=ZFHiZ2 z7yN3Q%soKgI8T)QnCIgy{FJ?1@mR}5Q>@E{;{Q70?*gZE>80ZUtX+bqgLjeAZ3v z&Gy0fahrGMTK*3fCx~C1H0vG3U61kLqOrf*mwe`1PM%7I^j6^T?Jsjir?#YbKl3kO z{`z8!BfhVF{8lLG?v;nB8ZJ$Vf0OGRpu>yU=g<)7{X2`;UXat;KbiCAJ^y*`!1w3N z2ap!cPr=V@w@X@7!uu0#Hs_T?lzVe6EjEgN1MJ^%<1s}Gjz2_O8~;sRn2UHEhqACD ziWe?C!uCjAll~fdqN^Ad;BCpa8%=+|)prWbSpf6K9*CdbP49m&32eO*z8w3i{@Q;C z$Uit7rVLJ0bhzbx^%uYYCl@gqK+3$0vuOW!OB|yCXRWa4#Oz?fWzb*T_6#bu+{hc5GzoqQv z>ynzFWaILhpyu~>NQ<{`9QF|Ezv&2IS;3ek=Xsf3rR$~^Lf)p0>TyjlB3Hx;S~YzT znLF`2xfboG*B7T~dM)!T7wb#Wy}o_93Gc`L^DMELD|n6dw3%&}NM%1Q0khi1rSW+~ zao6QugX0apNMcN?2a19f(0G%KMmd&K%6L&P)@OZt^B-cFdlJ|cf$>{!d4Fk7kJ%J? zwf2ELP(QaG7pn4f!j@7mrm9p*|A&eCU<+<#A*VR@Gg@GiAC%!&r^Rhmj;!3CdF*z` z>2>mQM>)p?B*e#N7d?BiSv?i+vN_(Vd;N=m5<6W(%_pexN8%SSX^}GJ9g5GsEOR_J z-N_Xgs1y0M{CzGykb%`Dou%N~<@U%QHJ>s>NVRuZ>MMg`{5V)N^Z}SHC0`F#Z260b zzoYc0%?;PWpW84m^P@cc-GW|;IrSJ--tsXC5iVg#U+Bf7{*%5#M)Mx{ijb)lp9dDu;z zPEBPyEjz8HJh4Z1?V8tRmmCTbEec#09$n8jRY-gvArn(t7u3p}_WJE+Qe}LdW{tQvG4Kh9d!-mcM03pebKk_l|&m21u_8L)+ zW8E#q#u+^2mX{`HXQ^&=3dV3O_k14Eh-ALOLwEJ_t<+&Rn2S~vY(?SYjD_+-PF~yv zjDpGqB^$3mQrZfY_u}t_j9`%G%KxznuuzY9VvG z9w5vhr=IhU3?!r3#Uy#+h|dEd5{NX%?18(Q?*o;Z+M6AxX`Hs6+K6F{Gm6DT=9J)t z#rCVsVNcKnvJ@MX3?dhwQCJ3L7#Jk@KV@j|L~`ps+x?Wi7owhe07&_$F4gYsIZbp( z5#=k#8@`^ad!g5(*-`uvlP#HyJ|cAeQDJW`{b+kC#B}S~@zIZFQQoI;kQ2t9-Vm|` z4Hs_J6EHG1+GUDXj=&55x#QF%9P%ZSUPn!Lqoag}+U$emlsT$G$$_88(>vAQW%$U( z)CV-g6SOvtz$7vM-JDCIp0N|7y_qv1`w78VYx9bOp?}jMd-BR;o$8 z6FGv)Bkz_Ftz+jMrY6l25Vf_L9gZocE&eX#0>tb%$m>6_f9w*Gy~(fV1unnTMkGq< zueGtgDKSjXt7vw7e?9V#NM^<n#M4>+8&XgbKe0Oa_;1H0I-9 zW?FoABk*wp^E3x4?DM(*Cfl?f{-D==4|2XzgQse zhbQq>v*YvJoFJvd=_y8z;R4$LjYw&IM68%TqRBQZy^mql1&k%=6@R^?m-m%euXy`X zw~6;MlTBtBy2HLg{pNjRGvA(8V!YY2{X7b>;Lk34fs zK9Q=Uh+PYd%&2x z^XS$sd3EE~MG6p67#SSXuDk9n#e>}7aM|n6O?SOJq8k&ej?6Sqw==%J!W28O>|to( zqlESDSz4Yz|FL!B5s~XgZV^8`br8{3EmeT@dE9sA;V0El;Q_L3pX|O$fUANm$ zcQG7g8w`(ac=vJFXTPuupTN1`t~lIXxWjA3biz|YwcqxGU6@ZBoLEiVK51wE{0+nj zCOZb1IT_(ngoeb5C9n6K%3!BJIs89)eSjeE00AtI1i1?$Rj~kKE8na}?uLySuqg=LrTgSzFue5Tyyz_yK#FI=WStt)))Wt@Yz77jQg~HjO}XMY04f zG6763Q;1EV!?h3cXwDPb4(i0iUTtk<+F;?Cu4S(?t;)!? zh+T8a8CTo4TX*it7pB^9D->7Tor3H#KD$=1n^6|WKn6*aoqiVzH%xAd0!;LmG3(L|SX&HL$-)1nOJwX=9;6IqQvl#}2PIF|77~ z%DIosmRKq-e zH#`c!<5Kw~nG|#R8x6jo#yeokTWZVVX z7o0I314~TMKpQ4#`ZkIO81+DyAKZmvw-7G)z;0-ibK(jv>OU}Km$%LposcnVJ@@X9t*(d8d>U0 zom*S?Xv*(gHyV?AQoO$1wnJgr+fBA+t2>K{Oz0ck6aoiDc+VkD+^pG?)w@_G zKLcF6G4$*13%@|-p+t6*$T1Iq*u^D*8u0*<(Uz@Gh|(vB9Z*`ne?HH^cM!pggQO3k zeKZou1$q^yEK_ClJTPgQ2ZT8ej4y0eK-qUaVK4s3CjejTD(%>=)u{)`UkdGHiR@&g z*HSDZa!fWhO`szsB6iF9_`qu&KAUO12#YlHJ%Bqx45#jrWlTq*eaKQ`4I_I}#(`O$ zzI5)teSwc)OTwpSVjuj4#k=$`IO>u&B(hWM8515!P69hw2=R^7K9E6YoUJ4-Ku4C) zZ}*I$b?QQ47{t(NPKP-~c!Md{R`yy4aK`e3;;ecO zw%U1-!^z_S;dw$^;aa^f%wLwsS}pA1)|lFc_LOfEhaU5qibe{he0df8@^;Ihirbl} zf+6$kjMzEyA^V(8=7x;gPs)o~3iJ6Bpor!Q(J1J---yk|SCD%6X zN^;O`*bcbAFB9+@a8%bzh!J7mSDpsW4x;&^WWuPP)X z3I*nLIR@#IzL6i3sSPicn7QxGD>h7NL)~B=md7j;mhSFy?SXGqvRj*mw3)8kEgJEv zWHkv-ho$ky+Z;{d_bYF>2ohiIG+$S~{v^b*7m3BOrK5%&@4?x0xy^WVVKPrXCyYY! zY7)@`>)DfB<2hHmB3BBRNg;D7?&-0>BA2CO!LLE4VW4tqVvBV*2vUmPfipx4xf^4fCVY51H@|;nm ziUsNXR7MAg3Y2nu{URZdgjU9nI=*^a%Au0n2u@oLYrRAVjYY>*Gqx!=iRkjI}6KgEyh-0Jx7vZ0!}=)R!Je7-DM zoD-8{miJ1Nrrsm#d>>Rqmfzv&_fR;0{;^qN>>__TQ%X@+tb6UN^Rf8bu zl}s&>0kX^VNg4$Aa5=8i#tT)q78(R+G7ER`agTQ2*n}4x>ySQOL7iE(O=X(?!iTuUUqY4&MI&)P4p~{WGwTShaK1>qmIN6P>*v3w25cJ zix1jZQ9YE(Got7)=b7c#TI+LS3K|Wig_g{2nNm)5E$JMxRb^*J-uD5jjIu9YR(IGL z7Djl~)YOA-Y;nE_hc?8(P_=65*Q&C}b!GedzM8qk6}p-6!=$YmW7GCTMhs2dQo3q<-vo zU8qg7VRowoC`pgt^6lNzy&z+ zH0hSd=~lH4HZB?UQTo0m%4td>Ta;4UDHHnm`iDfNq@R4vX45Y^TrTBfqv1N){K4?z z*_R$SpmLB+g|dbMR7d-0gMU}VXxzC@YGq|78znB?$t(w!;R*fUdz?SU*D&He;l15WuhiYN|0BaiqeMXp9klw11NwN`9rN16Lh z-39#2 zE(S8VxI!avF-AciL6?S^dzaEXr!dLaXmhdrtkOu%L9Ll)>e_=vJtvhercBV?VRM(~ zA=oi|s+}Qw^2}V)xmzgAec& zIkNff1!|O^_}#R`!iyhZs^=D_ChEXKoG^#eY$DkWxB6X&$Iv$jC=MWDJI0eMPEOv! zzK)jhpJ9=UOk8B1k0>_Q35x`|S z2(yFYLX6!saL99Z&&F^|YQDCgYM4jb{ap_!XcF10+14o^@OVUW)@_V}h`Vq4G!#lb+GQyY^JrXpS}+_A57LD+auoH59G=AM`T*u0A+g`` zTB4*bEH64(K6B$r(K5L}Wd>?qJ`LvGtK4mK$aY%%giYrzH-&^dU3Rn3WdxH7f$_P- z`7&>Miwq%UTadCfh{~$3B1Wb5>~l8!xT#xHkTN`<>51v%adjY&qR1&j1(cJ_jE#V1L75qeQMGvh*hH> z;<%R<_4paLBaRX{JsJRs3|-0YZ8iw3SDJc4U|A&tvo3uN+YE2JEW{#H*V}yl78eK- z*K@|Ddm2irUI^UaIjgMR_Tc#2zcliXE4Eoe+c_OMSW_b-$SV22Pl=huB8vr$l-31~ z@CKfT@MP1aG%_d&7a#^O!Z_CbG=TgR;7B6+wYlLj;?smE;7eAZ(KlY?eM;5sNf7pc zQY=hWNf~sINXMmw!yAMVcjF@!VsCcJf0BHCBb(tjlHX8+3~Q5lyx)&gR)o0)` zU%x6j!-TkR9mMeFfN-Qd!#_8SdXE0YiP5}sa~o}ZiulY1_fSos(bv?>=9W?>wU$VktAiL$~b zc13yB^_tAG+8+N04?M(3%m?$#U3@YgEZlLZAzNFvbdVCp$(U48H*2_%t|!JGzI$-r z-H5_R>-l!wd0j+Bgs6mYLvf!52+zbsk>?E=n~Nv+@ML2K2on%&j>A;JWlvp}M2|d1 zFU_3FeZHRvG~ENCOL%H!H%6WC!6C=@Y9K*A&2b6g8iek6BairSK8E_Y#kIjon3uQ8 z%X}G?mrhMUcD9XP{}o{WHopfU;naEfyFO0y~Zh;c$k0?mzVC4(&VWorhJvZmp;Dl@{QBf zrzT_yvJ~&spOxoY_PRbf(8T{6Vt$3Rnd|^}Oj{TE;B-y$v+EQ>H9$G&?mz?8Tp;lv zf8EUJsl<)a7?q^3ga_2nIFc@|zshva+NRT-x`EFYs&!Nq0rt|^edrWwN-m6v9e63*-%%`^kdvb&Ez?7~nD_y4 zm+Uu*FwvCX8C*|L3pX82Qcgi=JLVd5i;eWkTBX#dKFRAk={%h1bt`n|^WB?`}|Aa#T&+M&zrH=JGelvl5^>uQp4sNL2y9qYw} zy;^P|`R!xBxYBYOh+wL0>&597!`U9p&IN6`<@kFzEG>%WW*8HbmKXa5^TDpP>)-hSSeZ58{T=K9_ zTXH>1Psg49zDZY(wq7XN!Z9*p1r+auX4v16^c@8kLDTCnm)^G~q{(z^V{mGsTTv=I zMo*NIZ3MzqM!r9>7Xht;moFOL&xZ1~6*iNtF0ZT{KIAm5J}bujv6C@J=OF5-Bkd&1 z=}W&eYKki+n*456o?Saf6gy_-!EMuR_N-LIeMY@Ji46ArpV=Y^@H8zOn6WJN1l8`< zAydPc1FB>d(vf3(_#Xs>pV=|}seZHk3TOf!{(Th)XlNxw*SB`QoSyu%6sL(f#(V6w zS4ViL1pRMfb%3y&ZjOA7lBtIa(e0L0=ez;Q%D2vD+*dhxgaRq6IMSoqPI1VI)#=8G z)V#NjQ7(J1C5g7lZlu$paViP&)nb5 zo@IuvU+2H;d}zgsYY+N=+=MzBD56zbh|(sYo#Y()2(+Tj1K>vWcHBv z(8)t{`KGwu(mua=oI@29lTp#TSGH1Fg!@SpE8gSz}Mbde5kT|iJZ!KH1;q>@-d7a+uN>d!v#z8 ztEL z=o|r_KyA%)mXEqH7TnUqpJxFmqBXg$``z}5fMs~?VVr*|x)B4g@dHk0hqhCD=N@lv zZU=pD59!mUW4=<3m;$W{qLk<4xvuxA;3HY@<9qm& zRNaar`%C0JKaj{q6q6C=QQJU!vBM2V9oav6w=)EUf5C9^?nc!_gU^jmlBHujuYN9k z_b$p`k>4rV^u>16UUx$)NGw@?RBvvKy3^BvkcF& z88!jN+qAy5F|ZWnnoN1f>HRf-4Fr)BLr&*)uiXs>zaD;p5NYM2#-g|NDaCoh|JA$o zZp~dgY%C@E+XH_ov#K}%|8vV0PhIpuE5=&)7HYmFvHnJ%_M$;nmdn$+vvUXDo_m07 zd4VEwa>>{InSR&}bLtE7%HL44W>k(oL*Ud`!1A4sA`8<9Z*`7Kzst{0kX5CD2ji9v z>IH;6okEssq9v`4Zk@i^Dt6WaoXdH!_-KYYPv|R=y-erm`P zwAN(i(S60F1K6`Wsg=g&vv4Fz5bm~fcx`Elo3IS!)x&h~tky3s?xN}GQA0Y6x+>is zVZooQGza4kXtk#58G~CJby*Krd}jiNa^F1|fhtYgB!AmWi?ZD%Q2M5(AG*FTOEUie zAmo#!*ObBcLAmDM^W2Wp&3^-uISefNUq z_t)fa0}U?GQJe;0C0SXR}gqmML(Lfr>52_1(*LAYr2xdo}yP$M&r# zc3LFE?40f{X8nVKBicZgtqOCUEAjFFfR=C7Y{$82h8bqOF{hgh-b$S2h*P$C<@2m` zFHh24o`dMI(zkx679y!-9zfnjOuE`0#DB+;2N>3!OZBx@%zO8BAc%3sV8e?pkDjjw z{5RRU9ISVkMV&DxS}7}mb}&Mfpzu@w3S*`1V7mhMq@6g1=7 z4^jsR^+20d_!7&J=ItENHe|Oo!5#H=pkbLtq>OC&T2vQ@;xjshHYJ)p*ZtR@d_SSHWoVyh?FWkd5QG6jLuR5@rwq|VMZ(VYK(-R)&NI@K?i6~gU zREFXM3b7cnp;|tHdg^4LTXgZ42Kh^?U5+f-L9Zumep{oM2Z@eXt}67=cN5ZHoBtlV z|G(9hcM~=jx0N%qJ zZ2rgYxlD^YEM{<;9c@7wVLKx3^?UJ+exT%4V)8)>D8|L^*fLhliPUL?J zI(KAZe87gHToh~f{lva+gsB54(%MQFdhH|m zdugRVU-N?kK21(D>HicXK@adNfEe-0O_8g5Nc~_T*o?Rg{Q}f=&9vE#nVI=v&)xUc zkBP}m`aV7QXwKyQ!PgyKyaZ#!yk8irPo`4vf;i0NkB=bTwvr?4hhEOl*cC%;!FNu; z-=E&or2K=ZclaJOs|v;+m{Rw2hp}mdoYCSIc3d=zoj+5 zd$;;lcvjQkncEdrQsWUEtZP-uCaZH_^~^;q_tNzLfKAm$0I>*L{=z>i9P&LxB%EBG z9C1qCj7Qe4y%jl?PoZIbnnP(B>E>Vsxl9Fd>L`Nu88K2vVZ41L>o=5fUb6ud`SNMDaVF z%4g2XUMKu<6(5-QPzAq90u|Uw7M{8vtMRZWCgH5W-c3r)%CZ(7DQrBy5&G(k+%VzE z>r0dJ6l+l~%iZ%CzAmaHYaNvlsry6ZmtMFBJcq*m$7VnF#?KEB@zhW@soigulsiJ5 zvDyyfva-bUIVFWWJ3HIN<+w=ec5|FE6`;cBGtHb}uVp&Bx_ofHZ-L3Bb9!RJZ{e8Bjcc+?PcpHg574H1p|S)> z_gU9gsz5AqTWQMf&eD{YfA1=`lEtUUCv9%x_UGR9*65Df=Q=VTPyo|jh z!20nanVB}A0UTH%7dYWR($5iRa`=t9 zBYFZ6s4eOIX&>yt@b-=lE@f7i>^@uU4F>5H0q~)Jh|HewWbc*ez_~`4`X$NX_RQDy z7(Po{qKyusd#nJHY}z#DbC3g^9KEFV?^A{DQvIiL^pga5biisXA0{B0;u@q+PwK7c zRyX%Btqs0%EB0#>Q-YzV`=mYlAmjr5S&pkEI(%>hLdbihaPZTL$QwkA)jIhEYrw>d zu9=taNC;;iHtW30s}$FJTAxa_dBKVtM*P8w`tLDAo;oX*X;A;Sh_Gb>VXrbt!6W+e zU6Zcz*T)t82?!`wEUH32;v3M07bJqL>&-)}Z_c*)jf6rlD^s05#v1 z1Hcj1IDfM$DmUlw@d~jKr!O?hRceg$(}VW&7cJ)Bx?-jV^Xl$o6*HH*MlQy=VlPAl zEKnue_S@)8&ddaNbab4RykPL}7L!EG6Rb0F;*RZ60-~W~P0h4rL}Eh&f=gQ5!0Cte zze=GG2OsF-!5<#G`uWLKru^lO=jS9P9*$gwb7EePezJ?c=RKmr(@e{IP;{xUj0FA7 zCJ_9*tB{#Xezrd!Qm)tfHlsGbr!%79k~GPhBjcz|BFmHYM#`^K)?98Ln%AT*2^N8F zfXPXJh1fw@Xu|jPz9jwBXrv(-=xH6b;H&y>CK~3j!diB2rO*1^&LipV)nU_y2&v3j z^hOP(3{E|)a=)Hvk53?~tvGc>78F-*0;XrR_k-3y3=FUzd$8Djwv>vQr*k0l>ZSrZ zug^ZCXhZfXc*lRe<5Baa<1af&<~P*j)YFRBnU*Y7DRak;&qcOnMW0)$v-n27hF*xA z4Gb3_>Ftg>NMXzF`wn3K~c*dv9&3k;?oGXjb7343*sJZ-VM#nPZC-q@3Us_4s$P>={ga`~&(#BBN_%HGUwlhSepDxhbbFdHOmT#fa| z_S6&Ri~!bZt?f7-LXt1Mh9HT&HHt{6Ure(vv9{)8S&W+1JDd16YrHNsKT$6Wj*J^q zdir57G>$3+N_CqM%|3Tj(3lJkY&In;*I;tax+mUZvFG)Dx;S9HxpC2!ZG;JRoH6ei zZ^UO3bop_G2% zhq$`Vsa7u>Io#5w)bl3A{g5-9f%I&5hbcIX-Dqi2j$%zvKl?H$67}@xM|)?|?`>VG za)7MdvJ;X41>7s7p;kWK1eo|-71TC>-j)kvexN(Tc|Fm6_QHm55W;PAHIsrE2*$Rr zP}Mi?qG&(FTm1r9Xz6uu!~`}kodA~FFEo-2E5AjyHt;aq3zUis%uEag!151;MI~{~ z6wadOS$Dp%irlU^!XRRDIjKkKHX~ojnGW3@v3pw!(K|&GIh2~4gePI&!ow_ZK*FpW zWp%DejnXNgxj4u6%2I;X8*{Kpu^)NxDXe?Ph5Ju(06YAFrV+UF3bOX$D;T%pi8I6E zXC5O%UhNd|fJIgdjmS8!-iLF-fT35Y>Wr2ZS0P=k=%v>nL>G+4_|0yvEsSad{?B;j z#kK{L0sdoB7>fmrmT?aB8+pd%^RFDBqDAQZlH^-GV|S}oo-tYZy%jY(If=-Ah2DDH-oCUA z4HybJen$ed$vDt{r>nOsifgb}*R8dKbT!N*DV$XtNCYX_+SFwoPcUaws!djaJ+GHc%xGL(9oEb5?VGqj1&&T}Ob!E-gKaKsy(V*ocf_f8I+Kqb9I3*U0DtQvm$p+Q zQ};+#8)`8a^vpR#$W2{ZTFFu14m5z&p80mxa+j8?{gjLdU-L#^v`DWl@>$~)%dmK1 zk^}Q`Hx<#A(7lw-Jiaf0b*qccDVFs{hf_I|QXHPMlX zbi*IO0|jsA9iy9XN9!lCW6s1S!%xB!u9ZC6e6Xc6pp}Ttn7`2r10dcWrMhgUys`ZD z9Q-N)@-{yfSXo?^pNv|GBh++Dv!eR-jn3 zUC(}e$@?e>b-wk95x}kom%Pm(a?Y6tVm#3tH`;6GszZ5h;sAq4JarcfSvbu{sHR{QX1g=;J{d}Z+RcCC=t?MY)%0C9fF)Kjz|Df~JO3#8x!(ugX8Y=Sckgc+a z8QD2ZOxjeSm`K*P3+d-w4zt7Jj*=(Q2ygj(u$kmRJ5to{%JRxH5-)DYZns{^q8;JB zO1BA|T7Yd2aP}CQp0hjBsNicvKj@mPT^b6cq08(qP8{}jSH8W`PRGN36Vv6&KFEZ0 zJ4pbu)VN=@qRKb2fdo#?G&&3}+Fx{P5W;K;-gBzefx6l9;^4)$=Ybsz4o3vop|Iu* znFl%6q%AWSBf!3*Obfu|KNChk>wX0<_X)lpr&$zm2wu`UdYS6_!3dBC{s8;CEcuJ> z@B6DtfU)+8(n`XGf#%Af8So=33I5JzEUHzVFTOm!Oqht(G&|+YHP$vLg$K2KFcGc^ zCSd6k=MyvSHTdyFkK_4AkFCoZNruNx% z8a$EF@U@;FtgwG9PYKc}V;?ltSUbkek6Y*PdGUm$&9sq|ZH=$I{*+Y?zl++oi!Jg9=Z*hN_x8^1!$(vGynzJEi}?QFaRZ6W3jF) zBw16!Cbk7iL^jT>HM_hCfoGjfSL?jZIOo!PL3A{PDmAdgZ`Q!B_RCA!A#+L_lglMo zQ3^sd!&J!&wNs$CCZ%EsS{b5!r9@IX1y-`ss$I2W!8gcsVji`IF&$%{H33qynevrd z*dSc`RbZ4z(^JUPKS%g;NG|4wB$~c$;T=43pqJf*FlZ8&d@iqM)-f3IuFASG3*m6Q zX<=um{B=22TE0=rk#Z;^V8xDLBjhPSFy~FCo9`#f)(?q4uRjE;1{4R97K}9wlN#y` ztqBNM`ryRo&I)ab@}vZq16@y#Z*0BU&MMqd8*Av3`Fd<5dHw7`^y!5rWM{RL+?vbv zc%!ms)$be6YaWBebFBHy^Ox1FlZ7_u;2oUjpFp%nR_r%h<Q~@-KeMj6_)+-yg?8KvSZG?<`c(e4*(YC>4yT%h<1A_Ba!xVsn3KW9(5BfX0x@#F zVk7V{RlI8A%2=;kk*Od7tA2uK!D9e~_bGm}#tP_WC%$cQRn~X3bEyr|AP~*v6drL} zuzZNis1$yKu<7qe&ruy0B}9)J7#FZO4ZS^HL-vex&2@G|&JUHEj!;qQPAaPU=-w&i zcN7q$ZlxW`HGYQAJq5@N&Z?QWc1VUG0?m7xMwgkae;vVGw*Z8 zz&HOOdRd>yi>ad;=#PJcW`!6Ohse68V*edI7uAt?dOg~Rl7?d5&y zwru6AZHxJb2{02?ATgnD4Vs~FT^SGG12EwfeK|p}!53yr(JKD&ah-1ZW!DEQrcn}( zSvTc;@BEqCi2;70xM|K(26P?ILiG7+>(}qLRh#{g(AFtAP@6YTkw5tvy24`Klvh{? z2^KV%f_?#W<>>%iV-uE!1tjz2WuxXRnuUCcOnsYg0;2>)nPn#LvI{&QTisF}wzs~T zFrJU8=O#oqZ}5J^!v#2}t6!hjwm`&23Y8b1WhvDYu9Ig)WfgG|f?fN(@aD+sRL7PA zy6ibs2tMQ+aOr#S)cHx_Y5G%t0;?aGa#3Pun`vP0QF7}DsiDZH&b0{(E3e!U+v{xz zU(nh(`K7q-5hY0C^h<+83Y>~|zTMw>{G?Kqq$5IXoDn*%5_Cf5Zf?Y4_s*N8>#2S_ zPUVH?9-LUK3-EmV-Hl-<{uFZT=;^X@z?H8JwT7+9TW?FnEf2|rbB!yoy;Mh%DwTnw zLI8)lKnsvv;ia+Yw}Ov1OYgtMOdi3xPacvi=1h3Mpjvs&QU9_^1fi>3>)C;b8KUP35j5mh#BDO@>F%t$l3yeh0aITaOY?DA+Hwlewj7b)EGghx0 zb7*uLP^`*(XV)TG_UJsvAY?!%5qqW>$O|>_1CdT{{K_oft|`q6j5oCNAxn;Otd1qU zuC;yUj$lw85>g!(20JjRMu2(3g%jLyXsC{&Kji6LSWs}4W3537$D=gAL}!vHU??4I zlmE;~d?Y#GkhAy%R{Y!B>=3er?_O&u$$cRKuN%3w_aptLw`=vW@U6EBO_FfgYff3G z)f)|wy_cV$r7iNd1?!&5XW`S9%`zpAXhx1dt8JcnZ>u9%xM-Y@YdJ+fJsc8G_U2jr zT!DXz_XF^2BYZ4#_l92`MvWJeBXu+SWce1c1Ae_E2C>P5DN@!3*6&aevQE%JvMj}W z;3s{j!6brW-_}rH`dwxp$@e$5qBr{8{{S47dqcTu@fxH_EG;eHv14RFB;3sGa#%Oc z0!X}8D}zpoGH{z&=;l$AaA2!eSE?vw=)5dVFANj^i7Xd@EON3{3jv_dv1vQ6oQH;4 zxkn(^qQZ5c#UV6v^=d2HI1&O3*_(@Ip$$gk(XJ5@ZaoiPTFD@SbZF&}jCW(pX%jJ?S0TqZ|JB}E7PD6&y&l{{iyXAMWQ zOdr)+G>4{L9xB`&q*&i8dY9hSH|P*+WF8hmT}pq}&E3g4KuQ0n8S45VbC zu8FtoJ9hj}b-Hmc29vHoOMJ&h7)!k98qbWQK(#OtbK5n((Shx}PT6QhoYt1Ln#q6I zc`!t$j!N{2h70I4;%WpTg>_qD;%l~I;afhoayf;@WUCJ|-bM*7fbB^g?z(%A^=&D; z>z1+pbE;E}y&N0{6sE4AY$os~-RBDx;Q9s?;8agc2& zcJ8;CufFd5#t$LoBnWlbf&#z!xB)hww$gQryJe$NPKO^mJBtjh^y&dyhljRhGR5jt zFp#F5#p^qqI=3<}4y!8+C;TQc4#=+EJ9b}IaWdJ45f`$ar!9y{h%JU z%Ygp{Q3RM6iOh2mz`wYEE%2&#$)e~e0)Du3{5XUilYRk~h6tGEvp}#A$h0NMh(tT%yl&vC_Od(63!(SHFOYGCi&Ru)Zk<>DrLsR#fv;UKC$~ z*Fc1S*Ah}aTWz2x8q;Q5Z&;?v>Sv8Ra#VceCGW|hPN75_kjIE8KP+hhR0Z%EDMdV8 zZ#23iC2ByXrIQ6#yu}9+yem*kV$#))vGE*R*@Nf;7vQ9%8jp9AvZUCrP>MB2HO9I27-Z)cGgG|L z#rrCAv~%l`YPGa;OPVU|K)OEHO*qqldQ#)BWvx*G`M?Dp^Kg#5#s@-x8i^Cmd+ z^|UJM)%!TkBdqGEAI$r7m&QGE&bXLK15h&DDKnTXAm-@|fj}-H<+iuh`Qn{611Dt4 z82y0>QyL=E>Y|&IU9J)x3FWv(rKQ}au(G*w1N4v}7H+92qG207^$&+$avufi(E_L1 zy5{AMa3~H60LN{1j8-6|j=B`K*WqBIgC z5(6S24bm`#BA_CnbYsvl(mf2Vq*6oAAl*aGz!3Kt-}n2@`QLlaU9QCv7qH}sy??Ry zex6Pr;5V(Ogy2Sw`xS%QmyUy{Gl0eDtSsz5^jpqjAhTDqnKZ{8P*e+f?{`nh5MBWS z=I*)TbM+I>^~P5Z#(B{)V};n!Z=^Z1klQvlpInoOx{Zb`tVpQ&h@gH)?e5(Ax4y20 z*5|k{XZ0eMUMMN!C9uo~nR_QM>Q}XMVhz7i87cCD=P0gSJp@8 zB|ED$#vu`<#yN}=S%pf)25LY2Rn2yL@jw3zQsx}FQx@F^9QYG0XSuM=3+|8=?KiN{H3+0s$}aP9@x(f7-2BKAm# zw-f*rds3*bT%|oh6$Gmf^o`vY)TXr^4c^Q2Xiu#EPBJAmc1&H?WAHZEJXvMi{2Efk zQ(N615VdW^c>~U$i81hUhLrVM)PWwo$B;Q^qs)te3oy@XQ20B&!v=$-zl|;y7sb+J zsqrao1M-=~GKZYxr#5xoON`h`vo9YO3Nx8sEa?Ps161{D9Sj`tml~4yP1}R}SAhuO z?jOq4UqFK&AAa!*l55|B@atJE?RRr=*iI4viRt!=)}ZQn-u4c!TO{a?!|<`X<$E+y z>0*9A3ZolkcwY`rP9HHg| z>ceIIZz_=>)LsoyK`aW_-O*N_CA;P22>BN9#q}lW;pbm~R0&A~fj+IxgC3#C^63ZI$O4l;TV^sM`Q@L-Mtk2h>$4b z!ruxJ9~$C`oTtY7*=754k)S=X%4s5oT}KOdv9jpnAa^_48pU1 z`!%i<@?lnsqV)pO*%Vy7h>Pb(l_c_-T2Q(wmIf5WDff(M5#zy|-x?(!1UVjSArqm~ zpH_3lq*D8I>O?@B_3L?SHdgF#f~kM^4Wdib#ctg=5VJY=GZv@@41w4X#_nXj`e1bL z)Nu~?brkeCa$IKMxm1yPqk7+Ak07Ie0MYFdS4%S&SKeR+l1;kT_@h3Boz@ZYO5Oc& z2Y8K)R)8UOlFB|gtXo_QUFvQEAfchP|yLkQIHsiab=1t|xa^$d(xq<4*%h>rko`8V(MW*Bp&M+J**JsBMq9b6g@dCW_kg)9#g1 z9vALE{7qrFmunGB)rVFcCsw z0$M2sv2&}Fy0}F!En)ZH_NYpT8e?glRCZ2@6cAIKXRn zO9`;O2Y^PN{+a3Wkf@oNMonl(*zWSIk%Q8s+?4h8_X$ z=_9O>|3qwBBh>dV_ID;3tR-5kH;&;n_?)Kbl+U3#{^7tWSOM{fiYrhh0q%F}GCKXVBacN16K~>0SpM8w5ACBsttt1vQ94l`_k{~IJ>UK z!~ux$?K!(HeEG`Tm}g{it)-7h5$8ToDh)?4A6LT;w#$qfj<&nY{PtGLe2sm;;|Y^D zB_%xAZF@6>7Sn?)jeYfH-vC&z&u^a8r!(#Y&`-S>J&@8o0s8}m7$ZwxZ6BcTj)>7F zy=JW=C4pW|-FyuWTjeVNYjI7d$J_AJLm;;W_)8pw`U&SRO5*N=XQPSXfT%KF1ax9a zP5B1IG@lp!c|P)wAwPM8Gt(t(?|qb1!$)wa>X9CtK=c{yGrIy9opUGPwV)W~cg&_; zuEP&+0q=63bP19{@xuCiB0eyZutP#Z0OC^CXf#N1$XMRlS$J&WvcsQswQBjBG9=la znTQq~fa+K8)aU>V5Er+qMLvrddqL4jk!y!13$-)+#?KQyOS?ZF z^@swtez>QnwpYx*RJgh#^~Fz|H--;H1>=?C;Hnf57x0uwzJ{b=mlh;%p-cew{7cXG z!;df-{+pNN8yw}lfG3$R@oe$~@yG&QkW)?=tYR(nhj+^x8XyQNAI&hRp&yN4FJtR} zY9LXuH(hPOQ5@F=M>a}+SR!!+hmBZqGuI^q|76)peL#sb zs#6DnE@2_q1r+UF5L=^a+fcqEWt8-r!Yk{F{f?H*sHSo_O*X~Z|6cuTAV4`C0Ov50 zjV}?}@I+faBnDm&-QFGdXjoC|14Q12!#z-|-hVIossaj?*?71Umw!OMtC)+rW(kL; z=)6_z3kg2?P~Mbrn}INx7eHL!2^COesrdNViw1Ax(wVYFleEEU#Ez{rVUk)ZM5$;} zKR3PupDm=>2NTe?Wmlm%$7D%oncTG_+_nIj%LXj7-A4Lb zCkZq13)o0%R(Ci3hk7!z-uJ{7Y$QEB{ZvLG*m8EyzNmrRL&zmZaRSX{Q_cIV)}ny0 z8WL5^cqFF<7TXCR(zWt_CVCU`VG)csk=Le!K4x@oM*c}E$;QE!)3c{gP!*I3D>HR% zrnu$yr)|Y6snY(U7)3Wrb9I^&ktPdYpCk029`(O$XN3F|apB?@p6{JCd`|&RZSHN1 zMqb5<3;$rh<>8eYQB4%_0IyH-0jjvNUxKKHl$4(MYLW3mL7{+zQX60fcWx&wdMXV~ z4G{Ul^iu)OW&jjG=Bj%p8h_MRC`}y$y|7=zN{+t`F%eDe11QtM-KlLHSoWX$q^C$h zqyTFE`uh3ZNSufgWs6Q~_MoahEfUsX^52_Z8mzR`yT>p_o_=v{>DAH9egZ zxJ<7~2J3QWBZ6Jh<#?{9{#KqGUgZVh110?UV4N5gst0bo&{hPOgt95d#nzT6M5(?! zr0NG!)Q}~siL;!Mr;OxeAun2%pB``j3+3>~wjfdcWFfhval1MfjFFfoolM>F+Dcu; zxrYS9Gfm|H@7A(yk~AsQ#GNB*+_rHs@Px&{vKwOA#&`NVLDN+s!64cK?u5z%+#-Pf zu*4Su8Btg~?i}>G4Ob(WaC!jQP0XmleppzZ;IXafEfx>Ip>D)S?()7 zWo=4>9*%7Y-WwlOL9Niu5BUXcn=C}7rZ541RNMpoe(mr*{Zp$SF}&F^o1geJ=Grdl zIO+;*ml`*vHWJlhD}7{j-S|p*0_S$)Y?S;;`Q55p*DdOjRgvB?dAr}tNuX0tzuH>g(~&z}ASB^9RAy5`yHNDmnJMGa-78pOfR+9VzHyarUgoWHCY(Aq@dgNC%N z&p^nEoQ}*lZS_dAw*rk-JEJ@5>m_`1t8{xQpp7CmiEZ1ut2)^3avy9{%0C?GMG`pT zR839j*_BT%(Lu@-J==ZR4JmmC`tPMSYIg3{9`{ixXuRAU2f7yxH3$hEtk}v_PQE3L zA1I?)aV;X2U1eSI^pldqI8@K)FG8O3q8VQ%GU6lUJiWcUx%}67?fc){VFQOb=QHy& z`w)=Sf?YI`hDP_0)#^1Y5D89pap%_s&xqZ)S8db_w&f{_V)(Zrpx`k`&DEct5V}Q! z4!W1*d)PvNp#h@-gjdz?Q+qIcBP$GcdV*_G8;8)MM8HBy0dM5D3yTHY`8->YsaQbW z1aNA)f2(l{9wy@6zy=8-Qk>9;1?@!lMQw){t^a{nB`?UXau~&2QvyQ7UR%!ULy$7H zRzZ{<0Z~?7PVW9a1h<0zJ;V{O#_qHxicOC+@jvj&k4kd1emfj`D{6eT!O0v9HSXURbWn?ymPh_kV*FcrBK#hZrd;9 z&xwl|GdiE0U^lWhK7w-McHf;DSe>!Mbz6@>8u<0oCzJk~%!A^2 z|9GEeEZE`3Nzh+2mu#Ey4$)d9ejVd|DJpoAM}JPh{aosK-*m|$d{aSsd^nWa_||^{v3m|}y0u>hopjGX`m@o!miQ|A zgCpz1xMk8Dv4cSO{n){Nd9$4UjEv>uuUp_LHs_{YFXO}P>^|d={sUu~hZk1fEqhCw z-R>qqbIR$^SDs$pX2QsBds#Zmk_|g5p?Pl$X{Aac<#RbqR^j_~yf9){2 zKNvClbG9gGs@!^@CRx1dx}TvP9e&*Z^Id3q0S&?a7o3PveSoGOrYJn-~UA&_9h0b~&EWk|_tHka3})-flt$A><-| zy_SKg^$iTzhLZzSu0R1N+@yzVb~2NoC3m&`#sR}SM2MNGT?FOB<8OiRJ^<9C#mZ3b zgBaMt(b;cSyGcv(?QHbSVw_(C>Dhr8{^EyBj|YGhEtiFaT7M*1bJY4;`oZmLa4DyC z{4#`}Q9KwX+2?rP4dZOoQPIxxtu#A;E6&5`-Vlas3S4101D=?lT|^7Mbw;yAI` zyfOSrMZd>|ilw8o*1r!jBXlpR&JqJiu8pT>aumJ%!>4A&MTUg{NI#`0FINIxp9K4# z^XT|_p`gV47*Wt9fu_n~cY7)UDnkMwaC5%srzGf!jT*$EI-P@kQ%&x1Zqt-vH+~$h zMGlK}d?ze6)pG@7qCE{#NYxc^lDXR)en5hhffqOJ5*cVMhX9gXo$Wwn0LZDh@MXDA zfQ#$vBXMb8ZGYw#2R+gT(0Vp1AQ3k6N8?*=uE0+lU2js-oSiImP75UrrCh0UvdOr) zUD)N$+T`L>_B=3W8J2Ywm*I6{myA~SIztkRv=ZWq4OJ5@dZH;-`npZFb<;F{IC#_Y ztVpR6^fcWq)A+o0@Nccw+j<3$kshZAkWK1U^c@4ZEc&e*f977tHceC6s_;Alxb4x(NUbx_97xb|-ftY-14*Dt3a*xX zy8|AMVs)M>7+%1#*)QIpY6s#2>RmU4GWrhz`5tp?gQ+iFv0bt7&t60FJP@O|%pBT6 zHVxy-ZO;QMu^>ot>g^yKfR~#qruwajjYq9b8_U96&1S;o=)m|_=D0dN^Zwh8uS;cR zCL<;PyXK%+1qZz5oA4jll|+WqmhaCk_Tfgwap92U?w)jcZ?Ud_kr%w4`q3 zMzUBljFUA0Zv2%CsAWSH>)`&AGi$DlWx5=u!N!ET%bMKRNI)~7`B(h*j>wtb(ov8! znSR%kdh2M#{qyT4{N$g&aQ@Uc_8}{hMC|OO8{SZA3|pGYZSt3Xi$zGNXSnhTgY!Vl#deYFa)G68LWd6 z_k28c4tM$$hk;nE+%o(zEG@QgFi7)&Q7k}P8!6Ki(3j!#x19bsBwEhK?1B9`(H!{h;#=Z?)x2&pu$WI>GS$RESh{e^<;6993sufk10 zQYAuh8FNch+dIHGvmyxe6<@+e>Ps|w0&IqGIeN|de_Lo^n&fC%fP&xc5bmQpa&mG5 zBNaK0(aU&yP)fNObn0JwygxBt2t6)n*d5Ra*$iQKuDPo%w|s4RZH*&s5%zNKqgt1- zZ93m3Y*F6fT#@ot+of#R(veC4B;{lz@JHZh)SCwE4R7ZvA3svs{uupZ?q?a5EUx-qS1TLr%4fg$Hs_Tuq?0Xp2K5j11%T7b&WV_(b^-bGW_-? z#1T}4QP4=J&?c2=041$5l?xDbv;I|vy|qK1`^eouT**P*D%dCJ3)_!$~} zC<;=n-tfHIHGR^*aUzo}@9sk2i)JM%Qn1D3KHEBLQp8V!%(Js#<34XRX~EsuNYF3K zi-6iO9IbSiw+7k~=E2CvFzO^{rbR^zVY^$j8)VF@(9=WIJc6*XQ-N~b^hWG(`ht@^ zPsWYEvmCH$sH?VOTJb{9?m2p7#0Y)`u_IE0A+4SJq@ixj1;gG=R`mB$J!m@FgTFl5 z>SU<$1eJefHoa_CEHz(Vvw>(>%=OFTgIy#DL$x`4APk)Y1)DF>5Hn9hvk?4mNkL8{ zqH*iL!vgq^X@`b5LFz}+9XZxdF5%xrXh`;y=HN>y6~yMfF6A$ zsunu@VWs}V&bQc@MJZyBEs3a8;a_n?x+)h0j1g$Hvz#>eOO*@pv!)CaT-^n0mrPJJ z^zm2?EHCc!GAzf0|DMm~qW{bSz@#<=U9AFpcVXpZVplE~)WXPgKSm}ppt=YpAK{ua ztqq{Q<=Z3(A==J$%ltZC^z-!GN)4A+8V?pyRn^i)z`Xu0ouEVg-@c~@nWxLO8^p>z zA?U&TG}jiIKYJ404%y152mQBfdXkz|qm@r;G}Glgv6V``^YM^5Y%4<|^kh5HdMDF& zfprXE)vg7;z1JWXm4Qp(0Ptn66j5F91tNb?c+FM^yESOf`t21rkbe<-=xg2|yjh7d z7b5-$)l)G5lMi>7zbHp1u?biln!W-(p@(mTQ96JGI*kO*e7i~c%u{lrHR||_fI1Vd z(_QUZhVSTW(XQ?3t_zU|j(j#t`~`g;W}k9`;ZvM{rPaa-rhti<%C9KwT(*+YA%mlUPsgqy&Ai&otzGjPt3HC=E&MHy}hBc<7mTYbp=<}*4^>(fFcWMXp-dDBfFIH3Mu=l!7f80`G^b}M_R*Up#RbCkMe8dLezBt=< z0A@zu`9Xoey}e7BhHswPzgNk@)os8sR2r3yJ+1PUgBtfcw|DCd%Zl*~R;Me0I$Sau zOg)OIC;Vs6X7~kWGj5yldKg06v}#+}k) zG&Z=e&*e_+k_+Hm_SAMaZ~6vrk97E*ShQNxUA$WBr9WDu+!jdz9Unukb-WLQEB#dQgc)mP%~Dz zD{ev&3hM4;-RADSQf00)Ey$Ob-$IeQwWI*Y7w3xy{0)Pbz&;-&gZtMn+FcXScSAUE z;kUy#f79Ij7p;qzuueYAb0s7IfK>po?c7_r6u7z0s&dW(qgW$qM=Yf{%x^Q$m1nYU zZ*M(Q;KB0x>@@X(jpCXETP2W#wMo6y-T8E>HUtNXs+shAQ$ip z;G$ZK6^N!I;eg0&RX&nH?(Pou=F3Vw8c6Tufo!5z7w6egWL!R1RNnW=Dq7L#sfkH< zLa47>3yr`J90J$)SdX*Od-S_y_M$SFETFzuVpi|NV}}I7Jh2aS(q|S)&GGckMDfH62M2c}OBez?%sT+7b71l5) z8Vld%Ef6O)fIlxS(cAkB3Bk>SkSfYqpL6^yu#Tdk&y>wjLW)MU& zxJV0JA2hRLDX~$+K|%mylA3AfWQ+73?r-98Nw505F{z}8L;tInNQbEbJQ(1@R^p@p zPY#N@C6ybz+!#2qC~eKuc5dH@ePwTat0;QKzgp@&9Q6?3*G9io-(l*bp#5p`p7(!N zPL+eccCdtKAg^`sIFqPayw;1ev@wvhzr)kPclbW*x6I+6*Y8?_$!D9ACr^TD=Kj4?78t{Lx`ldnW6j!EUrlmL35 zXg(qNMKTHW^L?*pl(4BCK2IGhBuWuV6JgMAbp+*lqyDq=GAxM^iKc(^R)GOaLblj=lk_sr@~+dDDe zmz_a62M7H@=bDb@MR&qGqPZC&BO|Yv1n%jWwqH1Hlw*{38Yc%h=f!jt=pt+e<#NV9 z*t~`vDPdVO@{(@X@7SBH8Sprgi zBYpc4#YKU)~=OyyS-<$K+5ft1Q z-y%b(yjyFd!`CfptYmZq!~GE#nesC_eWI=#s4L}QT8qvgs2l&-hZaKMA{Xm(awyL~ zZ8vn!U)9hf>F>idhgkmpqIwVqGMVPnlQG-O-qU8G5^TgPxy63v)1#Ifjn(Ktc{*+- z4(Eog2P{(d1_+Lmad6)(54PvO1LN-UKwC#RcOdmrE2z4l-#`$YKbkUKy&Nn}PEUuY zG(5L`(p>xPrWi&@Uo~BN=@b78Cum{c+^C+paj$qdUQ*2~+5(BneVe8TKSNw}sM6gV zcS$H=t-i^i*62_#Tc)A*!hUhxwIKXVy-4V6)W$t)Ero#XT`Ek$GA?nfd8g;rBiUwM z5_|Jc(n#j|RjTdwqagyTLIySmojam09_lT3>O+Z`c}=hsaOoN3Mt)PNDO#;vC$F|CaYK{CXVCbU5cg(VLv2QJ{RWnk;j&CGPAC)oa zeUTyV(Ev9wE;0Q7#-%G}d2uk!fSX>dSk=@mj%{{V!ztr(o949J?p^OUK!6I~JDC$u zezjgTQLCKxlIpzz@xXzG(MGOK3-UA!n2E^=>Z z;KBUS33$(tLK)x8upI-WoKEGF3zX+Lu6srN@!eTy22!$dT-Y5^d-qAV&gSZ-W2fbn z7F7y5Zx*c7W%W<=%9uZb-1#+Ij}CJ-Tef`= zTPKK2Z1POaThz`N@XhvtUs#+p%DdaEF+hiZj}TuHio>ulX7%k=oI!jond*EoY!1Wm zln4%SGrX;Rlj)%S)M(opnb_$+L<_P@WsCTT4sk0Zf75)*G2o6n_^fv23HhU;(Y%uU z&~PTDfvk|I_Eh%bA`;0>#0M4=mFtF&C!x+%# z*wW4n)igW{dYc*2UUmtpc7h28G%}N}WVHpZdN$nhK2bk9Zbn--zAae1MG4zG%;u$P z=?Hls^GM4Y3AqsgzfZFN?50XuUDrYuIIlXcYicAbjih9VK!dpkJHucQ2Q$8AjD0tv zgvnO~fX5BKV}HkETT)~!!r_h#@PTtjAh+p~|jX5D~hOeDfsO! zHJGWs@wNC4c;TrBKL60_oq#-$u~dc>%MC`kQQ#pNJI_!fX2e zoH&L(y*$tVauo}z-fWB`4x|!T#(YXXj+PE@GqyS@^BI5*m3J1UD}hgzgEyP1VC0wB zH(#wf`)S_TsEj>5`yewYZFjWH_JppoQ>TmDWM@cn331AWO{|^{(O_R_1*HJAO&Ln0 z`}hu#finJg{fbQ_=WdtrX707;=C*p{VwDNWL%=+Ir|!J?d4a^da25u6Fc!;*geqNOLRV#kKig7gsPu=$ASXGsb^^xTnChsCL$pq4OmL*iObXcv{u-ap_VHOs1f* z{|b%0y?s`8cJ}wpsl4LiP%tO~b}@ln%03R~I<^l2@LA}oUuS_HHAB;Om(YO%P;EJO z*C(oPD)STD&RkdWbJe5c`A?R0f?hw011Wje0(ke1=r}h&t=}p1hT>91q+6nJSVY4X=KOO%_ z2GQnqxY=pB;p58$96;(EVOU}hKWPz(`U)ZLAf=I~t0-c%b!W&X%j2Tc?vfF`RsWOg zUg;66SkFGbWQn&VXu8_CBc244<(H1m?HcEPUe7wr>H)&1(r11$aPPh<57Uw%;%<$p zLjoRlWZVKUKj{Am^QF##AK4FSA>P8Ko6I-(xMMdZZjs_oR-vb_;emUF4^)%^tak;i zP)yqhK&!uZi_8mwp4^iIG5`HF7{V&*HsyB?#wy1#Qh_vAQp^nmiyv--zR;++%LNUp zIbDGUxma!8w9uPMWt-*}wiY89NgCj$igugrXa?di1@xc`^3B@w9jG+A{3W_Dqnibr z?Wr+%0dQcyh&s{us*5Po*Cgo2^S;V?q(eJ&WQe| z52Fk=@;XY3?&_U1-`h1 zgNP%Qvtj$ini3XzaqJy&%MyoVpZ}Mj=>_GTmv$|J1p8%*OOpkQsuXK!Y3Z7o0Zq)6 zYENi8OcZPPOTdjt4Btfv}k*$63HP_#g-I^PQ690l(o9Ou{$1UT2Bs@IGIY z(G@-Ly}251pqMC*DMX3V-)6y9&hl{x)fxIX3b%V6TLSDU=Mt_L?3V%Z!iNG=X<;Im z=+htCBXcM=K;pb{A|V$Rr{b8IW~OtwnrQ1hr)ck+Nv#Ja!FtIKW<7@$G4DbX7poha zm_&WKqLh98l$Kko7h0+&oXPNxxL;rUn|D{aUgw_f1_c8Jah-npS;qu-;4aPD?t1yo z0lNq%C1m7uw8qiy0WG#Ys5c5-f4c@AY|l|x6m8$9F&68IWUhShgSmB1^v_Y$ABcgH zuH?_~n>X&Hua=M^3Woh>F{#%ZfAOIYY($ehqEc}jlrZ6QY$fUiAPGD+9!P!|aNg>z zxN%T{()CU^GR-QSbA`4RsgL8eRc;Vfhm}i447or32CRH``~L1a_}v>jhK@BWdI0jd zA}>zlBcIs9iM&wA^5cr{2WZsU_)pRKI9B5`Fb3r#SD(Y>+_P7>gO0Yuz}pD2hf7~) zz*UulEo(qYO=oam_6a*ke56E$`bndor_!l+6T$3-i%VjOj|na5tQrPPlHewe`g&6B z%&D+9r0+T#DZ@0*$bMBV>J1twU_2J368#F7T;t6RM*-WJ0WkL!kZV4TO$QR3mIkuV zI+piJtNNZl_u$FK=WwnY>S_c~xc&W$|eNKH#OjKp8~&q+}9I3)cDX) zPQnm${Vhgi~JbxxWxb;{UVg2tc3r? z*}XkHJWOJqDBA{Dra~*|xIti;W+B9zpQ4XqD3Xc{79DSNTAX2t4;A2Q;8;1tUDv?z z!}iA(f92Y6G*oPe9?Zq}>VNj`|Ff7q>|h>_@Z{7LrC3&qF*(M~UEmZt?(&#CY7X~Z z^6rE?ZaN{pM|w4Ep@NXw&|_Ly0BZY`ZI%Ki+W10DQcIyy*T_+!{!r@4Nskb=*JX`V z;M@t~p-6;)PSBZ>l)xZkIWmOxC)w02wod*B6B&fZ``FZp0Zy*lxDE>;HvhPnx1 z3L3!qa^s>y0Hco9^XeM{sIX0UTy%gE*fcYhIEh4<5o6v11bC{&i3=2>4cL~W`s}fC zTjt`mi0@lq?#Rx8s`-VhrVZ#oLNhoWXY_M4Uwyt8p707mO$~Z(nu3YBU)=P!50DSP z(0&C3;A-->WP^TdI=nichS?P|M6+4gEE!ysg+TAt%lAA+7~ty0_5(eV72hx*@nmop zCX{rsPfw4ZyV?=?&{p7CMG80-&xohOkTNICK{#W;@(fyysk$|0FfdtWx2se@rPW&b zh6Gz??7l39)W>$#20yMXkMv)45$SgCRMfDXJ80T!RNqVp3ZhHYIZ^CMMw4QK+A*y$4+4=mKQ4>m7`K&u7=uF672@EU# zOPK#S0}ZT(5ml`g*&5DeOgBtszsKkrHOi=~7`=6NYjoS;frvC{!E>cM9xBxICx3{I0Z7xu?&xry0Az1$w>4-z`{dIp&BhaH9|z?bICELkAT zfxzCS^?X(f2sh)t+brl4=nbixzW*i~NKdQp9}^+?5DK+uX zODvTkg_NiGo@DH{kjta;o5n-B!LY{QO;HgJ^4!cJWtV7XI;MCY9$5F@qDm8j@c!Eu zATV9I{kZmCd?8}?bSy)pgy%%G>1fA#q=6-fTGTahaZ z{Y04e-(#@M^@|nT8`TfKX*#ihz*^M@YH5|;)>DA5HZnzX5dq$?L!!YsBEX~8=@Ob> zf4nJS3e*;*{!^f@2ZI4=tw6G;Tt&e}i#j6g*MONF%;VUfp*WNGHR?4`Kq{$|+6`SC z9C}aQo|zdhC@6phn(}!A;Y5UaGWPYX#lW{m2XX%rYE#+wLKNHqyT3yXy|<9oiDn5< znwps92>J@0qCOM9596m`K2jR=AW_ad1^iF*uPhM$=8IlWkJ|!dh?9m8tCr`(=H&kV z=QvW~4F@B<&#&tB3$kGOvKPse7_%~LS+IM8&sE0U+QAor4`vp>s!e%;A||yJt74}7 z{oEy<5gik^`D`Ps)N=m`)hUPh1oj+ zm0FG5E9VYLS>bZMzef^!2=!fe!k>Jk4>k1PLT31{QM#P$O{s9fwE)$O`TTtLy|_S2 ztu6f<7q4Z5m*0F6NnzrUix^0$XM^pRGg46$7@?K7mv-ZO-9tI2YX*#VoQ$_fB}JjL z@xhY|TWadFJtX#PwUl`6RGG^o&eu;{jbJVj{S9tZ!}Ud6wYGbuwl_aaT(kD4!{3;n z31PYnLc;5}B?fmvNI=RMGvq5>Cmi!6IqD2d$Vu6?Z~?%W1_h$FHbK_JNDJ}%9N78Y z2Nh0$wiY9v)Pum#3FI-xGDa2v8#Dh0Y;2Wnp^Vz2)!mdRBgd~8X1RmZX)6YXN+l@= z1qSk=H>T?EMRO@yfSHe>00XpS_-`biE>G4Tl7fFh^-_373cskd>Z*OlLVTr&=vA`^Wz7y z7kJ1J3ngDRZf8qzxDK(%VtGI9!k?9y6^!8@-3^Fj0(PFw0w?bMy<-bU31Nb|yVC zHNuF@hy)?PAWxTL8hCN79w$S)ULehaHO!BVvKo>Vp@5aj2zNt+6Br#Vw)dK3R??Oy z9~&hvA8~XWu`M}#2{Ju3yuTo?EukOG=3hcWl;x^Xp`k^z6CdZfS~=PI8wZ3{&8F!XuSKrlu#OFvipSGERo zwjTB2v|A_V_h;?2Ps(lRiJakJ_-*OU*FS-|Ae-j9kc^FVNeZ~XFAXdX2>t2Ad8CAUI=$>d1-066mjaE}*C&gE^mH~;f7QN#R$J+NkKuvXwn(BR zqW?GgLvj-2?P^|bF5Ue~#wzLlTXRVwODC($Sf~xOK}J6YJ6d^>1WH1c>838#^Yd=i z-PR?y>eixE=GK^5Ld)@vh@im^@{4lw&#NWfn?cwG7b0V)!YS_3rTLA(ckHN7lqM7y zi8@T^E!4cx(XnbPXE0_Xz1rhQoR0!OeUiE}v&+rg?@7yeft?P2vE|$KWVX&*NQg)G zraj`xIrJ8u{z*=4?HE29$+9bKl*Ak_G}2nB`1ihNNM#=-Gk5YUP4F`sq`h)D1UGlc z`0?#^K+k!hv+az~#|eNJXfVF>&Q!kMSy){pWk2x7mlYc?$fT2U4x8Q4m*hsTtL*%G z=*N^?fN_fR&e!)9B-+;)b+sBw&yyj(dF0CjJ7iZxf2#kxClMChofHNPqx#R`ex4i9 zR?X|bz0`GAuLxg!`^Y!D!E7ZU47eRJzR~w4!m7JJEJO=ff4ZLmeuV5h>tLP{AlJ*9 znF;m=!^bZT1>{u3c@IwySoKz)3yoauJ~gwmLktA4@6Z$XsM^48TIL^Chq~YReEZES z|Il{21F!4N!Eo8DuiJ_APD2PI@4960!r8=x|LvoouwxV)Gt-&TooH77=jo^m_h}IF zZD`Q`4!WhdT2%1$OYz3}{7Tm$8hy7*8OFCL5GV&xS?^t8*9p$vgNpFYy30}Wbu81J zj7(_%ptEZOcAb+!HV3EM2iU~mk7sFWBOhmt`hOU=K_))tS6m*F1sCy)ci>%US`ZaY z&W=}6eF;=2PjQ#$MU5E##`s-F&4LeIR7X)a1mQ`Q8j*^LXNHLu)HwZof4wY!EICZi za%@K|2bFoi3D{CsU3(Z8=!%d+}1sy1pt4I z`#uYnKDSUP8ypUQQuL!Y#+B6-Bg~1K;ojy>5uCg6kx+1u8e0kcw7UHgboi7!<{&Q4 z$%xZWCTE!c_^BEHjB(jeR@{I+PgXe6e8NwY%u~<&u803l(T~-~9F$ly#5QBzNppPY z#-!(js_Otv_~m^qnLa|h%S=M828K#)FHCI%QKaj*-W~KRC5AJy6{OG=0Qujp0FEDXckXMKNOMEa6Y$uMShn6sUMGWQ9qD+9?< z0XWlEyf?5_I*Mx$io~c_h!V-?C|E2N<7WG`qGL3XJ#AY^ai$iB4L+QWXz^E|jb9Mv2Y%|VUcEcfQss@@um za1N{8G0F1o-$ZsT6rLSB-I{K~uoE&9(aOG~madD!UoD2L-7zxNiXU|d=9|v0-sf&i z`@=!-DvLz-_+ETq=4W#9%T8H|!;f=_u>-nCP#K}xsJX_#9Aua3WC5&8!EfEt2En31 zn+bvo=4?_7STg)G`1$RB5sTH(qcO$qJPVQ7jp>Er+>3Y63G<>z@m2x9`gHVy_!Itj z`ZPnM8f2K&7J|-70OtF+xR@kvRvp6#GuyiFwGMXX9wM7x_CE+V!?Asq9mF&uQ;BAH z4v4IfKA%76K(Oi0HHZenUtDQoNnNI;5Rj(_$Kz7~Vrjo*vYyq!ONKC!dg-JA^yLMz z5Qk&Udc`3E9VKjV!?=E5zn@7Vq1PiP3;aXxuDc>L_+rItMx|{}80EYR!!pP^RTiCR zPO}AC5Tc5=&4x|K#FbGZf1{S>V{I|rznKMfr-7xM97HYOp@yd%@aI3k9=b7+M_A*o z=;;H6ar6{Ajivt3_(z*pRq4SK93cSrO6GK%>*yf9so8;j`e!bKY!TIwTXW5on31#V zf%uWh7V*uldS!(iTCRp2X5Bh_EXr$~q=bcFCF^VF<6mUNW^ZH1MqB{pJRos6s_H&C7Q7Q#45Wt~?yMuhDk<-n zXe$$qT!#q1Dom;4f*NfX0J)mVD0gwp#0>Asu}}W#>ymMkr#j0@uMsSome7Wo*~3xbOuV((RWGg`+%!K_lNL8h4Rb$KR%7fP4$ATdM2YAb3%jhiysRl z^6^P)=GR^t+`>e#75Qu!zBn9K@xd|(g>o8Rs@-hz%+IuOuO-{9!(%UKJi;G_+y^RItuPQ2sqP+#G8sRnZS8%#I~ za%HYN7>*76{=`f4qLiT=M#$qyC<*scWSd{cr;79+?{bCp_$^v zeY6#Xx;*fl{i-%R^d4d3>{pRSdXb||Zsdo;aOSdrv=CU3@9b4E%Pz&x3UBiR`n9^9 zsLdCv=L>;F;uQua<6=bdBFK(*hFFvFU z9kYAo@Ux7RdeXvTROYFue!n9_h8vxMH^HVN(!rkl)~4Hw32YTVGq?6*;B55{<^FF` z(GMvDNRa1ZnBY<&Den2Kj2*JtZQtrwp-W^5K}?D2T5(_%URe)Ii+DPArTb2AEwKAE zUqTrtUi+(ljcaP(QV#**1Y-sRQ}>4I7hn>GUThavshA%2i->lfGAY|t^|70Rel7`O zSn@??m@v#$nV)tmV^Yn$5=M5wD1VFJv;ECw^Fd*`Mh$dCZS70hy{OHzxc++blgjz4 zqgz4~AwA&^(37*1O-nkiUB6l_ngS4d1_%#;dLF9|RO?JJJCY#s-g5t7-dR2QMZcqa z7uVQkA|jLpR^1$)W1ITVHHjgGg%3x)`drHmX=zDV{?3EbXM6C0g~WZKM^I}azzA6l z5Zhrw&lKs`EX1N=-x0EJs9!xY4^2XI9x)FojF5R`ssgP`OCEl|1> z3Qu>)lcX^9`J8Onn{O-43`)Ry3oZOsSBlg#-vx?avsrksl98l}to9rpZ@6B(?(kU{ z7xKBTKuw9MsUw<&Us~dV1jrr_Ruph@w*`MNJBgX~R$qP@vlju-7{nK#G5fxFL!`I! ztZCAqWUgk$$GxjNu7yJ1T?|~|OS1Fc3g(Yuh`09x_22WIBS~8Nf`I(2|GBog*|*;_z2Bej z=bUq$b6vl`Tvr+8SzgP1->>_D>6R(b?Ug`xFqIfp7Gb_RHncCR=Nl%%8J{Dg+{Vk)3if` zVLMajSmr)!QoArty$jA`qQq&YU5f9OB~vL{ zcws3=44Xg6wmTy5+)LvSQ27)xDV1?}m-tr?)<|cori6&a%5>28=90s4Yf%Nc>PbtV z_5Uvm%K)^SE&A1#HQuJI^TWl|gI7H8K0V=yNtPFlKRoll?*(0BeQaC4r~AOF%693- zB9=fmT6embLLNv;G*n7c^@#{ASl<3UQ^tbP zDgNNwPF`@2)iBpdQZf&#(Yy5x5i7f&er*ZdbG}H&cVFLuP)!E#LWYep;+|U}F{sv8 zOQ~qah|Gi7B>w!H`TCh7PCyDl?r<=bW^8iqt?mzgx+o<%NbvC!zY=RAq{}P3niS>n?}Jd7E&Qq#hWMb1;<+i!7^^J@>zbhDQj}ic04F=L-5~Wt1DDMuS6XPomHsU6I+?D>LiGZrVJN zGmQ-@n+W%wn>0IOGv&__4GSoZhD^jt8XVV`B8cAmarO73C4DIAW7nldLWM^4l!vo9 zc5=BdEAtRTz_&$7&iRZ;Z^q9u)gtN%a)D-sV{n_nYQqs#~MJ1v*`C$lf-?iXEN z4k->v4DwB(e{K|#)=<9?l-J@8tKCeU?P?m0^eO*%VcfC#yl60ipQv0FVDEY3nB_ z&8?vgfV0)km~w)MYNJD z%z12X^K_S98h(~N2B7@)?BrVU8eOCB22Jv<&Ay#I#JC;R4%URphbqIKOIB08^aFtM z9!ccyP6)AEGv->$UwzVIU6y&qKn#>3=khQ=Js^!)#vok`Ht7SnIh(C~Mut73P5N3bdyuGGhhx=>aeK$15=RHqcQ?yL zvuz-0`th2~eiSlu9TY@#=>5}LjZZhC)o6BZG3=t2Zta^)-`GxL?TkvNFhg!SumWMJ zSqgX4J(sJmDjgZt7;+4LB^cRw2%E9W1?Y8AzM}{p@@GnD?^r97atLT8Bi1e*89_aZ! zY6S9iGXyTFK@n%}0l~grAy4p0-hU+gYsO#YL27C>QcrjbQUW-`w*_i3U?!rH+3`Zy zO1S44k(~>9qh-?DgqcoT;l7e&oOG1M9xi zwRtQ2`=`GW$iNPgpqd3oJI z-5r--p#!uJ9*A##2he#&h!FVVaiI5kCE6JlSe#Z1Nt(S9VbZQHwV08kAA4uC1bz*4 zSz4sD{9B@uVCIJQH&bfn$=0&Rr2&uNyEDBPCHY8SEz^Srn?m}1qjVR{;N`% z&CHNg~(R>2HY{t!Ys} zSnqe!Zuv#bv86G}I_20i%Cer_{js531t`%>p51&;mI7sCB?!8@x0@ljq41p6Z*RA< z+?{t3tM*~lp^wg;*Q#b;2C0lu3^kfAplcopMO6A2@_;L&g&fPb+WsvDZb9wU+?tE3 zbfUX6sfv^qT_GNP!-1@sAaOGrSHourEf_v?0;wr@v`jPCZfLqL)I%Hw-{xT@1{iIp zhRW=y=cgTFnj4gu^1l_Hxvp#yWIcqV@I^dD8fFV9->7CkKhXz{Q2vFc*+l(++HkkFbsbr&$ekjRSnuDV3Yk~VoRA77$TX`F`8jF(dq*12F$xAtrIjF5!`4cMb3@*bfT z+H{I2rP#ZkD^u>7ufOcjb5o^hH^lXy%|)4QF1_BoU7u9i?K28EKZc^b6>Cr*R%0v# z^Re9y8BZ-plt?j8P?ad`1H2)ahyQNP?Ckx<#bnJ#us@)$|7x}J3v3iRUAQu`*yW+S zF_QHsC=2R`o>B+8r@^cfbwG-CL}f`w`=39}wuXc=azlP%D=b|7yFwIHgECKo+mc^o zF$TI@Z7HH_j@=n@3wZ;+O&X{_C}-HwOA8j$!OkjgKfm)<{i(G)l(fe?KMF2akoIgX z1SAwz31X+nPNdf}dkEdVNai(Byba#e*>*hE6l(o(cZI9?s?{s@j3whEr|a{k_j63w zf5GRLcBB38gQ1YL)*?Khfev^wS}j0g>Bk40UQ*99tE4NYd#x^J#~pQlplwzBlPGGh z=|{M=4$!L#hL^0Y1Tcx)S`e&O8ZWj**i4i%{?=t7My2vTJ@PEs8~zPbnzDvALGY@Bw`ELz=T7z~&BYGg~c2 zAc#18z~POW)R=Sm-tDuOyxzCj$dl8O*nC=fDB-7}8<7URjM6#*g}EI8RYOh7U*B4i0PXp6tBvQ*nCoY&EUjNN;@u zL;rD|XUqYB_^U5^&)u;Ep~a!^&%T!m6BT=Is5Xxwfw$a|VDT|B!uzBb4HK2I3Q$BY_rdx4FiVMiZlzxi-=Z#PafZDCZ$z2;~7uEW|Czaw|CqP{Z1*i$EFOy9 zjr>vfN4-=iI(gLgT}}W?@+f9-PB+%2wg~&u5Rb&u%xu{43UWDp{{V%<;Ujx%Jl*GQ z=nS@5%|ORe1OV^#pC|o;la$$sF1Xm9pdC$dZreA42u6iGrzPfJh-8iqSiSoyp? z`i~a?P`=Oe_@W2uHHYlD)|SgA^zHiht!{0eqcH&DLr7+e$*t|jub`mvwpU;^?mfBL zrvvjGng0+HR$7$ya=}cpyjOr=SQO$WGYeIz;lOS!(bzDNAc3U=2QT{7LU6)-0MvaP z?Vd}nyAY#VwZ^<;kpl2paHb4|^SPd?HhXddUOd72Og+`JRd^o>u}Ja3jwh^qL=OZH zu!Su&Y%md_jr)PiH6^e;eC_-qCV=CG!>pD~2`rvw{)8Po1H{t%+X2kE0MK-Jv83D1$KIHvc$ai_4rzlqj+2N#po1sJbQtuMz zNst$AGV>3zp3~n6gRWZau$ZN2Irnw(yt01OV&)YqPNHo^GQg*|<-2movj`?qli0P) zmvS%vNv-QEpvWj5FB{}oi_d=-2zb+P`+SV_c>!?`z^5{v|JBLvWDYQowQ-ZLhp3X1 zu-_+tXEqt4)wiVDf$csJ{8zXiky;!Da7WGk;!m3w!o5ic)%$z5s{;lHZYY;#>XCaJwR6;w$vP#+Sij<3ArYeMZp z=c;^s;8l87Q1)!6!Ve_A4=(Qi<3U-_YURxYO{XwMNk+^V^Zpi1lcROn z=RT7og6jnO4J*mZbO!c4K+@bI^(;{0Q}v5iuADiBYK?UYCG4HKuoZPHvNTBkRzB$S z3{hNOV*`|IlsE~+%&cnw+osaITlL58n=3^zaS?uTd^tU2^*;iHp}zuz?x&tt;x8rK z0(TUM=P5^WlIjuu2Hg9~BC612ctVLRfXy=+Rh!N9|9y4o(CC51K z0QbJZ7DjU387V^ zKSx!FXm#9(c3@8muF#(xf_okF*^q}$LvOuxHz4D$w*~yBSi$jW0LR51$Seopu6Cf2 z%zt8My4peqM^3N*t(jwo8}RgitCc2Gq4@P{)urUI1KAL>I+Oqn~oNyZ5Mka4ay`=g%CGHV!p&E_~C z7lpBj2KCp<*>j3d4%5L**n7#994TV*YY!8G8D8moLgm19KKBqY(j7(mq2j7LTe8DX%=@Cx{;Hw_C-W%l5{5vbHe%90 zh6OjOWJs1+Ym?Z23$I=Znaq1hJsfu15LUxYg6hn$xJMjHe=S?Q2xRI0G5&A)uV5H< zBK_g(>2@z0XPIR4%R*k+U`T2=-SCcRo^1If&@Q=98J){%?Sxm&LArzY@GWQH8yRN= zOJ)~LtM7eX_tUQ3x_(mcljNt05vsNNsqfQvUB3s*6q*Ci38D*{Ssc8W-#>>=ETwV$ zZA3|a0=glqcrSL<{uG$edNT!J;!?dcZ9RcAHM;y5Vi(aLk~Q;HR)h)1gpMxc1%(CyD=RWOR=5l&SBy| z=^0Ng*;VrTM0ODS!qql_TWrTXybc0_^=OtBNFT3%4+IUlMw5bPyVxC)T~xrh>wE+6 z54k+dfD*vdCSfs1aL>depX8}zBm z?l1Fbt^kO96L@v~6=(5At$z@0WIu03XWlL@qh?uMQsrsE#cT;>)%rWktRF%u zE)*9Iu~E6Ad6p6jv2J*l+pge;p*5n}xrtGC^P+!jh`)$%pXaDx6eC6?n?>-&K61DfUwX~(JU6AJIyEopfyrYJ7yH`j3wV%B`fjyS&Ru9 zV5`?(7Qx4!01oJ;n$_K+LVxdII5#l+%#PXP*qn-|2t0kFe!mr+jjljhz2jB-6}tYY zBYgT7%{3V|n3mZX_gY+(U1L7`+-UG&c&^vo)#Axb8GUv)!(C9*hw6Z^=LP=di*eW2 zBLToV0UMyn20)Q8gg^~&7YA3Ae`Ds^&2`1@67~3H!$#fqtxrNy938P~(B^pmw6G6dD3ACBZtAtuG>{&4<)lu(IBpFJaDN zX9H;~sxZd0@RKtliZ8u!=he|Al-N?xfbHp>Apg#1k-Rwtr9sg~l`50X#u?|g<7R)9 z9l#z~lu`>IjsreF{6Nwk%LRb{yN^=8&X9FIP=vQ&(d_=u&Zi}edyHsq%9~y)&cKc! z2k*5|3JRt_J6f(nn;|kjKdCqri`1;rq;8R^<=%Nxt|`f0Vi0ts?k*`O`?+#=BrfCq zCdLll%IpI>FM;fSR>N%pJD?o6@yg*IX%BQAl1D!(QCtA+@)3MM|Gg3(+g*&BCIi!k zyq|6BIv^8$cw4+oGWRlo_4Rf?tv8D7CM~e4Uu_7^HeFR81(kI3%@(8)dF7*!9eqGV zxgHR=m;N5sNMH*D%kExzI(`n0Ne-@#>A)1=qk0?#0QojC@Y7_bSB<>TrD8y?dI6jD z`d>K4G)bc}DX#)@dCv%Rn$?UuDANqM&2-*TLgWj3&K@QBk7#>TLD55-db>$mJNBEn zYt{GXE{2~AH%A_wkzA*qAJ+td3Ks{0r$gX9T(Ic|ek`|_eV`u9 zNc2g?ZwtZlN)t^1_t#H7pFGPN5YBjkT%tF5nvemQ_%hl7ZKb;)wS-7r*F`}PBw#Pr z;@2Na0Zx2@!WE*n8j9_^Pf?^qjmnw4p2Mb)LRII>d{dZg(MQcpu|lqTjx?1?l26YG z#z|nPKxuFi46qBn)zPux1=M#dFV++}{R%q42y=f{gCX_q!jS-2j#nVr4@8;KtyR|e z7ndxdYMz1*MZ7f#+}iir`IO=Cv^GY&0o#kRa_wBX^8Kgd!5W>0U^ZK?^0 zhIlNZD#ago$C=1mtu6nXGWul2ds~H@9Dbc=0>7D2BS)n1hH-BN-ESnr+L&FN4e^U; z^7Do8oj+b;GzSv2(>*GTrYfUG33#e9L6NVtt)Ts&07OXoqvD6(zuoGuzqw_VyAn8v zccNcm4Z^G8P^)myQDHaCSu!Z5JEo`!0<3GY3HTCcn1*Fawq#-ZgO7H!)A+Zw&qwqU z&irBUZvg9^B5`gJocrJrD8i!wjOy6m@y+j_pP0+gJ z`yhm01LTYKp`_G1@_e>y4#I?tu4q|o_~*yAnBO#StSXeD$zT+IY0y=ut)a(i!c{>- zHuJb@^NXB4!rdFII)hchbypLMIX?TViMsW+3{ZX$bo$I-Xl5;e3_3}%qlcTd{UCj) z+~+-}&0NOED=3)$_mbRkBwrH!re{zf)??rsVOP(KjRmsTick?;!V8(6_ql!>25kX6 z@2r}(o7cw+>Qd7wPVC(i3=h&J+Foe`+T&C1>9hPGn4$yTRT_tg^_=~@K}VkHS~cy0 zpOTop*XWBjtz*@Jj!ALDcCpkuKEvhgbsMuAMyK5hrnuN_-*UFT@`L3CJVS?gq&zB? zSNERdjhRLlVd=z8VPh#T{O_|OM_vS|*8>Ovo*cex2NMo!O<%-xn`RQrz8};Nr@zHOm+78+%&kSYDDOw$(=U2Oc!>SsDOJ0hl$67O2Ra zXxBuSk1G^`7nRlp{7AXuD$o zfEj}By0GLbWkMJjDbTYt+k^yx(w4+WX9ap2EWg()&}4c`>xLAQEJF_s>mjEOk)k$+oJHe>p{&%4JW%_JX#n^C?WQrF>8WM!Fy`2-J~o4C%prorYdlHbwsD9IK|vLy-= zMJOfJ#qw*aU0%cSCTegb>v!j*i>SA)0>m?$-(Rs}89T@VEX7Ycfuc&9!dDDsiVOe( z-|u^aoS1v^_SZDTCO*lvQ{7@scv&Buzha-saM}s_N8SBj5GLXwYs)|#g_NNs82{lv zzbbepy*uN|mm)But#rQGwzz8Wt5tjv2>)jlDE}#_kU+uqKLr&-Bl}_#kdxkqcKB4J z;U+*9>2XrKlCz?l3E-J<=5JpIHs69!d63}>|GUTrKgf&BlZt=?dwk+9|A`b_jaDc;%> zJ3Pt`SAKeR(W?7Ia`yu@?D5V-!1!zrg1QMVd>&6hvay;j09gsj*^ z9hb=D;fmgW2E#IbMUS$X++L1?E3$}r8fXCy^)Mvxw6&Yfh_wS#k$F-)pQ%J4uUwOV zBADsDn!id%jyi938wpi*thN9a>5zw#!wh)2>z^#k0V8%YiBAR92r6r&-9L>V zjbM3n>LVVv{cQJmKR3M8W`{+G&sQ}oS-T)WAl|ARB;Qj>gCNB^J zIvkn4fCG}zxpMvJ1N_D9gVE2fNM-Pb5ajOX&=|)|Q&C2*AfJLLH@#-CSIH%m zZnFAHcR{!jI?zji2-1Vwk$P@ms8J%q{`7GK z@i}lMxK9NAylT@?;pdCtX~=bL$mo^k1ag4H^0l<2I4}jMZ%=Rmcx`a30dHJC#SSAji`H_K zpcRhJ2POCY0qxjbT@xs7^Ww{BLSrENAfgg(4BiglSkiUiVM_ByatTm9U z-culPOgQ{N%ja-CMY(bePIAtOETS1E=<3VPEd~MxX|E1+0Fr>X`2v!j`5kEfRJB~I zD<}22a<$Nh1p$jXi!QkLX(-?2_gAClbVEmiY5W{t1#&g5+P~1h;`(yJFhO(5ghGZ% zM~WDwy4pOr;FnqzSUrpyr`XZuTGEE%t}7R&@vzzet;GN#_jiKr#E$=PB1L| zd~$QD(kga*4WH`EXzQt^U1TwyWYpQ&wKCju@{Ew!*(=#|=Vcyk7ofAlcYmdrB&fdn z+YUaP(NPB@c+=9OHOnz%ro}`ZSnovP0USYQLjgDXnaUqNl1;m% zG96?Y=q<0x6My%`2$F{894gUn43{M}E0}Pi1fnn_bN&!;$MVRQ zDXHfNhECc45FmEii(XSP27QNN0tC^UdxdQFwxCY#1KAr*D9O_cZSI!p^-n5?sR=D~ zYn|5u<{*E`UZrRoG$@dFlx_s+fPvy6c431Avo+Dd$n7P;bpP&~?Nuwe^r?W6E1ZgQ ziBUP0H$QZXG8rmKkF4mU)5%(a6+v)`Cvj9GR-@TzhN+YT*32T3HtMp%?PfBqy*U0! zTQ}aD*<{xrpB=?^k1<71M2F=ixt#@7YwssewSr`FkIsO#t-h2rzR-1fwAFLs=g*T* zc%^8ap+Q22CKGS-UHK?+0#4L(YOuUrvh1JuTe$MXrtc&wJ_ zS&6P**~BT+FRS@Wj2m_;b&l3yuP9JX_1I6!C&0*)5c>YVlZRKe#dM!|jrAP}*Jdx| z%!M!U*o??lMv7r}*XTBUrF)9q!ae7>tz2B}CI)72K*t5a=$hVR=o%b~5uPyKmRkxu z^~4l*);_fwdub%BR5M+=txNz-pIl*gF_!T+d>1lmw#gEk96IfHv~7a1Z0K7i*E8;@0c%?pNnc7m-j>{Hw5Z zF=_(w0U&zEr=*iXig7jWdcK%xOZ5U8SgNfX9cSl1{r-6XJ7Qz?;ZeJb{h6;3(cqil zE@hI#&ov#$o#g4KI?+HJJ-w~|Uest?pf?Bj;VUZjysSFHX`1wSs)DUx5VZc<;!SP@ zJ1m;mRroKMw?8ulHdos(HWvx~XLnAB+$9?dibm|Hv;GY-Thcy-fD4=6KCx*h%kZCW zk3}e{D;^1OGcf%|zY_w32@)yI*2k)Gg$3b8?U*ZH*l9f>I21fEfY+MdY~qigqpgL) zP2kgDuAAOno&`mi5pB<7I(Xi;Q6*&lX;rWWQs+-)jJrTEe|;HvufEO9si4H2*wjn% zLEsd-+^qnHq!RaJ>I3hua@})YnVMT)f$}H$HGSTGC<3Gg{U#Jh4Qx+S?(NYCkC#;! z)-pcXsI5y9%l;TSoW9pBvg2RhJ*oh6SXH*6478yM(3bIWR-Zr?i-qNkb{>mYSyeq; z1EciuN!|Mes(j>y;!jQv7o5L%g(%`5mF_WWwA;|m?B~{6?f)nx1DnG}TD%Ntl_g&u zF4VV7T`uAJt6&=*0G9xR3r8Qw*Qa$gO3*3`RKB{SA*)A0VxgNJbr(@{^67wip+0kU z>K=CeYY31goc_1Hjkz1YrN4lV`b zoFGP4WQDBw6<3@EY<9(RJ`IksWwdQ8tp4i8^@AySIBX(K7^G)&6bt!12yK748 zUY=jpyf#3Jpqtb%fAxucj%SEOJ5b{a^{6N7jd6i7yJkx|OAxv|hZfUT{E6OT_u zB9lDsKFw1GhkjGmgWPlur0OBNgbT-v92zplaKXy_^{il$^C$q#TpXC zj?sAutNGMt0EfeX6_vHKvKeMy;iF* z>0>mxgE_5%278?G*pq8OvlREm)v!mI#O;hoI#%yEdgnsxK zfQ>mD{pjop>jib%U4h}JpqF}i*oNIV224v{)3_1R1{MVPxJ?tF1wK3v&TK(05$^eu z3%yc09$4mHC%~8DwAc0h64;bGEp7$`RS;&JXmSnd45n3_8+}LWGD>`I?mm5;vG;6Y z>Z6t$`gan~7nP2$A=9FF?r{FjhD?YPM#%ZZjjZE(aiFcn`UapRL&2qrbmass_LI#;- z$n=2n^?6 zf{di&xGhweZ@=cU>#{jf+wGim^G18-`3m+h#wS`@u6DW3De7c4*LXE?&e%p@!j}^= zQ4RKn)rRu#-WY(r2^wM}M%AZ7Oi?dU`=apMS&Kys7kFDw?B08@10;6*Yf*g~n+$JB zz>|SiUc5!uXwL9o*}whj#|d}W0NM~fmBJ~r!S$}sYlQonfq&37LGD@3#j4d5 zm47E*jDOI8nn>YeC`Yd_-a*0wU5+QCBLUxl5(Ssj3I-{^?ea$^Mfr*-g|eP1Gx>x`rRmT%gg@q;5pQdyQ9GLf9Xs6rd{CY9>4?}tB#(T zE!+rv&dPU2bGAf%_b+VseZp;U7<=c(cAqG0>CfJ91G0pijJ>G3X^oRoqEm{XfWckz!hpDF z!?ALK7==;VylcPWlq6m*sC@Xd6Z-Vvx%e9~SZ~DE^uZm`5gmmbX-x_RUCN(}n~5E^ z9?E!3VSG5;edv15V)=>Mrg++^}M?q~`z(xlKNjTy0G0SWt+xu-0fyKx85TF`@UrS$640zcxi}YOh z82}7OSjx{YIOXt(>cR|O$t2hU5HSu(G%Ig~q&5rHTKSyCw{bl@F1Z26unO(qC4!Ii?Ky2-1jH4TY2G-z^ll&B*C?a-sJ%mzYBTPpmZJC`w!SjVuNL_x5gD$OY$l=?%mNM~Sp|UrRmo z9G>-89Jp>Yv@A@FLT@pilZ1b8K0|l$8Fn5Bx)Vscq|CTo0nuaU55I4*0bx^+~p zQRTuB(0U_o%JjERa-x+ph>=?5+40kUu>>l!P{a>8y>{QR#m@AFt@B3tCImKTsGbAt zzl38Vr~(8QHOWD8CzL1F!y)d{KMvG8_T(VR)gYGX_7((5Su$(&m22?;h&$_!*7}o3 zY4<7m_MAKUi$zZinkqdgC-aE*nHS?IyZ1CE@ z`r+#QlN5Rxe4G}OS!Ox!hR>)c@#5lN5rg(sCM1NnYc8jZYzo<)a ziti~2O6*ZgU6%dRd4n5l%AYRqMxhZSHdXo%YL_X=27JFZgy{nq^^P)?jEQoZW8gn2 zf3;PY!G7E1r1MQUya!t|8>x5 zl;JY@ZWBCWzM<}e0j4MwpgV9`&XpxE|DuNa_68`E?SO*!#;YgxsR#gd5qN_wjmP={ z{A-*0rq(HWV zLtGAA_z6>)A2$v9HE_(<9B7lh7rUywtpZoM^q{11K^G+VT9Cbs2pOkr{#jQ2&n%#> z1@@}lU(h6z&VpA=GLKJKG;9Ff_X!(or&vv~4#s*yrO)Pl|F;VtDXUfx9T{{|<#o3u zNIicR2=AHK+_nNfmF~&f%jj0#k&pa&!u_v8&FcM5law;|R6mH%YK%LGj0UStWL12Y zp7J1=>OsqxNy_KIFv`hHqA92srfUq$%xL6~!l=fOU#u?Qz*WB7dQOH_;w}#7NE?WD z+};c~TZ{gx{;}vimij9+lSH&Vw;?aqEYHwH^(UG>EeA4j`r_?I-~`>rcX{*!Ggv9z zSD$shXYsrvS7u_^>!5Fu)39ECb@LBho^8Y^6}c--&v!1v+ys_cm40<+U8o2Ae6vUK zGTg{a27wHubGas_?D}7~TbOP-Id`eCBFoPK#AT2?UmGC@o?Zl_w}+FtGI%9WnlWDb z%W$nvPxJwNB|-QePCuFz%w)>Uq*$>UKye3ZPtba5RVEvNL<~cbf`Q~_di(UIApks| z1T}#h=$~67Lz*t0k6Q#jcf0c*$27LqvWO4XY&7BjRdFj@(K_!}FXv8RH#+ofFdL{@ z{D9I)_Xw_)%`L7B!WPEofrtCHz|Dk4TaUJ~6T7HxMUM%QAS)Sa!{jVJ_(z1`BYd)) zUcl)seBnG-uF?$}t;ddvZcQ(7r+E)Q|5^|}*ahAGcrvT^C z!D@8@CXusw`)u+fHm*zvlUuU@vDPYA%&KB!3`G^fHFyKyS$m@im_}MKJYVopVwy9A zB3g(ykLP;_Cy@$GMkjku{QtTtv-{wwaS^wXihS~Po#VSlq0cBlL3aFr<AySDjhG-7o25P&!<7NgoM!UFR*^s?11@qC_r zdn1Y;0`JAh<)p*k(wHp=-JAS%PYyE#inTuBsBPTPIrkrLL_&(i<3nvg{yw+$^M{Q_ zNg;3#n_TzX3$Lj2*&WN-E5Py^VA`PkiCe{vyY!E~uf59twyXYY=g{HfmF`Oh8zXsU zN$|q)s5!Zq)e7o?Gir#}`H!r=+$Oz=2NV(9#;id4d%Q^s=>6`u{~BN*54xu6sJ)i| zHpNQz7o&So2HP&zwjtND>E|ddx?s3IEApO!d zyjjD|!2n13Rqmc#u@tX2C<7G?P%^2U$HSW>AX0>BK1&TGS z#1k`$_8@U?HGxXmyDs~_?>_~0)r&jt1v&bGco_Gb8X<51WC`v``2FUx9 ztX8il?*_h%<*h5H(lOW$X^KDyg|P^pv9l z&pzhogib9O_+0z28nR>5h+1o6wby%2Yah(-V;`Fg41qQKSHfm7l@6nxq&o}Y_LPTR z=Q&|{H8wv%qYvJy6?yTEjJK36i_b+=9~tujoq%LZ6o~2|YDWi@)wsX^pIMuaWOcbk)zBRAx_4OC^3*hgSL?&!IO3r3tqxB(_u z_r9Zg*{gSgcPalPfQS&50>kFXe(YAud>}pndloGmn>%m_}=> z%@2E~39)3afcwq^^uSE8>(8`x3xIvB3SKTN;RAubY1F0Gc!H4oJ&yCss#aE+x#m|U zxoVWAa|E5dsCygLny6^pEa0X3my~?na%U?Z*a5(EiM*7Z`_Q1%YphKb)*K(yD}GtN z_7aavr}Wk(ok)6#^|cGSxTw)gPA8^azitShV*AZ7e4Rbk0*W`kWjsY?>}c~z`tax{ zuc;EB;(LAV5#?e#EIx3$ES@~iJ2jd!>jS#g%l60JKfC`+I>3D6;zJ4@xMla zsQ^;bSQyEj_pb%-W|0AUdrJO0#ZM6`<9ftVw_HF|mRIKP7gf47Zi#zT-#x}b?S)QpG%zNNBW)~>chzk^@;_rX*-!^Naw%xNaGSXI%k?gPVmy5HX|2?2 zg(cPbaYyzIx_jNAm--$$m-JaV6WpX|F;G0*CTOAI$k^|k#3&pC{q8;P6HB;~GFBjM z2(5Kg8+;)&6Ll&IV%Z@)OSK5Xsi<4Tr0wEk8k7Oe?i7Wb+GMdRwP!~A8G~pIH|eW_ z@WCeOruyb!))_0aG0HS^4mWs!VS^^%6JBX;Z5XVI*gyi+Jq|%R6m){w!4jz4%*gYL zw~OY|)7CR70(+ABEQB@L%o1BvTyl#vJr_GCzGmlgUw&Z%Dx^0QSd$d&S#oQU`1d>m zPI`UkHtZ%KA3;Kf99j$6{euyo_x8;%1!?htb1GU@h8l#4=`)u^nbfr!Km|J|G>sOG zp8)OQiE2{wKRqG-xb?|@`ow|`QzbB>nC+%}_ywHQ;8Q`@E0t>S|3dLN>2bih#LposM4~ zbqp#g`%esKyA@X;`bU4B?10B5ShH)(z@a%vOK$|0G@sYie}ebde!_mpJ3XtAIEX*9&~%ZKhRB`ehQ?1Y=cH{V;IOLlf^f>xj#~z+`TPO zXv~Uk<>yS|C)NVT+bmPf5p~YZ7AWhCqUZP6yHg)nknraUAchL=Qm`ke6`Ft42E2#$ zKzgwbJvf+q-rqyS3Uw`G3|7?wywNX!$6&<>(2mp-L^wAO&cTSY>^9S=U9n6{+kQdup-5@w~edB3|599JMxx(k9B+JkK zD4HL;BMBX5s5R5qkIFuU_}~fp5aWFlq!jR4ziicJV!!q(oFxO(z{&XMbL+oIOMBpBr$++#OYL-fn@ON|1oSRd0& z)b3%5ilk^C^^U+KNTTaH+E3r49uJ9(o&sa;_x*b5v_}8Y0gGGlwf{N!88Y#34BG@A zt-Ei@FlP^YP0kYL!rM|n6pVq~-e!H&e*@4u+fk(O^yPImXt`|2&`f-VD|my$XVbez z7SBO`BpA$F%*N;@N1oSA`DvUXEfz24<{uECXaNJLy7VjFbRNB4ah9P$eHu5lk|KN< z=TOU=`W>sFDWB-m0b{qyfB@7wNr71VYg(V!`b(^-tEV6= z+DiF*RKU_vK=Xld>}&Y^N&`0c2d--J#YGhH9kjkqvm0;{Tn!@s=PtB91hUQND&oWj z12_!tQj2;l8}l&~7p%(0l2;L9*`$Itt zdr3A=egERzK<$re$i$U~{#GZQl&beH1R)dUIY5sg(BR?E{hn?1mZ>WF5ePcZKA#Bx z#xe%+9NomMWdH^`GePo*H+Xoz?E(vvnTpt2M?6Ob!w%y)1zO95`ZK=lQe*?h6|BkD zl(sp*NUqI*uqy9b#%KN3i zC!iNWey7MjUU8%q$b6C^7b_L=oqH@3Ws~pYXI-sEGP~^GW%#SQD_b2_(+Ug+&4IXF za3C`a+s#?7D=c`ga|Yn=DCyX5N(@Jd z^#T$JJcQ%*Y`PG*=H0RKT4NI-BB0OZ*X&_5igbCvHv<-U8l^Oqph5gp!y2glpD4>Bko?xnI4sspTUi^e?o_u8a=`gG9on96M0S&t3j$j+jU3o>0F7AJ$i;-)5 zy%{b2Fcz|tsgmz%oL}4z*~|IGBcS+6YtzTQ;UC)ji%j_t&nt?5@c4yeP645J=FBRD zkJ`nxrW&-F9R((Zt*^O#~Y#pD{@BYk(l5Z=q;eAVYM8?*-^6@Y1# zWrt4aS}BJ_WM>#l4r#3B-`2r6DQ)W(*&$A?YAKPt+B<{5Ony8Jc|H=Ot!7?9 zc^r@G=6N0%z->)mzc)Dl9TOb*7xnON+QcUTpM2xdKQuM0XHEE7+SOBXy`)Ti<)F3Eha-%x$(6`=;g!9 zG5hjEFb`|i!|(dViT7aKjMJ@X3fyK>r2F6ZBI+t1_n0n}7tNjWD5JoO zPq;%~pUEQWtuC8~hKPmYWGS;#zs(HhaK~G+%UomjAL8K6yc%-|4B3iR>ibfxwj}QS z0mxj&b1U$Wz{A}yu0W8GT=)3Bd-9ELfIw)r(K5qKLYgim&X`_u5kthc|6!u`H$`aw%h?msH4+XdW$s1RrK&6yFK%D7$z$IMX%rH z#_}V-Lq|u|o16iwU^{$PVBo9B19-3x+$I zNKOmKSLPM2I1LK z*=XHHU%O6k^Edl%VT5C%@qMY>Yz+`P`lm$-vj-f+uQUz~$e+!T)9b~20wV*Sb-_QW z&^8$m#L*%HS~4!)P|nkkZ)xY$=A$J9_}HOq!XPvGr86sxwDsV(&oKqR1-mY6qGWle2`5a^J=CHp1 zmB_Er#|LuWp8%0dYEr^cn(^S-x5f+O{#4V0*wwlUBh>>R#{cA!0ZYe?(vpzkg%rUR zOulRU_icca3ZQpTvm0f~UsG>)6;ETnWYwdRggGo!-c8!Fl%DI7s`9)FbfVLFNbdU| zm6H|5UU9tcw@C*5L-Cz;Qhjr{C;3_w8z>E%%VFvUYp`TDHCr+D1|Y%u!`U`WAtP7( zPdE4Ufici`6f%!c1?J-ebdR;WP0>%HSl7t!J|6vftRN>tdy-xpQ#q^rTFdmqW6Ree zt)OW(kbfDKugvy&*sks67aQO43K02%H4_i=`Sv;Z3nIDJD%Ud{2{yjry2%}8r{VZ$ z!iCdoDLM)UOwrAMWuq%C!V?li`nxDlKdq#WI+sUqM%mr98c;=wWvt}djYxmI{A~oJ zeRrpV(Jd567!-kA$G)uAwceE`-xc)~C}MilDm|%PHpi>grqhE!|B?5&snc+kfeex4 zqrgazFmmSu0_N~{X`=v$fXo0Qtn2p_U4D?Mi$`-^#_=NCnW8yADoPYzeejRbY-U60 zQU?e*_I<>;EC>Uv@8uTer~XwR(GS;IYVUh7Z_O8d^Q6F?p(3Y>!E9%cvy&2WABuhN zG4jPI7@&7Plx-?8d z?y*|l~W=xsAqTw6?wZ3LJN*z<$xY zVuZ;i>gpjw9oug!=P-re&4qHOf%<2OSG=0pE^eEnHRjih| zah3vw$M_;A#4vdzE+0)6moKaLiu3)bucrJ$(>mks*DsN0TFKA^@*A9t63-n>VGnD7 zS0w*@$AOV>bap0yt3D$)hk^#L=w3qWgR2FqsQ8rU17WzbH?E{Q(vMu+U80t4_ppMaXyZ}fU>LzHJT(LaY zw!laTzCg-hppQfcxEPpX|3ZrTP+qkLtR}rW+P5S%ji^xHab+{NNCM5*LHTwt8EE!* zGB91Hc(J~(mC@h#tfIGSm#|Ps{h4l7*R$Ibx1_Rth?Hh^J8O!h5(^u_h|cFsw`AvN z$$`pYeh5-?9Be7udBl6zab7_7rpfOpnlW6+b<>&$lVCR%FS=K7NkeSmb#XZ5K1(;I;h`A-CIp)79FXsK+w)3c_z_4V!I^U)3Yof#{?RbE+vl5Ef91> zh0`FhjXr;|Lp?8%j$^e9T50QLdMX5!5QwH)`uc zKUd*LU8#$Qlf(;<m$V3%M{JwmmjMHriEW=a(pAQ5aLX6f&6_Y_dTrc^PBc7g%0B-p&_YY(Zsc?t9bRF&GGFzzOhy0#Pc7&(V#*1c!rdkL zxZ4~rvx=9eb{2RxvFEN*v`-lkWHGsYIp-qxK&XY^*ta! zR5ThR4K%C;;*e}$`pazogDmYg{Iz7e9AR*^#xvaAQrxzeV%Kl4wrz0FY~=F2bm1Gop^&vb@j&mizw$FwlQqNohdzg|>kkGuPAy%pEznE(?$ z!$@yRvk>ci!hjbu>9b0Af{dVs}!y(dFa~MS{h& ze6*Lt$_InjHtq&x$S%mGxc9|R`ij`(Rqo$&4jeEcW*c$Kw>ArLK{QEqE zWgphpT6HkY3mzabbtVI`+KG)j*e+(lNEK|Xh@9mF#+ME}SlRGco?KB@YWp3QI2142 zP$N6X{_9$My_NCiBRx6M3REdruEc?5I_BV71u{JcHZ*s|6y1w1GkYEA|MUpsbyb2} zeh?)2aSAntZe;cw*z^)jgK)`u?e2WSxVgQ(y>XVriJ<^5?1*AHLP2(G3v6O|_ZmXa z){3KT{=V{iTYR`@3HI6H88rR#iZSps=fgt>iY`;UW@s9 z1->(j-IXVN&cjmhU;wfk>NfU!wAeIJ+4l;LXOUVN071$;zLQQi`$iIi?n{}%afeN+ zkE2q-(#F1=jOi9s>kCvpFBlIVt%i9>&HAsf+>J3>%>EQ9S-jIqV<+chmAv`^w9H2M z`n^G9B5TK{d(GtuyR79;`({SH9hs~mJ3sz4RP!ytOsT&y^7Yx4*IFSHTE676A-jLH zUh?r8-KVh8Z)E|80v;0qyjdpb*VX(o@Mx?95(4imo42#_$v0uxjmON%9jg2g-+IJZ zN$uo^i_|neL?!L7;BRl#9lLOQx_$gDCrUQ7iEwGSlec0fq5b+Y@^Wq@VB&ho(UD1A z-QB|q5W$VU_#07yuMyH_F{2t7+uq268p&oA8&r44YIMB`0%to+<&oL8yk61~EfBW0 z1M6II+Q!|jEwXLlgCO^4<3&S8bO);53qa6;C3ko*;iJBUHD4|Li6Ogvy1pA3d12s} zZs>Ce6`p4G5!`FcQ0p3b=gAa4Sx7`5J#}BmflBS#v5?jmy?5M&+dwU;WJBiqiZXgm z*Qr$lvD3UWcERMP9#pQ~Kx{ATV2y8urA9cDlOc9&&#zkOo6j4$A;-242VjB65b4)t5RdaDF;J4a+;b*_n;?6$!J}eQNzWstu13M$8sNp~lQtTscRuMf2q9RbO zmi15C;*I)Z7s#<#Z*i+%P779Zv~Ae{SeE-~TQPVxZFH5stWf1DP8Eq0p&3z==e*f( zs*lRGwL~Xi)-H^>-86AgKj(-i-$WX=-|0n!wPuFPkJBuW z11K;|$_40^mgyxb%Z$rEYBxJp-$go@sC@pMB$uC zJ`#)vja6Q6^RyO2BX+3BWOUHTmF%}p*HQd3zFPduywtXf8!m&DnlA53fA}if(fRpL zd}moAVc`#WDXQJJGLO+~TxznwhLB3aD`w+YER=C}J@ZPPOn`sT>`MQtocEX{E00t5 z-T^r2(Yt77JAA`)xl&At3+1-ez1807u6x+`&8w$OmTFNlEe(5yvw~PtZ_>gaF7eqZc-3=kp@zQTO+~=ce=wk$Qh}h^wJqG6Diq9cuNO1 zJNS$b-#xmIgQ(C?_@dG^n>ruVjN^CZXa6ybPxR@!y~!RvMrts{9%|Lzic9-UAmA!H zpb5q_n5B2MWrWOl@UAOU>iMayC_&(N_3ytM^y9S+%a?m&Kkt*ko6n}%T-&W0wLX|3 zexGKg^D9BY(^VGr7j&aK1_d@&-JyOsyV^PM#$k;ej`Dbyb`wIK3(P4AWbKL)DnM=g__sa7-Bm-*JcTIc*Oh+}X`!fr88M>Zn zm}*cJrG!B+bm%w5$mj6#KCWUPv|0*eSF}|qy8P!~=kmF7t_fZsYp)}R)4k}JE{(4E zg$r_$%_br7?)l{GtoDO?a%yuq5NsC>>wOs!6&BLrw%X#VDYpMY<4v61Z%p5=uZBy@ zg5P-fshU~<_3J~os;vONlRif=*&4615eHHD*w3cO8;PToZ|@Z);+3ZS;xnVAYf|^Q z$`{XGDh3G!?Q{1TVpF-~a;m~4q=?maIO?|Pc9*&%OXaJwadKZj%X^Xi>64=QH+D`Q z2wLvl0?d;FY>E=EPyFj~ye2n6-(kJ6k7qWV-CBUiXNg=BA&_Yh$UyHhC7?O18@p~# zJgMO+x|LaUdLtj1hh*(~3LmB9S?zK{LeRec%?UcK@+8ex?iz#<(oc3~lzdOWMR*)9 zU1Pa$IYYUM_@Jd(_KM%Cj+Gam{s(6*7mK^3ovoM0qeSzCBLSO_>~dd0 z+Upi=Mq8C+;w8>BZUwMT1EM1RG$CyCmnUf;? zxBAf0Fb|!#WW5@VrK;;Px2lB=m|4o7>@W9cmp4*>Wh?pV&G6juG3a{aYsAZ@ANVh% zfUUPt46+4cL-kF6a+b`4YnUeUq9)84TCkfU#m+hXVwhfpqj}Qn)A3b`Glnc}F`KSN z0Dyh;ek7z{yWBdPs>0)d#lPFCts^D-`529JHBKuU3;>7FQEZk8zd0?~#Ky%aT^c4p!b$5^#OxW>)7|eq*k->f$l~RAz*4U;p1scpmHk4{z}5CU8K7QaC5J?O)sa-UY0=t*aGjq=kZWMTXlVP2Q-zt+=A#P{4Q1#_Wr4vR|1Pstb7vtDTzN)v%N2_V@ynhrkbQ;!qQXM*qZ`o{Z}$mF{F zr9S;q`OjAI2A=B)I2%A-`WTPI{NvN#HESZCQy;YQ+4^u^KluF;MIFde+Aw7$iey!bWk%z7(M10TQkJ9QLK$IesW7ZtG5Rd@(R3 zi$YuaC06AW_^n$XoKs9Y^xE<0`o3$~DWYC=AHU>YC|OCPD6*;Vh__#hM1%UP zXAManO`L9aWQ<~Pq1H4D6q!WbJ@PT+N9j7@Gko-M{=-8yqsC*#c^MX$5=K^S$gy$j z(nDF}D=TiuUBUP2_R_$J2G~q-lj})bS|Y}Rz^OU$5Vd(C&SOGBdiDcn5g3!_-FXeM z8nz6^SqpY@Wx<>rX(x;TvM4qzvP|>1bki>I_r9~$hE@Bd0m9hy%APJ&u*jS`@$s|6 zIc`gar3wlN`gi@)0FvCW_%%Ybzv9H_V#K7LLGvTXbX*oSG|QIKQIDjrfrdRU7k zB6cbQF3BTWkDPrDBgF3tF`f*~t|bUCDo&H4D&&~DUkEO2)uZ!?F(DqF-eOcHpsh!uS3DxAqm_vBNp} zeOzGu3o6=xbwB!Qi>4kmzPVj&rJTgj@Ym-ozd8&rKLg-Qa?;4JTSc=^%;b(ePJ8`c zPrEDCUx;r6fL2`!2e`>pk9X{E_q+QFwLH3Grz7*ub_|tI5_`!z_|n?6GRWl<%jg5+ z!#l9f!am4xmo#p0$@jXtz}ekfAWud(>}Z}JCES6D*nGkH)PrvQL)_#)m7%&qAz_hKAB^Zk)YV9sd2)yrdRq?N(LZk|jq)mdUJnde6niYEA zqqYsdw4QDKhBj?{Xv<4Sw^{hfOEX!V|8QBQYkS1FH|2Z5RWnUGt-3I*G;^>~ur`A) zJypsy=jxjxbZT&r7c8q|=49`3-x>n0p`@o7N*heZ@L)vDi5TGP4*2NG%Wazy`#sdC z-^1Bn#vNRrX;qkI(fHW=)~5k2J9N7?nNp0>G2$@@@0ho&dEt5;G>$ik-AyPTYc@t;pWGPcLJ0u@?&-Dj<0Yr3U}Cp#}3 zi~KZC>UwO7B_PP6=R6kdMocKNH&fRU0^%(rVGb>vP8-uNm=IT1l^$#LjVoP{S_`sJ zjwwD+J(6q?Ut4dL>>Ko=6yQkP&4NIz$XL$Y?s)QQ$q1csTMM5;F{v&k8!d-w)%u=ZPr+wTU7jR(}^1o(E5gr05A6g;DDQy&ne*Zv_qs`aAR}|DNq1pj? zixn4a(ni5>qxmrb2FNv1S-`x9#7#roI1gEHZ+@4s}s z08J-3+Y_ycZwCQ)Deo6W4gK7wKsAFP_vi2p<&w!&v=jT7Q4pw1r|0-YbL!Q9Q9<{& zzSI*}Mo2intt7$}W6V;I8dtfQ=%b8?OheZ3uIzxA0$9_E3I(rRfB(769Rr|dmXf_XnX~YkO68Z_xmjNPaIkF7w~NMZaMZ*K_MGrt7UCLkpC&y~ zj#L!xmHX%@I}?NKx({1!ct=Q$a6yXlM|nYzOZFwDZ1JFU9tSvsN0?!Hjk;=}Sa;|* zy84L4-wNn1J|^{dl|mgQu^L4az>*LEeGSar z;`?)a>l-2P%i22bZKK*zSEHww7G60vDAv^122h=!p;R|e`#oQztr%2; z>*lKX=ejzWtY(!8&?DMptv4=r*7IwawBn{^<0ttb=%3{5V$2b}kYn})6elhOT=$Ot zp$DZ(g96c9%geS_Qxow0ry9FUAJCniIN0%tlvMR86S1L2pyRToqRBwBq&~+_6BX8n zDgef&bzC{b4Wivz5@Z#c6(Mkzd4K8S6x>JCI>vyh1T&o+r<((QbYRtjpLz%jpqS}G z!^S8hKecI$Js@r2qPzRi0k=6ti0TdrFpfDso~Yg0F8BMzgF?9x`C5;%B&pt|2EaHD zWq2+LD})r0k`0L=Np7bK4wH7zl_-qfkZvy>yjCqXR7}$a20AsI{w!dha|It4b3|(a z=Wrs{R=(KDhKotHI{V9qpfj%GS0^x!*owz_+A+HP71rs3x|BZbMunW}^x#xa>jRo( z(rp_uP<(jyrzl`8aWNnVZ5)RDSteFv>H?9C9Mjtl)y-Fz7v&nT*M_@EjyACN7Z~ae z8+qh4rT8lujZTi#AE24F{HksL@uwVYod()Rfwm(_zdsA9U*G$kMC%TIvyP-3IFr~5 zGZ7ma+?l0!UGlM+mmN12V?6vMwdzGDt_%;mdt2mUC*TM26iAvXi~W!CTxlRy;-|Q~ zo}V169~&LM0}RH-XmlAkVohzl*O$-dMr{&kq-ipAV%=|YtMaE@c$GpVi{}98t2NwR zaA2KNRJVOx%%j&cZP=5iZ@nqRYa<9|#mzT)2a_i`sTq*9t~f=YlL}V+^Ap?mi%2ld zy}B+NJryPKHb2wLK|*p=27)%MikFPreoTj!NtSe!WzT*e?Lt43s>C$*5{2LVUVkAq zh?7L;wIN_yR6koZ$Fi?$%N{rJTUAe)_^H|P-E+mPydTtlkn{!WuQ3I;1pfiu)C~eA zeo$lBcpIT#K5D7CkdXMUSIPa6gwj^8UeM#igDlx%zmMy1k#bqZAO;AM=&MGu;eT0W z$tJ{!3vgAizx7u{Fw7|Kx48#JW||J5JNCIoY&`)XKiKNJG)7|=h$1w?UR8413jAGp+^ z8ktJ#cyWT}%_h9{Gb(pub(Ptl@H>7tmKy1;YHs}a?5qCiq_XwhHLbItm=A9&a}WnI zPZlW(baI9B)1Q7L`ydy{!pU4V(`yq>M@F6fN0}?w`Dd5S0sgTD&j#o;oe)iRek*Pb zgc0rfIiATW%W3k|K5@L{_Tk?!pppIgj_Hi%EXYmIhC4l)Ct6JRtLQ zDWv9d&X#Aye+wYtiYOg6A-boMP9KKfdlcXT^^}P_>9`j3Rc+LD5vly`Q!h4;eY|x4PCaiZf+1on(dGjE?6Faa8GLJ10 z1zq$q3G{DF39TEUzy?m80VPj4GP+L-&z{o9tR8{ zO|;b+A@b1Wv}0VACUqzbK<^C%bzo+Ry!alZeO#AL4Y-p=kD-92cx4$5#;o<0rqx}L z=WB)gK#Bo%u}QhPvtjAj*!+}-a&>1&tr0uy8(&(>dP?tVPo_ET=QmM$H`QBI)po?< zK2t-iScV-RiMrZPh3S@5W5bC^&nWe3MKi%#^ozYwZI5;~ak`TJ=$#ePnZ(FbunoCy z%#3D5Gj2C#YW>T1E@SEpS3{n3S%*ioaqoVYT4IH?Tk=vfeAcK0nJQkk6cf<7h`(?Y zj93G;QNG5cCPyD#6P`{m>c7Y&YWgu4`p@v8=Xay~0O-dj@Ty4XVxK3Vt{MQAC;J*Z zZ+?$o7e(OM5+_Bf_mvvS#yiWzI=X zc3KXcg=u{A@O9ZLJHK-9c@V2Uiidn>c*ahBlt zj53+b<##19uL;{5eivDOXU2yN7eljG{GQrJS^mAQwibDq?1%ESW;8!j7MopE9*hrC zt4}lMe`awiOD3aIy&ZmO;%<`uAf=0hfhAdklB``k#@v`(b3YVx4$2WVO-`T_Ywz?+ ze@F20K~zHNW`wzb>&}@PffA)oAn2?li#bg^sRf72%6d#StWmz%ALHe-v*mE`cP+J; z$9&a2_hZt&Qe%3Nk!vl{O};tj4AeH5czClfV?nZk4yb$iHs|c4*9qFki#Fp$`Dg3x zv-H!|An;G?tIGy-2JXH!Os>qeWU_^OAAjGz;&t=ijR5EB7^{*;{?W32ol&o^R@^}` z!1u{pmWCr4cZP>Mn8F*7#^M4?49D^8&$VQaB{|8p#;yomyQh^w6`8$?wC$Auu=!8P z{f-5@No&4u-F}n779Ko%!NzIcrgd6B$CF1o1=@x_0^sdR%_d*?TZa`|E6- z&Ly8)rsA2@Jm!4$*HjOZaQm}6H(f;X#ag6|$81jl?WeWvcbR#eM^mEYE<6is)Z!?# zz-D%s<=`ccEJ4Id!17B;pSGw8H*N6N>h?}*DdHYPX`$B$+r6r3`2m8WLoO88%DOHn zH7%z&mKyhXI6pp&oO8T zwK+9+{$@dxCBJcn4r)LMMh!v6rs)OJf~ZdhY?n2B#5B2NY1kO_4^VvW_|lQcbtfgJV=Aj7pZ zr;A`vB>R$^;635JaFJB1L6T>{sdl(&YNqXbH0E(L^b@|uG-1Iyg|BY6WN;zq1+f_5 zOM$Rq93<-X0Pv+E;7fipAJ@B&w7Qb@Ews@@+*x01$OKLzON7TsI@)lG>z4H!An?iU zD`a5eVy56I*3uJJ)Y$>7huZ&DN{c$ z+cLS|_B-6YMO$;2@Yz0&6?}`7_@wOfux~X%iRuP8!qJyS;t=#Ml}VW&BcV=;$h)yp zQ|r8E`Jn0+9+jd+RK(kIez>{4@8#u1VP*G^URE*ju0-_r>=tFUas-HTWhZ#>$xOV0 zQ+~q4Z^qH!=5`6zEt!pEkFU=-eAz{{$wgaUf0!2NO01GsPnAPi+6T9 zkX&7U{AB!<{!aDji99m2kttpkW8^#8#?M+NJe?9?Q8wrJ%O`jV#TDuPKYVCiTc$;l zlT-^?aK-s#e$nh0V9VL~^?!g|1q5E=#Q@p$oX)?pg;o93%2U6ko+Xbtadsv1)Gskt z2qz!$uVI1;{u;)_9ALFz3(o=iIM;jt)f^w;e@r@>XG6Ps*1d{p*JHsqg!K`aLy=Br zS6$p@|LO#O-(A7S$Rl0)c*)&jq!pb%#P4%m)lRuo%fLi`rucb$WW%#NW#|JR3r0)9 z`m@r{SB=K<@Ormh{-Fb3#+=?n*>X{@U#eaOJDCzeN3%Anu%;S7v*sJ|`^l`yxC5Wk z1hnFR`&45>5M&8OGKo7)+)y8p zY!W3;WPLVUz42n!eS^jHv6-3uE(Y*~N`+JyS=)ayygH22$v?XiuPzfvRJc^uIe^O$ zF9jSoo*i1&U29J?Dtr~aJuQI{DdGks3T51KY~GB%6m&P9(y{#Y+6Ad$(!a>JVem^(B%pJyZ*oC^uZJwU;Wj5dwsEe1M8=PVsf6t2+u!x74 z0am?S;({2ZyT2hRk^d5r87r1aZG{k!e0pMK))WYu9e<|3zaqNgSZn4o{(>~i5^j3v zE}t`x#*P>TXkxwQj=*Zct73~?i2{E8l6~+oXgk?jsQ=@-!<4K6lr7)yb+?y!Ivd}O z9b>4GS+!~+Xo9PhjJImZ25i!tt`!%sRKg_EyDxGTWz|<0hZLC5o&^)XR`@ovFX0Dk zYE73Z$23+g77}@l|mX4+M`EwV>^+aj50#e;Y>d{;7 zET30^a9Dqnv}`Cx`)L+y=l#JZXGGM!BD-Y8fjj zRZ^Os($fj-6k0MZKKu>_jRqizLGP*k_M$0d&zWrIqQ_L^_ALk<#D!?vr&Ry*>$3-!8ALe-umMSUK#JJu6WW&Zx}HB?r*d~!uF zom$^ZS71j)fpO^_<(GRbQKP<%FCcIdy27vBxo5`s5w=C_5VXg-x_d9TE9W_9GO?sZ~`P=?D7^y4+ zqa(=it$Mb+ZsGs!KkD?mpL~~-J_zSE;q3Bl*QT3v=L)8}w2PZxnp~U`od14T>v8_6 zE$^G-#SIyomdkmD`hsZ=dR@g=Cvp}U5dq%8q-}8d=e3w~hqa~wTFsql+l{6JQ_=+4 zOG3%{lxAz~O)E3k;{q#*_aM@JpQsT70)yJ@XF_x+zh!a>l0>~A-A?Tv{=2u$p|J``-kCp+QbpLwj7KDs;sg73Bpt5}MUR|nq>+9? zo@4&8p%M-MRh-6*@S+r<6;~}_?s_vj)bk*KA?n#2oi{)Wxbzk>rQ)nhg1Rnt6 zC-qU!7dudZ%&^C@_iH^dBl3v|cFEn#c&BTuf5k8q`GyhtiCOTddjfvbpzQvD`cB;g zaHdJK<0gIosk_zl(Fi)*%p&mL@oNdEUkR(e&4#e_{sPBCjmyUJGGm_v1AnZ?XB;ADJ-ICH?M}9y{D6F<9O_5oe z`M`4Cn{$+^bZw8Z<1xwFS{^a7a#x7+jymP(ns4KN*wVQf1)Jj%6 z-2Y7B_+vwWAIxTEQTAjAxvx21`l)~8mNLpxa#`&Zbtm)fSGx#_qwj55RBZtdMT8d% zjV0S)w}&7By!o}>Kysoj+UKPHZx*UzEAg@ad9wfIp?1Bj=fdmLF`C8nQQvLR`k|Q; zm+*=2%>pb$EDFSG=TmyIKgu=R6Vsp0Jz%-dLR^dxQNhKS4H77R7UE()WSe|p35eB< zSx?Qa72~;xH#rJdG$_v=kRjdm{LBD?V_nX-YL33ONrD$JKr0vzy}FM(t&9f`uF6|W zB=m*?ov+A_@p>Y```keIj-@|7-y@VVpnGm74LNJeqgkH_NNF`Y2&|+BSinRVM5S9w zt}BLK&$i3(R`Di{(aU-PJdE#g_;b#vgF&&A-INZ#sxHo~)98NFdk<;urhOYmeFZ#n zI7`7j{C4q>7i@hj+N>`o7$tc8Js2i4Nvef(=X-cqjSB?ZURSgm)=LIobGkDWdqE{7#X>FS+OXi zy>eX^AvTJ^%$xA$qG2xn2f0HHlAdb)2)uCSQ#C!twiOt%?1d+E|uI7 zBE8C^qjh;_KS3p$_w0vPcaPZB3^nBBblF3 zq(q-4Nph)p=w#pI&>3c1lf}+f6JU2zx1Y?-7;@$InnBktKzqikx4rEeelC*rtcPOP z=90X$XXQP?Nmoa!uwx17KxvuGrleZRJj3Q=?|`Gc`_j`Gr^jH5$?B6rU9Z`5Flf>& zvO}L{^0yglkCZ5Kq2-LZ^)WpFMiT319QqvhtRtafJ~EPY!G&zT9oxG0zBR1^q%6oSF6j1NmD(NK9^NZEp+5#a z?qQ#$16KtlqkO;X zWZaj}pa#k!eE4>OP@Vo9`R|eCX70=ESFz?8LBl3!M^``_8ti0G}oZAARkfeGH^z; zN{5E#VQ35E|147pK#}H*Kl6T#a(}~?q*ljhTnfleF{i)(dbyeNx#s9yUd(H*F9&2D zxqse<;mA-$BfFuyhm+tmrzW&Wp!;H<0Djd87hL40%;O~fWkw152RIlWOaA!#cMkWQ zaI1j|>(*nRC^0dG@4L7~=Wl~woO&0CWoW?73EC5RzHNP%y{?#o2SKuPi>jtHoN@F0 z>Ro5M+M~FXiSf}v;h6TEe0A?#t^2lL%Y$_@eTinZznRMy1n8XbLE?lSX6aIG*_OxL z1V2SMA!?uHqv=WW-Vs8O_eE5G zeVUou*dyAe@BJ~a&2A1L{}9(+?)Ens!Cn3|e_ES}S^;0NSx^r+Jav~s`5C_gJ;cEE z0z8b_x!5Zs#iGpai=(MF|B#cI#e9{8g`UIqE%&vPxj?E+k9Gl>s;;7U`^x%qOLD5KvJVQ4fN7`yyL116n#lC z%;{q*FKB%R82Al_ky3UhP(_~>21=S@Ohe(BXPp)H!y9zC?#A^k4m%%-uIW!dZ}6Ja z?Xusmud73SqqdqMWOAffV5Q_t0%o~3#%%iju5=+Vi|roluE1rgb!>!NC1i80) zV)pU=2D25EBZC@NW%p-nKkn{m&*L9)C?y%4?7f1h26JoYhy3;>c=MM1EBm zxvR&Ou=5KTIG+^t$0{g0YHYWu{hVhRJXzo89c>*?PJ90k@w zBaLs_#4W{Np#b*bp;rz>7?LM9*B)#$O!LjpWr)PIXI#x-pzDF(Y*QuvAFqgGH-a9k zh=LvT`6a8x8D5~3U+hZtEcfJt5Ds#NQ2=>FeL6baRsp-zP~C2onw>l2wW!+s+?4-p z@aQr_q=~;gnG^&%7;HA_I-&wy_i_!^U2I9~e(@mh+Rr>r8~NvI&{_9qN-LDKt_wAK z>j%wfX~#;TYWqvQWSST~W)A#Z8D0oFjTX;2otxQWXS0SAA&~eDuW-qu++=^r?Po@S zuqXDQN=on-6zldBa{OLfKbb-w&3}!)g#7z0OmCexGQP9awEDEpwGB0rR|>k*Ar3mU z0j$)8!LUy_#T$8o+6klZ$y>oldY?>&<0wAAo1tyDMYids*(fG@%a^Uj)mt48Y#=j^ z6P#+skFMUj&OVk}OT0nDndH7X58|EYI5F9TNgxz{3$cSx@!nL20I~Rk$DOUZYchEM ziS+B>s7lgrKI0nYV0GS;8h*g7WoqaFuHnIlB*1y+F5rXvpluW zjciGANysv5rM@(l2(S8)nuvpv#%&j`xCOQb6RV9&o$-P;>ylsy{C*j1aGY|}2JPL^ zx=9C!RlP0d2|Xa%^?$rMZpNMU+CpZQ7L`=>8}6`Z$m zxtWE~fX1D1@%-xbk3%4@AJ%PJ;QBh29NCiSKB)h&eE!_Llm@Z;nf#7C5g6k!3sMo{ z%W-~b4d46ElG z4_WB0HlO`dVNl394M&R{xe>K4&&Qd`fp;r)9Q<7@31qq>MPtUJW9FU35M;pqC#e^s zO^Ii8TXVv?Sz}}Ym6r`PAWELXJpX?W_q}ceYu%7|2VEmih<%1+E-U_#2=*^9HZ?aJ zq?$ImvB4%P@5HuwdN{K3tAA~>8y{*~8B?y>Ew&COEHUYbWWB|5Rm3qfBjkeQ-rk~$ z2JQRo7r?RKOgU1`4q#Nra?I-HYK+U7^9=j_0*=FOe-kw?XF| z;jpb~-7W+{%7-T*-pM~6_A!F)lB|E=qr7L-{>!oodP1=DbOPJGk~loQ(?X?m=u(h| zytnVr8u%GMym_^5O4q#owXn{eG=XpsjC)b~1Yprw`Sq&1?^N-3$@e&4^~H+_lC+Pz zNQwBwL9BGXV^juWzeSjyMnb>y*l@vfJ($4b!iU-TnE}-s%hu04t9wh7dQK3vN7Y4p>u7ArTJt1Q-;WQ}&2AjNH?zkNy)cIImwMpo2pCyX(pF1r7uS-2oA z_8*q9BuzhJgp0$PM%=^o;pCvod@9e+K=ZZ3y0g`A3f+YnQh)EuL9F>bzjgpXImco7 z+PfdVqX&mE4Y@Z#l7oe>@N9`sDP&#=coR(lSz+Vc8d{~vP#gMOf)|1FI_4OLNA#xY zoI1P^X4s7W(r(se#KO{e_|s&b!1h7U1uh6nzuY2QrO~OmE5YZwI7$5V&d_>U(k$1p&uUo)C*4bL(dK z@1}F>9`>%7VVyaJFufZ4+|BAH6P`l~!W@}O;AiuT$FtNc;~F6W zcW89Wy3BYltJ)zR5REXR3{9c4XAcU+6_I?3Dnl2(QP*?GT`Zu;*x4?e-vP*m{+iBZ zLRWhTroaB;8d!9}7xk@D!@P|?WHM$7w40x%{P2;796wGpbksX-0N{Yfj7xX>^?{O2 zf&`!`di&`V!<_{?m^K(=@#<%m7Svd9BK^{pcs-^!pdQg!z;Tc;a|L{bh z2p;;92BuKMz`)SsunJ%8=Zd(4O2$FGsUf5DWe>5^`jd$L60@xc)KD;kA|Tgf zv)TNnc_p0RvNxyH1YU|FA97Q}W~*Q72f;isTv6^51>9Q6Vv1C=O!Xf2;)lGZGXeB- zf@@|)>r-M)=_(QHSzN>UVSUO+c};E#ab(^4mx1Ts@r(NNk&By)Ef)BO<21l1jwUNC zrl54G7tO`~0~~q_(@6)hV&WDLBU@|isAU?uAEJ^b(WmxyCqwj6vd%cx$X1S(vN8+7 z_47%Gk-b*Z2O>=a8p@F(98^+7dIJ6~3Epq$>8qOB!Jk|Odf$H}w8OIN0Dv5&*CD=p zxMVo!n1FK(MzyQhy$ixNt^+gNn6@=x?w0;Ya8QeXEVDOLMLQvt=`((CTa@OsPDSX> zLv^46HA96ee$Iwa-ZPQI!PSCXcNb7gHUuuCPQ~Np^Y{C<{mLlbf3NEMT2lXHjKOU~ zWma{DJKCT|`|sRx&{6l`p|kK;c~Vrfh$f9zMjt2y!maupfA1t&jNI2*DLquU@Bn$# zbXtZfC!~NWv!BzA>M{5j|5VkhuN(-l$Ge~;59{e$RP>Rp#%XykP_bNctqE*wI1Q4V zmLSAOab`8=4^NaRFn)+C9(?b=o`oCOLvS2QWj#^fys|f?>vAOtQf#GvfBlGn>PqzT zaJEc@VDq*=^Q<52pbeg91PCW_&Wi0x%3Pwl>!)r>?voMnhL>Z{?*v_Z1?q4r{j1l) zJLn@mX2kCYxSE4rDI)nDBt z!>{7RJcGyNJy8BZCr-+zK=Uy487FhZWP^td0pOOd1d7{vC1(h$$Z`2L=+EA@+p9}@ zBcncN#N)fyt|qmp9(?RVn2$m{(k~cyQ}Rx|agpY)X7{^moY~!T^XsBXi9c!w=3=9X zR#R{7Y7-xUo7+|iTkjRUhq*y-yUgteA$S~n%4qf+FdsJk2J}JX4Ut<5JSwySZogz>0$@(KV%1byKZV3;sWDlUnc%8I`1w3PJ6(u81|)O6TMR`nYi-!55tZCAHV`BFAs z&~ayIy5)S@La=rhecF;+JlMJ$Hz_8TcJEGbOYz|B{_Vc)ru%NhOxIarb0ed-3q}ZH zK6?;j{CnEEwb;AkKB8*0<8-RldY-Ju!6y8-S@&QDj#^_If>S%G<^D7o)-kk zKN>NWu>IMd;`?xoaF?nubAHVgp} zkC)lN0a%75&ft;EUnp_F8Acil34pn9_p+We{qeAC<2=_^!EA?kVcfPiRj&Qav;Il| z7{p6(;Flhtw>;3DNo5azLtFuc{U+;R% zZgIcQ*kaN}Rh7_{AvW}ho8XVc=dps}=ys+$XR7FOM=&SsIWQ~4{wu`@eyrh%*%S?fGu>LJVp#Js%-R}WVVtt^!fvAYlQX5esUY$FWq`7?rqY)~u zquu#4Roo&SlWWk{qW6GWz4WczrO_SYoD`M7OEL)L^3ojAaQL5{1DolDEsRFVmxK?; zEsZTK=$e`~-cu<*Qp)AzX2D_McetVSHi#h5Ng*gu#HWv){P90HrvISGgL3rA|4}0e zk46xzIV*6I%Rls(KOgYAIJ-XJyIoU1dzOGcfp4dK+v%k4q$S^9^2(^z;d>R6_Rm|y zKL>(=eDyOzkE@<{NAvtSj~lptKOeVK^+_|VKC;}*r!N}s(3CbPSAg`FbeNy3Ytvf| zJa*j-dhKMPyjUx5ZuYE|qND-lw9)+$3$J9aX&`By)yo${!}ni=^VO++nW)SD=h=Yi z;dF~cixtXIOxrhV`p}J`&c)uH*CQbV*14a4+UH5;_5V0~3#d4{Y;70|fuIe+-ARC; z!7XSKEFl3JcMH%0fPx6S+ZSlJ~UL8ipbXPbDo=`Dr z5n-=qH}he$G@pG9m6nPmw-b)P4NQVZ%VWMIWIe=OVX2c2?39I!GEra=ZdRvMEx|Qs z(h%17dkyBLl#ux2W``;>OzyF~pCr`P9<7|m=uN)9Vdn_Ux?Oh3|K8i$VzJqr(@B!& z?)L#+cyY6>Mf&?S$6HM#a`+*Y=;l(6itTyxb}U6p3qi#2zK5ODMI_tqjmwAm=H04- z{pm%iTYUdG$w_looRRNTle!{k>Z6xW{he2%IK8izMH*p-f~v8tu7a$1w&b$jBKl;mmggh=xuX|Y(fw$QG49O7RFsA& zUNjw;;S6cHQ7pqjipAQ>Xyi;^r_C6sF})~qg--97j;Y|DTm(YBRwQN{kG>!MxGMSM z+|W9Gs_$~J3lUJA`y$EjUvADS@^Mf8W%Yuvbk~Rk8sqUnrsr=q5HhHj?&~z1-5-Km zYADR^f5hc~_|4>tFj50!c)wf-=C8mXO@8wE+>6kHch-aW??U8wC!Wfo$z;uAu1^v% z9UokZOncn{Gm@dpOLw*#MR3J0UD43-dU<{kmPg}IGbFsn=SB9ZgD*l6Sw<4LZVzL< zYPRtRFuV@t%8PZK)pBx*FyaW8$&3OhF9MY7 z@O-a2Mlo5^veZI_AAwIU*^x25JTIr7tsA0Ya$^j>i5x`Wsxw}e#(!|-a?M*x# zaYxyu-JvvI3cb7@{5g}qk>axceYO#%TARHQQ}&Zk_zW_C+~^bThY>1SLb!bWs)56a z;3`-Ku?BO7Nl<#*LpP*f1Rtd(KxhJTIUb+R84+AnKZ0n;I&>yuWi$9x8obUh5p%{M zptATT2^A3G6mFV7>)o#Xn|%`dXTw_1nwy+C`-*^nT!-h<*Bbea^n~4|mh=d(9K3Zz zqRQ~Qizl`+mPaH&7NLG(@*nb)ZCG_;%rQN~x3|+UWMa^opFaX#&|4*o1g&{m*HcMj zDMU%)q_28Q1!bw4;UF2K32Ef0<{Cwd=sEF5UM&2@+nPzogWX@f1hm2K6_wT$8PgCD zD+AZ|uqGKzItenM!anq^BKjyxKJ__W)Aarj?o8#hL5jxPz6i`wGaAISGUIPS`A|Q! zW(G8$)^CG*R)g)d_U^<)RDd4i`$HK3&0Bj5XiwAiGk?g@VukXCU`bI@yk0_Y4auWs z&FP=Uzyl8wE{ly7s_A!sWvYZ^G)dtS1LAtT=d<@o)_@js-B}n?ulaIx*IA)K+iRXp z;N^OHPnCMXML*NCOyKH&K5S^e3V_+Mnr-~%Kfhnc^s3sG-m+DPW&v}WZY-|Kt*|Vj zy8f7R(RoTU?>zeg*n5l-@8S9pUbNFWYaQS{#2hKcBa|f&s}{SXB)Kg%@@tNAM#t6e zHr?z5dfRON5MnwiLjN|Vce49d2_D}W8QuqKt2=E(5`Jy1u{V1{2aIKmb_$H+LWUZv z>r9LNr)cW)7eHtJ>yUg)tH26q@|V047sg|%yZK}RFq{9;@lv>Q9;-hj50lM@;T_|7 zJ05Bk_$dBi9+5%YG$@)j{Kd*BOVEZMf~!lY7)2E2hn6)6fM?>rNpE`&Dh=cmKy==3 z>0h)8QzGb4P!;@|Sr=|$OPSWLOWv>_=}9+$-r)NO1KK~&#q;Q4(3dE{P*YT~tu-}= zi4JW4GLr|iUtfX&I~VD7(WfN1!o3&k+aqyzIEfCH;C)+`gO+f&J3U%`O(n#L6O@2m zIS?i(g1O$L{aV_zIQ<7v;=}toJ@zl;rw{^$7dcd6r~@Vlq!qX&012>ABOtg^XJYdG zhv?p=0~0@~*_Q`cRx|(O$yT)GDqdIm6vwxX(}3YE!)fko^6nDoUOQzep_Ml;J%`u? z6j=Pt!3zOBxK+OMtd0iuLF@LIc30&vVXUMw70L1PMlMx~S2b8wiH~o*8v)EmJM8yY|AMAZ$e_2CiG$8r|wT`@`SFNlquO$t>v5de>(SNEYG zP@umJ?8J#Y`K$p3tz?)Yu_VAxpgg=df$9|GzGolajQlt~S`;ef`mVe|LUdEt!n~k#Do4 zR`WI)h?NgEM)+W3eUTHU|Lp+&diRH4BufZ@TWB2VT%Q)TtzVY?`SRNWiGQFa>H-fL zrxZ5aG!g%g3tL|kyi2uV+;J7RnIsb#uBG|V`^mhYz)1pXsT=#Z_ilX+I3d>`M}A`u zp8G!?q2=hoL8e?77F_OEhrI3BX*RJmN;eIN-GWq%Nr_hvfDTkgn$Td=`Es`V!QS)Pu zt^W8A|9A#cabOB?X{>S?|6P`UnZM*R1hz<|3u{ervisF)^^u>|3RO|oep}}ZEgyT1 z2xmmwZJs}q@blAudnf-Grgv4hnv?nk8XqQxo(oTZs(-@yPd6Fy0q$I#`P=BD56%RE!P4s~rCq9(*-Nfgj}1sY7|Wtp?C805L#$@+g+kWfj_ z2W_Q}5+fRL92O&4wC|xB-Jh+<`Jf{9k{8(DsoAsg{_a|y=4SaZ@9^G#x|0)yKTD<; zkH=(z-lWZ}s?Wa|!~bMIS%!dm^G6icpcd*i3D5alALOJ$n%#RpH=f3qM07_{Q5@85 zWsGV=PZQ|2od44(_3=n-6JE=xG@(=azPHoQ;ySnM`3ShC*)yv<3Ql5Pim)!ZUur99``S z&(Vf7y+kDX!V@j74{D;BmD5N~HP5^b>$b8X-h0}hgdk}c-u*G&C|w98f%lfkQrMKP z>*BG~ybr?*@vSt+?4|oV*eE)MHNH*FqRXg)-s!iuC$ab8FX~x^FD5=PbVhJ`&JJWp z3P1%Bz(%+&)e{Pxaxke71z=uCi285RTA^8{@cO$|7dbN z?PHN2|EWY#PX3mRdhYFDFq7>ZghbEHu(AVJM5e@Zaj*Xy#noA=b@>}7V9FUOeQwrh z;n4RU<{R5~!EM|<0r%cu(M5HoX|}=*5gB^|Ky<^f!+bC*0R4mS#RV{XH_*p@EB9)yaiUdl_<2@kF_%A#U06 zrNq;B*GDviN!$$CgHlc5lPZ5WbnE%b-d&GI`|u@8zL~f9O0^VB(zGr6EC5g*?m6>J z@3E!1|CZDLhcWx>F#xk$vRKBy={j3(I{H8jC|cFupR3$I1^6u{&@tkbssc4zwa7tb z?Cqb0C~7fb;f=TtV~#C18=N^x0*idNgS0Odel`PtMpJtox5+eGv5?91JU?z&x;@;k zX}Psog>3Z4yVTn@OsZ?kZl$~Gog=1Onh)KsQ1>)L6VE7D&Cog@oJWY8X0w<3`AM{| zsK@T_yx_F6$F?oEm)oc`SAG&dyg}PmC~1K3fw)T72pNGO zdg;}87HnRK@yz11YxQ<5`cPNRXODW`KEonrFa0n&Q)ZmA7A+wTZ(55IVP=c`!SdK{ ztNHpc+@dE%Wamfhw@JhOn(F@;8~^>s5vvDosY+zZ|3>`gxMLLfpkKCTg_&%$=;>?S zCm0T7GMB5f1;zExAB0EFrUE=#bD>S|^@1J2IqhBf+_SF4}9I0YvQiU}Ey%!7k9!L+_!LKMc^n zXefNhmD%T3VineIcp8DOc2j@i0SUAi&T*FVbATL7ja)r+C_POeL-yO7ErpJb_^>2i)OjGv7&bf^Vz zzp|m2EGijkU&`5O(r3!O5V?tZDd4!K(|)!+_IZZWuDLZlpW{Gf#`MmxTC5n7DAR?O z^IT4*>)V?nEd48~^VrNA#yXi@?()_7bl24SSwMKrjE?e@_A;u&XF|7YH1|oNmF~)= zstCyaCFHIWcvLowG~YY;8zClMK>um7HJWzS;?E}Ef(bA=#vH6C^VN2qKfBZIt!ueG zfV`i5&VIG0(es2i^Qx0&Y`~blx8?r2C9kE5;eQninjT_-UV|~!pCjI3(O9sz@JbM2 zHW0dnlrTs4hJnfKOnH$&Se6ffEg16_sDbBeKR0FI2>489q^<6y)f8h7yvLw6%p@FL zo5pj$%ml94Df|%mp9@1h_KFxz&7@-TGJS;0Fw8M2okm`P)IBOF+gBd;?mV0r5;;l( zXPbPbd=(a!+7n4ZLgjrv`dhycl}9kT;}g?rkwDXqQs2>oaff}~EgyKH;6oU7>_E@O ziq>d)2*4YGGK3iGwD{|3IACyM>0T$>Inh%}h+qBRsqDWd__8oO@mltOyvuC54}gyy znb-$WjRL`k_#wQhRcLPG(H&Jy>!q^C0v~?aV~!Q|xva>{T?)-^^c> z73!*7_j6xbU*k*C_A_>KRL{_B8vck8tt3crDrbHMY|gS+mqx6`OxSCI1SV(36mY_qo=Cf65f#;@Z6$$^9GIiAMlXH;*A& zc@+_%iC=K0hzGg3`jK{k4W6@c1QtecSq8dnOWV+QATIc=^fQt{RfGKVj9ZPQB(I1A zhfM?6I9H8?w9u#YZ&pe2Ea?*3zZAQ=tKXCgDt;am;Z(&cB48lxkm4FxDQ7Qs1+(3* zvq+FNEk6QG1wiK?Nh31f)vk6}`j%n<-++G6=g@a+{l?~GwQCdry73DO4-sh2FoX(y zPY$nl(&&477!6qLeh}g_KA~2_fJ@LF(4fS#B9O3vC%BB{LKw0k4(-~+zv4OhK^29` ztZ49iHHk9NS(Q+aNXhy)fqo1Pdk#(4&)D_2k1DkLeH3!+Voqn!rB+M;?Sh^ZK{KU z%)9fvm`oGXtxUJoutB74b|-%N^qU-Bq*v{@DhnuaMYMKW@T?Qt95BApth=7Z6g*8r9Kj z_UA?`Bm-BW(Zk}``!dH%m}~(Uy`_}VxgQ-Be{hGxT_O(tLe+QgR1N`j*%Dp7;4rX# zIS=6+y#GSPMzD9COTF=(0WAr?%4uh;zP2pL##y++_7nB#uDy$%Hi*;~1xjz7V>GJX zKIcsBvHg9t?J3+w$znFHRn!sts#g@RA>+X9 zWlc*YkOwI=G3wjXzekviJtE|T6XWKsc`Bsh_Q<4q9@eqUhLEu9pJ$UW5gPqs3yQKb zkYbC~)I^q4{+}s{tQL%hSHBf3V8<{uy-m+E(!t_$WZE{MGfSq8eD+Oc{j- z=W0-C-xqYze`myAS!FV^DidVAr3&}8Dl0AHqZVBN)_<0@HO&`AxrQ&A&c`MsEInXb z68CldW`B~-McvPQ2NF49P^%L^+eP=AZgsW2n`4IR-grd#=aecpe2H77jJn_w?*pqH z?o6}B2x*S$Z-R`VNL6K+T1T4BR96sWP5r< z8-rXwpv{3EV?F>ROTEH{vdE%pW;n3q^?A8|1H}9Bj~NvxC!*I*#(O`iz-aLsRmMF= zWhOSjmp_WSV&8p(g%7LD`mlTLf}4fOxG(%>@_ki(!&opU7X!HTUF>hZx%oR~dI$1J`a!vw#yUc$LaPM6J)gJaI-a{p?pYqi?3TDQrg>XD5bg|@WH zc=^>0e}+@CSBQ7#$@-`k`=4R##}#hnVX%zDo>(C(Uh|>Cu2|(y_U0ZO8G>4=g7!v{ z9?nE0CC@*E7fnw$pQRs7tYRiu3g=~bd`d~2DfcFyj3DD3TPu^jJ+D0&IlP!ASJ^E4 z0A%T&AtPiqhtjpfStMAz@p6&ZU;QWHXF)JJ2@kDKzO=lo#Ie8xSdY5qx@wvjBQz( zN?TJ^DX#~SPUGSBB&3ej-n1UPm#2z1p_E)=wn5PVWyY z$CV2>G|PWnuE(nU+^XaSqv1!nmwo2c_i53PbyhoMmf~obdQSK_zw?>IqO8F~BKnA1 zD#J`iWR+GI+N2J>Nq}+Nj{SHeJv5vuz$r8xyUfA^qM_TW^L3yf;Xt#Xm~G&%{sw(lt%YfYc>=9I{|(WzYkRZ<5uvsP z7KO@K8Mc-r+?Z_D9(rqa5ge0K$3U#VE$;4oLmA*H;U9p>5o!*DdLGU7h+me~J6Ynb zT2N?(xFz9Ti@EI2be7z^E;>(dOl3-(#zg-(5U1(Bw3McHTlRN`({Y?eB?;>FF^S0w zZ>|zyjIC+XTPz(ih|hI9(0`TnR-@w+exOXAdQP z2?gQnv3rO|+damL?0WE;VE&^CKk7uZvN;W`$wVdqkYoIQW) zIae7uD@|zFJ_^c&h`0FAoNxtB@0GOpq-Sc|*5wj&4genpNlfDCFCN07MeaNf?JE*J zq2#I!2a7&^u^hd2yKRv~j!Ln9*t17DVI_rT408p@0IMg`BCDlAXU)d^vz@+=*_RY& zEE3NGT+qtgqaaM`pFtnEErTI%p{{tNFVuqz6p3GbHJy&H=^~&f)b5o|mTrCObI-2V zT$iZt~c$->N8uP0p+{Rjub?@RRmMLlt6H>M1cFeHteHl-ooI z-hmt;7Vk;BNXuI+L-(ztnkAnq<7lz!VEw2Q4!8ZX9OzNgRN(Jt!b;+Q@12j8DS!+SMkFFM_(Uk!JS)LTlRB3fUc>s9RHcybNg*w+^hD! zN8*nAg8|tl*-BEF4F_)Xp53DRR(k9k3B{Y0^jZLw{FvJR1A!8Qz%g!#wV>a$`Me;W zh$xZ6+D;E~LI`^aI3Rvc93}2?gZ#?b?q~nakRnYOM?LwWK(^g(HpXZTfRNT}G>JH# z3<&jC)SO!GFg#y}_Yo2&bQPT4%KEv+c{{R}MAic9df`y_TgSjhdX%eLY!%k-ob>87 z#-Ra&-1dxCtKo3d0eAKrdY#HOFO5^mgd2B(5aQb8k${fpjhQ!H0z;HLU~=mYkIzke zLLstuBXlztD`oG71>!OJLbJh7Ml-ikZH5k+Y$is;PC5ylSNK{92*?+f;&7P1oTQ(a z4jg?T>EO_B_Ue_fIEv=!U0=_x*8^ELs}SqYNEolUZF=j0_DI6~Int_|RFJr9;-l>b zJTvNro%51;&6#7l+v~lL2)7T|S_S?k8#vY>O@ zHlB3z;i3nSImwIe-+}}JVS1h6Ex@vf9Vh(0(TRgohsDPl9~xyfuaCxRu1-B`r1Z=Q z8|opUXM&H|5Lm@VBf_pPQ#_W&6N}p@vWlYn)v}!({)r{sN!26$Q>e?}fu**bH8a~$ zOXJZiO{0U*Kt3s}{ZNW_Sgr4-^?fKq6zhP|#+hE>A7{Ex8zOBr<6~^|P!dJ-<2%@# zRSBJY-SuGuugK@^Ah+YT$AtHaKg-G#`hg=jC*uJzMVyQgwks zP6lMMcoNMcXL~-I8dI7^djKC686JeNR$ty#O7Nil!h>(cUT?5;&op7xV8otMni>Fv zq2iGzX@UnkN&(MSQFgHXajFN~l1W_~^Rfob_r-}&E2hHSHI_KH81^Hhd))36Q5$Bq zfrkC^aM>8ZoARvK1LS}*bD+TriB%)mRfgT`fh@RlgK1Nm21w4m<`?Q}FRi-#A<@i0#w0l( zC7c{TZKt3cas(QtFCa}`)09c;Va(-|X& zhb)UG+=oM5x6HBH)GLg?Y2g^e#ef<~?o8TD`J4$vN{vDq!cKujnvupz1mwc<0cx!1 z?b|yZOxo>^0G0&*TZ0NU#aW}pxM);`VBXM?ZCasrYBUP$8+8!=70B1*t%qg`4;L7J^fBMVIvCrLG+y~0cb90QFWLpK= z@ozSSBtAF}r3ocNSv~bWt_K%F#9Lo9lu zZM&6viU64ZC(lfjAo~g`t|09o-V#xNY7PD;chkKAQ=;&!1i9G+NRkQ*gr^guAYX1-Y1_dzSHPAZVf3F51_4dex~IZrc7A71+!amnzyqC$=uaU5(y)nSB!Ybg@e$x11lU=zmYeNX_a&s8cA{joD~TerG@7j`{etSx-}CDB;KQyJTsMD_1j|PsguQ9 z7+%_s_G7S>{pITZN;2}P_=%fZK|XtXboPUzXcXEwZ=nQm{^7>auTw|l>U5X9&3tRMbNP$+HuqkcGVB_|4b#@S6 zkdY7?7CR)mH0qVe9lklO8+*ls4C+`AZe+sbyR51upKGS+US9fXeY8e5;xcpIVt|RW zt^HH0`5UJQaoT9u77Z&A^Ze6olX#Zx9llfzrj~u(Mz9_!$ecWLW?|25$@rA_G-8Wm zc2_nW9LMH}@s^#VTCugyF-^PC&E80DA)K{Nk5lp0*UdGPXFKd$OW&_ps3GaonVbY{ zymh@}sX&{Lx~edL09tK{i!ZnhxyRZ-KY0HpU#)Ygncn3eD1v%f5eMQ66yk4IAf=;m;bs=wM5{#+1Q#W{QDjXH^;vhrzS*> z)o?5=t7pAisj?W&<||ToQO9`l7Rv68{{EMSr*B#;>DVt^XaaynLe$k~#=2h>UO%if z=(yDNo{f7MZ}Zg{i~Gzunnv{*5i?smo?r)Ee8`A7*49c09)gh|GZU+t-54;L1|VWH z!KNEucx1$sY)vq(5SCNB?>b`G*sv5mVkyW_z*gqpFb}Bp@5z!ESk?Y1;pK0JX#oDS zJ1v`nrvBz)4i$T$7Y7e{&lfuq3tMIPtt?tO=7>aCj0|FmJ17W0-QE!4Cd^IdcDmlT z)6gG%VYa0#54h3#&w$n~Iz_-l)qMC1$G^m5VnuOOrs{AtsN)LY}|Mr^UyP94e&d9e%nYk$i zg6wVxbQaS^8R~VFVqc77+zK8khIXu-d_7ZO?NC|DXaYA3$C%I&d~GwO({%5mnR`kv z6;AV>2&Y7ya=6m@Ik?eh;3Ac%v(O0xa#z64CSyEuqfuoSfic!ji-J+q^Lc9HnBql= zGE}p8arL|P?FybRsl%JlLN~lQ5lCpr6%uTARR~D?RWxqTV1O_ltwUA)?}Hug@p)Fg zsP=$hT{Hj>W64N@TJ>XRyB$&c-kh8_#)mb&vgb77Yq|4u--y?k>gsY;vE{-HS|Wz} zuk}Q$Kn)e!vv7mHCibaepgv0tZP$GBBfeDeS6O*q^)4DZ&0+0Yl-1Qw_r0CM6xDS_ zLAkpx%=K`=E8J^gM$sZ%%V{imG>a9_cJ;Pndb#2lNV4Wdm7oP|C27Bd8!cWC%C^Y@ zp=j3r=7$i#NApT=?|(epc>_OVOFY4!P!>6eOAumyl`3sOR=ws{^K|meY$z?c`*W_I zC<_1F~`-Ez75S*R_8Ca}h-l8rP-C#a_9%a0^P=?kZj1 z&~0^@AlcP6rEyo9ieH)IT~cNAC>esOA_8T`v$L8MY@OHfI<_m_Tqi$$$JtmXLiPyA zuvTTOy=vfI>K`|FDD^2lx|h@uLsh@@&+e!s3YiIgB7|y*&C2k@DAOPCWvXnJ&#hfS z$Eb!ta?IkHqYrHex@;|_AiW%Cwjhw8{#3KzppY&ZP(FzLB8b38lVlaE=uXQF)G^E3 zMjWnc8udz_^UlC7xu1nzbk*Sa6`+M~$gCUrWYbZCuSu*_G4cxksLo#EauW?BZMn0g z{z7KUApWUXxW2|VR}(C;u~?~!A7Yrk zap4Ay62DlWkGnkDBjYujfai+qJk79zrKjmJS|?n%Lpz)fYDkoV;ZsL!U))9ezRp}NCL_+$RKe2T@c5-E*&c)Jbjqz|?ruU>AN7VN(T6YDHICggwX|wUZ zOd{t`y?H`7-lv1&LtofJv;SeZjT8yBB~mMHSm_lk->dYn62j@X2Di?CJpB0ZhyFYejJ6AE^1t+-VL}2 zO8_7A`z{2P4m^I5jrWJQ=EQL=s|dr1?V&*HVu)4WhQaT+S8oI(ZCxgd2<0K+r15;l64emp`3h>FHjgQdYT*X*x$mNjYU83Y5l(fHH6}6dM90oax%6 zPD(X5`3f>ZeK!yxNCW2DWUqjpork>g-e=!+OCV7$T^9=8nD?2gUf z37_cIsy7!U_|#q;v5uXr_I2P%G_vn3oQbRO$ut>{jGwk$E8AXSaE(vTRLK4y5tc#2 zF!z2uN~EC^6TH@7G?aMoZXr7J*mH9n_j9}Wi?a|UHYaHQHn1xPWyaJIGJPwWxV}b( z!-vW)iKBD%_A$7MXtQzA6BokaScy)-NI}T6d4Td++1b9p?%(YIZo3lcee~ zX@lv)CBPW;?n|@1H5MFtpLl^@xJ}U|NgNx;ogkMQa#HfV`=?eP=Ry2KHerY4!Whpm zO})nKn{FPZTnMSQsUj*W^SQ|T=wa=X>C;y9qTYVK}fAsx5}JK z8n+gk?@Gd(^oRm|faB~F6r96HG}@^3pAAd26p@{-8JyFMCX$h(F4vIlSX!;>;%f>f zV4}9nnUk2j*TnZ%1oBU2Mx$*$3BwIqT!O1j)-CixyW* zH>|W!Ma!pKLj5ZMQ2RFY2%_|p%07sAvb=D7b0eJTWno%Pj+fE<`PH0k3B&fFP6P{% zlaxV0=;Pi=db#pguygkxPB8H@QxTY&LhgT;k>21K}y1|T9;zvG{b{(d9Y zLAkFfjx`WVoR^MO)0lXgR+E*&67dJCVC`1kFJ7e`66(!H`r6RcPi~j3vW(qpG)t8= zDNl?CrMzkBku; z$zbtCsMGH$z3hN_tttEO zFM)U{hG?OkJ){IijiT}*A>8yTHlL}`o3iPkdAHL8EgmN_1alO z+uPF0(8qs(g?&7MJztYx^&Gd-IuJJ;adu<+beA|bg>jgptMKBhu%+nKj%1fh%e-`c zlgW?6fv4Rje(ID*1Q934%GvCm9V9L-XVvG@mjyYE*DyPV0-{ScLR}5mL$As8QH;Y3 zItsJY`q~fw0)8Z_WmeLW6C!I)>Q89lB;@biSsek|LWymX5BKVTDj~lD-J(2!28N&hk8`+{UIyxBiW-jMko(1O>|~@d9rt*aRPOSYkq|ur#OfZrKW-2DVjvXdHaZ-D zd$uE6#azjZQ(^F0T&Zw7(f`%N`^{}Zj1u0bmkv;28T^jV)25M<5Z&oj6Lf)!TnLby zTo{VsiWaBt&vrI5d^T2rQ6wCEvh8|}LEB#8+0S2s%F8JmbVW>Ub-LNOoN%T|W#V~I zr9t1gNe^FCz;tzv|$i%57|DkBa0u$=Z0Y4caC~@nUgD?~bNf8f70PQzl=j zh=Zk?hbtf2Ixu@!f5cu?n$Bvh%tSi6Ev;DWe|dqmtE$%Nf2aM?>=qe|G6DUdy|j6& z#dBKU*ISn=+)h!n*U`rzKljyQbQ`M7r5bbcW>?YLqC)HLigh@pX>wnXIzLdvp}XF+ zHDs8cdWSV99p~AgT&+0w%mF*=J4G``GTp z+x!kRw#N#d$PuBlwPidCT%K;JxMfs_PM*t;^bkA|z2b*sA_80AOtb!Xx_?oz753^w zPoDT7jSc>;=Xu`jbLwVgYVujbv<}!~x0dSRD|U3fKcn}ORM6ku|DULNWXv`46 zj>Wp5+@86wssWj#+qb$vU8t?4cFtR0rd#S}m9Ep_gc~}U%P8@;66l+^^Wq1u_O@UJ zH-<(0!`yd4_+zWb+TdVyrO5BF#3~x)k&arGDX=za0uL{xx8MU1D@i_45Pa92fBt!+ zY~o}K^NQ%%s2aAsNGe}r=u39(6-YmWHyuzK?2yhKyH%2XA=XbAyyeE3aFJU5jodUb z>;PliUBg5O>R$LR)Gf1lj6I|^#a?c1=lFX|r1>{kd>RI}0Dz7{QftR3Kq!CSmf<~> zk`n@1d`41W1iKq~j=XxDyIP)qVgw&s{Y77THZYnRIyBZ!pv4+gSv&hezj>jNSv%oX z*%_GAct`WvMw>i?&&}HI>sknNA1&4qA(90~>gZqQL63>d*kbiMHRCI5 z+=A7n)zkE~`cC6&a?4~`(1t<7%<3RAt{QU?awQ2b0iK~(9#Xov;f<_9Y0^vvSL27I z*?kHdCt>gwHkFY0;q%DX#cd&&unwlV3OCKiLqJ8%E|kMir*s|wX#`g8;2*pv#Vx*0 z>sNNzlGlBrY4bp8#|!_!HA(X7PipQ~exJv{*n+R)Lvhby(e)A2OE!!vM}LXcP~AAe+F?{YXV|wbH4g`rPrUP)lX@XZcp0+(V70 zB}2{v$X2LNF}*+~!GT@ST_N?nU$Gy0%xD=0-rr|O95lh8eGO2&N(bBl8Ejmh?Qmz< zirsmn4v-khMQxm*k(N&l+%uD0fCbJ}7;8h34;*~xnJ818*Qt!r&x!19!09U#HU|>n z0uaBs;z*7I;e?5J3LgNGzgUeHmG|sfQpkYI&Fi__@BOWp@*kN@Nh`EpiPs3{c<*j& zX_8_d8H&%%!KF*7^~*rf(pT^~$EO~!xWyX0o`KNsfO0}v04I(awY9d4u+ofOV3m)@Yh8>KxHinxgKJomt8G@&v<6<2kxn2j+3DN4m! z^@iREwb{*WGng{8RPd=rEdPNDma+CB_pR811~#)nBdbbUQvAJK)M&6t8k#hQd;6z> z#G{f=j5u)b3?$P5w-n^+tzq27oIkWEdDwAc62vJyLvMYjX!=MCa+7OJt>d?%9p*xF zmK^SG;4iU{@gEPf+{#c6JDgK++fr`^@~D8JM4C}seokCGAE#Ip+z>AdA%F6^1cmKC z+8161_s+sj5VbQs7Vn6rS@WLT_YW@vwOy`uN4Ju?Lh)*zJE}Y12>sPD-hDTkHDvo> z{Y)b?)~hSP{{y=wkMTGshvM9K{C;v2cev54u|hO5la(LcQ(TJYu`DXZR_mkqCj-@p zjXjX0y5%{y+gCMrzV~$Yy%!|NG)(OpI|E1qiTf24rupN$w*PU_0Z_GdSEe`;-C%h(K1BD&b z7YCEzL;7<8rcp;}-l)GTQr2e6*H)%fg1PTk5+F)ib{l`~1)w`+09ummJXpL*3!cti zn#8=~r;JVcrNG;UH!6I<=L_whthHa;B{0ZNOE;EsOz7~agd1A1xs1U4n8x`2PC6J=sT1Do> zF@kpn*5KbchG`)JjWAGkLDAt+!sUOZIuKz|B8i3vq z!Zqg2E$<^w{d8XuWnZL=R=ZsKIkal>5ulVvgJf#XSr!%)AL7+;+QsQ(uS;3U`RnZE zYU#Qq+)QB}>pN|?(al@N^~LV-+1258!=Tm*nWsCEizC!Bt=4xt1K3piw^cFJ#`l$r zT&uKMs%Q!c3=k>|xHcjf; zF?F1-_Bbj?+`x5>cX6_~s^hu)Raog}v@1DbWra?18|`XV}JCJGAFf}m1NHUc`p9dE4YAB(L%t|S7aQ{51wa2gb*M|L4ggU>tq;cSs({FJ! zfNKlC#A?E1!TW5!``R7siu0=j^QtlDQ*K`32X*v?Bm$lib4NBPe1gANd9(Tp1@6FO z0dmZNHKDR(UfJA=fvLerJ&nM+Uc5Q~`s2v0<48V{zVLgCWydBU8I!66PLL2L-gjs4 z#R$}PULO<~3(c%ezQDL28TFTD_%&K-`OYvbX(&7!{cEl~1Fj)7m199;8*kEM}%7imvFEpb;X9_H^}p?k2k5#w2MzllVo>)hiT_{|*J$8$`5P(=Fl z)H;{CeXEM(hL-Klf<`sL`Ri3;=^9Cy^q@!nFnaw%_T~7rBfI1%&xsct{Y>m$AnOz zBMx!C5+5zssvHKNq^h(^xSsEDzf+hsKBWDnZ3KnM?Ry)WAxB}Qkj4KsZ@?E+pH z86w`|`*Goi*Jkt~VSggVwC|(HHpz%BcX?XC@|AP$HX*N3?di5BJ&kC~+2-KDkwlDQ&-rxGdCQkH2ccOWS zno60Eg8pkw{V$($Y(7_b_9T%HsnGJbT&=jbB@rK?Rd3!nD$JLFLN2Tn7$5`vK2Z4N z6lTZ8>0!<-S$CU^na+C}kfD^cw2ppnj5rTC>O;>tA%Pon$IY?l`SxgE zL)3{2^pUK$qRZl-U3n=`I7bcOMPTCvw=Ulg58G@gPw-{ zanCHZStxs}xxj@|lo3S9KpRrt3tQ4L`M8awcb4|F1YZHh(OH` zlE;v`N2b@l_cHyA^H_<4`~Ht165T-A+wgm4#1)>z&2b^E^O6h{8K|%0MB6?RWT;Nd zU=iO<+_TvWP^A&GSls+#!h34L*31 z!xjgD2#w#2QJ42&M>x;eN94r`wT#g@i|iV>M^z_vDN#jj>>j{?{;60r<3Ig#cGkB+ z?^93Pyv)`DFbufZh6qDWwWZzsE1svLfs?X^MU>6i zNp)K4I=Y!@_mcE+1rhlh5$l?3QL7dql**Puoj4nJYVH8(Oq>lQZuX6(40!}*XduV1 zAy}N_PV^RY%6o8ledd#$+E%7W`&lD>Ru3!SlkA3?H!8($E842yVLgpBji0SE9!1D$ z-rewfDc^uIAvpxhI+$O~7xoOcv6${VErZM5n7usqWA!&6>2^q5wlWpEsK&_%Z; zvrW<5A1Fv{H6Un}qIcFnC*^x)!ZqO6@(KM5E1uK0vvgL+@6Kb5YT(8ys?}i!eq^D2 zKbAtFPL-*t{&^n%-1i|P)2+{fx zd%(POTzVICwJ9Y|-Y*?ZU-(j?HenzQ&zCv)+Gu>fSvBfUa)&nJBDD~PIE8(iKYA!% zP&nyvrB+6ES1SJQ&Q`%2BXi2gLqjWa4Q;!Wj990z2l9PrMJub+TVv`7({%CTYNwbI zFY+ z4_lEKd%{#!zEj>>&Djgd^3PPUo6|k%rF~I)Y+G7ljn!oE&EyVDb5>;;mt9#&N?_2q z*X+?YUnd28nBTwCpYsveO(px)Ek%r493iDV7rPTZ5MG7T{SBgbkAL9O$dcJ16HZlv z-7fmMgZiFEHQ0a5XL7xgeiOghvxu^(GF&(86+4z;XODu{?{Cj@@43{ntCYJhD(z9G zR#g~Y>0H?+NQIbkWr(GYMa3|ltX<1qGpoNFyig#k6(k&*>tM=?d!<<@bxrmMt-d zm{r0ksGg-)q_BoE@yqVHu(#}@%ZGNg4q1wj-l+Ipkj1-JJ0zLg&7d1vPh7bro{FVs zl-`u~B^`8J2i#FHuzO*C4wgn8;Fi&yj4b8N@9;fB@}c&Kj7!{PJTfs5vv9vT?3`#< zAw5~3F)QLq9c0U?zuB%vbw`|{#hefQNB&C6(2S8+7A36k`OU9%yrD4D4*XtzRqDLg z-QP3hLQ%_8Y*iuk(HsxEct~{|F^C5p2Nd6eGx_flybL0Sq|SH4ls9wkpVjL-a z)b#_~|JS2!$uPdHhb08~H$Hqei5Q>NP3h|aG)pjP!}3$N0OchiUD(FpS72sia_ZP z%rki(Zk`Z1Hq{RX-xxVYSWX{xKy_xIGSw38+we0-)OIXC0xEdRN9YsfG_VZW>Sb<`iYY=rB7nNU0M5V zAgIT+(yNP(dm`fPzeTw*CTEVqx3L5_8X#<0O0=_~XICbO^8#yYR)_Pig5a(2u zsiYzG7n4GM!hU-MaJE#_gl4cPM4=hYFRzi5W?ee0(cIAFq3`6wwdiVGyxZBnWf@uW zbHA!Kw6t6LhfpLbZmt*l9c{Z~=v}O@V5{(3TL0SZ!9-`9x^tK%JNq8xIJ7}-xW7~y zb@Tc?8&(r|9Gv_h8nGt7Q(^T);MvlWV^^5j_&#*|JxWw?&0p(vQ3KM=vE}(!Q38#b z?(HhpS1D&P?MqdP2PZzr6~D^JcDd0lZ@Ix4{`^XY9YZ&jU|!6-+ymJsLHdJjzixS! zU*_+j(6tfc1&v4=wgG=WJtM~}*H;7*!oS(c=R@a*C|9m0ep60J*H!-99bsckYdn*t zf&*AQ%s%|ZmifPiW;!S8V6Mey2SM^x@FsE;!ZNWJ_A`EJM#Ss=CO;j90^7H9o~-eT zdu9iZ4+&@tkba5o)hoLxK(HJH{aPgwRr%Y`(GS(Frieu5gmNjE_g(%&rfm<}-8``Wr) zJdJsv@2XW&(E<&nBPE=U)6A`V(EUqKw#m0lyCL+}Yl5=Px=tpl4Ys_61<4l@Z+fhw z>p=B$m`h?x+*`tn`NOQA{UhclpGx#fh)T2s%Gnp5OEsYHLPh>wOVUU;O=R27X+TYC({Xj>zsQDz_d##WKCuC%Qp57mqXG5JFrlBi zLU4n^8v#md#22~YT$&cVS73x+p*I_C`6fsasl+_;sy5DE%HDVZaS8W^x|!SWh4v)6 zcfyB1k384;W-Tf)e=3NA+4h%ZudcalVz^_|9JPolFFceT6?>$@6vz`A@!F#1Us)|P z-)-y+Rs`nra~_;7TVE-#Q5N*rVBtQq(`WOfae|>a0%d2PPOg}Y4{68AjMj}UF1BfM zubzfhP-6vVix-#Yi52?z*=~YmYQlP~6VrqN;=JICTiS}=L`Q>f+r0$lqsig=%wF1& zcSPurv4cVCpBr-~CLb7n#+};hc-uBmE$Ghlmoxnt>Fi30g7~EB?|a`)&!>p`*^Cz) z&O4v8D{9;m1ss9jX18U7#u#1I!ps*dNR1c?wj@@F>%Zy>e{ajHSbvuvYnV*u8|piG z;H^&xh;ln*Kb-ZIk%*~|T3)bHO6xY#8z6+DvXa>{Ki(fOCc%9W>jDjRYeT2(i;22K z3r=WDqMv=~{b$4sXR8IBQ&B1zqjBi)lR7#S@$|06e1Toxt}YD$QME1}%tJB#eikTc zARKYZl!}f%Fb;W-TYP;a>U(p+bH_$qgbk5RC4wY9AV%id@*5b!LDK>*J-Cd zIfd%8Q3N%buaRXGSB01xPrazcd!u2XB6fRna<$lVoVa#FwfIx*`!-S|h75r~s-u`T z!lB~kO~dhc#lZ&3P3e1%2@=G{WaG+Ok`0s<72t>7!i007>tSKUr7&xQYjiqsW1%gx zQt!w4&9L!}aRThu2pj8bB0UF*L}dcprZttBCB|l>S~TQ8zM;Bp-@^LKkz`uaR-#p+ z&BH$94kYUdJZK#1_{6@zRddHs(#ugdI(MwvHL^WsYCPZbPYd27+WC<{fuI1x3PKEC?i;#)AQle)F@b6E?w zy~RC)>z2+{c%KxqY&lGCr#L-P@E%_A(?wjADR~dBa!86(^_#|LiTn^-5aS9;E`RIT*!08d zi09T=o6yI2k&z$PlKaYcO%^C~)k*$~AF}C%XfW#tMj6j;Pqz{40uczgjs#a~X+r~J z(c4%*TpUk)GOzzd&Fiii(M~3qO=4?8g@MaSK72EzrjW2i_K+;Y92xn&x2)@s=%JB% zNWVe3r;gP1mzP-bE2+;-Wl}Vcme|Q--a`)}#50;Qg{KA3F1H=dN3nx^Tq*7Rt2Ys` zKzJP$3*_MEyfXI>Dhnf^g-pQar!fc(|5V#~_~LXY!pDJMiwPTQiE%akFje@$rL~4q zdsq&p_{WJqaNXMyap+m8<>)rnX+!f8R+rd~wS~h(em{3e^KS>2;cL1bfG@5Y8%B$ltt<-GoTRE0_WU?PQg()Le<-Ddi$cbZlT@|_#GD6f+N@5#h;kYU*wXH+0cdR61Jo`0z2E)#j2e34w_G8$ z5k)E+iL~MkiD(E6uS>2;-`8x^Uwh3x*A8-;vz=PKwy-Tj+H2^~!M{9@B{T~4e_7;* z=8ju0!fMf7uCKW({ppPCs89OqOCKC#h6UQBfw(r>PqnG1kkI-$>PN3Jnhc7Z!>UfJ1F@yCtHvvNgR zS0laTK|Hj;uA82 zp(k9}-IXzPO;=jYG$1K{>G6?T6tN!uA+;J=opdI6s&sGAOgVtx$^2#QUCaA~b8z3E@+y-*l|JTl9JXc1?yMc#u(a5D z(&EbGsFOOSvDZuArzYFwxu9m4?$?teJ+Js-;W49vt}ZSUe8LNtJmufzC*UdA zL`U=;tm{!+YOJ=>ZwtdsFYm;5r;U8~Bi6Hxt0)MZ;!$PH!4XSN?BD0yi9ysz8Uc$xm07=H*>9O#g`pgkca>h7j>1rP>R=IkUF zRVx@$7Ke%=HaYZ&sGmPo67}=pG54P9kcGh@$wW@7{Uxb{f~f+WB`Z^?j5$=v*uA-H z5AxfVss?n+SG3m~f?XIWQT7pP=4t$`m@6=>)|z3cc%%e9AMhWsO*yg)qev4LA0Pb* z{#akU)`NBt{=Q^mhGU`$=|+mk%l_cKb^J=wI{Wt2v90xGVfeb_5{|FcOn11r92J{x zqpa?yAVW8S3{ zig)k2kR?x)7am<$>Z$5EdSk9ILlFANqDMUL*?MSgyxu-;$@RpaafYAn3Zq|B*la3= zo1j>B?Q9#pDQM0fciw)8v3L%?N1VHDGI~s_oahQ-QEPMd-JCx3<8G!K1%{oz<$XF0 zOsx7>;%sEQy3Q5)9@tbb`th!4YR_Yz7u**TC_`Z|8dWL4pa5J_e1{DvFoH3x7+|ya?`G$(|>;aOSR#w;cZ|u@R`6y{j!CaDRNmXe}-au6h zhoPmY(>FEq)XhFs0jM)q67qY`mywY0y#1h@h6Z)Z5v+I*Jek2GK^<( z+9^{85}+j%#WaWgWXf@dCa6kchKqGlEAuFDFznl9s*av5$*M_7f&`IY@-J(sj(PKw zUbP?HRE1S?5q`u=H%9z_w3cIjfda_{gR%V6XebeF;+W~m;3i>4qaoh##=?GVw&B@x zNZ|SYImvIv6k($Ihw~`D;!)K&(Y-E_{>6M6HF$yb7VOQDqWwopM*Fz!D7ve0*KsO`xm4x)kCyS1ulc9&B&2_bwKQ&eBU zp8uHUw8#lTiPOU-ps3hPcJ7xCvV~JvAoYZ)rkR_TV%Jl@Ed|Y5q^xZ=)Z7j>tgn+@ zr>*6$Lt(`D?4VZtMR?-G_}c<}Bqd?-l?V^S*ucWWz&+I($f*>w+Vu?r-n+n+NLNyo@*Gq>E`*9ZqdTZ0P zJDDiGn=X(KMUu!SGM*cl`VcjOlB#KB_ivtCq48{PV~5QlKU(Ha;rPazVLqdN6{dE3 z+WqVjzwT5Yh|nwrfSNF~GH^_~n3ZPE{W``uqK{N#56`z4bU=r{7m^tPIP@ z&3RTjyAl4vnKhZs&Q3ZVPTH0J@ZrD4{K1U3R%dSkzhdcvloe0rdW6`!{|~D zovPiY(e1)Wan~Y6Eo`pD$@&NvOgD?;EsAWT;3v+>{_aLl+a#_xEBX2>NfqZ$ybN9P zSZUY8$Ojts%<(H25zk6Yp~iSohhl6f*8w^=F&6V>dy3P`_4=H~eVMPmWnVH+Cu*@i zVIvGKb^feVRBCgk+g9b9s{u)3cR@u?E@La8>0;CyHzGR4>{q`W=lZ$(C|lI$O!hOW zI^V(WanbD=&cPZbt3w}u6B~D6v^hsFm$Cc)rP9>1o%y!zV>~Dk6v`Va&i|5 zeYD;^t~gz+deoAK2PlIlHZHQn=KM+cQJ{vhdM$1Endn*@8HecTy$!tzp{5Wlj8{86 z5KY7&xt*7q<0~jwpU*n0n9!5n!L{{B9;j<2ERYf`)~JenttEdnwQ|b=xODi^wl+=uRzxVjtiQX$?ft(=OQ-t|0|yr6j?<|Fu{(~G$`c-b?JuZJQwtB>01bO3o_fGe6acGw#INV%js z!TZg6y&Tk@`|v=tlz(zKq~of3pY@QH{8w2-Anl=}-QCm?w9ltj`ntkTXGzNEr7}_v z7{}F>k^4`M9pJQnL?F-kpjTmn_JdP=lSc5>;^-R{C86XJ|6d5vX}QXwxEfDT#-@&f z!&G`K{$R7v>Q})Lb8o4e#gQfZGarOxItNLAc=Gst`EWLtx#RO@VnO8M-FmT3ag5%+ zD{v6bpOK838<7ne$30+2Ho!qJ?Q6&WMEk^;L|5#L)EstY-Bd;T>o%Djb@ki>+<~gsz*Km5Wf>9n)P3{TNP2>8kS8Nfy zD>!4i+I>%(F;}ugdyJgMk6ZlN5kHn-gh;vvt!BLZVYAGFY*6aDPX4eDm70Hp-I5j| zdPfOxJ$Pct8%$LfdIICr=F1J9E#6{ph%VsGC)Gb35WByj{z%&kNo(%r9{dTP`ML5Q z7X8Y~xVV7%HL1W&$%E}^Xcq}5f66Q1Jgf>iRg-eHMw&^5Wa(Pk2Wp>?<_$g7FaI!! z7b2KLbX3t2QOmTCh7>Vc2+(b_+4GKCAG4Z*++J^9P&iXqNKb_Nt12UT#v!zv)~h4@ zA<4cuZam0WsJQmbeX6Qi$#V=7wy%&KGkAASzPW#a<7k4L2qTP*7t;5QD5x_nol!}J z+0&isFASKtA*nU^QCy_bD6S_(Sf#+&7E1O(uigdU-D>~Hn|tV&Uorc+>nOe5Q?ND3VqEq!^ zPxX`3^LoZHNBBai-;1m@M)I z-2mTZSTN)-D&-FmIiUkUW^+G@(5yG%FU#POO8<_uc^}4@OSjPOF8yN@P~FV-xI-4S zx=3#qQN*m6R7$1y&43j(Rb#+m5VXxpnE9EaWcYwJsmgjD~850vA+cW$vI#_ zlK)mZm;Q`HXAr~IO`ORrW$_Pf^_)^OB$jwqitZ0NB4!Zj9L>S0$4Da(Zd{{2v!;Um ze9{1?BR1aO?#sdjqzciUy~Q%6=KXY73>}vNLr@xxMPvE{I5&%+y1jR4UD;oJux<^i z%Mh#DWn??Whl{j<)-&URTbc!cM>4h|e9OgM?S!3p5~=6M;Q3@oIlZ#&SW6t#a{GP;L)!ioef9SC-Y7==cJHP~oj9{J;(5#8R_i+{ynYy5j#HDZ zP8q*O8sHjm*u){2ZgXt=oB?5HlQFzIivK=|=vze0it&BnA8PeMmH;?)G~wG8qp5j& zcboB5gew4rg)jtj2{6FglBA}3?b?)`drXQkCQ2+%uzgBXa1&(vSditO74M|8+Owh6Nw~ zIi77I;mA#zeH{Rhx*_-vYPph_j%)IBa5D0G4nYif+M=-Fib`=g1Xf#A3Z>O2sT2G2 zRQ@GTelbS|{891Qep5qY`EdabqC63XrYDdkY+Pg~d#oqm=4Vj3zSQcJ>aMC^WB&`+ zrbu>z*jV6>(`;N?)%9HB(Q_)oT1lHERCGnXQ1fz|qMB@`Jn9X2w^LfHrrr3kJV9Nr zivtXAu)4#9{7?M~9*g$eN*e%ldE4#H)>KYP>axQ)V)cYR9o?DO@ZUOJ)?&eT+NK9$sW`Qd zVJBkB@15{RsJUrl`I?qjLlW@6JU{|95!gw?bIX1*zgO2HGE^|zg!x`DumWAb1CSHP z?X8!GN%w(uAie~PXimyt&qzp& z6>bTbQ9tbf5I4cD(KJ$kA)E!~DM**R7GTrfjtB~v^@xg2iuYH6;%pz3_Sx5_#W(@Y zSjn^B(2h+0`-^Ovb$FhE@NpOlX zOn5--c1vZa$Em)+!zuDStbJ#}@#{65QnO*QWKU`-94;*K-gxS$f2jugk97?21!%S@ zl_n2wg*9+sr!wM*H06u@&*Kjc0if4?mfj<-z<;r`Qoo&-AM@+o)8>dwvU-ab>O5R z@b5Fqj(SlVwRElRH-Cc`#6I~iP}Emk{zdB0EZ~i=@={ITgh*R%JSLzFkAr zIk;u^Ws6oL4x#ArZ*EcdIZ;XT<6c5zdo9_pLf(nPr7K^+IUNr^b}Ui3jsyw-;Ro*e zAt&D%Q7`08ziJ2V@n3P{9J?FynI9|$Vk;vZHhPZ`;nbo~dY8b!9Nh#URp6%PUGJ_G z3#lEn_L$W7sBW_wybxyp-qu|nx8aV(j+M%tJP?{<+Uwl@s}KYa}B;h+$_`A-9+mj{uD! z9<2ekqp=T)${r-TCrpz*G#uY*he~`0qAJExnYj3V2l6y7^H=)w-l9{MIFS-NG!Sl( zC>!GC8-yX;G^a+P=j;3 zqJCQs8|MIV0rW}=Uq?2Wb2M@vzCN?A1}JOp+iZu(GAF>38dtI~YsbD8EtQr_T4S=& zCbS91cLc?|*%1}7l=wcK_Aa?UVyg06VB2~e%?f1wW{h* z1>JC$N@8p_aRppQu60 z94L}QE2(w+FI|~wEXWXbC{>tD+Elv2+>PSby3*Q1rH5D|&%Os6Lr#J4`2CshBp+dh z;q4wdB4`xnvc@lo^j|fAxng?rIo;+pf#*Y>+?rA8lf#lf_Q_40`_=Xujt8!Qh9WWf z2<%=Ju{kfwGL`%AEcvU|1>YIM-tqYq#}1O*w1P&eW5}QzT&Cu$W z3DhIt`P%=x6K{zz9;L@nYx_LdK| z!DNqqUOI%H6&`;!p{}W~9&cD3Cot^}HJnb35;@blN?I|li@C6hr=fs3z*2w}g@se9 zM8cytG~!R}9Lw~p9gHpN;gK^%4E;^}u#-iQApbHFQ>XD1?7GLf>~vh05$X)5u?pr- zYAA21JiE8u>Z}IW!^B#S`yxqxTP)lr?P{z?jRwEp-x8h3Yo%7h3Iw?%4dyY|qu68D zN;LMOj4K5{=%rnPPWhYRk&;DiOmay>HYZ);nUI#HXUV>h=5)MfLD7XjaFw$v?q;oY z`VJZ#h`r_Zx!VDLM&74^(kctuzaHWLdof7!pnxFcA^(e#m`Za5mp_xG)|PRX5~H5I z>{X*hV5QAGq=otkjX45UmZ80fjK4hL`EouPHS1@nj^SwifXOj`5YbefYbep*9&4oes#^LuHX%+8D*h5~bvd?>AQ-AHXZ=Bo;C;|@)&@{& zmi5H?Ww@!OMrUubW|3qEt%z%RwqzgV14ItQCcN?9RQx|0g3LYlZ%!}awdbsPt%kO= zI$%-~is~IDu>Nh-uNprhDM6g%v3EY|GTcs#hf-@on#uZ7a`5XKFp5mRdfsHaBr(JN zVgAaAAYea+USaqmoO@N_X4XCpwlJXkoZ)x8uN%>WD_n2w3{e5Evp=M_2o*b9GY5Nc zQb2EW01CyoRCp5aIotDyuq5HfY*!v`Q0alzIB)2exyWOzqyBSwJ;!!}ZsuVgus?DZ zNis1gLFHH8FkEb)QOV@s)F$#+34{EjE4s@47lA7ycV8Sv3p5}2U^Df2E!g9Z=tYH~ z@_mn6vS9-#S0=z}b1)0(?xIsH+XYo)<_k!?F7Dxtefht0=l}Q_@B+74u-Jvr^m%ZA zPC?PL>YjA&K%L~y#C@KJ4{vhGaSQl8u|Qkt3f+kXAyyXi~bM1g)JPo*q3UVMqrmR<$5s6ySIhr0|!EA}F&BKX< zFF7J!U}o}&Sr7IE5FJrp!B#BTM`&7KWg8Yv++4yXJ=XIIRj4RkZze@)L<=H{^y^-P z(^iPuY>@+pgO|ex6QK^9pvK7-zrDVW9T&hN=$JB$A0%65HcmDR@ofuO8v^f8L6Vgm5LW&DAq~Vv)t5j>F6)dn)4o2lg;Y1 z!tXgH3@Awp;3|g;@*wHnIs(npFaZYSi#VF!QFh@;Xnm-gxUnjlq@Oc!BmC z(-`s%HZk8Z28b+}1eQTmTqNPJ96*6y!_(^8c!Yc$4*l8+FYceo;mxQxU>Xi7omKvS z^6Y>+I)@=?qO3q>I3ORL(+OOgat)zxe>-DX%rtX9EWpsy4dr>LB^~*|^>_!x^UpU* zsj2{m{waUKWwYa6s(d#s=waaG1#9{ACi{g2`(>he9~!Ifl0TpIuNKb2t>bK;zkX-H zINGe7tOlR=b!q*m3 zsn`bKP(MGrK3?XPvO!(_lVoO$8^%pSZ@Xf~uHXk}U~*J1ek3)S8>Ie}1k@xtAnY)3 zI@y6}ZSQI4`hG--uduhOeTz{n(f<~d;dJyb;FcgbL`m0huCi2p7q{fo0{|lWi4HD( zGZq}ozAK;k#`!HTO(7*jq5j>G_?#MYxO(MRJ#a32YAg1x#I((lQFZlUcuUjw7h~y*(VL_&} zdU?g9XHY2IsW4JkGng<4gIikn0&0!bHiWsNG}pkSkqof8s0i@5TTT&x!%jp zp8fkV!(aikAtwrAQl`jUWkD@?7OIvD@! znB`%JvB5v`%=B-~z)?wE7c|;S8B^e~J^&f5L`vXeUjco`(@(GFLlY28c&GWbB18Zk z`5o#qP9A~wL`7}W!mx$-4WVF9@OgaIz^YvNS*SbTiEL7Ll|0Vsawx0=RIXdiVi&^B zB_EJOx4Ltp()f?2Dd6Mm+eCf`El1z5?XQ!u+q+%+bz8Bg@UVs`X_z!T1G0k4*)z3m z>Zm8->Xm}^#Gl)Tos^V#oFAp9v-ppU9I4wloEt@KmH?_a`Ph zu+kiK%8goFHMsJwigF5crZR;s1bTRV7)?7P3)MmjgoAYp+AV8dVhNbZeIs|9E$t{z z+W3iWi5Ai)HXyJc1Z3Yr2{@{!p6uT9FIXw&W!0towRmCY;s&I*?#Xx5h7>pu`|`dQ z;bC9Vu)QCCBg4~S*q0@2a8S~w@tA*T-tjQ8;YF$@E2ZzF0E?A3{J5KR54KeTFDUlu z-n6qLHo)!544MLan*PoH=x;op_ymrT_^=@4p3-aML6NLl)h*gqgIi4OD^E!|IZ0e3 zH3OUDr;UeKFxws3QuzJnzvqPX*w9 zX`h~Hhx}fnqbnCS3x1;v6=i2j<7CasP{5zD=zZx^MtQp_OR{XS7PnuUD^~4ueO97n z-#u~l(b!}8$=U6b92`bSm$7KKUHHLzUs8PZ7+CM8IYYc{V`PYBqJ_R*9w|jN8n`-O zM@h)julfO9-OZ1OXeZ`qJn>qZ33&#d3_S7@#DK8!YabbH+VGsW0{oD6Qr5cLbbrkEv)?YQN zYt^S=4fw+h_N8*rcf1FM51#fa+|9Wy?8?JajL25==dAGxkGv@y?X)``uq> z0a#}wdobKwNV~V{Az)iY^#WgpvPbdnbhpam=+)-8$MRP$mO_Gh;FSK{g0<`{O|e~I zfYaNfh0i+{ub4f?TMM!T6PqsMglXvFiwc~AyHuoaOQ)C96OB+sXzl%alD&mqS_#KS znxC&NIe)?h07IUh!tSh(_^;t#xFlj(Hd{tjP~#=Tr%5ll{g!Q<*pO3`I<^3~_1EEj z)>3O|e2?cXXd4oQZe6V=y?ygyVwQTT$Gls;-6?043F!>;f%YzDQ*r2iZa6;UUeYVo z$4FGF6-e8;7DJE+DF1pc{ zI$or|L-V{8!Zvx~b2#p?{MY|2{Q^yjM!xF9)@wHa9(&?ve*dP`N_7(4O>H>v3NP;s zd9#<_R=h%$%EaY!{i8*MS$}mVLnnD~4QBOY4)1$={ULrBs&qH*7HRo;d{ z7{1oZq5c6g{6)UES49f(97WO)!eBZVLG0&qHO@Ir8stK*zmqu1zHBGt*%PDH-(E_! zSTBTrW}^~H$=@~v{{ZxA2S@wG8Cl$(Ddn}+yj$1DN9f+(bDI3{<1`p=&`WC}YISsf z@E)~t?Fofa$oaxu;aF_E9GT``85u=R$*A!3i8x|JuNRzXbKLv#L80X6(y3AkeAeCx{d{8=h zhsHDM1rpktV(&Fpko>z6{sGu>+SO-X(@%RkvrS-n;n3^4#yeOP_FqqA zyS(@^16tN94|4|$QVzQ>1SPq`n|ppG@k`*YcDL}*b_`T#P_vcjUVF*ev7kLx5#0LW zp848EQcZUcbcp zYc*Pyt!qnn0`C2^U#{kZG#WW}Z+g2A&w^wm5{>YE6>dh-?p9|SNTOLvzB_rI{n>i6 z?N!(~vz!;-4A|bQjPlkdKYV(N=J9M?_pY*wV0p=(oHvu2J_U9)hY0w1-M2A{8rMm0 z45SE6qyX5w!D7ChRHL+8_)k?#2LI+NWJBRRtp7f~+GWKv^|iTUv~pDy6a^u>GgWmi zJL5H0XzGj))R?xFQr7@QX5H*LpU!KJViz{d=6q$$r!N#OVrz?!;yMaeSz?e|%`=ps ziYL#7H4Ow&wE4*R9nzn2naL$^nLU3sPWnBoec2S=(rgG`w(^+b|NVl_3i@rb;6%rL zc%suJJPTMY`L9x+QNuHi^e$9694n1Pj+#|Gv1$V|b;ZI+?UM*7n;U6C3vh@i!|?23 z&RG?)*8Wj{GORFXDSDicY?ob6vKDp#8owPt0$?s0!r-`z&02dV+?AJDkiSHwI)ZBD z^qs9-p@8jB!sF^a@2*}hCw!r5WvD8Ugq+tE`P>f{-y#9@CgggZuZe_mq>8a}$ze=k z@lD#**H|@#LgZ&YS0`M-4Z!=DgtY;THFRPUKTB7vKSJuFsjp0Uj-yK9L;zvMZD3ID z?p6(C(2V=~^(|YuV9cvnnaX!T3}4>Sn1kW`TRsn;du&YvMed(!Y+;fCqWIq-Yk8_s z(~<}Q_I5Qf$teH*P>QD`;|_+-gU#%b|FApDW<>cVA%*#nCgrKQ%gf%Cck6khs;MtmZA62?MMJT!Jx8ec@s~doi{fSG!C-EJbsZ!5F)UA(qq8d1(O>ur_-jU`V zS~hQVE@17fq`L)l#6O^g$RV~U=(H-w`DBQWDZGLAkfo-~AK8MQ^m2AXRCDbckWU!a zzyrPL0_d@VgSkW9gXEH{Vgq=hC_8}n7o26{UFv}WUr+b;K5!$_6zfXket`1V(*E`F z=uv-S7KH~n0S1h#HGU7fm$q}Jp#NNJXFxwc-I<7r z?WSoc=-|{YYtvnfckxmZ9pWk{{Weo!9h(+rf@VgdDEUyYk1n15HK5cR73cUL=eaDULI9QEu-!R?;~wl8caaR6MaTg7)+K0PGg0I}>(o~W_tJcIk! z)7O=`tSfNOIbA1c@FJ-1)N)Nl!w?X4T>h#v3gf=cR1ep5G)D`%&4*jE?tuwTHhlT~ zY#_=N5Y>y7H%I!8^PR+;E3X6LG+agaP|ENlgYqoUGWhB)yg4ZsHRII?D4;A$-RZR@tO)ZJitIJ`DY!?o&3V z{m`yVl{5M~G&*i`h)m%vX?WsBn(x)t9z4ew?7Fvcpm16P0%`60x!SCbDZ>Cfl)TRj zrD+VF-PVTrY+bMSfQ{2^B)QPdYB4-4mkw46Zr6F8zi(;@*6HxE5{h&WSa?*=>tyW zcHvq3Z1JeGn*OTT+&EpBVTb4?se^d`)E%>F%ZMgGGrXw-Q@S{YD(Zgai!^o9F1ra+0~ zVowb)_12xsxXyrFs!cZbKsWG)z!6VG4>UpMCEq0Ci0Y=QpWbZVeIY&v?Wq$DSb1DL zMtt)HNH3jeWg}0<6X{Z;=~=XGV%L~4w+RDB8j3AlfeDTQKaH^{);}&$0f|bBAy3~s z02ZX%jlA3{i?Q*rylRfSrlF6E_a-R%I{`n;`=|RhFEjh#R>LpTvtcQ0%{`J?NVe9> zu}Hj!3bEx+ntuFfyS8>@S9D>q_tI<8FZ@E)n~+zx?E*Rk{Mxxn&dW5td6&uCnP@gZ zWeA(8b;&2%Nd?ZZ@4LQG=gRYz{(+}4_B+A2qbP7KBxTh-XrHdYZx(6*?)ny5aKE4BAIJx(o@27Z z_sny%Z?fjFF;|Tg7sDuWp%?0H`q4KoXcJ9){Ac{Sahg&{^+ zSDWFJ?%a&$`6hXWz3A1}bvVP5C(Cei%Nh_L^1l3R3qU}+dx;RhS%vujKXU~Y#RKKb z@-hQt?RsNkkZF=YH`kh_?9&Oj1`>XaV*3ckbj>HeUTE}bXrH}0-68r4W1;Qti7D_U zSmwFN6NDG^-w&osH1>B(cxbbFg7a2yqdx`GG96#=4uqKdqrfiGFIdtOS{h1C+htN- z9VC_)fn{v$HoxJ%RSj!fa3|hFs{AY{0k8I9+$p_)HuH47M>#o1ycp>E*FVgvy{F?) zn+2lBItmd_er=`v$LBZ4133{UE%m(B_wab(1596!*LH5By|G#@uAOyFl0g0Y^71S+g2r#>JVcp2p5BRn#S7 zouJ@Qv{T%e+t&TA)op)%?%dg_wCZGgVw-f<`?!yAm2S8ZXdLD=o37L+N=-ZYg(pH7 zTHettf@k;7HG4#z(Jw^!UA4{587(_@K)y)W=IXgVwpZw8N@h38RPt2@=&UAQ&n(<& z&@b4Q#l>>!u~u%1=!A{qVt13C-`@<}zfE{1%w~v>O1F3q)E={g887uivB+Cm z)oTmGmuj&R|If%x z<_Rnx3Uw-9_023>sjBs^zY`a+yVQWhuiIFwvrn-Ym$4~!zup9Wnk{yz?vL+ZTGr7T^c9=X$Hzwq00|qn=g;^{*2BR`|6N zZaKcRR6paX*_By>EPDFs-QozF3_X4m$gRxTMz%aAn(LEOmm+i+v3gm)hGq>3$c^GP z^6`|gc}kora@TJj}>v2~1&Xm4WT4TED^&5uEq4XR&7SD%1fq|D00CrjvJF|Z-d z2+e(rvhprVDr!5a$Jzw~cdX|X)90+rbx|^}oapDX9k5wW>B;-_6x57}%tAeW`u()Y zZ`6zD(Dyb}o8T|BXOYM$x(3*DGK=Wa!s9PldOP^g^VCt7VY)YQ^ckhoE!N8CVHZKW z+h+B4Gq6j0^l=n93L^Vco0tX4QEc@zp+9_v?n}XkEPU@qb`rO08TSd|vKk$St>ehG zGc=miZy$try7)GDoh!?*U->l1(jmt=+yYN*Xz{*g;}OBVXs+B^`%6&lGw|##mZ4+V zR!IfPQvX#vK{p?}Qlq_yr{T8X*3}RF&I(~x(rhoqfmn^tXerwcU|7Zz(p4)vqUZ5# z%Xi%ES;PK42t<}QeyGO=z?+cG0K0W6L_7;sic=BPBL1(${9i9ejS!>bVur0Q{~u>p z8CBJ~wFME0gXpFNM39t51w}wW6qLpwHjUB^(v5-&DAL^uNC-&h1_>pU?oR2Ju5Ye$ zujk(Ho*4JsKMuzk$FbLX*L-I@^O?`=XrFm(^_e?&4o$@=9qptqMM7k%Un`%|blmzGI5!q>8;hE*tI0j@%H*C zAY069KjmA7tvHcSMpN)aD`l>?ix}*xu@ASDw|eqb4p2q7Z1&Rw1#{8 znPYNWF2*V@tZ1iA`=Jr9;X`J%)z=&@p%O}O#vcdPU(d>G&lqkSec&RbI)26Ex^MOq zKiosM{ajTnu$dNz=`fsQ-Ji>-!tuKrM;iTq_*FQcfj{(I6*gQNX9H}y)l ziZ|r=6fQJ2BM(1^hQA0+-ijU8YNN{{_6;LwcEGjKgpxCFCOI&GPG29onkfGeiKiaz z^|`7>x{=o>N_J`Lh`y#jT{u*IwAWY2aLF=q-LE!~((HvHN263l;o)E)L zgK{$+<0s-z(Ia-Rm`U`{y-|{7eU0;4Siay6$LoE=y4SCA+wD5Je(7K=>Y{y_iB4hq zMr8-G+4_@|XcbN*>6B&|(@!h>@Cq5vusUztgf4a|aIL{a^yki%sl?o!p|ns_;Sz~@ ziB5Nm*?mpE>^!oksqSWd%j}k4s);Ta%Js{b&cL95c#r(%TiXwB8=jodd+Vmkh~q#1 zb~j!2!{Phpo;hxJU`JTRO;xZkNPkUMD>DC9Y!_D4=%>``({z78(<-mTH zj-PPIz0{oG&FvDaZG^pFtiHjSyRpb1U0_8u&|2if=*(WGa2Te1X*;l0L|>%J z@A{1wVeY&Qi4T|G)MGoIBgg-e$Mg+Pxo<8*y!w0lUBfslj*__{X<={|SdTAoOZI z*{RF7yZswubnc(d#4;x0HLg;6CZ*$qKg!!n@_eel+au&UQ9sj3evU;bpL in7kM3v`>vz*o!y`dQN)W! z_Ln+5@@a@&iAb4G5;CCzBIv)dYDv?g-^u#jXXeN~r9p+Z^Y1|KwqC;$kE~_Iy81HR ziF};*I$bV(_@x;w%JXc>wms_!)1vf|E9&_lZa#X_@Ho)B;Hmi8u#>_+{R-bRkN!^$ zM+ZBbU8e~4IOa&uR0Y@CV`}x=e3_G>6f-#2)Bb+>%m?d$K$%4liBH6LHJ*NfeWw8b zTiQX2kBj8Z=v{Z6J(1jEHL4Gf6jDwFp6G3V$J4hax7#*HLOB~e-k39#Vi!`Pz@Jfi zM@fHC;d%CE9!_dtm*U6>uN*HpnFB8Br1vt{qAASySE75WYT;Y z>(9;%R%va#q~&ToMA&y$B@!uyB~*u;$M`Uyyf8jdYVb95I<6d=?gzI&|W^lkDxZriWLS^}<5F|`?; zy;kx8QyzDHV<_l=rzy2}-*74+X8PTePCnG*iWfg(8B{N&d^JL_2gmh5%WK~tpS(1m zm%4_;kbGTXnHV$d_t-y%>azTXO}zPioH$<(o0P;;t(lef>N=AGMCdY}_^t?L; z0W-hs&vbH~@otxPFj20AbhLhLk&8v~7_J4rxFgGly1N|b2p)Ve-3iOy7mMX6V?1f( z229qlEX|5i8`wrddE5;}8`VK4-Od(-&EhshCs8RD(K*t4vzEYJ=_6r_IPU_&hAea;sy6t_d=;>sYs~1z5QVGGBT( zMWOuabtaBYJhdSH&=jNpDQ_|XuBN+Glbv)9QPL!l(I-^v!pVvzt8@(0`qnyai*{!y z>!%sd;+<@=Z85QY0)TKb6&{l2Ozbo6bJg}$d~ou56k%#)_EglcA?-4(fK1sF8n}}1 zNrPIqVj;I_{0Xm}K#y46ZeO#XlM z6CFWuqxI`Pww$z6u^(P1+C}#Rqn|J6MW0y z9`S=~_*%AF5oeWmLLjM;;{ANf78Tz*Taq)eR;LfWY8}$b1L%bq)ILy0oA9N&B@bl? z=qqX2W~&~G?6$q(aC(k6;NPU|YRmniC5WlrYl0_hyS_C4cBlaH(8ye{*9alOVxAi1 zUQN%9B!i2S6mbTHX6Qz~RD+AJQtjhzPofs*DNnue-U@Dik-?4@8e-qoM7fQ$3Favn z?98}v)_Om+2#x&Kfcav_Q%Jo~sz7=&BKh=(^0yd zY&^P^A?0_hCx7dx{-5i$l!!$#^Fx!YTLoO zQG<}TH)-+D-AesLMwIFOw9j&s`&Gsdr4b&Xo=qqPQK@ZJAP;)W;8|g#^6`Q1LFDCG z#p48mlLxI-p}1suC6<(3Vw~80$D9otW&jEDYpFRa1ncAUd_xOTEKe(UV@fJxZ(WLMl!;aelby-1 zi`W%z6tPLqE=XP58Alb8L7*Pw|4{qXn_o6J-}+&^i(E#Aipx%4e_YgE{Z&R3Q7p6F zY{Z;a7guzvvYup4?1DwwH{*@`L&poYY|A$&Rl7rrZ_Fb6+~V^}@qjLWX)x5xH2w0e zhYvrNIj#TF#*MFiB~mZV<>+PnOfa{zzE}G=K1$vDHr^0ED#ZFwvHzW(t}J#ItK2jt z=2q=pSE`MUS331oOOJifHTK>(6H#Z)a+Q^P#+uFERLmWfOo!HG2!eO?<@ zPmfGhsAA{DlklS1D9?`Aljhn_Y_o>PgR-fTc~X!Xt9-T=0n)g>r-Lcrj_|G3&@axl5$} z#yqS&$0Q6jcn1>X`1vD0eC1I;-mgOyn}KV_7h%28r5xT!{~@;0h4Y>XmpX0_?$d<` z7oyWmPFXlFOK!U*){%48slCk0<(tu8YP&=G?fpD^cKM?hjoShi_(F?=yrLARlGKX# z5eOlbSUh+vD61))o15X7k^s@Rlnl75@lUG{4~n9gd%gAO zctvEwm*YCE*tNElXH3VVY2CFwc6xnT*skPbn^dpy$QUv5p33&Q z9T67J+&yQow7{7VqHc%9&BX0o825HKIy|`urQ{&g zxoHo(-vmvcKUL^W{wYX$m~GEgOlvC5C1^`tDs)c06~_DqB6fhpy4txO&d*TZV?!HN z-;MGb4^(h5`|>Tki;3uEzpdOUIg{2Qf618hrLdZA+vz@RL#j> z)582iQJ}&i=<5|!-fvFx9*F{VeLwcvEx#}$GGb15)d4mct(S)Eq>JgCf!{la@dwtV z-ZrQtR$UuBH6Ji|xK(+ik%G73DRZ0h{KEluBR)klakW`GQVCTj*~8%8B?MAS+x1H? zV=iQhl{I{L0AkOHlVhE(=-`5Nz_l^KR4kMZ(0r{m>g>sPi)A)wn9H1^*iC%7lb(^~ zJMpl~i)-U?aG0s0#SnLt3U#;mK&~}AOGdk$%yf`?IU#$O~&9JZ_>vU2pb;3^+kbR*o8DcuTcNu4vEy89~4 z&awAedm&C-DNCJP(ms!TvSMk}_v&RLZy8M2`gu3|OZt8L0J`%@&+3BoM;xa9y| znNvC-=<9=>y^tl|OlNIf_)g5!; zLr;19^0lH?q$4TUdhM06FMg^S^+a6vOjs|p2mWm@KFOsP?!*}i>e z{1IwP>qottsB&3tx0Lj}=Ew?0=JM%11-iuIu~ALK>m1Pqmwm{@u%0h%ybJC!D$|tA z=tM;hTtc5Ybz&r!*Ho_M-nP1tU}hPv>JuW4rpDmVE6^El>h_c35F(8efw4I&7m|vH zAunpNl%BibUkz#=zLZ!+M|o$j&kIAPEiWg0S6cq;&zQ3p?{PZ~2uo{9_q)yq)A5*2 zzeDfmt4imPtBw>XUaBsXuogciO}(oVEp1e=`+}M3253OIp|p#CsqoJj(qk9)VyE$% z(jRCpM0xddqeRl(4aA0W{Ed&iyDt~ad8s=b?s&b`aqI8)63uYQDMeOsTxf@3}`e_-ACUtXu`ePEa{qk zO2~Cl(fPU&<+ZOZ;micol@@3&)G&?``?51Q-bZdN{?Q7W7ubC zNQaN?WQ2lZ3Lg2<08)|I;dW81$YIU)!+OceQ1mMkk_*)?)somot|L&BT+7PnsvDji zbG_iUqplZ`JY^=PRi^d9ZLNzUMI=(jVGd?IH=hZhETWv} z(o)B<&JZotdq1=3^LiepiQ%wPJ*2uQquO`x9KK17-ea<(f*#F|#LHY9Be({nlFuBO zFXxv&?|$(W`{TIrX~pBwR9-#WhF6>NBngvIbjr0p!Iz3*`teF`K*3o-{qA?97t{~i za3f$EZIM5#okKPMA_MoL;pjR-@XM)Zm`7WWjRs~F;D(B^e9W%HuZz#^j(U}0Q>t9O zJK*Tmtz5G|KVDdS`m?y{lk4bJf)nRTCG>~bSY=uh;+HhgUftS{FCJ5Hbi!=Nyt`iZ znM&mvOH-kfc}RD)t%&N-lyot*uscD=mDD|L;89UNIy-e{xJAi-zt#7CaY#Cci<+a( zHgrNKR^kDa6wdHn!~1|TId**!b9KH?z+7&b0N-7YMyNrN{@Bp^XT|B$-6LyNFaBAo zCT!!WyKZz@3En!Y39gnXIAYq@IET;GRTv5MMv5A5@++Zo-4c2S%Ht{~E%iOdr*>l6 zC3&b*nhFp4THLO_t=t5*L5O6kn|v>BL3!s$<9D%pas5`15k+)0-#h43`h_KnReqyR zEHDf9OMQvdmUm2hjeeikpt957t>dN|Tg%VrI>j&_uM_=1F9K;`kf(MreoeO=)8z|w z2L|A7gXtS)^GOm>BP2cCesp#tCI~Y92&a!@LtDst9 zrqiDIjft9VPQwd~FF#s~@H%@!Hg zP3K&0vhTEV^mv@=FETP`Xbic8ET*!ZWkz@(TKI(3?5b4QXf-wz<+zXtl8Fc{uU&rN zR5Iyh?VTo)zIVl?Tao+5moZ(_y1JVQNeFi>at0B;{U4XQ~#kG0@^;^YO6VOUks0is0gqK7OSyhJzG6s9;I$Ysh3C z5=?f_FJoPbI-ya?zxtME-qhPq2AtFHu@1#fsy~#M&fbjz-N82<)i~ktaDaIE` z$76$W#rUc~VxlO_I8t@$v14P8Rmo&kmY1(lGj4xq5&Cr-Upof(Q#(#zZf7v*^~v1P zC?#_3Xp!L3A0W?y6m9x=XRrvI3uYTVkbRbTYk zQzL??s(O^Y!xE~(t9!^s_*WVtQV{sYi)Izd^)59OaNbkxl{n?C{UAk#74K=g37>aP ztiE3QVvCYFZ)n^NWq{WvcWtsEW3-=ZE3q#FpD$#;!tK4M^ zxhlu9Tqbon2XOksQa3wxe<`gbz2m4_`A|&WN1#ij{yRb{=wRd&k`77BFYW1i6Tqap zk+|_nMI#ROmzwsG+ZHdK6g(-bT0dPaS&C$sm>J41wpuFawGElx+a`p{5s=9&lizrT z`j}d>LLK5`xY3Natfg`5iPP2K5s}POiL~AFJ!)K}0WY4qH3`ltKaX5%-&3;<^0s?S z5|(&`Rk`!c;&LyuvOB><9+BtTY4FtNRcpO}hkO0BT!q^>x*MNyZ}3)8o%)O}-a%pY zC)gTlu;u$k&v!CT7L;4>P;C>QfHYBM5T9;p9X zZA-?1)=>1?Y>2^GmdGNa@6Tsnz2G2^&bUa-*~La^_^Zio47Wb{AYja@d)?<$wbS>3 zFop9!>DHpFBmv$-E=N62XnJr2(P~7oyRzZx){?V`2oq27s9AvsqeSRwr2{`xGx8<{ zY{d_#B*}+lwrsZ!augOK=F4j&l^cBC&Q0IF$bK`rS^Uw}g5*vL{Jj`?;T_$yT?5p9 zUpkq0w^mq4Q*=sedB=D|X?A9)el%lb7_(MHe|h4GN%itB31w=&S;h!O+A3zFl&K4c zWECt-nO&~$H}eYof^G4(_^{-*0+qhC-q1Tpc*mo`PF z;ts*Q5~Y44i;?&~Fe6inf-(&xf?aG|(z$6c@*Df3bP8lRbF9HL&%gXRLB~Ush`|!} z0F@wo-5%9=C!kKu(>KL)p<3$?6r#jMC1cZeKif@kgf`hP z?7U5&Ui{^I3|MJpNYfs;nv^{WH=kYpQsZIB^BH0^`!9{X{u}}&_2Z`JQg+3keOz_Z zz1}OHH0mHya)G=}`>h{t@*ohG)S-{1p=;D5{#1ej{XrX6XJ zCWP?utT<4&OYip1%=aWqhrfDUNP?ELeS7DPU0_b9c(*jt_untMqvXXYb<>UUl^vXe z-A$``F{n}dVXtqddTP2!TaHdmCvV%a1V7va4c^nUs`-XvNuQcK9gI8!d3tDoAuq19RHpBNA+x944sIeM(OeoE)9+So|BP?t*$9UYDy zIMbo7?-nY3@u!J(+tm^=?d74spBNXuzZ$2pW1Qk9%}BcGT8+-rFS$E7yJuj09H@sq zuW1Ckw~{;W@UUnae-UR7P*H$GlCy>J1J;Y?X8w$)0u>*QY~4SS-iqF}0xFShpsoJM zmQp2eeBP!tJ-|NwBC*AFL*Fo+KK)O@2ETC!xWUGQ(I` zypuTQcDGH%%a&@wRX%M>^q3XZWS~P9?A@Fq7nU+*`(!r5yKMy$CuQ~6@@E$qGtcp8 zd#GVt9M7FcAGf8@bV#Jt4#c~+5ojp?OaSBsU4&*(oO<)T3i@)#<)FFbMX@aT!{qoQ zJ9RhR*hXT@7dri?yxZ(#ju%qt2DM(2eVB%|S-%#VT?tYMD0=CfG?Vw9oA2IVqpS=# zsIPuaTTfi~no6bJBNq4TT1mZefHR=hPkcq!r#`LDF(vFxO@(W1Cau?xD>pl5mKrD7 ze`9zfG=9>DPhkMK^K)$JQtPe&xT))g4iNuhQtf%W*GbuUWgCdCTgNA_UPzK6WWMqf zccW8yhr`HK0aD}Yx&{1iM?y2rb;UafmAbNLfY-amvsdt{3MhJW^wQrnEz|Jp>b^6| zg*#}-(c5lJnWT5u$`P5O(XEg>K*vO-}B>XPaHw@?q9PYiu z1RLfitJkyU<}Acjc8@ zoSE!sIZ!S48Jpkr@^i*FUiMQOutKPjk?Ydz(n5S~Y9#EIr{8k`;1$Yr*Xg%+`j-j( zGlAXn@gORcj7ahP5cOR*^e(xv?oEkDQ|jb0OhKXVi%we^*N=huS))W!uH!T6wb=lj zPje>Klhem9&&M|OPQFW4k?PxgScjWS&2Oo#+AU6<;^~YIM^VIXfvSwF1`}J@1~vMy ztKhD0rr)O5``@bbUyV~i4~IAM0SU9g*N8FBLgR65m<&U>PRNi&ffjS9VqZhxy%K!G z`yZ5g?`gS$D)y9|))2m!-^Zd6x3WFR^E8LV$z>1c2Bb}QpV96osN0spR9gAzVs)QJ zN_3An$t%6p%~}%quSV$N7_>5*lj5b|)~z@0`)`kglP%bg*_&L(A&jf@HQ<~d5lJa| z1EK0++HTNUY>8u@*@3@RU;n?sjZD=k)ZoPcC&i7jD~fG}liM7fumfE88T0bh&oPp9 z6EH`SPn(}^OnB)!;di>9n8a^aG9m1AF|geSw=i2b+^?!0q`;7mxUIfw>>YNj24q}_ z#=8E3xMHR2JxcUPgzP;8vhLR8srZe%&uyBoPru6&4!L5*DTtS+YIk$KY8IHv$0;Ho zc>P&^XUyVhIeQC=2)*y~PX}^zo;m+$yHim>irx$jWkr)x3A+>*u`&6v?u3P962|b~ z`JKgxisQrP9)_h?SDIHESmfgtPTly?>Z`AqW~170hulffW@Z$cLB87qFF1OQwdA@} z=5y;hVR)@{b>fSlE5i=(qsqn6vJS#P!W)dZ^jIChFLqlZTF1ul} zK!emKyL`_NgicU&c(J!PaBn(d%w3TC498Mz7!d^NW@9mlV>2m zg$&0ZvgFfd+-duR1Nt>ohz63hq9S4Hd81$-i0umqv76lSb=vtqcvL=b4;$HQBQAN_ zOBvaiIe+2Z8CT5>1m0KzOtw*V3HaZ0$RvN}m+Of0Nnj=s9WA5H9^PJ^I1otytFJ2v zs4s$fA9L7!xU`VhVn@5z`(^yG>F=Y~&zc(#-%u~-6pcV_w~Z1|{z`3TQ4%`Te*}v@ zd|3r7)FJRFuKh^j0I^r`VtgaB(`KJR#hE&V$9j&yfbVvv{^EgqoESC$|Gg&x2% z{`u&L?~wX`i~iUk~BE8DI`e zU)Xn{lT`vN=`!F*wjmRcddBpD7O8TvgD|Q&grq1i2%^k<3hTFr`Om8|#LNSm7`6w- zfGlcnUyaYwX+{@w6%Ap6$p=b|Te2nz7c=nSZHAvX1wfWbLRV?Y@z z3ZYs)Ng1R2gX8NmSgj_z{UtrqTzA@YJlqGCS_@>qbc}y7$nU+B9 zR01}|DB#UKBx$E0L^*=n(kZ|m*rvwO0bUO7B3BzInQH|xvMZ& zh)Nm?PV|3%6aVsOA95p2)25>SCnLpHZ`JY*11WYv+pg(byvRsHuEhm8CD)TgRFh6W zX2$9;aQ-&HtWG#03yS0agLn9EUR_ZhY_{vl5cl=Qq-9Vy90e<#{{A&`0)d)f^YkP+ z?DbV})ooM8);a(YBTMeJ#qJN$9mGFzn*YjD|MhKimJ#-hedZQg66TDy#72PZ*77ax z#(|0uAFDC)f<|l{ZCv-1QcVkg;sILno$gGX9T7>w5+exl87J%w6EV z24QGvSsYA-OTD<2(3R53Wq7InU4$o(*%ZBLeaq_s#f4zm{1j6W7fDk}wN5}plKc^=o;+g#KD z^dr`v4}wR4>C2z}-@kdz0p4(BIJD=)R@SDPDthf<4WT)FR?P+If$1-218t5D2epmV z^tMZbeV`gp@8<^S$OIBqCa;+f7ALs8#QE)A{?-w!xI@gcW^uqqTrFIlCu&=vdbl`+i~_C$-f60g;P z+OG~>OoGN2cmJmwm5_yCzsgxQOfBorh=o_gMotit7lUe}vq>dt7fF9+dz)hg3E2|C zQE1K#B;GoSNGVuM^>|@&mj)pFQLc15c)CW9`hBFEp7tpJsRi&y`PYBk_xG>iJ9K}} zC?#%|tk%SLorW9z9pJnZmGj*n*2T1T)Cx^mi^-9M`x!2gH4_sPk`&6(Yva38^BRuU zVbI!HLnZtac<_t1lX)$IG80U$eb}tO-U~|5w1(~eUJf?1m`@g=o7rZx>b)xYtKG_)v z_)gc{_8qRdAG&aFhqfvwjQ}iG0_jdp!h=w<5zxgbgM7d-(IQ^h_18iT@J9kE`Txz& zDP+PIC!Y^ZW7yaQ`p2R-26oq2kF&{c0}3mFOP!>Rl%D>m|+h}y~10#Ocu%NN73e;!ok;Ii0dDQ9b%!~M*H z4;Y5?FAbajF!jHTsiZ_;2I5ax-{0lJMkcwUL9C=`Q&6WG$71%H^8Vx{d^e`eP;V`K zH@Fb9ATqchB_%Zo7;kxR(VaKvkzva$7{_mzT=;F!_wP7HPglS;mW6N0-c*zn4Rc}I z4(+ucDz!C-dc+*|p_>DyJH;mPNDJSI>6rO&d5OtD;fMKwqHJm*haBQ;_(t=?6=l=_ z{F%T~gOWAo%fELF|BW+1ID%w}5>ZKX3aYAej6b@TVc^K7%zk+@NsD9`iG&q9-=odJ z8%(7`;s#@WtK@D>E1}1Vp)?&{TLuNjncc1B%#c^YW1i!nW*mn6p#a+!5ukg z$^UR2F9Pla9KAgxu*mSL*vDn4H>P26$kJ7dVf7`FU+OTqn*+(x6i_8zVfknQg-}Ir z5o7~dFg?o$k2$y#{2!Iqe;F$2R;JFsC&7_ffJ4ya-W|%-^P@P%4Y2}o5D+Lq2Irbd zH%IE@DO=Q@)Ffb59YAEqC1&lr{BIn>zjTZLew{9I zVe5P3LYeGVw+*+DLit3R>~GVAT4-#iPg1Sy?3EBHY!RdeuCX;-XrDR^@F(pdNq7R1 z1nl-!s12!6b`uC~n@HMg*jDZ5K?eFhg5YpORl?j6jBV-i-9N9Q|Lpko!m1!a7`g>Y zB$^ImCoE2{!Jdp5q>PBEg(B(U2{0+en}I^cG*~{a*GzjyKkS16_kDmMw>VhR!Kqfr zLB5xzTF}S|)}@BS#eK}jHJZ=lJ3t8I2aq!(szA=G*|iwZ*!giq0HmU(RSS%#NW0&ezyMno1)VAvsYLkeIW6`Dw~*`6$2L03#~DYGz8?)b#M3hN|)#Ws?DBWXmTQO9RL zS6dnSu`o_G0huyoFJ7bn`osUzUy;g*ve%g~Z7XX!)`9qs29lvs`he7}sXd4cl!61) z_sTo&>5HTkXN=!@iazzI#pKRpL{jE%mbvU$bMJ>9>#fBkCntldfUVpkZG12Gam0Cz zetH(`daFg+1~cFG{qGU3x`c)aj$qaO;Vupz-#EBxx`3SMN?9?`Isc{GaD#qeKF z0}fLJ+0NSzk*B_;Aiq-zX}3cL2DN<(qR(q6;R?)$`XM*RhT-DGqF+quwQDQMl4S+{ zp1VolIvvoZ>EmnxBk>T1w>ux0cfJqYgGDoQ7+8Rle4gbr_7oH3>E*RAi9Olq=n}JW zkOFgMTx$m@PrBVUTC}0|b9?^cb|q+?**R}5*+P_9brbia1=SmH$=<;uxq8yqbn8!Z z@6!m_yx(Fv{0kW7PfohZ9UsQMyKd|);0 z!%0Wt=3IhQ-06#tupwJQWOX*XAxy_!di>lOSYhSIiZ6F{zamBNhw^Cwy=_SOk@mqA z6EJ$h6P3Q~?4O_yZH*#8yy9LcYkdCS^#W|t+K?gqDx)$!%p@Ha(vSsU@)9h|wlRpY zdJZs?M&tyJXhJ$v`5)MVCXWM}C`#^@3gMhtCSsKHSX1sYf)PV|1ra-8vwe=*mD(Oi zS7cxA$xv=RPSN!OO@-=llb-i)K_>5q6xl;89YkRJ?R-GC#vMi^dpqp6F#K# z5O`sih-whE*0^TY&x@qdv<5IBskzVY4uwOgfGa+DsioG$XGMqXXDktb7!j+!upN4~ z7;p>CbD8{r2ik#uUN>S_#6tbe9MU$Z3JUn4V0qzLM0vkVuPIEACm_duvOZ9=7wIaM zT{Y|1J!fzdyXz((IkpN+@_3z(ut7n4RZNovOS+T&%nCprE3VzotN$pIe~@ekhl!Ygh^0R8uhM3ZM%Vnx0~}~ zf5FFAd$<$`)wG!*( z5zPYQ&^HWca`Zbd#`(jG98(b+h7`6=Y!pCkr@&>Ta5}AmJ&NdLwOK&$tl<8Xt5*E$pI@1sZE~~mI$RGvm7G~st$7K)801;KC&tG~ysxY9# zVXsA!Dg?1g3U7w?gI9T!BmlMNS){8_$G9N}ZP?Amchg%{eyXzrz38p6-1C_eic?66 zB#xlBFTJZ!oBro-^!s@dU!Oe9O`4UyR{!Tyet-1Lr!Ufaf}xjzYb=Ic0*yC7piC~~ zb#w>82;v3WkM0f%0^EbxrQj=K&jXNwTXy>-mv|Z;pDw|@9<|8@D21N~ay^@h+^WEs2|NBm1ofsbG<|Dcv!14y|xch+MOZvWFl9J;J~AplWAo? zlbyNW?*?(C;#e^>D2Bt&s)#@hzE1-cFU6tiK0xZ*2M96(hyS++*&WSmxUT2W$DFy+g4s%!mFPC99LH84`O4I}$ z!w(neJhRb1LW`9Lb|n|s{VJIWl!VYnNMa+BbpUOi9t&lbUr5#(ufc6T5grm!yZ0+j z=*hGOnXVfXK{uwhc5UN3@n`fdQp`Wg{p!{+d`a1<-P>rEhm--8u_E1~K?tm$4c4i#F+3lL)KV{lf zjDo#_pRu*;lqYK2VR8M4h4Qt0Vb|T-r^C=^&{H|20S*txVd#uKAWR?Yv=>bxkCWh- zI36Fkde0JW$79c&=38Qq{z^CH_i1E1UM7^^^Tr^_EV(uA$6X3lcx{{Rd%6BFM6e3@ zRm}+U4$TEd;bvSMUj(Fmgc0Z@g6JYD3O5@Z>Dh!a?{Ykq0n}d9 z)F`C#pOds{)Ii!L=1Z<#lMCkNy{^)<=RujrngDI*C`6fJkHFqCJv=XBAgrj+nUX(e zBKU_=2u7V~y3VYsQz!%vb#*f)V z=muMgC%(Dd?>)4JbM_OjLg7lUWJD^VjtyY;L2;h3kT=s6kxAR*-^*AEA^d%M^QSMP zV1?bDQw~U)W-o#;7J)t$pssQJHQ*P@U|C%?4G4TMfi~Sc1kQ7LMTFPM2=~{dy(=jT zNkY;iJ#JT^mgbOsY$~JdIxFLKevkdC^l5rIaHLmEu*V@iGkEtvGgrX$sWzHEi-6@* zmXF<3w5^sI)O}jgY=(mSe2_kTQFM`J(CYT8ap*q}W)`CyksW50W18N>YMEBq&Y0B}ii)&>+n{e+^4H3gq5gIuoE`52+|ds=r&F=+eMd&*jO6;48PQ z2$X9+Fki=aVX6^S5tz+!Y(CWty%P(|<2Z1X zimB|$dv4bd#Dgp5uBo1EYP2-7MJ1jC${f1xB5EuBV}=9s57~k4kI2m@E05^an`uof zAY+<_v=}k6UTz#niv!k=ve^ni=zrVYwg1S9Q$G1Dh!k!q+Rx1Y`@vLKoY&`_&hky+APB~nupJzIYufpUMbr&a~KrjetF)veX)Y( z%80{MKk*yA0WRKU)7cDmb?vpO#j(}DO5Omq#COzkKd`mfmz_XCjj^ng1-7GE$pZ+Yl7$K^q8g?j_dOt0nZ*-A9xY#Q>(STo9vs<*wvwBnu+taIxMX5Koh zHTLWzadWXn-q4eVm7t)YZ8Up|b3o~zyPWsm+EfEkN!q*EixE#BtSPt$poV8_6*9eMYrIal4J16ayhZo;abkO>+^cPvs`q9Z$ zB?{;dgl&tx-c<0qH*7YgG0dyulA(4s#mpzMGqAPrfFo-^e_e{OiM&Xly6J(Nj<~I?J`_2&UG`g?fPoq%`##ZriAed-*ju5vH+0$OA|9n zQ5FCGlPR4q05V10OVy<*;LbQ87YRQC$=HxHEo^SNaR4k{8#b%9tmEI&*r zkP`1r@%F5($WS^dp{F&;UEyb&`%E>#dwt{HW6)ZEJ#yUXPVc3i5;x;i)%}&BB;Nj> z+XeYQz`^=A;mt$G`Nb>y=X3>-c@Jj(MOPp}1>Z1p5=n&sja+|x?{aOD6}i!cuQU#$ zp2PREZ&x5g6F@=vF`~Z6FO2iE*=h~R97IQjw|MV$(j`nT)^`=g;9!8#zVTXFFl&3P zaJ(EQNcC{4nGZufRIphW*|Xdl%oZoo<2>+r+r7@|s3fjd^M(!$)9R7mpB9@-=HKj4 zlr~TGW^Kk0WE);5lck6ofb3m}_~TVBZ*ynSz126vCQcO3r%B9Nb__>uhj_=lWi}&q zB5o`-z1Vuc)KY68eE`_e>)(y0dGs1Xwxcd`z%Gau(A!=6ZZP|!^3z;sX0&%x591kW z29d?o-Lp-)L#A3A9lB(T^DN=ejC)_ISd^DLl_4|GqQvdJ3~6&|h9bi{h2V9oOdq+Y zQe8uH76T0jE9W@VuQnbtc7&RTtWES2){!dAr< zd1S(tqg9ue7cbm;9VJnw_&r@tF@>b&V}4<{zi8OzOxi54g-j@Z*?e2M~g+0=0zWt zzbH15oIIqZ6PlJ6$DZe#?y`{jByFZ}jpKH0)MsfUrXL{=7fBP?%rDFArN8yL*Rdx3 zob}L`jD7q`2WI?TY{n96UWoA*vamw4bGGGdFq2d$&SgNkf^>$Sxg4ms1#0@**WIq%(Pot*iI5Jh^W6G$zSY@L+>3W zxtbwDv%@`wZn<`64&{~3{`q$jwLKjH0_I2l2LO{6gO2*P#VS>0N4igg1qMhLOYh(M zBWdUpWVb9ZGop`D&@>zL@_3|IAd#wbv3>u~+EJU9B)~yTq@=aTj4$1Ov483ftIXb7 zM!Hl9JJhNS*h@@G@MB$ zG;hD7Q1J+=bKQfH6yDq%%I*thYO6m2CBIZ3DUKao>biC1D`&-~*?`LwLt6$-Ug>;s z;S^HCTc}0Uby%6rmoMy-g|(n}j4)X~Qx+SoZp9y5(L|wn#YO`SIglW;zwZ|K7|>M5 zs21a!Lb#scD^$IoYo=^)-N-R@C{>Xri7zio!gc#r(2HvDWaSZ>ORjOyqUpFg`Q} z{j!_O1fd_8nh8rr7W>kDcclzNNp0BNud{#Iv=!bxhU&ierP?)_P&`E-Z1X`maMN^U z^6~1@rQW)hh~ea%H!~O*rMnJ}-1R|Tp@A;) zD#gRM*p(06HR0)O;TtqWt|h5nMn)u znN?XQNHr#3&~%cq(!YOynojiY-MbfP+9k#AzV)gf>DgM|f(0URf=>)CUulrh9QpFt zx_Kw=uzV{{Fmg{gzO3BFRk9~?At0SpCeO6gJehhpx9O4vJhD~%k5pHlW+Gdos4=K3 zqH$v9&x-J}d88U;mAopl7!f|@s5SnaXqVCeja|u6!{1fjA@Iwc*HmQFmxNumSD3FW zT}$%7Cb*dJyFUcukHs4yItHs235YD;z96&v6^%+a*K;kjs{t2oRS&@0c^;dY*6m2r z_!r1b-r6%_*(J{x&&*juQv-!Bv;iZmo)HdM@=sj+CWg=TyE zG+ie$YkC&mO~^D+P?AsytW+I0xvG5xtps-O)ZClt3;amYo%yJ2HX_#!Cm4^Dj7Cc} z?!vTqXiaDMkc3;zI@J|@nb)(4K? z+vGQ)l%Kve<6Wj?l$QN*qN`ha`m@!U_hTj&xfgo(`J2c#$h;Nu2J|Zyh`-#z3&+@) zcYVg+dpGyqsLy!%mWVmdGdr7%DU~z=(+9tP?1VHNhc|NWD$}knXVV^}e%|W;AuP=k zK|nR@8aWFf9Bx$NE{-DxQR!9i=>C;&-)ZxBYJI=-jY!n! z#`D42>VU;Qu%KVf=>PwbFpC4d5b~pDhXJ#q6Q8wUmFIR*F;pF0JF<^mY9^eA*W(=M zAa%wYT#%ojCf#vPki|N|cH)<9rKsMw|2pd`F7^PTB$V$*)_$yr>;(Or# ztn3}0!GQf?a-V^d_;;#d2tx__P-Iq5o&OVbE7l(^U+eZ%j0v-6tv@i#NE!W`0#XQso#oE{t(i)Jh(bu95h0qFc zd8SAfuZKz>e@f+pBD>lUK+h2`cGAGZ)8e3IBF=w@CosP3l!|yN`9IopqY~fIwFK(L zHdcZ=%*7NL=S-K3Gv*nccUUa|VG!?L#gF={P0#BX0a&o@1VAPmwOy-4`ol1IimtbUJy zK-2E7fm$?r>^jjN#hSt!eKEYD!!Myl#1B|3k+l#;aqQi-Ia#U$4v(YgA*&QcgI=Iz z0!~hPjD=|o4NcUcaK&0g2Tgfds0Q?D;zc*?lnXS_OvxLJSXn$*!E=%64 z!jkB_P9?Q0Sp6UGlf8t-~#c1KPyXHMwU$JREou zf4uqxw$ujZ=JHI+Pew4pZUyjHY~jYtWd+UR_Cok2HDpF9J2Ci`?o3j6k!-B)-coqZ zFj0N+$qOJXYQ{7l0Q%2EBmxp|EvJb`2W>)OA3QJVVeJ$Zn+WSH!&vM5TSI*(XLZY+ zZD&Wa>qA zgi~D}?8)u_0q5_a(5e-M*XEV_SZX06OW^h)ee6;w=Ay4BU}w~aD}B9D+e<@);v7AZ zHH7>i?)if;*M9o@CC|?40B$H^tp}@L!EXIXzOXP`71m0lfY1gc#**%AD2t{@+lFxnFX*(Z5nUQbw8&@JpV}5 z)ku5Gjbg@bSFJM<@V+n) zf}vqTAyBqFW8_%$X@n*PX%H(%4`1%}|L4m!Umz&9jWH{^)cbY7e!K~MG~=Wu9RTW1 zvX!d1ln;uZ?oP5Qkovuq!8F$K6Rq;gj1+HZE13but2kI=v-TlqI3IAtPeX9<4vW(X z)VctZUhBE9EaB760`b#9RF5I!pT8Hm&hoDu61zWC{R-XX1a*`Fz!p3Uh+smKN{~}p zBGT*w3GKGSv%3spm0e!@yGCvL_&9g+j~#3jKjwfL-&uGX$c5e(g)D2t9lsKgjpubh z7>br%tNrW9ca9x#R(xM79<<(ea_@6Up3YYE7`aGYb7xc#3X=Poj>2mD0I+MIx zK%QEr{MQY0;ZXk*mykrBQK$43G4+n4n%sT&8kPv0Hq|fGxZuXXCy#Kcn)glr$@Vd8 zep7~R7{4JsMbn<8&*)TVOFnJ+wMJ|cd>WtINek50?ro^Ec0)_15#z*_N28xF8-D!5 zdG>h^oc=i!&4Q&{Nb19P2j54+C3uYW3kH#PRBoTA`es5vyK!@*Jc6PyA!NOPH{Q1235k*__nu&9$$AENBWwtFNtQ=ug=vYt zWX9K&L_GwYooc&@;n)ZVsB_bEjc)XOK7KcRIaO{1BEaOO4{3gRovWz?l6Rh<5A}J3x0!D_C&DN?H7X+x+>J-{y&v71c1C1Hu zcBD8sP+Xq09{76QNc3?9|B;iOs|0ZLT|-J9rXReL8P~nZT%j#+Y=gNC(#YWeN%zhI zXgk>Wa8i;|tkN(F;q`_Prim!hhKMm7Awhc_3cD&H8S;?EY}AF4MMo+&M+{m+6U}hI z9bd21E7ck@-G2qt!3Wg+?@fHEh%^>FKM9JL7eQgpiw$^3esta%^e%sRY&NsZhk){4 zN=JY*-oHML`ns_=I$vSiI5PMtbFIuDg{idLLmMrgc_VbH(+_cwa{H2t74F94spp$L zs2|}e;JeMVy_3!#R}WKalj1GE=js#~M;q>H6)Dd=>N2vd7wTFyr=R$i;cz!NwIn$0 zQc=u+x}(r*YoQ5$(K4r?Bo`c0@r~->_^q1dYjt0kx!icI@cPI+gF7un(DG=FsekX~*KvftuC0p;oVU96@-4LD z?`0v^&_#;NgWB&5;}4Ev7n=hKHY)$f=4K2<1Pna&RUN{VDZ?}QD7#@<95WJ4iS~ z5qh&O+ZsC^X<8`V$qdBe_E1I8`eh_RX{IJ@8TKHVJZurP&oWT2$)K9EkSKcK} z@y7YLjdyf3&7bRCd0AK&IwIe?8ni5Yc;8E50zGq?UISC3S7+qSf@}R=6k>loR@*hW z-GM$t31y90r33ZR!>R;M^GYUcpP!=WCG<*e} zhSE+S4db4p>|sCd;x<$N8O|Jo=(|NSdsH9 zb)lj2%mH*1vd?KwRO42b-<*~l;vGHA_cP&>$iee@JgY%xr!XOVT#A4T>KsBR=I#u^vcaBi)3%BY{i2Rh+p!czo*agD zD~TzQmQ9QKP$Pw4k@!u6m@WfHkvVQ0N6#02&-XL{C_IV9Y<_Fvp&0!lUPGs-mDTO6 zJ99Z?LffwMLR!u7Yy2_z88^LW9<7O*$OYR4C6>CIThk=3@09u%a77Fp6>~pFx@&RH zk~L`J{c@f0>sBF)yl`7jZmpdn;ymHRq!D@P%%bP5D*E39J$B-X*m>zTUK7Twczz5W zkW*o-yFW0}@@OOb2aK?|HM3MkF2Il{G6IcbY#DU}bQ^tv6O=uJ^VFtg7SC|n?=0b_ zipty&TvU*OFsYP~1SwKse)0BS#rr>#UPq}zRr;4Y<&E|6_0S21H&zGS2P4SJR5l{m zZUmV^m%?+&0p9bPdm;xM~rbj0VIqs*Tw`V5sJ8MXYdy`l!ZwT`R z_X2w@{>`hpyn5d?@;N4gN7f3@vNx7R-^7Z)w#JGp*tv`7N8UNAz}IM30E4+~%Vt?E zaVF&{S)U6sEx(tF)Fmxv@uwR{naw177JeA%=(JqiX72EwojEVwh+XoyJ+a+t@2Qz@ z;pt}jdMn)c+RtdweWZBpksWc+jDWal?7+>9FxDWbjL&Oj?-{1H=2nNm%Uo?c#SF<=(&MgQ3SLwm+U-UE zNjmc=!&q3yyz98QuR}?Dp?^4A;j-8VvLHO!LJGFl)KUv#1M*U)g-HA}k?*oZ@6^3B zAYRXb6Nyz}3iC+D&I)crAF#MCXZH)~wMap{#_HtC#tWeyanzzt%gcFnHe3!3Ou_xKY`I)!apqxvu!%2; z3wP?+wcb}*sQy!`NiL1Aci&H_e{9{95 zndI z`{P5syEV=cYvst~9rL(EXCTG<&ZJM};UV2BW`grX3otK4==fd#y?iSL;ekd!!3$DZ zIxja3_E(Y?-6fv%oiO{No}i%CemC51jCTnX!m5(V+G0*=-$95qfPb&ZM;6*hO6N z!*Krb>TsH-`L|fpsu<~j;~A|fpA)qhajI}zq;c42&DlPn?W&D)*p&62Eo@zUxZNVy zh+a{wyPR$wJK)l?FLO6Iy!}gXYi%wtubERKnsG)qYt<}2KiV9~n>2Fqp9&bLD7Fop zvez+GL>)Xzw+LN zK>gncR`b@8mTh6_{_VW#M3>FY^y>KwQ|l+Mx3~t;Ppn5DJ}#KhHZtrT^cMi&<4*)t z&mZA+zY7$^yd`)au1Vp1$Dvo6HXsBHSt9w&)k#`U;9WpRh>2u6;<;Oq);dO1m)Q1#0c=x`NiU?|rc$~$L6yQy^I_F@hN$DP6^Anz;20IyJTCV|qn55T z@;#RQBf_&#elS9bSh}6H80i3h@F5|WUjI~s=jKg;OXT;4gu?Lo=xE>Qf#ZeA=Cskc zttGU8O3<-NNFUKX({r|5zD?wx08zdbAZ{7ki_4P-gH!+la7s)%LDwnIth`N7U=##6 zli1AFkEqWlyw8!#IS1bgBN*Q>>jSxvpyv^GA0dp!Qb7^iohEk=;inO7sv56N8^ zJ?*1|U*_vH#p=8bo=BhYtoiHf%4bV`?Moc96~c`e-t;G1>F94vo2-2zGtgmcI5BE` zLn4BlTR*5?a2F{GYpZ#3L}v2nv9dyz0p_RP=>79$W?d0trJ}a?s`@`~>YtT|Rsr6J zq5|^{e-J32ua6TIJ~~9SCk6wgd1f_czMP;r-znc2F{lgZq0yv(p9iY##8Czw!|Noy zZKb*q01fR^GToDJ)2S*T&`u4G!HnmgT%MArazrgc%FI^};_FQ5P^ZS&Yx_kXa3Lo~ z6@o17beS&pd=D}!1{qUO42wI$FVj@9Jcm$2;85b?ZBFE|W25j!%9?m1_mEUMhxq}S zhS>?o{-3?5I{+pB>om^VH9c<-h&XQS&QM4AVp3X!@%?FH zK6!bt?A&T!(Qk@@1c_%vN>5)lTVuGs+CcjQkME7ih-_|y>s!bW?Qkw#tz}!-w3#Q@ zSeQD2ZfM4N?bDnW8T3AEXcCGA;eFP)WoZIFCyRt%A2^PFOaOtg`N+(9+DVqdnDHzy zwC1V}9Tf5DPc|X2NcELJ_gTI&T&$|-QoX!p#2}Xk=zNZ&Tkja?ZAB`oS%xdlkw3?O!vIc1#MA>N!>sH!56$;#0ww%Oec?KqO< zck(;PiKXQY`c4jW_6!bO=u{`(zcRW*wD2zs;%^UJcouNN43QUc0l!_Mm&8ho5qRef z^s7RA=~FGf@)``lhVcg_zaMZcey}Gy^5AHCNQ7gT@dROefr5u3i}Q#a&o zS_;jH7u*p+H%X-GU7;hcr=iP+)88dkR)Oes0(X!Ex6jI)B|#HvxP@prd16e@ZH|f^ zy<9%JDPJ%RHejD7xQBEfPeVNH>+dT&pg_)+C#G5GV_|eu)cS4Wcj-vVt6R-40#gXw zQHCmu5oW1qbdvSv(io5X-5F2SPVlVg=joMiYX!TkTHD93eA!V-6xjR%(nD1xNJBi8 zBUoB06R$wKF^4*iC0jl+0*||?1eEwSp)RViqRf48thmMEof}U~CJ}QFJz~2}Ic*&M zQ=0-k6(_1E>hlgb8^pX7kTxX?I6V7j82I-Yx_(Fsi!^ML+~?9OFGdd`_l&6uX>mJS z8hY_(1P$zl=db|~I{*a4wO>;Su0a26li;0=j=~Xw6g_z$bob>I;>1TVT+5E4{LgSM6;~qB3bIsGoV>^JN939P$d_a#Ht#89D1Fl6DBG@*4#yOK}lj&uL&emceLf- z=FWpbT{Va+84({!7)WNdFf8w<@>Z2?3P)Vf`h*{A<%l819C%Em-yPdSS{!zOkZs}63*OQXXl$;d=OJ} zZvrs0GCNTQhM?b_=}Im(NMV7fG5}Z%u~8)ko%tod>!>Y?t)zGiu7DT+^+;_98MGA> zArO!tbgmov=}W~Sk!I?BxaC}69XS>;tQ}5$&sMtfEv0@3#(D?9XnIlgVGy62Y39OU z!Mq2+blDi_T-beyUvc~WSFEX?Q2>D*tO`BRaxikf98f=0UPoMZf_{*Z&H0XaqaMJFo_O-ENo4u9CY)4f=&X^4(1io$Fa4%J z-lQ*z($H@J#$)@Bgowj`j>0^6lEP=19>j6|kBEg8@Sl2nP8z=gx|IUM!gAi6od@k> zqNtinJhT?|`NO zERIINO2@fQyI-Z%HN%uygh-0gFj?gav+m<^f~2Sm_3{p*NGE|3w~wBMcH)WB9|JhA zPHWt56#}Ut?Vvz2P$Om<>|CmrQFzusL=4+QE1ULkcJD#=LFI93*jY~)@tl4mNrAI3 zp-O~M@V9|4R*!lO)j*_(%cD!PF_;#WjI}JUUGt$IL^zds{A!Yz(a$RxWAV08r9{gEE%&#G-Uub^eu^!{3Q z^+7#R>w85q0@n#3QLD>+J)$~vg6yVXp3NjMqa?hPRzDz187$wJ0+oFFXZhm4P?o{9F*$KEPg(~h?Ox$E-HMey_nVCYH)iZ<>e9#OE)nzM>MQGjG@faAv*an#yK#p(D}SkSE(6EBhCFc~HFT{-e@7tAbh& zv{^K3NL+J9van_+$YTtkt{DRE9hM%|rNV;P7cIUAkxtHI#O{Lurgnx%tm+l!f$-K9 z;5&Q^9H>xo@VU3&svw3()qtfLCOCm|!d6eW$0qK;&m?>c!a(o~C){tN0CD~!AIL?H zKE@Jwp+>=2g&B!x{vOa#a~ir@+YBETLn=eyNkjtqmMEycSdY3Ak}=RpSNOVb{#!Bj z_ifaC28z8xf4MJ4=-(fJFr*`wXJNTTp4%^g!S$lQd%Vkd)MWtYFNi;Z?$BDZa<~9E zj<^yYr=vk42NJ@)qTSzHDIKs~Kwt^`r3e4>kKZDR5_7`9PMHpBSCp?STnMyJDnNwyQ|{9WyPLGW=%Dz7FkJA4esbUJ{(Ft__vgCDdWvZj z@pjFIjjBq5+Yuo>S((St6RsksLc4@nu z7bW{2KF7i9@59dwH-Li3^SFb>ivQEYNN58V)T7RDvq}2z8|t@b=|4Y6P(ad0O0u`C zZ~plmfA^mMdYEQ)WKW05wJaa}PXE)x{NM8a@7wrapa1`s_kY?*f8TolcY6PqGy9)i z_kXANKT+_1`KSM{;LMB+2^85`s5WR$kTmEmwlx6rt%#(sV;248*LEOtBnIaHXL)J@ zRRX*CIWqkN_H_a^TwMhhOP(K?oc#>E!X=e3?U}BBDZI!RaMO?HMCbBbtDxtV))7FU z*E^p@%S#Ek$``6^I&_^r$$$u)+H*Bc-_K*cuP9S3_Wc2IzWFE;$$bdFss8;q{r9W# z9$4=o>8&w2V$fahzLgU6?~DrsfbclezL|0Jvs{%1 zb}6hS|J*_MuB9qm^F8i5qU^uB=6!T;CtMtKYvAkfC37DLz=N=8p>zI~`D8ojgXf!0DKyalmf*>jV3!vt(OzK(uXJ%pMQpB!;$TvbbmL4#>wll@Su} zRr&V6nxhW*6bYy!KxmN%!+7{yPVgD^8_KQYVxO@~bU-~n!^Z}Pu*AL@9gUc>%C`pd z-WddwqT^t3VAS;$Ji5Fh0hqfRPx;RmuM9>Y!?&Y-SqGoU{FF)XG1G>HZ+9!LBpmmFk%Ihwk21OMBlmx0(s9X`gfdNkXLbEiPA0hOw&YI8B<`w}=9{_73J_}8Lq<0&PsXE`_i>xa` z^H{FYM%5`Sd86ohFY;U=cmixBr?0%5^$vmMw*jlQSMV(_`+z_o0M>vi^L}Mf=%+jX zIIW$BkZpxmT6S>N=`(k#6W~R#Ig^bP^Kf;B4^j<%BE6Fm*5AJEK=>d%Cl|~SZ5)p; zoK%TS&#?g?u`v1`H4R;=Mv>)uOaFZuL{JUb3-}@!NCv_wA2x= z(VwgQryRpmHN=6kFP+e)Np#&8z#)MdvEE{Qg-a4`Q{V>r-r^x_LJn9CF9XH%MAhnyXynuy)aaV`;!hDl~WX zj__PQ%tzjg|La8njW~Wv$3}AK7P<<-mxe@y|5@bugaZZI^P>BQ`~k-q-s2314cI06 zG42c!^7Qr$hRtZc0l)*fUEe^TO3o5@n0MYBjMpKB;T87^>uirWn)wVs6d!_ViEP3( za4LCX?OSdlPZWfwR=v1er@yCCD2ZPI`;~L%ZQzx*SG3*gnL4L~VMbPNo7mR35P$Zp zawKB;quj^X^-%-c{p-XUr7*Lt3c#{s6I}-^M*cgf7sJ0h>i;uzAStnB7orS^K zd|B0x0josS(%Iea5<@3wGD;$ZZ7K-pG#Vv@5mJL`xo@&gq>!3`yvJRkNb?h~} zD)`&9{V)lMP_GHmJO1q!*OG$0Cx5Z*Hx;f)b|tiPooo~wd44oYzA8 z4e`qj(NjAmS6dX+K0;1X7NlfBVtZiabha1Tg-Nx%{(T{6PryEXv{CD|Qwssa>{KTV zb3pX6ZB(8xzuO<$pVVK^Ki8G>zBdmiQNy+hnKMz;`$!(pKT)>vf7}fcs=%CD-cl$^ z`e!6T!nMK$B~Hzkj);E12Ed*Gr-~WfHvlT8x|R6@$#qc&nq1b&fQ|RMXQF!VA=L)# zP{z#irLVcwZa_y?LtoEV*jL4s2#AXe4fD^6)PABg2s7X!W-`RCK_jknP#u>oFk=3# zisGU@W6p;O0x@SFYIwm%Dx8u1bGDGqx(~mo-_;yu(XkUN4#Na+>lfxyLp5{12Ij$e zh}a%%|F~v;zFOrS_60EZV)LYO+`~i8u78%toH>BB$2ac=#UbQ~VNxpqP^YY(1cD%I zF+>`WTjGhH)XHG$iHUj8+B-5$O5q^z7(qH`ZJ&6`P7oIStm)7JpiP3@p zL&E`k)sfzZac?1DXgPl0y^RtJ0|l#KB&8Po@IXFQ_wFtpQV{L0rs|f%nH{s9iK6FfNLRi{|{*uzk1%z^%{GQqxERwEXF%Or} zCvVHWGaqj%e~ZC4^f>JLv`iH+Pl|>eW%Q-6h z-vHny>ORQPG5`so@wQ2(H@9%qHMj&Lex5BR3F8hJU{^Y; z1OtEQkm?6HF_Z}yfi^qfUi5ORz(bb?rn$xY0kVm%fBWPflK?9?+Z7t_`JK|E;ewgq zfqRL;2&EMZ38-4O zfcB6b}U#=0)N*uyB5$Ls%o+3(@)k-w^{u$C9N-@~K8tKnwVD2H+ zQ8ES{9`2qGMjwn`SkF7x3RgMdw)@3;Yz56mS!WRedzZ!9c}-r0V#|&5?#IuJN!5=M&Fd^v#<$#QXuWYGiLmtpHZy>XI0}h!6;DO8O zFe$w2nQyiu?b`=kh=&gqfF1l%whhGHD~VLaIJ`0;V+bGqQjpp;gs=__OYKGc*wj`g zg#VrHPLvs4-as;A882nRAxSHa&xK!JvIcc{q} zGn)xsm-`RET^V8dw1cbZPoNK9tqR;9hT&evkc_YztTON=zXQ2DGxU*C(c<%kXv@27 zI!)1h>7hUs!gN@}OD=#=f2=-RgU= z&dpy^rT>k@IKvI+es?v?i#{4tYA_O#{-^=cu>fwb&AObg`eVKOGgCl(Z&VVBks~P_ zjmZ}78Y$yBiE_j@Fey}G5zN%d5d;DJSH4erqxUZAL0jDd^lE=>zM@9*Q=?UR1`vQw z*tpBjI(RG2rJD(5tLKu35H>MBw#|oOqBVmT^;zQ2Fgr_G75X3Bfw`2&5nBd!^_{O8 zSPHSFKSR(fryR%)>8{+r@hyELnltv#nQ|!84!>!&c7=0uqBTuhX*_hN>Rx zVA|PE7N|(I(X(^dgABrnWMI6=3)Bn&WFC{!n?ItH^&!z-XfML@jUJx%eF{P={Z@WW z6EH|I0D}Pogh_*n*JqupUq3~&Cp1Q{<+zK24*Z=QVyjZeB$qB`pHbwh!YZ~ux=PlQ zG`xmf*GL7%T$buui#n4h{tpq1@LLFMuNRdUBY%zJeaN0jIsb=U?#udsr$UZiMAg4c#p^1Y=Wh9{ud@~Rj#b>W1x+1>6 zEM7g^m6yilv9~!906BQEWY`7mZp`!tfkhHFQOX22C?>0c!tPFB(v?2W!uZ2jH%^{l zi^w-wHta5@q5BLM6#lGek$IGWidq44m<;6Y``3D51*r%nF5<$~Px9)4FaMKZpIs$| z4M=C~%1_)w_<9g`z5#_w4#j1)Pbf zs2Y_CnjsAfZh|bb8ElyoW!Sb-0zF`sMf+x)T29M=5Q1;md|`<|``TQ)6W39$*atb3 z0cf9O(mHy;Up-&KZ+F}(R(xm#Y{G>W7;czE+a;V4po-bN22CI%Bdw7O_MEjL9h8b9 zCy2Z$JH;R)H^<28)fMW>{{4g{xBxQ7ACjhhF!JDYf#M?Z28GDkdQJ1U_;ZLL7I!@M z9%-0WD#MLzOVFC~47P+~{5N+gCf%Fd`c;)Vd2l)fQ;69^&SN9sz|uHXrddB;54(q< z>F7?0yutTTi&l&vhXDpBMXSH5IvZw`%Alifev>`ekGvI3v&3h?otizBFbX-S7*jXw4;RC%|0UdAB~R z%I&KFb}BuB<}`u84nuJ$-C22XQ1u#@ao`_br|}ts>rGJN$l?zmd|Sm;LGnUFgX() zoTMq~0um@cI0qqSqgYtQAF zA($A0UN5F<-8!=d z?b8k$9%#cHpysr}Gf`NQa z{(`B7He_FJ&&6lPO#zr=r|J`;=Mlsdu{L^Kvflyzbr_rBr>yEaMyMi63M-S5qes~= zYd`vgZAK(K<{P!g3mEiRa!AC)_a*pk)-^DEIO6 z{;^hA$5qDR-(2GIhG$5x|2;Br*m{yF^iJ5wljA}rZe(dT9&DuppiW!d?C6}X`U(0X zzs?^b4)u>GjqWmLR=h(wjfGIQb3z`i|JgAtXjw#lXe#&gonNxrSsMg=+hJb7`%UoY z`kBt~h`}&MnhPu&xgcI6z}%pi_J9g%rdy}Y1w6hm8^jBkV-FuFMTIcwsPah(Le*&X zX`D!jYE^?nQ?3?rIh}_@Gr-4_BACdrCyyK~3VCM`)TR2N_;y!|z3026^dBJfs?6bi zr_4+-0@(=f{et3I7*e978I{|9`9X$oy86`1UGix+K!WwD_jV`c#LpnboR+}+Bd=2l zkETJdQ76I$NPgu=Z)Z&q;qpWi0$n|&dS~9(7y;vMBm~9z@B_VeMi^V~f3g906Q0{$ zowU?TwC|>;k>UuEbWS|{G-dWdGA?c6wUgeEP@51+5<6#U&93W3BpVM?NZP2$&~oFp zsoaw{@xt(oF#gd3#)N2vzWx|q41!+zXS!4Ngurhxunlu05|ya( zx->k7nT>yIJs6P<&rAT+q?bdK#iFOG&;~xj6i8RY{P=?KOnM$ZPTIy@hgBQX@LE_v z$MzGI!cy$8NO9^E0>cjtg`DBT%$B_L;<9V4{lXfrQ*1T;F<`yHzrQlH-6`eo#d5Pd z)SWR+?VSH*4E49uEKhQz1+hAG`QX}R%L?R9g z2(I(Bim%Hr1uJ(i=&Qd)%p7?2Uj$0U3x(X3j^A<@k^2~hz=HA~;D4=@j?tdJNW)BU zy-gl-O2b@#cUr?}Fx5n?O&{eyCjCIBPJL#CCWWgT65c==t*?QYir9xnXQ-r6nm7Pn z%$ld&E3OSKCCZX#OqQtTTj+Wabf9Lb3xwy(uadhmLh^Qr?#m<4%^M^Pj|E#G22SaQ zo1j^(%?h0;k-Uh5ESm@YW#wxa`{6K6)R`+G6DP!hC24MjiQ zwhH3rt$zUBwbl9)=b!VhjjAn6=>zsD=j_y4x&O05`2qusZy}&?hK>a`ZUwYdZ2X~9 z303S=&+;<5X979|vsSfeLoP8p>phc^+;l1*p&x=9#VRAld}%CYTIXa9tNrbJdF`@c zQ;?)p7%oC>Elh5~1vbw_4vwi+yDoTF<~KVS9YxJ}8Fl*rmfx;#fa zup#?|?atE1f*=*&V;q56(R)sPtjL1ksUrd)q??xT4(7jC>7Cn0{)1(-?CqJM>uf?W zB_sP=K;<&1w9!*mPYa=%Kwty(Y@p2ma2cxK272kyiOIPS%#iDDq=C>>h!nN9OqZHO z4|bS7N?VB2NaTx-O1OB{CT8}b_#_NbjyDhT`?K|f0=A8FpY;hu zV;UCz3xXfMnFXyEU2PORhZU!&oJc;E8G;sWDoR}cf)q=@+>D+IYF$;MnQ(XJMrO{a zJI}HAP7VmAe%}TrbQRL{ zAuzrmgq-v!-FJqu>#l5O54Dv)Edp}%>DCas(sR&|qLT;Fj zk)dzDTs{B#qr{yAEpLt@&5D=kZWcVbuy&;T3iC&*p|$52$O#K)XTGND-ALgZ2xODY zC1FmR$-;@@oQHTnD6y?kRJsCKG<+rXG4~->%)HvM8xNjTZU(86zQBZbh3`-Q;LWXEs8cssvfx6>!)5p`VeTz}i$_i07>KKfCwlz>q zrr!kb@0|+~QPIrFxA=flB)=aEoYS{-n%-`xHjYiSwcKpGU8sNZ`gknDF%(B3F!rL$ z>%I$3K4xlOd3AiXf>ds_TBwq4+jkw(1)B<8;)t8XbeBb0_e}Dob!|&+Ffw@1Rq$Rb zsY^>b3r2`3moqalMvlA48V@3yOD#)4hPg9w`c3RH$>u}}(}LymM7RDBWHfBjpwF83 zC#*_G`!y08cX10?(b-fUTHJ~iKW=4%q(n=7oI<+jPZ6~~KH6bYra5|8ZTdh)pKsE}9Ks^`u>VPGfh&a7 zz_v$enJY!7eMh>SR0)<{^pxQyy?_N5j-+qoH%vvT4hG{N3jDnW2fm?5W1w@pF&Vl6JR%LV4 zY2WsGaJ$-{TxXw*V+;>D7cD5G8mMs^iK%ZEhR_eA~&^^wK#NC%pU<3?w# zn7T53D6y|hZCkAR?8~qIGC_MVG>eIxs7?U?yR<*a7pK~(M6*|Z$M!Oo_4mf~s}Qgx zunXOPJ~G_e_cV+RqGnHlTOPaFW!y?f_@TmmBxh)5731fc_WFq~I(IXgQd zUuzt@$p1ilHA6LHcoK>wsRCO3>7Mbm`X-zercXp7JI?I2Lje?;TD`_t<0Im%ilehiT|N6%ctx%e-`KS%|gy zrpc>>Q2?5%5oFi-_BHORAE)%)ZOg?bI>vT=LOqH1CMJqL@=nP3ja!UVq=i>bLW5*A zxFzSN)hr^g{|?OI3Yll!j6%{p0~S52=>!T(5JoPgeVh^p$ZiN+&#fU4<;;xO`<20F$4))CAvsl7p51yTt z2<%C#lCf|LjV?i4gc8aPWGTkGo2WIXnT(b zMHG;ftnQ1?^lv|_W!C?KU|ALEA*e$6Rouz57|lUzdZ1h44>PBIcP2?WuX&QhoGY!A z&9(iMjmKEJgnI2Qp73jQi75HE0?x zGZ+rzM8Y~Tk;A}T50Kxh2aC>#As}x9QIi3FqHyAg%+b~2iXco+tQ=>USdo){`aS@{ z0Z>h(7^;eBK(~e(a<9dTtMRG6O`Hc7IP8LaH8jE0nCqvWyIQrJlu3(!83t8#2CnAO zqi??dYEhs*H|}dzCm`qbZClqvO%28E1t|eX`q@3mlDLq|XjmzA!L{b}L-b0k4^Mcx zXI{rX2!(tR*1@=KE{}Hbf&Y#Omvlnxf=ywY+RZZTpuMB8X)Ns{vCS`v$zNJRvG%IRI(8G+w_B>B zPOweI19iZ2zQH`ImbWLxS8<`|7cbB3Wl)^n?As_~u8Wuqh0fmD0#l0Rpk zm0KW&MAg7U>$b~{=}Qd?V@^btk4KxcN!_O-3*&I{JR$ZjkVowrVk(+;Cpr5}+cOmN z4*-Zc9f=8YQiEf?=-X6w3$Go&vcGlSU5}55 zu~ma`oasecN5wj|G^_g6;PvljZ40eiGS!JOHY$93i6d;?M?E$dxH`svsve#Rj2WpG z8nFJ7?403*@bb+*^Oqv*-`Yh^W`I&z@>d|k5`lTAd`BCiw*&(*3u9>?kk-xHv4S~< z9SSEF5Z~nr{UP0)5mzB1sflV4#5_z!=~V--8r87cd z7kxjK$UO96R^!bnPHBDRiy?zQ=86XNw34*QqmHBu$7G0Y0XZQkyj3}n;7^OR`z%vHYE;-}hSu8&#Q56ltw zG~J>)^~@_DNO^Y{URk3rC+FI%Rh+}!j!y2j#!@T*Q_jGC=DjfuKug5E|EQSckcw$p z)A{_t6_XMZEWU#NRo!gom-f3b5%M@Xq@JHmje_Y`kXZ@qYkQSw4$_#jgW71x;`}GF z&y_?M01jl|E_L9SPWb_ftCY@R$XKa^uZ>3r?$tQ*Mo{?}KUVFM&0doe-%>_+jJ!Uo z5#%mU-?__ku0xqqDM~l};$RHvY1PFQ`gvzL$FN*mjAC*q!=V}=>Ag61x;H~SW{rFn zOqJb$L1)LTs&p;G*5cCY#R~n@Qb`Hu$@!WOEXS>X&v9&R{7Ptn`c9z^$>bsZDp%jN zGp*qqcU7F}5kb}@v?aC?W1V_!lj7Gb{!b=-_v6Xr$sL12l*cy|= zPz7Ruf}1`n3$XUu?m0Yv7g$~GPlbr`>{|!jp@DqL$5@yUSvlECe9U~7Gp+zM@5Rj{`l z9kC<7EF>p4obIuPzPAVGr@RsXm~KOnbj_cMHaqGlr`AdL2923Cbcjdp#B9P| zN7Q(&mksy<=qI4iE_|K@8DNB|^MLS4Dlp(~*X546jWE7M2(6B!41h*P>eYiCYfbhSz<00E!kC$mV%QQfA^3Jt^t>t>=Z8)Mau zZRA?$4lV6C=CpULEj! z6@>B3t@lkq_jsy?mLTPMd{(ixR?0nJS{7$Two)$ToU+N%FJeHzG@Z1_m@0*lq(PP2 zjYuqzklE7$D)OVLu2g03#g~@7E7c9+sqpM7Q$_8!KA38gq&~PKxCbEjJSHlPTy}2dx7e{I*oaoaU7{C}PCqD2M!m`e!R|$fs=%>ioFJy@}gL=(S zS4tVFRkB)Arp9veHeqw=zS8gMS zNnJPS);j`U9E|W*BPh`fX>LQ#Lo*}_n3zkz>Re~>erR4?36nw%SGyCZQ!IjPSUzkb zGU_2(>&i2me*rhn$0vGVKvoq{=F5GHG^jdow|BT)(kx}gxPX8d4oEU!JdGax$8q0F)$ z#OQr4VzT$E&I}71HX}C+*8VG7<)4=ayK@&AFFOYthJlS50W(bJ9X|cwj7crUjhG3C z!ZG%3jWp`MPGYRg_Wk`-;z!>`qC~L>yyp`mjp4)GUoN{|xJY_bCM+VaPB8Po;H;lF zbn_!vb?wG((O)_LzwU2|bI7Xu#vzpxrbaqO~e!E{7F*wu^(EUoF1d zDz3)yGHX&JM?#cd#y}H$CC*z3t2`EQk5wGG$w4x0E%F0tR|kiUiBnmTnbehrl2wlp zWzRA8eRYeJxZv^JvpRf$9_Ln2mhS)|tkz!p5$J5g&?)3@jFcm!I^kl0nxW>a@+J@h z$)g4j;`%~@FcWoKD8?m`;cIynb3jMA2tvX&(_4>Jd+at3P6>fYb~TdX&pj$XgfH`^ zWoH_l#=@bc(aeLpBb~f_XuzxkACt61*GTvR6<}v1l_7aQF3-0u^`h=~qUeWDK02rg zB|n-3$LBi!>NQmM|7-8c= zkI$rJ7zjn$e^8W;MC1t81fIC^FHJ{lXPRC@ntdhMrni-B@}c}0&ow6BZ9NvbE8+a6 zfE*)Q%Aw2A(zJu&SdTf>&LI)okZ_MJYWv9cqd-{R?j?`R8|;?Oou1N z0Zr#W-6zplKZ{Q2nK8+G(RBKBk}CFS0a6s6$tFBY{8=SrOf#;$jgL1;tFv(LwaZw0 z_bdB#%gzu=Vl8C#GE56(%LM1gBg&6qo{5Drz+2gjBneKLq)o=1Ir{8EoE%4b&Yc%z zjz7LFBcXN!iIMkR8Qb;^;QRa6$G5Z%Dwwf!Z`*^dMbZQnqMAsm)&7o*vGlWh>3s21 z2lRUL?AIK6EM&nWIRNlwWqO1&jLlP+`s)5{lTj8Lkb@j_A+(}9o3JUHR6{VlDQ)1NJpRGHp1e{3z~1L`;z7 z33WD=Y@q$}7mid!e_0?OlWKH>DIm?o_X3ud6(Wk0e)t-Hq9dwi4hZ5`BxyatAgiK> zgU&lYOe;dMVcQ zQ$NmnTUSB#Jabp$){~ZbWma3&b$evas3%n-Mbkx)#PiBC#&J9YP?P*919jo2kxQSd zugv+5k{w<>wVq!yR%?diKOXkH*YY4Qs`-}sx%A|guoJuAa!H%~W+`z0#!gOt3!?yC zY4@8tDyMEAJJ2X-?0=9dE!?=PJ?z-YBVq^mLZet^c$TpZ)X9pAf=MVHnd!=`Q&3>~ zkjtmd+UrJc)8}JgR`E|Pk(}mi$e&&}5;JD38kKbFxl=1AZ(HpgqER4dzPhlu!wPN)7J7{b`=3L=?|E_{m! zV$B4j=^7E}%a=e+Kp}@0RL!P&pZ^#NLbqacBU4NLlhTA1-I<+ZEwmBxV_Sv1Fv?+P zFLP+??D*=vr4J)do;zKkqV-e!-A^V!kO@lillk!8+^Ri)Q?2#I5wj8O|trV77 z+uP$Hcn`etyX}IPbIVLw zPXHRkR^#CY#Sw^vIT*Dm{m+RCY(nHM@6{IsYGyB5uGc49y zHN{xB8*H!tPB%%cngNnf9>G*WaD?v*%8yIFw;r zq@MQz?Jr{papQ{g`B;S-oYOogcGdHD3{^bdkGC22j3kfe45H{eUp~}-QX_=#dM6v# z*U*WcPf|e8mkGFZ+1a;0=omkX!&=VuOKI&*&%H~EW6z6O8`d5aztk~GxiuM)MD6Idm?xEjW^Mq6J{L^uo1Z;4!(0RE z30j^oG+uQT9v_Txb|WY8J$z(?M7G*+Yz>_3IXF3b(7xt<8bTM7$>XZLAtUta_gd8{ zq`TNp`YJtnrFV+={E}y#&e*F=wf)19+`{nm1%@^Zd(s?Zswzzq_%K>e^=_DebrOz{ znQdacjT9)ZL!>9xD|b3pUq!7=KIO_zOe9Wmm22@`5GgZy(r)SLK2{^*TQa+_v7O&* zDzbwnB_3PhQl@sjAi~`9dH?yl3l^Jo3-S4v+N)}Y7SakRCS^bfa;b+agRdBz?_{t- z!#2b_tcBz9(0ZlGEJQatVA?&?7VLwVrtCiiOzYgcRug2$K!JqA?1ub@3*?FT?P9+} zWG%ZNT5nCX(|zr$yAvyuJE#ErVwB!3K%Jm4v|LFhdKGEo<$yKW<~ct!Yq*=)(s~mp zaGs6>?Uy?&sb-idFE$xSa+TL+xi#ylgk#syy2mv;VCmE%{#JVNw3!uCB!<+7-Qxpf zQ8#NWxI6qhiq9piFq@jtkO;Qv?R2Ptp0 z5^Idjf-)J~+?#xsdR5k2H-drvMVKNCHFtMQW7SH!3ujR|bV%^%bKB%D>q?|}4u^vj zf=~h}E3W2s_w_5NJIUqG2+qDY=MfPWGO}5`@ie}&3hLx7NaJH0yv5yP(9zZPNJ`Q> z+#?dxZ4&7*X4tliDq^uP9&uE4nwf8Aw5uzvkom}T`El^yt$=9nOGrJn?@2*Yiw3TN z@}|NHeMM?tBS&2JGd@cq*Q7JGe>zcQK82M45%`EOk`K*wb*P?~ZP)P_Des6~R5&A* zM>8>nJH#kh&0`-YTMaE_z*qaCJtZSn%JGSJtO(K|lQj>sR-q~+UxjCW(lMTBh|s@| zx|WO{NRi2VYI-rCw3(y!0~TO5>0@w=YroP&Kqacd2lm`fYG7BnL}a9r;+*SeRQa!>$3Ir~x{7IRgmZ?yx7MC8?SQ^m&k&&L|Rcvd(kYPpQSeL^1(nQ#?VBMRG4= z`Nz7we~u;zL+~E+(ar2}CaciEcc?TH{Sy+nOW$u_`P74DJz}Pqg{3xS`7=1S@_nhd zJ*;{!$;;09=uLUcmjN7{E9@TWvhX5KAcTViCWk?mrwYtM zo@}Wju;UMecV}8?=D1Kw(gU*@^X1ZlQR~)y;`$RuEFLeZ0;^Vaw^m-{S+>6Z{#4+k zsQ*)hQ^Jhx6haHWxHg*7epkN?H6}5o*u|-o5FL2iQ)s?PX17*oscL{c>PvmwPs+H# zT%!??((`QG6r4|_4>~7R^k?}Y>XY`>_6DzNb!v>0e7gG(X%om7ep1vrm^y}x`8>m&y|RJKw>gPho$#b zQDz}<#vCPDIhRhf@#Q>oz*a_h_?AHYmt4*)8PXtoxyCr<8#`f4rDQJ(t3mm*^n9S5 zM2Jk!$}-fKL7%A1IY|C|i`j6W*^_HZ(JD$2@1@$wazsw5J9x1;A`xt8T&pWM5a?Hf zyWF7qtKqDf_T8~hDQF#Dzxtso8LO{oK5(J*qJ?c_E(A!8m+1~M?9}W+H>)PhA(Vyn z13l%aEK^lB+ff-Pu2?)~Tg?TScc=$D(6I4yZIJM^_bZmnxMp3XgTv8XTJfkvbH@7@ zuRT!zL5iZlEgkYIK+R6521<&D zo#+H4PR1D=Yp>|N^rW=Uvw&)^X1L}hR;(@(bqCRkj9hAtbxoHMlIIg2?iQ$g;p^?o zG8A=Hj@MSGnc>)-+)?M?1_y{fpRrS|Uf|_L))|7&4#}qewK&cOTZ5KTLw02xv z@M=H%J)~N-xH=K~xykYr^ZZUV^K*gRrH9t6qFpv;{si0Z{@6AR>eB~rwr`+y}K!;v5L3%|~t3;iMqmKHLUkqgKtyr>l`9 zC=7EN8v;tjp`-T3knd9q!TVat7{?||a5jjUXrGVNT9!S{p@=Xun8X@OFqW*C5x-mAHFb7l3vfW&{?-oE1fG z0TQD}%PhF|^!Ry%TL1Yx|1&F%L_a_0;T;vjQp+FIpYz&-k;sH)FkU6HaQQ{ZZ6pL- z3&9|xrYw8}#%((!$$l>Wq_pE9;t>g)?WtGk-id@BrXe!Et2Ib`+8jZpwa}HZ7N`It z<#*?<1GXd);}N1PA$MpJ)A>w+KA1_CUyi-gEjcP z+lnrWlY1aJswznOkHT+;cNY;{-hU9FZHxsw+duumH(&DCi0pZJ#?tJR zTXg>QPQJSB#u6AG<0#u-&hp}|e)R|6yIXBN*ab@U*G>LU8XZ~x0R>d|Ybl*>8T{$+BOkF-Txw8ymBF(qjE+yDIdT_Gx#;MEFIv1CbCh>As1x|CFd0^QB&!UWn$55n+R$BM0>lr0bju_#bW$en