mirror of
https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl
synced 2025-08-12 10:29:00 +08:00
add initial codeowners
This commit is contained in:
Gergő Móricz
parent
51bc7757ef
commit
115b6b61c4
91
.github/CODEOWNERS
vendored
Normal file
91
.github/CODEOWNERS
vendored
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
# Firecrawl Ownership Chart
|
||||||
|
|
||||||
|
# api
|
||||||
|
|
||||||
|
## V1 API
|
||||||
|
/apps/api/src/routes/v1.ts @mogery @nickscamara
|
||||||
|
|
||||||
|
### /scrape
|
||||||
|
/apps/api/src/controllers/v1/scrape* @mogery
|
||||||
|
|
||||||
|
### /crawl
|
||||||
|
/apps/api/src/controllers/v1/crawl* @mogery
|
||||||
|
|
||||||
|
### /batch/scrape
|
||||||
|
/apps/api/src/controllers/v1/batch-scrape* @mogery
|
||||||
|
|
||||||
|
### /extract
|
||||||
|
/apps/api/src/controllers/v1/extract* @nickscamara
|
||||||
|
/apps/api/src/lib/extract/* @nickscamara
|
||||||
|
/apps/api/src/lib/generic-ai.ts @mogery @nickscamara # (AI SDK)
|
||||||
|
|
||||||
|
### /map
|
||||||
|
/apps/api/src/controllers/v1/map* @nickscamara
|
||||||
|
/apps/api/src/lib/map-cosine.ts @nickscamara
|
||||||
|
|
||||||
|
### /search
|
||||||
|
/apps/api/src/controllers/v1/search* @nickscamara
|
||||||
|
/apps/api/src/search/* @nickscamara
|
||||||
|
|
||||||
|
### /llmstxt
|
||||||
|
/apps/api/src/controllers/v1/generate-llmstxt* @ericciarla
|
||||||
|
/apps/api/src/lib/generate-llmstxt/* @ericciarla
|
||||||
|
|
||||||
|
### /deep-research
|
||||||
|
/apps/api/src/controllers/v1/deep-research* @nickscamara
|
||||||
|
/apps/api/src/lib/deep-research/* @nickscamara
|
||||||
|
|
||||||
|
### Input Validation/Zod
|
||||||
|
/apps/api/src/controllers/v1/types.ts @mogery
|
||||||
|
|
||||||
|
## V0 API, deprecated
|
||||||
|
/apps/api/src/controllers/v0/* @mogery @nickscamara
|
||||||
|
/apps/api/src/routes/v0.ts @mogery @nickscamara
|
||||||
|
|
||||||
|
# Worker
|
||||||
|
|
||||||
|
## scrapeURL
|
||||||
|
/apps/api/src/scraper/scrapeURL/* @mogery
|
||||||
|
|
||||||
|
### crawler
|
||||||
|
/apps/api/src/lib/crawl-redis* @mogery
|
||||||
|
|
||||||
|
### remnants of WebScraper/WebCrawler
|
||||||
|
/apps/api/src/scraper/WebScraper/* @mogery @nickscamara
|
||||||
|
|
||||||
|
## concurrency limits
|
||||||
|
/apps/api/src/lib/concurrency-limit.ts @mogery @nickscamara
|
||||||
|
|
||||||
|
## BullMQ-related code
|
||||||
|
/apps/api/src/services/queue-worker.ts @mogery @nickscamara
|
||||||
|
/apps/api/src/main/runWebScraper.ts @mogery @nickscamara
|
||||||
|
/apps/api/src/services/queue* @mogery @nickscamara
|
||||||
|
/apps/api/src/lib/job-priority.ts @nickscamara @mogery
|
||||||
|
|
||||||
|
# Shared Libraries
|
||||||
|
/apps/api/sharedLibs/go-html-to-md/* @tomkosm
|
||||||
|
/apps/api/src/lib/html-to-markdown.ts @tomkosm
|
||||||
|
/apps/api/sharedLibs/html-transformer/* @mogery
|
||||||
|
/apps/api/src/lib/html-transformer.ts @mogery
|
||||||
|
|
||||||
|
# playwright-serice-ts
|
||||||
|
/apps/playwright-service-ts/* @mogery
|
||||||
|
|
||||||
|
# self-hosting
|
||||||
|
/docker-compose.yaml @mogery
|
||||||
|
/SELF_HOST.md @mogery
|
||||||
|
|
||||||
|
# SDKs
|
||||||
|
/apps/python-sdk/* @rafaelsideguide @nickscamara
|
||||||
|
/apps/js-sdk/* @mogery @nickscamara
|
||||||
|
/apps/rust-sdk/* @mogery
|
||||||
|
/apps/go-sdk/* @rafaelsideguide
|
||||||
|
|
||||||
|
# CI/CD and GitHub Workflows
|
||||||
|
/.github/* @mogery @rafaelsideguide
|
||||||
|
|
||||||
|
# Tests
|
||||||
|
/apps/api/src/__tests__/snips/* @mogery
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
/examples/* @ericciarla @nickscamara
|
||||||
@ -1,66 +0,0 @@
|
|||||||
# fly.toml app configuration file generated for firecrawl-scraper-js on 2024-04-07T21:09:59-03:00
|
|
||||||
#
|
|
||||||
# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
|
|
||||||
#
|
|
||||||
|
|
||||||
app = 'staging-firecrawl-scraper-js'
|
|
||||||
primary_region = 'mia'
|
|
||||||
kill_signal = 'SIGINT'
|
|
||||||
kill_timeout = '30s'
|
|
||||||
|
|
||||||
[build]
|
|
||||||
|
|
||||||
[processes]
|
|
||||||
app = 'node dist/src/index.js'
|
|
||||||
worker = 'node dist/src/services/queue-worker.js'
|
|
||||||
|
|
||||||
[http_service]
|
|
||||||
internal_port = 8080
|
|
||||||
force_https = true
|
|
||||||
auto_stop_machines = true
|
|
||||||
auto_start_machines = true
|
|
||||||
min_machines_running = 2
|
|
||||||
processes = ['app']
|
|
||||||
|
|
||||||
[http_service.concurrency]
|
|
||||||
type = "requests"
|
|
||||||
# hard_limit = 100
|
|
||||||
soft_limit = 100
|
|
||||||
|
|
||||||
[[http_service.checks]]
|
|
||||||
grace_period = "10s"
|
|
||||||
interval = "30s"
|
|
||||||
method = "GET"
|
|
||||||
timeout = "5s"
|
|
||||||
path = "/"
|
|
||||||
|
|
||||||
|
|
||||||
[[services]]
|
|
||||||
protocol = 'tcp'
|
|
||||||
internal_port = 8080
|
|
||||||
processes = ['worker']
|
|
||||||
|
|
||||||
[[services.ports]]
|
|
||||||
port = 80
|
|
||||||
handlers = ['http']
|
|
||||||
force_https = true
|
|
||||||
|
|
||||||
[[services.ports]]
|
|
||||||
port = 443
|
|
||||||
handlers = ['tls', 'http']
|
|
||||||
|
|
||||||
[services.concurrency]
|
|
||||||
type = 'connections'
|
|
||||||
# hard_limit = 25
|
|
||||||
soft_limit = 100
|
|
||||||
|
|
||||||
[[vm]]
|
|
||||||
size = 'performance-2x'
|
|
||||||
processes = ['app','worker']
|
|
||||||
memory = 8192
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -1,63 +0,0 @@
|
|||||||
# fly.toml app configuration file generated for firecrawl-scraper-js on 2024-04-07T21:09:59-03:00
|
|
||||||
#
|
|
||||||
# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
|
|
||||||
#
|
|
||||||
|
|
||||||
app = 'firecrawl-scraper-js'
|
|
||||||
primary_region = 'iad'
|
|
||||||
kill_signal = 'SIGINT'
|
|
||||||
kill_timeout = '30s'
|
|
||||||
|
|
||||||
[build]
|
|
||||||
|
|
||||||
[processes]
|
|
||||||
app = 'node --max-old-space-size=8192 dist/src/index.js'
|
|
||||||
worker = 'node --max-old-space-size=8192 dist/src/services/queue-worker.js'
|
|
||||||
|
|
||||||
[http_service]
|
|
||||||
internal_port = 8080
|
|
||||||
force_https = true
|
|
||||||
auto_stop_machines = false
|
|
||||||
auto_start_machines = true
|
|
||||||
min_machines_running = 2
|
|
||||||
processes = ['app']
|
|
||||||
|
|
||||||
[http_service.concurrency]
|
|
||||||
type = "requests"
|
|
||||||
# hard_limit = 200
|
|
||||||
soft_limit = 200
|
|
||||||
|
|
||||||
[[http_service.checks]]
|
|
||||||
grace_period = "20s"
|
|
||||||
interval = "30s"
|
|
||||||
method = "GET"
|
|
||||||
timeout = "15s"
|
|
||||||
path = "/"
|
|
||||||
|
|
||||||
[[services]]
|
|
||||||
protocol = 'tcp'
|
|
||||||
internal_port = 8080
|
|
||||||
processes = ['app']
|
|
||||||
|
|
||||||
[[services.ports]]
|
|
||||||
port = 80
|
|
||||||
handlers = ['http']
|
|
||||||
force_https = true
|
|
||||||
|
|
||||||
[[services.ports]]
|
|
||||||
port = 443
|
|
||||||
handlers = ['tls', 'http']
|
|
||||||
|
|
||||||
[services.concurrency]
|
|
||||||
type = 'connections'
|
|
||||||
# hard_limit = 30
|
|
||||||
soft_limit = 200
|
|
||||||
|
|
||||||
[[vm]]
|
|
||||||
size = 'performance-4x'
|
|
||||||
processes = ['app']
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
30
apps/api/utils/find_uncovered_files.sh
Executable file
30
apps/api/utils/find_uncovered_files.sh
Executable file
@ -0,0 +1,30 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Get all files tracked by git
|
||||||
|
git ls-files > /tmp/all_files.txt
|
||||||
|
|
||||||
|
# Get files matched by CODEOWNERS
|
||||||
|
while read -r line; do
|
||||||
|
# Skip comments and empty lines
|
||||||
|
[[ "$line" =~ ^#.*$ ]] && continue
|
||||||
|
[[ -z "$line" ]] && continue
|
||||||
|
|
||||||
|
# Extract the path pattern
|
||||||
|
pattern=$(echo "$line" | awk '{print $1}')
|
||||||
|
|
||||||
|
# Convert the pattern to a form git understands
|
||||||
|
# Remove leading slash if present
|
||||||
|
pattern=${pattern#/}
|
||||||
|
|
||||||
|
# List files matching this pattern
|
||||||
|
git ls-files "$pattern" 2>/dev/null >> /tmp/covered_files.txt
|
||||||
|
done < .github/CODEOWNERS
|
||||||
|
|
||||||
|
# Sort and get unique entries
|
||||||
|
sort -u /tmp/covered_files.txt > /tmp/covered_files_unique.txt
|
||||||
|
|
||||||
|
# Find files that are in all_files but not in covered_files
|
||||||
|
comm -23 /tmp/all_files.txt /tmp/covered_files_unique.txt
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
rm /tmp/all_files.txt /tmp/covered_files.txt /tmp/covered_files_unique.txt
|
||||||
Loading…
x
Reference in New Issue
Block a user