diff --git a/docker-compose.yaml b/docker-compose.yaml index 0fc22fca..32179b94 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,11 @@ name: firecrawl x-common-service: &common-service - build: apps/api + # NOTE: If you want to build the service locally, + # uncomment the build: statement and comment out the image: statement + image: ghcr.io/mendableai/firecrawl + # build: apps/api + ulimits: nofile: soft: 65535 @@ -11,6 +15,31 @@ x-common-service: &common-service extra_hosts: - "host.docker.internal:host-gateway" +x-common-env: &common-env + REDIS_URL: ${REDIS_URL:-redis://redis:6379} + REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379} + PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape} + USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION} + OPENAI_API_KEY: ${OPENAI_API_KEY} + OPENAI_BASE_URL: ${OPENAI_BASE_URL} + MODEL_NAME: ${MODEL_NAME} + SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL} + BULL_AUTH_KEY: ${BULL_AUTH_KEY} + TEST_API_KEY: ${TEST_API_KEY} + POSTHOG_API_KEY: ${POSTHOG_API_KEY} + POSTHOG_HOST: ${POSTHOG_HOST} + SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN} + SUPABASE_URL: ${SUPABASE_URL} + SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN} + SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY} + SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL} + SERPER_API_KEY: ${SERPER_API_KEY} + SEARCHAPI_API_KEY: ${SEARCHAPI_API_KEY} + LOGGING_LEVEL: ${LOGGING_LEVEL} + PROXY_SERVER: ${PROXY_SERVER} + PROXY_USERNAME: ${PROXY_USERNAME} + PROXY_PASSWORD: ${PROXY_PASSWORD} + services: playwright-service: build: apps/playwright-service-ts @@ -26,71 +55,21 @@ services: api: <<: *common-service environment: - REDIS_URL: ${REDIS_URL:-redis://redis:6379} - REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379} - PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape} - USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION} - PORT: ${PORT:-3002} - NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE} - OPENAI_API_KEY: ${OPENAI_API_KEY} - OPENAI_BASE_URL: ${OPENAI_BASE_URL} - MODEL_NAME: ${MODEL_NAME:-gpt-4o} - SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL} - LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY} - LOGTAIL_KEY: ${LOGTAIL_KEY} - BULL_AUTH_KEY: ${BULL_AUTH_KEY} - TEST_API_KEY: ${TEST_API_KEY} - POSTHOG_API_KEY: ${POSTHOG_API_KEY} - POSTHOG_HOST: ${POSTHOG_HOST} - SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN} - SUPABASE_URL: ${SUPABASE_URL} - SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN} - SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY} + <<: *common-env HOST: ${HOST:-0.0.0.0} - SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL} - SERPER_API_KEY: ${SERPER_API_KEY} - SEARCHAPI_API_KEY: ${SEARCHAPI_API_KEY} - LOGGING_LEVEL: ${LOGGING_LEVEL} - PROXY_SERVER: ${PROXY_SERVER} - PROXY_USERNAME: ${PROXY_USERNAME} - PROXY_PASSWORD: ${PROXY_PASSWORD} + PORT: 3002 FLY_PROCESS_GROUP: app depends_on: - redis - playwright-service ports: - - "3002:3002" + - "${PORT:-3002}:3002" command: [ "pnpm", "run", "start:production" ] worker: <<: *common-service environment: - REDIS_URL: ${REDIS_URL:-redis://redis:6379} - REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379} - PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape} - USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION} - PORT: ${PORT:-3002} - NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE} - OPENAI_API_KEY: ${OPENAI_API_KEY} - OPENAI_BASE_URL: ${OPENAI_BASE_URL} - MODEL_NAME: ${MODEL_NAME:-gpt-4o} - SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL} - LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY} - LOGTAIL_KEY: ${LOGTAIL_KEY} - BULL_AUTH_KEY: ${BULL_AUTH_KEY} - TEST_API_KEY: ${TEST_API_KEY} - POSTHOG_API_KEY: ${POSTHOG_API_KEY} - POSTHOG_HOST: ${POSTHOG_HOST} - SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN} - SUPABASE_URL: ${SUPABASE_URL} - SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN} - SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY} - HOST: ${HOST:-0.0.0.0} - SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL} - LOGGING_LEVEL: ${LOGGING_LEVEL} - PROXY_SERVER: ${PROXY_SERVER} - PROXY_USERNAME: ${PROXY_USERNAME} - PROXY_PASSWORD: ${PROXY_PASSWORD} + <<: *common-env FLY_PROCESS_GROUP: worker depends_on: - redis