mirror of
https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl
synced 2025-04-19 12:39:42 +08:00

* feat(self-host): proxy support * fix(playwright-service-ts): return untreated text/plain
139 lines
4.5 KiB
YAML
139 lines
4.5 KiB
YAML
name: Self-hosted Server Test Suite
|
|
|
|
on:
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
paths:
|
|
- apps/api/**
|
|
- apps/playwright-service-ts/**
|
|
|
|
env:
|
|
PORT: 3002
|
|
REDIS_URL: redis://localhost:6379
|
|
HOST: 0.0.0.0
|
|
ENV: ${{ secrets.ENV }}
|
|
TEST_SUITE_SELF_HOSTED: true
|
|
USE_GO_MARKDOWN_PARSER: true
|
|
|
|
jobs:
|
|
test:
|
|
name: Run tests
|
|
strategy:
|
|
matrix:
|
|
ai: ["openai", "no-ai"]
|
|
search: ["searxng", "google"]
|
|
engine: ["playwright", "fetch"]
|
|
proxy: ["proxy", "no-proxy"]
|
|
fail-fast: false
|
|
runs-on: ubuntu-latest
|
|
services:
|
|
redis:
|
|
image: redis
|
|
ports:
|
|
- 6379:6379
|
|
env:
|
|
OPENAI_API_KEY: ${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }}
|
|
SEARXNG_ENDPOINT: ${{ matrix.search == 'searxng' && 'http://localhost:3434' || '' }}
|
|
PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && 'http://localhost:3003/scrape' || '' }}
|
|
PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }}
|
|
PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}
|
|
PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@v4
|
|
with:
|
|
version: 10
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@v3
|
|
with:
|
|
node-version: "20"
|
|
cache: "pnpm"
|
|
cache-dependency-path: './apps/api/pnpm-lock.yaml'
|
|
- name: Install dependencies
|
|
run: pnpm install
|
|
working-directory: ./apps/api
|
|
- name: Install Playwright dependencies
|
|
if: matrix.engine == 'playwright'
|
|
run: |
|
|
pnpm install
|
|
pnpm exec playwright install-deps
|
|
pnpm exec playwright install
|
|
working-directory: ./apps/playwright-service-ts
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v5
|
|
with:
|
|
go-version: '1.19'
|
|
cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum
|
|
- name: Build go-html-to-md
|
|
run: |
|
|
go mod tidy
|
|
go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go
|
|
chmod +x html-to-markdown.so
|
|
working-directory: ./apps/api/sharedLibs/go-html-to-md
|
|
- name: Set up SearXNG
|
|
if: matrix.search == 'searxng'
|
|
run: |
|
|
mkdir searxng
|
|
|
|
echo "use_default_settings: true
|
|
search:
|
|
formats: [html, json, csv]
|
|
server:
|
|
secret_key: 'fcsecret'" > searxng/settings.yml
|
|
|
|
docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng
|
|
pnpx wait-on tcp:3434 -t 30s
|
|
working-directory: ./
|
|
- name: Start server
|
|
run: npm start > api.log 2>&1 &
|
|
working-directory: ./apps/api
|
|
- name: Start worker
|
|
run: npm run workers > worker.log 2>&1 &
|
|
working-directory: ./apps/api
|
|
- name: Start playwright
|
|
if: matrix.engine == 'playwright'
|
|
run: npm run dev > playwright.log 2>&1 &
|
|
working-directory: ./apps/playwright-service-ts
|
|
env:
|
|
PORT: 3003
|
|
- name: Wait for server
|
|
run: pnpx wait-on tcp:3002 -t 15s
|
|
- name: Wait for playwright
|
|
if: matrix.engine == 'playwright'
|
|
run: pnpx wait-on tcp:3003 -t 15s
|
|
- name: Run snippet tests
|
|
run: |
|
|
npm run test:snips
|
|
working-directory: ./apps/api
|
|
- name: Kill instances
|
|
if: always()
|
|
run: pkill -9 node
|
|
- name: Kill SearXNG
|
|
if: always() && matrix.search == 'searxng'
|
|
run: |
|
|
docker logs searxng > searxng/searxng.log 2>&1
|
|
docker kill searxng
|
|
working-directory: ./
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: Logs (${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})
|
|
path: |
|
|
./apps/api/api.log
|
|
./apps/api/worker.log
|
|
- uses: actions/upload-artifact@v4
|
|
if: always() && matrix.playwright
|
|
with:
|
|
name: Playwright Logs (${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.proxy }})
|
|
path: |
|
|
./apps/playwright-service-ts/playwright.log
|
|
- uses: actions/upload-artifact@v4
|
|
if: always() && matrix.search == 'searxng'
|
|
with:
|
|
name: SearXNG (${{ matrix.ai }}, ${{ matrix.engine }}, ${{ matrix.proxy }})
|
|
path: |
|
|
./searxng/searxng.log
|
|
./searxng/settings.yml
|