add package crawl4ai (#2918)

### What problem does this PR solve?



### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
Kevin Hu 2024-10-21 15:07:45 +08:00 committed by GitHub
parent 2ced25c676
commit 3e702aa4ac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 834 additions and 423 deletions

1250
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -24,6 +24,7 @@ cachetools = "5.3.3"
chardet = "5.2.0"
cn2an = "0.5.22"
cohere = "5.6.2"
Crawl4AI = "0.3.8"
dashscope = "1.20.11"
deepl = "1.18.0"
demjson3 = "3.0.6"
@ -55,7 +56,7 @@ nltk = "3.9.1"
numpy = "1.26.4"
ollama = "0.2.1"
onnxruntime = "1.19.2"
openai = "1.12.0"
openai = "1.45.0"
opencv-python = "4.10.0.84"
opencv-python-headless = "4.10.0.84"
openpyxl = "3.1.2"
@ -63,7 +64,7 @@ ormsgpack = "1.5.0"
pandas = "2.2.2"
pdfplumber = "0.10.4"
peewee = "3.17.1"
pillow = "10.3.0"
pillow = "10.4.0"
protobuf = "5.27.2"
psycopg2-binary = "2.9.9"
pyclipper = "1.3.0.post5"
@ -92,7 +93,7 @@ strenum = "0.4.15"
tabulate = "0.9.0"
tencentcloud-sdk-python = "3.0.1215"
tika = "2.6.0"
tiktoken = "0.6.0"
tiktoken = "0.7.0"
umap_learn = "0.5.6"
vertexai = "1.64.0"
volcengine = "1.0.146"