mirror of
https://git.mirrors.martin98.com/https://github.com/infiniflow/ragflow.git
synced 2025-06-04 11:24:00 +08:00
aaefc3f44c
### What problem does this PR solve? There are two main changes: 1. Update xgboost to 1.6.0 to build the project on MacOS with Apple chips, this change refers to the issue: https://github.com/infiniflow/ragflow/issues/5114. 2. When `use_china_mirrors` is set in `download_deps.py`, the names of chrome files downloaded by the script will be different from the file names used in Dockerfile, so I added the file name in `get_urls` function to solve this problem. I think it's better to add testing for Docker image `infiniflow/ragflow_deps` to the test workflow, but since the workflow is currently running on a self-hosted runner, I'm not sure how to modify it. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [ ] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [ ] Refactoring - [ ] Performance Improvement - [ ] Other (please describe):
77 lines
3.3 KiB
Python
77 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# PEP 723 metadata
|
|
# /// script
|
|
# requires-python = ">=3.10"
|
|
# dependencies = [
|
|
# "huggingface-hub",
|
|
# "nltk",
|
|
# "argparse",
|
|
# ]
|
|
# ///
|
|
|
|
from huggingface_hub import snapshot_download
|
|
from typing import Union
|
|
import nltk
|
|
import os
|
|
import urllib.request
|
|
import argparse
|
|
|
|
def get_urls(use_china_mirrors=False) -> Union[str, list[str]]:
|
|
if use_china_mirrors:
|
|
return [
|
|
"http://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb",
|
|
"http://mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb",
|
|
"https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar",
|
|
"https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar.md5",
|
|
"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
|
|
["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"],
|
|
["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"],
|
|
]
|
|
else:
|
|
return [
|
|
"http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb",
|
|
"http://ports.ubuntu.com/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb",
|
|
"https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar",
|
|
"https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar.md5",
|
|
"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
|
|
"https://bit.ly/chrome-linux64-121-0-6167-85",
|
|
"https://bit.ly/chromedriver-linux64-121-0-6167-85",
|
|
]
|
|
|
|
repos = [
|
|
"InfiniFlow/text_concat_xgb_v1.0",
|
|
"InfiniFlow/deepdoc",
|
|
"InfiniFlow/huqie",
|
|
"BAAI/bge-large-zh-v1.5",
|
|
"maidalun1020/bce-embedding-base_v1",
|
|
]
|
|
|
|
def download_model(repo_id):
|
|
local_dir = os.path.abspath(os.path.join("huggingface.co", repo_id))
|
|
os.makedirs(local_dir, exist_ok=True)
|
|
snapshot_download(repo_id=repo_id, local_dir=local_dir, local_dir_use_symlinks=False)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description='Download dependencies with optional China mirror support')
|
|
parser.add_argument('--china-mirrors', action='store_true', help='Use China-accessible mirrors for downloads')
|
|
args = parser.parse_args()
|
|
|
|
urls = get_urls(args.china_mirrors)
|
|
|
|
for url in urls:
|
|
download_url = url[0] if isinstance(url, list) else url
|
|
filename = url[1] if isinstance(url, list) else url.split("/")[-1]
|
|
print(f"Downloading {filename} from {download_url}...")
|
|
if not os.path.exists(filename):
|
|
urllib.request.urlretrieve(download_url, filename)
|
|
|
|
local_dir = os.path.abspath('nltk_data')
|
|
for data in ['wordnet', 'punkt', 'punkt_tab']:
|
|
print(f"Downloading nltk {data}...")
|
|
nltk.download(data, download_dir=local_dir)
|
|
|
|
for repo_id in repos:
|
|
print(f"Downloading huggingface repo {repo_id}...")
|
|
download_model(repo_id) |