diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 822717b6c..1d438fb23 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,7 +48,7 @@ jobs: - name: Build ragflow:dev-slim run: | RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME} - cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data . + cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data ${RUNNER_WORKSPACE_PREFIX}/*.tar.gz . sudo docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim . - name: Build ragflow:dev diff --git a/Dockerfile b/Dockerfile index b3c1bfd46..a064e01e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,14 +12,22 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \ RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \ apt update && apt-get --no-install-recommends install -y ca-certificates -# if you located in China, you can use tsinghua mirror to speed up apt -RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources +# If you are at mainland China, you can use tsinghua mirror to speed up apt and poetry +RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources +ENV POETRY_PYPI_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple/ RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \ - apt update && apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus python3-poetry \ + apt update && apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus python3-pip python3-poetry \ + && pip3 install --user --break-system-packages poetry-plugin-pypi-mirror --index-url https://pypi.tuna.tsinghua.edu.cn/simple/ \ && rm -rf /var/lib/apt/lists/* -RUN curl -o libssl1.deb http://archive.ubuntu.com/ubuntu/pool/main/o/openssl1.0/libssl1.0.0_1.0.2n-1ubuntu5_amd64.deb && dpkg -i libssl1.deb && rm -f libssl1.deb +RUN --mount=type=bind,source=openssl-1.1.1w.tar.gz,target=/root/openssl-1.1.1w.tar.gz \ + echo '/usr/local/lib' >> /etc/ld.so.conf.d/local.conf \ + && echo '/usr/local/lib64' >> /etc/ld.so.conf.d/local.conf \ + && cd /root && tar xzf openssl-1.1.1w.tar.gz \ + && cd openssl-1.1.1w && ./config --prefix=/usr/local/openssl11 --openssldir=/usr/local/openssl11 shared \ + && make -j && make install \ + && ldconfig && cd /root && rm -rf openssl-1.1.1w ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 diff --git a/Dockerfile.slim b/Dockerfile.slim index 8dba33382..d0430121d 100644 --- a/Dockerfile.slim +++ b/Dockerfile.slim @@ -12,14 +12,22 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \ RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \ apt update && apt-get --no-install-recommends install -y ca-certificates -# if you located in China, you can use tsinghua mirror to speed up apt -RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources +# If you are at mainland China, you can use tsinghua mirror to speed up apt and poetry +RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources +ENV POETRY_PYPI_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple/ RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \ - apt update && apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus python3-poetry \ + apt update && apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus python3-pip python3-poetry \ + && pip3 install --user --break-system-packages poetry-plugin-pypi-mirror \ && rm -rf /var/lib/apt/lists/* -RUN curl -o libssl1.deb http://archive.ubuntu.com/ubuntu/pool/main/o/openssl1.0/libssl1.0.0_1.0.2n-1ubuntu5_amd64.deb && dpkg -i libssl1.deb && rm -f libssl1.deb +RUN --mount=type=bind,source=openssl-1.1.1w.tar.gz,target=/root/openssl-1.1.1w.tar.gz \ + echo '/usr/local/lib' >> /etc/ld.so.conf.d/local.conf \ + && echo '/usr/local/lib64' >> /etc/ld.so.conf.d/local.conf \ + && cd /root && tar xzf openssl-1.1.1w.tar.gz \ + && cd openssl-1.1.1w && ./config --prefix=/usr/local/openssl11 --openssldir=/usr/local/openssl11 shared \ + && make -j && make install \ + && ldconfig && cd /root && rm -rf openssl-1.1.1w ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 diff --git a/docker/.env b/docker/.env index 8f61de3d7..6368a6c8a 100644 --- a/docker/.env +++ b/docker/.env @@ -33,7 +33,7 @@ REDIS_PASSWORD=infini_rag_flow SVR_HTTP_PORT=9380 -RAGFLOW_IMAGE=infiniflow/ragflow:dev-slim +RAGFLOW_IMAGE=infiniflow/ragflow:dev # If inside mainland China, decomment either of the following hub.docker.com mirrors: # RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:dev-slim diff --git a/download_deps.py b/download_deps.py index 6ff72f6ce..265515c2b 100644 --- a/download_deps.py +++ b/download_deps.py @@ -3,6 +3,11 @@ from huggingface_hub import snapshot_download import nltk import os +import urllib.request + +urls = [ + "https://github.com/openssl/openssl/releases/download/OpenSSL_1_1_1w/openssl-1.1.1w.tar.gz", +] repos = [ "InfiniFlow/text_concat_xgb_v1.0", @@ -20,10 +25,17 @@ def download_model(repo_id): if __name__ == "__main__": + for url in urls: + filename = url.split("/")[-1] + print(f"Downloading {url}...") + if not os.path.exists(filename): + urllib.request.urlretrieve(url, filename) + local_dir = os.path.abspath('nltk_data') - for data in ['wordnet', 'punkt', 'wordnet']: + for data in ['wordnet', 'punkt', 'punkt_tab']: print(f"Downloading nltk {data}...") nltk.download(data, download_dir=local_dir) + for repo_id in repos: print(f"Downloading huggingface repo {repo_id}...") download_model(repo_id)