AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-06-04 03:04:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

build: introduce uv as Python package manager (#16317)

Browse Source 
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
This commit is contained in:
Bowen Liang 2025-04-15 16:16:49 +08:00 committed by GitHub
parent f27a956c71
commit 12de1d175c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 6576 additions and 10991 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.devcontainer/post_create_command.sh
View File

@ -2,10 +2,10 @@
npm add -g pnpm@10.8.0
cd web && pnpm install
pipx install poetry
pipx install uv
echo 'alias start-api="cd /workspaces/dify/api && poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc
echo 'alias start-worker="cd /workspaces/dify/api && poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc
echo 'alias start-api="cd /workspaces/dify/api && uv run python -m flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc
echo 'alias start-worker="cd /workspaces/dify/api && uv run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc
echo 'alias start-web="cd /workspaces/dify/web && pnpm dev"' >> ~/.bashrc
echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env up -d"' >> ~/.bashrc
echo 'alias stop-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env down"' >> ~/.bashrc

2
.devcontainer/post_start_command.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash
cd api && poetry install
cd api && uv sync

36
.github/actions/setup-poetry/action.yml vendored
View File

@ -1,36 +0,0 @@
name: Setup Poetry and Python
inputs:
python-version:
description: Python version to use and the Poetry installed with
required: true
default: '3.11'
poetry-version:
description: Poetry version to set up
required: true
default: '2.0.1'
poetry-lockfile:
description: Path to the Poetry lockfile to restore cache from
required: true
default: ''
runs:
using: composite
steps:
- name: Set up Python ${{ inputs.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ inputs.python-version }}
cache: pip
- name: Install Poetry
shell: bash
run: pip install poetry==${{ inputs.poetry-version }}
- name: Restore Poetry cache
if: ${{ inputs.poetry-lockfile != '' }}
uses: actions/setup-python@v5
with:
python-version: ${{ inputs.python-version }}
cache: poetry
cache-dependency-path: ${{ inputs.poetry-lockfile }}

34
.github/actions/setup-uv/action.yml vendored Normal file
View File

@ -0,0 +1,34 @@
name: Setup UV and Python
inputs:
python-version:
description: Python version to use and the UV installed with
required: true
default: '3.12'
uv-version:
description: UV version to set up
required: true
default: '0.6.14'
uv-lockfile:
description: Path to the UV lockfile to restore cache from
required: true
default: ''
enable-cache:
required: true
default: true
runs:
using: composite
steps:
- name: Set up Python ${{ inputs.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ inputs.python-version }}
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
version: ${{ inputs.uv-version }}
python-version: ${{ inputs.python-version }}
enable-cache: ${{ inputs.enable-cache }}
cache-dependency-glob: ${{ inputs.uv-lockfile }}

34
.github/workflows/api-tests.yml vendored
View File

@ -17,6 +17,9 @@ jobs:
test:
name: API Tests
runs-on: ubuntu-latest
defaults:
run:
shell: bash
strategy:
matrix:
python-version:
@ -30,37 +33,32 @@ jobs:
fetch-depth: 0
persist-credentials: false
- name: Setup Poetry and Python ${{ matrix.python-version }}
uses: ./.github/actions/setup-poetry
- name: Setup UV and Python
uses: ./.github/actions/setup-uv
with:
python-version: ${{ matrix.python-version }}
poetry-lockfile: api/poetry.lock
uv-lockfile: api/uv.lock
- name: Check Poetry lockfile
run: |
poetry check -C api --lock
poetry show -C api
- name: Check UV lockfile
run: uv lock --project api --check
- name: Install dependencies
run: poetry install -C api --with dev
- name: Check dependencies in pyproject.toml
run: poetry run -P api bash dev/pytest/pytest_artifacts.sh
run: uv sync --project api --group dev
- name: Run Unit tests
run: poetry run -P api bash dev/pytest/pytest_unit_tests.sh
run: uv run --project api bash dev/pytest/pytest_unit_tests.sh
- name: Run dify config tests
run: poetry run -P api python dev/pytest/pytest_config_tests.py
run: uv run --project api dev/pytest/pytest_config_tests.py
- name: Cache MyPy
- name: MyPy Cache
uses: actions/cache@v4
with:
path: api/.mypy_cache
key: mypy-${{ matrix.python-version }}-${{ runner.os }}-${{ hashFiles('api/poetry.lock') }}
key: mypy-${{ matrix.python-version }}-${{ runner.os }}-${{ hashFiles('api/uv.lock') }}
- name: Run mypy
run: dev/run-mypy
- name: Run MyPy Checks
run: dev/mypy-check
- name: Set up dotenvs
run: |
@ -80,4 +78,4 @@ jobs:
ssrf_proxy
- name: Run Workflow
run: poetry run -P api bash dev/pytest/pytest_workflow.sh
run: uv run --project api bash dev/pytest/pytest_workflow.sh

12
.github/workflows/db-migration-test.yml vendored
View File

@ -24,13 +24,13 @@ jobs:
fetch-depth: 0
persist-credentials: false
- name: Setup Poetry and Python
uses: ./.github/actions/setup-poetry
- name: Setup UV and Python
uses: ./.github/actions/setup-uv
with:
poetry-lockfile: api/poetry.lock
uv-lockfile: api/uv.lock
- name: Install dependencies
run: poetry install -C api
run: uv sync --project api
- name: Prepare middleware env
run: |
@ -54,6 +54,4 @@ jobs:
- name: Run DB Migration
env:
DEBUG: true
run: |
cd api
poetry run python -m flask upgrade-db
run: uv run --directory api flask upgrade-db

1
.github/workflows/docker-build.yml vendored
View File

@ -42,6 +42,7 @@ jobs:
with:
push: false
context: "{{defaultContext}}:${{ matrix.context }}"
file: "${{ matrix.file }}"
platforms: ${{ matrix.platform }}
cache-from: type=gha
cache-to: type=gha,mode=max

17
.github/workflows/style.yml vendored
View File

@ -29,24 +29,27 @@ jobs:
api/**
.github/workflows/style.yml
- name: Setup Poetry and Python
- name: Setup UV and Python
if: steps.changed-files.outputs.any_changed == 'true'
uses: ./.github/actions/setup-poetry
uses: ./.github/actions/setup-uv
with:
uv-lockfile: api/uv.lock
enable-cache: false
- name: Install dependencies
if: steps.changed-files.outputs.any_changed == 'true'
run: poetry install -C api --only lint
run: uv sync --project api --only-group lint
- name: Ruff check
if: steps.changed-files.outputs.any_changed == 'true'
run: |
poetry run -C api ruff --version
poetry run -C api ruff check ./
poetry run -C api ruff format --check ./
uv run --directory api ruff --version
uv run --directory api ruff check ./
uv run --directory api ruff format --check ./
- name: Dotenv check
if: steps.changed-files.outputs.any_changed == 'true'
run: poetry run -P api dotenv-linter ./api/.env.example ./web/.env.example
run: uv run --project api dotenv-linter ./api/.env.example ./web/.env.example
- name: Lint hints
if: failure()

20
.github/workflows/vdb-tests.yml vendored
View File

@ -8,7 +8,7 @@ on:
- api/core/rag/datasource/**
- docker/**
- .github/workflows/vdb-tests.yml
- api/poetry.lock
- api/uv.lock
- api/pyproject.toml
concurrency:
@ -32,19 +32,17 @@ jobs:
fetch-depth: 0
persist-credentials: false
- name: Setup Poetry and Python ${{ matrix.python-version }}
uses: ./.github/actions/setup-poetry
- name: Setup UV and Python
uses: ./.github/actions/setup-uv
with:
python-version: ${{ matrix.python-version }}
poetry-lockfile: api/poetry.lock
uv-lockfile: api/uv.lock
- name: Check Poetry lockfile
run: |
poetry check -C api --lock
poetry show -C api
- name: Check UV lockfile
run: uv lock --project api --check
- name: Install dependencies
run: poetry install -C api --with dev
run: uv sync --project api --group dev
- name: Set up dotenvs
run: |
@ -80,7 +78,7 @@ jobs:
elasticsearch
- name: Check TiDB Ready
run: poetry run -P api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py
run: uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py
- name: Test Vector Stores
run: poetry run -P api bash dev/pytest/pytest_vdb.sh
run: uv run --project api bash dev/pytest/pytest_vdb.sh

19
api/Dockerfile
View File

@ -3,20 +3,11 @@ FROM python:3.12-slim-bookworm AS base
WORKDIR /app/api
# Install Poetry
ENV POETRY_VERSION=2.0.1
# Install uv
ENV UV_VERSION=0.6.14
# if you located in China, you can use aliyun mirror to speed up
# RUN pip install --no-cache-dir poetry==${POETRY_VERSION} -i https://mirrors.aliyun.com/pypi/simple/
RUN pip install --no-cache-dir uv==${UV_VERSION}
RUN pip install --no-cache-dir poetry==${POETRY_VERSION}
# Configure Poetry
ENV POETRY_CACHE_DIR=/tmp/poetry_cache
ENV POETRY_NO_INTERACTION=1
ENV POETRY_VIRTUALENVS_IN_PROJECT=true
ENV POETRY_VIRTUALENVS_CREATE=true
ENV POETRY_REQUESTS_TIMEOUT=15
FROM base AS packages
@ -27,8 +18,8 @@ RUN apt-get update \
&& apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev
# Install Python dependencies
COPY pyproject.toml poetry.lock ./
RUN poetry install --sync --no-cache --no-root
COPY pyproject.toml uv.lock ./
RUN uv sync --locked
# production stage
FROM base AS production

29
api/README.md
View File

@ -3,7 +3,10 @@
## Usage
> [!IMPORTANT]
> In the v0.6.12 release, we deprecated `pip` as the package management tool for Dify API Backend service and replaced it with `poetry`.
>
> In the v1.3.0 release, `poetry` has been replaced with
> [`uv`](https://docs.astral.sh/uv/) as the package manager
> for Dify API backend service.
1. Start the docker-compose stack
@ -37,19 +40,19 @@
4. Create environment.
Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. First, you need to add the poetry shell plugin, if you don't have it already, in order to run in a virtual environment. [Note: Poetry shell is no longer a native command so you need to install the poetry plugin beforehand]
Dify API service uses [UV](https://docs.astral.sh/uv/) to manage dependencies.
First, you need to add the uv package manager, if you don't have it already.
```bash
poetry self add poetry-plugin-shell
pip install uv
# Or on macOS
brew install uv
```
Then, You can execute `poetry shell` to activate the environment.
5. Install dependencies
```bash
poetry env use 3.12
poetry install
uv sync --group lint --group dev
```
6. Run migrate
@ -57,21 +60,21 @@
Before the first launch, migrate the database to the latest version.
```bash
poetry run python -m flask db upgrade
uv run flask db upgrade
```
7. Start backend
```bash
poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug
uv run flask run --host 0.0.0.0 --port=5001 --debug
```
8. Start Dify [web](../web) service.
9. Setup your application by visiting `http://localhost:3000`...
9. Setup your application by visiting `http://localhost:3000`.
10. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service.
```bash
poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion
uv run celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion
```
## Testing
@ -79,11 +82,11 @@
1. Install dependencies for both the backend and the test environment
```bash
poetry install -C api --with dev
uv sync --group lint --group dev
```
2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
```bash
poetry run -P api bash dev/pytest/pytest_all_tests.sh
uv run -P api bash dev/pytest/pytest_all_tests.sh
```

10583
api/poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

388
api/pyproject.toml
View File

@ -1,215 +1,203 @@
[project]
name = "dify-api"
version = "1.2.0"
requires-python = ">=3.11,<3.13"
dynamic = ["dependencies"]
[build-system]
requires = ["poetry-core>=2.0.0"]
build-backend = "poetry.core.masonry.api"
dependencies = [
"authlib==1.3.1",
"azure-identity==1.16.1",
"beautifulsoup4==4.12.2",
"boto3==1.35.99",
"bs4~=0.0.1",
"cachetools~=5.3.0",
"celery~=5.4.0",
"chardet~=5.1.0",
"flask~=3.1.0",
"flask-compress~=1.17",
"flask-cors~=4.0.0",
"flask-login~=0.6.3",
"flask-migrate~=4.0.7",
"flask-restful~=0.3.10",
"flask-sqlalchemy~=3.1.1",
"gevent~=24.11.1",
"gmpy2~=2.2.1",
"google-api-core==2.18.0",
"google-api-python-client==2.90.0",
"google-auth==2.29.0",
"google-auth-httplib2==0.2.0",
"google-cloud-aiplatform==1.49.0",
"googleapis-common-protos==1.63.0",
"gunicorn~=23.0.0",
"httpx[socks]~=0.27.0",
"jieba==0.42.1",
"langfuse~=2.51.3",
"langsmith~=0.1.77",
"mailchimp-transactional~=1.0.50",
"markdown~=3.5.1",
"numpy~=1.26.4",
"oci~=2.135.1",
"openai~=1.61.0",
"openpyxl~=3.1.5",
"opik~=1.3.4",
"opentelemetry-api==1.27.0",
"opentelemetry-distro==0.48b0",
"opentelemetry-exporter-otlp==1.27.0",
"opentelemetry-exporter-otlp-proto-common==1.27.0",
"opentelemetry-exporter-otlp-proto-grpc==1.27.0",
"opentelemetry-exporter-otlp-proto-http==1.27.0",
"opentelemetry-instrumentation==0.48b0",
"opentelemetry-instrumentation-celery==0.48b0",
"opentelemetry-instrumentation-flask==0.48b0",
"opentelemetry-instrumentation-sqlalchemy==0.48b0",
"opentelemetry-propagator-b3==1.27.0",
# opentelemetry-proto1.28.0 depends on protobuf (>=5.0,<6.0),
# which is conflict with googleapis-common-protos (1.63.0)
"opentelemetry-proto==1.27.0",
"opentelemetry-sdk==1.27.0",
"opentelemetry-semantic-conventions==0.48b0",
"opentelemetry-util-http==0.48b0",
"pandas-stubs~=2.2.3.241009",
"pandas[excel,output-formatting,performance]~=2.2.2",
"pandoc~=2.4",
"psycogreen~=1.0.2",
"psycopg2-binary~=2.9.6",
"pycryptodome==3.19.1",
"pydantic~=2.9.2",
"pydantic-extra-types~=2.9.0",
"pydantic-settings~=2.6.0",
"pyjwt~=2.8.0",
"pypdfium2~=4.30.0",
"python-docx~=1.1.0",
"python-dotenv==1.0.1",
"pyyaml~=6.0.1",
"readabilipy==0.2.0",
"redis[hiredis]~=5.0.3",
"resend~=0.7.0",
"sentry-sdk[flask]~=1.44.1",
"sqlalchemy~=2.0.29",
"starlette==0.41.0",
"tiktoken~=0.8.0",
"tokenizers~=0.15.0",
"transformers~=4.35.0",
"unstructured[docx,epub,md,ppt,pptx]~=0.16.1",
"validators==0.21.0",
"yarl~=1.18.3",
]
# Before adding new dependency, consider place it in
# alphabet order (a-z) and suitable group.
[tool.poetry]
package-mode = false
############################################################
# [ Main ] Dependency group
############################################################
[tool.poetry.dependencies]
authlib = "1.3.1"
azure-identity = "1.16.1"
beautifulsoup4 = "4.12.2"
boto3 = "1.35.99"
bs4 = "~0.0.1"
cachetools = "~5.3.0"
celery = "~5.4.0"
chardet = "~5.1.0"
flask = "~3.1.0"
flask-compress = "~1.17"
flask-cors = "~4.0.0"
flask-login = "~0.6.3"
flask-migrate = "~4.0.7"
flask-restful = "~0.3.10"
flask-sqlalchemy = "~3.1.1"
gevent = "~24.11.1"
gmpy2 = "~2.2.1"
google-api-core = "2.18.0"
google-api-python-client = "2.90.0"
google-auth = "2.29.0"
google-auth-httplib2 = "0.2.0"
google-cloud-aiplatform = "1.49.0"
googleapis-common-protos = "1.63.0"
gunicorn = "~23.0.0"
httpx = { version = "~0.27.0", extras = ["socks"] }
jieba = "0.42.1"
langfuse = "~2.51.3"
langsmith = "~0.1.77"
mailchimp-transactional = "~1.0.50"
markdown = "~3.5.1"
numpy = "~1.26.4"
oci = "~2.135.1"
openai = "~1.61.0"
openpyxl = "~3.1.5"
opentelemetry-api = "1.27.0"
opentelemetry-distro = "0.48b0"
opentelemetry-exporter-otlp = "1.27.0"
opentelemetry-exporter-otlp-proto-common = "1.27.0"
opentelemetry-exporter-otlp-proto-grpc = "1.27.0"
opentelemetry-exporter-otlp-proto-http = "1.27.0"
opentelemetry-instrumentation = "0.48b0"
opentelemetry-instrumentation-celery = "0.48b0"
opentelemetry-instrumentation-flask = "0.48b0"
opentelemetry-instrumentation-sqlalchemy = "0.48b0"
opentelemetry-propagator-b3 = "1.27.0"
opentelemetry-proto = "1.27.0" # 1.28.0 depends on protobuf (>=5.0,<6.0), conflict with googleapis-common-protos (1.63.0)
opentelemetry-sdk = "1.27.0"
opentelemetry-semantic-conventions = "0.48b0"
opentelemetry-util-http = "0.48b0"
opik = "~1.3.4"
pandas = { version = "~2.2.2", extras = ["performance", "excel", "output-formatting"] }
pandas-stubs = "~2.2.3.241009"
pandoc = "~2.4"
psycogreen = "~1.0.2"
psycopg2-binary = "~2.9.6"
pycryptodome = "3.19.1"
pydantic = "~2.9.2"
pydantic-settings = "~2.6.0"
pydantic_extra_types = "~2.9.0"
pyjwt = "~2.8.0"
pypdfium2 = "~4.30.0"
python = ">=3.11,<3.13"
python-docx = "~1.1.0"
python-dotenv = "1.0.1"
pyyaml = "~6.0.1"
readabilipy = "0.2.0"
redis = { version = "~5.0.3", extras = ["hiredis"] }
resend = "~0.7.0"
sentry-sdk = { version = "~1.44.1", extras = ["flask"] }
sqlalchemy = "~2.0.29"
starlette = "0.41.0"
tiktoken = "~0.8.0"
tokenizers = "~0.15.0"
transformers = "~4.35.0"
unstructured = { version = "~0.16.1", extras = ["docx", "epub", "md", "ppt", "pptx"] }
validators = "0.21.0"
yarl = "~1.18.3"
# Before adding new dependency, consider place it in alphabet order (a-z) and suitable group.
############################################################
# [ Indirect ] dependency group
# Related transparent dependencies with pinned version
# required by main implementations
############################################################
[tool.poetry.group.indirect.dependencies]
kaleido = "0.2.1"
rank-bm25 = "~0.2.2"
safetensors = "~0.4.3"
############################################################
# [ Tools ] dependency group
############################################################
[tool.poetry.group.tools.dependencies]
cloudscraper = "1.2.71"
nltk = "3.9.1"
############################################################
# [ Storage ] dependency group
# Required for storage clients
############################################################
[tool.poetry.group.storage.dependencies]
azure-storage-blob = "12.13.0"
bce-python-sdk = "~0.9.23"
cos-python-sdk-v5 = "1.9.30"
esdk-obs-python = "3.24.6.1"
google-cloud-storage = "2.16.0"
opendal = "~0.45.16"
oss2 = "2.18.5"
supabase = "~2.8.1"
tos = "~2.7.1"
############################################################
# [ VDB ] dependency group
# Required by vector store clients
############################################################
[tool.poetry.group.vdb.dependencies]
alibabacloud_gpdb20160503 = "~3.8.0"
alibabacloud_tea_openapi = "~0.3.9"
chromadb = "0.5.20"
clickhouse-connect = "~0.7.16"
couchbase = "~4.3.0"
elasticsearch = "8.14.0"
opensearch-py = "2.4.0"
oracledb = "~2.2.1"
pgvecto-rs = { version = "~0.2.1", extras = ['sqlalchemy'] }
pgvector = "0.2.5"
pymilvus = "~2.5.0"
pymochow = "1.3.1"
pyobvector = "~0.1.6"
qdrant-client = "1.7.3"
tablestore = "6.1.0"
tcvectordb = "~1.6.4"
tidb-vector = "0.0.9"
upstash-vector = "0.6.0"
volcengine-compat = "~1.0.156"
weaviate-client = "~3.21.0"
xinference-client = "~1.2.2"
[tool.uv]
default-groups = ["storage", "tools", "vdb"]
[dependency-groups]
############################################################
# [ Dev ] dependency group
# Required for development and running tests
############################################################
[tool.poetry.group.dev]
optional = true
[tool.poetry.group.dev.dependencies]
coverage = "~7.2.4"
faker = "~32.1.0"
lxml-stubs = "~0.5.1"
mypy = "~1.15.0"
pytest = "~8.3.2"
pytest-benchmark = "~4.0.0"
pytest-env = "~1.1.3"
pytest-mock = "~3.14.0"
types-aiofiles = "~24.1.0"
types-beautifulsoup4 = "~4.12.0"
types-cachetools = "~5.5.0"
types-colorama = "~0.4.15"
types-defusedxml = "~0.7.0"
types-deprecated = "~1.2.15"
types-docutils = "~0.21.0"
types-flask-cors = "~5.0.0"
types-flask-migrate = "~4.1.0"
types-gevent = "~24.11.0"
types-greenlet = "~3.1.0"
types-html5lib = "~1.1.11"
types-markdown = "~3.7.0"
types-oauthlib = "~3.2.0"
types-objgraph = "~3.6.0"
types-olefile = "~0.47.0"
types-openpyxl = "~3.1.5"
types-pexpect = "~4.9.0"
types-protobuf = "~5.29.1"
types-psutil = "~7.0.0"
types-psycopg2 = "~2.9.21"
types-pygments = "~2.19.0"
types-pymysql = "~1.1.0"
types-python-dateutil = "~2.9.0"
types-pywin32 = "~310.0.0"
types-pyyaml = "~6.0.12"
types-regex = "~2024.11.6"
types-requests = "~2.32.0"
types-requests-oauthlib = "~2.0.0"
types-shapely = "~2.0.0"
types-simplejson = "~3.20.0"
types-six = "~1.17.0"
types-tensorflow = "~2.18.0"
types-tqdm = "~4.67.0"
types-ujson = "~5.10.0"
dev = [
"coverage~=7.2.4",
"faker~=32.1.0",
"lxml-stubs~=0.5.1",
"mypy~=1.15.0",
"pytest~=8.3.2",
"pytest-benchmark~=4.0.0",
"pytest-env~=1.1.3",
"pytest-mock~=3.14.0",
"types-aiofiles~=24.1.0",
"types-beautifulsoup4~=4.12.0",
"types-cachetools~=5.5.0",
"types-colorama~=0.4.15",
"types-defusedxml~=0.7.0",
"types-deprecated~=1.2.15",
"types-docutils~=0.21.0",
"types-flask-cors~=5.0.0",
"types-flask-migrate~=4.1.0",
"types-gevent~=24.11.0",
"types-greenlet~=3.1.0",
"types-html5lib~=1.1.11",
"types-markdown~=3.7.0",
"types-oauthlib~=3.2.0",
"types-objgraph~=3.6.0",
"types-olefile~=0.47.0",
"types-openpyxl~=3.1.5",
"types-pexpect~=4.9.0",
"types-protobuf~=5.29.1",
"types-psutil~=7.0.0",
"types-psycopg2~=2.9.21",
"types-pygments~=2.19.0",
"types-pymysql~=1.1.0",
"types-python-dateutil~=2.9.0",
"types-pywin32~=310.0.0",
"types-pyyaml~=6.0.12",
"types-regex~=2024.11.6",
"types-requests~=2.32.0",
"types-requests-oauthlib~=2.0.0",
"types-shapely~=2.0.0",
"types-simplejson~=3.20.0",
"types-six~=1.17.0",
"types-tensorflow~=2.18.0",
"types-tqdm~=4.67.0",
"types-ujson~=5.10.0",
]
############################################################
# [ Lint ] dependency group
# Required for code style linting
############################################################
[tool.poetry.group.lint]
optional = true
[tool.poetry.group.lint.dependencies]
dotenv-linter = "~0.5.0"
ruff = "~0.11.0"
lint = [
"dotenv-linter~=0.5.0",
"ruff~=0.11.0",
]
############################################################
# [ Storage ] dependency group
# Required for storage clients
############################################################
storage = [
"azure-storage-blob==12.13.0",
"bce-python-sdk~=0.9.23",
"cos-python-sdk-v5==1.9.30",
"esdk-obs-python==3.24.6.1",
"google-cloud-storage==2.16.0",
"opendal~=0.45.16",
"oss2==2.18.5",
"supabase~=2.8.1",
"tos~=2.7.1",
]
############################################################
# [ Tools ] dependency group
############################################################
tools = [
"cloudscraper~=1.2.71",
"nltk~=3.9.1",
]
############################################################
# [ VDB ] dependency group
# Required by vector store clients
############################################################
vdb = [
"alibabacloud_gpdb20160503~=3.8.0",
"alibabacloud_tea_openapi~=0.3.9",
"chromadb==0.5.20",
"clickhouse-connect~=0.7.16",
"couchbase~=4.3.0",
"elasticsearch==8.14.0",
"opensearch-py==2.4.0",
"oracledb~=2.2.1",
"pgvecto-rs[sqlalchemy]~=0.2.1",
"pgvector==0.2.5",
"pymilvus~=2.5.0",
"pymochow==1.3.1",
"pyobvector~=0.1.6",
"qdrant-client==1.7.3",
"tablestore==6.1.0",
"tcvectordb~=1.6.4",
"tidb-vector==0.0.9",
"upstash-vector==0.6.0",
"volcengine-compat~=1.0.156",
"weaviate-client~=3.21.0",
"xinference-client~=1.2.2",
]

49
api/tests/artifact_tests/dependencies/test_dependencies_sorted.py
View File

@ -1,49 +0,0 @@
from typing import Any
import toml # type: ignore
def load_api_poetry_configs() -> dict[str, Any]:
pyproject_toml = toml.load("api/pyproject.toml")
return pyproject_toml["tool"]["poetry"]
def load_all_dependency_groups() -> dict[str, dict[str, dict[str, Any]]]:
configs = load_api_poetry_configs()
configs_by_group = {"main": configs}
for group_name in configs["group"]:
configs_by_group[group_name] = configs["group"][group_name]
dependencies_by_group = {group_name: base["dependencies"] for group_name, base in configs_by_group.items()}
return dependencies_by_group
def test_group_dependencies_sorted():
for group_name, dependencies in load_all_dependency_groups().items():
dependency_names = list(dependencies.keys())
expected_dependency_names = sorted(set(dependency_names))
section = f"tool.poetry.group.{group_name}.dependencies" if group_name else "tool.poetry.dependencies"
assert expected_dependency_names == dependency_names, (
f"Dependencies in group {group_name} are not sorted. "
f"Check and fix [{section}] section in pyproject.toml file"
)
def test_group_dependencies_version_operator():
for group_name, dependencies in load_all_dependency_groups().items():
for dependency_name, specification in dependencies.items():
version_spec = specification if isinstance(specification, str) else specification["version"]
assert not version_spec.startswith("^"), (
f"Please replace '{dependency_name} = {version_spec}' with '{dependency_name} = ~{version_spec[1:]}' "
f"'^' operator is too wide and not allowed in the version specification."
)
def test_duplicated_dependency_crossing_groups() -> None:
all_dependency_names: list[str] = []
for dependencies in load_all_dependency_groups().values():
dependency_names = list(dependencies.keys())
all_dependency_names.extend(dependency_names)
expected_all_dependency_names = set(all_dependency_names)
assert sorted(expected_all_dependency_names) == sorted(all_dependency_names), (
"Duplicated dependencies crossing groups are found"
)

6241
api/uv.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

7
dev/mypy-check Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
set -x
# run mypy checks
uv run --directory api --group dev \
python -m mypy --install-types --non-interactive .

14
dev/reformat
View File

@ -2,20 +2,14 @@
set -x
# style checks rely on commands in path
if ! command -v ruff &> /dev/null || ! command -v dotenv-linter &> /dev/null; then
echo "Installing linting tools (Ruff, dotenv-linter ...) ..."
poetry install -C api --only lint
fi
# run ruff linter
poetry run -C api ruff check --fix ./
uv run --directory api --group lint ruff check --fix ./
# run ruff formatter
poetry run -C api ruff format ./
uv run --directory api --group lint ruff format ./
# run dotenv-linter linter
poetry run -P api dotenv-linter ./api/.env.example ./web/.env.example
uv run --project api --group lint dotenv-linter ./api/.env.example ./web/.env.example
# run mypy check
dev/run-mypy
dev/mypy-check

6
dev/run-mypy
View File

@ -2,10 +2,6 @@
set -x
if ! command -v mypy &> /dev/null; then
poetry install -C api --with dev
fi
# run mypy checks
poetry run -C api \
uv run --directory api --group dev \
python -m mypy --install-types --non-interactive .

18
dev/sync-poetry
View File

@ -1,18 +0,0 @@
#!/bin/bash
# rely on `poetry` in path
if ! command -v poetry &> /dev/null; then
echo "Installing Poetry ..."
pip install poetry
fi
# check poetry.lock in sync with pyproject.toml
poetry check -C api --lock
if [ $? -ne 0 ]; then
# update poetry.lock
# refreshing lockfile only without updating locked versions
echo "poetry.lock is outdated, refreshing without updating locked versions ..."
poetry lock -C api
else
echo "poetry.lock is ready."
fi

10
dev/sync-uv Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
# rely on `uv` in path
if ! command -v uv &> /dev/null; then
echo "Installing uv ..."
pip install uv
fi
# check uv.lock in sync with pyproject.toml
uv lock --project api

13
dev/update-poetry
View File

@ -1,13 +0,0 @@
#!/bin/bash
# rely on `poetry` in path
if ! command -v poetry &> /dev/null; then
echo "Installing Poetry ..."
pip install poetry
fi
# refreshing lockfile, updating locked versions
poetry update -C api
# check poetry.lock in sync with pyproject.toml
poetry check -C api --lock

22
dev/update-uv Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# Update dependencies in dify/api project using uv
set -e
set -o pipefail
SCRIPT_DIR="$(dirname "$0")"
REPO_ROOT="$(dirname "${SCRIPT_DIR}")"
# rely on `poetry` in path
if ! command -v uv &> /dev/null; then
echo "Installing uv ..."
pip install uv
fi
cd "${REPO_ROOT}"
# refreshing lockfile, updating locked versions
uv lock --project api --upgrade
# check uv.lock in sync with pyproject.toml
uv lock --project api --check

6
web/.husky/pre-commit
View File

@ -30,14 +30,14 @@ if $api_modified; then
# python style checks rely on `ruff` in path
if ! command -v ruff > /dev/null 2>&1; then
echo "Installing linting tools (Ruff, dotenv-linter ...) ..."
poetry install -C api --only lint
uv sync --project api --only-group lint
fi
# run Ruff linter auto-fixing
ruff check --fix ./api
uv run --project api ruff check --fix ./api
# run Ruff linter checks
ruff check ./api || status=$?
uv run --project api ruff check ./api || status=$?
status=${status:-0}