From d463b82aba65e7daeed5c719e65a1e13092fc7f6 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Thu, 18 Apr 2024 13:43:15 +0800 Subject: [PATCH] test: add scripts for running tests on api module both locally and CI jobs (#3497) --- .github/workflows/api-tests.yml | 6 +++--- api/README.md | 13 +++++++++++++ api/pyproject.toml | 20 ++++++++++++++++++++ api/requirements-dev.txt | 5 +++-- dev/pytest/pytest_all_tests.sh | 11 +++++++++++ dev/pytest/pytest_model_runtime.sh | 8 ++++++++ dev/pytest/pytest_tools.sh | 4 ++++ dev/pytest/pytest_workflow.sh | 4 ++++ 8 files changed, 66 insertions(+), 5 deletions(-) create mode 100755 dev/pytest/pytest_all_tests.sh create mode 100755 dev/pytest/pytest_model_runtime.sh create mode 100755 dev/pytest/pytest_tools.sh create mode 100755 dev/pytest/pytest_workflow.sh diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 3a4d1fe2ea..8ace97c744 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -50,10 +50,10 @@ jobs: run: pip install -r ./api/requirements.txt -r ./api/requirements-dev.txt - name: Run ModelRuntime - run: pytest api/tests/integration_tests/model_runtime/anthropic api/tests/integration_tests/model_runtime/azure_openai api/tests/integration_tests/model_runtime/openai api/tests/integration_tests/model_runtime/chatglm api/tests/integration_tests/model_runtime/google api/tests/integration_tests/model_runtime/xinference api/tests/integration_tests/model_runtime/huggingface_hub/test_llm.py + run: dev/pytest/pytest_model_runtime.sh - name: Run Tool - run: pytest api/tests/integration_tests/tools/test_all_provider.py + run: dev/pytest/pytest_tools.sh - name: Run Workflow - run: pytest api/tests/integration_tests/workflow + run: dev/pytest/pytest_workflow.sh diff --git a/api/README.md b/api/README.md index 4069b3d88b..3d73c63dbb 100644 --- a/api/README.md +++ b/api/README.md @@ -55,3 +55,16 @@ 9. If you need to debug local async processing, please start the worker service by running `celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail`. The started celery app handles the async tasks, e.g. dataset importing and documents indexing. + + +## Testing + +1. Install dependencies for both the backend and the test environment + ```bash + pip install -r requirements.txt -r requirements-dev.txt + ``` + +2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml` + ```bash + dev/pytest/pytest_all_tests.sh + ``` diff --git a/api/pyproject.toml b/api/pyproject.toml index 3ec759386b..801a39cff9 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -25,3 +25,23 @@ ignore = [ "UP007", # non-pep604-annotation "UP032", # f-string ] + + +[tool.pytest_env] +OPENAI_API_KEY = "sk-IamNotARealKeyJustForMockTestKawaiiiiiiiiii" +AZURE_OPENAI_API_BASE = "https://difyai-openai.openai.azure.com" +AZURE_OPENAI_API_KEY = "xxxxb1707exxxxxxxxxxaaxxxxxf94" +ANTHROPIC_API_KEY = "sk-ant-api11-IamNotARealKeyJustForMockTestKawaiiiiiiiiii-NotBaka-ASkksz" +CHATGLM_API_BASE = "http://a.abc.com:11451" +XINFERENCE_SERVER_URL = "http://a.abc.com:11451" +XINFERENCE_GENERATION_MODEL_UID = "generate" +XINFERENCE_CHAT_MODEL_UID = "chat" +XINFERENCE_EMBEDDINGS_MODEL_UID = "embedding" +XINFERENCE_RERANK_MODEL_UID = "rerank" +GOOGLE_API_KEY = "abcdefghijklmnopqrstuvwxyz" +HUGGINGFACE_API_KEY = "hf-awuwuwuwuwuwuwuwuwuwuwuwuwuwuwuwuwu" +HUGGINGFACE_TEXT_GEN_ENDPOINT_URL = "a" +HUGGINGFACE_TEXT2TEXT_GEN_ENDPOINT_URL = "b" +HUGGINGFACE_EMBEDDINGS_ENDPOINT_URL = "c" +MOCK_SWITCH = "true" +CODE_MAX_STRING_LENGTH = "80000" \ No newline at end of file diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt index 2ac72f3797..0391ac5969 100644 --- a/api/requirements-dev.txt +++ b/api/requirements-dev.txt @@ -1,4 +1,5 @@ coverage~=7.2.4 -pytest~=7.3.1 -pytest-mock~=3.11.1 +pytest~=8.1.1 pytest-benchmark~=4.0.0 +pytest-env~=1.1.3 +pytest-mock~=3.14.0 diff --git a/dev/pytest/pytest_all_tests.sh b/dev/pytest/pytest_all_tests.sh new file mode 100755 index 0000000000..ff031a753c --- /dev/null +++ b/dev/pytest/pytest_all_tests.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -x + +# ModelRuntime +dev/pytest/pytest_model_runtime.sh + +# Tools +dev/pytest/pytest_tools.sh + +# Workflow +dev/pytest/pytest_workflow.sh diff --git a/dev/pytest/pytest_model_runtime.sh b/dev/pytest/pytest_model_runtime.sh new file mode 100755 index 0000000000..2e113346c7 --- /dev/null +++ b/dev/pytest/pytest_model_runtime.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -x + +pytest api/tests/integration_tests/model_runtime/anthropic \ + api/tests/integration_tests/model_runtime/azure_openai \ + api/tests/integration_tests/model_runtime/openai api/tests/integration_tests/model_runtime/chatglm \ + api/tests/integration_tests/model_runtime/google api/tests/integration_tests/model_runtime/xinference \ + api/tests/integration_tests/model_runtime/huggingface_hub/test_llm.py diff --git a/dev/pytest/pytest_tools.sh b/dev/pytest/pytest_tools.sh new file mode 100755 index 0000000000..5b1de8b6dd --- /dev/null +++ b/dev/pytest/pytest_tools.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -x + +pytest api/tests/integration_tests/tools/test_all_provider.py diff --git a/dev/pytest/pytest_workflow.sh b/dev/pytest/pytest_workflow.sh new file mode 100755 index 0000000000..db8fdb2fb9 --- /dev/null +++ b/dev/pytest/pytest_workflow.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -x + +pytest api/tests/integration_tests/workflow