AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-12 11:49:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add queue to celery task (#688)

Browse Source
This commit is contained in:
takatost 2023-07-31 13:13:08 +08:00 committed by GitHub
parent 396197e881
commit 0c18cab111
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 20 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/README.md
View File

@ -38,4 +38,4 @@
flask run --host 0.0.0.0 --port=5001 --debug flask run --host 0.0.0.0 --port=5001 --debug
``` ```
7. Setup your application by visiting http://localhost:5001/console/api/setup or other apis... 7. Setup your application by visiting http://localhost:5001/console/api/setup or other apis...
8. If you need to debug local async processing, you can run `celery -A app.celery worker`, celery can do dataset importing and other async tasks. 8. If you need to debug local async processing, you can run `celery -A app.celery worker -Q dataset,generation,mail`, celery can do dataset importing and other async tasks.

3
api/docker/entrypoint.sh
View File

@ -8,7 +8,8 @@ if [[ "${MIGRATION_ENABLED}" == "true" ]]; then
fi fi
if [[ "${MODE}" == "worker" ]]; then if [[ "${MODE}" == "worker" ]]; then
celery -A app.celery worker -P ${CELERY_WORKER_CLASS:-gevent} -c ${CELERY_WORKER_AMOUNT:-1} --loglevel INFO celery -A app.celery worker -P ${CELERY_WORKER_CLASS:-gevent} -c ${CELERY_WORKER_AMOUNT:-1} --loglevel INFO \
-Q ${CELERY_QUEUES:-dataset,generation,mail}
else else
if [[ "${DEBUG}" == "true" ]]; then if [[ "${DEBUG}" == "true" ]]; then
flask run --host=${DIFY_BIND_ADDRESS:-0.0.0.0} --port=${DIFY_PORT:-5001} --debug flask run --host=${DIFY_BIND_ADDRESS:-0.0.0.0} --port=${DIFY_PORT:-5001} --debug

2
api/tasks/add_document_to_index_task.py
View File

@ -14,7 +14,7 @@ from models.dataset import DocumentSegment
from models.dataset import Document as DatasetDocument from models.dataset import Document as DatasetDocument
@shared_task @shared_task(queue='dataset')
def add_document_to_index_task(dataset_document_id: str): def add_document_to_index_task(dataset_document_id: str):
""" """
Async Add document to index Async Add document to index

2
api/tasks/clean_dataset_task.py
View File

@ -10,7 +10,7 @@ from models.dataset import DocumentSegment, Dataset, DatasetKeywordTable, Datase
AppDatasetJoin, Document AppDatasetJoin, Document
@shared_task @shared_task(queue='dataset')
def clean_dataset_task(dataset_id: str, tenant_id: str, indexing_technique: str, index_struct: str): def clean_dataset_task(dataset_id: str, tenant_id: str, indexing_technique: str, index_struct: str):
""" """
Clean dataset when dataset deleted. Clean dataset when dataset deleted.

2
api/tasks/clean_document_task.py
View File

@ -9,7 +9,7 @@ from extensions.ext_database import db
from models.dataset import DocumentSegment, Dataset from models.dataset import DocumentSegment, Dataset
@shared_task @shared_task(queue='dataset')
def clean_document_task(document_id: str, dataset_id: str): def clean_document_task(document_id: str, dataset_id: str):
""" """
Clean document when document deleted. Clean document when document deleted.

2
api/tasks/clean_notion_document_task.py
View File

@ -10,7 +10,7 @@ from extensions.ext_database import db
from models.dataset import DocumentSegment, Dataset, Document from models.dataset import DocumentSegment, Dataset, Document
@shared_task @shared_task(queue='dataset')
def clean_notion_document_task(document_ids: List[str], dataset_id: str): def clean_notion_document_task(document_ids: List[str], dataset_id: str):
""" """
Clean document when document deleted. Clean document when document deleted.

2
api/tasks/create_segment_to_index_task.py
View File

@ -14,7 +14,7 @@ from extensions.ext_redis import redis_client
from models.dataset import DocumentSegment from models.dataset import DocumentSegment
@shared_task @shared_task(queue='dataset')
def create_segment_to_index_task(segment_id: str, keywords: Optional[List[str]] = None): def create_segment_to_index_task(segment_id: str, keywords: Optional[List[str]] = None):
""" """
Async create segment to index Async create segment to index

2
api/tasks/deal_dataset_vector_index_task.py
View File

@ -11,7 +11,7 @@ from models.dataset import DocumentSegment, Dataset
from models.dataset import Document as DatasetDocument from models.dataset import Document as DatasetDocument
@shared_task @shared_task(queue='dataset')
def deal_dataset_vector_index_task(dataset_id: str, action: str): def deal_dataset_vector_index_task(dataset_id: str, action: str):
""" """
Async deal dataset from index Async deal dataset from index

2
api/tasks/document_indexing_sync_task.py
View File

@ -14,7 +14,7 @@ from models.dataset import Document, Dataset, DocumentSegment
from models.source import DataSourceBinding from models.source import DataSourceBinding
@shared_task @shared_task(queue='dataset')
def document_indexing_sync_task(dataset_id: str, document_id: str): def document_indexing_sync_task(dataset_id: str, document_id: str):
""" """
Async update document Async update document

2
api/tasks/document_indexing_task.py
View File

@ -11,7 +11,7 @@ from extensions.ext_database import db
from models.dataset import Document from models.dataset import Document
@shared_task @shared_task(queue='dataset')
def document_indexing_task(dataset_id: str, document_ids: list): def document_indexing_task(dataset_id: str, document_ids: list):
""" """
Async process document Async process document

2
api/tasks/document_indexing_update_task.py
View File

@ -12,7 +12,7 @@ from extensions.ext_database import db
from models.dataset import Document, Dataset, DocumentSegment from models.dataset import Document, Dataset, DocumentSegment
@shared_task @shared_task(queue='dataset')
def document_indexing_update_task(dataset_id: str, document_id: str): def document_indexing_update_task(dataset_id: str, document_id: str):
""" """
Async update document Async update document

2
api/tasks/enable_segment_to_index_task.py
View File

@ -13,7 +13,7 @@ from extensions.ext_redis import redis_client
from models.dataset import DocumentSegment from models.dataset import DocumentSegment
@shared_task @shared_task(queue='dataset')
def enable_segment_to_index_task(segment_id: str): def enable_segment_to_index_task(segment_id: str):
""" """
Async enable segment to index Async enable segment to index

2
api/tasks/generate_conversation_summary_task.py
View File

@ -10,7 +10,7 @@ from extensions.ext_database import db
from models.model import Conversation, Message from models.model import Conversation, Message
@shared_task @shared_task(queue='generation')
def generate_conversation_summary_task(conversation_id: str): def generate_conversation_summary_task(conversation_id: str):
""" """
Async Generate conversation summary Async Generate conversation summary

2
api/tasks/mail_invite_member_task.py
View File

@ -8,7 +8,7 @@ from flask import current_app
from extensions.ext_mail import mail from extensions.ext_mail import mail
@shared_task @shared_task(queue='mail')
def send_invite_member_mail_task(to: str, token: str, inviter_name: str, workspace_id: str, workspace_name: str): def send_invite_member_mail_task(to: str, token: str, inviter_name: str, workspace_id: str, workspace_name: str):
""" """
Async Send invite member mail Async Send invite member mail

2
api/tasks/recover_document_indexing_task.py
View File

@ -10,7 +10,7 @@ from extensions.ext_database import db
from models.dataset import Document from models.dataset import Document
@shared_task @shared_task(queue='dataset')
def recover_document_indexing_task(dataset_id: str, document_id: str): def recover_document_indexing_task(dataset_id: str, document_id: str):
""" """
Async recover document Async recover document

2
api/tasks/remove_document_from_index_task.py
View File

@ -11,7 +11,7 @@ from extensions.ext_redis import redis_client
from models.dataset import DocumentSegment, Document from models.dataset import DocumentSegment, Document
@shared_task @shared_task(queue='dataset')
def remove_document_from_index_task(document_id: str): def remove_document_from_index_task(document_id: str):
""" """
Async Remove document from index Async Remove document from index

2
api/tasks/remove_segment_from_index_task.py
View File

@ -11,7 +11,7 @@ from extensions.ext_redis import redis_client
from models.dataset import DocumentSegment from models.dataset import DocumentSegment
@shared_task @shared_task(queue='dataset')
def remove_segment_from_index_task(segment_id: str): def remove_segment_from_index_task(segment_id: str):
""" """
Async Remove segment from index Async Remove segment from index

2
api/tasks/update_segment_index_task.py
View File

@ -14,7 +14,7 @@ from extensions.ext_redis import redis_client
from models.dataset import DocumentSegment from models.dataset import DocumentSegment
@shared_task @shared_task(queue='dataset')
def update_segment_index_task(segment_id: str, keywords: Optional[List[str]] = None): def update_segment_index_task(segment_id: str, keywords: Optional[List[str]] = None):
""" """
Async update segment index Async update segment index

2
api/tasks/update_segment_keyword_index_task.py
View File

@ -14,7 +14,7 @@ from extensions.ext_redis import redis_client
from models.dataset import DocumentSegment from models.dataset import DocumentSegment
@shared_task @shared_task(queue='dataset')
def update_segment_keyword_index_task(segment_id: str): def update_segment_keyword_index_task(segment_id: str):
""" """
Async update segment index Async update segment index