From c227f3d98566041f0c0a9820d87b0a7a445a2e7f Mon Sep 17 00:00:00 2001 From: LIU HONGWEI <1327374483@qq.com> Date: Fri, 12 Apr 2024 16:22:24 +0800 Subject: [PATCH] feat: Deprecate datetime.utcnow() in favor of datetime.now(timezone.utc).replace(tzinfo=None) for better timezone handling (#3408) (#3416) --- api/controllers/console/app/conversation.py | 4 +-- api/controllers/console/auth/activate.py | 4 +-- api/controllers/console/auth/oauth.py | 4 +-- .../console/datasets/data_source.py | 4 +-- .../console/datasets/datasets_document.py | 18 ++++++------ .../console/datasets/datasets_segments.py | 4 +-- api/controllers/console/explore/completion.py | 6 ++-- .../console/explore/installed_app.py | 4 +-- api/controllers/console/workspace/account.py | 6 ++-- api/controllers/service_api/wraps.py | 4 +-- api/core/agent/base_agent_runner.py | 4 +-- .../task_pipeline/workflow_cycle_manage.py | 10 +++---- api/core/entities/provider_configuration.py | 4 +-- api/core/indexing_runner.py | 28 +++++++++---------- .../event_handlers/create_document_index.py | 2 +- ...vider_last_used_at_when_messaeg_created.py | 4 +-- api/extensions/ext_storage.py | 4 +-- api/models/task.py | 8 +++--- api/services/account_service.py | 16 +++++------ api/services/annotation_service.py | 2 +- api/services/app_service.py | 12 ++++---- api/services/dataset_service.py | 24 ++++++++-------- api/services/file_service.py | 6 ++-- api/services/workflow_service.py | 18 ++++++------ api/tasks/add_document_to_index_task.py | 2 +- .../enable_annotation_reply_task.py | 2 +- .../batch_create_segment_to_index_task.py | 4 +-- api/tasks/create_segment_to_index_task.py | 6 ++-- api/tasks/document_indexing_sync_task.py | 2 +- api/tasks/document_indexing_task.py | 4 +-- api/tasks/document_indexing_update_task.py | 2 +- api/tasks/enable_segment_to_index_task.py | 2 +- 32 files changed, 112 insertions(+), 112 deletions(-) diff --git a/api/controllers/console/app/conversation.py b/api/controllers/console/app/conversation.py index f7890de1cb..71b52d5ceb 100644 --- a/api/controllers/console/app/conversation.py +++ b/api/controllers/console/app/conversation.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone import pytz from flask_login import current_user @@ -262,7 +262,7 @@ def _get_conversation(app_model, conversation_id): raise NotFound("Conversation Not Exists.") if not conversation.read_at: - conversation.read_at = datetime.utcnow() + conversation.read_at = datetime.now(timezone.utc).replace(tzinfo=None) conversation.read_account_id = current_user.id db.session.commit() diff --git a/api/controllers/console/auth/activate.py b/api/controllers/console/auth/activate.py index 20e028af99..8efb55cdb6 100644 --- a/api/controllers/console/auth/activate.py +++ b/api/controllers/console/auth/activate.py @@ -1,6 +1,6 @@ import base64 +import datetime import secrets -from datetime import datetime from flask_restful import Resource, reqparse @@ -66,7 +66,7 @@ class ActivateApi(Resource): account.timezone = args['timezone'] account.interface_theme = 'light' account.status = AccountStatus.ACTIVE.value - account.initialized_at = datetime.utcnow() + account.initialized_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() return {'result': 'success'} diff --git a/api/controllers/console/auth/oauth.py b/api/controllers/console/auth/oauth.py index 05b1c36873..e5b80e9a57 100644 --- a/api/controllers/console/auth/oauth.py +++ b/api/controllers/console/auth/oauth.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime +from datetime import datetime, timezone from typing import Optional import requests @@ -73,7 +73,7 @@ class OAuthCallback(Resource): if account.status == AccountStatus.PENDING.value: account.status = AccountStatus.ACTIVE.value - account.initialized_at = datetime.utcnow() + account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() TenantService.create_owner_tenant_if_not_exist(account) diff --git a/api/controllers/console/datasets/data_source.py b/api/controllers/console/datasets/data_source.py index f3e639c6ac..8b210cc756 100644 --- a/api/controllers/console/datasets/data_source.py +++ b/api/controllers/console/datasets/data_source.py @@ -80,7 +80,7 @@ class DataSourceApi(Resource): if action == 'enable': if data_source_binding.disabled: data_source_binding.disabled = False - data_source_binding.updated_at = datetime.datetime.utcnow() + data_source_binding.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.add(data_source_binding) db.session.commit() else: @@ -89,7 +89,7 @@ class DataSourceApi(Resource): if action == 'disable': if not data_source_binding.disabled: data_source_binding.disabled = True - data_source_binding.updated_at = datetime.datetime.utcnow() + data_source_binding.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.add(data_source_binding) db.session.commit() else: diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index c383cdc762..3d6daa7682 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from flask import request from flask_login import current_user @@ -637,7 +637,7 @@ class DocumentProcessingApi(DocumentResource): raise InvalidActionError('Document not in indexing state.') document.paused_by = current_user.id - document.paused_at = datetime.utcnow() + document.paused_at = datetime.now(timezone.utc).replace(tzinfo=None) document.is_paused = True db.session.commit() @@ -717,7 +717,7 @@ class DocumentMetadataApi(DocumentResource): document.doc_metadata[key] = value document.doc_type = doc_type - document.updated_at = datetime.utcnow() + document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return {'result': 'success', 'message': 'Document metadata updated.'}, 200 @@ -755,7 +755,7 @@ class DocumentStatusApi(DocumentResource): document.enabled = True document.disabled_at = None document.disabled_by = None - document.updated_at = datetime.utcnow() + document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() # Set cache to prevent indexing the same document multiple times @@ -772,9 +772,9 @@ class DocumentStatusApi(DocumentResource): raise InvalidActionError('Document already disabled.') document.enabled = False - document.disabled_at = datetime.utcnow() + document.disabled_at = datetime.now(timezone.utc).replace(tzinfo=None) document.disabled_by = current_user.id - document.updated_at = datetime.utcnow() + document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() # Set cache to prevent indexing the same document multiple times @@ -789,9 +789,9 @@ class DocumentStatusApi(DocumentResource): raise InvalidActionError('Document already archived.') document.archived = True - document.archived_at = datetime.utcnow() + document.archived_at = datetime.now(timezone.utc).replace(tzinfo=None) document.archived_by = current_user.id - document.updated_at = datetime.utcnow() + document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() if document.enabled: @@ -808,7 +808,7 @@ class DocumentStatusApi(DocumentResource): document.archived = False document.archived_at = None document.archived_by = None - document.updated_at = datetime.utcnow() + document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() # Set cache to prevent indexing the same document multiple times diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 7b58120a58..0a88a0d8d4 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime +from datetime import datetime, timezone import pandas as pd from flask import request @@ -192,7 +192,7 @@ class DatasetDocumentSegmentApi(Resource): raise InvalidActionError("Segment is already disabled.") segment.enabled = False - segment.disabled_at = datetime.utcnow() + segment.disabled_at = datetime.now(timezone.utc).replace(tzinfo=None) segment.disabled_by = current_user.id db.session.commit() diff --git a/api/controllers/console/explore/completion.py b/api/controllers/console/explore/completion.py index 292b4ed2a0..869b56e13b 100644 --- a/api/controllers/console/explore/completion.py +++ b/api/controllers/console/explore/completion.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime +from datetime import datetime, timezone from flask_login import current_user from flask_restful import reqparse @@ -47,7 +47,7 @@ class CompletionApi(InstalledAppResource): streaming = args['response_mode'] == 'streaming' args['auto_generate_name'] = False - installed_app.last_used_at = datetime.utcnow() + installed_app.last_used_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() try: @@ -110,7 +110,7 @@ class ChatApi(InstalledAppResource): args['auto_generate_name'] = False - installed_app.last_used_at = datetime.utcnow() + installed_app.last_used_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() try: diff --git a/api/controllers/console/explore/installed_app.py b/api/controllers/console/explore/installed_app.py index 7d6231270f..ec7bbed307 100644 --- a/api/controllers/console/explore/installed_app.py +++ b/api/controllers/console/explore/installed_app.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from flask_login import current_user from flask_restful import Resource, inputs, marshal_with, reqparse @@ -81,7 +81,7 @@ class InstalledAppsListApi(Resource): tenant_id=current_tenant_id, app_owner_tenant_id=app.tenant_id, is_pinned=False, - last_used_at=datetime.utcnow() + last_used_at=datetime.now(timezone.utc).replace(tzinfo=None) ) db.session.add(new_installed_app) db.session.commit() diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 656a4d4cee..198409bba7 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -1,4 +1,4 @@ -from datetime import datetime +import datetime import pytz from flask import current_app, request @@ -59,7 +59,7 @@ class AccountInitApi(Resource): raise InvalidInvitationCodeError() invitation_code.status = 'used' - invitation_code.used_at = datetime.utcnow() + invitation_code.used_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) invitation_code.used_by_tenant_id = account.current_tenant_id invitation_code.used_by_account_id = account.id @@ -67,7 +67,7 @@ class AccountInitApi(Resource): account.timezone = args['timezone'] account.interface_theme = 'light' account.status = 'active' - account.initialized_at = datetime.utcnow() + account.initialized_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() return {'result': 'success'} diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index a75583469e..70733d63f4 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -1,5 +1,5 @@ from collections.abc import Callable -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from functools import wraps from typing import Optional @@ -183,7 +183,7 @@ def validate_and_get_api_token(scope=None): if not api_token: raise Unauthorized("Access token is invalid") - api_token.last_used_at = datetime.utcnow() + api_token.last_used_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return api_token diff --git a/api/core/agent/base_agent_runner.py b/api/core/agent/base_agent_runner.py index dabc13374a..e5b4b9a4cd 100644 --- a/api/core/agent/base_agent_runner.py +++ b/api/core/agent/base_agent_runner.py @@ -1,7 +1,7 @@ import json import logging import uuid -from datetime import datetime +from datetime import datetime, timezone from typing import Optional, Union, cast from core.agent.entities import AgentEntity, AgentToolEntity @@ -440,7 +440,7 @@ class BaseAgentRunner(AppRunner): ToolConversationVariables.conversation_id == self.message.conversation_id, ).first() - db_variables.updated_at = datetime.utcnow() + db_variables.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db_variables.variables_str = json.dumps(jsonable_encoder(tool_variables.pool)) db.session.commit() db.session.close() diff --git a/api/core/app/task_pipeline/workflow_cycle_manage.py b/api/core/app/task_pipeline/workflow_cycle_manage.py index 49c79a5c05..48ff34fef9 100644 --- a/api/core/app/task_pipeline/workflow_cycle_manage.py +++ b/api/core/app/task_pipeline/workflow_cycle_manage.py @@ -1,6 +1,6 @@ import json import time -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Optional, Union, cast from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom, WorkflowAppGenerateEntity @@ -120,7 +120,7 @@ class WorkflowCycleManage: workflow_run.elapsed_time = time.perf_counter() - start_at workflow_run.total_tokens = total_tokens workflow_run.total_steps = total_steps - workflow_run.finished_at = datetime.utcnow() + workflow_run.finished_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() db.session.refresh(workflow_run) @@ -149,7 +149,7 @@ class WorkflowCycleManage: workflow_run.elapsed_time = time.perf_counter() - start_at workflow_run.total_tokens = total_tokens workflow_run.total_steps = total_steps - workflow_run.finished_at = datetime.utcnow() + workflow_run.finished_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() db.session.refresh(workflow_run) @@ -223,7 +223,7 @@ class WorkflowCycleManage: workflow_node_execution.outputs = json.dumps(outputs) if outputs else None workflow_node_execution.execution_metadata = json.dumps(jsonable_encoder(execution_metadata)) \ if execution_metadata else None - workflow_node_execution.finished_at = datetime.utcnow() + workflow_node_execution.finished_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() db.session.refresh(workflow_node_execution) @@ -251,7 +251,7 @@ class WorkflowCycleManage: workflow_node_execution.status = WorkflowNodeExecutionStatus.FAILED.value workflow_node_execution.error = error workflow_node_execution.elapsed_time = time.perf_counter() - start_at - workflow_node_execution.finished_at = datetime.utcnow() + workflow_node_execution.finished_at = datetime.now(timezone.utc).replace(tzinfo=None) workflow_node_execution.inputs = json.dumps(inputs) if inputs else None workflow_node_execution.process_data = json.dumps(process_data) if process_data else None workflow_node_execution.outputs = json.dumps(outputs) if outputs else None diff --git a/api/core/entities/provider_configuration.py b/api/core/entities/provider_configuration.py index b83ae0c8e7..303034693d 100644 --- a/api/core/entities/provider_configuration.py +++ b/api/core/entities/provider_configuration.py @@ -203,7 +203,7 @@ class ProviderConfiguration(BaseModel): if provider_record: provider_record.encrypted_config = json.dumps(credentials) provider_record.is_valid = True - provider_record.updated_at = datetime.datetime.utcnow() + provider_record.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() else: provider_record = Provider( @@ -351,7 +351,7 @@ class ProviderConfiguration(BaseModel): if provider_model_record: provider_model_record.encrypted_config = json.dumps(credentials) provider_model_record.is_valid = True - provider_model_record.updated_at = datetime.datetime.utcnow() + provider_model_record.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() else: provider_model_record = ProviderModel( diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 130884679b..51e77393d6 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -81,7 +81,7 @@ class IndexingRunner: except ProviderTokenNotInitError as e: dataset_document.indexing_status = 'error' dataset_document.error = str(e.description) - dataset_document.stopped_at = datetime.datetime.utcnow() + dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() except ObjectDeletedError: logging.warning('Document deleted, document id: {}'.format(dataset_document.id)) @@ -89,7 +89,7 @@ class IndexingRunner: logging.exception("consume document failed") dataset_document.indexing_status = 'error' dataset_document.error = str(e) - dataset_document.stopped_at = datetime.datetime.utcnow() + dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() def run_in_splitting_status(self, dataset_document: DatasetDocument): @@ -140,13 +140,13 @@ class IndexingRunner: except ProviderTokenNotInitError as e: dataset_document.indexing_status = 'error' dataset_document.error = str(e.description) - dataset_document.stopped_at = datetime.datetime.utcnow() + dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() except Exception as e: logging.exception("consume document failed") dataset_document.indexing_status = 'error' dataset_document.error = str(e) - dataset_document.stopped_at = datetime.datetime.utcnow() + dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() def run_in_indexing_status(self, dataset_document: DatasetDocument): @@ -202,13 +202,13 @@ class IndexingRunner: except ProviderTokenNotInitError as e: dataset_document.indexing_status = 'error' dataset_document.error = str(e.description) - dataset_document.stopped_at = datetime.datetime.utcnow() + dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() except Exception as e: logging.exception("consume document failed") dataset_document.indexing_status = 'error' dataset_document.error = str(e) - dataset_document.stopped_at = datetime.datetime.utcnow() + dataset_document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() def indexing_estimate(self, tenant_id: str, extract_settings: list[ExtractSetting], tmp_processing_rule: dict, @@ -382,7 +382,7 @@ class IndexingRunner: after_indexing_status="splitting", extra_update_params={ DatasetDocument.word_count: sum([len(text_doc.page_content) for text_doc in text_docs]), - DatasetDocument.parsing_completed_at: datetime.datetime.utcnow() + DatasetDocument.parsing_completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) } ) @@ -467,7 +467,7 @@ class IndexingRunner: doc_store.add_documents(documents) # update document status to indexing - cur_time = datetime.datetime.utcnow() + cur_time = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) self._update_document_index_status( document_id=dataset_document.id, after_indexing_status="indexing", @@ -482,7 +482,7 @@ class IndexingRunner: dataset_document_id=dataset_document.id, update_params={ DocumentSegment.status: "indexing", - DocumentSegment.indexing_at: datetime.datetime.utcnow() + DocumentSegment.indexing_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) } ) @@ -685,7 +685,7 @@ class IndexingRunner: after_indexing_status="completed", extra_update_params={ DatasetDocument.tokens: tokens, - DatasetDocument.completed_at: datetime.datetime.utcnow(), + DatasetDocument.completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), DatasetDocument.indexing_latency: indexing_end_at - indexing_start_at, } ) @@ -706,7 +706,7 @@ class IndexingRunner: ).update({ DocumentSegment.status: "completed", DocumentSegment.enabled: True, - DocumentSegment.completed_at: datetime.datetime.utcnow() + DocumentSegment.completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) }) db.session.commit() @@ -739,7 +739,7 @@ class IndexingRunner: ).update({ DocumentSegment.status: "completed", DocumentSegment.enabled: True, - DocumentSegment.completed_at: datetime.datetime.utcnow() + DocumentSegment.completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) }) db.session.commit() @@ -838,7 +838,7 @@ class IndexingRunner: doc_store.add_documents(documents) # update document status to indexing - cur_time = datetime.datetime.utcnow() + cur_time = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) self._update_document_index_status( document_id=dataset_document.id, after_indexing_status="indexing", @@ -853,7 +853,7 @@ class IndexingRunner: dataset_document_id=dataset_document.id, update_params={ DocumentSegment.status: "indexing", - DocumentSegment.indexing_at: datetime.datetime.utcnow() + DocumentSegment.indexing_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) } ) pass diff --git a/api/events/event_handlers/create_document_index.py b/api/events/event_handlers/create_document_index.py index 0b281c9271..68dae5a553 100644 --- a/api/events/event_handlers/create_document_index.py +++ b/api/events/event_handlers/create_document_index.py @@ -29,7 +29,7 @@ def handle(sender, **kwargs): raise NotFound('Document not found') document.indexing_status = 'parsing' - document.processing_started_at = datetime.datetime.utcnow() + document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) documents.append(document) db.session.add(document) db.session.commit() diff --git a/api/events/event_handlers/update_provider_last_used_at_when_messaeg_created.py b/api/events/event_handlers/update_provider_last_used_at_when_messaeg_created.py index ae983cc5d1..81cb861180 100644 --- a/api/events/event_handlers/update_provider_last_used_at_when_messaeg_created.py +++ b/api/events/event_handlers/update_provider_last_used_at_when_messaeg_created.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, ChatAppGenerateEntity from events.message_event import message_was_created @@ -17,5 +17,5 @@ def handle(sender, **kwargs): db.session.query(Provider).filter( Provider.tenant_id == application_generate_entity.app_config.tenant_id, Provider.provider_name == application_generate_entity.model_config.provider - ).update({'last_used': datetime.utcnow()}) + ).update({'last_used': datetime.now(timezone.utc).replace(tzinfo=None)}) db.session.commit() diff --git a/api/extensions/ext_storage.py b/api/extensions/ext_storage.py index 943cf4f58d..c2104cdf55 100644 --- a/api/extensions/ext_storage.py +++ b/api/extensions/ext_storage.py @@ -2,7 +2,7 @@ import os import shutil from collections.abc import Generator from contextlib import closing -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Union import boto3 @@ -38,7 +38,7 @@ class Storage: account_key=app.config.get('AZURE_BLOB_ACCOUNT_KEY'), resource_types=ResourceTypes(service=True, container=True, object=True), permission=AccountSasPermissions(read=True, write=True, delete=True, list=True, add=True, create=True), - expiry=datetime.utcnow() + timedelta(hours=1) + expiry=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(hours=1) ) self.client = BlobServiceClient(account_url=app.config.get('AZURE_BLOB_ACCOUNT_URL'), credential=sas_token) diff --git a/api/models/task.py b/api/models/task.py index 2a1bfa124f..618d831d8e 100644 --- a/api/models/task.py +++ b/api/models/task.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from celery import states @@ -15,8 +15,8 @@ class CeleryTask(db.Model): task_id = db.Column(db.String(155), unique=True) status = db.Column(db.String(50), default=states.PENDING) result = db.Column(db.PickleType, nullable=True) - date_done = db.Column(db.DateTime, default=datetime.utcnow, - onupdate=datetime.utcnow, nullable=True) + date_done = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc).replace(tzinfo=None), + onupdate=lambda: datetime.now(timezone.utc).replace(tzinfo=None), nullable=True) traceback = db.Column(db.Text, nullable=True) name = db.Column(db.String(155), nullable=True) args = db.Column(db.LargeBinary, nullable=True) @@ -35,5 +35,5 @@ class CeleryTaskSet(db.Model): autoincrement=True, primary_key=True) taskset_id = db.Column(db.String(155), unique=True) result = db.Column(db.PickleType, nullable=True) - date_done = db.Column(db.DateTime, default=datetime.utcnow, + date_done = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc).replace(tzinfo=None), nullable=True) diff --git a/api/services/account_service.py b/api/services/account_service.py index 8b08a5f816..1fe8da760c 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -2,7 +2,7 @@ import base64 import logging import secrets import uuid -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from hashlib import sha256 from typing import Any, Optional @@ -59,8 +59,8 @@ class AccountService: available_ta.current = True db.session.commit() - if datetime.utcnow() - account.last_active_at > timedelta(minutes=10): - account.last_active_at = datetime.utcnow() + if datetime.now(timezone.utc).replace(tzinfo=None) - account.last_active_at > timedelta(minutes=10): + account.last_active_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return account @@ -70,7 +70,7 @@ class AccountService: def get_account_jwt_token(account): payload = { "user_id": account.id, - "exp": datetime.utcnow() + timedelta(days=30), + "exp": datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30), "iss": current_app.config['EDITION'], "sub": 'Console API Passport', } @@ -91,7 +91,7 @@ class AccountService: if account.status == AccountStatus.PENDING.value: account.status = AccountStatus.ACTIVE.value - account.initialized_at = datetime.utcnow() + account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() if account.password is None or not compare_password(password, account.password, account.password_salt): @@ -163,7 +163,7 @@ class AccountService: # If it exists, update the record account_integrate.open_id = open_id account_integrate.encrypted_token = "" # todo - account_integrate.updated_at = datetime.utcnow() + account_integrate.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) else: # If it does not exist, create a new record account_integrate = AccountIntegrate(account_id=account.id, provider=provider, open_id=open_id, @@ -197,7 +197,7 @@ class AccountService: @staticmethod def update_last_login(account: Account, request) -> None: """Update last login time and ip""" - account.last_login_at = datetime.utcnow() + account.last_login_at = datetime.now(timezone.utc).replace(tzinfo=None) account.last_login_ip = get_remote_ip(request) db.session.add(account) db.session.commit() @@ -431,7 +431,7 @@ class RegisterService: password=password ) account.status = AccountStatus.ACTIVE.value if not status else status.value - account.initialized_at = datetime.utcnow() + account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None) if open_id is not None or provider is not None: AccountService.link_account_integrate(provider, open_id, account) diff --git a/api/services/annotation_service.py b/api/services/annotation_service.py index db4639d40b..8b00b28c4f 100644 --- a/api/services/annotation_service.py +++ b/api/services/annotation_service.py @@ -415,7 +415,7 @@ class AppAnnotationService: raise NotFound("App annotation not found") annotation_setting.score_threshold = args['score_threshold'] annotation_setting.updated_user_id = current_user.id - annotation_setting.updated_at = datetime.datetime.utcnow() + annotation_setting.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.add(annotation_setting) db.session.commit() diff --git a/api/services/app_service.py b/api/services/app_service.py index 673fa1e86b..c2f7cbb02c 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -1,6 +1,6 @@ import json import logging -from datetime import datetime +from datetime import datetime, timezone from typing import cast import yaml @@ -251,7 +251,7 @@ class AppService: app.description = args.get('description', '') app.icon = args.get('icon') app.icon_background = args.get('icon_background') - app.updated_at = datetime.utcnow() + app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return app @@ -264,7 +264,7 @@ class AppService: :return: App instance """ app.name = name - app.updated_at = datetime.utcnow() + app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return app @@ -279,7 +279,7 @@ class AppService: """ app.icon = icon app.icon_background = icon_background - app.updated_at = datetime.utcnow() + app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return app @@ -295,7 +295,7 @@ class AppService: return app app.enable_site = enable_site - app.updated_at = datetime.utcnow() + app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return app @@ -311,7 +311,7 @@ class AppService: return app app.enable_api = enable_api - app.updated_at = datetime.utcnow() + app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return app diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 78b191ac31..fe95a22cac 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -415,7 +415,7 @@ class DocumentService: # update document to be paused document.is_paused = True document.paused_by = current_user.id - document.paused_at = datetime.datetime.utcnow() + document.paused_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.add(document) db.session.commit() @@ -739,7 +739,7 @@ class DocumentService: document.parsing_completed_at = None document.cleaning_completed_at = None document.splitting_completed_at = None - document.updated_at = datetime.datetime.utcnow() + document.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) document.created_from = created_from document.doc_form = document_data['doc_form'] db.session.add(document) @@ -1062,8 +1062,8 @@ class SegmentService: word_count=len(content), tokens=tokens, status='completed', - indexing_at=datetime.datetime.utcnow(), - completed_at=datetime.datetime.utcnow(), + indexing_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), + completed_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), created_by=current_user.id ) if document.doc_form == 'qa_model': @@ -1078,7 +1078,7 @@ class SegmentService: except Exception as e: logging.exception("create segment index failed") segment_document.enabled = False - segment_document.disabled_at = datetime.datetime.utcnow() + segment_document.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) segment_document.status = 'error' segment_document.error = str(e) db.session.commit() @@ -1128,8 +1128,8 @@ class SegmentService: word_count=len(content), tokens=tokens, status='completed', - indexing_at=datetime.datetime.utcnow(), - completed_at=datetime.datetime.utcnow(), + indexing_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), + completed_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), created_by=current_user.id ) if document.doc_form == 'qa_model': @@ -1147,7 +1147,7 @@ class SegmentService: logging.exception("create segment index failed") for segment_document in segment_data_list: segment_document.enabled = False - segment_document.disabled_at = datetime.datetime.utcnow() + segment_document.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) segment_document.status = 'error' segment_document.error = str(e) db.session.commit() @@ -1208,10 +1208,10 @@ class SegmentService: segment.word_count = len(content) segment.tokens = tokens segment.status = 'completed' - segment.indexing_at = datetime.datetime.utcnow() - segment.completed_at = datetime.datetime.utcnow() + segment.indexing_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) + segment.completed_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) segment.updated_by = current_user.id - segment.updated_at = datetime.datetime.utcnow() + segment.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) if document.doc_form == 'qa_model': segment.answer = args['answer'] db.session.add(segment) @@ -1221,7 +1221,7 @@ class SegmentService: except Exception as e: logging.exception("update segment index failed") segment.enabled = False - segment.disabled_at = datetime.datetime.utcnow() + segment.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) segment.status = 'error' segment.error = str(e) db.session.commit() diff --git a/api/services/file_service.py b/api/services/file_service.py index ab71ef02a0..f9df21f544 100644 --- a/api/services/file_service.py +++ b/api/services/file_service.py @@ -81,7 +81,7 @@ class FileService: mime_type=file.mimetype, created_by_role=('account' if isinstance(user, Account) else 'end_user'), created_by=user.id, - created_at=datetime.datetime.utcnow(), + created_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), used=False, hash=hashlib.sha3_256(file_content).hexdigest() ) @@ -111,10 +111,10 @@ class FileService: extension='txt', mime_type='text/plain', created_by=current_user.id, - created_at=datetime.datetime.utcnow(), + created_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), used=True, used_by=current_user.id, - used_at=datetime.datetime.utcnow() + used_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) ) db.session.add(upload_file) diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index d15c383b0b..e1cffdd1bd 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -1,6 +1,6 @@ import json import time -from datetime import datetime +from datetime import datetime, timezone from typing import Optional from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager @@ -93,7 +93,7 @@ class WorkflowService: workflow.graph = json.dumps(graph) workflow.features = json.dumps(features) workflow.updated_by = account.id - workflow.updated_at = datetime.utcnow() + workflow.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) # commit db session changes db.session.commit() @@ -123,7 +123,7 @@ class WorkflowService: tenant_id=app_model.tenant_id, app_id=app_model.id, type=draft_workflow.type, - version=str(datetime.utcnow()), + version=str(datetime.now(timezone.utc).replace(tzinfo=None)), graph=draft_workflow.graph, features=draft_workflow.features, created_by=account.id @@ -202,8 +202,8 @@ class WorkflowService: elapsed_time=time.perf_counter() - start_at, created_by_role=CreatedByRole.ACCOUNT.value, created_by=account.id, - created_at=datetime.utcnow(), - finished_at=datetime.utcnow() + created_at=datetime.now(timezone.utc).replace(tzinfo=None), + finished_at=datetime.now(timezone.utc).replace(tzinfo=None) ) db.session.add(workflow_node_execution) db.session.commit() @@ -230,8 +230,8 @@ class WorkflowService: elapsed_time=time.perf_counter() - start_at, created_by_role=CreatedByRole.ACCOUNT.value, created_by=account.id, - created_at=datetime.utcnow(), - finished_at=datetime.utcnow() + created_at=datetime.now(timezone.utc).replace(tzinfo=None), + finished_at=datetime.now(timezone.utc).replace(tzinfo=None) ) else: # create workflow node execution @@ -249,8 +249,8 @@ class WorkflowService: elapsed_time=time.perf_counter() - start_at, created_by_role=CreatedByRole.ACCOUNT.value, created_by=account.id, - created_at=datetime.utcnow(), - finished_at=datetime.utcnow() + created_at=datetime.now(timezone.utc).replace(tzinfo=None), + finished_at=datetime.now(timezone.utc).replace(tzinfo=None) ) db.session.add(workflow_node_execution) diff --git a/api/tasks/add_document_to_index_task.py b/api/tasks/add_document_to_index_task.py index a26ecf5526..e0a1b21909 100644 --- a/api/tasks/add_document_to_index_task.py +++ b/api/tasks/add_document_to_index_task.py @@ -70,7 +70,7 @@ def add_document_to_index_task(dataset_document_id: str): except Exception as e: logging.exception("add document to index failed") dataset_document.enabled = False - dataset_document.disabled_at = datetime.datetime.utcnow() + dataset_document.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) dataset_document.status = 'error' dataset_document.error = str(e) db.session.commit() diff --git a/api/tasks/annotation/enable_annotation_reply_task.py b/api/tasks/annotation/enable_annotation_reply_task.py index 666fa8692f..fda8b7a250 100644 --- a/api/tasks/annotation/enable_annotation_reply_task.py +++ b/api/tasks/annotation/enable_annotation_reply_task.py @@ -50,7 +50,7 @@ def enable_annotation_reply_task(job_id: str, app_id: str, user_id: str, tenant_ annotation_setting.score_threshold = score_threshold annotation_setting.collection_binding_id = dataset_collection_binding.id annotation_setting.updated_user_id = user_id - annotation_setting.updated_at = datetime.datetime.utcnow() + annotation_setting.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.add(annotation_setting) else: new_app_annotation_setting = AppAnnotationSetting( diff --git a/api/tasks/batch_create_segment_to_index_task.py b/api/tasks/batch_create_segment_to_index_task.py index 9e41c6aee3..d6dc970477 100644 --- a/api/tasks/batch_create_segment_to_index_task.py +++ b/api/tasks/batch_create_segment_to_index_task.py @@ -85,9 +85,9 @@ def batch_create_segment_to_index_task(job_id: str, content: list, dataset_id: s word_count=len(content), tokens=tokens, created_by=user_id, - indexing_at=datetime.datetime.utcnow(), + indexing_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), status='completed', - completed_at=datetime.datetime.utcnow() + completed_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) ) if dataset_document.doc_form == 'qa_model': segment_document.answer = segment['answer'] diff --git a/api/tasks/create_segment_to_index_task.py b/api/tasks/create_segment_to_index_task.py index f33a1e91bf..d31286e4cc 100644 --- a/api/tasks/create_segment_to_index_task.py +++ b/api/tasks/create_segment_to_index_task.py @@ -38,7 +38,7 @@ def create_segment_to_index_task(segment_id: str, keywords: Optional[list[str]] # update segment status to indexing update_params = { DocumentSegment.status: "indexing", - DocumentSegment.indexing_at: datetime.datetime.utcnow() + DocumentSegment.indexing_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) } DocumentSegment.query.filter_by(id=segment.id).update(update_params) db.session.commit() @@ -75,7 +75,7 @@ def create_segment_to_index_task(segment_id: str, keywords: Optional[list[str]] # update segment to completed update_params = { DocumentSegment.status: "completed", - DocumentSegment.completed_at: datetime.datetime.utcnow() + DocumentSegment.completed_at: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) } DocumentSegment.query.filter_by(id=segment.id).update(update_params) db.session.commit() @@ -85,7 +85,7 @@ def create_segment_to_index_task(segment_id: str, keywords: Optional[list[str]] except Exception as e: logging.exception("create segment to index failed") segment.enabled = False - segment.disabled_at = datetime.datetime.utcnow() + segment.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) segment.status = 'error' segment.error = str(e) db.session.commit() diff --git a/api/tasks/document_indexing_sync_task.py b/api/tasks/document_indexing_sync_task.py index a646158dbd..c35c18799a 100644 --- a/api/tasks/document_indexing_sync_task.py +++ b/api/tasks/document_indexing_sync_task.py @@ -67,7 +67,7 @@ def document_indexing_sync_task(dataset_id: str, document_id: str): # check the page is updated if last_edited_time != page_edited_time: document.indexing_status = 'parsing' - document.processing_started_at = datetime.datetime.utcnow() + document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() # delete all document segment and index diff --git a/api/tasks/document_indexing_task.py b/api/tasks/document_indexing_task.py index b776207050..43d1cc13f9 100644 --- a/api/tasks/document_indexing_task.py +++ b/api/tasks/document_indexing_task.py @@ -47,7 +47,7 @@ def document_indexing_task(dataset_id: str, document_ids: list): if document: document.indexing_status = 'error' document.error = str(e) - document.stopped_at = datetime.datetime.utcnow() + document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.add(document) db.session.commit() return @@ -62,7 +62,7 @@ def document_indexing_task(dataset_id: str, document_ids: list): if document: document.indexing_status = 'parsing' - document.processing_started_at = datetime.datetime.utcnow() + document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) documents.append(document) db.session.add(document) db.session.commit() diff --git a/api/tasks/document_indexing_update_task.py b/api/tasks/document_indexing_update_task.py index e59c549a65..b27274be37 100644 --- a/api/tasks/document_indexing_update_task.py +++ b/api/tasks/document_indexing_update_task.py @@ -33,7 +33,7 @@ def document_indexing_update_task(dataset_id: str, document_id: str): raise NotFound('Document not found') document.indexing_status = 'parsing' - document.processing_started_at = datetime.datetime.utcnow() + document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() # delete all document segment and index diff --git a/api/tasks/enable_segment_to_index_task.py b/api/tasks/enable_segment_to_index_task.py index a6254a822d..e37c06855d 100644 --- a/api/tasks/enable_segment_to_index_task.py +++ b/api/tasks/enable_segment_to_index_task.py @@ -69,7 +69,7 @@ def enable_segment_to_index_task(segment_id: str): except Exception as e: logging.exception("enable segment to index failed") segment.enabled = False - segment.disabled_at = datetime.datetime.utcnow() + segment.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) segment.status = 'error' segment.error = str(e) db.session.commit()