add retry count to task (#2152)

### What problem does this PR solve?


### Type of change

- [x] New Feature (non-breaking change which adds functionality)

api/db/db_models.py

@@ -788,6 +788,7 @@ class Task(DataBaseModel):
         null=True,
         help_text="process message",
         default="")
+    retry_count = IntegerField(default=0)
 
 
 class Dialog(DataBaseModel):
@@ -982,3 +983,10 @@ def migrate_db():
             DB.execute_sql('ALTER TABLE llm ADD PRIMARY KEY (llm_name,fid);')
         except Exception as e:
             pass
+        try:
+            migrate(
+                migrator.add_column('task', 'retry_count', IntegerField(default=0))
+            )
+        except Exception as e:
+            pass
+

api/db/services/task_service.py

@@ -64,9 +64,20 @@ class TaskService(CommonService):
         docs = list(docs.dicts())
         if not docs: return []
 
-        cls.model.update(progress_msg=cls.model.progress_msg + "\n" + "Task has been received.",
+        msg = "\nTask has been received."
-                         progress=random.random() / 10.).where(
+        prog = random.random() / 10.
+        if docs[0]["retry_count"] >= 3:
+            msg = "\nERROR: Task is abandoned after 3 times attempts."
+            prog = -1
+
+        cls.model.update(progress_msg=cls.model.progress_msg + msg,
+                         progress=prog,
+                         retry_count=docs[0]["retry_count"]+1
+                         ).where(
             cls.model.id == docs[0]["id"]).execute()
+
+        if docs[0]["retry_count"] >= 3: return []
+
         return docs
 
     @classmethod