From f7a3fd364a83c16144eb803b72747eed6b473c66 Mon Sep 17 00:00:00 2001 From: ymq Date: Wed, 16 Sep 2026 18:21:19 +0800 Subject: [PATCH] =?UTF-8?q?fix(task):=20reset=5Ftask=5Fretry=E5=81=87?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E7=A9=BA=E8=BD=AC=E2=80=94=E2=80=94=E8=A7=A3?= =?UTF-8?q?=E9=93=BE=E4=B8=89=E6=AD=A5=E6=9B=B2(resolve=5Fproblem=20resume?= =?UTF-8?q?=5Ftask=3DTrue=E5=85=88=E6=8A=8Awaiting=E2=86=92submitted,?= =?UTF-8?q?=E5=86=8Dreset=5Ftask=5Fretry)=E4=B8=8B=E6=97=A7CAS=E5=8F=AA?= =?UTF-8?q?=E8=AE=A4waiting/failed=E2=86=92UPDATE=200=E8=A1=8C,=E6=88=90?= =?UTF-8?q?=E5=8A=9F=E5=88=A4=E5=AE=9A=E5=8D=B4=E5=8F=AA=E7=9C=8B=E7=BB=88?= =?UTF-8?q?=E6=80=81=3D=3Dsubmitted=E2=86=92=E7=A9=BA=E8=BD=AC=E6=8A=A5Tru?= =?UTF-8?q?e:retry=5Fcount=E6=9C=AA=E6=B8=85=E9=9B=B6+=E9=99=88=E6=97=A7la?= =?UTF-8?q?st=5Ferror=E6=AE=8B=E7=95=99,=E4=BB=BB=E5=8A=A1=E5=B8=A6?= =?UTF-8?q?=E6=BB=A1=E8=AE=A1=E6=95=B0=E9=87=8D=E6=96=B0=E8=AE=A4=E9=A2=86?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E9=80=80=E5=9B=9E=E5=8D=B3=E5=86=8D=E8=A7=A6?= =?UTF-8?q?=E9=A1=B6fault(2026-09-16=20pbls=20M1b=E5=AE=9E=E6=B5=8B17:22?= =?UTF-8?q?=20fault=E8=A7=A3=E9=93=BE=E5=90=8E18:08=20retry=5Fcount?= =?UTF-8?q?=E5=8F=88=3D3=E5=AE=9E=E9=94=A4)=E3=80=82=E4=BF=AE:state=20IN?= =?UTF-8?q?=E5=8A=A0submitted(running=E4=BB=8D=E6=8E=92=E9=99=A4=E9=98=B2?= =?UTF-8?q?=E6=89=93=E6=96=AD=E5=9C=A8=E8=B7=91=E5=8D=8F=E7=A8=8B)+?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E5=88=A4=E5=AE=9A=E6=94=B9state=E4=B8=8Eretr?= =?UTF-8?q?y=5Fcount=E5=8F=8C=E6=A0=B8=E5=AF=B9+=E5=85=A5=E5=8F=82?= =?UTF-8?q?=E6=80=81=E5=89=8D=E7=BD=AE=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline_service/task_capability.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pipeline_service/task_capability.py b/pipeline_service/task_capability.py index 58aae0a..64b4b0d 100644 --- a/pipeline_service/task_capability.py +++ b/pipeline_service/task_capability.py @@ -292,9 +292,16 @@ async def retry_task(task_id, tenant_id, who=None, agent_id=None): async def reset_task_retry(task_id, tenant_id, who=None, agent_id=None): - """人工处理后恢复：retry_count 归零，waiting/failed → submitted 重新执行。 + """人工处理后恢复：retry_count 归零，waiting/failed/submitted → submitted 重新执行。 任务重复超限后任务链暂停、故障抛给人工；人工处理完毕调用本函数把该任务重复数清零并重新执行。 + + state IN 含 submitted（2026-09-16 实测 bug）：标准解链三步 resolve_problem + (resume_task=True) → reset_task_retry → resume_project 中，resolve 已把 + waiting→submitted，本函数旧 CAS 只认 waiting/failed → UPDATE 0 行；而成功 + 判定只看「终态==submitted」→ 空转也报 True，retry_count 从未清零、陈旧 + last_error 残留，任务带着满重试计数重新认领，一次退回即再触顶 fault。 + running 态仍排除（防中途拽回 submitted 打断在跑协程）。 """ db, dbname = _get_db() async with db.sqlorContext(dbname) as sor: @@ -302,15 +309,20 @@ async def reset_task_retry(task_id, tenant_id, who=None, agent_id=None): if not recs: return False, "任务不存在" from_state = getattr(recs[0], 'state', '') + if from_state not in ('waiting', 'failed', 'submitted'): + return False, f"恢复失败(任务状态 {from_state} 不在 waiting/failed/submitted)" await sor.sqlExe( "UPDATE pipeline_tasks SET retry_count=0, state='submitted', " "claimed_by=NULL, last_error=NULL, updated_at=NOW() " - "WHERE id=${tid}$ AND tenant_id=${tn}$ AND state IN ('waiting','failed')", + "WHERE id=${tid}$ AND tenant_id=${tn}$ AND state IN ('waiting','failed','submitted')", {"tid": task_id, "tn": tenant_id}) chk = await sor.R('pipeline_tasks', {'id': task_id}) await sor.sqlExe("COMMIT", {}) - if not chk or getattr(chk[0], 'state', '') != S_SUBMITTED: - return False, "恢复失败(任务不在 waiting/failed 状态)" + # 成功判定=UPDATE 真实生效（state 与 retry_count 双核对），不只看终态—— + # 终态可能本来就是 submitted（空转假成功，见 docstring 实测事故）。 + if (not chk or getattr(chk[0], 'state', '') != S_SUBMITTED + or int(getattr(chk[0], 'retry_count', -1) or -1) != 0): + return False, "恢复失败(计数未清零或状态竞态，请重试)" await record_audit(tenant_id, 'pipeline_tasks', task_id, 'reset_retry', from_state=from_state, to_state=S_SUBMITTED, who=who, agent_id=agent_id, sor=sor)