fix(poller): 依赖门控提前到候选筛选 + 项目轮询,修复队列饥饿

agent_poller 原来 SELECT 后 [:avail] 截断只取队列头 avail 个任务,依赖门控却
在认领阶段(_claim_task)才检查——依赖未满足的僵尸 submitted 任务(created_at 最早)
永久占住队头,导致其他项目/新任务饿死(一项目卡死全平台瘫痪)。

改动:
1. 候选 SELECT 带上 depends_on,批量查依赖状态,排除依赖未满足的任务(终态=completed/approved)
2. 项目轮询(round-robin): 按 tenant_id 分组轮流取就绪任务,保证项目间公平
3. SELECT LIMIT 20->200 容纳僵尸任务;批量查依赖替代 N+1
This commit is contained in:
ymq 2026-08-20 11:34:49 +08:00
parent 7f05a3b467
commit 77f5aa45dc

View File

@ -610,28 +610,82 @@ def load_pipeline_service():
# 达到并发上限,本轮不派发(退出 context 后末尾 sleep 再 poll
return
def _deps_of(depends_on_raw):
"""解析 depends_onJSON 数组字符串或 list→ 依赖任务 ID 列表。空/非法 → []"""
if not depends_on_raw:
return []
try:
d = json.loads(depends_on_raw) if isinstance(depends_on_raw, str) else depends_on_raw
return [str(x) for x in d if x] if isinstance(d, list) else []
except (json.JSONDecodeError, TypeError, ValueError):
return []
# 候选任务(含 depends_onLIMIT 放大以容纳「依赖未满足排在前面」的僵尸任务)
recs = await sor.sqlExe(
"SELECT id, tenant_id, role FROM pipeline_tasks "
"SELECT id, tenant_id, role, depends_on FROM pipeline_tasks "
"WHERE state='submitted' AND pipeline_id='role_task' "
"AND claimed_by IS NULL ORDER BY created_at ASC LIMIT 20",
"AND claimed_by IS NULL ORDER BY created_at ASC LIMIT 200",
{})
for rec in (recs or [])[:avail]:
# 依赖门控提前到候选筛选:批量查 depends_on 里所有依赖任务的状态,
# 排除依赖未满足的「僵尸 submitted」任务——否则它们占住队列头、饿死其他就绪任务
# (一项目卡死 → 全平台瘫痪)。终态 = completed/approved与 _task_deps_satisfied 一致。
all_dep_ids = set()
for rec in (recs or []):
all_dep_ids.update(_deps_of(getattr(rec, 'depends_on', '') or ''))
dep_state = {}
if all_dep_ids:
dep_ids = ",".join("'" + x.replace("'", "''") + "'" for x in all_dep_ids)
dep_rows = await sor.sqlExe(
"SELECT id, state FROM pipeline_tasks WHERE id IN (" + dep_ids + ")", {})
dep_state = {getattr(r_, 'id', ''): getattr(r_, 'state', '') for r_ in (dep_rows or [])}
await sor.sqlExe("COMMIT", {})
ready = []
for rec in (recs or []):
deps = _deps_of(getattr(rec, 'depends_on', '') or '')
# 依赖 id 查不到(任务已删)→ 视为未满足,保守不放行
if all(dep_state.get(d, '') in ('completed', 'approved') for d in deps):
ready.append(rec)
# 项目轮询round-robin每个项目轮流取一个就绪任务保证项目间公平
# 避免单项目大量就绪任务占满全局并发名额、饿死其他项目。dict 保持插入序)
by_project = {}
for rec in ready:
by_project.setdefault(getattr(rec, 'tenant_id', ''), []).append(rec)
selected = []
while len(selected) < avail and by_project:
progressed = False
for pid in list(by_project.keys()):
if len(selected) >= avail:
break
tasks = by_project[pid]
if tasks:
selected.append(tasks.pop(0))
progressed = True
if not tasks:
del by_project[pid]
if not progressed:
break
for rec in selected:
tid = getattr(rec, 'id', '')
pid = getattr(rec, 'tenant_id', '')
r = getattr(rec, 'role', '')
if tid and pid and r and tid not in _dispatched:
_dispatched.add(tid)
debug(f"agent_poller dispatching task={tid} role={r}")
if not (tid and pid and r) or tid in _dispatched:
continue
_dispatched.add(tid)
debug(f"agent_poller dispatching task={tid} role={r}")
async def _dispatch(tid, pid, r):
try:
await role_agent_loop(pid, r, agent_id=f"poller-{r}")
except Exception as e:
debug(f"agent_poller dispatch error: task={tid} err={e}")
finally:
_dispatched.discard(tid)
async def _dispatch(tid, pid, r):
try:
await role_agent_loop(pid, r, agent_id=f"poller-{r}")
except Exception as e:
debug(f"agent_poller dispatch error: task={tid} err={e}")
finally:
_dispatched.discard(tid)
asyncio.ensure_future(_dispatch(tid, pid, r))
asyncio.ensure_future(_dispatch(tid, pid, r))
async def _poll_loop():
while True: