fix(poller): 依赖门控提前到候选筛选 + 项目轮询,修复队列饥饿
agent_poller 原来 SELECT 后 [:avail] 截断只取队列头 avail 个任务,依赖门控却 在认领阶段(_claim_task)才检查——依赖未满足的僵尸 submitted 任务(created_at 最早) 永久占住队头,导致其他项目/新任务饿死(一项目卡死全平台瘫痪)。 改动: 1. 候选 SELECT 带上 depends_on,批量查依赖状态,排除依赖未满足的任务(终态=completed/approved) 2. 项目轮询(round-robin): 按 tenant_id 分组轮流取就绪任务,保证项目间公平 3. SELECT LIMIT 20->200 容纳僵尸任务;批量查依赖替代 N+1
This commit is contained in:
parent
7f05a3b467
commit
77f5aa45dc
@ -610,28 +610,82 @@ def load_pipeline_service():
|
|||||||
# 达到并发上限,本轮不派发(退出 context 后末尾 sleep 再 poll)
|
# 达到并发上限,本轮不派发(退出 context 后末尾 sleep 再 poll)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
def _deps_of(depends_on_raw):
|
||||||
|
"""解析 depends_on(JSON 数组字符串或 list)→ 依赖任务 ID 列表。空/非法 → []"""
|
||||||
|
if not depends_on_raw:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
d = json.loads(depends_on_raw) if isinstance(depends_on_raw, str) else depends_on_raw
|
||||||
|
return [str(x) for x in d if x] if isinstance(d, list) else []
|
||||||
|
except (json.JSONDecodeError, TypeError, ValueError):
|
||||||
|
return []
|
||||||
|
|
||||||
|
# 候选任务(含 depends_on,LIMIT 放大以容纳「依赖未满足排在前面」的僵尸任务)
|
||||||
recs = await sor.sqlExe(
|
recs = await sor.sqlExe(
|
||||||
"SELECT id, tenant_id, role FROM pipeline_tasks "
|
"SELECT id, tenant_id, role, depends_on FROM pipeline_tasks "
|
||||||
"WHERE state='submitted' AND pipeline_id='role_task' "
|
"WHERE state='submitted' AND pipeline_id='role_task' "
|
||||||
"AND claimed_by IS NULL ORDER BY created_at ASC LIMIT 20",
|
"AND claimed_by IS NULL ORDER BY created_at ASC LIMIT 200",
|
||||||
{})
|
{})
|
||||||
for rec in (recs or [])[:avail]:
|
|
||||||
|
# 依赖门控提前到候选筛选:批量查 depends_on 里所有依赖任务的状态,
|
||||||
|
# 排除依赖未满足的「僵尸 submitted」任务——否则它们占住队列头、饿死其他就绪任务
|
||||||
|
# (一项目卡死 → 全平台瘫痪)。终态 = completed/approved,与 _task_deps_satisfied 一致。
|
||||||
|
all_dep_ids = set()
|
||||||
|
for rec in (recs or []):
|
||||||
|
all_dep_ids.update(_deps_of(getattr(rec, 'depends_on', '') or ''))
|
||||||
|
dep_state = {}
|
||||||
|
if all_dep_ids:
|
||||||
|
dep_ids = ",".join("'" + x.replace("'", "''") + "'" for x in all_dep_ids)
|
||||||
|
dep_rows = await sor.sqlExe(
|
||||||
|
"SELECT id, state FROM pipeline_tasks WHERE id IN (" + dep_ids + ")", {})
|
||||||
|
dep_state = {getattr(r_, 'id', ''): getattr(r_, 'state', '') for r_ in (dep_rows or [])}
|
||||||
|
await sor.sqlExe("COMMIT", {})
|
||||||
|
|
||||||
|
ready = []
|
||||||
|
for rec in (recs or []):
|
||||||
|
deps = _deps_of(getattr(rec, 'depends_on', '') or '')
|
||||||
|
# 依赖 id 查不到(任务已删)→ 视为未满足,保守不放行
|
||||||
|
if all(dep_state.get(d, '') in ('completed', 'approved') for d in deps):
|
||||||
|
ready.append(rec)
|
||||||
|
|
||||||
|
# 项目轮询(round-robin):每个项目轮流取一个就绪任务,保证项目间公平,
|
||||||
|
# 避免单项目大量就绪任务占满全局并发名额、饿死其他项目。(dict 保持插入序)
|
||||||
|
by_project = {}
|
||||||
|
for rec in ready:
|
||||||
|
by_project.setdefault(getattr(rec, 'tenant_id', ''), []).append(rec)
|
||||||
|
selected = []
|
||||||
|
while len(selected) < avail and by_project:
|
||||||
|
progressed = False
|
||||||
|
for pid in list(by_project.keys()):
|
||||||
|
if len(selected) >= avail:
|
||||||
|
break
|
||||||
|
tasks = by_project[pid]
|
||||||
|
if tasks:
|
||||||
|
selected.append(tasks.pop(0))
|
||||||
|
progressed = True
|
||||||
|
if not tasks:
|
||||||
|
del by_project[pid]
|
||||||
|
if not progressed:
|
||||||
|
break
|
||||||
|
|
||||||
|
for rec in selected:
|
||||||
tid = getattr(rec, 'id', '')
|
tid = getattr(rec, 'id', '')
|
||||||
pid = getattr(rec, 'tenant_id', '')
|
pid = getattr(rec, 'tenant_id', '')
|
||||||
r = getattr(rec, 'role', '')
|
r = getattr(rec, 'role', '')
|
||||||
if tid and pid and r and tid not in _dispatched:
|
if not (tid and pid and r) or tid in _dispatched:
|
||||||
_dispatched.add(tid)
|
continue
|
||||||
debug(f"agent_poller dispatching task={tid} role={r}")
|
_dispatched.add(tid)
|
||||||
|
debug(f"agent_poller dispatching task={tid} role={r}")
|
||||||
|
|
||||||
async def _dispatch(tid, pid, r):
|
async def _dispatch(tid, pid, r):
|
||||||
try:
|
try:
|
||||||
await role_agent_loop(pid, r, agent_id=f"poller-{r}")
|
await role_agent_loop(pid, r, agent_id=f"poller-{r}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
debug(f"agent_poller dispatch error: task={tid} err={e}")
|
debug(f"agent_poller dispatch error: task={tid} err={e}")
|
||||||
finally:
|
finally:
|
||||||
_dispatched.discard(tid)
|
_dispatched.discard(tid)
|
||||||
|
|
||||||
asyncio.ensure_future(_dispatch(tid, pid, r))
|
asyncio.ensure_future(_dispatch(tid, pid, r))
|
||||||
|
|
||||||
async def _poll_loop():
|
async def _poll_loop():
|
||||||
while True:
|
while True:
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user