fix(normalize): 原子化解析容错+产出率兜底——LLM输出no为字符串/浮点时isinstance(int)全过滤致46文档仅6原子;int()转换+范围校验;批LLM失败用标题作原子;原子覆盖<50%文档时缺者标题兜底(众包标题即浓缩语义,保证覆盖率可算)

This commit is contained in:
yumoqing 2026-09-12 12:01:04 +08:00
parent eba6285841
commit ca921d3550

View File

@ -128,17 +128,33 @@ async def _atomize(sor, snaps, cfg, ctx):
arr = json.loads(raw)
items = []
for it in arr if isinstance(arr, list) else []:
if not isinstance(it, dict):
continue
no = it.get("no")
atom = str(it.get("atom") or "").strip()[:255]
if isinstance(no, int) and 1 <= no <= len(chunk) and atom:
items.append((chunk[no - 1]["id"], atom))
# 容错:LLM 可能输出 "1"/1.0/True,统一 int() 转换后范围校验
try:
no_i = int(no)
except (TypeError, ValueError):
continue
if 1 <= no_i <= len(chunk) and atom:
items.append((chunk[no_i - 1]["id"], atom))
async with lock:
results.extend(items)
except Exception as e:
logger.debug("atomize chunk failed: %s", e)
async with lock:
results.extend([(s["id"], s["title"][:255]) for s in chunk])
chunks = [snaps[i:i + bs] for i in range(0, len(snaps), bs)]
await asyncio.gather(*(_one(ch) for ch in chunks))
# 产出率兜底:原子数 < 文档数 50% 时,缺原子的文档用标题作原子
# (众包标题本身即浓缩需求语义;保证覆盖率统计可算,2026-09-12 实测 46 文档仅 6 原子)
covered = set(sid for sid, _ in results)
if len(covered) < len(snaps) * 0.5:
for s in snaps:
if s["id"] not in covered:
results.append((s["id"], s["title"][:255]))
return results