fix(normalize): 原子化解析容错+产出率兜底——LLM输出no为字符串/浮点时isinstance(int)全过滤致46文档仅6原子;int()转换+范围校验;批LLM失败用标题作原子;原子覆盖<50%文档时缺者标题兜底(众包标题即浓缩语义,保证覆盖率可算)
This commit is contained in:
parent
eba6285841
commit
ca921d3550
@ -128,17 +128,33 @@ async def _atomize(sor, snaps, cfg, ctx):
|
|||||||
arr = json.loads(raw)
|
arr = json.loads(raw)
|
||||||
items = []
|
items = []
|
||||||
for it in arr if isinstance(arr, list) else []:
|
for it in arr if isinstance(arr, list) else []:
|
||||||
|
if not isinstance(it, dict):
|
||||||
|
continue
|
||||||
no = it.get("no")
|
no = it.get("no")
|
||||||
atom = str(it.get("atom") or "").strip()[:255]
|
atom = str(it.get("atom") or "").strip()[:255]
|
||||||
if isinstance(no, int) and 1 <= no <= len(chunk) and atom:
|
# 容错:LLM 可能输出 "1"/1.0/True,统一 int() 转换后范围校验
|
||||||
items.append((chunk[no - 1]["id"], atom))
|
try:
|
||||||
|
no_i = int(no)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if 1 <= no_i <= len(chunk) and atom:
|
||||||
|
items.append((chunk[no_i - 1]["id"], atom))
|
||||||
async with lock:
|
async with lock:
|
||||||
results.extend(items)
|
results.extend(items)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("atomize chunk failed: %s", e)
|
logger.debug("atomize chunk failed: %s", e)
|
||||||
|
async with lock:
|
||||||
|
results.extend([(s["id"], s["title"][:255]) for s in chunk])
|
||||||
|
|
||||||
chunks = [snaps[i:i + bs] for i in range(0, len(snaps), bs)]
|
chunks = [snaps[i:i + bs] for i in range(0, len(snaps), bs)]
|
||||||
await asyncio.gather(*(_one(ch) for ch in chunks))
|
await asyncio.gather(*(_one(ch) for ch in chunks))
|
||||||
|
# 产出率兜底:原子数 < 文档数 50% 时,缺原子的文档用标题作原子
|
||||||
|
# (众包标题本身即浓缩需求语义;保证覆盖率统计可算,2026-09-12 实测 46 文档仅 6 原子)
|
||||||
|
covered = set(sid for sid, _ in results)
|
||||||
|
if len(covered) < len(snaps) * 0.5:
|
||||||
|
for s in snaps:
|
||||||
|
if s["id"] not in covered:
|
||||||
|
results.append((s["id"], s["title"][:255]))
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user