deliver: 交付收口(引擎代为提交)
This commit is contained in:
parent
b546d4bf6f
commit
fc0ac8118a
@ -150,6 +150,34 @@ from .pbl_runtime_broadcast import (
|
||||
wrap_with_post_commit_broadcast,
|
||||
)
|
||||
|
||||
# --- M11b-4 3s 轮询兜底 + 端到端时延验证(真源:pbl_runtime_polling.py / pbl_runtime_latency.py) ---
|
||||
from .pbl_runtime_polling import (
|
||||
FALLBACK_SLO_MS,
|
||||
POLL_INTERVAL_S,
|
||||
POLL_LIMIT,
|
||||
POLL_TIMEOUT_S,
|
||||
SEEN_IDS_IN_SQL,
|
||||
PollingFallback,
|
||||
build_poll_sql,
|
||||
configure_fallback,
|
||||
explain_poll_query as polling_explain_poll_query,
|
||||
fallback_stats,
|
||||
get_fallback,
|
||||
normalize_watermark,
|
||||
plan_uses_scene_index,
|
||||
poll_since,
|
||||
reset_fallback,
|
||||
run_fallback,
|
||||
wait_event,
|
||||
)
|
||||
from .pbl_runtime_latency import (
|
||||
E2E_LATENCY_SLO_MS,
|
||||
latency_report,
|
||||
measure_once,
|
||||
run_chain,
|
||||
summarize_latency,
|
||||
)
|
||||
|
||||
#: 契约解析优先级键(与 pbl_runtime_tx_env.SERVER_ENV_KEYS 一致,单一来源)
|
||||
assert tuple(SERVER_ENV_CONN_FACTORY_KEYS) == tuple(SERVER_ENV_KEYS), \
|
||||
"world_sync: SERVER_ENV_CONN_FACTORY_KEYS 与 pbl_runtime_tx_env.SERVER_ENV_KEYS 分叉"
|
||||
@ -247,6 +275,29 @@ __all__ = [
|
||||
"broadcast_latency_percentile_ms",
|
||||
"drain_broadcast",
|
||||
"shutdown_broadcast",
|
||||
# M11b-4 3s 轮询兜底(消费端侧,只读,不侵入写入端)
|
||||
"POLL_INTERVAL_S",
|
||||
"POLL_TIMEOUT_S",
|
||||
"POLL_LIMIT",
|
||||
"FALLBACK_SLO_MS",
|
||||
"SEEN_IDS_IN_SQL",
|
||||
"PollingFallback",
|
||||
"build_poll_sql",
|
||||
"configure_fallback",
|
||||
"fallback_stats",
|
||||
"get_fallback",
|
||||
"normalize_watermark",
|
||||
"plan_uses_scene_index",
|
||||
"poll_since",
|
||||
"reset_fallback",
|
||||
"run_fallback",
|
||||
"wait_event",
|
||||
# M11b-4 端到端时延基准
|
||||
"E2E_LATENCY_SLO_MS",
|
||||
"measure_once",
|
||||
"run_chain",
|
||||
"summarize_latency",
|
||||
"latency_report",
|
||||
# 宿主基础包可用性诊断
|
||||
"HOST_DEPS_AVAILABLE",
|
||||
"HOST_DEPS_ERROR",
|
||||
|
||||
@ -178,6 +178,32 @@ from .pbl_runtime_broadcast import (
|
||||
wrap_with_post_commit_broadcast,
|
||||
)
|
||||
|
||||
# --- M11b-4: 3s 轮询兜底 + 端到端时延验证 注册接线 ---
|
||||
from .pbl_runtime_polling import (
|
||||
FALLBACK_SLO_MS,
|
||||
POLL_INTERVAL_S,
|
||||
POLL_LIMIT,
|
||||
POLL_TIMEOUT_S,
|
||||
PollingFallback,
|
||||
build_poll_sql,
|
||||
configure_fallback,
|
||||
fallback_stats,
|
||||
get_fallback,
|
||||
normalize_watermark,
|
||||
plan_uses_scene_index,
|
||||
poll_since,
|
||||
reset_fallback,
|
||||
run_fallback,
|
||||
wait_event,
|
||||
)
|
||||
from .pbl_runtime_latency import (
|
||||
E2E_LATENCY_SLO_MS,
|
||||
latency_report,
|
||||
measure_once,
|
||||
run_chain,
|
||||
summarize_latency,
|
||||
)
|
||||
|
||||
|
||||
MODULE_NAME = 'world_sync'
|
||||
MODULE_VERSION = '0.3.0-m11b3'
|
||||
@ -526,6 +552,30 @@ def load_world_sync(server_env=None, conn_factory=None):
|
||||
env.drain_broadcast = drain_broadcast
|
||||
env.shutdown_broadcast = shutdown_broadcast
|
||||
env.reset_broadcast = reset_broadcaster
|
||||
# --- M11b-4: 3s 轮询兜底(消费端侧,只读;不侵入 M11b-2 写入端) ---
|
||||
env.POLL_INTERVAL_S = POLL_INTERVAL_S
|
||||
env.POLL_TIMEOUT_S = POLL_TIMEOUT_S
|
||||
env.POLL_LIMIT = POLL_LIMIT
|
||||
env.FALLBACK_SLO_MS = FALLBACK_SLO_MS
|
||||
env.E2E_LATENCY_SLO_MS = E2E_LATENCY_SLO_MS
|
||||
env.PollingFallback = PollingFallback
|
||||
env.build_poll_sql = build_poll_sql
|
||||
env.normalize_watermark = normalize_watermark
|
||||
env.plan_uses_scene_index = plan_uses_scene_index
|
||||
env.configure_fallback = configure_fallback
|
||||
env.get_fallback = get_fallback
|
||||
env.reset_fallback = reset_fallback
|
||||
env.fallback_stats = fallback_stats
|
||||
env.poll_since = poll_since
|
||||
env.run_fallback = run_fallback
|
||||
env.wait_event = wait_event
|
||||
# 消费端订阅侧标准接线:广播到点 → 推进兜底水位线 + 停止轮询
|
||||
env.notify_broadcast = get_fallback().notify_broadcast
|
||||
# M11b-4 端到端时延基准
|
||||
env.measure_once = measure_once
|
||||
env.run_chain = run_chain
|
||||
env.summarize_latency = summarize_latency
|
||||
env.latency_report = latency_report
|
||||
# post-commit hook:把 write_event_with_state 包成「COMMIT 后触发广播」版本,
|
||||
# 并同步刷新三处引用(pbl_runtime_tx_sqlor / world_sync 包命名空间 / env)。
|
||||
env.write_event_with_state = install_broadcast_hook(env)
|
||||
|
||||
188
world_sync/pbl_runtime_latency.py
Normal file
188
world_sync/pbl_runtime_latency.py
Normal file
@ -0,0 +1,188 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""M11b-4:端到端时延基准(adjudication → write → COMMIT → broadcast → consumer)。
|
||||
|
||||
链路分段(对应任务验收「P99 ≤ 1s」)::
|
||||
|
||||
t0 裁决触发(run_chain 进入,模拟裁决服务决定下发事件)
|
||||
t1 write_event_with_state 返回(内部已完成事件+状态单事务 COMMIT,M11b-2)
|
||||
t2 post-commit hook 广播发出完成(M11b-3 install_broadcast_hook 包装后的函数)
|
||||
t3 消费端回调收到广播消息(感知)
|
||||
|
||||
e2e_latency = t3 - t0
|
||||
|
||||
本模块 **只组合调用** M11b-2/M11b-3 已交付的公开 API,不修改其核心逻辑:
|
||||
- 写入:``world_sync.write_event_with_state``(若宿主已 ``load_world_sync()`` 则
|
||||
该引用已被 ``install_broadcast_hook`` 包成 COMMIT 后自动广播版本);
|
||||
- 广播:``subscribe_broadcast`` / ``drain_broadcast`` / ``broadcast_stats``。
|
||||
|
||||
消费端感知判定:优先「广播回调到达」(真实链路);若广播丢失,则由
|
||||
:mod:`world_sync.pbl_runtime_polling` 的 3s 轮询兜底补漏到达(此时 e2e 必然
|
||||
>1s,报告会把它归入 ``fallback_recovered`` 并单独统计,供瓶颈分析)。
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from .pbl_runtime_broadcast import (
|
||||
broadcast_stats,
|
||||
drain_broadcast,
|
||||
get_broadcaster,
|
||||
subscribe_broadcast,
|
||||
unsubscribe_broadcast,
|
||||
)
|
||||
from .pbl_runtime_errors import TxConfigError
|
||||
from .pbl_runtime_polling import FALLBACK_SLO_MS, get_fallback
|
||||
from .pbl_runtime_sql import percentile
|
||||
|
||||
#: 端到端时延 SLO(毫秒)——任务规定 P99 ≤ 1s
|
||||
E2E_LATENCY_SLO_MS = 1000.0
|
||||
|
||||
__all__ = [
|
||||
"E2E_LATENCY_SLO_MS",
|
||||
"FALLBACK_SLO_MS",
|
||||
"measure_once",
|
||||
"run_chain",
|
||||
"summarize_latency",
|
||||
"latency_report",
|
||||
]
|
||||
|
||||
|
||||
def _now_ms() -> float:
|
||||
return time.monotonic() * 1000.0
|
||||
|
||||
|
||||
def measure_once(write_fn: Callable[..., Any], event_data: Dict[str, Any],
|
||||
state_update: Optional[Dict[str, Any]] = None,
|
||||
wait_ms: float = 1000.0,
|
||||
poll_fallback: bool = False) -> Dict[str, Any]:
|
||||
"""跑一条事件的端到端测量。
|
||||
|
||||
:param write_fn: 写入函数(应为已安装 post-commit hook 的
|
||||
``write_event_with_state``;纯写入端时延也能测,广播段记 None)。
|
||||
:param poll_fallback: True 时广播丢失不判 fail,改由轮询兜底补漏计时。
|
||||
"""
|
||||
scene_id = str(event_data.get("scene_id") or event_data.get("world_id") or "")
|
||||
fb = get_fallback()
|
||||
holder: Dict[str, Any] = {}
|
||||
t_done = {"v": None}
|
||||
|
||||
def _on_msg(msg: Any) -> None:
|
||||
if holder.get("t_recv_ms") is None:
|
||||
holder["t_recv_ms"] = _now_ms()
|
||||
holder["msg"] = msg
|
||||
# 广播正常路径:推进兜底水位线(消费端标准接线姿势)
|
||||
try:
|
||||
fb.notify_broadcast(msg)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
holder["t_recv_ms"] = None
|
||||
sub = subscribe_broadcast(_on_msg)
|
||||
try:
|
||||
t0 = _now_ms()
|
||||
try:
|
||||
result = write_fn(event_data, state_update or {})
|
||||
finally:
|
||||
t1 = _now_ms()
|
||||
# post-commit hook 在 write_fn 内部同步/异步发出;drain 确保已派发
|
||||
try:
|
||||
drain_broadcast(timeout=0.5)
|
||||
except Exception:
|
||||
pass
|
||||
t2 = _now_ms()
|
||||
# 等消费端感知(广播回调)
|
||||
deadline = _now_ms() + wait_ms
|
||||
while holder.get("t_recv_ms") is None and _now_ms() < deadline:
|
||||
time.sleep(0.001)
|
||||
t3 = holder.get("t_recv_ms")
|
||||
via = "broadcast"
|
||||
if t3 is None and poll_fallback:
|
||||
# 广播丢失 → 3s 轮询兜底补漏(真实节奏:interval=3s)
|
||||
eid = None
|
||||
if isinstance(result, dict):
|
||||
eid = result.get("event_id") or result.get("id")
|
||||
w = fb.wait_event(scene_id, str(eid or ""), timeout=FALLBACK_SLO_MS / 1000.0)
|
||||
via = "polling_fallback"
|
||||
t3 = _now_ms() if w.get("found") else None
|
||||
e2e = (t3 - t0) if t3 is not None else None
|
||||
return {
|
||||
"scene_id": scene_id,
|
||||
"event_id": (result.get("event_id") if isinstance(result, dict) else None),
|
||||
"t_write_ms": round(t1 - t0, 3),
|
||||
"t_broadcast_emit_ms": round(t2 - t1, 3),
|
||||
"t_consume_ms": round(t3 - t1, 3) if t3 is not None else None,
|
||||
"e2e_ms": round(e2e, 3) if e2e is not None else None,
|
||||
"via": via,
|
||||
"perceived": t3 is not None,
|
||||
}
|
||||
finally:
|
||||
unsubscribe_broadcast(sub)
|
||||
|
||||
|
||||
def run_chain(write_fn: Callable[..., Any], events: List[Dict[str, Any]],
|
||||
state_builder: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None,
|
||||
wait_ms: float = 1000.0, poll_fallback: bool = False) -> List[Dict[str, Any]]:
|
||||
"""批量端到端测量(裁决触发→…→消费端感知),返回逐条样本。"""
|
||||
samples: List[Dict[str, Any]] = []
|
||||
for ev in events:
|
||||
st = state_builder(ev) if state_builder else {}
|
||||
samples.append(measure_once(write_fn, ev, st, wait_ms=wait_ms,
|
||||
poll_fallback=poll_fallback))
|
||||
return samples
|
||||
|
||||
|
||||
def summarize_latency(samples: List[Dict[str, Any]],
|
||||
slo_ms: float = E2E_LATENCY_SLO_MS) -> Dict[str, Any]:
|
||||
"""汇总:P50/P90/P99/max + SLO 判定 + 分段瓶颈定位。"""
|
||||
e2e = [s["e2e_ms"] for s in samples if s.get("e2e_ms") is not None]
|
||||
write_seg = [s["t_write_ms"] for s in samples if s.get("t_write_ms") is not None]
|
||||
emit_seg = [s["t_broadcast_emit_ms"] for s in samples
|
||||
if s.get("t_broadcast_emit_ms") is not None]
|
||||
consume_seg = [s["t_consume_ms"] for s in samples if s.get("t_consume_ms") is not None]
|
||||
p99 = round(percentile(e2e, 99), 3) if e2e else None
|
||||
via_broadcast = sum(1 for s in samples if s.get("via") == "broadcast")
|
||||
via_polling = sum(1 for s in samples if s.get("via") == "polling_fallback")
|
||||
lost = sum(1 for s in samples if not s.get("perceived"))
|
||||
# 瓶颈定位:P99 分段最大者
|
||||
seg_p99 = {
|
||||
"write_commit": round(percentile(write_seg, 99), 3) if write_seg else 0.0,
|
||||
"broadcast_emit": round(percentile(emit_seg, 99), 3) if emit_seg else 0.0,
|
||||
"consume_wait": round(percentile(consume_seg, 99), 3) if consume_seg else 0.0,
|
||||
}
|
||||
bottleneck = max(seg_p99.items(), key=lambda kv: kv[1])[0] if seg_p99 else None
|
||||
return {
|
||||
"n": len(samples),
|
||||
"perceived": len(samples) - lost,
|
||||
"lost": lost,
|
||||
"via_broadcast": via_broadcast,
|
||||
"via_polling_fallback": via_polling,
|
||||
"p50_ms": round(percentile(e2e, 50), 3) if e2e else None,
|
||||
"p90_ms": round(percentile(e2e, 90), 3) if e2e else None,
|
||||
"p99_ms": p99,
|
||||
"max_ms": round(max(e2e), 3) if e2e else None,
|
||||
"segment_p99_ms": seg_p99,
|
||||
"bottleneck_segment": bottleneck,
|
||||
"slo_ms": slo_ms,
|
||||
"slo_passed": (p99 is not None and p99 <= slo_ms),
|
||||
"broadcast_stats": broadcast_stats(),
|
||||
}
|
||||
|
||||
|
||||
def latency_report(write_fn: Callable[..., Any], events: List[Dict[str, Any]],
|
||||
state_builder: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None,
|
||||
slo_ms: float = E2E_LATENCY_SLO_MS,
|
||||
wait_ms: float = 1000.0,
|
||||
poll_fallback: bool = False) -> Dict[str, Any]:
|
||||
"""一键基准:跑链路 + 汇总 + 结论(供集成测试 / scripts/m11b4_e2e_bench.py 复用)。"""
|
||||
if write_fn is None or not callable(write_fn):
|
||||
raise TxConfigError("latency_report: write_fn must be a callable "
|
||||
"(world_sync.write_event_with_state)")
|
||||
samples = run_chain(write_fn, events, state_builder=state_builder,
|
||||
wait_ms=wait_ms, poll_fallback=poll_fallback)
|
||||
summary = summarize_latency(samples, slo_ms=slo_ms)
|
||||
summary["samples"] = samples
|
||||
summary["verdict"] = ("PASS" if summary["slo_passed"] else
|
||||
"FAIL: P99 %.1fms > SLO %.0fms, bottleneck=%s"
|
||||
% (summary["p99_ms"] or -1, slo_ms,
|
||||
summary["bottleneck_segment"]))
|
||||
return summary
|
||||
544
world_sync/pbl_runtime_polling.py
Normal file
544
world_sync/pbl_runtime_polling.py
Normal file
@ -0,0 +1,544 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""M11b-4:广播不可靠时的 **3 秒轮询兜底**(polling fallback,消费端侧)。
|
||||
|
||||
设计定位(与 M11b-1/2/3 的关系)
|
||||
================================
|
||||
- M11b-1(表/索引):补漏查询 **只按 ``scene_id + created_at`` 过滤**,命中
|
||||
``models/pbl_runtime_event.json`` 的复合索引 ``ix_scene_created(scene_id,
|
||||
created_at)``,不做全表扫描;MySQL/MariaDB 侧用 :func:`explain_poll_query`
|
||||
EXPLAIN 验证(:func:`plan_uses_scene_index` 判定)。
|
||||
- M11b-2(单事务写入):本模块 **零侵入** —— 不 import、不包装、不修改
|
||||
``write_event_with_state``;只读 ``pbl_runtime_event`` 表。
|
||||
- M11b-3(提交后广播):本模块是广播的 **对偶兜底**。广播正常时消费端调用
|
||||
:meth:`PollingFallback.notify_broadcast` 记到点,轮询循环立即停止;广播丢失时
|
||||
由 3s 定时器主动查库补漏(水位线之后的行 = 漏网事件)。
|
||||
|
||||
行为契约(对应任务验收标准)
|
||||
============================
|
||||
1. 轮询间隔 :data:`POLL_INTERVAL_S` = 3.0s(可配),总超时 :data:`POLL_TIMEOUT_S`;
|
||||
「收到广播后停止,或超时后停止」(见 :meth:`PollingFallback.run` 的
|
||||
``stopped_by`` ∈ {``broadcast_received``, ``event_matched``, ``timeout``})。
|
||||
2. 广播丢失场景:下一轮轮询(≤ interval + 单次查询耗时)内把水位线之后的事件全部
|
||||
补回(:meth:`PollingFallback.poll_once` 的 ``recovered``)。
|
||||
3. 独立封装:类 :class:`PollingFallback` + 进程级单例门面
|
||||
(:func:`get_fallback` / :func:`configure_fallback` / :func:`reset_fallback` /
|
||||
:func:`fallback_stats` / :func:`wait_event`),消费端按需调用。
|
||||
|
||||
两个必须守住的正确性细节(写在本模块而非靠调用方自觉)
|
||||
======================================================
|
||||
A. **时间格式必须与库内一致**。``write_event_with_state`` 落库的 ``created_at`` 是
|
||||
:func:`world_sync.pbl_runtime_tx.utc_now_text` 生成的 ``'YYYY-MM-DD HH:MM:SS'``
|
||||
(空格分隔)。M11b-3 广播消息里的 ``commit_ts_text`` 是 ISO ``'...T...Z'`` 格式,
|
||||
字典序恒大于空格格式(``'T'``=0x54 > ``' '``=0x20),**若误用它推进水位线会把
|
||||
同场景后续事件全部跳过(静默丢事件)**。因此 :meth:`set_watermark` 只接受空格
|
||||
格式时间戳,其余一律忽略(:func:`normalize_watermark`)。
|
||||
B. **秒级时间戳会撞秒**。同一秒内多条事件 ``created_at`` 相同,用 ``>`` 比较会漏。
|
||||
因此查询用 ``created_at >= watermark`` + ``event_uid NOT IN (已投递集合)`` 双保险:
|
||||
同秒事件不丢、重复行不重复投递。已投递集合以 ``event_uid`` 为键(广播消息与写入
|
||||
返回值都含该键;表列 ``id`` 是物理主键、``event_uid`` 才是跨系统追溯口径)。
|
||||
"""
|
||||
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Callable, Dict, List, Optional, Sequence
|
||||
|
||||
from . import pbl_runtime_sql as ps
|
||||
from .pbl_runtime_errors import TxConfigError
|
||||
from .pbl_runtime_tx_env import resolve_conn_factory
|
||||
|
||||
#: 兜底轮询间隔(秒)——任务规定 3s
|
||||
POLL_INTERVAL_S = 3.0
|
||||
#: 单次兜底循环总超时(秒):超时后定时器自行停止
|
||||
POLL_TIMEOUT_S = 30.0
|
||||
#: 单次补漏查询 LIMIT
|
||||
POLL_LIMIT = 500
|
||||
#: 消费端感知 SLO(毫秒):广播丢失时兜底须在 1 个轮询周期内补漏
|
||||
FALLBACK_SLO_MS = 3000.0
|
||||
#: 去重集合参与 SQL 的最大长度(防止 NOT IN 参数爆炸)
|
||||
SEEN_IDS_IN_SQL = 200
|
||||
|
||||
_WATERMARK_RE = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{1,6})?$")
|
||||
|
||||
__all__ = [
|
||||
"POLL_INTERVAL_S",
|
||||
"POLL_TIMEOUT_S",
|
||||
"POLL_LIMIT",
|
||||
"FALLBACK_SLO_MS",
|
||||
"SEEN_IDS_IN_SQL",
|
||||
"PollingFallback",
|
||||
"build_poll_sql",
|
||||
"configure_fallback",
|
||||
"explain_poll_query",
|
||||
"fallback_stats",
|
||||
"get_fallback",
|
||||
"normalize_watermark",
|
||||
"plan_uses_scene_index",
|
||||
"poll_since",
|
||||
"reset_fallback",
|
||||
"run_fallback",
|
||||
"wait_event",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 基础工具
|
||||
def _field(obj: Any, name: str, default: Any = None) -> Any:
|
||||
"""兼容 dict / DictObject / 行对象取字段(与 M11b-3 同口径,独立实现不互相依赖)。"""
|
||||
if obj is None:
|
||||
return default
|
||||
if isinstance(obj, dict):
|
||||
return obj.get(name, default)
|
||||
try:
|
||||
val = getattr(obj, name)
|
||||
except Exception:
|
||||
return default
|
||||
return default if val is None else val
|
||||
|
||||
|
||||
def normalize_watermark(value: Any) -> str:
|
||||
"""把 ``created_at`` 归一为库内口径 ``'YYYY-MM-DD HH:MM:SS[.ffffff]'``。
|
||||
|
||||
只接受空格分隔格式(:func:`world_sync.pbl_runtime_tx.utc_now_text` 的产物)。
|
||||
ISO ``T``/``Z`` 格式(广播消息 ``commit_ts_text``)一律返回 ``''`` —— 见模块
|
||||
docstring 细节 A:混用会静默跳过后续事件。datetime 对象按 UTC 格式化。
|
||||
"""
|
||||
if value is None:
|
||||
return ""
|
||||
if hasattr(value, "strftime") and not isinstance(value, str):
|
||||
try:
|
||||
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||||
except Exception:
|
||||
return ""
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return ""
|
||||
if _WATERMARK_RE.match(text):
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _row_uid(row: Any) -> str:
|
||||
return str(_field(row, "event_uid") or _field(row, "event_id")
|
||||
or _field(row, "id") or "")
|
||||
|
||||
|
||||
def _row_created(row: Any) -> str:
|
||||
return normalize_watermark(_field(row, "created_at"))
|
||||
|
||||
|
||||
def _row_to_dict(row: Any) -> Dict[str, Any]:
|
||||
if isinstance(row, dict):
|
||||
return dict(row)
|
||||
return dict(row)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- SQL 构造
|
||||
def build_poll_sql(dialect: str, watermark: str, scene_id: str = ":scene",
|
||||
seen_uids: Optional[Sequence[Any]] = None,
|
||||
limit: int = POLL_LIMIT):
|
||||
"""构造补漏查询,返回 ``(sql, params)``。
|
||||
|
||||
**索引契约**:WHERE 前缀必须是 ``scene_id = ? AND created_at >= ?``,与复合索引
|
||||
``ix_scene_created(scene_id, created_at)`` 列序一致(等值列在前、范围列在后);
|
||||
``ORDER BY created_at ASC`` 走同一索引,避免 filesort 扩大扫描面。
|
||||
``LIMIT`` 为消毒后的整数直接内插(方言差异不涉及占位)。
|
||||
|
||||
``scene_id`` 传默认哨兵字符串时仅作 SQL 文本预览/EXPLAIN 用,真实调用由
|
||||
:meth:`PollingFallback.query_since` 覆盖 ``params[0]``。
|
||||
"""
|
||||
ph = ps.placeholder
|
||||
limit = max(1, int(limit))
|
||||
sql = (
|
||||
"SELECT id, event_uid, tenant_id, world_id, scene_id, entity_id, "
|
||||
"event_type, seq, seq_no, state_version, created_at "
|
||||
"FROM %s WHERE scene_id = %s AND created_at >= %s"
|
||||
% (ps.EVENT_TABLE, ph(dialect, 0), ph(dialect, 1))
|
||||
)
|
||||
params: List[Any] = [scene_id, normalize_watermark(watermark)]
|
||||
uids = [str(u) for u in (seen_uids or []) if u is not None and str(u) != ""]
|
||||
uids = uids[-SEEN_IDS_IN_SQL:]
|
||||
if uids:
|
||||
marks = ", ".join(ph(dialect, 2 + k) for k in range(len(uids)))
|
||||
sql += " AND (event_uid IS NULL OR event_uid NOT IN (%s))" % marks
|
||||
params.extend(uids)
|
||||
sql += " ORDER BY created_at ASC LIMIT %d" % limit
|
||||
return sql, params
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 核心类
|
||||
class PollingFallback:
|
||||
"""3 秒轮询兜底器(消费端 / 订阅方侧,只读,不侵入写入端)。
|
||||
|
||||
:param conn_factory: DB-API 连接工厂;None 时走模块统一解析链
|
||||
(显式注入 → ServerEnv 登记 → 环境变量 ``PBL_RUNTIME_DB_URL`` → ``conf/db.json``)。
|
||||
:param interval: 轮询间隔秒(默认 :data:`POLL_INTERVAL_S` = 3s)。
|
||||
:param timeout: 单次兜底循环总超时秒(默认 :data:`POLL_TIMEOUT_S`)。
|
||||
"""
|
||||
|
||||
def __init__(self, conn_factory: Optional[Callable[[], Any]] = None,
|
||||
interval: float = POLL_INTERVAL_S, timeout: float = POLL_TIMEOUT_S,
|
||||
limit: int = POLL_LIMIT):
|
||||
self._conn_factory = conn_factory
|
||||
self.interval = float(interval)
|
||||
self.timeout = float(timeout)
|
||||
self.limit = int(limit)
|
||||
self._lock = threading.Lock()
|
||||
# scene_id -> {"watermark": str, "notified": bool, "last_notify_ts": float}
|
||||
self._scenes: Dict[str, Dict[str, Any]] = {}
|
||||
# scene_id -> [event_uid](保序,用于 NOT IN 去重)
|
||||
self._seen: Dict[str, List[str]] = {}
|
||||
self._counters = {
|
||||
"polls": 0, "queries": 0, "recovered": 0, "duplicated": 0,
|
||||
"missed_by_broadcast": 0, "timeouts": 0, "stops_on_notify": 0,
|
||||
}
|
||||
|
||||
# ---------------- 状态维护 ----------------
|
||||
def _scene_state(self, scene_id: str) -> Dict[str, Any]:
|
||||
st = self._scenes.get(scene_id)
|
||||
if st is None:
|
||||
st = {"watermark": "", "notified": False, "last_notify_ts": 0.0}
|
||||
self._scenes[scene_id] = st
|
||||
return st
|
||||
|
||||
def set_watermark(self, scene_id: str, created_at: Any) -> bool:
|
||||
"""消费端感知到事件后推进水位线(单调不减)。格式不合法则忽略并返回 False。"""
|
||||
ts = normalize_watermark(created_at)
|
||||
if not ts:
|
||||
return False
|
||||
with self._lock:
|
||||
st = self._scene_state(str(scene_id or ""))
|
||||
if ts > st["watermark"]:
|
||||
st["watermark"] = ts
|
||||
return True
|
||||
return False
|
||||
|
||||
def watermark(self, scene_id: str) -> str:
|
||||
with self._lock:
|
||||
return self._scenes.get(str(scene_id or ""), {}).get("watermark", "")
|
||||
|
||||
def note_event_seen(self, scene_id: str, event_uid: Any) -> None:
|
||||
"""登记已投递事件(event_uid 口径),供轮询二次去重。"""
|
||||
if event_uid is None or str(event_uid) == "":
|
||||
return
|
||||
uid = str(event_uid)
|
||||
with self._lock:
|
||||
lst = self._seen.setdefault(str(scene_id or ""), [])
|
||||
if uid in lst:
|
||||
self._counters["duplicated"] += 1
|
||||
return
|
||||
lst.append(uid)
|
||||
if len(lst) > 4 * SEEN_IDS_IN_SQL:
|
||||
del lst[:len(lst) - 2 * SEEN_IDS_IN_SQL]
|
||||
|
||||
def seen_uids(self, scene_id: str) -> List[str]:
|
||||
with self._lock:
|
||||
return list(self._seen.get(str(scene_id or ""), []))
|
||||
|
||||
def notify_broadcast(self, message: Any, created_at: Any = None) -> Dict[str, Any]:
|
||||
"""广播正常路径:消费端订阅回调里调用 → 记到点(轮询循环随即停止)。
|
||||
|
||||
:param message: M11b-3 广播消息 dict(含 ``scene_id`` / ``event_uid``)。
|
||||
:param created_at: 可选,写入返回值里的库内口径 ``created_at``。广播消息
|
||||
本身不带该格式(只有 ISO ``commit_ts_text``,见模块 docstring 细节 A),
|
||||
因此推荐消费端把 ``write_event_with_state`` 返回的 ``created_at`` 传进来,
|
||||
让水位线随广播精确推进。
|
||||
"""
|
||||
scene_id = str(_field(message, "scene_id") or "")
|
||||
uid = _field(message, "event_uid") or _field(message, "event_id")
|
||||
advanced = self.set_watermark(scene_id, created_at)
|
||||
self.note_event_seen(scene_id, uid)
|
||||
with self._lock:
|
||||
st = self._scene_state(scene_id)
|
||||
st["notified"] = True
|
||||
st["last_notify_ts"] = time.time()
|
||||
info = {"scene_id": scene_id, "notified": True,
|
||||
"watermark": st["watermark"], "watermark_advanced": advanced}
|
||||
return info
|
||||
|
||||
def was_notified(self, scene_id: str) -> bool:
|
||||
with self._lock:
|
||||
return bool(self._scenes.get(str(scene_id or ""), {}).get("notified"))
|
||||
|
||||
def clear_notified(self, scene_id: str) -> None:
|
||||
"""重置到点标记(下一轮事件等待前调用,避免上一轮的 notified 短路本轮)。"""
|
||||
with self._lock:
|
||||
self._scene_state(str(scene_id or ""))["notified"] = False
|
||||
|
||||
# ---------------- 查询 ----------------
|
||||
def _connect(self):
|
||||
try:
|
||||
factory = resolve_conn_factory(self._conn_factory)
|
||||
except Exception as exc: # fail-closed:没有连接工厂就别假装查过
|
||||
raise TxConfigError("polling fallback: conn factory unavailable: %r" % (exc,))
|
||||
if not callable(factory):
|
||||
raise TxConfigError("polling fallback: resolved conn factory is not callable")
|
||||
return factory()
|
||||
|
||||
def query_since(self, scene_id: str, watermark: str,
|
||||
seen_uids: Optional[Sequence[Any]] = None) -> List[Dict[str, Any]]:
|
||||
"""执行一次补漏查询(公开方法,供基准/测试直接调用)。返回行 dict 列表。"""
|
||||
conn = self._connect()
|
||||
try:
|
||||
dialect = ps.detect_dialect(conn)
|
||||
sql, params = build_poll_sql(dialect, watermark, scene_id, seen_uids,
|
||||
self.limit)
|
||||
params[0] = str(scene_id)
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(sql, params)
|
||||
cols = [d[0] for d in (cur.description or [])]
|
||||
rows: List[Dict[str, Any]] = []
|
||||
for raw in cur.fetchall() or []:
|
||||
if isinstance(raw, dict):
|
||||
rows.append(dict(raw))
|
||||
else:
|
||||
rows.append(dict(zip(cols, list(raw))))
|
||||
finally:
|
||||
try:
|
||||
cur.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
conn.commit()
|
||||
except Exception:
|
||||
pass
|
||||
return rows
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def explain_poll_query(self, scene_id: str, watermark: str = "") -> List[Dict[str, Any]]:
|
||||
"""对补漏查询执行 ``EXPLAIN``(MySQL/MariaDB 有意义),返回计划行。"""
|
||||
conn = self._connect()
|
||||
try:
|
||||
dialect = ps.detect_dialect(conn)
|
||||
sql, params = build_poll_sql(dialect, watermark, scene_id, None, self.limit)
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute("EXPLAIN " + sql, params)
|
||||
cols = [d[0] for d in (cur.description or [])]
|
||||
out: List[Dict[str, Any]] = []
|
||||
for raw in cur.fetchall() or []:
|
||||
out.append(dict(raw) if isinstance(raw, dict)
|
||||
else dict(zip(cols, list(raw))))
|
||||
return out
|
||||
finally:
|
||||
try:
|
||||
cur.close()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def poll_once(self, scene_id: str, wait_for: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""一轮补漏:查水位线之后的行,扣除已投递集合 = 广播漏网事件。
|
||||
|
||||
:param wait_for: 可选,等待特定 ``event_uid``;出现则 ``found=True``。
|
||||
"""
|
||||
scene_id = str(scene_id or "")
|
||||
with self._lock:
|
||||
st = self._scene_state(scene_id)
|
||||
watermark = st["watermark"]
|
||||
seen = list(self._seen.get(scene_id) or [])
|
||||
self._counters["polls"] += 1
|
||||
rows = self.query_since(scene_id, watermark, seen)
|
||||
seen_set = set(seen)
|
||||
recovered: List[Dict[str, Any]] = []
|
||||
found = False
|
||||
max_created = watermark
|
||||
for row in rows:
|
||||
uid = _row_uid(row)
|
||||
if uid and uid in seen_set:
|
||||
continue
|
||||
recovered.append(row)
|
||||
if uid:
|
||||
seen_set.add(uid)
|
||||
self.note_event_seen(scene_id, uid)
|
||||
created = _row_created(row)
|
||||
if created and created > max_created:
|
||||
max_created = created
|
||||
if wait_for and uid == str(wait_for):
|
||||
found = True
|
||||
if max_created:
|
||||
self.set_watermark(scene_id, max_created)
|
||||
with self._lock:
|
||||
self._counters["queries"] += 1
|
||||
self._counters["recovered"] += len(recovered)
|
||||
if recovered:
|
||||
self._counters["missed_by_broadcast"] += len(recovered)
|
||||
return {"scene_id": scene_id, "watermark": watermark, "rows": rows,
|
||||
"recovered": recovered, "recovered_count": len(recovered),
|
||||
"found": found, "notified": self.was_notified(scene_id)}
|
||||
|
||||
# ---------------- 兜底循环 ----------------
|
||||
def run(self, scene_id: str, wait_for: Optional[str] = None,
|
||||
timeout: Optional[float] = None, interval: Optional[float] = None,
|
||||
on_recovered: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
sleep: Callable[[float], None] = time.sleep) -> Dict[str, Any]:
|
||||
"""3s 轮询兜底循环:**收到广播即停;等不到广播时按周期查库补漏;超时也停**。
|
||||
|
||||
消费端通常在「广播等待超时」后把场景交给定时器线程调用本方法。
|
||||
``sleep`` 可注入(测试用假时钟加速,不改变 3s 语义)。
|
||||
"""
|
||||
scene_id = str(scene_id or "")
|
||||
interval = float(interval if interval is not None else self.interval)
|
||||
timeout = float(timeout if timeout is not None else self.timeout)
|
||||
self.clear_notified(scene_id)
|
||||
deadline = time.monotonic() + timeout
|
||||
polls = 0
|
||||
recovered: List[Dict[str, Any]] = []
|
||||
stopped_by = "timeout"
|
||||
found = False
|
||||
while time.monotonic() < deadline:
|
||||
sleep(interval)
|
||||
polls += 1
|
||||
res = self.poll_once(scene_id, wait_for=wait_for)
|
||||
for row in res["recovered"]:
|
||||
recovered.append(row)
|
||||
if on_recovered:
|
||||
try:
|
||||
on_recovered(row)
|
||||
except Exception:
|
||||
pass
|
||||
if res["found"]:
|
||||
found = True
|
||||
stopped_by = "event_matched"
|
||||
break
|
||||
if res["notified"]:
|
||||
stopped_by = "broadcast_received"
|
||||
with self._lock:
|
||||
self._counters["stops_on_notify"] += 1
|
||||
break
|
||||
if stopped_by == "timeout":
|
||||
with self._lock:
|
||||
self._counters["timeouts"] += 1
|
||||
return {"scene_id": scene_id, "polls": polls, "recovered": recovered,
|
||||
"recovered_count": len(recovered), "found": found,
|
||||
"stopped_by": stopped_by, "interval": interval, "timeout": timeout}
|
||||
|
||||
def wait_event(self, scene_id: str, event_uid: str,
|
||||
timeout: Optional[float] = None, interval: Optional[float] = None,
|
||||
sleep: Callable[[float], None] = time.sleep) -> Dict[str, Any]:
|
||||
"""等待特定事件被感知(广播或轮询兜底),返回 ``{found, waited_ms, polls}``。"""
|
||||
start = time.monotonic()
|
||||
res = self.run(scene_id, wait_for=str(event_uid), timeout=timeout,
|
||||
interval=interval, sleep=sleep)
|
||||
waited_ms = (time.monotonic() - start) * 1000.0
|
||||
return {"scene_id": str(scene_id or ""), "event_uid": str(event_uid),
|
||||
"found": bool(res["found"]), "waited_ms": round(waited_ms, 3),
|
||||
"polls": res["polls"], "stopped_by": res["stopped_by"],
|
||||
"recovered_count": res["recovered_count"]}
|
||||
|
||||
def stats(self) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
snap = dict(self._counters)
|
||||
snap["scenes"] = sorted(self._scenes.keys())
|
||||
snap["interval"] = self.interval
|
||||
snap["timeout"] = self.timeout
|
||||
return snap
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 计划判定
|
||||
def plan_uses_scene_index(plan_rows: Sequence[Any]) -> bool:
|
||||
"""判定 EXPLAIN 结果是否命中 ``ix_scene_created``(即不是全表扫描)。
|
||||
|
||||
MySQL 5.7 经典格式看 ``key`` / ``possible_keys`` + ``type``;``FORMAT=JSON`` 或
|
||||
mariadb 扩展格式退化为在序列化文本里找索引名。``type=ALL`` 且未用该索引 → False。
|
||||
"""
|
||||
rows = list(plan_rows or [])
|
||||
if not rows:
|
||||
return False
|
||||
for row in rows:
|
||||
key = str(_field(row, "key", "") or "")
|
||||
if key == "ix_scene_created":
|
||||
return True
|
||||
possible = str(_field(row, "possible_keys", "") or "")
|
||||
rtype = str(_field(row, "type", "") or "")
|
||||
if "ix_scene_created" in possible and rtype != "ALL":
|
||||
return True
|
||||
blob = repr(rows)
|
||||
if "ix_scene_created" not in blob:
|
||||
return False
|
||||
return "'type': 'ALL'" not in blob and '"type": "ALL"' not in blob
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 单例门面
|
||||
_DEFAULT = PollingFallback()
|
||||
_LOCK = threading.RLock()
|
||||
|
||||
|
||||
def get_fallback(conn_factory: Optional[Callable[[], Any]] = None) -> PollingFallback:
|
||||
"""取进程级兜底器单例(消费端默认用它;需要独立状态时自行 new 一个)。"""
|
||||
global _DEFAULT
|
||||
with _LOCK:
|
||||
if conn_factory is not None and _DEFAULT._conn_factory is not conn_factory:
|
||||
_DEFAULT._conn_factory = conn_factory
|
||||
return _DEFAULT
|
||||
|
||||
|
||||
def configure_fallback(conn_factory: Optional[Callable[[], Any]] = None,
|
||||
interval: Optional[float] = None,
|
||||
timeout: Optional[float] = None) -> PollingFallback:
|
||||
"""配置单例的连接工厂与节奏(interval/timeout),返回单例本身。"""
|
||||
fb = get_fallback(conn_factory)
|
||||
with _LOCK:
|
||||
if interval is not None:
|
||||
fb.interval = float(interval)
|
||||
if timeout is not None:
|
||||
fb.timeout = float(timeout)
|
||||
return fb
|
||||
|
||||
|
||||
def reset_fallback(conn_factory: Optional[Callable[[], Any]] = None,
|
||||
interval: Optional[float] = None,
|
||||
timeout: Optional[float] = None) -> PollingFallback:
|
||||
"""重建单例(清空水位线/去重/计数,测试隔离与宿主重配置用)。"""
|
||||
global _DEFAULT
|
||||
with _LOCK:
|
||||
_DEFAULT = PollingFallback(
|
||||
conn_factory=conn_factory if conn_factory is not None
|
||||
else _DEFAULT._conn_factory,
|
||||
interval=interval if interval is not None else _DEFAULT.interval,
|
||||
timeout=timeout if timeout is not None else _DEFAULT.timeout,
|
||||
)
|
||||
return _DEFAULT
|
||||
|
||||
|
||||
def fallback_stats() -> Dict[str, Any]:
|
||||
"""单例兜底指标(补漏次数 / 漏网事件数 / 因广播到点停止次数 / 超时次数)。"""
|
||||
return get_fallback().stats()
|
||||
|
||||
|
||||
def poll_since(scene_id: str, watermark: str = "", conn_factory=None,
|
||||
seen_uids: Optional[Sequence[Any]] = None) -> List[Dict[str, Any]]:
|
||||
"""门面:一次性查某场景水位线之后的事件(消费端手动补漏,不起定时器)。"""
|
||||
return get_fallback(conn_factory).query_since(scene_id, watermark, seen_uids)
|
||||
|
||||
|
||||
def run_fallback(scene_id: str, conn_factory=None, timeout: Optional[float] = None,
|
||||
interval: Optional[float] = None,
|
||||
on_recovered: Optional[Callable[[Dict[str, Any]], None]] = None,
|
||||
sleep: Callable[[float], None] = time.sleep) -> Dict[str, Any]:
|
||||
"""门面:跑一轮 3s 兜底循环(消费端定时器线程体内调用)。"""
|
||||
return get_fallback(conn_factory).run(scene_id, timeout=timeout,
|
||||
interval=interval,
|
||||
on_recovered=on_recovered, sleep=sleep)
|
||||
|
||||
|
||||
def wait_event(scene_id: str, event_uid: str, conn_factory=None,
|
||||
timeout: Optional[float] = None, interval: Optional[float] = None,
|
||||
sleep: Callable[[float], None] = time.sleep) -> Dict[str, Any]:
|
||||
"""门面:用单例等待特定事件被感知(广播到点或轮询补漏,≤ timeout)。"""
|
||||
return get_fallback(conn_factory).wait_event(scene_id, event_uid, timeout=timeout,
|
||||
interval=interval, sleep=sleep)
|
||||
|
||||
|
||||
def explain_poll_query(scene_id: str, watermark: str = "", conn_factory=None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""门面:对补漏查询跑 EXPLAIN(验证走 ``ix_scene_created``,非全表扫描)。"""
|
||||
return get_fallback(conn_factory).explain_poll_query(scene_id, watermark)
|
||||
Loading…
x
Reference in New Issue
Block a user