develop: 重做 fault-log-analyzer 分析模块（遵守模块开发规范）
This commit is contained in:
Pipeline Agent
parent
661fa2f02b
commit
81e2447016
@ -1 +1,51 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""CLI 入口。
|
||||
|
||||
用法：
|
||||
python -m fault_log_analyzer --storage memory --api-host 127.0.0.1 --api-port 8080
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from typing import List, Optional
|
||||
|
||||
from .api import run_server
|
||||
from .config import Config
|
||||
from .storage import MemoryStorage, Storage
|
||||
|
||||
|
||||
def build_storage(storage_name: str) -> Storage:
|
||||
"""根据配置构建存储。生产后端（mysql 等）在 integrations.py 中。"""
|
||||
if storage_name in ("memory", "mem"):
|
||||
return MemoryStorage()
|
||||
if storage_name == "mysql":
|
||||
from .integrations import MySQLFaultRepository
|
||||
|
||||
cfg = Config.from_env()
|
||||
return MySQLFaultRepository(
|
||||
host=cfg.mysql_host,
|
||||
port=cfg.mysql_port,
|
||||
user=cfg.mysql_user,
|
||||
password=cfg.mysql_password,
|
||||
db=cfg.mysql_db,
|
||||
)
|
||||
raise ValueError(f"unknown storage: {storage_name}")
|
||||
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> int:
|
||||
cfg = Config.from_env()
|
||||
parser = argparse.ArgumentParser(prog="fault-log-analyzer", description="HMS 故障日志捕获与分析模块")
|
||||
parser.add_argument("--storage", default=cfg.storage, help="存储后端（memory/mysql）")
|
||||
parser.add_argument("--api-host", default=cfg.api_host, help="REST API 监听地址")
|
||||
parser.add_argument("--api-port", type=int, default=cfg.api_port, help="REST API 监听端口")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
storage = build_storage(args.storage)
|
||||
print(f"fault-log-analyzer serving on http://{args.api_host}:{args.api_port} (storage={args.storage})")
|
||||
run_server(args.api_host, args.api_port, storage)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
@ -1 +1,191 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""REST API（标准库 http.server 实现）。
|
||||
|
||||
依据 api-design.md 第 6 节，提供：
|
||||
- GET /healthz
|
||||
- GET /api/v1/fault-logs 故障日志列表（分页/过滤）
|
||||
- GET /api/v1/fault-logs/{id} 故障日志详情
|
||||
- GET /api/v1/fault-logs/{id}/root-cause 根因结论
|
||||
- GET /api/v1/fault-types 故障类型列表
|
||||
- POST /api/v1/fault-types 新建故障类型
|
||||
- GET /api/v1/fault-filters 过滤规则列表
|
||||
- POST /api/v1/fault-filters 新建过滤规则
|
||||
|
||||
统一响应包装：{"code": 0, "message": "ok", "data": ...}。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Any, Dict, Optional, Type
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from .models import FaultFilterRule, FaultType
|
||||
from .storage import Storage
|
||||
|
||||
|
||||
class FaultLogAPIHandler(BaseHTTPRequestHandler):
|
||||
"""绑定 MemoryStorage 的请求处理器。storage 由 create_handler 注入。"""
|
||||
|
||||
storage: Storage = None # type: ignore[assignment]
|
||||
server_version = "FaultLogAnalyzer/0.1"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 工具方法
|
||||
# ------------------------------------------------------------------
|
||||
def _send_json(self, status: int, payload: Dict[str, Any]) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _ok(self, data: Any, status: int = 200) -> None:
|
||||
self._send_json(status, {"code": 0, "message": "ok", "data": data})
|
||||
|
||||
def _error(self, code: int, message: str, status: int) -> None:
|
||||
self._send_json(status, {"code": code, "message": message, "data": None})
|
||||
|
||||
def _read_body(self) -> Dict[str, Any]:
|
||||
length = int(self.headers.get("Content-Length", "0") or "0")
|
||||
if length <= 0:
|
||||
return {}
|
||||
raw = self.rfile.read(length)
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
def _query(self) -> Dict[str, str]:
|
||||
parsed = urlparse(self.path)
|
||||
qs = parse_qs(parsed.query)
|
||||
return {k: v[0] for k, v in qs.items() if v}
|
||||
|
||||
def log_message(self, fmt: str, *args: Any) -> None: # noqa: A002
|
||||
# 静默访问日志，避免污染测试输出
|
||||
return
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 路由
|
||||
# ------------------------------------------------------------------
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
if path == "/healthz":
|
||||
return self._send_json(200, {"status": "ok"})
|
||||
|
||||
if path == "/api/v1/fault-logs":
|
||||
return self._list_fault_logs()
|
||||
|
||||
if path.startswith("/api/v1/fault-logs/"):
|
||||
rest = path[len("/api/v1/fault-logs/") :]
|
||||
if rest.endswith("/root-cause"):
|
||||
return self._get_root_cause(rest[: -len("/root-cause")])
|
||||
return self._get_fault_log(rest)
|
||||
|
||||
if path == "/api/v1/fault-types":
|
||||
return self._list_fault_types()
|
||||
|
||||
if path == "/api/v1/fault-filters":
|
||||
return self._list_fault_filters()
|
||||
|
||||
return self._error(40401, "not found", 404)
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
if path == "/api/v1/fault-types":
|
||||
return self._create_fault_type()
|
||||
|
||||
if path == "/api/v1/fault-filters":
|
||||
return self._create_fault_filter()
|
||||
|
||||
return self._error(40401, "not found", 404)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 处理器
|
||||
# ------------------------------------------------------------------
|
||||
def _list_fault_logs(self) -> None:
|
||||
q = self._query()
|
||||
try:
|
||||
page = max(1, int(q.get("page", "1")))
|
||||
page_size = max(1, min(int(q.get("page_size", "20")), 200))
|
||||
except ValueError:
|
||||
return self._error(40001, "invalid param: page/page_size", 400)
|
||||
|
||||
result = self.storage.query_fault_logs(
|
||||
host_id=q.get("host_id") or None,
|
||||
fault_type=q.get("fault_type") or None,
|
||||
level=q.get("level") or None,
|
||||
keyword=q.get("keyword") or None,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
items = [l.to_dict() for l in result["items"]]
|
||||
return self._ok({"total": result["total"], "items": items})
|
||||
|
||||
def _get_fault_log(self, fault_log_id: str) -> None:
|
||||
log = self.storage.get_fault_log(fault_log_id)
|
||||
if log is None:
|
||||
return self._error(40401, "fault log not found", 404)
|
||||
return self._ok(log.to_dict())
|
||||
|
||||
def _get_root_cause(self, fault_log_id: str) -> None:
|
||||
root_cause = self.storage.get_root_cause(fault_log_id)
|
||||
if root_cause is None:
|
||||
return self._error(40401, "root cause not found", 404)
|
||||
return self._ok(root_cause.to_dict())
|
||||
|
||||
def _list_fault_types(self) -> None:
|
||||
return self._ok({"items": [ft.to_dict() for ft in self.storage.list_fault_types()]})
|
||||
|
||||
def _create_fault_type(self) -> None:
|
||||
body = self._read_body()
|
||||
if not body.get("fault_type") or not body.get("name"):
|
||||
return self._error(40002, "fault_type and name are required", 400)
|
||||
try:
|
||||
ft = FaultType.from_dict(body)
|
||||
except (TypeError, ValueError):
|
||||
return self._error(40001, "invalid param", 400)
|
||||
self.storage.add_fault_type(ft)
|
||||
return self._ok(ft.to_dict(), status=201)
|
||||
|
||||
def _list_fault_filters(self) -> None:
|
||||
return self._ok({"items": [r.to_dict() for r in self.storage.list_filter_rules()]})
|
||||
|
||||
def _create_fault_filter(self) -> None:
|
||||
body = self._read_body()
|
||||
if not body.get("name"):
|
||||
return self._error(40002, "name is required", 400)
|
||||
try:
|
||||
rule = FaultFilterRule.from_dict(body)
|
||||
except (TypeError, ValueError):
|
||||
return self._error(40001, "invalid param", 400)
|
||||
self.storage.add_filter_rule(rule)
|
||||
return self._ok(rule.to_dict(), status=201)
|
||||
|
||||
|
||||
def create_handler(storage: Storage) -> Type[FaultLogAPIHandler]:
|
||||
"""创建绑定存储的处理器类。"""
|
||||
|
||||
class Handler(FaultLogAPIHandler):
|
||||
pass
|
||||
|
||||
Handler.storage = storage
|
||||
return Handler
|
||||
|
||||
|
||||
def run_server(host: str, port: int, storage: Storage) -> None:
|
||||
"""启动 HTTP 服务（阻塞）。"""
|
||||
handler = create_handler(storage)
|
||||
server = ThreadingHTTPServer((host, port), handler)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
|
||||
@ -1 +1,199 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""可选真实后端适配（Kafka / Elasticsearch / MySQL / Redis）。
|
||||
|
||||
所有外部 SDK 均为惰性导入：未安装对应依赖时，仅在实例化时抛出明确异常，
|
||||
不影响核心逻辑与单元测试运行。
|
||||
|
||||
- KafkaProducerAdapter / KafkaConsumerAdapter -> kafka-python
|
||||
- ElasticsearchSink -> elasticsearch
|
||||
- MySQLFaultRepository -> pymysql
|
||||
- RedisDedupCache -> redis
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Iterator, List, Optional
|
||||
|
||||
from .models import Event, FaultFilterRule, FaultLog, FaultType, RootCause
|
||||
from .storage import Storage
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kafka
|
||||
# ---------------------------------------------------------------------------
|
||||
class KafkaProducerAdapter:
|
||||
"""Kafka 生产者封装（用于生产 logs.fault）。"""
|
||||
|
||||
def __init__(self, bootstrap_servers: str) -> None:
|
||||
try:
|
||||
from kafka import KafkaProducer # type: ignore
|
||||
except ImportError as exc: # pragma: no cover - 依赖未安装
|
||||
raise RuntimeError("Kafka 后端需要安装 kafka-python") from exc
|
||||
self._producer = KafkaProducer(
|
||||
bootstrap_servers=bootstrap_servers,
|
||||
value_serializer=lambda v: json.dumps(v, ensure_ascii=False, default=str).encode("utf-8"),
|
||||
)
|
||||
|
||||
def send(self, topic: str, value: Dict[str, Any]) -> None:
|
||||
self._producer.send(topic, value)
|
||||
|
||||
def flush(self) -> None:
|
||||
self._producer.flush()
|
||||
|
||||
def close(self) -> None:
|
||||
self._producer.close()
|
||||
|
||||
|
||||
class KafkaConsumerAdapter:
|
||||
"""Kafka 消费者封装（用于消费 logs.raw）。"""
|
||||
|
||||
def __init__(self, bootstrap_servers: str, group_id: str, topic: str) -> None:
|
||||
try:
|
||||
from kafka import KafkaConsumer # type: ignore
|
||||
except ImportError as exc: # pragma: no cover - 依赖未安装
|
||||
raise RuntimeError("Kafka 后端需要安装 kafka-python") from exc
|
||||
self._consumer = KafkaConsumer(
|
||||
topic,
|
||||
bootstrap_servers=bootstrap_servers,
|
||||
group_id=group_id,
|
||||
value_deserializer=lambda raw: json.loads(raw.decode("utf-8")),
|
||||
auto_offset_reset="earliest",
|
||||
)
|
||||
|
||||
def poll(self, timeout_ms: int = 1000) -> List[Dict[str, Any]]:
|
||||
batch = self._consumer.poll(timeout_ms=timeout_ms)
|
||||
return [record.value for records in batch.values() for record in records]
|
||||
|
||||
def close(self) -> None:
|
||||
self._consumer.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Elasticsearch
|
||||
# ---------------------------------------------------------------------------
|
||||
class ElasticsearchSink:
|
||||
"""故障日志写入 Elasticsearch（索引 hms-fault-log-{yyyy.MM}）。"""
|
||||
|
||||
def __init__(self, hosts: str, index_prefix: str = "hms-fault-log") -> None:
|
||||
try:
|
||||
from elasticsearch import Elasticsearch # type: ignore
|
||||
except ImportError as exc: # pragma: no cover - 依赖未安装
|
||||
raise RuntimeError("Elasticsearch 后端需要安装 elasticsearch") from exc
|
||||
host_list = [h.strip() for h in hosts.split(",") if h.strip()]
|
||||
self._client = Elasticsearch(host_list or ["http://localhost:9200"])
|
||||
self.index_prefix = index_prefix
|
||||
|
||||
@staticmethod
|
||||
def _index_name(prefix: str, occurred_at: Optional[datetime]) -> str:
|
||||
dt = occurred_at or datetime.utcnow()
|
||||
return f"{prefix}-{dt:%Y.%m}"
|
||||
|
||||
def write(self, log: FaultLog) -> None:
|
||||
index = self._index_name(self.index_prefix, log.occurred_at)
|
||||
self._client.index(index=index, id=log.fault_log_id, document=log.to_dict())
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MySQL
|
||||
# ---------------------------------------------------------------------------
|
||||
class MySQLFaultRepository:
|
||||
"""故障日志 / 根因结果 MySQL 落库（简化子集，生产可按需扩展）。"""
|
||||
|
||||
def __init__(self, host: str, port: int, user: str, password: str, db: str) -> None:
|
||||
try:
|
||||
import pymysql # type: ignore
|
||||
except ImportError as exc: # pragma: no cover - 依赖未安装
|
||||
raise RuntimeError("MySQL 后端需要安装 pymysql") from exc
|
||||
self._conn = pymysql.connect(
|
||||
host=host, port=port, user=user, password=password, database=db
|
||||
)
|
||||
|
||||
def save_fault_log(self, log: FaultLog) -> None:
|
||||
sql = (
|
||||
"INSERT INTO fault_log (fault_log_id, host_id, fault_type, cluster_id, "
|
||||
"fingerprint, level, service, message, trace_id, occurred_at, count) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
|
||||
"ON DUPLICATE KEY UPDATE fault_type=VALUES(fault_type), "
|
||||
"cluster_id=VALUES(cluster_id), count=VALUES(count)"
|
||||
)
|
||||
with self._conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
sql,
|
||||
(
|
||||
log.fault_log_id,
|
||||
log.host_id,
|
||||
log.fault_type,
|
||||
log.cluster_id,
|
||||
log.fingerprint,
|
||||
log.level,
|
||||
log.service,
|
||||
log.message,
|
||||
log.trace_id,
|
||||
log.occurred_at,
|
||||
log.count,
|
||||
),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def save_root_cause(self, root_cause: RootCause) -> None:
|
||||
sql = (
|
||||
"INSERT INTO root_cause (fault_log_id, cause_type, evidence, confidence, analysis_at) "
|
||||
"VALUES (%s, %s, %s, %s, %s) "
|
||||
"ON DUPLICATE KEY UPDATE cause_type=VALUES(cause_type), "
|
||||
"evidence=VALUES(evidence), confidence=VALUES(confidence)"
|
||||
)
|
||||
with self._conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
sql,
|
||||
(
|
||||
root_cause.fault_log_id,
|
||||
root_cause.cause_type,
|
||||
json.dumps(root_cause.evidence, ensure_ascii=False, default=str),
|
||||
root_cause.confidence,
|
||||
root_cause.analysis_at,
|
||||
),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def close(self) -> None:
|
||||
self._conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Redis
|
||||
# ---------------------------------------------------------------------------
|
||||
class RedisDedupCache:
|
||||
"""基于 Redis 的指纹去重（对应 Redis 键 hms:log:fp:{fingerprint}）。"""
|
||||
|
||||
def __init__(self, url: str, window_seconds: int = 300) -> None:
|
||||
try:
|
||||
import redis # type: ignore
|
||||
except ImportError as exc: # pragma: no cover - 依赖未安装
|
||||
raise RuntimeError("Redis 后端需要安装 redis") from exc
|
||||
self._redis = redis.Redis.from_url(url)
|
||||
self.window_seconds = window_seconds
|
||||
|
||||
def is_duplicate(self, fingerprint: str) -> bool:
|
||||
key = f"hms:log:fp:{fingerprint}"
|
||||
# SET NX EX：成功写入返回 True 表示首次出现；失败表示窗口内重复
|
||||
return not bool(self._redis.set(key, "1", nx=True, ex=self.window_seconds))
|
||||
|
||||
def mark(self, fingerprint: str) -> None:
|
||||
key = f"hms:log:fp:{fingerprint}"
|
||||
self._redis.set(key, "1", ex=self.window_seconds)
|
||||
|
||||
def close(self) -> None:
|
||||
self._redis.close()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KafkaProducerAdapter",
|
||||
"KafkaConsumerAdapter",
|
||||
"ElasticsearchSink",
|
||||
"MySQLFaultRepository",
|
||||
"RedisDedupCache",
|
||||
]
|
||||
|
||||
@ -1 +1,97 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""故障日志捕获管道。
|
||||
|
||||
依据 architecture.md 5.3.2：
|
||||
```
|
||||
logs.raw → 级别过滤(ERROR/FATAL) → 关键字/正则过滤 → 结构化解析
|
||||
→ 去重打标 → Kafka(logs.fault) → Elasticsearch(hms-fault-log-*)
|
||||
```
|
||||
|
||||
本模块把「过滤 → 解析 → 指纹去重 → 启发式归类 → 落库 → 可选生产」串成
|
||||
一个可独立测试的管道。聚类与根因分析由 workers.py 异步完成。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Any, Callable, List, Optional
|
||||
|
||||
from .classifier import Classifier
|
||||
from .filters import FaultFilter
|
||||
from .fingerprint import Deduplicator, MinHash
|
||||
from .models import FaultLog
|
||||
from .parser import LogParser
|
||||
from .storage import Storage
|
||||
|
||||
|
||||
class CapturePipeline:
|
||||
"""故障日志捕获管道。
|
||||
|
||||
参数：
|
||||
storage: 故障日志存储。
|
||||
capture_filter: 过滤规则，默认 FaultFilter（ERROR/FATAL）。
|
||||
parser: 日志解析器，默认 LogParser。
|
||||
classifier: 启发式归类器，默认 Classifier。
|
||||
dedup: 去重器（实现 is_duplicate 接口），默认进程内 Deduplicator。
|
||||
minhash: 指纹器，默认 MinHash。
|
||||
producer: 可选回调，捕获成功后调用（生产到 logs.fault / ES）。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
storage: Storage,
|
||||
capture_filter: Optional[FaultFilter] = None,
|
||||
parser: Optional[LogParser] = None,
|
||||
classifier: Optional[Classifier] = None,
|
||||
dedup: Optional[Deduplicator] = None,
|
||||
minhash: Optional[MinHash] = None,
|
||||
producer: Optional[Callable[[FaultLog], None]] = None,
|
||||
) -> None:
|
||||
self.storage = storage
|
||||
self.capture_filter = capture_filter or FaultFilter()
|
||||
self.parser = parser or LogParser()
|
||||
self.classifier = classifier or Classifier()
|
||||
self.dedup = dedup or Deduplicator()
|
||||
self.minhash = minhash or MinHash()
|
||||
self.producer = producer
|
||||
self.captured = 0
|
||||
self.duplicates = 0
|
||||
|
||||
def process(self, raw: Any) -> Optional[FaultLog]:
|
||||
"""处理一条原始日志；不是故障日志或重复时返回 None。"""
|
||||
parsed = self.parser.parse(raw)
|
||||
|
||||
if not self.capture_filter.should_capture(parsed):
|
||||
return None
|
||||
|
||||
fingerprint = self.minhash.fingerprint(parsed.message or "")
|
||||
if self.dedup.is_duplicate(fingerprint):
|
||||
self.duplicates += 1
|
||||
return None
|
||||
|
||||
fault_type = self.classifier.classify(parsed.message or "")
|
||||
log = FaultLog(
|
||||
fault_log_id=uuid.uuid4().hex,
|
||||
host_id=parsed.host_id or "",
|
||||
message=parsed.message or "",
|
||||
level=parsed.level or "ERROR",
|
||||
fingerprint=fingerprint,
|
||||
fault_type=fault_type,
|
||||
service=parsed.service or None,
|
||||
trace_id=parsed.trace_id,
|
||||
occurred_at=parsed.timestamp,
|
||||
count=1,
|
||||
)
|
||||
self.storage.save_fault_log(log)
|
||||
if self.producer is not None:
|
||||
self.producer(log)
|
||||
self.captured += 1
|
||||
return log
|
||||
|
||||
def process_batch(self, raws: List[Any]) -> List[FaultLog]:
|
||||
"""批量处理原始日志，返回捕获到的 FaultLog 列表。"""
|
||||
results: List[FaultLog] = []
|
||||
for raw in raws:
|
||||
log = self.process(raw)
|
||||
if log is not None:
|
||||
results.append(log)
|
||||
return results
|
||||
|
||||
@ -1 +1,180 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""存储抽象与内存实现。
|
||||
|
||||
依据 docs/01-design/database-design.md 中 fault_log / root_cause /
|
||||
fault_type / fault_filter_rule / event 表结构，提供统一的进程内存储接口。
|
||||
生产环境通过 integrations.py 对接 MySQL / Elasticsearch / Redis；
|
||||
本模块保证零第三方依赖，测试与离线运行使用 MemoryStorage。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .models import Event, FaultFilterRule, FaultLog, FaultType, RootCause
|
||||
|
||||
|
||||
class Storage:
|
||||
"""存储接口（抽象基类）。
|
||||
|
||||
设计上把五类实体（故障日志 / 根因 / 故障类型 / 过滤规则 / 事件）收敛到
|
||||
一个 Storage 门面，便于 pipeline / workers / api 复用同一份存储。
|
||||
"""
|
||||
|
||||
# ---- 故障日志 ------------------------------------------------------
|
||||
def save_fault_log(self, log: FaultLog) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_fault_log(self, fault_log_id: str) -> Optional[FaultLog]:
|
||||
raise NotImplementedError
|
||||
|
||||
def list_fault_logs(self) -> List[FaultLog]:
|
||||
raise NotImplementedError
|
||||
|
||||
def query_fault_logs(
|
||||
self,
|
||||
host_id: Optional[str] = None,
|
||||
fault_type: Optional[str] = None,
|
||||
level: Optional[str] = None,
|
||||
keyword: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
) -> Dict:
|
||||
raise NotImplementedError
|
||||
|
||||
# ---- 根因 ----------------------------------------------------------
|
||||
def save_root_cause(self, root_cause: RootCause) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_root_cause(self, fault_log_id: str) -> Optional[RootCause]:
|
||||
raise NotImplementedError
|
||||
|
||||
# ---- 故障类型 ------------------------------------------------------
|
||||
def list_fault_types(self) -> List[FaultType]:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_fault_type(self, fault_type: str) -> Optional[FaultType]:
|
||||
raise NotImplementedError
|
||||
|
||||
def add_fault_type(self, fault_type: FaultType) -> FaultType:
|
||||
raise NotImplementedError
|
||||
|
||||
# ---- 过滤规则 ------------------------------------------------------
|
||||
def list_filter_rules(self) -> List[FaultFilterRule]:
|
||||
raise NotImplementedError
|
||||
|
||||
def add_filter_rule(self, rule: FaultFilterRule) -> FaultFilterRule:
|
||||
raise NotImplementedError
|
||||
|
||||
# ---- 事件（根因分析关联输入） --------------------------------------
|
||||
def save_event(self, event: Event) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def list_events_by_host(
|
||||
self, host_id: str, start: datetime, end: datetime
|
||||
) -> List[Event]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MemoryStorage(Storage):
|
||||
"""进程内内存实现，用于单元测试与离线运行。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._fault_logs: Dict[str, FaultLog] = {}
|
||||
self._root_causes: Dict[str, RootCause] = {}
|
||||
self._fault_types: Dict[str, FaultType] = {}
|
||||
self._filter_rules: Dict[str, FaultFilterRule] = {}
|
||||
self._events: List[Event] = []
|
||||
|
||||
# ---- 故障日志 ------------------------------------------------------
|
||||
def save_fault_log(self, log: FaultLog) -> None:
|
||||
with self._lock:
|
||||
self._fault_logs[log.fault_log_id] = log
|
||||
|
||||
def get_fault_log(self, fault_log_id: str) -> Optional[FaultLog]:
|
||||
with self._lock:
|
||||
return self._fault_logs.get(fault_log_id)
|
||||
|
||||
def list_fault_logs(self) -> List[FaultLog]:
|
||||
with self._lock:
|
||||
return sorted(
|
||||
self._fault_logs.values(),
|
||||
key=lambda l: (l.occurred_at, l.fault_log_id),
|
||||
)
|
||||
|
||||
def query_fault_logs(
|
||||
self,
|
||||
host_id: Optional[str] = None,
|
||||
fault_type: Optional[str] = None,
|
||||
level: Optional[str] = None,
|
||||
keyword: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
) -> Dict:
|
||||
with self._lock:
|
||||
items = list(self._fault_logs.values())
|
||||
if host_id:
|
||||
items = [l for l in items if l.host_id == host_id]
|
||||
if fault_type:
|
||||
items = [l for l in items if l.fault_type == fault_type]
|
||||
if level:
|
||||
items = [l for l in items if l.level.upper() == level.upper()]
|
||||
if keyword:
|
||||
needle = keyword.lower()
|
||||
items = [l for l in items if needle in (l.message or "").lower()]
|
||||
items.sort(key=lambda l: (l.occurred_at, l.fault_log_id), reverse=True)
|
||||
total = len(items)
|
||||
page = max(1, page)
|
||||
page_size = max(1, min(page_size, 200))
|
||||
start = (page - 1) * page_size
|
||||
return {"total": total, "items": items[start : start + page_size]}
|
||||
|
||||
# ---- 根因 ----------------------------------------------------------
|
||||
def save_root_cause(self, root_cause: RootCause) -> None:
|
||||
with self._lock:
|
||||
self._root_causes[root_cause.fault_log_id] = root_cause
|
||||
|
||||
def get_root_cause(self, fault_log_id: str) -> Optional[RootCause]:
|
||||
with self._lock:
|
||||
return self._root_causes.get(fault_log_id)
|
||||
|
||||
# ---- 故障类型 ------------------------------------------------------
|
||||
def list_fault_types(self) -> List[FaultType]:
|
||||
with self._lock:
|
||||
return list(self._fault_types.values())
|
||||
|
||||
def get_fault_type(self, fault_type: str) -> Optional[FaultType]:
|
||||
with self._lock:
|
||||
return self._fault_types.get(fault_type)
|
||||
|
||||
def add_fault_type(self, fault_type: FaultType) -> FaultType:
|
||||
with self._lock:
|
||||
self._fault_types[fault_type.fault_type] = fault_type
|
||||
return fault_type
|
||||
|
||||
# ---- 过滤规则 ------------------------------------------------------
|
||||
def list_filter_rules(self) -> List[FaultFilterRule]:
|
||||
with self._lock:
|
||||
return list(self._filter_rules.values())
|
||||
|
||||
def add_filter_rule(self, rule: FaultFilterRule) -> FaultFilterRule:
|
||||
with self._lock:
|
||||
self._filter_rules[rule.name] = rule
|
||||
return rule
|
||||
|
||||
# ---- 事件 ----------------------------------------------------------
|
||||
def save_event(self, event: Event) -> None:
|
||||
with self._lock:
|
||||
self._events.append(event)
|
||||
|
||||
def list_events_by_host(
|
||||
self, host_id: str, start: datetime, end: datetime
|
||||
) -> List[Event]:
|
||||
with self._lock:
|
||||
return [
|
||||
e
|
||||
for e in self._events
|
||||
if e.host_id == host_id and start <= e.fired_at <= end
|
||||
]
|
||||
|
||||
@ -1 +1,108 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""聚类与根因分析 worker。
|
||||
|
||||
依据 architecture.md 5.3.3 / 5.3.4：
|
||||
- ClusterWorker：批量对未归类故障日志做 TF-IDF + DBSCAN 聚类，映射到 fault_type。
|
||||
- RootCauseWorker：批量关联同主机近期指标事件，产出 root_cause。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from typing import Dict, List, Optional, Sequence
|
||||
|
||||
from .classifier import Classifier
|
||||
from .cluster import Clusterer
|
||||
from .models import FaultLog, RootCause
|
||||
from .root_cause import RootCauseAnalyzer
|
||||
from .storage import Storage
|
||||
|
||||
|
||||
def _cluster_label(cluster_id: int) -> str:
|
||||
return f"c-{cluster_id}"
|
||||
|
||||
|
||||
class ClusterWorker:
|
||||
"""批量聚类并归类故障日志。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
storage: Storage,
|
||||
clusterer: Optional[Clusterer] = None,
|
||||
classifier: Optional[Classifier] = None,
|
||||
) -> None:
|
||||
self.storage = storage
|
||||
self.clusterer = clusterer or Clusterer()
|
||||
self.classifier = classifier or Classifier()
|
||||
|
||||
def run(self, logs: Optional[Sequence[FaultLog]] = None) -> Dict:
|
||||
"""对故障日志聚类并持久化归类结果。
|
||||
|
||||
返回：{"clustered": n, "clusters": m, "noise": k}
|
||||
"""
|
||||
items: List[FaultLog] = list(logs) if logs is not None else self.storage.list_fault_logs()
|
||||
if not items:
|
||||
return {"clustered": 0, "clusters": 0, "noise": 0}
|
||||
|
||||
messages = [l.message or "" for l in items]
|
||||
labels = self.clusterer.fit_predict(messages)
|
||||
|
||||
# 按簇收集代表消息，用于簇级归类
|
||||
cluster_messages: Dict[int, List[str]] = {}
|
||||
for label, message in zip(labels, messages):
|
||||
if label >= 0:
|
||||
cluster_messages.setdefault(label, []).append(message)
|
||||
|
||||
fault_type_by_cluster: Dict[int, Optional[str]] = {}
|
||||
for label, msgs in cluster_messages.items():
|
||||
fault_type_by_cluster[label] = self.classifier.classify_cluster(label, msgs)
|
||||
|
||||
clustered = 0
|
||||
for log, label in zip(items, labels):
|
||||
if label >= 0:
|
||||
log.cluster_id = _cluster_label(label)
|
||||
log.fault_type = fault_type_by_cluster[label] or log.fault_type
|
||||
clustered += 1
|
||||
self.storage.save_fault_log(log)
|
||||
|
||||
noise = sum(1 for lbl in labels if lbl < 0)
|
||||
return {
|
||||
"clustered": clustered,
|
||||
"clusters": self.clusterer.n_clusters,
|
||||
"noise": noise,
|
||||
}
|
||||
|
||||
|
||||
class RootCauseWorker:
|
||||
"""批量执行根因分析并持久化结论。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
storage: Storage,
|
||||
analyzer: Optional[RootCauseAnalyzer] = None,
|
||||
window_minutes: int = 5,
|
||||
) -> None:
|
||||
self.storage = storage
|
||||
self.window_minutes = window_minutes
|
||||
self.analyzer = analyzer or RootCauseAnalyzer(window_minutes=window_minutes)
|
||||
|
||||
def run(self, logs: Optional[Sequence[FaultLog]] = None) -> List[RootCause]:
|
||||
"""对尚未分析过的故障日志执行根因分析，返回新产出的结论列表。"""
|
||||
items: List[FaultLog] = list(logs) if logs is not None else self.storage.list_fault_logs()
|
||||
results: List[RootCause] = []
|
||||
window = timedelta(minutes=self.window_minutes)
|
||||
|
||||
for log in items:
|
||||
if self.storage.get_root_cause(log.fault_log_id) is not None:
|
||||
continue
|
||||
start = log.occurred_at - window
|
||||
end = log.occurred_at + window
|
||||
events = (
|
||||
self.storage.list_events_by_host(log.host_id, start, end)
|
||||
if log.host_id
|
||||
else []
|
||||
)
|
||||
root_cause = self.analyzer.analyze(log, events)
|
||||
self.storage.save_root_cause(root_cause)
|
||||
results.append(root_cause)
|
||||
|
||||
return results
|
||||
|
||||
@ -1 +1,75 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""Config 配置模型与环境变量加载测试。"""
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from fault_log_analyzer.config import Config, _env, _env_bool, _env_float, _env_int
|
||||
|
||||
|
||||
class TestEnvHelpers(unittest.TestCase):
|
||||
def test_env_default(self):
|
||||
self.assertEqual(_env("HMS_TEST_UNSET_XYZ", "dflt"), "dflt")
|
||||
|
||||
def test_env_int_parse_and_fallback(self):
|
||||
os.environ["HMS_TEST_INT"] = "42"
|
||||
self.assertEqual(_env_int("HMS_TEST_INT", 1), 42)
|
||||
os.environ["HMS_TEST_INT"] = "abc"
|
||||
self.assertEqual(_env_int("HMS_TEST_INT", 7), 7)
|
||||
del os.environ["HMS_TEST_INT"]
|
||||
|
||||
def test_env_float_parse_and_fallback(self):
|
||||
os.environ["HMS_TEST_FLOAT"] = "0.75"
|
||||
self.assertAlmostEqual(_env_float("HMS_TEST_FLOAT", 1.0), 0.75)
|
||||
os.environ["HMS_TEST_FLOAT"] = "x"
|
||||
self.assertEqual(_env_float("HMS_TEST_FLOAT", 2.5), 2.5)
|
||||
del os.environ["HMS_TEST_FLOAT"]
|
||||
|
||||
def test_env_bool_variants(self):
|
||||
for raw in ("1", "true", "TRUE", "yes", "on"):
|
||||
os.environ["HMS_TEST_BOOL"] = raw
|
||||
self.assertTrue(_env_bool("HMS_TEST_BOOL", False))
|
||||
for raw in ("0", "false", "no", "off"):
|
||||
os.environ["HMS_TEST_BOOL"] = raw
|
||||
self.assertFalse(_env_bool("HMS_TEST_BOOL", True))
|
||||
del os.environ["HMS_TEST_BOOL"]
|
||||
|
||||
|
||||
class TestConfig(unittest.TestCase):
|
||||
def test_defaults(self):
|
||||
cfg = Config()
|
||||
self.assertEqual(cfg.kafka_bootstrap_servers, "localhost:9092")
|
||||
self.assertEqual(cfg.kafka_logs_raw_topic, "logs.raw")
|
||||
self.assertEqual(cfg.kafka_logs_fault_topic, "logs.fault")
|
||||
self.assertEqual(cfg.cluster_eps, 0.75)
|
||||
self.assertEqual(cfg.cluster_min_samples, 5)
|
||||
self.assertEqual(cfg.api_port, 8080)
|
||||
self.assertEqual(cfg.storage, "memory")
|
||||
|
||||
def test_from_env_defaults(self):
|
||||
cfg = Config.from_env()
|
||||
self.assertEqual(cfg.api_host, "127.0.0.1")
|
||||
self.assertEqual(cfg.root_cause_window_minutes, 5)
|
||||
|
||||
def test_from_env_override(self):
|
||||
os.environ["KAFKA_BOOTSTRAP_SERVERS"] = "kafka:29092"
|
||||
os.environ["CLUSTER_EPS"] = "0.6"
|
||||
os.environ["API_PORT"] = "9090"
|
||||
os.environ["STORAGE"] = "mysql"
|
||||
try:
|
||||
cfg = Config.from_env()
|
||||
self.assertEqual(cfg.kafka_bootstrap_servers, "kafka:29092")
|
||||
self.assertAlmostEqual(cfg.cluster_eps, 0.6)
|
||||
self.assertEqual(cfg.api_port, 9090)
|
||||
self.assertEqual(cfg.storage, "mysql")
|
||||
finally:
|
||||
for k in ("KAFKA_BOOTSTRAP_SERVERS", "CLUSTER_EPS", "API_PORT", "STORAGE"):
|
||||
os.environ.pop(k, None)
|
||||
|
||||
def test_to_dict_roundtrip(self):
|
||||
cfg = Config(api_port=9001)
|
||||
data = cfg.to_dict()
|
||||
self.assertEqual(data["api_port"], 9001)
|
||||
self.assertIn("cluster_eps", data)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@ -1 +1,63 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""故障日志过滤规则测试。"""
|
||||
import unittest
|
||||
|
||||
from fault_log_analyzer.filters import DEFAULT_LEVELS, FaultFilter
|
||||
from fault_log_analyzer.models import FaultFilterRule, ParsedLog
|
||||
|
||||
|
||||
def _log(level="ERROR", message="boom"):
|
||||
return ParsedLog(level=level, message=message)
|
||||
|
||||
|
||||
class TestDefaultFilter(unittest.TestCase):
|
||||
def test_error_captured(self):
|
||||
self.assertTrue(FaultFilter().should_capture(_log("ERROR", "x")))
|
||||
|
||||
def test_fatal_captured(self):
|
||||
self.assertTrue(FaultFilter().should_capture(_log("FATAL", "x")))
|
||||
|
||||
def test_info_not_captured(self):
|
||||
self.assertFalse(FaultFilter().should_capture(_log("INFO", "x")))
|
||||
|
||||
def test_warn_not_captured(self):
|
||||
self.assertFalse(FaultFilter().should_capture(_log("WARN", "x")))
|
||||
|
||||
def test_default_levels_set(self):
|
||||
self.assertEqual(DEFAULT_LEVELS, {"ERROR", "FATAL"})
|
||||
|
||||
|
||||
class TestRuleFilter(unittest.TestCase):
|
||||
def test_level_rule(self):
|
||||
f = FaultFilter([FaultFilterRule(name="warn", level="WARN")])
|
||||
self.assertTrue(f.should_capture(_log("WARN", "x")))
|
||||
self.assertFalse(f.should_capture(_log("ERROR", "x")))
|
||||
|
||||
def test_pattern_rule(self):
|
||||
f = FaultFilter([FaultFilterRule(name="oom", level="INFO", pattern="out of memory")])
|
||||
self.assertTrue(f.should_capture(_log("INFO", "out of memory occurred")))
|
||||
self.assertFalse(f.should_capture(_log("INFO", "nothing")))
|
||||
|
||||
def test_exclude_pattern(self):
|
||||
f = FaultFilter(
|
||||
[FaultFilterRule(name="no_heartbeat", level="ERROR", pattern=".*", exclude_pattern="heartbeat")]
|
||||
)
|
||||
self.assertFalse(f.should_capture(_log("ERROR", "heartbeat timeout")))
|
||||
self.assertTrue(f.should_capture(_log("ERROR", "disk full")))
|
||||
|
||||
def test_disabled_rule_skipped(self):
|
||||
f = FaultFilter([FaultFilterRule(name="off", level="WARN", enabled=False)])
|
||||
self.assertFalse(f.should_capture(_log("WARN", "x")))
|
||||
|
||||
def test_add_rule(self):
|
||||
f = FaultFilter()
|
||||
f.add_rule(FaultFilterRule(name="warn", level="WARN"))
|
||||
self.assertTrue(f.should_capture(_log("WARN", "x")))
|
||||
|
||||
def test_compile_rules(self):
|
||||
rules = FaultFilter.compile_rules([{"name": "r1", "level": "WARN"}])
|
||||
self.assertEqual(len(rules), 1)
|
||||
self.assertEqual(rules[0].name, "r1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@ -1 +1,125 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""数据模型测试。"""
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fault_log_analyzer.models import (
|
||||
Event,
|
||||
FaultFilterRule,
|
||||
FaultLog,
|
||||
FaultType,
|
||||
ParsedLog,
|
||||
RootCause,
|
||||
_to_dt,
|
||||
)
|
||||
|
||||
|
||||
class TestToDt(unittest.TestCase):
|
||||
def test_none_returns_now(self):
|
||||
self.assertIsInstance(_to_dt(None), datetime)
|
||||
|
||||
def test_datetime_naive_gets_utc(self):
|
||||
dt = datetime(2025, 1, 1, 10, 0, 0)
|
||||
out = _to_dt(dt)
|
||||
self.assertEqual(out.tzinfo, timezone.utc)
|
||||
|
||||
def test_datetime_aware_kept(self):
|
||||
dt = datetime(2025, 1, 1, 10, 0, 0, tzinfo=timezone.utc)
|
||||
self.assertEqual(_to_dt(dt), dt)
|
||||
|
||||
def test_timestamp(self):
|
||||
ts = 1735722000.0
|
||||
out = _to_dt(ts)
|
||||
self.assertEqual(out.tzinfo, timezone.utc)
|
||||
self.assertEqual(int(out.timestamp()), int(ts))
|
||||
|
||||
def test_iso_string_z(self):
|
||||
out = _to_dt("2025-01-01T10:18:00Z")
|
||||
self.assertEqual(out.tzinfo, timezone.utc)
|
||||
self.assertEqual(out.hour, 10)
|
||||
|
||||
def test_iso_string_space(self):
|
||||
out = _to_dt("2025-01-01 10:18:00")
|
||||
self.assertEqual(out.hour, 10)
|
||||
|
||||
|
||||
class TestFaultLog(unittest.TestCase):
|
||||
def test_to_from_dict_roundtrip(self):
|
||||
log = FaultLog(
|
||||
fault_log_id="fl-1",
|
||||
host_id="h-1",
|
||||
message="No space left on device",
|
||||
level="ERROR",
|
||||
fingerprint="abc",
|
||||
fault_type="disk_full",
|
||||
cluster_id="c-1",
|
||||
occurred_at=datetime(2025, 1, 1, 10, 18, 0, tzinfo=timezone.utc),
|
||||
)
|
||||
data = log.to_dict()
|
||||
self.assertEqual(data["fault_log_id"], "fl-1")
|
||||
restored = FaultLog.from_dict(data)
|
||||
self.assertEqual(restored.fault_log_id, "fl-1")
|
||||
self.assertEqual(restored.occurred_at, log.occurred_at)
|
||||
self.assertEqual(restored.fault_type, "disk_full")
|
||||
|
||||
def test_from_dict_ignores_unknown_fields(self):
|
||||
log = FaultLog.from_dict({"fault_log_id": "x", "host_id": "h", "message": "m", "extra": 1})
|
||||
self.assertEqual(log.fault_log_id, "x")
|
||||
self.assertFalse(hasattr(log, "extra"))
|
||||
|
||||
|
||||
class TestRootCause(unittest.TestCase):
|
||||
def test_to_from_dict(self):
|
||||
rc = RootCause(
|
||||
fault_log_id="fl-1",
|
||||
cause_type="disk_full",
|
||||
evidence=[{"type": "event", "metric": "disk_used_percent"}],
|
||||
confidence=0.92,
|
||||
)
|
||||
data = rc.to_dict()
|
||||
restored = RootCause.from_dict(data)
|
||||
self.assertEqual(restored.cause_type, "disk_full")
|
||||
self.assertEqual(restored.confidence, 0.92)
|
||||
self.assertEqual(restored.evidence[0]["metric"], "disk_used_percent")
|
||||
|
||||
|
||||
class TestFaultType(unittest.TestCase):
|
||||
def test_to_from_dict(self):
|
||||
ft = FaultType(fault_type="disk_full", name="磁盘空间不足", pattern="no space left")
|
||||
restored = FaultType.from_dict(ft.to_dict())
|
||||
self.assertEqual(restored.fault_type, "disk_full")
|
||||
self.assertEqual(restored.pattern, "no space left")
|
||||
|
||||
|
||||
class TestFaultFilterRule(unittest.TestCase):
|
||||
def test_to_from_dict(self):
|
||||
rule = FaultFilterRule(name="oom", level="ERROR", pattern="out of memory")
|
||||
restored = FaultFilterRule.from_dict(rule.to_dict())
|
||||
self.assertEqual(restored.name, "oom")
|
||||
self.assertEqual(restored.level, "ERROR")
|
||||
|
||||
|
||||
class TestEvent(unittest.TestCase):
|
||||
def test_to_from_dict(self):
|
||||
event = Event(
|
||||
event_id="e-1",
|
||||
host_id="h-1",
|
||||
metric="disk_used_percent",
|
||||
value=97.2,
|
||||
threshold=90.0,
|
||||
fired_at=datetime(2025, 1, 1, 10, 18, 0, tzinfo=timezone.utc),
|
||||
)
|
||||
restored = Event.from_dict(event.to_dict())
|
||||
self.assertEqual(restored.event_id, "e-1")
|
||||
self.assertEqual(restored.value, 97.2)
|
||||
|
||||
|
||||
class TestParsedLog(unittest.TestCase):
|
||||
def test_to_dict(self):
|
||||
pl = ParsedLog(host_id="h-1", level="ERROR", message="boom", raw="boom")
|
||||
data = pl.to_dict()
|
||||
self.assertEqual(data["host_id"], "h-1")
|
||||
self.assertIn("timestamp", data)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@ -1 +1,101 @@
|
||||
（已提交至远程 main 分支）
|
||||
"""日志解析与模板化测试。"""
|
||||
import unittest
|
||||
|
||||
from fault_log_analyzer.models import ParsedLog
|
||||
from fault_log_analyzer.parser import LogParser
|
||||
|
||||
|
||||
class TestParseJson(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.parser = LogParser()
|
||||
|
||||
def test_json_dict(self):
|
||||
raw = {
|
||||
"timestamp": "2025-01-01T10:18:00Z",
|
||||
"level": "ERROR",
|
||||
"message": "disk full",
|
||||
"host_id": "h-1",
|
||||
"service": "svc",
|
||||
"trace_id": "tr-1",
|
||||
}
|
||||
parsed = self.parser.parse(raw)
|
||||
self.assertEqual(parsed.level, "ERROR")
|
||||
self.assertEqual(parsed.message, "disk full")
|
||||
self.assertEqual(parsed.host_id, "h-1")
|
||||
self.assertEqual(parsed.service, "svc")
|
||||
self.assertEqual(parsed.trace_id, "tr-1")
|
||||
|
||||
def test_json_dict_uses_msg_alias(self):
|
||||
parsed = self.parser.parse({"msg": "hello", "severity": "warn"})
|
||||
self.assertEqual(parsed.message, "hello")
|
||||
self.assertEqual(parsed.level, "WARN")
|
||||
|
||||
def test_json_line(self):
|
||||
line = '{"timestamp":"2025-01-01T10:00:00Z","level":"FATAL","message":"oom"}'
|
||||
parsed = self.parser.parse(line)
|
||||
self.assertEqual(parsed.level, "FATAL")
|
||||
self.assertEqual(parsed.message, "oom")
|
||||
|
||||
|
||||
class TestParseLog4j(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.parser = LogParser()
|
||||
|
||||
def test_log4j(self):
|
||||
line = "2025-01-01 10:18:00 ERROR [app] No space left on device"
|
||||
parsed = self.parser.parse(line)
|
||||
self.assertEqual(parsed.level, "ERROR")
|
||||
self.assertEqual(parsed.service, "app")
|
||||
self.assertEqual(parsed.message, "No space left on device")
|
||||
|
||||
def test_log4j_with_trace(self):
|
||||
line = "2025-01-01 10:18:00 ERROR [svc] trace_id=abc-123-456 connection refused"
|
||||
parsed = self.parser.parse(line)
|
||||
self.assertEqual(parsed.trace_id, "abc-123-456")
|
||||
|
||||
|
||||
class TestParseSyslog(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.parser = LogParser()
|
||||
|
||||
def test_syslog(self):
|
||||
line = "<134>Jan 1 10:18:00 web-01 nginx[123]: connect timeout to upstream"
|
||||
parsed = self.parser.parse(line)
|
||||
self.assertEqual(parsed.host_id, "web-01")
|
||||
self.assertEqual(parsed.service, "nginx")
|
||||
self.assertIn("connect timeout", parsed.message)
|
||||
|
||||
|
||||
class TestParseFallback(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.parser = LogParser()
|
||||
|
||||
def test_fallback_level_guess(self):
|
||||
parsed = self.parser.parse("some random FATAL failure")
|
||||
self.assertEqual(parsed.level, "FATAL")
|
||||
self.assertEqual(parsed.message, "some random FATAL failure")
|
||||
|
||||
def test_passthrough_parsed_log(self):
|
||||
pl = ParsedLog(level="ERROR", message="x")
|
||||
self.assertIs(self.parser.parse(pl), pl)
|
||||
|
||||
|
||||
class TestTemplate(unittest.TestCase):
|
||||
def test_ip_uuid_number_replacement(self):
|
||||
msg = "connect from 10.0.0.1 uuid 123e4567-e89b-12d3-a456-426614174000 retry 42"
|
||||
templated = LogParser.template(msg)
|
||||
self.assertIn("<IP>", templated)
|
||||
self.assertIn("<UUID>", templated)
|
||||
self.assertIn("<NUM>", templated)
|
||||
self.assertNotIn("10.0.0.1", templated)
|
||||
|
||||
def test_tokenize_filters_short(self):
|
||||
tokens = LogParser.tokenize("a b disk full error")
|
||||
self.assertIn("disk", tokens)
|
||||
self.assertIn("error", tokens)
|
||||
self.assertNotIn("a", tokens)
|
||||
self.assertNotIn("b", tokens)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user