develop: 重做 fault-log-analyzer 分析模块(遵守模块开发规范)

This commit is contained in:
Pipeline Agent 2026-08-15 11:23:29 +08:00
parent c62453ef70
commit de5638076c
25 changed files with 613 additions and 0 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

130
tests/test_api.py Normal file
View File

@ -0,0 +1,130 @@
import _bootstrap # noqa: F401
import json
import time
import unittest
import urllib.error
import urllib.request
from datetime import datetime, timezone
from fault_log_analyzer.api import FaultLogApiServer
from fault_log_analyzer.models import FaultLog, RootCause
from fault_log_analyzer.storage import (
InMemoryFaultLogRepository,
InMemoryFaultTypeRepository,
InMemoryFilterRuleRepository,
InMemoryRootCauseRepository,
)
class TestFaultLogApiServer(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.fl_repo = InMemoryFaultLogRepository()
cls.rc_repo = InMemoryRootCauseRepository()
cls.ft_repo = InMemoryFaultTypeRepository()
cls.fr_repo = InMemoryFilterRuleRepository()
cls.fl_repo.save(
FaultLog(
fault_log_id="fl-1",
host_id="h-1",
fingerprint="fp",
level="ERROR",
message="disk full",
occurred_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
fault_type="disk_full",
)
)
cls.rc_repo.save(RootCause(fault_log_id="fl-1", cause_type="disk_full", confidence=0.9))
cls.server = FaultLogApiServer(
cls.fl_repo, cls.rc_repo, cls.ft_repo, cls.fr_repo, host="127.0.0.1", port=0
)
cls.server.start()
# 等待监听线程就绪
time.sleep(0.05)
@classmethod
def tearDownClass(cls):
cls.server.stop()
def _get(self, path):
with urllib.request.urlopen(f"http://127.0.0.1:{self.server.port}{path}", timeout=5) as resp:
return json.loads(resp.read().decode("utf-8"))
def _post(self, path, body):
req = urllib.request.Request(
f"http://127.0.0.1:{self.server.port}{path}",
data=json.dumps(body).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=5) as resp:
return json.loads(resp.read().decode("utf-8"))
def test_healthz(self):
data = self._get("/healthz")
self.assertEqual(data["code"], 0)
self.assertEqual(data["data"]["status"], "ok")
def test_readyz(self):
data = self._get("/readyz")
self.assertEqual(data["code"], 0)
self.assertIn("deps", data["data"])
def test_list_fault_logs(self):
data = self._get("/api/v1/fault-logs")
self.assertEqual(data["data"]["total"], 1)
self.assertEqual(data["data"]["items"][0]["fault_log_id"], "fl-1")
def test_list_fault_logs_filter(self):
data = self._get("/api/v1/fault-logs?host_id=h-1&level=ERROR")
self.assertEqual(data["data"]["total"], 1)
def test_get_fault_log(self):
data = self._get("/api/v1/fault-logs/fl-1")
self.assertEqual(data["data"]["fault_log_id"], "fl-1")
def test_get_fault_log_404(self):
with self.assertRaises(urllib.error.HTTPError) as ctx:
self._get("/api/v1/fault-logs/nope")
self.assertEqual(ctx.exception.code, 404)
def test_get_root_cause(self):
data = self._get("/api/v1/fault-logs/fl-1/root-cause")
self.assertEqual(data["data"]["cause_type"], "disk_full")
def test_list_fault_types(self):
data = self._get("/api/v1/fault-types")
self.assertEqual(data["code"], 0)
self.assertIn("items", data["data"])
def test_create_fault_type(self):
data = self._post("/api/v1/fault-types", {"fault_type": "oom", "name": "OOM"})
self.assertEqual(data["code"], 0)
self.assertEqual(data["data"]["fault_type"], "oom")
def test_create_fault_type_missing_fields(self):
with self.assertRaises(urllib.error.HTTPError) as ctx:
self._post("/api/v1/fault-types", {"name": "OOM"})
self.assertEqual(ctx.exception.code, 400)
def test_list_fault_filters(self):
data = self._get("/api/v1/fault-filters")
self.assertEqual(data["code"], 0)
def test_create_fault_filter(self):
data = self._post(
"/api/v1/fault-filters", {"name": "disk", "level": "ERROR", "pattern": "disk"}
)
self.assertEqual(data["code"], 0)
def test_unknown_route_404(self):
with self.assertRaises(urllib.error.HTTPError) as ctx:
self._get("/nope")
self.assertEqual(ctx.exception.code, 404)
if __name__ == "__main__":
unittest.main()

48
tests/test_classifier.py Normal file
View File

@ -0,0 +1,48 @@
import _bootstrap # noqa: F401
import unittest
from datetime import datetime, timezone
from fault_log_analyzer.classifier import FaultClassifier
from fault_log_analyzer.models import FaultLog, FaultType
def make_log(message, cluster_id=""):
return FaultLog(
fault_log_id="fl-1",
host_id="h-1",
fingerprint="fp",
level="ERROR",
message=message,
occurred_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
cluster_id=cluster_id,
)
class TestFaultClassifier(unittest.TestCase):
def test_pattern_match(self):
c = FaultClassifier([FaultType(fault_type="disk_full", name="磁盘满", pattern=r"no space|disk full")])
self.assertEqual(c.match(make_log("No space left on device")), "disk_full")
def test_name_match(self):
c = FaultClassifier([FaultType(fault_type="oom", name="OOMKilled")])
self.assertEqual(c.match(make_log("OOMKilled process")), "oom")
def test_cluster_type_map_priority(self):
c = FaultClassifier([FaultType(fault_type="disk_full", name="磁盘满", pattern="")])
self.assertEqual(c.match(make_log("anything", cluster_id="c1"), {"c1": "disk_full"}), "disk_full")
def test_no_match(self):
c = FaultClassifier([])
self.assertIsNone(c.match(make_log("unknown message")))
def test_guess_candidate(self):
c = FaultClassifier()
self.assertEqual(c.guess_candidate("No space left on device"), "disk_full")
self.assertEqual(c.guess_candidate("Out of memory"), "oom")
self.assertEqual(c.guess_candidate("Connection refused"), "connection_refused")
self.assertEqual(c.guess_candidate("nothing recognizable"), "unknown")
if __name__ == "__main__":
unittest.main()

89
tests/test_cluster.py Normal file
View File

@ -0,0 +1,89 @@
import _bootstrap # noqa: F401
import unittest
from datetime import datetime, timezone
from fault_log_analyzer.cluster import (
ClusterEngine,
TfidfVectorizer,
cosine_distance,
cosine_similarity,
dbscan,
)
from fault_log_analyzer.models import FaultLog
def make_log(fid, message, host="h-1"):
return FaultLog(
fault_log_id=fid,
host_id=host,
fingerprint="fp",
level="ERROR",
message=message,
occurred_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
)
class TestTfidfVectorizer(unittest.TestCase):
def test_fit_transform(self):
vec = TfidfVectorizer().fit([["a", "b"], ["b", "c"]])
out = vec.transform([["a", "b"], ["b", "c"]])
self.assertEqual(len(out), 2)
self.assertTrue(all(isinstance(v, dict) for v in out))
def test_empty_documents(self):
vec = TfidfVectorizer().fit([])
self.assertEqual(vec.transform([]), [])
class TestCosineSimilarity(unittest.TestCase):
def test_identical(self):
self.assertEqual(cosine_similarity({"a": 1.0}, {"a": 1.0}), 1.0)
def test_disjoint(self):
self.assertEqual(cosine_similarity({"a": 1.0}, {"b": 1.0}), 0.0)
def test_distance(self):
self.assertAlmostEqual(cosine_distance({"a": 1.0}, {"a": 1.0}), 0.0)
class TestDbscan(unittest.TestCase):
def test_empty(self):
result = dbscan([])
self.assertEqual(result.labels, [])
self.assertEqual(result.cluster_ids, {})
def test_single_noise(self):
result = dbscan([{"a": 1.0}], eps=0.75, min_samples=2)
self.assertEqual(result.labels, [-1])
def test_two_similar_cluster(self):
result = dbscan([{"a": 1.0}, {"a": 1.0}], eps=0.75, min_samples=1)
self.assertEqual(result.labels[0], result.labels[1])
self.assertNotEqual(result.labels[0], -1)
def test_two_dissimilar_noise(self):
result = dbscan([{"a": 1.0}, {"b": 1.0}], eps=0.75, min_samples=2)
self.assertEqual(result.labels, [-1, -1])
class TestClusterEngine(unittest.TestCase):
def test_cluster_batch_empty(self):
ce = ClusterEngine()
self.assertEqual(ce.cluster_batch([]), {})
def test_cluster_batch_groups_identical(self):
ce = ClusterEngine(eps=0.75, min_samples=1)
logs = [make_log("fl1", "No space left on device"), make_log("fl2", "No space left on device")]
assignment = ce.cluster_batch(logs)
self.assertEqual(assignment["fl1"], assignment["fl2"])
def test_incremental_match_existing_cluster(self):
ce = ClusterEngine(eps=0.75, min_samples=1)
first = ce.cluster_batch([make_log("fl1", "No space left on device")])
second = ce.cluster_batch([make_log("fl2", "No space left on device")])
self.assertEqual(second["fl2"], first["fl1"])
if __name__ == "__main__":
unittest.main()

56
tests/test_pipeline.py Normal file
View File

@ -0,0 +1,56 @@
import _bootstrap # noqa: F401
import unittest
from fault_log_analyzer.filters import CaptureFilter
from fault_log_analyzer.pipeline import CapturePipeline
from fault_log_analyzer.storage import InMemoryDedupCache, InMemoryLogSink
class TestCapturePipeline(unittest.TestCase):
def test_process_raw_fault(self):
produced = []
sink = InMemoryLogSink()
p = CapturePipeline(
capture_filter=CaptureFilter(),
dedup=InMemoryDedupCache(),
log_sink=sink,
fault_producer=produced.append,
)
raw = {
"timestamp": "2025-01-01T10:00:00Z",
"level": "ERROR",
"message": "disk full",
"host_id": "h-1",
}
fl = p.process_raw(raw)
self.assertIsNotNone(fl)
self.assertEqual(fl.host_id, "h-1")
self.assertEqual(fl.level, "ERROR")
self.assertEqual(len(sink.search()), 1)
self.assertEqual(len(produced), 1)
def test_process_raw_non_fault(self):
p = CapturePipeline(capture_filter=CaptureFilter())
self.assertIsNone(p.process_raw({"level": "INFO", "message": "hello"}))
def test_dedup_same_fingerprint(self):
p = CapturePipeline(capture_filter=CaptureFilter(), dedup=InMemoryDedupCache())
raw = {"level": "ERROR", "message": "disk full"}
self.assertIsNotNone(p.process_raw(raw))
self.assertIsNone(p.process_raw(raw))
def test_process_batch(self):
p = CapturePipeline(capture_filter=CaptureFilter())
out = p.process_batch(
[
{"level": "ERROR", "message": "disk full"},
{"level": "INFO", "message": "ok"},
]
)
self.assertEqual(len(out), 1)
self.assertEqual(out[0].message, "disk full")
if __name__ == "__main__":
unittest.main()

87
tests/test_root_cause.py Normal file
View File

@ -0,0 +1,87 @@
import _bootstrap # noqa: F401
import unittest
from datetime import datetime, timezone
from fault_log_analyzer.models import Event, FaultLog
from fault_log_analyzer.root_cause import RootCauseAnalyzer, RootCauseRule
def make_log(message="No space left on device", fid="fl-1", host="h-1", trace_id=""):
return FaultLog(
fault_log_id=fid,
host_id=host,
fingerprint="fp",
level="ERROR",
message=message,
occurred_at=datetime(2025, 1, 1, 10, 0, 0, tzinfo=timezone.utc),
trace_id=trace_id,
)
def make_event(event_id="e-1", host="h-1", metric="disk_used_percent", value=97.2, minute=1):
return Event(
event_id=event_id,
host_id=host,
metric=metric,
value=value,
fired_at=datetime(2025, 1, 1, 10, minute, 0, tzinfo=timezone.utc),
)
class TestRootCauseRule(unittest.TestCase):
def test_match_log(self):
rule = RootCauseRule("disk_full", r"no space left on device")
self.assertTrue(rule.match_log("No space left on device"))
self.assertFalse(rule.match_log("connection refused"))
def test_match_metric_threshold(self):
rule = RootCauseRule("disk_full", metric_pattern=r"disk.*used", metric_threshold=90.0)
self.assertTrue(rule.match_metric("disk_used_percent", 95.0))
self.assertFalse(rule.match_metric("disk_used_percent", 80.0))
self.assertFalse(rule.match_metric("cpu_usage", 95.0))
class TestRootCauseAnalyzer(unittest.TestCase):
def test_disk_full_with_metric_event(self):
rc = RootCauseAnalyzer().analyze(
make_log("No space left on device"),
[make_event()],
)
self.assertEqual(rc.cause_type, "disk_full")
self.assertGreaterEqual(rc.confidence, 0.95)
self.assertTrue(any(e.type == "event" for e in rc.evidence))
self.assertTrue(any(e.type == "log" for e in rc.evidence))
def test_rule_without_matching_event(self):
rc = RootCauseAnalyzer().analyze(
make_log("No space left on device"),
[make_event(metric="cpu_usage", value=80.0)],
)
self.assertEqual(rc.cause_type, "disk_full")
# 日志规则命中但指标事件不匹配,置信度为基础值
self.assertAlmostEqual(rc.confidence, 0.95)
def test_unknown_with_related_event(self):
rc = RootCauseAnalyzer().analyze(
make_log("mysterious failure"),
[make_event(metric="cpu_usage", value=50.0)],
)
self.assertEqual(rc.cause_type, "related_cpu_usage")
self.assertEqual(rc.confidence, 0.4)
def test_unknown_with_trace_only(self):
rc = RootCauseAnalyzer().analyze(make_log("mysterious failure", trace_id="t-1"), [])
self.assertEqual(rc.cause_type, "trace_linked")
self.assertEqual(rc.confidence, 0.3)
def test_event_out_of_window_ignored(self):
old_event = make_event(event_id="e-old", minute=20) # 超出 ±5 分钟窗口
rc = RootCauseAnalyzer().analyze(make_log("No space left on device"), [old_event])
# 日志规则仍命中,但没有指标事件证据增强
self.assertEqual(rc.cause_type, "disk_full")
self.assertFalse(any(e.type == "event" for e in rc.evidence))
if __name__ == "__main__":
unittest.main()

131
tests/test_storage.py Normal file
View File

@ -0,0 +1,131 @@
import _bootstrap # noqa: F401
import unittest
from datetime import datetime, timezone
from fault_log_analyzer.models import (
Event,
FaultFilterRule,
FaultLog,
FaultType,
RootCause,
)
from fault_log_analyzer.storage import (
InMemoryDedupCache,
InMemoryEventRepository,
InMemoryFaultLogRepository,
InMemoryFaultTypeRepository,
InMemoryFilterRuleRepository,
InMemoryLogSink,
InMemoryRootCauseRepository,
)
def make_log(fid, host="h-1", fault_type="disk_full", level="ERROR", message="disk full", minute=0):
return FaultLog(
fault_log_id=fid,
host_id=host,
fingerprint=f"fp-{fid}",
level=level,
message=message,
occurred_at=datetime(2025, 1, 1, 0, minute, 0, tzinfo=timezone.utc),
fault_type=fault_type,
)
class TestInMemoryFaultLogRepository(unittest.TestCase):
def setUp(self):
self.repo = InMemoryFaultLogRepository()
self.repo.save(make_log("fl1", host="h-1", fault_type="disk_full", level="ERROR", message="disk full", minute=1))
self.repo.save(make_log("fl2", host="h-2", fault_type="oom", level="FATAL", message="oom killed", minute=2))
def test_get(self):
self.assertIsNotNone(self.repo.get("fl1"))
self.assertIsNone(self.repo.get("nope"))
def test_list_filter_by_host(self):
total, items = self.repo.list(host_id="h-1")
self.assertEqual(total, 1)
self.assertEqual(items[0].fault_log_id, "fl1")
def test_list_filter_by_fault_type(self):
total, items = self.repo.list(fault_type="oom")
self.assertEqual(total, 1)
self.assertEqual(items[0].fault_log_id, "fl2")
def test_list_filter_by_level_and_keyword(self):
total, _ = self.repo.list(level="fatal")
self.assertEqual(total, 1)
total, items = self.repo.list(keyword="disk")
self.assertEqual(total, 1)
self.assertEqual(items[0].fault_log_id, "fl1")
def test_pagination(self):
total, items = self.repo.list(page=1, page_size=1)
self.assertEqual(total, 2)
self.assertEqual(len(items), 1)
class TestInMemoryRootCauseRepository(unittest.TestCase):
def test_save_and_get(self):
repo = InMemoryRootCauseRepository()
repo.save(RootCause(fault_log_id="fl1", cause_type="disk_full", confidence=0.9))
self.assertEqual(repo.get("fl1").cause_type, "disk_full")
self.assertIsNone(repo.get("nope"))
class TestInMemoryFaultTypeRepository(unittest.TestCase):
def test_add_list_get(self):
repo = InMemoryFaultTypeRepository()
repo.add(FaultType(fault_type="disk_full", name="磁盘满"))
self.assertEqual(repo.get("disk_full").name, "磁盘满")
self.assertEqual(len(repo.list()), 1)
def test_add_empty_raises(self):
repo = InMemoryFaultTypeRepository()
with self.assertRaises(ValueError):
repo.add(FaultType(fault_type="", name="x"))
class TestInMemoryFilterRuleRepository(unittest.TestCase):
def test_add_assigns_id(self):
repo = InMemoryFilterRuleRepository()
rule = repo.add(FaultFilterRule(name="disk", pattern="disk"))
self.assertIsNotNone(rule.id)
self.assertEqual(len(repo.list()), 1)
class TestInMemoryEventRepository(unittest.TestCase):
def test_list_by_host_window(self):
e1 = Event(event_id="e1", host_id="h-1", metric="cpu", value=1.0,
fired_at=datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc))
e2 = Event(event_id="e2", host_id="h-2", metric="cpu", value=1.0,
fired_at=datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc))
repo = InMemoryEventRepository([e1, e2])
start = datetime(2024, 12, 31, tzinfo=timezone.utc)
end = datetime(2025, 1, 2, tzinfo=timezone.utc)
self.assertEqual(len(repo.list_by_host("h-1", start, end)), 1)
self.assertEqual(repo.list_by_host("h-1", start, end)[0].event_id, "e1")
class TestInMemoryLogSink(unittest.TestCase):
def test_write_and_search(self):
sink = InMemoryLogSink()
sink.write(make_log("fl1", host="h-1", level="ERROR", message="disk full"))
sink.write(make_log("fl2", host="h-2", level="FATAL", message="oom"))
self.assertEqual(len(sink.search()), 2)
self.assertEqual(len(sink.search(host_id="h-1")), 1)
self.assertEqual(len(sink.search(level="fatal")), 1)
self.assertEqual(len(sink.search(keyword="disk")), 1)
class TestInMemoryDedupCache(unittest.TestCase):
def test_seen_before(self):
cache = InMemoryDedupCache()
self.assertFalse(cache.seen_before("fp1"))
self.assertTrue(cache.seen_before("fp1"))
self.assertFalse(cache.seen_before("fp2"))
if __name__ == "__main__":
unittest.main()

72
tests/test_workers.py Normal file
View File

@ -0,0 +1,72 @@
import _bootstrap # noqa: F401
import unittest
from datetime import datetime, timezone
from fault_log_analyzer.models import Event, FaultLog, FaultType
from fault_log_analyzer.storage import (
InMemoryEventRepository,
InMemoryFaultLogRepository,
InMemoryFaultTypeRepository,
InMemoryRootCauseRepository,
)
from fault_log_analyzer.workers import ClusteringWorker, RootCauseWorker
def make_log(fid, message, host="h-1"):
return FaultLog(
fault_log_id=fid,
host_id=host,
fingerprint=f"fp-{fid}",
level="ERROR",
message=message,
occurred_at=datetime(2025, 1, 1, 10, 0, 0, tzinfo=timezone.utc),
)
class TestClusteringWorker(unittest.TestCase):
def test_run_classifies_and_persists(self):
fl_repo = InMemoryFaultLogRepository()
ft_repo = InMemoryFaultTypeRepository([FaultType(fault_type="disk_full", name="磁盘满", pattern=r"no space")])
worker = ClusteringWorker(fl_repo, ft_repo, eps=0.75, min_samples=1)
logs = [make_log("fl1", "No space left on device")]
out = worker.run(logs)
self.assertEqual(out[0].fault_type, "disk_full")
self.assertNotEqual(out[0].cluster_id, "")
persisted = fl_repo.get("fl1")
self.assertEqual(persisted.fault_type, "disk_full")
self.assertEqual(persisted.cluster_id, out[0].cluster_id)
def test_run_empty(self):
worker = ClusteringWorker(InMemoryFaultLogRepository(), InMemoryFaultTypeRepository())
self.assertEqual(worker.run([]), [])
def test_guess_candidate_when_no_type_matches(self):
fl_repo = InMemoryFaultLogRepository()
worker = ClusteringWorker(fl_repo, InMemoryFaultTypeRepository(), eps=0.75, min_samples=1)
out = worker.run([make_log("fl1", "No space left on device")])
self.assertEqual(out[0].fault_type, "disk_full")
class TestRootCauseWorker(unittest.TestCase):
def test_run_persists_root_cause(self):
rc_repo = InMemoryRootCauseRepository()
event_repo = InMemoryEventRepository(
[
Event(
event_id="e1",
host_id="h-1",
metric="disk_used_percent",
value=97.0,
fired_at=datetime(2025, 1, 1, 10, 1, 0, tzinfo=timezone.utc),
)
]
)
worker = RootCauseWorker(rc_repo, event_repo, window_minutes=5)
results = worker.run([make_log("fl1", "No space left on device")])
self.assertEqual(results[0].cause_type, "disk_full")
self.assertEqual(rc_repo.get("fl1").cause_type, "disk_full")
if __name__ == "__main__":
unittest.main()