"""特征提取与日志聚类。 核心为纯 Python 实现(无第三方依赖),保证离线可运行;生产环境可通过 ``use_sklearn=True`` 切换到 scikit-learn 的 TF-IDF + DBSCAN 后端。 """ from __future__ import annotations import math import uuid from dataclasses import dataclass from typing import Optional from .models import FaultLog from .parser import templatize, tokenize # --------------------------------------------------------------------------- # TF-IDF # --------------------------------------------------------------------------- class TfidfVectorizer: """极简 TF-IDF 向量化器。""" def __init__(self): self._idf: dict[str, float] = {} self._vocab: list[str] = [] def fit(self, documents: list[list[str]]) -> "TfidfVectorizer": n = len(documents) if n == 0: self._vocab = [] self._idf = {} return self df: dict[str, int] = {} for doc in documents: for token in set(doc): df[token] = df.get(token, 0) + 1 self._vocab = sorted(df.keys()) self._idf = { token: math.log((1 + n) / (1 + df[token])) + 1.0 for token in self._vocab } return self def transform(self, documents: list[list[str]]) -> list[dict[str, float]]: vectors: list[dict[str, float]] = [] for doc in documents: tf: dict[str, float] = {} if doc: for token in doc: tf[token] = tf.get(token, 0.0) + 1.0 norm = math.sqrt(sum(v * v for v in tf.values())) or 1.0 vectors.append( {token: (tf.get(token, 0.0) / norm) * self._idf.get(token, 0.0) for token in tf} ) return vectors def cosine_similarity(a: dict[str, float], b: dict[str, float]) -> float: """两个稀疏向量(L2 归一化后)的余弦相似度。""" if not a or not b: return 0.0 dot = 0.0 if len(a) <= len(b): for k, v in a.items(): dot += v * b.get(k, 0.0) else: for k, v in b.items(): dot += v * a.get(k, 0.0) return max(0.0, min(1.0, dot)) def cosine_distance(a: dict[str, float], b: dict[str, float]) -> float: return 1.0 - cosine_similarity(a, b) # --------------------------------------------------------------------------- # DBSCAN(余弦距离) # --------------------------------------------------------------------------- @dataclass class ClusterResult: """聚类结果。""" labels: list[int] # -1 表示噪声 cluster_ids: dict[int, str] # 簇索引 -> 簇 id representative_vectors: dict[str, dict[str, float]] # 簇 id -> 代表向量 def dbscan( vectors: list[dict[str, float]], eps: float = 0.75, min_samples: int = 5, ) -> ClusterResult: """DBSCAN(余弦距离),纯 Python 实现,O(n^2)。 ``eps`` 为余弦距离阈值(等价于相似度 >= 1 - eps)。 """ n = len(vectors) if n == 0: return ClusterResult(labels=[], cluster_ids={}, representative_vectors={}) # 邻接表(核心点判定:邻域内点数 >= min_samples,含自身) neighbors: list[list[int]] = [[] for _ in range(n)] for i in range(n): neighbors[i].append(i) for j in range(i + 1, n): if cosine_distance(vectors[i], vectors[j]) <= eps: neighbors[i].append(j) neighbors[j].append(i) labels = [-1] * n visited = [False] * n cluster = 0 for i in range(n): if visited[i]: continue visited[i] = True if len(neighbors[i]) < min_samples: labels[i] = -1 # 噪声 continue # 核心点:扩展新簇 labels[i] = cluster seeds = list(neighbors[i]) for q in seeds: if not visited[q]: visited[q] = True if len(neighbors[q]) >= min_samples: seeds.extend(neighbors[q]) if labels[q] == -1: labels[q] = cluster cluster += 1 cluster_ids: dict[int, str] = {} representative_vectors: dict[str, dict[str, float]] = {} for c in range(cluster): members = [i for i in range(n) if labels[i] == c] cid = f"c-{uuid.uuid4().hex[:8]}" cluster_ids[c] = cid representative_vectors[cid] = _centroid([vectors[i] for i in members]) return ClusterResult(labels=labels, cluster_ids=cluster_ids, representative_vectors=representative_vectors) def _centroid(vectors: list[dict[str, float]]) -> dict[str, float]: if not vectors: return {} acc: dict[str, float] = {} for vec in vectors: for k, v in vec.items(): acc[k] = acc.get(k, 0.0) + v n = len(vectors) return {k: v / n for k, v in acc.items()} # --------------------------------------------------------------------------- # 增量聚类引擎 # --------------------------------------------------------------------------- class ClusterEngine: """日志聚类引擎:特征提取 + DBSCAN + 增量分配。""" def __init__(self, eps: float = 0.75, min_samples: int = 5, use_sklearn: bool = False): self.eps = eps self.min_samples = min_samples self.use_sklearn = use_sklearn # 已存在簇的代表向量:cluster_id -> 向量 self._representatives: dict[str, dict[str, float]] = {} def _vectors(self, logs: list[FaultLog]) -> list[dict[str, float]]: docs = [tokenize(templatize(log.message)) for log in logs] if self.use_sklearn: return self._sklearn_vectors(docs) vectorizer = TfidfVectorizer().fit(docs) return vectorizer.transform(docs) @staticmethod def _sklearn_vectors(docs: list[list[str]]) -> list[dict[str, float]]: try: from sklearn.feature_extraction.text import TfidfVectorizer as SkTfidf except ImportError as exc: # pragma: no cover raise RuntimeError("use_sklearn=True 需要安装 scikit-learn") from exc joined = [" ".join(doc) for doc in docs] if not any(joined): return [{} for _ in docs] m = SkTfidfVectorizer().fit_transform(joined) rows = m.toarray() return [{str(i): float(v) for i, v in enumerate(row) if v != 0.0} for row in rows] def cluster_batch(self, logs: list[FaultLog]) -> dict[str, str]: """对一批日志聚类,返回 {fault_log_id: cluster_id}。 优先增量匹配已有簇,剩余未匹配日志再做一次 DBSCAN 形成新簇。 """ assignment: dict[str, str] = {} if not logs: return assignment vectors = self._vectors(logs) unresolved_idx: list[int] = [] for i, log in enumerate(logs): matched = self._match_representative(vectors[i]) if matched is not None: assignment[log.fault_log_id] = matched else: unresolved_idx.append(i) if unresolved_idx: sub_vectors = [vectors[i] for i in unresolved_idx] result = dbscan(sub_vectors, self.eps, self.min_samples) for local_idx, global_idx in enumerate(unresolved_idx): label = result.labels[local_idx] if label == -1: # 噪声日志独立成单点簇 cid = f"c-{uuid.uuid4().hex[:8]}" self._representatives[cid] = sub_vectors[local_idx] assignment[logs[global_idx].fault_log_id] = cid else: cid = result.cluster_ids[label] assignment[logs[global_idx].fault_log_id] = cid # 更新新簇代表向量 for cid, vec in result.representative_vectors.items(): self._representatives[cid] = vec return assignment def _match_representative(self, vector: dict[str, float]) -> Optional[str]: if not vector or not self._representatives: return None best_cid: Optional[str] = None best_sim = 1.0 - self.eps # 相似度阈值 for cid, rep in self._representatives.items(): sim = cosine_similarity(vector, rep) if sim > best_sim: best_sim = sim best_cid = cid return best_cid