236 lines
8.2 KiB
Python
236 lines
8.2 KiB
Python
"""特征提取与日志聚类。
|
||
|
||
核心为纯 Python 实现（无第三方依赖），保证离线可运行；生产环境可通过
|
||
``use_sklearn=True`` 切换到 scikit-learn 的 TF-IDF + DBSCAN 后端。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import math
|
||
import uuid
|
||
from dataclasses import dataclass
|
||
from typing import Optional
|
||
|
||
from .models import FaultLog
|
||
from .parser import templatize, tokenize
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TF-IDF
|
||
# ---------------------------------------------------------------------------
|
||
class TfidfVectorizer:
|
||
"""极简 TF-IDF 向量化器。"""
|
||
|
||
def __init__(self):
|
||
self._idf: dict[str, float] = {}
|
||
self._vocab: list[str] = []
|
||
|
||
def fit(self, documents: list[list[str]]) -> "TfidfVectorizer":
|
||
n = len(documents)
|
||
if n == 0:
|
||
self._vocab = []
|
||
self._idf = {}
|
||
return self
|
||
df: dict[str, int] = {}
|
||
for doc in documents:
|
||
for token in set(doc):
|
||
df[token] = df.get(token, 0) + 1
|
||
self._vocab = sorted(df.keys())
|
||
self._idf = {
|
||
token: math.log((1 + n) / (1 + df[token])) + 1.0 for token in self._vocab
|
||
}
|
||
return self
|
||
|
||
def transform(self, documents: list[list[str]]) -> list[dict[str, float]]:
|
||
vectors: list[dict[str, float]] = []
|
||
for doc in documents:
|
||
tf: dict[str, float] = {}
|
||
if doc:
|
||
for token in doc:
|
||
tf[token] = tf.get(token, 0.0) + 1.0
|
||
norm = math.sqrt(sum(v * v for v in tf.values())) or 1.0
|
||
vectors.append(
|
||
{token: (tf.get(token, 0.0) / norm) * self._idf.get(token, 0.0) for token in tf}
|
||
)
|
||
return vectors
|
||
|
||
|
||
def cosine_similarity(a: dict[str, float], b: dict[str, float]) -> float:
|
||
"""两个稀疏向量（L2 归一化后）的余弦相似度。"""
|
||
if not a or not b:
|
||
return 0.0
|
||
dot = 0.0
|
||
if len(a) <= len(b):
|
||
for k, v in a.items():
|
||
dot += v * b.get(k, 0.0)
|
||
else:
|
||
for k, v in b.items():
|
||
dot += v * a.get(k, 0.0)
|
||
return max(0.0, min(1.0, dot))
|
||
|
||
|
||
def cosine_distance(a: dict[str, float], b: dict[str, float]) -> float:
|
||
return 1.0 - cosine_similarity(a, b)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# DBSCAN（余弦距离）
|
||
# ---------------------------------------------------------------------------
|
||
@dataclass
|
||
class ClusterResult:
|
||
"""聚类结果。"""
|
||
|
||
labels: list[int] # -1 表示噪声
|
||
cluster_ids: dict[int, str] # 簇索引 -> 簇 id
|
||
representative_vectors: dict[str, dict[str, float]] # 簇 id -> 代表向量
|
||
|
||
|
||
def dbscan(
|
||
vectors: list[dict[str, float]],
|
||
eps: float = 0.75,
|
||
min_samples: int = 5,
|
||
) -> ClusterResult:
|
||
"""DBSCAN（余弦距离），纯 Python 实现，O(n^2)。
|
||
|
||
``eps`` 为余弦距离阈值（等价于相似度 >= 1 - eps）。
|
||
"""
|
||
n = len(vectors)
|
||
if n == 0:
|
||
return ClusterResult(labels=[], cluster_ids={}, representative_vectors={})
|
||
|
||
# 邻接表（核心点判定：邻域内点数 >= min_samples，含自身）
|
||
neighbors: list[list[int]] = [[] for _ in range(n)]
|
||
for i in range(n):
|
||
neighbors[i].append(i)
|
||
for j in range(i + 1, n):
|
||
if cosine_distance(vectors[i], vectors[j]) <= eps:
|
||
neighbors[i].append(j)
|
||
neighbors[j].append(i)
|
||
|
||
labels = [-1] * n
|
||
visited = [False] * n
|
||
cluster = 0
|
||
|
||
for i in range(n):
|
||
if visited[i]:
|
||
continue
|
||
visited[i] = True
|
||
if len(neighbors[i]) < min_samples:
|
||
labels[i] = -1 # 噪声
|
||
continue
|
||
# 核心点：扩展新簇
|
||
labels[i] = cluster
|
||
seeds = list(neighbors[i])
|
||
for q in seeds:
|
||
if not visited[q]:
|
||
visited[q] = True
|
||
if len(neighbors[q]) >= min_samples:
|
||
seeds.extend(neighbors[q])
|
||
if labels[q] == -1:
|
||
labels[q] = cluster
|
||
cluster += 1
|
||
|
||
cluster_ids: dict[int, str] = {}
|
||
representative_vectors: dict[str, dict[str, float]] = {}
|
||
for c in range(cluster):
|
||
members = [i for i in range(n) if labels[i] == c]
|
||
cid = f"c-{uuid.uuid4().hex[:8]}"
|
||
cluster_ids[c] = cid
|
||
representative_vectors[cid] = _centroid([vectors[i] for i in members])
|
||
|
||
return ClusterResult(labels=labels, cluster_ids=cluster_ids, representative_vectors=representative_vectors)
|
||
|
||
|
||
def _centroid(vectors: list[dict[str, float]]) -> dict[str, float]:
|
||
if not vectors:
|
||
return {}
|
||
acc: dict[str, float] = {}
|
||
for vec in vectors:
|
||
for k, v in vec.items():
|
||
acc[k] = acc.get(k, 0.0) + v
|
||
n = len(vectors)
|
||
return {k: v / n for k, v in acc.items()}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 增量聚类引擎
|
||
# ---------------------------------------------------------------------------
|
||
class ClusterEngine:
|
||
"""日志聚类引擎：特征提取 + DBSCAN + 增量分配。"""
|
||
|
||
def __init__(self, eps: float = 0.75, min_samples: int = 5, use_sklearn: bool = False):
|
||
self.eps = eps
|
||
self.min_samples = min_samples
|
||
self.use_sklearn = use_sklearn
|
||
# 已存在簇的代表向量：cluster_id -> 向量
|
||
self._representatives: dict[str, dict[str, float]] = {}
|
||
|
||
def _vectors(self, logs: list[FaultLog]) -> list[dict[str, float]]:
|
||
docs = [tokenize(templatize(log.message)) for log in logs]
|
||
if self.use_sklearn:
|
||
return self._sklearn_vectors(docs)
|
||
vectorizer = TfidfVectorizer().fit(docs)
|
||
return vectorizer.transform(docs)
|
||
|
||
@staticmethod
|
||
def _sklearn_vectors(docs: list[list[str]]) -> list[dict[str, float]]:
|
||
try:
|
||
from sklearn.feature_extraction.text import TfidfVectorizer as SkTfidf
|
||
except ImportError as exc: # pragma: no cover
|
||
raise RuntimeError("use_sklearn=True 需要安装 scikit-learn") from exc
|
||
joined = [" ".join(doc) for doc in docs]
|
||
if not any(joined):
|
||
return [{} for _ in docs]
|
||
m = SkTfidfVectorizer().fit_transform(joined)
|
||
rows = m.toarray()
|
||
return [{str(i): float(v) for i, v in enumerate(row) if v != 0.0} for row in rows]
|
||
|
||
def cluster_batch(self, logs: list[FaultLog]) -> dict[str, str]:
|
||
"""对一批日志聚类，返回 {fault_log_id: cluster_id}。
|
||
|
||
优先增量匹配已有簇，剩余未匹配日志再做一次 DBSCAN 形成新簇。
|
||
"""
|
||
assignment: dict[str, str] = {}
|
||
if not logs:
|
||
return assignment
|
||
|
||
vectors = self._vectors(logs)
|
||
unresolved_idx: list[int] = []
|
||
|
||
for i, log in enumerate(logs):
|
||
matched = self._match_representative(vectors[i])
|
||
if matched is not None:
|
||
assignment[log.fault_log_id] = matched
|
||
else:
|
||
unresolved_idx.append(i)
|
||
|
||
if unresolved_idx:
|
||
sub_vectors = [vectors[i] for i in unresolved_idx]
|
||
result = dbscan(sub_vectors, self.eps, self.min_samples)
|
||
for local_idx, global_idx in enumerate(unresolved_idx):
|
||
label = result.labels[local_idx]
|
||
if label == -1:
|
||
# 噪声日志独立成单点簇
|
||
cid = f"c-{uuid.uuid4().hex[:8]}"
|
||
self._representatives[cid] = sub_vectors[local_idx]
|
||
assignment[logs[global_idx].fault_log_id] = cid
|
||
else:
|
||
cid = result.cluster_ids[label]
|
||
assignment[logs[global_idx].fault_log_id] = cid
|
||
# 更新新簇代表向量
|
||
for cid, vec in result.representative_vectors.items():
|
||
self._representatives[cid] = vec
|
||
|
||
return assignment
|
||
|
||
def _match_representative(self, vector: dict[str, float]) -> Optional[str]:
|
||
if not vector or not self._representatives:
|
||
return None
|
||
best_cid: Optional[str] = None
|
||
best_sim = 1.0 - self.eps # 相似度阈值
|
||
for cid, rep in self._representatives.items():
|
||
sim = cosine_similarity(vector, rep)
|
||
if sim > best_sim:
|
||
best_sim = sim
|
||
best_cid = cid
|
||
return best_cid
|