"""日志结构化解析与模板化。""" from __future__ import annotations import json import re from datetime import datetime from typing import Any, Optional from .models import LogEntry, from_iso, utcnow # 常见时间格式(顺序尝试) _TIMESTAMP_FORMATS = ( "%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y/%m/%d %H:%M:%S", "%b %d %H:%M:%S", "%b %d %Y %H:%M:%S", ) _IP_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b") _UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b") _NUM_RE = re.compile(r"\b\d+(?:\.\d+)?\b") _PATH_RE = re.compile(r"(?:/[A-Za-z0-9._-]+)+") _HEX_RE = re.compile(r"\b0x[0-9a-fA-F]+\b") # 常见 trace id 字段名 _TRACE_KEYS = ("trace_id", "traceId", "traceid", "x-request-id", "request_id", "requestId") # 常见主机字段名 _HOST_KEYS = ("host_id", "hostId", "host", "hostname", "instance") # 常见服务字段名 _SERVICE_KEYS = ("service", "service_name", "app", "component", "logger") def _find_value(fields: dict[str, Any], keys: tuple[str, ...]) -> Optional[str]: for key in keys: if key in fields and fields[key] is not None: return str(fields[key]) return None def parse_timestamp(value: Any) -> Optional[datetime]: """解析多种时间戳表达,失败返回 None。""" if value is None: return None if isinstance(value, (int, float)): # 秒/毫秒/纳秒时间戳 if value > 1e17: # 纳秒 value = value / 1e9 elif value > 1e14: # 毫秒 value = value / 1e3 try: return datetime.utcfromtimestamp(value) except (ValueError, OSError, OverflowError): return None if isinstance(value, datetime): return value text = str(value).strip() if not text: return None parsed = from_iso(text) if parsed is not None: return parsed for fmt in _TIMESTAMP_FORMATS: try: return datetime.strptime(text, fmt) except ValueError: continue return None def parse_log(raw: dict[str, Any]) -> LogEntry: """将一条原始日志解析为结构化 LogEntry。 支持两种形态: 1. 结构化 JSON 日志:message / level / timestamp / host_id / service / trace_id 等字段。 2. 半结构化文本:以 message 为主体,从 fields 提取主机/服务/链路信息。 """ fields = dict(raw.get("fields") or {}) message = str(raw.get("message", "") or raw.get("msg", "") or raw.get("log", "") or "") # 若 message 本身是 JSON 文本,尝试展开 if not fields and message.lstrip().startswith("{"): try: nested = json.loads(message) if isinstance(nested, dict): if not nested.get("message") and not nested.get("msg"): message = json.dumps(nested, ensure_ascii=False) else: message = str(nested.get("message") or nested.get("msg") or message) fields = {**nested, **fields} except (json.JSONDecodeError, TypeError): pass level = str(raw.get("level", "") or fields.get("level", "") or fields.get("severity", "") or "") if not level: # 从 message 常见前缀提取级别 m = re.match(r"^\s*(TRACE|DEBUG|INFO|WARN|WARNING|ERROR|FATAL|CRITICAL)\b", message, re.I) if m: level = m.group(1).upper() host_id = str(raw.get("host_id", "") or _find_value(fields, _HOST_KEYS) or "") service = str(raw.get("service", "") or _find_value(fields, _SERVICE_KEYS) or raw.get("source", "") or "") trace_id = str(raw.get("trace_id", "") or _find_value(fields, _TRACE_KEYS) or "") source = str(raw.get("source", "") or fields.get("source", "") or fields.get("file", "") or "") timestamp = parse_timestamp(raw.get("timestamp")) or parse_timestamp( raw.get("time") ) or parse_timestamp(fields.get("timestamp")) or parse_timestamp(fields.get("@timestamp")) or utcnow() return LogEntry( timestamp=timestamp, level=level.upper(), message=message, host_id=host_id, service=service, source=source, trace_id=trace_id, fields=fields, ) def templatize(message: str) -> str: """将消息中的变量(数字/IP/UUID/路径/十六进制)替换为占位符,用于聚类模板化。""" text = message text = _UUID_RE.sub("", text) text = _IP_RE.sub("", text) text = _HEX_RE.sub("", text) text = _NUM_RE.sub("", text) text = _PATH_RE.sub("", text) # 合并连续空白 text = re.sub(r"\s+", " ", text).strip() return text def tokenize(message: str) -> list[str]: """简单分词:按非字母数字下划线切分,过滤空串与过短 token。""" tokens = re.findall(r"[A-Za-z_][A-Za-z0-9_]{1,}|<[A-Z]+>", message) return [t for t in tokens if len(t) >= 2]