143 lines
4.9 KiB
Python
143 lines
4.9 KiB
Python
"""日志结构化解析与模板化。"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from datetime import datetime
|
||
from typing import Any, Optional
|
||
|
||
from .models import LogEntry, from_iso, utcnow
|
||
|
||
# 常见时间格式（顺序尝试）
|
||
_TIMESTAMP_FORMATS = (
|
||
"%Y-%m-%dT%H:%M:%S.%f%z",
|
||
"%Y-%m-%dT%H:%M:%S%z",
|
||
"%Y-%m-%d %H:%M:%S.%f",
|
||
"%Y-%m-%d %H:%M:%S",
|
||
"%Y/%m/%d %H:%M:%S",
|
||
"%b %d %H:%M:%S",
|
||
"%b %d %Y %H:%M:%S",
|
||
)
|
||
|
||
_IP_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
|
||
_UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
|
||
_NUM_RE = re.compile(r"\b\d+(?:\.\d+)?\b")
|
||
_PATH_RE = re.compile(r"(?:/[A-Za-z0-9._-]+)+")
|
||
_HEX_RE = re.compile(r"\b0x[0-9a-fA-F]+\b")
|
||
|
||
# 常见 trace id 字段名
|
||
_TRACE_KEYS = ("trace_id", "traceId", "traceid", "x-request-id", "request_id", "requestId")
|
||
|
||
# 常见主机字段名
|
||
_HOST_KEYS = ("host_id", "hostId", "host", "hostname", "instance")
|
||
|
||
# 常见服务字段名
|
||
_SERVICE_KEYS = ("service", "service_name", "app", "component", "logger")
|
||
|
||
|
||
def _find_value(fields: dict[str, Any], keys: tuple[str, ...]) -> Optional[str]:
|
||
for key in keys:
|
||
if key in fields and fields[key] is not None:
|
||
return str(fields[key])
|
||
return None
|
||
|
||
|
||
def parse_timestamp(value: Any) -> Optional[datetime]:
|
||
"""解析多种时间戳表达，失败返回 None。"""
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, (int, float)):
|
||
# 秒/毫秒/纳秒时间戳
|
||
if value > 1e17: # 纳秒
|
||
value = value / 1e9
|
||
elif value > 1e14: # 毫秒
|
||
value = value / 1e3
|
||
try:
|
||
return datetime.utcfromtimestamp(value)
|
||
except (ValueError, OSError, OverflowError):
|
||
return None
|
||
if isinstance(value, datetime):
|
||
return value
|
||
text = str(value).strip()
|
||
if not text:
|
||
return None
|
||
parsed = from_iso(text)
|
||
if parsed is not None:
|
||
return parsed
|
||
for fmt in _TIMESTAMP_FORMATS:
|
||
try:
|
||
return datetime.strptime(text, fmt)
|
||
except ValueError:
|
||
continue
|
||
return None
|
||
|
||
|
||
def parse_log(raw: dict[str, Any]) -> LogEntry:
|
||
"""将一条原始日志解析为结构化 LogEntry。
|
||
|
||
支持两种形态：
|
||
1. 结构化 JSON 日志：message / level / timestamp / host_id / service / trace_id 等字段。
|
||
2. 半结构化文本：以 message 为主体，从 fields 提取主机/服务/链路信息。
|
||
"""
|
||
fields = dict(raw.get("fields") or {})
|
||
message = str(raw.get("message", "") or raw.get("msg", "") or raw.get("log", "") or "")
|
||
|
||
# 若 message 本身是 JSON 文本，尝试展开
|
||
if not fields and message.lstrip().startswith("{"):
|
||
try:
|
||
nested = json.loads(message)
|
||
if isinstance(nested, dict):
|
||
if not nested.get("message") and not nested.get("msg"):
|
||
message = json.dumps(nested, ensure_ascii=False)
|
||
else:
|
||
message = str(nested.get("message") or nested.get("msg") or message)
|
||
fields = {**nested, **fields}
|
||
except (json.JSONDecodeError, TypeError):
|
||
pass
|
||
|
||
level = str(raw.get("level", "") or fields.get("level", "") or fields.get("severity", "") or "")
|
||
if not level:
|
||
# 从 message 常见前缀提取级别
|
||
m = re.match(r"^\s*(TRACE|DEBUG|INFO|WARN|WARNING|ERROR|FATAL|CRITICAL)\b", message, re.I)
|
||
if m:
|
||
level = m.group(1).upper()
|
||
|
||
host_id = str(raw.get("host_id", "") or _find_value(fields, _HOST_KEYS) or "")
|
||
service = str(raw.get("service", "") or _find_value(fields, _SERVICE_KEYS) or raw.get("source", "") or "")
|
||
trace_id = str(raw.get("trace_id", "") or _find_value(fields, _TRACE_KEYS) or "")
|
||
source = str(raw.get("source", "") or fields.get("source", "") or fields.get("file", "") or "")
|
||
|
||
timestamp = parse_timestamp(raw.get("timestamp")) or parse_timestamp(
|
||
raw.get("time")
|
||
) or parse_timestamp(fields.get("timestamp")) or parse_timestamp(fields.get("@timestamp")) or utcnow()
|
||
|
||
return LogEntry(
|
||
timestamp=timestamp,
|
||
level=level.upper(),
|
||
message=message,
|
||
host_id=host_id,
|
||
service=service,
|
||
source=source,
|
||
trace_id=trace_id,
|
||
fields=fields,
|
||
)
|
||
|
||
|
||
def templatize(message: str) -> str:
|
||
"""将消息中的变量（数字/IP/UUID/路径/十六进制）替换为占位符，用于聚类模板化。"""
|
||
text = message
|
||
text = _UUID_RE.sub("<UUID>", text)
|
||
text = _IP_RE.sub("<IP>", text)
|
||
text = _HEX_RE.sub("<HEX>", text)
|
||
text = _NUM_RE.sub("<NUM>", text)
|
||
text = _PATH_RE.sub("<PATH>", text)
|
||
# 合并连续空白
|
||
text = re.sub(r"\s+", " ", text).strip()
|
||
return text
|
||
|
||
|
||
def tokenize(message: str) -> list[str]:
|
||
"""简单分词：按非字母数字下划线切分，过滤空串与过短 token。"""
|
||
tokens = re.findall(r"[A-Za-z_][A-Za-z0-9_]{1,}|<[A-Z]+>", message)
|
||
return [t for t in tokens if len(t) >= 2]
|