pipeline_core/pipeline_core/skill_loader.py

529 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""
pipeline-core: Skill Loader — 六级可插拔技能隔离
技能目录结构(每级可独立存在,同名高优先级覆盖低优先级):
skills/
global/ ← 通用技能core 内核,所有产线所有角色)
skill-name/SKILL.md
pipelines/ ← 产线技能
{pipeline_id}/
common/ ← 产线通用技能(该产线所有角色可见)
skill-name/SKILL.md
roles/ ← 角色技能(产线内角色专属)
{role}/
skill-name/SKILL.md
projects/ ← 项目/会话技能(临时、项目特定)
{project_id}/
skill-name/SKILL.md
orgs/{org_id}/ ← 组织技能
users/{user_id}/ ← 用户个人技能
加载优先级(低 → 高,同名后者覆盖前者):
global → org → pipeline(common) → project → role → user
"""
import logging
import os
import re
from dataclasses import dataclass, field
from typing import Dict, List, Optional
logger = logging.getLogger("pipeline.skill_loader")
# scope 优先级(数值越大优先级越高)
SCOPE_PRIORITY = {
"global": 0,
"project_common": 1,
"pipeline": 2,
"org": 3,
"project": 4,
"role": 5,
"user": 6,
}
SCOPE_TAG = {
"global": "通用",
"project_common": "项目通用",
"pipeline": "产线",
"org": "组织",
"project": "项目",
"role": "角色",
"user": "个人",
}
# ── 技能数据模型 ──
@dataclass
class Skill:
"""单个技能"""
name: str
path: str # SKILL.md 绝对路径
description: str = ""
content: str = ""
scope: str = "global" # global/org/pipeline/role/project/user
scope_id: str = "" # org_id / pipeline_id / pipeline_id:role / project_id / user_id
trigger_keywords: List[str] = field(default_factory=list)
version: str = "1.0.0"
capability: str = "" # 配套能力名(如 task_capability / communication
tools: List[str] = field(default_factory=list) # 配套工具名列表
essential: bool = False # 基础规范:所有角色目录层必显示(如 project-directory-spec
@classmethod
def from_file(cls, filepath: str, scope: str = "global",
scope_id: str = "") -> "Skill":
name = os.path.basename(os.path.dirname(filepath))
content = ""
description = ""
trigger_keywords = []
version = "1.0.0"
capability = ""
tools = []
essential = False
try:
with open(filepath, "r", encoding="utf-8") as f:
content = f.read()
except Exception as e:
logger.warning(f"Skill read failed: {filepath} err={e}")
return cls(name=name, path=filepath, scope=scope, scope_id=scope_id)
fm = _parse_frontmatter(content)
if fm:
description = fm.get("description", description)
trigger_keywords = fm.get("trigger_keywords", trigger_keywords)
version = fm.get("version", version)
scope = fm.get("scope", scope)
capability = fm.get("capability", "") or ""
tools = fm.get("tools", []) or []
essential = str(fm.get("essential", "")).lower() in ("true", "1", "yes")
if not description:
for line in content.split("\n"):
line = line.strip()
if line and not line.startswith("#") and not line.startswith("---"):
description = line[:200]
break
return cls(
name=name, path=filepath, description=description,
content=content, scope=scope, scope_id=scope_id,
trigger_keywords=trigger_keywords, version=version,
capability=capability, tools=tools, essential=essential,
)
def to_prompt_block(self) -> str:
scope_tag = SCOPE_TAG.get(self.scope, self.scope)
body = self.content
# 剥离 frontmatter--- 之间),只保留正文,避免 name/description 与目录层重复
if body.startswith("---"):
end = body.find("---", 3)
if end != -1:
body = body[end + 3:].strip()
# 配套能力/工具声明frontmatter 里 capability/tools剥离后这里显式注入让 LLM 知道工具在哪)
cap_line = ""
if self.capability or self.tools:
parts = []
if self.capability:
parts.append(f"配套能力: {self.capability}")
if self.tools:
parts.append(f"配套工具: {', '.join(self.tools)}")
cap_line = " | ".join(parts) + "\n"
return f"""## [{scope_tag}] {self.name}
{self.description}
{cap_line}{body}
"""
_LINKED_DIRS = ('references', 'scripts', 'templates', 'assets')
def list_linked_files(self) -> List[str]:
"""列出 skill 目录下 references/scripts/templates/assets 内的子文件(相对路径)。"""
base = os.path.dirname(self.path)
result = []
for sub in self._LINKED_DIRS:
d = os.path.join(base, sub)
if os.path.isdir(d):
for root, _, files in os.walk(d):
for f in files:
result.append(os.path.relpath(os.path.join(root, f), base))
return sorted(result)
def read_linked_file(self, rel_path: str) -> str:
"""读取 skill 目录下 references/scripts/templates/assets 内的子文件(相对路径)。"""
base = os.path.realpath(os.path.dirname(self.path))
full = os.path.realpath(os.path.join(base, rel_path or ''))
rel = os.path.relpath(full, base)
if rel == os.pardir or rel.startswith(os.pardir + os.sep) or os.path.isabs(rel):
return 'FAIL: 路径越界'
if not any(rel == sub or rel.startswith(sub + os.sep) for sub in self._LINKED_DIRS):
return f'FAIL: 只允许加载 references/scripts/templates/assets 下的文件(收到 {rel}'
if not os.path.isfile(full):
return f'FAIL: 文件不存在 {rel_path}'
try:
with open(full, 'r', encoding='utf-8') as f:
return f.read()
except Exception as e:
return f'ERROR: {str(e)[:300]}'
def _parse_frontmatter(content: str) -> Optional[dict]:
if not content.startswith("---"):
return None
end = content.find("---", 3)
if end == -1:
return None
fm_text = content[3:end].strip()
result = {}
for line in fm_text.split("\n"):
line = line.strip()
if ":" in line:
key, _, val = line.partition(":")
key = key.strip()
val = val.strip().strip('"').strip("'")
if val.startswith("[") and val.endswith("]"):
val = [v.strip().strip('"').strip("'")
for v in val[1:-1].split(",") if v.strip()]
result[key] = val
return result
# ── LLM 技能名归一化(目录层装饰容错)──
def normalize_skill_name(name: str) -> str:
"""清洗 LLM 传入技能名时携带的目录层装饰噪音:
'[产线] bid-workflow' / '**[产线] bid-workflow**: 总纲' / 'bid-workflow总纲''bid-workflow'
技能名只含字母/数字/连字符/下划线,其余皆是装饰。"""
n = (name or '').strip().replace('**', '').replace('`', '')
n = re.sub(r'^\s*(\[[^\]]*\]\s*)+', '', n) # scope 标签:[产线]/[通用]/[角色](可能多个)
n = re.split(r'[:]', n, 1)[0] # 冒号后的描述尾巴
n = re.split(r'\s+[—–-]\s+', n, 1)[0] # 破折号描述(两侧须有空白,不伤 bid-workflow 自带连字符)
n = re.sub(r'\s*[(][^)]*[)]\s*$', '', n) # 尾部括号注释
return n.strip()
def resolve_skill(merged: Dict[str, "Skill"], name: str) -> Optional["Skill"]:
"""按名称在 merged 中查技能,容忍 LLM 装饰。
匹配顺序:精确 → 清洗后精确 → 忽略大小写 → 子串唯一命中。"""
if not name:
return None
if name in merged:
return merged[name]
cleaned = normalize_skill_name(name)
if cleaned in merged:
return merged[cleaned]
lower_map = {k.lower(): k for k in merged}
if cleaned.lower() in lower_map:
return merged[lower_map[cleaned.lower()]]
if len(cleaned) >= 4:
hits = [k for k in merged if k in cleaned or cleaned in k]
if len(hits) == 1:
return merged[hits[0]]
return None
def format_skill_names(merged: Dict[str, "Skill"], limit: int = 60) -> str:
"""FAIL 提示用技能名列表:高优先级 scoperole/pipeline/project/org排前超出截断。"""
ordered = sorted(merged.values(), key=lambda s: (-SCOPE_PRIORITY.get(s.scope, 0), s.name))
names = [s.name for s in ordered[:limit]]
text = ", ".join(names)
if len(merged) > len(names):
text += f" …(还有 {len(merged) - len(names)} 个)"
return text or "(无可用技能)"
# ── 六级技能加载器 ──
class SkillLoader:
"""七级技能加载器global → project_common → pipeline → org → project → role → user"""
def __init__(self, base_dir: str = ""):
self.base_dir = base_dir
self._global: Dict[str, Skill] = {}
self._projects_common: Dict[str, Skill] = {} # 项目通用projects/,所有项目共享)
self._orgs: Dict[str, Dict[str, Skill]] = {} # {org_id: {name: Skill}}
self._pipelines: Dict[str, Dict[str, Skill]] = {} # {pipeline_id: {name: Skill}}
self._roles: Dict[str, Dict[str, Skill]] = {} # {pipeline_id:role: {name: Skill}}
self._projects: Dict[str, Dict[str, Skill]] = {} # {org_id:project_id: {name: Skill}} 项目私有(挂在机构下)
self._users: Dict[str, Dict[str, Skill]] = {} # {user_id: {name: Skill}}
if base_dir:
self.reload()
def set_base_dir(self, base_dir: str):
self.base_dir = base_dir
if base_dir:
self.reload()
# ── 目录扫描 ──
def reload(self):
self._global = {}
self._projects_common = {}
self._orgs = {}
self._pipelines = {}
self._roles = {}
self._projects = {}
self._users = {}
if not self.base_dir or not os.path.isdir(self.base_dir):
return
# 通用: skills/global/
global_dir = os.path.join(self.base_dir, "global")
if os.path.isdir(global_dir):
self._global = self._scan_scope_dir(global_dir, "global")
# 项目通用: skills/projects/(所有项目共享,不分 project_id
projects_dir = os.path.join(self.base_dir, "projects")
if os.path.isdir(projects_dir):
self._projects_common = self._scan_scope_dir(projects_dir, "project_common")
# 产线: skills/pipelines/{pipeline_id}/common/ + roles/{role}/
pipelines_dir = os.path.join(self.base_dir, "pipelines")
if os.path.isdir(pipelines_dir):
for pid in os.listdir(pipelines_dir):
pdir = os.path.join(pipelines_dir, pid)
if not os.path.isdir(pdir):
continue
# 产线通用技能
common_dir = os.path.join(pdir, "common")
if os.path.isdir(common_dir):
self._pipelines[pid] = self._scan_scope_dir(common_dir, "pipeline", pid)
# 角色技能
roles_dir = os.path.join(pdir, "roles")
if os.path.isdir(roles_dir):
for role in os.listdir(roles_dir):
rdir = os.path.join(roles_dir, role)
if os.path.isdir(rdir):
key = f"{pid}:{role}"
self._roles[key] = self._scan_scope_dir(rdir, "role", key)
# 组织 + 项目私有: skills/orgs/{org_id}/
# 第一层含 SKILL.md 的是 org 技能;否则是 {project_id} 项目私有子目录(挂在机构下)
orgs_dir = os.path.join(self.base_dir, "orgs")
if os.path.isdir(orgs_dir):
for org_id in os.listdir(orgs_dir):
org_path = os.path.join(orgs_dir, org_id)
if not os.path.isdir(org_path):
continue
self._orgs[org_id] = {}
for entry in os.listdir(org_path):
entry_path = os.path.join(org_path, entry)
if not os.path.isdir(entry_path):
continue
if os.path.isfile(os.path.join(entry_path, "SKILL.md")):
# org 技能(第一层直接是 skill 目录)
skill = Skill.from_file(os.path.join(entry_path, "SKILL.md"), "org", org_id)
self._orgs[org_id][skill.name] = skill
else:
# {project_id} 项目私有目录
key = f"{org_id}:{entry}"
self._projects[key] = self._scan_scope_dir(entry_path, "project", key)
# 用户: skills/users/{user_id}/
users_dir = os.path.join(self.base_dir, "users")
if os.path.isdir(users_dir):
for user_id in os.listdir(users_dir):
p = os.path.join(users_dir, user_id)
if os.path.isdir(p):
self._users[user_id] = self._scan_scope_dir(p, "user", user_id)
logger.debug(f"SkillLoader reloaded: global={len(self._global)}, "
f"projects_common={len(self._projects_common)}, "
f"orgs={sum(len(s) for s in self._orgs.values())}, "
f"pipelines={sum(len(s) for s in self._pipelines.values())}, "
f"roles={sum(len(s) for s in self._roles.values())}, "
f"projects={sum(len(s) for s in self._projects.values())}, "
f"users={sum(len(s) for s in self._users.values())}")
def _scan_scope_dir(self, scope_dir: str, scope: str,
scope_id: str = "") -> Dict[str, Skill]:
skills = {}
for entry in os.listdir(scope_dir):
entry_path = os.path.join(scope_dir, entry)
if not os.path.isdir(entry_path):
continue
skill_file = os.path.join(entry_path, "SKILL.md")
if os.path.isfile(skill_file):
skill = Skill.from_file(skill_file, scope, scope_id)
skills[skill.name] = skill
for sub in os.listdir(entry_path):
sub_path = os.path.join(entry_path, sub)
if os.path.isdir(sub_path):
sf = os.path.join(sub_path, "SKILL.md")
if os.path.isfile(sf):
skill = Skill.from_file(sf, scope, scope_id)
skills[skill.name] = skill
return skills
# ── 查询 ──
def get_merged(self, pipeline_id: str = "", role: str = "",
project_id: str = "", org_id: str = "",
user_id: str = "") -> Dict[str, Skill]:
"""按优先级合并七级技能。同名技能高优先级覆盖低优先级。
优先级低→高global → project_common → pipeline → org → project → role → user
org scope 缺省继承org_id='0' 是 ocai 组织(缺省机构),其机构技能是所有组织共享的
缺省内容其他组织org_id≠'0')先继承 ocai 的机构技能,再用自己 org scope 的同名技能覆盖
(很多组织有自己的开发规范)。"""
merged = dict(self._global)
# 项目通用projects/,所有项目共享)
merged.update(self._projects_common)
if pipeline_id and pipeline_id in self._pipelines:
merged.update(self._pipelines[pipeline_id])
# 缺省机构技能ocaiorg_id='0')的机构技能,所有组织缺省共享
if '0' in self._orgs:
merged.update(self._orgs['0'])
# 组织自有机构技能覆盖 ocai 缺省org_id='0' 已在上一步加载,跳过重复)
if org_id and org_id != '0' and org_id in self._orgs:
merged.update(self._orgs[org_id])
# 项目私有orgs/{org_id}/{project_id}/
if org_id and project_id:
proj_key = f"{org_id}:{project_id}"
if proj_key in self._projects:
merged.update(self._projects[proj_key])
if pipeline_id and role:
role_key = f"{pipeline_id}:{role}"
if role_key in self._roles:
merged.update(self._roles[role_key])
if user_id and user_id in self._users:
merged.update(self._users[user_id])
return merged
def get_skill_catalog(self, pipeline_id: str = "", role: str = "",
project_id: str = "", org_id: str = "",
user_id: str = "") -> List[tuple]:
"""返回可用技能目录 [(name, description)],供 LLM 语义检索(技能检索必须 LLM 做)。"""
merged = self.get_merged(pipeline_id, role, project_id, org_id, user_id)
return [(s.name, (s.description or "")[:150]) for s in sorted(
merged.values(), key=lambda s: (SCOPE_PRIORITY.get(s.scope, 9), s.name))]
def build_prompt_block_by_names(self, names: List[str],
pipeline_id: str = "", role: str = "",
project_id: str = "", org_id: str = "",
user_id: str = "") -> str:
"""按技能名构建目录层LLM 检索结果 → 目录层)。"""
merged = self.get_merged(pipeline_id, role, project_id, org_id, user_id)
skills = [merged[n] for n in names if n in merged]
if not skills:
return ""
blocks = ["## 可用技能\n"]
for s in skills:
blocks.append(f"- **[{SCOPE_TAG.get(s.scope, s.scope)}] {s.name}**: {s.description[:200]}")
blocks.append("")
return "\n".join(blocks)
def get_by_trigger(self, user_input: str, pipeline_id: str = "",
role: str = "", project_id: str = "",
org_id: str = "", user_id: str = "",
max_skills: int = 5) -> List[Skill]:
"""根据用户输入匹配相关技能trigger_keywords + 技能名拆分 + scope 加权)。
兼容 Hermes 技能格式:多数 Hermes 技能只有 trigger_conditions自然语言而无
trigger_keywords因此额外用技能名拆分匹配module-development-spec → module/development/spec
兜底,保证英文技能名也能被中文输入触达。
"""
all_skills = list(self.get_merged(pipeline_id, role, project_id, org_id, user_id).values())
scored = []
user_lower = user_input.lower()
for skill in all_skills:
score = 0
# 1. 显式 trigger_keywords中文/英文关键词)
for kw in skill.trigger_keywords:
if kw.lower() in user_lower:
score += 3
# 2. 技能名整体匹配
if skill.name.lower() in user_lower:
score += 2
# 3. 技能名拆分匹配module-development-spec → module/development/spec
for part in re.split(r'[-_]+', skill.name.lower()):
if len(part) >= 3 and part in user_lower:
score += 1
# 4. scope 优先级加权
score += SCOPE_PRIORITY.get(skill.scope, 0)
if score > 0:
scored.append((score, skill))
scored.sort(key=lambda x: x[0], reverse=True)
return [s[1] for s in scored[:max_skills]]
def get_skill_count(self, pipeline_id: str = "", role: str = "",
project_id: str = "", org_id: str = "",
user_id: str = "") -> int:
return len(self.get_merged(pipeline_id, role, project_id, org_id, user_id))
def list_by_scope(self) -> Dict[str, int]:
return {
"global": len(self._global),
"project_common": len(self._projects_common),
"orgs": sum(len(s) for s in self._orgs.values()),
"pipelines": sum(len(s) for s in self._pipelines.values()),
"roles": sum(len(s) for s in self._roles.values()),
"projects": sum(len(s) for s in self._projects.values()),
"users": sum(len(s) for s in self._users.values()),
}
# ── Prompt 构建 ──
def build_prompt_block(self, pipeline_id: str = "", role: str = "",
project_id: str = "", org_id: str = "",
user_id: str = "", user_input: Optional[str] = None,
max_skills: int = 5) -> str:
if user_input:
skills = self.get_by_trigger(user_input, pipeline_id, role,
project_id, org_id, user_id, max_skills)
if not skills:
# 匹配不到时兜底:按 scope 优先级 + 名字稳定排序,避免目录层为空
merged = self.get_merged(pipeline_id, role, project_id, org_id, user_id)
skills = sorted(merged.values(),
key=lambda s: (SCOPE_PRIORITY.get(s.scope, 9), s.name))
skills = skills[:max_skills]
else:
merged = self.get_merged(pipeline_id, role, project_id, org_id, user_id)
skills = sorted(merged.values(),
key=lambda s: (SCOPE_PRIORITY.get(s.scope, 9), s.name))
skills = skills[:max_skills]
if not skills:
return ""
blocks = ["## 可用技能\n"]
for s in skills:
blocks.append(f"- **[{SCOPE_TAG.get(s.scope, s.scope)}] {s.name}**: {s.description[:200]}")
blocks.append("")
return "\n".join(blocks)
def build_full_prompt_block(self, skill_names: List[str],
pipeline_id: str = "", role: str = "",
project_id: str = "", org_id: str = "",
user_id: str = "") -> str:
merged = self.get_merged(pipeline_id, role, project_id, org_id, user_id)
blocks = []
for name in skill_names:
if name in merged:
blocks.append(merged[name].to_prompt_block())
return "\n".join(blocks)
# ── 全局单例 ──
_loader: Optional[SkillLoader] = None
def get_skill_loader(base_dir: str = "") -> SkillLoader:
global _loader
if _loader is None:
_loader = SkillLoader(base_dir)
elif base_dir and base_dir != _loader.base_dir:
_loader.set_base_dir(base_dir)
return _loader