diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..9cf22fdb Binary files /dev/null and b/.DS_Store differ diff --git a/hotspot/__init__.py b/hotspot/__init__.py deleted file mode 100644 index 8a0dfa42..00000000 --- a/hotspot/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -热点雷达 (Hotspot Radar) — Sage 框架模块 - -用法: - from hotspot.engine import run_fetch - from hotspot.analysis import run_analysis - from hotspot.db import get_stats -""" -from .engine import run_fetch -from .analysis import run_analysis -from .db import get_stats -from .parsers import BUILTIN as builtin_parsers - -__all__ = ['run_fetch', 'run_analysis', 'get_stats', 'builtin_parsers'] diff --git a/hotspot/analysis.py b/hotspot/analysis.py deleted file mode 100644 index aecae172..00000000 --- a/hotspot/analysis.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -五维分析引擎 + 自动状态分类 - -维度: time(时效) / heat(热度) / content(内容) / propagation(传播) / audience(受众) -状态: emerging → rising → hot → cooling → expired -""" -import json, math -from datetime import datetime, timedelta -from appPublic.uniqueID import getID -from sqlor.dbpools import DBPools - -def now_str(): - return datetime.now().strftime('%Y-%m-%d %H:%M:%S') - -DIM_NAMES = { - 'time': '时效维度', - 'heat': '热度指标', - 'content': '内容属性', - 'propagation': '传播路径', - 'audience': '受众画像', -} - -async def analyze_item(item): - """分析单条热点，返回 (new_status, dimensions_dict, detail)""" - now = datetime.now() - try: - first_seen = datetime.strptime(str(item.get('first_seen', '')), '%Y-%m-%d %H:%M:%S') - except Exception: - first_seen = now - - hours_alive = max(0.1, (now - first_seen).total_seconds() / 3600) - heat = float(item.get('heat_score', 0)) - engagement = int(item.get('engagement_count', 0)) - comments = int(item.get('comment_count', 0)) - shares = int(item.get('share_count', 0)) - - # ---- 五维评分 (0-100) ---- - dims = {} - - # 时效: 越新越高 - dims['time'] = 95 if hours_alive < 1 else 85 if hours_alive < 6 else \ - 70 if hours_alive < 24 else 50 if hours_alive < 72 else \ - 30 if hours_alive < 168 else 10 - - # 热度: 绝对值 - dims['heat'] = 95 if heat > 1e4 else 85 if heat > 5e3 else \ - 70 if heat > 1e3 else 55 if heat > 500 else \ - 35 if heat > 100 else 15 - - # 内容: 标题长度 + 摘要丰富度 + 标签 - title_len = len(item.get('title', '')) - summary_len = len(item.get('summary', '')) - dims['content'] = min(100, (20 if title_len > 15 else 10) + - (30 if summary_len > 100 else 15) + - (25 if item.get('tags') else 0) + - (25 if item.get('category') else 0)) - - # 传播: 互动总量 - total_eng = engagement + comments * 2 + shares * 3 - dims['propagation'] = 95 if total_eng > 1e4 else 80 if total_eng > 5e3 else \ - 60 if total_eng > 1e3 else 35 if total_eng > 100 else 10 - - # 受众: 互动率 - rate = total_eng / heat if heat > 0 else 0 - dims['audience'] = 90 if rate > 0.5 else 70 if rate > 0.2 else \ - 45 if rate > 0.05 else 25 if total_eng > 0 else 5 - - # ---- 状态分类 ---- - decay = math.exp(-hours_alive / 168) # 7天半衰期 - adjusted_heat = heat * decay - velocity = round(heat / hours_alive, 2) - - if hours_alive > 336: # >14天 - new_status = 'expired' - elif hours_alive > 168: # 7-14天 - new_status = 'cooling' - elif adjusted_heat > 5000: - new_status = 'hot' - elif adjusted_heat > 500: - new_status = 'rising' if velocity > 50 else 'emerging' - elif adjusted_heat > 100: - new_status = 'rising' if velocity > 100 else 'emerging' - else: - new_status = 'emerging' - - detail = { - 'hours_alive': round(hours_alive, 1), - 'adjusted_heat': round(adjusted_heat, 2), - 'heat_velocity': velocity, - 'total_engagement': total_eng, - 'decay_factor': round(decay, 4), - } - - return new_status, dims, detail - - -async def save_analysis(item_id, dims, detail, now=None): - """保存/更新五维分析记录""" - if now is None: - now = now_str() - db = DBPools() - - async with db.sqlorContext('sage') as sor: - for dim, score in dims.items(): - analysis_data = json.dumps({ - 'dimension': dim, - 'dimension_cn': DIM_NAMES.get(dim, dim), - 'score': score, - 'detail': detail, - }, ensure_ascii=False) - - old = await sor.sqlExe( - "select id from hotspot_analysis where item_id=${iid}$ and dimension=${dim}$", - {'iid': item_id, 'dim': dim}) - if old: - await sor.U('hotspot_analysis', { - 'id': old[0]['id'], 'score': score, - 'analysis_data': analysis_data, 'analyzed_at': now, - }) - else: - await sor.C('hotspot_analysis', { - 'id': getID(), 'item_id': item_id, - 'dimension': dim, 'score': score, - 'analysis_data': analysis_data, 'analyzed_at': now, - }) - - -async def run_analysis(limit=500): - """批量分析最近的热点 (默认500条)""" - db = DBPools() - now = now_str() - results = {'analyzed': 0, 'status_changes': 0} - - async with db.sqlorContext('sage') as sor: - items = await sor.sqlExe( - "select * from hotspot_item order by first_seen desc limit ${n}$", - {'n': limit}) - - for item in items: - current_status = item.get('status', 'emerging') - new_status, dims, detail = await analyze_item(item) - - await save_analysis(item['id'], dims, detail, now) - - if new_status != current_status: - results['status_changes'] += 1 - - async with db.sqlorContext('sage') as sor: - await sor.U('hotspot_item', { - 'id': item['id'], - 'heat_score': round(detail['adjusted_heat'], 2), - 'heat_velocity': detail['heat_velocity'], - 'status': new_status, - 'last_updated': now, - }) - - results['analyzed'] += 1 - - return results diff --git a/hotspot/db.py b/hotspot/db.py deleted file mode 100644 index 24fd6355..00000000 --- a/hotspot/db.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -数据库操作: 保存热点条目、写抓取日志、统计查询 -""" -from datetime import datetime -from appPublic.uniqueID import getID -from sqlor.dbpools import DBPools - -def now_str(): - return datetime.now().strftime('%Y-%m-%d %H:%M:%S') - -async def get_enabled_sources(): - """获取所有启用的来源""" - db = DBPools() - async with db.sqlorContext('sage') as sor: - sources = await sor.R('hotspot_source', {}) - return [s for s in sources if s.get('enabled') == '1'] - -async def save_items(items, source_id, fetch_time=None): - """批量保存热点条目 (去重)""" - if fetch_time is None: - fetch_time = now_str() - new_count = 0 - dup_count = 0 - db = DBPools() - - async with db.sqlorContext('sage') as sor: - for item in items: - url = item.get('url', '') - if url: - existing = await sor.sqlExe( - "select id from hotspot_item where url=${u}$ limit 1", {'u': url}) - if existing: - dup_count += 1 - continue - - await sor.C('hotspot_item', { - 'id': getID(), 'source_id': source_id, - 'title': item.get('title', '')[:500], - 'url': (url or '')[:2000], - 'summary': item.get('summary', '')[:1000], - 'heat_score': item.get('heat_score', 100), - 'heat_velocity': 0, - 'publish_time': fetch_time, - 'first_seen': fetch_time, - 'last_updated': fetch_time, - 'status': 'emerging', - 'category': item.get('category', ''), - 'tags': item.get('tags', ''), - 'engagement_count': item.get('engagement_count', 0), - 'comment_count': item.get('comment_count', 0), - 'share_count': item.get('share_count', 0), - 'sentiment': 'neutral', - }) - new_count += 1 - - return new_count, dup_count - -async def update_source_last_fetch(source_id): - db = DBPools() - async with db.sqlorContext('sage') as sor: - await sor.U('hotspot_source', {'id': source_id, 'last_fetch': now_str()}) - -async def write_fetch_log(source_id, start_time, elapsed_ms, - status, items_total, items_new, items_duplicate, - error_msg='', response_code=0, response_size_bytes=0): - db = DBPools() - async with db.sqlorContext('sage') as sor: - await sor.C('hotspot_fetch_log', { - 'id': getID(), - 'source_id': source_id, - 'start_time': start_time, - 'end_time': now_str(), - 'duration_ms': elapsed_ms, - 'status': status, - 'items_total': items_total, - 'items_new': items_new, - 'items_duplicate': items_duplicate, - 'error_msg': error_msg[:1000] if error_msg else '', - 'response_code': response_code, - 'response_size_bytes': response_size_bytes, - }) - -async def get_stats(): - """全维度统计""" - from datetime import timedelta - now = datetime.now() - day_ago = (now - timedelta(hours=24)).strftime('%Y-%m-%d %H:%M:%S') - - db = DBPools() - async with db.sqlorContext('sage') as sor: - items = await sor.R('hotspot_item', {'order': 'heat_score desc'}) - sources = await sor.R('hotspot_source', {}) - logs = await sor.sqlExe( - "select * from hotspot_fetch_log where start_time >= ${t}$", - {'t': day_ago}) - - status_count = {'emerging': 0, 'rising': 0, 'hot': 0, 'cooling': 0, 'expired': 0} - for item in items: - s = item.get('status', 'emerging') - status_count[s] = status_count.get(s, 0) + 1 - - return { - 'total': len(items), - **status_count, - 'sources': sum(1 for s in sources if s.get('enabled') == '1'), - 'failures': sum(1 for log in logs if log.get('status') == 'failed'), - } diff --git a/hotspot/engine.py b/hotspot/engine.py deleted file mode 100644 index bdd7d1c0..00000000 --- a/hotspot/engine.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -抓取引擎 — 调度所有来源，fetch + extract + save -""" -import json, time -import aiohttp -from datetime import datetime - -from .parsers import (BUILTIN, extract_json_path, extract_css, - extract_regex, extract_rss, fetch_hn) -from .db import (save_items, update_source_last_fetch, - write_fetch_log, get_enabled_sources) - -def now_str(): - return datetime.now().strftime('%Y-%m-%d %H:%M:%S') - -async def fetch_and_extract(session, src): - """根据 source 的 parser_config 抓取并提取条目""" - url = src.get('url', '') - headers = {} - if src.get('headers'): - try: - headers = json.loads(src.get('headers', '{}')) - except Exception: - pass - - parser_config = {} - if src.get('parser_config'): - try: - parser_config = json.loads(src.get('parser_config', '{}')) - except Exception: - pass - - parser_type = parser_config.get('parser', 'json_path') - builtin_name = parser_config.get('builtin_name', '') - - # === 内置解析器 === - if parser_type == 'builtin' and builtin_name in BUILTIN: - bp = BUILTIN[builtin_name] - fetch_url = bp['url'] or url - fetch_headers = {**headers, **bp.get('headers', {})} - - if builtin_name == 'hackernews': - items = await fetch_hn(session) - return items, 200, 0 - - if builtin_name == 'rsshub': - try: - async with session.get(url, headers=fetch_headers, - timeout=aiohttp.ClientTimeout(total=30)) as resp: - text = await resp.text() - items = extract_rss(text) - return items, resp.status, len(text.encode()) - except Exception: - return [], 0, 0 - - # 标准内置: GET → JSON → parser_fn - try: - async with session.get(fetch_url, headers=fetch_headers, - timeout=aiohttp.ClientTimeout(total=30)) as resp: - if resp.status != 200: - return [], resp.status, 0 - text = await resp.text() - size = len(text.encode()) - try: - data = json.loads(text) - except json.JSONDecodeError: - return [], resp.status, size - items = bp['parser'](data) - return items, resp.status, size - except Exception: - return [], 0, 0 - - # === 通用提取器 === - try: - async with session.get(url, headers=headers, - timeout=aiohttp.ClientTimeout(total=30)) as resp: - if resp.status != 200: - return [], resp.status, 0 - text = await resp.text() - size = len(text.encode()) - - if parser_type == 'json_path': - try: - data = json.loads(text) - except json.JSONDecodeError: - return [], 0, size - items = extract_json_path(data, parser_config) - return items, resp.status, size - - elif parser_type == 'css': - items = extract_css(text, parser_config) - return items, resp.status, size - - elif parser_type == 'regex': - items = extract_regex(text, parser_config) - return items, resp.status, size - - elif parser_type == 'rss': - items = extract_rss(text) - return items, resp.status, size - - return [], resp.status, size - - except Exception: - return [], 0, 0 - - -async def run_fetch(): - """遍历所有启用的来源，执行抓取""" - sources = await get_enabled_sources() - now = now_str() - results = {'total': len(sources), 'new_items': 0, 'errors': 0, 'details': []} - - async with aiohttp.ClientSession() as session: - for src in sources: - t0 = time.time() - try: - items, code, size = await fetch_and_extract(session, src) - except Exception as e: - await write_fetch_log(src['id'], now, int((time.time() - t0) * 1000), - 'failed', 0, 0, 0, str(e)[:1000]) - results['errors'] += 1 - continue - - elapsed = int((time.time() - t0) * 1000) - status = 'success' if code in (0, 200) else ('partial' if items else 'failed') - - new_count, dup_count = await save_items(items, src['id'], now) - await update_source_last_fetch(src['id']) - await write_fetch_log(src['id'], now, elapsed, status, - len(items), new_count, dup_count, - response_code=code, response_size_bytes=size) - - results['new_items'] += new_count - results['details'].append({ - 'source': src.get('name', src['id']), - 'items': len(items), 'new': new_count, 'dup': dup_count, - 'elapsed_ms': elapsed, 'code': code, - }) - - return results diff --git a/hotspot/parsers.py b/hotspot/parsers.py deleted file mode 100644 index c746a549..00000000 --- a/hotspot/parsers.py +++ /dev/null @@ -1,297 +0,0 @@ -""" -内置平台解析器 — 每个返回 (fetch_url, headers_override, extract_fn) - -添加新平台: 在模块底部调用 _reg() 注册即可。 -""" -import json -import re -import xml.etree.ElementTree as ET -import aiohttp - -BUILTIN = {} - -def _reg(name, url, parser_fn, headers=None): - BUILTIN[name] = {'url': url, 'parser': parser_fn, 'headers': headers or {}} - -# ---- 国内平台 ---- - -_reg('weibo_hot', - 'https://weibo.com/ajax/side/hotSearch', - lambda data: [ - {'title': item.get('word', ''), - 'url': f'https://s.weibo.com/weibo?q={item.get("word","")}', - 'heat_score': item.get('raw_hot', item.get('num', 0))} - for item in data.get('data', {}).get('realtime', [])[:50] - if isinstance(item, dict) and item.get('word') - ], - {'User-Agent': 'Mozilla/5.0', 'X-Requested-With': 'XMLHttpRequest'}) - -_reg('zhihu_hot', - 'https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50', - lambda data: [ - {'title': item.get('target', {}).get('title', ''), - 'url': f'https://www.zhihu.com/question/{item.get("target",{}).get("id","")}', - 'heat_score': int(str(item.get('detail_text', '0')) - .replace('万','0000').replace('亿','00000000') - .replace('热度','').strip() or 0), - 'summary': (item.get('target', {}).get('excerpt', '') or '')[:500]} - for item in data.get('data', []) if isinstance(item, dict) - ], - {'User-Agent': 'Mozilla/5.0'}) - -_reg('baidu_hot', - 'https://top.baidu.com/board?tab=realtime', - lambda data: [ - {'title': item.get('word', ''), - 'url': item.get('url', ''), - 'heat_score': int(item.get('hotScore', 0)), - 'summary': item.get('desc', '')} - for item in (data.get('data', {}).get('cards', [{}])[0].get('content', [])) - if isinstance(item, dict) - ], - {'User-Agent': 'Mozilla/5.0'}) - -_reg('toutiao_hot', - 'https://www.toutiao.com/hot-event/hot-board/?origin=toutiao_pc', - lambda data: [ - {'title': item.get('Title', ''), - 'url': item.get('Url', ''), - 'heat_score': int(float(item.get('HotValue', 0)))} - for item in (data.get('data', []) if isinstance(data.get('data'), list) else [])[:50] - if isinstance(item, dict) and item.get('Title') - ], - {'User-Agent': 'Mozilla/5.0', 'Referer': 'https://www.toutiao.com/'}) - -_reg('bilibili_hot', - 'https://api.bilibili.com/x/web-interface/popular?ps=50', - lambda data: [ - {'title': item.get('title', ''), - 'url': f'https://www.bilibili.com/video/{item.get("bvid","")}', - 'heat_score': item.get('stat', {}).get('view', 0), - 'summary': item.get('desc', '')[:500], - 'engagement_count': item.get('stat', {}).get('like', 0), - 'comment_count': item.get('stat', {}).get('reply', 0), - 'share_count': item.get('stat', {}).get('share', 0), - 'category': item.get('tname', '')} - for item in data.get('data', {}).get('list', []) - if isinstance(item, dict) and item.get('title') - ], - {'User-Agent': 'Mozilla/5.0', 'Referer': 'https://www.bilibili.com/'}) - -_reg('douyin_hot', - 'https://dy.feigua.cn/api/hot/board', - lambda data: [ - {'title': item.get('title', item.get('word', '')), - 'url': item.get('url', ''), - 'heat_score': int(item.get('hot_value', item.get('heat', 0)))} - for item in (data.get('data', []) if isinstance(data.get('data'), list) else [])[:50] - if isinstance(item, dict) - ], - {'User-Agent': 'Mozilla/5.0'}) - -_reg('36kr_hot', - 'https://www.36kr.com/api/search/list/info-flow/hot?pageSize=30', - lambda data: [ - {'title': item.get('title', ''), - 'url': f'https://www.36kr.com/p/{item.get("id","")}', - 'heat_score': item.get('total_count', 0), - 'summary': (item.get('summary', '') or '')[:500]} - for item in data.get('data', {}).get('items', []) - if isinstance(item, dict) and item.get('title') - ], - {'User-Agent': 'Mozilla/5.0'}) - -# ---- 海外平台 ---- - -_reg('github_trending', - 'https://api.github.com/search/repositories?q=stars:>1&sort=stars&order=desc&per_page=20', - lambda data: [ - {'title': item.get('full_name', ''), - 'url': item.get('html_url', ''), - 'heat_score': item.get('stargazers_count', 0), - 'summary': (item.get('description', '') or '')[:500], - 'category': item.get('language', '')} - for item in data.get('items', []) if isinstance(item, dict) - ], - {'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'HotspotRadar/1.0'}) - -_reg('hackernews', - 'https://hacker-news.firebaseio.com/v0/topstories.json', - None, # 特殊: 两步抓取, 见 engine.py - {'User-Agent': 'Mozilla/5.0'}) - -_reg('v2ex_hot', - 'https://www.v2ex.com/api/topics/hot.json', - lambda data: [ - {'title': item.get('title', ''), - 'url': item.get('url', ''), - 'heat_score': item.get('replies', 0) * 10, - 'summary': (item.get('content', '') or '')[:500], - 'comment_count': item.get('replies', 0), - 'category': item.get('node', {}).get('title', '') - if isinstance(item.get('node'), dict) else ''} - for item in data if isinstance(item, dict) and item.get('title') - ], - {'User-Agent': 'Mozilla/5.0'}) - -_reg('rsshub', None, None, {}) # url 来自 source 配置 - -# ---- 通用提取器 ---- - -def extract_json_path(data, config): - """简易 JSONPath: $.data.list[*] → {title, url, heat_score, ...}""" - path = config.get('item_path', '$[*]') - fm = config.get('field_map', {}) - - if path.startswith('$.'): - path = path[2:] - elif path.startswith('$'): - path = path[1:] - - parts = [] - for p in path.split('.'): - p = p.strip() - if not p: continue - if p.endswith('[*]'): - parts.append(p[:-3]); parts.append('*') - else: - parts.append(p) - - items = data - for part in parts: - if part == '*': - continue - if isinstance(items, dict): - items = items.get(part) - elif isinstance(items, list): - items = [i.get(part) for i in items if isinstance(i, dict)] - items = [x for sub in items for x in (sub if isinstance(sub, list) else [sub])] - else: - return [] - if not isinstance(items, list): - items = [items] if items else [] - - result = [] - for item in items: - if not isinstance(item, dict): - continue - entry = { - 'title': _field(item, fm.get('title', 'title')), - 'url': _field(item, fm.get('url', 'url')), - 'heat_score': float(_field(item, fm.get('heat_score', '0')) or 0), - 'summary': str(_field(item, fm.get('summary', '')) or '')[:500], - 'category': str(_field(item, fm.get('category', '')) or ''), - 'tags': str(_field(item, fm.get('tags', '')) or ''), - 'engagement_count': int(_field(item, fm.get('engagement_count', '0')) or 0), - 'comment_count': int(_field(item, fm.get('comment_count', '0')) or 0), - 'share_count': int(_field(item, fm.get('share_count', '0')) or 0), - } - if entry['title']: - result.append(entry) - return result - -def _field(obj, path): - if not path or not isinstance(obj, dict): - return obj if not isinstance(obj, dict) else '' - val = obj - for p in path.split('.'): - if isinstance(val, dict): - val = val.get(p, '') - else: - return '' - return val - -def extract_css(html, config): - """CSS 选择器提取 (轻量正则版)""" - results = [] - for m in re.finditer( - r']+href=["\']([^"\']+)["\'][^>]*>([^<]*)', - html, re.IGNORECASE | re.DOTALL - ): - url, title = m.group(1), m.group(2).strip() - if len(title) >= 3: - results.append({ - 'title': title[:200], 'url': url[:2000], - 'heat_score': 60, 'summary': '', - 'category': '', 'tags': '', - 'engagement_count': 0, 'comment_count': 0, 'share_count': 0, - }) - return results[:100] - -def extract_regex(text, config): - pattern = config.get('item_regex', '') - fm = config.get('field_map', {}) - if not pattern: return [] - results = [] - for m in re.finditer(pattern, text, re.DOTALL): - gd = m.groupdict() - entry = { - 'title': gd.get(fm.get('title', 'title'), '')[:200], - 'url': gd.get(fm.get('url', 'url'), ''), - 'heat_score': float(gd.get(fm.get('heat_score', '0'), 0) or 0), - 'summary': str(gd.get(fm.get('summary', ''), ''))[:500], - 'category': '', 'tags': '', - 'engagement_count': 0, 'comment_count': 0, 'share_count': 0, - } - if entry['title']: - results.append(entry) - return results - -def extract_rss(xml_text): - items = [] - try: - root = ET.fromstring(xml_text) - ns_a = 'http://www.w3.org/2005/Atom' - for item in root.findall('.//item'): - items.append({ - 'title': item.findtext('title', '')[:200], - 'url': item.findtext('link', ''), - 'summary': (item.findtext('description', '') or '')[:500], - 'heat_score': 80, 'category': item.findtext('category', ''), - 'tags': '', 'engagement_count': 0, 'comment_count': 0, 'share_count': 0, - }) - if not items: - for e in root.findall(f'.//{{{ns_a}}}entry'): - link = e.find(f'{{{ns_a}}}link') - items.append({ - 'title': e.findtext(f'{{{ns_a}}}title', '')[:200], - 'url': link.get('href', '') if link is not None else '', - 'summary': (e.findtext(f'{{{ns_a}}}summary', '') or '')[:500], - 'heat_score': 70, 'category': '', - 'tags': '', 'engagement_count': 0, 'comment_count': 0, 'share_count': 0, - }) - except ET.ParseError: - pass - return items - -async def fetch_hn(session): - """HackerNews 两步抓取""" - items = [] - try: - async with session.get( - 'https://hacker-news.firebaseio.com/v0/topstories.json', - timeout=aiohttp.ClientTimeout(total=15) - ) as resp: - ids = await resp.json() - for sid in ids[:30]: - try: - async with session.get( - f'https://hacker-news.firebaseio.com/v0/item/{sid}.json', - timeout=aiohttp.ClientTimeout(total=10) - ) as r2: - s = await r2.json() - if s and s.get('title'): - items.append({ - 'title': s.get('title', '')[:200], - 'url': s.get('url', f'https://news.ycombinator.com/item?id={sid}'), - 'heat_score': s.get('score', 0), - 'engagement_count': s.get('score', 0), - 'comment_count': s.get('descendants', 0), - 'summary': '', 'category': s.get('type', ''), - 'tags': '', 'share_count': 0, - }) - except Exception: - continue - except Exception: - pass - return items diff --git a/init/data.xlsx b/init/data.xlsx new file mode 100644 index 00000000..82098619 Binary files /dev/null and b/init/data.xlsx differ diff --git a/json/hotspot_alert.json b/json/hotspot_alert.json deleted file mode 100644 index 13575a03..00000000 --- a/json/hotspot_alert.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "models_dir": "${HOME}$/py/sage/models", - "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_alert", - "dbname": "sage", - "tblname": "hotspot_alert", - "title": "预警规则", - "params": { - "sortby": "name", - "browserfields": { - "exclouded": ["id"], - "cwidth": {} - }, - "editexclouded": ["id", "last_triggered"], - "record_toolbar": null - } -} diff --git a/json/hotspot_analysis.json b/json/hotspot_analysis.json deleted file mode 100644 index 901ed07d..00000000 --- a/json/hotspot_analysis.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "models_dir": "${HOME}$/py/sage/models", - "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_analysis", - "dbname": "sage", - "tblname": "hotspot_analysis", - "title": "热点分析", - "params": { - "sortby": "analyzed_at desc", - "browserfields": { - "exclouded": ["id"], - "cwidth": {"analysis_data": 400} - }, - "editexclouded": ["id", "analyzed_at"], - "record_toolbar": null - } -} diff --git a/json/hotspot_fetch_log.json b/json/hotspot_fetch_log.json deleted file mode 100644 index 8a92fb58..00000000 --- a/json/hotspot_fetch_log.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "models_dir": "${HOME}$/py/sage/models", - "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_fetch_log", - "dbname": "sage", - "tblname": "hotspot_fetch_log", - "title": "抓取日志", - "params": { - "sortby": "start_time desc", - "browserfields": { - "exclouded": ["id", "error_msg"], - "cwidth": {"error_msg": 400} - }, - "editexclouded": ["id"], - "record_toolbar": null - } -} diff --git a/json/hotspot_item.json b/json/hotspot_item.json deleted file mode 100644 index 5a584ba4..00000000 --- a/json/hotspot_item.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "models_dir": "${HOME}$/py/sage/models", - "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_item", - "dbname": "sage", - "tblname": "hotspot_item", - "title": "热点条目", - "params": { - "sortby": "heat_score desc", - "browserfields": { - "exclouded": ["id", "content"], - "cwidth": {"title": 300, "summary": 400, "url": 200} - }, - "editexclouded": ["id", "first_seen", "last_updated"], - "record_toolbar": null - } -} diff --git a/json/hotspot_schedule.json b/json/hotspot_schedule.json deleted file mode 100644 index 30722963..00000000 --- a/json/hotspot_schedule.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "models_dir": "${HOME}$/py/sage/models", - "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_schedule", - "dbname": "sage", - "tblname": "hotspot_schedule", - "title": "调度配置", - "params": { - "sortby": "source_id", - "browserfields": { - "exclouded": ["id"], - "cwidth": {} - }, - "editexclouded": ["id", "last_run", "next_run"], - "record_toolbar": null - } -} diff --git a/json/hotspot_source.json b/json/hotspot_source.json deleted file mode 100644 index ab994b36..00000000 --- a/json/hotspot_source.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "models_dir": "${HOME}$/py/sage/models", - "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_source", - "dbname": "sage", - "tblname": "hotspot_source", - "title": "热点来源", - "params": { - "sortby": "priority asc", - "browserfields": { - "exclouded": ["id", "api_secret", "headers", "request_body", "crawler_rules", "parser_config"], - "cwidth": {"name": 150, "url": 300} - }, - "editexclouded": ["id", "created_at", "last_fetch"], - "record_toolbar": null - } -} diff --git a/skills/kboss-promote/SKILL.md b/skills/kboss-promote/SKILL.md new file mode 100644 index 00000000..26857e0a --- /dev/null +++ b/skills/kboss-promote/SKILL.md @@ -0,0 +1,13 @@ +--- +name: kboss产品推荐 +description: 根据用户输入需求，从kboss平台获取产品数据，交给大模型选择推荐产品 +--- +# kboss产品推荐 +使用kboss提供的获取全量产品清单，获取产品列表，根据客户输入的需求用大模型匹配5个推荐产品 + +## 获取kboss全量产品清单 +使用以下命令 +``` +script/get_kboss_products.sh +``` + diff --git a/skills/kboss-promote/scripts/get_kboss_products.sh b/skills/kboss-promote/scripts/get_kboss_products.sh new file mode 100644 index 00000000..1329677c --- /dev/null +++ b/skills/kboss-promote/scripts/get_kboss_products.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + + diff --git a/skills/media-convert/SKILL.md b/skills/media-convert/SKILL.md new file mode 100644 index 00000000..40834c67 --- /dev/null +++ b/skills/media-convert/SKILL.md @@ -0,0 +1,35 @@ +--- +name: media-convert +description: 媒体转换，支持视频分离出音频，音频分离人声和伴奏，音视频合并等操作 +--- +# media-convert +媒体转换，收集多种媒体之间的转换，如视频分离出音频，音频分离人声和伴奏，音视频合并 + +## 视频分离 +能将输入的视频文件转换为无音频的视频和无音频的视频 +### 输入 +输入是一个json +{ + "video_file":视频文件路径 +} +执行以下操作 +``` +scripts/videosplitor.sh +``` + +## 音频文件分离人声和伴奏 +本功能使用了Meta (Facebook) 开源的 Demucs，它是目前公认的音质保留最好的工具之一。 +安装： +```Bash +pip install demucs +``` +上述安装需要在有GPU的宿主机上，Demucs第一次运行时需要下载模型，需要有外网环境：https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/955717e8-8726e21a.th + +输入是一个json +{ + "audio_file": 音频文件路径 +} +分离命令： +```Bash +scripts/audiosplitor.sh +``` diff --git a/skills/media-convert/ktvmake.sh b/skills/media-convert/ktvmake.sh new file mode 100644 index 00000000..70b7acfd --- /dev/null +++ b/skills/media-convert/ktvmake.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +cd /tmp +cdir=$(pwd) +adir=${cdir}/${id}/separated/htdemucs/output_audio +id=$$ +ffmpeg -i $1 -c:v copy -an output_video${id}.mp4 -c:a pcm_s16le -f wav output_audio${id}.wav +mkdir ${id} +cd ${id} +demucs --two-stems=vocals ../output_audioi${id}.wav +cd ${cdir} +ffmpeg -i output_video${id}.mp4 -i ${adir}/no_vocals.wav -i ${adir}/vocals.wav \ + -map 0:v:0 \ + -map 1:a:0 \ + -map 2:a:0 \ + -c:v copy \ + -c:a aac -b:a 192k \ + -metadata:s:a:0 title="伴奏" \ + -metadata:s:a:1 title="原唱" \ + output_ktv.mkv -y +rm -rf ${id} output_video${id}.mp4 output_audio${id}.wav +echo ${cdir}/output_ktv.mkv diff --git a/skills/media-convert/scripts/audiosplitor.sh b/skills/media-convert/scripts/audiosplitor.sh new file mode 100644 index 00000000..90c775c1 --- /dev/null +++ b/skills/media-convert/scripts/audiosplitor.sh @@ -0,0 +1,4 @@ + + +demucs --two-stems=vocals $audio_file + diff --git a/skills/media-convert/scripts/videosplitor.sh b/skills/media-convert/scripts/videosplitor.sh new file mode 100644 index 00000000..d411c51d --- /dev/null +++ b/skills/media-convert/scripts/videosplitor.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +cdir=$(pwd) +read -r INPUT + +error() { + echo "{\"error\":\"$1\"}" + exit 0 +} + +video_file=$(echo "$INPUT" | jq -r '.video_file // empty') +[[ -z "$video_file" ]] && error "missing video_file" +[[ ! -f "$video_file" ]] && error "$video_file not exists" + +ffmpeg -i $video_file -c:v copy -an output_video$$.mp4 -c:a pcm_s16le -f wav output_audio$$.wav + +echo "{\"video_file\": \"$cdir/output_video$$.mp4\", \"audio_file\": \"$cdir/output_audio$$.wav\"}" diff --git a/skills/media-convert/videosplitor.sh b/skills/media-convert/videosplitor.sh new file mode 100644 index 00000000..e69de29b diff --git a/wwwroot/hotspot/analyze.dspy b/wwwroot/hotspot/analyze.dspy deleted file mode 100644 index 918270b5..00000000 --- a/wwwroot/hotspot/analyze.dspy +++ /dev/null @@ -1,2 +0,0 @@ -from hotspot.analysis import run_analysis -return await run_analysis() diff --git a/wwwroot/hotspot/fetch_now.dspy b/wwwroot/hotspot/fetch_now.dspy deleted file mode 100644 index 939d9f64..00000000 --- a/wwwroot/hotspot/fetch_now.dspy +++ /dev/null @@ -1,2 +0,0 @@ -from hotspot.engine import run_fetch -return await run_fetch() diff --git a/wwwroot/hotspot/index.ui b/wwwroot/hotspot/index.ui deleted file mode 100644 index d4d0c83f..00000000 --- a/wwwroot/hotspot/index.ui +++ /dev/null @@ -1,182 +0,0 @@ -{% set roles = get_user_roles(get_user()) %} -{ - "widgettype": "VBox", - "options": { - "children": [ - { - "widgettype": "HBox", - "options": { - "children": [ - { - "widgettype": "VBox", - "options": { - "id": "stat_emerging", - "children": [ - {"widgettype": "Text", "options": {"text": "潜在热点", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#52c41a;"}} - ], - "style": "padding:14px;background:#f6ffed;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #b7eb8f;" - } - }, - { - "widgettype": "VBox", - "options": { - "id": "stat_rising", - "children": [ - {"widgettype": "Text", "options": {"text": "上升中", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#1890ff;"}} - ], - "style": "padding:14px;background:#e6f7ff;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #91d5ff;" - } - }, - { - "widgettype": "VBox", - "options": { - "id": "stat_hot", - "children": [ - {"widgettype": "Text", "options": {"text": "正热点", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#ff4d4f;"}} - ], - "style": "padding:14px;background:#fff1f0;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffa39e;" - } - }, - { - "widgettype": "VBox", - "options": { - "id": "stat_cooling", - "children": [ - {"widgettype": "Text", "options": {"text": "降温中", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#faad14;"}} - ], - "style": "padding:14px;background:#fffbe6;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffe58f;" - } - }, - { - "widgettype": "VBox", - "options": { - "id": "stat_expired", - "children": [ - {"widgettype": "Text", "options": {"text": "已过期", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#bfbfbf;"}} - ], - "style": "padding:14px;background:#fafafa;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #d9d9d9;" - } - }, - { - "widgettype": "VBox", - "options": { - "id": "stat_sources", - "children": [ - {"widgettype": "Text", "options": {"text": "活跃来源", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#722ed1;"}} - ], - "style": "padding:14px;background:#f9f0ff;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #d3adf7;" - } - }, - { - "widgettype": "VBox", - "options": { - "id": "stat_failures", - "children": [ - {"widgettype": "Text", "options": {"text": "24h 抓取失败", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#ff7875;"}} - ], - "style": "padding:14px;background:#fff2f0;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffccc7;" - } - } - ] - } - }, - { - "widgettype": "HBox", - "options": { - "children": [ - { - "widgettype": "Button", - "options": { - "label": "立即抓取全部来源", - "actiontype": "script", - "script": "fetch('/hotspot/fetch_now.dspy').then(r=>r.json()).then(d=>{var m='完成: '+d.total+'来源, 新增'+d.new_items+'条';if(d.errors)m+='\\\\n失败: '+d.errors;alert(m);setTimeout(()=>location.reload(),1000);}).catch(e=>alert('异常: '+e))", - "style": "margin:6px;" - } - }, - { - "widgettype": "Button", - "options": { - "label": "执行分析", - "actiontype": "script", - "script": "fetch('/hotspot/analyze.dspy').then(r=>r.json()).then(d=>{alert('已分析: '+d.analyzed+'条热点');setTimeout(()=>location.reload(),1000);}).catch(e=>alert('分析异常: '+e))", - "style": "margin:6px;" - } - }, - { - "widgettype": "Button", - "options": { - "label": "刷新统计", - "actiontype": "script", - "script": "fetch('/hotspot/stats.dspy').then(r=>r.json()).then(d=>{['emerging','rising','hot','cooling','expired','sources','failures'].forEach(k=>{var el=document.querySelectorAll('[id=stat_'+k+'] div')[1];if(el)el.textContent=d[k]||0;});}).catch(e=>console.error('stats err',e))", - "style": "margin:6px;" - } - } - ] - } - }, - { - "widgettype": "Tab", - "options": { - "tabs": [ - { - "title": "来源管理", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_source') }}"} - } - }, - { - "title": "调度配置", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_schedule') }}"} - } - }, - { - "title": "抓取日志", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_fetch_log') }}"} - } - }, - { - "title": "热点条目", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_item') }}"} - } - }, - { - "title": "五维分析", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_analysis') }}"} - } - }, - { - "title": "预警规则", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_alert') }}"} - } - } - ] - } - }, - { - "widgettype": "Text", - "options": { - "text": "工作流: ① 添加来源 → parser_config选parser=builtin,builtin_name=weibo/zhihu/baidu/bilibili/... 或自定义json_path/css/regex → ② 调度配置 → ③ 抓取 → ④ 分析 → ⑤ 浏览 | 内置解析器: weibo_hot zhihu_hot baidu_hot toutiao_hot bilibili_hot douyin_hot 36kr_hot github_trending hackernews v2ex_hot rsshub", - "style": "font-size:12px;color:#bbb;padding:16px;border-top:1px solid #eee;margin-top:8px;" - } - } - ] - } -} diff --git a/wwwroot/hotspot/stats.dspy b/wwwroot/hotspot/stats.dspy deleted file mode 100644 index cd064e5e..00000000 --- a/wwwroot/hotspot/stats.dspy +++ /dev/null @@ -1,2 +0,0 @@ -from hotspot.db import get_stats -return await get_stats() diff --git a/wwwroot/imgs/ocai1.svg b/wwwroot/imgs/ocai1.svg new file mode 100644 index 00000000..a08000df --- /dev/null +++ b/wwwroot/imgs/ocai1.svg @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +