108 lines
3.9 KiB
Python
108 lines
3.9 KiB
Python
"""
|
|
数据库操作: 保存热点条目、写抓取日志、统计查询
|
|
"""
|
|
from datetime import datetime
|
|
from appPublic.uniqueID import getID
|
|
from sqlor.dbpools import DBPools
|
|
|
|
def now_str():
|
|
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
|
|
|
async def get_enabled_sources():
|
|
"""获取所有启用的来源"""
|
|
db = DBPools()
|
|
async with db.sqlorContext('sage') as sor:
|
|
sources = await sor.R('hotspot_source', {})
|
|
return [s for s in sources if s.get('enabled') == '1']
|
|
|
|
async def save_items(items, source_id, fetch_time=None):
|
|
"""批量保存热点条目 (去重)"""
|
|
if fetch_time is None:
|
|
fetch_time = now_str()
|
|
new_count = 0
|
|
dup_count = 0
|
|
db = DBPools()
|
|
|
|
async with db.sqlorContext('sage') as sor:
|
|
for item in items:
|
|
url = item.get('url', '')
|
|
if url:
|
|
existing = await sor.sqlExe(
|
|
"select id from hotspot_item where url=${u}$ limit 1", {'u': url})
|
|
if existing:
|
|
dup_count += 1
|
|
continue
|
|
|
|
await sor.C('hotspot_item', {
|
|
'id': getID(), 'source_id': source_id,
|
|
'title': item.get('title', '')[:500],
|
|
'url': (url or '')[:2000],
|
|
'summary': item.get('summary', '')[:1000],
|
|
'heat_score': item.get('heat_score', 100),
|
|
'heat_velocity': 0,
|
|
'publish_time': fetch_time,
|
|
'first_seen': fetch_time,
|
|
'last_updated': fetch_time,
|
|
'status': 'emerging',
|
|
'category': item.get('category', ''),
|
|
'tags': item.get('tags', ''),
|
|
'engagement_count': item.get('engagement_count', 0),
|
|
'comment_count': item.get('comment_count', 0),
|
|
'share_count': item.get('share_count', 0),
|
|
'sentiment': 'neutral',
|
|
})
|
|
new_count += 1
|
|
|
|
return new_count, dup_count
|
|
|
|
async def update_source_last_fetch(source_id):
|
|
db = DBPools()
|
|
async with db.sqlorContext('sage') as sor:
|
|
await sor.U('hotspot_source', {'id': source_id, 'last_fetch': now_str()})
|
|
|
|
async def write_fetch_log(source_id, start_time, elapsed_ms,
|
|
status, items_total, items_new, items_duplicate,
|
|
error_msg='', response_code=0, response_size_bytes=0):
|
|
db = DBPools()
|
|
async with db.sqlorContext('sage') as sor:
|
|
await sor.C('hotspot_fetch_log', {
|
|
'id': getID(),
|
|
'source_id': source_id,
|
|
'start_time': start_time,
|
|
'end_time': now_str(),
|
|
'duration_ms': elapsed_ms,
|
|
'status': status,
|
|
'items_total': items_total,
|
|
'items_new': items_new,
|
|
'items_duplicate': items_duplicate,
|
|
'error_msg': error_msg[:1000] if error_msg else '',
|
|
'response_code': response_code,
|
|
'response_size_bytes': response_size_bytes,
|
|
})
|
|
|
|
async def get_stats():
|
|
"""全维度统计"""
|
|
from datetime import timedelta
|
|
now = datetime.now()
|
|
day_ago = (now - timedelta(hours=24)).strftime('%Y-%m-%d %H:%M:%S')
|
|
|
|
db = DBPools()
|
|
async with db.sqlorContext('sage') as sor:
|
|
items = await sor.R('hotspot_item', {'order': 'heat_score desc'})
|
|
sources = await sor.R('hotspot_source', {})
|
|
logs = await sor.sqlExe(
|
|
"select * from hotspot_fetch_log where start_time >= ${t}$",
|
|
{'t': day_ago})
|
|
|
|
status_count = {'emerging': 0, 'rising': 0, 'hot': 0, 'cooling': 0, 'expired': 0}
|
|
for item in items:
|
|
s = item.get('status', 'emerging')
|
|
status_count[s] = status_count.get(s, 0) + 1
|
|
|
|
return {
|
|
'total': len(items),
|
|
**status_count,
|
|
'sources': sum(1 for s in sources if s.get('enabled') == '1'),
|
|
'failures': sum(1 for log in logs if log.get('status') == 'failed'),
|
|
}
|