yumoqing
45cec8987e
- 汇总主题/分项条目整行点击(cursor:pointer)弹下级弹窗,撤「查看分项」按钮
- 明细每条来源写进说明文字(source_text),删除 source_button_widget 来源按钮
- 新 opp_item_detail.dspy: 爬虫平台条目全量字段+公告全文+附件列表
- 新 opp_attachment_dl.dspy: 服务端附件下载代理,归属校验防SSRF,
降级链 download_url→/api/attachments/{id}/download→原始url,失败如实报错
- opp_data_capability: tender_attachments/normalize_attachment 容错对接
爬虫平台附件子系统(/api/tenders/{id}/attachments,另一会话开发中,未就绪时降级提示)
- opp_common: crawler_get_raw 原始字节下载辅助; load_path 注册两个新端点
- README/prompt 同步新交互描述
316 lines
12 KiB
Python
316 lines
12 KiB
Python
"""商机产线公共层：DB 上下文、爬虫平台 HTTP 客户端、报告/审批状态、人工任务。
|
||
|
||
所有 opp_* capability 模块共用本模块，禁止各自重复实现。
|
||
|
||
爬虫平台接入：内网 HTTP（http_api.py），配置优先级
|
||
1. appbase params 表: tender_api_base / tender_api_token（系统级配置禁硬编码）
|
||
2. 兜底默认: http://192.168.16.2:9085（内网地址，仅默认值）
|
||
"""
|
||
|
||
import json
|
||
import logging
|
||
import re
|
||
import urllib.parse
|
||
import urllib.request
|
||
|
||
from sqlor.dbpools import DBPools
|
||
from appPublic.uniqueID import getID
|
||
|
||
DBNAME = "pipeline"
|
||
PIPELINE_ID = "opportunity_general"
|
||
|
||
logger = logging.getLogger("pipeline.opportunity")
|
||
|
||
# 爬虫平台默认接入点（内网网卡地址，公网不可达）；可被 params 表覆盖
|
||
DEFAULT_API_BASE = "http://192.168.16.2:9085"
|
||
|
||
# ── 数据来源登记表（code → 显示名称/来源主页）──
|
||
# 名称与主页对齐爬虫平台数据源注册表 tender_opp_build/sites.yaml（事实源），
|
||
# 众包/海外平台补齐。明细弹窗用名称展示、点击直跳来源 url（记录级 source_url
|
||
# 优先，缺失回退来源主页）。新增采集源时在此登记。
|
||
SOURCE_REGISTRY = {
|
||
"ccgp": ("中国政府采购网", "http://www.ccgp.gov.cn/"),
|
||
"ggzy": ("全国公共资源交易平台", "https://www.ggzy.gov.cn/"),
|
||
"cebpub": ("中国招标投标公共服务平台", "https://www.cebpubservice.com/"),
|
||
"zycg": ("中央国家机关政府采购中心", "https://www.zycg.gov.cn/"),
|
||
"weain": ("军队采购网", "http://www.weain.mil.cn/"),
|
||
"chinabidding": ("中国招标网", "https://www.chinabidding.com.cn/"),
|
||
"qianlima": ("千里马招标网", "https://www.qianlima.com/"),
|
||
"bidcenter": ("中国采购与招标网", "http://www.bidcenter.com.cn/"),
|
||
"jianyu360": ("剑鱼标讯", "https://www.jianyu360.cn/"),
|
||
"zbj": ("猪八戒", "https://www.zbj.com/"),
|
||
"epwk": ("一品威客", "https://www.epwk.com/"),
|
||
"freelancer": ("Freelancer.com", "https://www.freelancer.com/"),
|
||
"pph": ("PeoplePerHour", "https://www.peopleperhour.com/"),
|
||
}
|
||
|
||
|
||
def source_display(code):
|
||
"""来源 code → (显示名称, 来源主页 url)。未登记来源原样回显 code。"""
|
||
code = (code or "").strip()
|
||
if code in SOURCE_REGISTRY:
|
||
return SOURCE_REGISTRY[code]
|
||
return (code or "", "")
|
||
|
||
|
||
# 域名 → 来源 code（从 source_url 反推来源名称用；爬虫平台记录不带 source code，
|
||
# 只有 source_url，故按域名匹配。主域匹配：去 www. 后按注册表主页域名后缀比对）
|
||
def _registry_hosts():
|
||
hosts = {}
|
||
for code, (_name, home) in SOURCE_REGISTRY.items():
|
||
h = home.split("//", 1)[-1].split("/", 1)[0].lower()
|
||
h = h[4:] if h.startswith("www.") else h
|
||
hosts[h] = code
|
||
return hosts
|
||
|
||
|
||
def infer_source_code(source_url):
|
||
"""source_url → 来源 code（域名后缀匹配注册表主页）。匹配不到返回 ''。"""
|
||
url = (source_url or "").strip()
|
||
if not url or "//" not in url:
|
||
return ""
|
||
host = url.split("//", 1)[-1].split("/", 1)[0].split(":", 1)[0].lower()
|
||
host = host[4:] if host.startswith("www.") else host
|
||
for rh, code in _registry_hosts().items():
|
||
if host == rh or host.endswith("." + rh):
|
||
return code
|
||
return ""
|
||
|
||
|
||
def source_text(code, source_url=""):
|
||
"""来源 code/url → 说明文字用的来源名称（2026-09-08 用户定夺：
|
||
明细列表每项的来源写进说明文字，不再渲染来源按钮）。
|
||
|
||
解析顺序：记录 source code → source_url 域名反推；解析不出返回「未知来源」。
|
||
"""
|
||
code = (code or "").strip()
|
||
if not code:
|
||
code = infer_source_code(source_url)
|
||
name, _home = source_display(code)
|
||
return name or "未知来源"
|
||
|
||
# ── 报告状态 ──
|
||
RP_DRAFT = "draft" # 草稿（agent 编写中/待审阅）
|
||
RP_CONFIRMED = "confirmed" # 人工确认通过（门禁）
|
||
RP_APPROVAL_INITIATED = "approval_initiated" # 已发起研发审批
|
||
RP_APPROVED = "approved" # 审批通过
|
||
RP_REJECTED = "rejected" # 审批驳回
|
||
|
||
# ── 审批状态 ──
|
||
AP_INITIATED = "initiated"
|
||
AP_APPROVED = "approved"
|
||
AP_REJECTED = "rejected"
|
||
|
||
# ── 人工任务类型 ──
|
||
HT_REPORT_CONFIRM = "opp_report_confirm" # 报告人工确认门禁
|
||
HT_DEV_APPROVAL = "opp_dev_approval" # 研发审批人工决策
|
||
|
||
# 报告状态流转（合法迁移）
|
||
REPORT_TRANSITIONS = {
|
||
RP_DRAFT: (RP_CONFIRMED,),
|
||
RP_CONFIRMED: (RP_APPROVAL_INITIATED,),
|
||
RP_APPROVAL_INITIATED: (RP_APPROVED, RP_REJECTED),
|
||
RP_REJECTED: (RP_DRAFT,), # 驳回可改回草稿重来
|
||
}
|
||
|
||
|
||
def get_db():
|
||
"""取 DBPools（databases 为空时从 config 兜底注入，与引擎其他模块一致）。"""
|
||
db = DBPools()
|
||
if not db.databases:
|
||
from appPublic.jsonConfig import getConfig
|
||
config = getConfig()
|
||
if config and config.databases:
|
||
db.databases = config.databases
|
||
return db, DBNAME
|
||
|
||
|
||
def new_id():
|
||
return getID()
|
||
|
||
|
||
def rec_to_dict(rec):
|
||
"""sqlor 行对象 → dict（过滤方法名）。"""
|
||
if rec is None:
|
||
return {}
|
||
if isinstance(rec, dict):
|
||
return {k: v for k, v in rec.items() if not callable(v)}
|
||
d = {}
|
||
for k, v in vars(rec).items():
|
||
if k.startswith("_") or callable(v):
|
||
continue
|
||
d[k] = v
|
||
return d
|
||
|
||
|
||
def rows_to_dicts(recs, limit=200):
|
||
return [rec_to_dict(r) for r in (recs or [])[:limit]]
|
||
|
||
|
||
def json_loads(s, default=None):
|
||
if not s:
|
||
return default if default is not None else []
|
||
if isinstance(s, (list, dict)):
|
||
return s
|
||
try:
|
||
return json.loads(s)
|
||
except (json.JSONDecodeError, TypeError, ValueError):
|
||
return default if default is not None else []
|
||
|
||
|
||
async def get_param(sor, key, default=""):
|
||
"""读 appbase params 表配置 + 默认兜底（系统级配置禁硬编码）。"""
|
||
try:
|
||
from pipeline_service.workspace import get_param as _gp
|
||
v = await _gp(sor, key, "")
|
||
if v not in (None, ""):
|
||
return str(v)
|
||
except Exception:
|
||
pass
|
||
return default
|
||
|
||
|
||
async def get_crawler_config(sor):
|
||
"""爬虫平台接入配置 (base, token)。params 表优先，默认内网地址兜底。"""
|
||
base = await get_param(sor, "tender_api_base", DEFAULT_API_BASE)
|
||
token = await get_param(sor, "tender_api_token", "")
|
||
return (base or DEFAULT_API_BASE).rstrip("/"), token
|
||
|
||
|
||
async def crawler_get(sor, path, params=None, timeout=20):
|
||
"""调爬虫平台只读接口。返回 (ok, payload|错误信息)。"""
|
||
base, token = await get_crawler_config(sor)
|
||
qs = ""
|
||
if params:
|
||
clean = {k: v for k, v in params.items() if v not in (None, "")}
|
||
if clean:
|
||
qs = "?" + urllib.parse.urlencode(clean)
|
||
url = base + path + qs
|
||
headers = {"X-API-Token": token} if token else {}
|
||
try:
|
||
import httpx
|
||
except ImportError:
|
||
httpx = None
|
||
try:
|
||
if httpx is not None:
|
||
async with httpx.AsyncClient(timeout=timeout) as cli:
|
||
r = await cli.get(url, headers=headers)
|
||
if r.status_code != 200:
|
||
return False, "爬虫平台返回 %d: %s" % (r.status_code, r.text[:200])
|
||
return True, r.json()
|
||
# 无 httpx 时退回 urllib（同步阻塞，仅在缺依赖时用）
|
||
import asyncio
|
||
|
||
def _blocking():
|
||
req = urllib.request.Request(url, headers=headers)
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
return json.loads(resp.read().decode("utf-8"))
|
||
|
||
return True, await asyncio.get_event_loop().run_in_executor(None, _blocking)
|
||
except Exception as e:
|
||
logger.warning("crawler_get %s failed: %s", path, str(e)[:200])
|
||
return False, "爬虫平台调用失败: %s" % str(e)[:200]
|
||
|
||
|
||
async def crawler_get_raw(sor, url, timeout=60):
|
||
"""下载爬虫平台上的原始字节（附件代理下载用）。
|
||
|
||
url 为绝对地址（含平台下载端点）。返回 (ok, bytes, content_type, 文件名或错误)。
|
||
带 X-API-Token；文件名优先从 Content-Disposition 解析。
|
||
"""
|
||
base, token = await get_crawler_config(sor)
|
||
# 相对下载端点补全为绝对地址（download_url 可能是 /api/... 形态）
|
||
if url.startswith("/"):
|
||
url = base + url
|
||
headers = {"X-API-Token": token} if token else {}
|
||
try:
|
||
import httpx
|
||
except ImportError:
|
||
httpx = None
|
||
|
||
def _parse_disp(cd):
|
||
if not cd:
|
||
return ""
|
||
m = re.search(r"filename\*=UTF-8''([^;]+)", cd, re.I)
|
||
if m:
|
||
return urllib.parse.unquote(m.group(1)).strip()
|
||
m = re.search(r'filename="?([^";]+)"?', cd, re.I)
|
||
if m:
|
||
return m.group(1).strip()
|
||
return ""
|
||
|
||
try:
|
||
if httpx is not None:
|
||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as cli:
|
||
r = await cli.get(url, headers=headers)
|
||
if r.status_code != 200:
|
||
return False, b"", "", "爬虫平台返回 %d" % r.status_code
|
||
ct = r.headers.get("Content-Type", "")
|
||
fn = _parse_disp(r.headers.get("Content-Disposition", ""))
|
||
return True, r.content, ct, fn
|
||
import asyncio
|
||
|
||
def _blocking():
|
||
req = urllib.request.Request(url, headers=headers)
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
return (resp.read(), resp.headers.get("Content-Type", ""),
|
||
_parse_disp(resp.headers.get("Content-Disposition", "")))
|
||
|
||
data, ct, fn = await asyncio.get_event_loop().run_in_executor(None, _blocking)
|
||
return True, data, ct, fn
|
||
except Exception as e:
|
||
logger.warning("crawler_get_raw failed: %s", str(e)[:200])
|
||
return False, b"", "", "附件下载失败: %s" % str(e)[:200]
|
||
|
||
|
||
async def find_human_task(sor, project_id, task_type, status=None):
|
||
"""查项目下指定类型的人工任务（供门禁判断与幂等创建）。"""
|
||
sql = ("SELECT id, title, status, task_type FROM pipeline_human_tasks "
|
||
"WHERE project_id=${pid}$ AND task_type=${tt}$")
|
||
args = {"pid": project_id, "tt": task_type}
|
||
if status:
|
||
sql += " AND status=${st}$"
|
||
args["st"] = status
|
||
sql += " ORDER BY created_at DESC LIMIT 1"
|
||
recs = await sor.sqlExe(sql, args)
|
||
await sor.sqlExe("COMMIT", {})
|
||
return rec_to_dict(recs[0]) if recs else None
|
||
|
||
|
||
async def create_human_task(sor, project_id, task_type, title, description="",
|
||
assignee_role="owner.superuser", assignee_id=""):
|
||
"""创建项目级人工任务（确认/审批门禁）。幂等由调用方先 find 保证。
|
||
|
||
pipeline_human_tasks NOT NULL 列：id/task_id/step_name/version/task_type/status。
|
||
task_id 与步骤任务解耦（确认/审批是产线级门禁，非步骤产物）→ 填空串。
|
||
"""
|
||
hid = new_id()
|
||
await sor.C("pipeline_human_tasks", {
|
||
"id": hid,
|
||
"task_id": "",
|
||
"step_name": task_type,
|
||
"version": 1,
|
||
"task_type": task_type,
|
||
"assignee_role": assignee_role or "owner.superuser",
|
||
"assignee_id": assignee_id or "",
|
||
"status": "pending",
|
||
"project_id": project_id or "",
|
||
"title": (title or "")[:480],
|
||
"description": description or "",
|
||
})
|
||
await sor.sqlExe("COMMIT", {})
|
||
logger.info("opp create_human_task type=%s id=%s project=%s",
|
||
task_type, hid, project_id)
|
||
return hid
|
||
|
||
|
||
async def get_report(sor, report_id):
|
||
recs = await sor.sqlExe(
|
||
"SELECT * FROM opp_reports WHERE id=${i}$", {"i": report_id})
|
||
await sor.sqlExe("COMMIT", {})
|
||
return rec_to_dict(recs[0]) if recs else None
|
||
|
||
|
||
async def can_transition(cur_status, new_status):
|
||
return new_status in REPORT_TRANSITIONS.get(cur_status, ())
|