yumoqing
a08bac91b6
关键修复：accounting/pricing/discount/unipay 的表和前端早已就位， 但 app/pipeline_app.py 从未 load 它们 —— 相关页面一点即 NameError 500。 - 入口补 load_pricing/accounting/discount/unipay + load_account_resource/storage_resource 顺序：pricing 先于 accounting 与资源模块(资源计费依赖 pricing_program_charging) - build.sh 四处列表 + 8b DDL 循环加 account_resource/storage_resource - wwwroot/index.ui 加「账务管理」菜单 11 子项(参考 sage menu.ui 的配置账务结构) 按 pipeline-app 实际存在的页面取项：sage 菜单里的 open_account.ui 本模块不存在，已剔除， 另补上 sage 未列但存在的 我的账户/余额/明细/账单/信用额度/币种/汇率/记账日志 - i18n 四语言补「账务管理」
281 lines
12 KiB
Bash
281 lines
12 KiB
Bash
#!/usr/bin/env bash
|
||
set -e
|
||
cdir=$(cd "$(dirname "$0")" && pwd)
|
||
cd "$cdir"
|
||
|
||
echo "=== Pipeline App Build ==="
|
||
|
||
# 1. Create venv
|
||
if [ ! -d py3 ]; then
|
||
python3 -m venv py3
|
||
fi
|
||
source py3/bin/activate
|
||
|
||
# 2. Install foundation packages (clone to pkgs)
|
||
mkdir -p pkgs
|
||
for m in apppublic sqlor ahserver bricks-for-python xls2ddl rbac appbase dapi; do
|
||
echo "install $m ..."
|
||
cd "$cdir/pkgs"
|
||
if [ ! -d "$m" ]; then
|
||
git clone https://git.opencomputing.cn/yumoqing/$m || echo "SKIP: $m clone failed"
|
||
fi
|
||
if [ -d "$m" ]; then
|
||
cd "$m"
|
||
"$cdir/py3/bin/pip" install . 2>&1 | tail -1
|
||
fi
|
||
cd "$cdir"
|
||
done
|
||
|
||
# 3. Build bricks frontend
|
||
cd "$cdir/pkgs"
|
||
if [ ! -d bricks ]; then
|
||
git clone https://git.opencomputing.cn/yumoqing/bricks || true
|
||
fi
|
||
if [ ! -f bricks/dist/bricks.js ]; then
|
||
mkdir -p bricks/dist
|
||
if [ -d bricks/bricks ]; then
|
||
cd bricks/bricks
|
||
bash build.sh 2>&1 | tail -3
|
||
cd "$cdir/pkgs"
|
||
fi
|
||
fi
|
||
# Link dist to wwwroot
|
||
rm -rf "$cdir/wwwroot/bricks" 2>/dev/null
|
||
ln -sf "$cdir/pkgs/bricks/dist" "$cdir/wwwroot/bricks"
|
||
echo "bricks: $(readlink "$cdir/wwwroot/bricks")"
|
||
|
||
# 4. Move local business modules into pkgs/
|
||
cd "$cdir"
|
||
for mod in pipeline_core pipeline_ops pipeline_dist; do
|
||
if [ -d "$mod" ]; then
|
||
mv "$mod" "pkgs/$mod"
|
||
fi
|
||
done
|
||
|
||
# 5. Install business modules (clone external, local already in pkgs)
|
||
for mod in pipeline-sdlc showcase evaluate pipeline-service pipeline-task tenant app_audit product_management discount pricing unipay smssend accounting uapi rag dingdingflow account_resource storage_resource; do
|
||
cd "$cdir/pkgs"
|
||
if [ ! -d "$mod" ]; then
|
||
git clone https://git.opencomputing.cn/yumoqing/$mod || echo "SKIP: $mod"
|
||
fi
|
||
cd "$cdir"
|
||
done
|
||
|
||
# 6. pip install all modules from pkgs/（pipeline-service 是普通 install 非 editable，部署必 pull+pip install 都做）
|
||
for mod in pipeline_core pipeline_ops pipeline_dist pipeline-sdlc showcase pipeline-service tenant app_audit product_management discount pricing unipay smssend accounting uapi rag dingdingflow account_resource storage_resource; do
|
||
if [ -d "pkgs/$mod" ]; then
|
||
"$cdir/py3/bin/pip" install "pkgs/$mod/" 2>&1 | tail -1
|
||
fi
|
||
done
|
||
|
||
# 6c. Runtime deps: pymupdf（office-text-extract 技能的 pdf 提取脚本依赖）
|
||
"$cdir/py3/bin/pip" install pymupdf 2>&1 | tail -1
|
||
|
||
# 6b. Deploy skills: global(public skills) + packs(skill pack manifests)
|
||
if [ -d "pkgs/pipeline_core/skills_library" ]; then
|
||
mkdir -p "$cdir/skills/global"
|
||
# 公共技能：预置技能全量复制到 global（所有 agent 默认加载，目录层按触发词匹配，正文按需 load_skill）
|
||
cp -r "pkgs/pipeline_core/skills_library/all/." "$cdir/skills/global/"
|
||
# 技能集清单（manifest 引用 global 里的技能名）
|
||
rm -rf "$cdir/skills/packs"
|
||
cp -r "pkgs/pipeline_core/skills_library/packs" "$cdir/skills/packs"
|
||
# 产线技能：pipelines/{pipeline_id}/common/ + roles/{role}/（SDLC 等产线默认规范，loader 按此路径扫描）
|
||
rm -rf "$cdir/skills/pipelines"
|
||
if [ -d "pkgs/pipeline_core/skills_library/pipelines" ]; then
|
||
cp -r "pkgs/pipeline_core/skills_library/pipelines" "$cdir/skills/pipelines"
|
||
echo " skills: pipelines(产线技能) 已部署，共 $(ls "$cdir/skills/pipelines" | wc -l) 个产线"
|
||
fi
|
||
echo " skills: global(公共技能)+packs(技能集) 已部署，global 共 $(ls "$cdir/skills/global" | wc -l) 个技能"
|
||
fi
|
||
|
||
# 7. Regenerate CRUD from json for all pipeline modules
|
||
for mod in appbase pipeline_core pipeline_ops pipeline_dist pipeline-sdlc rbac dapi accounting uapi rag dingdingflow account_resource storage_resource; do
|
||
json_dir="pkgs/$mod/json"
|
||
models_dir="pkgs/$mod/models"
|
||
wwwroot_dir="pkgs/$mod/wwwroot"
|
||
if [ -d "$json_dir" ] && ls "$json_dir"/*.json >/dev/null 2>&1; then
|
||
cd "$json_dir"
|
||
"$cdir/py3/bin/xls2ui" -m "../models" -o "../wwwroot" "$mod" *.json 2>&1 | grep -c 'handle' | xargs -I{} echo " xls2ui $mod: {} tables"
|
||
cd "$cdir"
|
||
fi
|
||
done
|
||
|
||
|
||
# 8. Link module wwwroot directories (symlinks from pkgs/)
|
||
for mod in pipeline-sdlc showcase tenant pipeline_core pipeline_ops pipeline_dist appbase app_audit product_management discount pricing unipay rbac dapi accounting uapi rag dingdingflow account_resource storage_resource; do
|
||
src="pkgs/$mod/wwwroot"
|
||
dst="wwwroot/$mod"
|
||
if [ -d "$src" ]; then
|
||
rm -rf "$dst" 2>/dev/null
|
||
ln -sf "../pkgs/$mod/wwwroot" "$dst"
|
||
echo " wwwroot: $mod linked"
|
||
fi
|
||
done
|
||
# pipeline_task has no separate module
|
||
mkdir -p wwwroot/pipeline_task
|
||
|
||
# 8b. uapi + rag 模块建表 + 导入外部服务配置种子
|
||
# uapi 表(upapp/uapi/uapiio...)必须先建，rag 的 uapi_seed.sql 才有表可插
|
||
# rag 表统一 rag_ 前缀，与产线业务表隔离；数据独立于 ragserver（同代码不同宿主，各自的数据）
|
||
for m in uapi rag dingdingflow account_resource storage_resource; do
|
||
if [ -d "pkgs/$m/models" ] && ls pkgs/$m/models/*.json >/dev/null 2>&1; then
|
||
cd "$cdir/pkgs/$m/models"
|
||
"$cdir/py3/bin/json2ddl" mysql . > "/tmp/pipeline_${m}_ddl.sql" || {
|
||
echo "ERROR: $m json2ddl failed" >&2; cd "$cdir"; exit 1; }
|
||
if [ ! -s "/tmp/pipeline_${m}_ddl.sql" ]; then
|
||
echo "ERROR: $m DDL empty — check models/*.json (need summary+fields)" >&2
|
||
cd "$cdir"; exit 1
|
||
fi
|
||
cd "$cdir"
|
||
fi
|
||
done
|
||
if [ -f /tmp/pipeline_uapi_ddl.sql ] || [ -f /tmp/pipeline_rag_ddl.sql ]; then
|
||
"$cdir/py3/bin/python" - <<'PYEOF'
|
||
import sys, os, subprocess
|
||
sys.path.insert(0, os.getcwd())
|
||
from appPublic.jsonConfig import getConfig
|
||
from appPublic.aes import aes_decode_b64
|
||
cfg = getConfig('.', {'workdir': '.'})
|
||
kw = cfg.databases['pipeline'].kwargs
|
||
pwd = aes_decode_b64(cfg.password_key, kw.password)
|
||
# 顺序关键：uapi 建表 → rag 建表 → rag 的 uapi 配置种子
|
||
for sqlfile in ['/tmp/pipeline_uapi_ddl.sql', '/tmp/pipeline_rag_ddl.sql',
|
||
'/tmp/pipeline_dingdingflow_ddl.sql',
|
||
'/tmp/pipeline_account_resource_ddl.sql',
|
||
'/tmp/pipeline_storage_resource_ddl.sql',
|
||
'pkgs/rag/init/uapi_seed.sql']:
|
||
if not os.path.exists(sqlfile):
|
||
print(' SKIP (missing): %s' % sqlfile)
|
||
continue
|
||
with open(sqlfile, 'rb') as f:
|
||
r = subprocess.run(['mysql', '-h', str(kw.host), '-P', str(kw.port),
|
||
'-u', str(kw.user), '-p%s' % pwd, str(kw.db)],
|
||
stdin=f, capture_output=True)
|
||
tag = os.path.basename(sqlfile)
|
||
if r.returncode == 0:
|
||
print(' rag/uapi: %s applied' % tag)
|
||
else:
|
||
err = r.stderr.decode('utf-8', 'replace').strip().split(chr(10))[0]
|
||
print(' WARN %s: %s' % (tag, err[:160]))
|
||
PYEOF
|
||
fi
|
||
|
||
# 9. Sync password_key from Sage (not databases — those are app-specific)
|
||
if [ -f /d/apitest/sage/conf/config.json ]; then
|
||
python3 -c "
|
||
import json
|
||
cs=json.load(open('/d/apitest/sage/conf/config.json'))
|
||
c=json.load(open('$cdir/conf/config.json'))
|
||
c['password_key']=cs.get('password_key','')
|
||
json.dump(c,open('$cdir/conf/config.json','w'),ensure_ascii=False,indent=2)
|
||
print(' config: password_key synced')
|
||
"
|
||
fi
|
||
|
||
# 10. Create runtime dirs
|
||
mkdir -p "$cdir/logs" "$cdir/files" "$cdir/conf" "$cdir/bin"
|
||
|
||
# 10.5 Install bwrap (zero-root: 系统已有则复用，否则 apt download + dpkg -x)
|
||
if [ ! -x "$cdir/bin/bwrap" ]; then
|
||
if command -v bwrap >/dev/null 2>&1; then
|
||
cp "$(command -v bwrap)" "$cdir/bin/bwrap" 2>/dev/null || true
|
||
else
|
||
tmpd=$(mktemp -d)
|
||
( cd "$tmpd" && apt download bubblewrap >/dev/null 2>&1 && dpkg -x bubblewrap*.deb . && cp usr/bin/bwrap "$cdir/bin/bwrap" ) 2>/dev/null || true
|
||
rm -rf "$tmpd"
|
||
fi
|
||
fi
|
||
if [ -x "$cdir/bin/bwrap" ]; then
|
||
echo " bwrap: $("$cdir/bin/bwrap" --version 2>/dev/null | head -1)"
|
||
else
|
||
echo " WARN: bwrap not available (deploy sandbox will degrade to dir isolation)"
|
||
fi
|
||
|
||
# 10.8 Install Node（office 技能脚本依赖，无 sudo：官方预编译二进制）
|
||
# 解压到 bin/nodejs/，symlink node/npm/npx 到 bin/（start.sh 已把 $cdir/bin 加进 PATH）
|
||
NODE_VER="v22.23.2"
|
||
if [ ! -x "$cdir/bin/nodejs/bin/node" ]; then
|
||
echo " node: downloading $NODE_VER (linux-x64) ..."
|
||
tmpn=$(mktemp -d)
|
||
if curl -sL --max-time 180 "https://nodejs.org/dist/$NODE_VER/node-$NODE_VER-linux-x64.tar.xz" -o "$tmpn/node.tar.xz" && \
|
||
tar -xJf "$tmpn/node.tar.xz" -C "$tmpn"; then
|
||
mkdir -p "$cdir/bin/nodejs"
|
||
cp -a "$tmpn/node-$NODE_VER-linux-x64/." "$cdir/bin/nodejs/"
|
||
else
|
||
echo " WARN: node 下载失败（office 技能脚本不可用）"
|
||
fi
|
||
rm -rf "$tmpn"
|
||
fi
|
||
ln -sf nodejs/bin/node "$cdir/bin/node"
|
||
ln -sf nodejs/bin/npm "$cdir/bin/npm"
|
||
ln -sf nodejs/bin/npx "$cdir/bin/npx"
|
||
if [ -x "$cdir/bin/node" ]; then
|
||
echo " node: $("$cdir/bin/node" --version)"
|
||
fi
|
||
|
||
# 6. Fix auto-assign created_by in CRUD dspys (xls2ui overwritten)
|
||
for d in sd_projects sd_iterations; do
|
||
for t in add update; do
|
||
f="$cdir/wwwroot/pipeline-sdlc/$d/${t}_${d}.dspy"
|
||
[ -f "$f" ] && grep -q 'created_by' "$f" || sed -i "s/ns\['org_id'\] = userorgid/ns['org_id'] = userorgid\nns['created_by'] = userorgid/" "$f"
|
||
done
|
||
done
|
||
|
||
# 6b. Inject session-current-project filter into generated get dspys (迭代/Bug/测试用例 用会话当前项目，无项目报错)
|
||
if [ -f "$cdir/scripts/inject_project_filter.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/inject_project_filter.py" "$cdir/wwwroot/pipeline-sdlc" || echo " WARN: inject_project_filter.py failed"
|
||
fi
|
||
|
||
|
||
# 10.6 Create module tables (models -> DDL -> execute)
|
||
echo "=== Module table creation ==="
|
||
if [ -f "$cdir/scripts/create_tables.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/create_tables.py" || echo " WARN: create_tables.py failed (DB not ready?)"
|
||
else
|
||
echo " WARN: scripts/create_tables.py not found"
|
||
fi
|
||
|
||
# 10.7 Import module init data (appcodes dictionaries)
|
||
echo "=== Module init data import ==="
|
||
if [ -f "$cdir/scripts/import_init.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/import_init.py" || echo " WARN: import_init.py failed (DB not ready?)"
|
||
else
|
||
echo " WARN: scripts/import_init.py not found"
|
||
fi
|
||
|
||
# 11. Import RBAC permissions from conf/rp.json (r:p 数据 -> permission + rolepermission)
|
||
echo "=== RBAC permission import ==="
|
||
if [ -f "$cdir/scripts/import_rp.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/import_rp.py" || echo " WARN: import_rp.py failed (DB not ready?)"
|
||
else
|
||
echo " WARN: scripts/import_rp.py not found"
|
||
fi
|
||
|
||
# 11.5 汇率自动获取定时任务（每日 8:00 抓取中国银行外汇牌价）
|
||
# accounting 模块 fetch_forex_rates.dspy：USD/JPY/GBP → CNY，幂等写入 exchange_rate
|
||
if [ -f "$cdir/pkgs/accounting/wwwroot/api/fetch_forex_rates.dspy" ]; then
|
||
forex_cron="0 8 * * * curl -s http://localhost:9090/accounting/api/fetch_forex_rates.dspy"
|
||
cc=$(crontab -l 2>/dev/null || true)
|
||
if ! echo "$cc" | grep -Fq "fetch_forex_rates"; then
|
||
(echo "$cc"; echo "$forex_cron") | crontab -
|
||
echo " cron: 汇率任务已添加（每日 8:00）"
|
||
else
|
||
echo " cron: 汇率任务已存在，未重复添加"
|
||
fi
|
||
fi
|
||
|
||
chmod +x "$cdir/start.sh" "$cdir/stop.sh" 2>/dev/null || true
|
||
|
||
# 12. 部署期安全检查：扫「any 授权 + 端点无鉴权 + 危险动作」等危险组合（防回归）
|
||
# 不阻断部署（与本脚本其它步骤的 WARN 风格一致），但会打印清单；
|
||
# CI 里可直接调 scripts/security_check.py（A/B/D 类返回退出码 1）做硬门禁。
|
||
echo "=== Security check (RBAC any grants × endpoint auth) ==="
|
||
if [ -f "$cdir/scripts/security_check.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/security_check.py" --warn-only \
|
||
|| echo " WARN: security_check.py failed (DB not ready?)"
|
||
else
|
||
echo " WARN: scripts/security_check.py not found"
|
||
fi
|
||
|
||
echo "=== Build complete ==="
|