276 lines
13 KiB
Bash
276 lines
13 KiB
Bash
#!/usr/bin/env bash
|
||
set -e
|
||
cdir=$(cd "$(dirname "$0")" && pwd)
|
||
cd "$cdir"
|
||
|
||
echo "=== Pipeline App Build ==="
|
||
|
||
# 1. Create venv
|
||
if [ ! -d py3 ]; then
|
||
python3 -m venv py3
|
||
fi
|
||
source py3/bin/activate
|
||
|
||
# 2. Install foundation packages (clone to pkgs)
|
||
mkdir -p pkgs
|
||
for m in apppublic sqlor ahserver bricks-for-python xls2ddl rbac appbase dapi; do
|
||
echo "install $m ..."
|
||
cd "$cdir/pkgs"
|
||
if [ ! -d "$m" ]; then
|
||
git clone https://git.opencomputing.cn/yumoqing/$m || echo "SKIP: $m clone failed"
|
||
fi
|
||
if [ -d "$m" ]; then
|
||
cd "$m"
|
||
"$cdir/py3/bin/pip" install . 2>&1 | tail -1
|
||
fi
|
||
cd "$cdir"
|
||
done
|
||
|
||
# 3. Build bricks frontend
|
||
cd "$cdir/pkgs"
|
||
if [ ! -d bricks ]; then
|
||
git clone https://git.opencomputing.cn/yumoqing/bricks || true
|
||
fi
|
||
if [ ! -f bricks/dist/bricks.js ]; then
|
||
mkdir -p bricks/dist
|
||
if [ -d bricks/bricks ]; then
|
||
cd bricks/bricks
|
||
bash build.sh 2>&1 | tail -3
|
||
cd "$cdir/pkgs"
|
||
fi
|
||
fi
|
||
# Link dist to wwwroot
|
||
rm -rf "$cdir/wwwroot/bricks" 2>/dev/null
|
||
ln -sf "$cdir/pkgs/bricks/dist" "$cdir/wwwroot/bricks"
|
||
echo "bricks: $(readlink "$cdir/wwwroot/bricks")"
|
||
|
||
# 4. Move local business modules into pkgs/
|
||
cd "$cdir"
|
||
for mod in pipeline_core pipeline_ops pipeline_dist; do
|
||
if [ -d "$mod" ]; then
|
||
mv "$mod" "pkgs/$mod"
|
||
fi
|
||
done
|
||
|
||
# 5. Install business modules (clone external, local already in pkgs)
|
||
# 注：evaluate 只 clone 不 install——其 wwwroot/json/models 嵌套在包内(evaluate/evaluate/)，
|
||
# 与本脚本 pkgs/<mod>/wwwroot 软链约定不兼容，且依赖 VBench/torch 重环境；
|
||
# 待营销产线开发时连同结构规范化一起接入。
|
||
for mod in pipeline-sdlc showcase evaluate pipeline-service pipeline-task tenant app_audit product_management discount pricing unipay smssend accounting uapi rag dingdingflow account_resource storage_resource filemgr supplychain pipeline-bidding pipeline-llm pipeline-platform pipeline-opportunity; do
|
||
cd "$cdir/pkgs"
|
||
if [ ! -d "$mod" ]; then
|
||
git clone https://git.opencomputing.cn/yumoqing/$mod || echo "SKIP: $mod"
|
||
fi
|
||
cd "$cdir"
|
||
done
|
||
|
||
# 6. pip install all modules from pkgs/（pipeline-service 是普通 install 非 editable，部署必 pull+pip install 都做）
|
||
# 注意与第 5 步 clone 清单对齐：clone 了却不 install 的模块，主程序 try/except 会静默降级——
|
||
# 模块整个消失且零报错（account_resource 等 4 个此前正是这种状态，测试机靠手工补装）。
|
||
for mod in pipeline_core pipeline_ops pipeline_dist pipeline-sdlc showcase pipeline-service pipeline-task tenant app_audit product_management discount pricing unipay smssend accounting uapi rag dingdingflow account_resource storage_resource filemgr supplychain pipeline-bidding pipeline-llm pipeline-platform pipeline-opportunity; do
|
||
if [ -d "pkgs/$mod" ]; then
|
||
"$cdir/py3/bin/pip" install "pkgs/$mod/" 2>&1 | tail -1
|
||
fi
|
||
done
|
||
|
||
# 6c. Runtime deps: pymupdf（office-text-extract 技能的 pdf 提取脚本依赖）
|
||
"$cdir/py3/bin/pip" install pymupdf 2>&1 | tail -1
|
||
|
||
# 6b. Deploy skills: global(public skills) + packs(skill pack manifests)
|
||
if [ -d "pkgs/pipeline_core/skills_library" ]; then
|
||
mkdir -p "$cdir/skills/global"
|
||
# 公共技能：预置技能全量复制到 global（所有 agent 默认加载，目录层按触发词匹配，正文按需 load_skill）
|
||
cp -r "pkgs/pipeline_core/skills_library/all/." "$cdir/skills/global/"
|
||
# 技能集清单（manifest 引用 global 里的技能名）
|
||
rm -rf "$cdir/skills/packs"
|
||
cp -r "pkgs/pipeline_core/skills_library/packs" "$cdir/skills/packs"
|
||
# 产线技能：pipelines/{pipeline_id}/common/ + roles/{role}/（SDLC 等产线默认规范，loader 按此路径扫描）
|
||
rm -rf "$cdir/skills/pipelines"
|
||
if [ -d "pkgs/pipeline_core/skills_library/pipelines" ]; then
|
||
cp -r "pkgs/pipeline_core/skills_library/pipelines" "$cdir/skills/pipelines"
|
||
echo " skills: pipelines(产线技能) 已部署，共 $(ls "$cdir/skills/pipelines" | wc -l) 个产线"
|
||
fi
|
||
echo " skills: global(公共技能)+packs(技能集) 已部署，global 共 $(ls "$cdir/skills/global" | wc -l) 个技能"
|
||
fi
|
||
|
||
# 7. Regenerate CRUD from json for all pipeline modules
|
||
# 注意：凡 json/ 下有 CRUD 定义的模块都必须在列——漏掉则页面从未生成，
|
||
# 菜单点进去 500（fpath is None）。
|
||
# 历史事故：pricing 漏列 → 定价管理 500（2026-08）；
|
||
# discount 漏列 → 折扣管理 4 个子菜单全 500（2026-09-09，已补入）。
|
||
# ⚠ 清单不能盲目扩成"全模块动态扫描"：product_management/dingdingflow 的生成目录
|
||
# 是 git 跟踪的（重生成会弄脏工作树），且第 6 步有 xls2ui 覆盖后的手工修补——
|
||
# 新模块入列前先确认其生成目录在模块 .gitignore 内。
|
||
for mod in appbase pipeline_core pipeline_ops pipeline_dist pipeline-sdlc rbac dapi accounting uapi rag dingdingflow account_resource storage_resource filemgr supplychain pipeline-bidding pricing pipeline-llm pipeline-opportunity discount; do
|
||
json_dir="pkgs/$mod/json"
|
||
if [ -d "$json_dir" ] && ls "$json_dir"/*.json >/dev/null 2>&1; then
|
||
cd "$json_dir"
|
||
"$cdir/py3/bin/xls2ui" -m "../models" -o "../wwwroot" "$mod" *.json 2>&1 | grep -c 'handle' | xargs -I{} echo " xls2ui $mod: {} tables"
|
||
cd "$cdir"
|
||
fi
|
||
done
|
||
|
||
|
||
# 8. Link module wwwroot directories (symlinks from pkgs/)
|
||
for mod in pipeline-sdlc showcase tenant pipeline_core pipeline_ops pipeline_dist appbase app_audit product_management discount pricing unipay rbac dapi accounting uapi rag dingdingflow account_resource storage_resource filemgr supplychain pipeline-bidding pipeline-llm pipeline-platform pipeline-opportunity pipeline-task; do
|
||
src="pkgs/$mod/wwwroot"
|
||
dst="wwwroot/$mod"
|
||
if [ -d "$src" ]; then
|
||
rm -rf "$dst" 2>/dev/null
|
||
# pipeline-task 的菜单/RBAC 路径用下划线(/pipeline_task)，软链名保持下划线
|
||
[ "$mod" = "pipeline-task" ] && dst="wwwroot/pipeline_task"
|
||
ln -sf "../pkgs/$mod/wwwroot" "$dst"
|
||
echo " wwwroot: $mod linked"
|
||
fi
|
||
done
|
||
# pipeline_task 兜底：模块缺失时保留空目录防菜单 404（正常路径已由上方软链覆盖）
|
||
[ -e wwwroot/pipeline_task ] || mkdir -p wwwroot/pipeline_task
|
||
|
||
# 8b. uapi/rag 等模块建表已统一交给 create_tables.py（步骤 10.6，动态扫描 pkgs/*/models，幂等不 DROP）。
|
||
# 历史事故（2026-09-07）：本段曾对这 7 个模块跑含 `drop table if exists` 的 json2ddl DDL，
|
||
# 每次部署清空 upapp/uapi/storres_spec/supplychain 等表——与 TABLE_MODULES 清库同源隐患。
|
||
# rag 的 uapi_seed.sql 数据种子（INSERT IGNORE，依赖 uapi 表先建）移到步骤 10.65 执行。
|
||
|
||
# 9. Sync password_key from Sage (not databases — those are app-specific)
|
||
if [ -f /d/apitest/sage/conf/config.json ]; then
|
||
python3 -c "
|
||
import json
|
||
cs=json.load(open('/d/apitest/sage/conf/config.json'))
|
||
c=json.load(open('$cdir/conf/config.json'))
|
||
c['password_key']=cs.get('password_key','')
|
||
json.dump(c,open('$cdir/conf/config.json','w'),ensure_ascii=False,indent=2)
|
||
print(' config: password_key synced')
|
||
"
|
||
fi
|
||
|
||
# 10. Create runtime dirs
|
||
mkdir -p "$cdir/logs" "$cdir/files" "$cdir/conf" "$cdir/bin"
|
||
|
||
# 10.5 Install bwrap (zero-root: 系统已有则复用，否则 apt download + dpkg -x)
|
||
if [ ! -x "$cdir/bin/bwrap" ]; then
|
||
if command -v bwrap >/dev/null 2>&1; then
|
||
cp "$(command -v bwrap)" "$cdir/bin/bwrap" 2>/dev/null || true
|
||
else
|
||
tmpd=$(mktemp -d)
|
||
( cd "$tmpd" && apt download bubblewrap >/dev/null 2>&1 && dpkg -x bubblewrap*.deb . && cp usr/bin/bwrap "$cdir/bin/bwrap" ) 2>/dev/null || true
|
||
rm -rf "$tmpd"
|
||
fi
|
||
fi
|
||
if [ -x "$cdir/bin/bwrap" ]; then
|
||
echo " bwrap: $("$cdir/bin/bwrap" --version 2>/dev/null | head -1)"
|
||
else
|
||
echo " WARN: bwrap not available (deploy sandbox will degrade to dir isolation)"
|
||
fi
|
||
|
||
# 10.8 Install Node（office 技能脚本依赖，无 sudo：官方预编译二进制）
|
||
# 解压到 bin/nodejs/，symlink node/npm/npx 到 bin/（start.sh 已把 $cdir/bin 加进 PATH）
|
||
NODE_VER="v22.23.2"
|
||
if [ ! -x "$cdir/bin/nodejs/bin/node" ]; then
|
||
echo " node: downloading $NODE_VER (linux-x64) ..."
|
||
tmpn=$(mktemp -d)
|
||
if curl -sL --max-time 180 "https://nodejs.org/dist/$NODE_VER/node-$NODE_VER-linux-x64.tar.xz" -o "$tmpn/node.tar.xz" && \
|
||
tar -xJf "$tmpn/node.tar.xz" -C "$tmpn"; then
|
||
mkdir -p "$cdir/bin/nodejs"
|
||
cp -a "$tmpn/node-$NODE_VER-linux-x64/." "$cdir/bin/nodejs/"
|
||
else
|
||
echo " WARN: node 下载失败（office 技能脚本不可用）"
|
||
fi
|
||
rm -rf "$tmpn"
|
||
fi
|
||
ln -sf nodejs/bin/node "$cdir/bin/node"
|
||
ln -sf nodejs/bin/npm "$cdir/bin/npm"
|
||
ln -sf nodejs/bin/npx "$cdir/bin/npx"
|
||
if [ -x "$cdir/bin/node" ]; then
|
||
echo " node: $("$cdir/bin/node" --version)"
|
||
fi
|
||
|
||
# 6. Fix auto-assign created_by in CRUD dspys (xls2ui overwritten)
|
||
for d in sd_projects sd_iterations; do
|
||
for t in add update; do
|
||
f="$cdir/wwwroot/pipeline-sdlc/$d/${t}_${d}.dspy"
|
||
[ -f "$f" ] && grep -q 'created_by' "$f" || sed -i "s/ns\['org_id'\] = userorgid/ns['org_id'] = userorgid\nns['created_by'] = userorgid/" "$f"
|
||
done
|
||
done
|
||
|
||
# 6b. Inject session-current-project filter into generated get dspys (迭代/Bug/测试用例 用会话当前项目，无项目报错)
|
||
if [ -f "$cdir/scripts/inject_project_filter.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/inject_project_filter.py" "$cdir/wwwroot/pipeline-sdlc" || echo " WARN: inject_project_filter.py failed"
|
||
fi
|
||
|
||
|
||
# 10.6 Create module tables (models -> DDL -> execute)
|
||
echo "=== Module table creation ==="
|
||
if [ -f "$cdir/scripts/create_tables.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/create_tables.py" || echo " WARN: create_tables.py failed (DB not ready?)"
|
||
else
|
||
echo " WARN: scripts/create_tables.py not found"
|
||
fi
|
||
|
||
# 10.65 rag 模块 uapi 外部服务配置种子（INSERT IGNORE 幂等；依赖 uapi 表已由 10.6 建好）
|
||
echo "=== rag uapi seed ==="
|
||
if [ -f "$cdir/pkgs/rag/init/uapi_seed.sql" ]; then
|
||
"$cdir/py3/bin/python" - <<'PYEOF'
|
||
import sys, os, subprocess
|
||
sys.path.insert(0, os.getcwd())
|
||
from appPublic.jsonConfig import getConfig
|
||
from appPublic.aes import aes_decode_b64
|
||
cfg = getConfig('.', {'workdir': '.'})
|
||
kw = cfg.databases['pipeline'].kwargs
|
||
pwd = aes_decode_b64(cfg.password_key, kw.password)
|
||
sqlfile = 'pkgs/rag/init/uapi_seed.sql'
|
||
with open(sqlfile, 'rb') as f:
|
||
r = subprocess.run(['mysql', '-h', str(kw.host), '-P', str(kw.port),
|
||
'-u', str(kw.user), '-p%s' % pwd, str(kw.db)],
|
||
stdin=f, capture_output=True)
|
||
if r.returncode == 0:
|
||
print(' rag uapi_seed applied')
|
||
else:
|
||
err = r.stderr.decode('utf-8', 'replace').strip().split(chr(10))[0]
|
||
print(' WARN uapi_seed: %s' % err[:160])
|
||
PYEOF
|
||
fi
|
||
|
||
# 10.7 Import module init data (appcodes dictionaries)
|
||
echo "=== Module init data import ==="
|
||
if [ -f "$cdir/scripts/import_init.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/import_init.py" || echo " WARN: import_init.py failed (DB not ready?)"
|
||
else
|
||
echo " WARN: scripts/import_init.py not found"
|
||
fi
|
||
|
||
# 11. Import RBAC permissions from conf/rp.json (r:p 数据 -> permission + rolepermission)
|
||
echo "=== RBAC permission import ==="
|
||
if [ -f "$cdir/scripts/import_rp.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/import_rp.py" || echo " WARN: import_rp.py failed (DB not ready?)"
|
||
else
|
||
echo " WARN: scripts/import_rp.py not found"
|
||
fi
|
||
|
||
# 11.5 汇率自动获取定时任务（每日 8:00 抓取中国银行外汇牌价）
|
||
# accounting 模块 fetch_forex_rates.dspy：USD/JPY/GBP → CNY，幂等写入 exchange_rate
|
||
if [ -f "$cdir/pkgs/accounting/wwwroot/api/fetch_forex_rates.dspy" ]; then
|
||
forex_cron="0 8 * * * curl -s http://localhost:9090/accounting/api/fetch_forex_rates.dspy"
|
||
cc=$(crontab -l 2>/dev/null || true)
|
||
if ! echo "$cc" | grep -Fq "fetch_forex_rates"; then
|
||
(echo "$cc"; echo "$forex_cron") | crontab -
|
||
echo " cron: 汇率任务已添加（每日 8:00）"
|
||
else
|
||
echo " cron: 汇率任务已存在，未重复添加"
|
||
fi
|
||
fi
|
||
|
||
chmod +x "$cdir/start.sh" "$cdir/stop.sh" 2>/dev/null || true
|
||
|
||
# 12. 部署期安全检查：扫「any 授权 + 端点无鉴权 + 危险动作」等危险组合（防回归）
|
||
# 不阻断部署（与本脚本其它步骤的 WARN 风格一致），但会打印清单；
|
||
# CI 里可直接调 scripts/security_check.py（A/B/D 类返回退出码 1）做硬门禁。
|
||
echo "=== Security check (RBAC any grants × endpoint auth) ==="
|
||
if [ -f "$cdir/scripts/security_check.py" ]; then
|
||
"$cdir/py3/bin/python" "$cdir/scripts/security_check.py" --warn-only \
|
||
|| echo " WARN: security_check.py failed (DB not ready?)"
|
||
else
|
||
echo " WARN: scripts/security_check.py not found"
|
||
fi
|
||
|
||
echo "=== Build complete ==="
|