diff --git a/app/pipeline_app.py b/app/pipeline_app.py index f608e6d..ac36351 100644 --- a/app/pipeline_app.py +++ b/app/pipeline_app.py @@ -181,5 +181,9 @@ def init(): # Permission cache warms up on first request -if __name__ == '__main__': +def main(): webapp(init) + + +if __name__ == '__main__': + main() diff --git a/scripts/build_binary.sh b/scripts/build_binary.sh new file mode 100644 index 0000000..6ce6c2d --- /dev/null +++ b/scripts/build_binary.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# Nuitka 二进制化构建(2026-09-17 立项):把自有 Python 包逐模块编译成 .so, +# 产出 site-packages 覆盖层 binpkg_overlay/(保留包目录结构 + 包内数据文件)。 +# +# 实测定型的编译形态(测试机 2026-09-17 实验 out1-out5): +# - 逐模块 --module 编译,保留包目录结构(整包单 .so 会断 importlib 动态导入子模块,弃) +# - __init__.py 不能直接 --module(Nuitka 拒),用改名 trick:cp 成 <目录名>.py 编译 +# (PyInit 符号=文件 stem=目录名,天然正确;objcopy 改动态符号无效,弃) +# - 数据文件(.json/.tmpl 等)不进编译,原样拷进覆盖层;运行时 __file__ 是虚拟 .py 路径 +# (与 .so 同目录),dirname(__file__) 定位数据目录的模式成立 +# - 相对导入/绝对导入/inspect.signature/importlib 动态导入 编译后全验证通过 +# +# 用法:bash scripts/build_binary.sh [输出目录] (在应用根目录或任意位置跑,自定位) +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +PY=$ROOT/py3/bin/python +OUT=${1:-$ROOT/binpkg_overlay} +TAG="cpython-310-x86_64-linux-gnu" +WORK=$(mktemp -d /tmp/nuitka_build.XXXXXX) +JOBS=$WORK/jobs.tsv +FAILLOG=$WORK/fail.log +OKLOG=$WORK/ok.log +: > "$JOBS"; : > "$FAILLOG"; : > "$OKLOG" +mkdir -p "$OUT" + +# repo目录:包目录(测试机 pkgs/ 布局;包目录名以仓库实查为准,勿凭记忆) +MANIFEST=" +pipeline-service:pipeline_service +pipeline-core:pipeline_core +pipeline-ops:pipeline_ops +pipeline-sdlc:pipeline_sdlc +pipeline-bidding:pipeline_bidding +pipeline-opportunity:pipeline_opportunity +pipeline-platform:pipeline_platform +pipeline-task:pipeline_task +pipeline-llm:pipeline_llm +apppublic:appPublic +sqlor:sqlor +ahserver:ahserver +rbac:rbac +appbase:appbase +app_audit:app_audit +xls2ddl:xls2ddl +bricks-for-python:bricks_for_python +accounting:accounting +account_resource:account_resource +storage_resource:storage_resource +supplychain:supplychain +discount:discount +pricing:pricing +product_management:product_management +unipay:unipay +smssend:smssend +filemgr:filemgr +dapi:dapi +uapi:uapi +rag:rag +dingdingflow:dingdingflow +showcase:showcase +ticket:ticket +voucher:voucher +tenant:tenant +" + +# 生成编译任务:src dest mode stem +gen_jobs() { + local pkgdir=$1 outpkg=$2 + find "$pkgdir" -name "*.py" -not -path "*__pycache__*" -not -name "test_*" \ + -not -path "*/tests/*" -not -path "*/.git/*" | while read -r f; do + local rel=${f#$pkgdir/} dir stem src mode + dir=$(dirname "$rel"); stem=$(basename "$f" .py) + local destdir="$OUT/$outpkg"; [ "$dir" != "." ] && destdir="$OUT/$outpkg/$dir" + if [ "$stem" = "__init__" ]; then + # 改名 trick:副本名=该包目录最后一段(顶层=包名,子包=子包名) + local comp; comp=$(basename "$dir"); [ "$dir" = "." ] && comp=$outpkg + src="$WORK/init_${outpkg}_${dir//\//_}.py" + cp "$f" "$src" + mode=init; stem=$comp + else + src=$f; mode=mod + fi + printf "%s\t%s\t%s\t%s\n" "$src" "$destdir" "$mode" "$stem" >> "$JOBS" + done + # 包内数据文件原样进覆盖层(不用 rsync:目标机可能没装) + mkdir -p "$OUT/$outpkg" + (cd "$pkgdir" && find . -type f -not -name "*.py" -not -name "*.pyc" \ + -not -path "*__pycache__*" -not -path "./.git/*") | while read -r rel; do + mkdir -p "$OUT/$outpkg/$(dirname "$rel")" + cp "$pkgdir/$rel" "$OUT/$outpkg/$rel" + done +} + +echo "$MANIFEST" | grep -v '^$' | while read -r entry; do + repo=${entry%%:*}; pkg=${entry#*:} + pkgdir=$ROOT/pkgs/$repo/$pkg + [ -d "$pkgdir" ] || { echo "SKIP(目录不存在): $pkgdir"; continue; } + gen_jobs "$pkgdir" "$pkg" +done + +# 入口与 app 顶层模块(script 形态,编译成顶层模块进 _appbin) +mkdir -p "$OUT/_appbin" +for f in "$ROOT"/app/*.py; do + stem=$(basename "$f" .py) + printf "%s\t%s\tmod\t%s\n" "$f" "$OUT/_appbin" "$stem" >> "$JOBS" +done + +cat > "$WORK/worker.sh" <<'EOS' +src=$1; dest=$2; mode=$3; stem=$4 +tag="cpython-310-x86_64-linux-gnu" +mkdir -p "$dest" +out=$("$PYBIN" -m nuitka --module "$src" --output-dir="$dest" --remove-output 2>&1) +rc=$? +if [ $rc -ne 0 ]; then echo "FAIL $src :: $(echo "$out" | tail -2)" >> "$FAILLOG"; exit 0; fi +if [ "$mode" = init ]; then + mv "$dest/$stem.$tag.so" "$dest/__init__.$tag.so" 2>/dev/null || { echo "FAIL(mv) $src" >> "$FAILLOG"; exit 0; } +fi +echo "OK $src" >> "$OKLOG" +EOS +export PYBIN=$PY +export FAILLOG OKLOG +total=$(wc -l < "$JOBS") +echo "编译任务 $total 个,并行 4 …" +xargs -a "$JOBS" -P 4 -n 4 bash "$WORK/worker.sh" +ok=$(wc -l < "$OKLOG"); fail=$(wc -l < "$FAILLOG") +echo "完成: OK=$ok FAIL=$fail" +if [ "$fail" -gt 0 ]; then echo "== 失败清单 =="; cat "$FAILLOG"; exit 1; fi + +# runner(工具脚本,非业务代码,保留源码形态) +cat > "$OUT/_appbin/run_compiled.py" <<'EOS' +#!/usr/bin/env python3 +"""编译态入口 runner:sys.argv 原样透传给 ahserver webapp 的参数解析。 +sys.path 顺序:_appbin(入口编译模块) > overlay 根(编译业务包,遮蔽 site-packages 源码) > 其余。""" +import os, sys +_here = os.path.dirname(os.path.abspath(__file__)) +_overlay = os.path.dirname(_here) +sys.path.insert(0, _overlay) +sys.path.insert(0, _here) +import pipeline_app +pipeline_app.main() +EOS + +# BUILD_INFO:逐仓哈希(部署对账/回滚点用) +HASHF=$WORK/hashes.jsonl +: > "$HASHF" +echo "$MANIFEST" | grep -v '^$' | while read -r entry; do + repo=${entry%%:*} + h=$(git -C "$ROOT/pkgs/$repo" rev-parse HEAD 2>/dev/null || echo unknown) + printf ' "%s": "%s"\n' "$repo" "$h" >> "$HASHF" +done +printf ' "pipeline-app": "%s"\n' "$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || echo unknown)" >> "$HASHF" +{ + echo "{" + echo " \"built_at\": \"$(date -Is)\"," + echo " \"python\": \"$($PY --version 2>&1)\"," + awk 'NR>1{printf ",\n"} {printf "%s", $0} END{print ""}' "$HASHF" + echo "}" +} > "$OUT/BUILD_INFO.json" + +so=$(find "$OUT" -name "*.$TAG.so" | wc -l) +echo "覆盖层: $OUT (.so=$so)" +du -sh "$OUT" | cut -f1