refactor: build.sh matches deployment spec (pkgs/py3, WORKDIR fix)

This commit is contained in:
yumoqing 2026-08-11 15:16:42 +08:00
parent 23e11cdadb
commit 6bca07f4a6

359
build.sh
View File

@ -1,313 +1,244 @@
#!/usr/bin/env bash
# ============================================================
# pccs build.sh — 一键部署脚本 (符合 web-application-spec)
# pccs build.sh — 一键部署 (符合部署目录规范)
#
# 在部署节点上运行:
# cd /d/pccs/repos/pccs && bash build.sh
# 目标结构:
# /d/pccs/
# ├── app/ ← 应用入口
# ├── conf/ ← config.json
# ├── pkgs/ ← 所有模块 clone
# ├── wwwroot/ ← 软链接
# ├── py3/ ← venv
# ├── logs/
# ├── files/
# └── scripts/
#
# 前提:
# - MariaDB 已安装, pccs 库已创建, test/test123 已授权
# - /d 已作为 NFS 共享盘导出
# - git, python3.10+, pip, npm 已安装
# 运行: cd /d/pccs && bash build.sh
# ============================================================
set -e
# --------------- 配置 ---------------
WORKDIR="$(cd "$(dirname "$0")/../.." && pwd)"
CDIR="${WORKDIR}" # 应用根目录 (/d/pccs)
PKGDIR="${CDIR}/repos" # 模块 clone 目录
VENV="${CDIR}/venv"
# --------------- 路径 ---------------
# 假设 build.sh 放在 /d/pccs/build.sh (从 pccs repo 复制)
CDIR="$(cd "$(dirname "$0")" && pwd)" # /d/pccs
PKGDIR="${CDIR}/pkgs"
VENV="${CDIR}/py3"
APPNAME="pccs"
APP_PORT=9180
# Git 仓库基地址
GIT_BASE="https://git.opencomputing.cn/yumoqing"
# --------------- 颜色 ---------------
red() { echo -e "\033[31m$1\033[0m"; }
green() { echo -e "\033[32m$1\033[0m"; }
blue() { echo -e "\033[34m$1\033[0m"; }
blue "=========================================="
blue " pccs — 算力中心集群系统 部署"
blue " pccs 部署 — ${CDIR}"
blue "=========================================="
# ============================================================
# Phase 0: 创建目录
# 0: 目录
# ============================================================
mkdir -p "${CDIR}/app" "${CDIR}/conf" "${CDIR}/files" "${CDIR}/logs" \
"${CDIR}/wwwroot/imgs" "${PKGDIR}"
"${CDIR}/wwwroot/imgs" "${CDIR}/scripts" "${PKGDIR}"
# 复制 app/pccs.py 和 conf/config.json 到顶层
if [ -d "${PKGDIR}/pccs" ]; then
cp -f "${PKGDIR}/pccs/app/pccs.py" "${CDIR}/app/pccs.py" 2>/dev/null || true
cp -f "${PKGDIR}/pccs/conf/config.json" "${CDIR}/conf/config.json" 2>/dev/null || true
fi
# ============================================================
# Phase 1: 安装系统依赖
# 1: 系统依赖
# ============================================================
blue "[1/10] 系统依赖..."
sudo apt-get update -qq
sudo apt-get install -y -qq python3-venv python3-pip python3-dev \
build-essential nfs-common
sudo apt-get install -y -qq python3-venv python3-pip python3-dev build-essential nfs-common
green " OK"
# ============================================================
# Phase 2: Python 虚拟环境
# 2: venv
# ============================================================
blue "[2/10] Python 虚拟环境..."
python3 -m venv "${VENV}" --clear
PIP="${VENV}/bin/pip"
PYTHON="${VENV}/bin/python"
"${PIP}" install --upgrade pip setuptools wheel -q
# 配置 pip 国内镜像 (阿里云)
"${PIP}" config set global.index-url https://mirrors.aliyun.com/pypi/simple/ 2>/dev/null || true
"${PIP}" install --upgrade pip setuptools wheel -q
# 安装 xls2ddl (DDL 生成工具)
"${PIP}" install xls2ddl -q --timeout=30 2>/dev/null || {
red " WARN: xls2ddl install failed (PyPI unreachable?), skip"
}
green " OK"
"${PIP}" install xls2ddl -q --timeout=30 2>/dev/null || red " WARN: xls2ddl skip"
green " OK: ${VENV}"
# ============================================================
# Phase 3: Clone/Pull 所有模块
# 3: Clone/Pull
# ============================================================
blue "[3/10] 同步代码仓库..."
ALL_REPOS=(
# 基础层
apppublic sqlor ahserver
# 数据库层
appbase rbac
# 前端
bricks bricks_for_python
# 业务模块
pcpool pcc storage_mgr image_mgr
# 主应用
pccs
)
for repo in "${ALL_REPOS[@]}"; do
blue "[3/10] 同步仓库..."
ALL=(apppublic sqlor ahserver appbase rbac bricks bricks_for_python
pcpool pcc storage_mgr image_mgr pccs)
for repo in "${ALL[@]}"; do
if [ -d "${PKGDIR}/${repo}/.git" ]; then
(cd "${PKGDIR}/${repo}" && git pull origin main 2>/dev/null) || true
else
echo " clone ${repo}..."
git clone "${GIT_BASE}/${repo}.git" "${PKGDIR}/${repo}" 2>/dev/null || {
red " WARN: ${repo} clone failed"
}
git clone "${GIT_BASE}/${repo}.git" "${PKGDIR}/${repo}" 2>/dev/null || red " WARN: ${repo}"
fi
done
# 确保 app/conf 文件存在
cp -f "${PKGDIR}/pccs/app/pccs.py" "${CDIR}/app/pccs.py" 2>/dev/null || true
cp -f "${PKGDIR}/pccs/conf/config.json" "${CDIR}/conf/config.json" 2>/dev/null || true
# 更新 config.json 路径占位符
if [ -f "${CDIR}/conf/config.json" ]; then
sed -i "s|\$\[workdir\]\$|${CDIR}|g" "${CDIR}/conf/config.json"
fi
green " OK"
# ============================================================
# Phase 4: pip install 所有模块
# 4: pip install
# ============================================================
blue "[4/10] 安装 Python 模块..."
MODULES=(
apppublic sqlor ahserver
appbase rbac
bricks_for_python
pcpool pcc storage_mgr image_mgr
)
blue "[4/10] 安装模块..."
MODULES=(apppublic sqlor ahserver appbase rbac bricks_for_python
pcpool pcc storage_mgr image_mgr)
for m in "${MODULES[@]}"; do
mp="${PKGDIR}/${m}"
if [ -f "${mp}/setup.py" ] || [ -f "${mp}/setup.cfg" ] || [ -f "${mp}/pyproject.toml" ]; then
"${PIP}" install -e "${mp}" -q 2>&1 | tail -1
fi
[ -f "${mp}/setup.py" ] || [ -f "${mp}/setup.cfg" ] || [ -f "${mp}/pyproject.toml" ] || continue
"${PIP}" install -e "${mp}" -q 2>&1 | tail -1
done
green " OK"
# ============================================================
# Phase 5: 数据库 DDL — json2ddl 生成建表 SQL
# 5: DDL
# ============================================================
blue "[5/10] 数据库 DDL 生成..."
# 收集所有有 models/ 的模块
DB_MODULES=(appbase rbac pcpool pcc storage_mgr image_mgr)
for m in "${DB_MODULES[@]}"; do
blue "[5/10] DDL 建表..."
for m in appbase rbac pcpool pcc storage_mgr image_mgr; do
mdir="${PKGDIR}/${m}/models"
if [ -d "$mdir" ]; then
has_json=$(ls "$mdir"/*.json 2>/dev/null | wc -l)
has_xlsx=$(ls "$mdir"/*.xlsx 2>/dev/null | wc -l)
if [ "$has_json" -gt 0 ]; then
echo " [${m}] json2ddl → DDL"
(cd "$mdir" && "${VENV}/bin/json2ddl" mysql . > /tmp/pccs_${m}.ddl.sql 2>/dev/null) || true
mysql -u test -ptest123 pccs < /tmp/pccs_${m}.ddl.sql 2>/dev/null || true
[ -d "$mdir" ] || continue
for jsonf in "$mdir"/*.json; do
[ -f "$jsonf" ] || continue
tbl=$(python3 -c "import json;d=json.load(open('$jsonf'));print(d['summary'][0]['name'])" 2>/dev/null) || continue
exists=$(mysql -u test -ptest123 pccs -sNe "SHOW TABLES LIKE '$tbl'" 2>/dev/null)
if [ -z "$exists" ]; then
echo " [${m}] create table ${tbl}..."
"${PYTHON}" -c "
import sys,json
sys.path[:0]=['${PKGDIR}/apppublic','${PKGDIR}/sqlor','${PKGDIR}/ahserver']
from appPublic.jsonConfig import getConfig
from sqlor.dbpools import DBPools
from sqlor.ddl_template_mysql import DDLTemplateMySQL
import asyncio
async def go():
config=getConfig('${CDIR}')
db=DBPools(config.databases)
ddl=DDLTemplateMySQL()
m=json.load(open('${jsonf}'))
sql=ddl.create_table(m['summary'][0]['name'],m.get('fields',[]))
async with db.sqlorContext('pccs') as s:
await s.sqlExe(sql,{})
print(' done')
asyncio.run(go())
" 2>&1 | tail -1
fi
if [ "$has_xlsx" -gt 0 ]; then
echo " [${m}] xls2ddl → DDL"
(cd "$mdir" && xls2ddl mysql . > /tmp/pccs_${m}_xlsx.ddl.sql 2>/dev/null) || true
mysql -u test -ptest123 pccs < /tmp/pccs_${m}_xlsx.ddl.sql 2>/dev/null || true
fi
fi
done
done
green " OK"
# ============================================================
# Phase 6: 导入初始数据 (data/ 目录)
# 6: xls2ui
# ============================================================
blue "[6/10] 初始数据导入..."
for m in "${DB_MODULES[@]}"; do
ddir="${PKGDIR}/${m}/data"
if [ -d "$ddir" ]; then
for xlsx in "$ddir"/*.xlsx; do
[ -f "$xlsx" ] || continue
echo " [${m}] dbloader → ${xlsx}"
"${VENV}/bin/dbloader" "${CDIR}" pccs "$xlsx" 2>/dev/null || true
done
fi
done
green " OK"
# ============================================================
# Phase 7: CRUD 生成 — xls2ui
# ============================================================
blue "[7/10] CRUD 界面生成..."
for m in "${DB_MODULES[@]}"; do
blue "[6/10] CRUD 生成..."
for m in appbase rbac pcpool pcc storage_mgr image_mgr; do
jdir="${PKGDIR}/${m}/json"
mdir="${PKGDIR}/${m}/models"
odir="${PKGDIR}/${m}/wwwroot"
if [ -d "$jdir" ] && [ -d "$mdir" ]; then
echo " [${m}] xls2ui..."
(cd "$jdir" && "${VENV}/bin/xls2ui" -m "$mdir" -o "$odir" "$m" *.json 2>/dev/null) || true
fi
[ -d "$jdir" ] && [ -d "$mdir" ] || continue
echo " [${m}] xls2ui..."
(cd "$jdir" && "${VENV}/bin/xls2ui" -m "$mdir" -o "$odir" "$m" *.json 2>/dev/null) || true
done
green " OK"
# ============================================================
# Phase 8: 创建 wwwroot 软链接
# 7: wwwroot symlinks
# ============================================================
blue "[8/10] 创建 wwwroot 软链接..."
for m in "${DB_MODULES[@]}"; do
wdir="${PKGDIR}/${m}/wwwroot"
if [ -d "$wdir" ]; then
ln -sfn "$wdir" "${CDIR}/wwwroot/${m}" 2>/dev/null || true
fi
blue "[7/10] 软链接..."
for m in appbase rbac pcpool pcc storage_mgr image_mgr; do
[ -d "${PKGDIR}/${m}/wwwroot" ] || continue
ln -sfn "${PKGDIR}/${m}/wwwroot" "${CDIR}/wwwroot/${m}" 2>/dev/null || true
done
# pccs 自身的 wwwroot
ln -sfn "${PKGDIR}/pccs/wwwroot" "${CDIR}/wwwroot/${APPNAME}" 2>/dev/null || true
# Bricks 前端
ln -sfn "${PKGDIR}/pccs/wwwroot" "${CDIR}/wwwroot/pccs" 2>/dev/null || true
# bricks
if [ -f "${PKGDIR}/bricks/build.sh" ]; then
blue "[bricks] 构建前端..."
(cd "${PKGDIR}/bricks" && npm install --silent 2>/dev/null && bash build.sh 2>&1 | tail -3) || true
(cd "${PKGDIR}/bricks" && npm install --silent 2>/dev/null && bash build.sh 2>&1 | tail -2) || true
ln -sfn "${PKGDIR}/bricks/dist" "${CDIR}/wwwroot/bricks" 2>/dev/null || true
fi
green " OK"
# ============================================================
# Phase 9: 种子应用数据 (appcode / RBAC)
# 8: 种子数据
# ============================================================
blue "[9/10] 种子数据..."
blue "[8/10] 种子数据..."
"${PYTHON}" - "${CDIR}" "${PKGDIR}" <<'PYEOF'
import sys, os, json
CDIR = sys.argv[1]
PKGDIR = sys.argv[2]
sys.path.insert(0, os.path.join(PKGDIR, 'apppublic'))
sys.path.insert(0, os.path.join(PKGDIR, 'sqlor'))
sys.path.insert(0, os.path.join(PKGDIR, 'ahserver'))
import sys,os,json,asyncio,datetime
CDIR=sys.argv[1]; PKGDIR=sys.argv[2]
sys.path[:0]=[os.path.join(PKGDIR,'apppublic'),os.path.join(PKGDIR,'sqlor'),os.path.join(PKGDIR,'ahserver')]
from appPublic.jsonConfig import getConfig
from sqlor.dbpools import DBPools
from appPublic.uniqueID import getID
import datetime, asyncio
APPCODES = [
('cluster_type', 'k8s', 'Kubernetes集群'),
('cluster_type', 'slurm', 'Slurm集群'),
('cluster_type', 'ray', 'Ray集群'),
('cluster_status', 'deploying', '部署中'),
('cluster_status', 'running', '运行中'),
('cluster_status', 'failed', '失败'),
('cluster_status', 'stopped', '已停止'),
('cluster_status', 'destroyed', '已销毁'),
('node_role', 'control', '控制节点'),
('node_role', 'compute', '算力节点'),
('node_role', 'storage', '存储节点'),
('node_status', 'joining', '加入中'),
('node_status', 'active', '活跃'),
('node_status', 'draining', '排空中'),
('node_status', 'removed', '已移除'),
('node_type', 'control', '控制节点'),
('node_type', 'compute', '算力节点'),
('node_type', 'storage', '存储节点'),
('storage_type', 'nfs', 'NFS'),
('storage_type', 'glusterfs', 'GlusterFS'),
('storage_type', 'cephfs', 'CephFS'),
('storage_status', 'online', '在线'),
('storage_status', 'offline', '离线'),
('storage_status', 'maintenance', '维护中'),
('storage_access_mode', 'rw', '读写'),
('storage_access_mode', 'ro', '只读'),
('export_status', 'active', '活跃'),
('export_status', 'inactive', '停用'),
('mount_status', 'unmounted', '未挂载'),
('mount_status', 'mounted', '已挂载'),
('mount_status', 'failed', '失败'),
('mirror_source_type', 'public_mirror', '公共镜像'),
('mirror_source_type', 'private_registry', '私有仓库'),
('mirror_source_type', 'hub', '官方Hub'),
('mirror_region', 'cn', '中国'),
('mirror_region', 'us', '美国'),
('mirror_region', 'eu', '欧洲'),
('mirror_status', 'active', '活跃'),
('mirror_status', 'inactive', '停用'),
('mirror_status', 'error', '异常'),
('registry_type', 'docker-registry', 'Docker Registry'),
('registry_type', 'harbor', 'Harbor'),
('registry_status', 'deploying', '部署中'),
('registry_status', 'running', '运行中'),
('registry_status', 'failed', '失败'),
('registry_status', 'destroyed', '已销毁'),
('image_type', 'vm', '虚拟机镜像'),
('image_type', 'base', '基础镜像'),
('image_type', 'app', '应用镜像'),
('image_type', 'custom', '自定义镜像'),
('sync_status', 'pending', '待同步'),
('sync_status', 'syncing', '同步中'),
('sync_status', 'synced', '已同步'),
('sync_status', 'failed', '失败'),
CODES=[
('cluster_type','k8s','Kubernetes集群'),('cluster_type','slurm','Slurm集群'),('cluster_type','ray','Ray集群'),
('cluster_status','deploying','部署中'),('cluster_status','running','运行中'),('cluster_status','failed','失败'),
('cluster_status','stopped','已停止'),('cluster_status','destroyed','已销毁'),
('node_role','control','控制节点'),('node_role','compute','算力节点'),('node_role','storage','存储节点'),
('node_status','joining','加入中'),('node_status','active','活跃'),('node_status','draining','排空中'),('node_status','removed','已移除'),
('node_type','control','控制节点'),('node_type','compute','算力节点'),('node_type','storage','存储节点'),
('storage_type','nfs','NFS'),('storage_type','glusterfs','GlusterFS'),('storage_type','cephfs','CephFS'),
('storage_status','online','在线'),('storage_status','offline','离线'),('storage_status','maintenance','维护中'),
('storage_access_mode','rw','读写'),('storage_access_mode','ro','只读'),
('export_status','active','活跃'),('export_status','inactive','停用'),
('mount_status','unmounted','未挂载'),('mount_status','mounted','已挂载'),('mount_status','failed','失败'),
('mirror_source_type','public_mirror','公共镜像'),('mirror_source_type','private_registry','私有仓库'),('mirror_source_type','hub','官方Hub'),
('mirror_region','cn','中国'),('mirror_region','us','美国'),('mirror_region','eu','欧洲'),
('mirror_status','active','活跃'),('mirror_status','inactive','停用'),('mirror_status','error','异常'),
('registry_type','docker-registry','Docker Registry'),('registry_type','harbor','Harbor'),
('registry_status','deploying','部署中'),('registry_status','running','运行中'),('registry_status','failed','失败'),('registry_status','destroyed','已销毁'),
('image_type','vm','虚拟机镜像'),('image_type','base','基础镜像'),('image_type','app','应用镜像'),('image_type','custom','自定义镜像'),
('sync_status','pending','待同步'),('sync_status','syncing','同步中'),('sync_status','synced','已同步'),('sync_status','failed','失败'),
]
async def seed():
config = getConfig(CDIR)
db = DBPools(config.databases)
now = datetime.datetime.now().isoformat()
config=getConfig(CDIR)
db=DBPools(config.databases)
now=datetime.datetime.now().isoformat()
async with db.sqlorContext('pccs') as sor:
rows = await sor.R('appcodes_kv', {'parentid': 'cluster_type'})
rows=await sor.R('appcodes_kv',{'parentid':'cluster_type'})
if rows:
print(' appcode 已存在, 跳过')
print(' appcode exists, skip')
return
for parentid, k, v in APPCODES:
await sor.C('appcodes_kv', {
'id': getID(), 'parentid': parentid,
'k': k, 'v': v, 'created_at': now,
})
print(f' 已写入 {len(APPCODES)} 条 appcode')
for pid,k,v in CODES:
await sor.C('appcodes_kv',{'id':getID(),'parentid':pid,'k':k,'v':v,'created_at':now})
print(f' seeded {len(CODES)} codes')
asyncio.run(seed())
PYEOF
green " OK"
# ============================================================
# Phase 10: 创建 systemd 服务 + 启动脚本
# 9: 启动脚本
# ============================================================
blue "[10/10] 服务配置..."
# start.sh
cat > "${CDIR}/start.sh" <<STARTEOF
blue "[9/10] 启动脚本..."
cat > "${CDIR}/start.sh" <<SHEOF
#!/usr/bin/env bash
set -e
cd "${CDIR}"
source "${CDIR}/.env" 2>/dev/null || true
export PYTHONPATH="${PKGDIR}/ahserver:${PKGDIR}/apppublic:${PKGDIR}/sqlor:${PKGDIR}/appbase:${PKGDIR}/rbac:${PKGDIR}/bricks_for_python:\$PYTHONPATH"
exec "${VENV}/bin/python" "${CDIR}/app/${APPNAME}.py" 2>&1 | tee -a "${CDIR}/logs/${APPNAME}.log"
STARTEOF
exec "${VENV}/bin/python" "${CDIR}/app/pccs.py"
SHEOF
chmod +x "${CDIR}/start.sh"
# stop.sh
cat > "${CDIR}/stop.sh" <<STOPEOF
cat > "${CDIR}/stop.sh" <<SHEOF
#!/usr/bin/env bash
pkill -f "${CDIR}/app/${APPNAME}.py" 2>/dev/null && echo "pccs stopped" || echo "pccs not running"
STOPEOF
pkill -f "${CDIR}/app/pccs.py" 2>/dev/null && echo "stopped" || echo "not running"
SHEOF
chmod +x "${CDIR}/stop.sh"
# systemd service
# systemd
sudo tee /etc/systemd/system/pccs.service > /dev/null <<SERVEOF
[Unit]
Description=PCCS 算力中心集群系统
@ -322,37 +253,31 @@ ExecStart=${CDIR}/start.sh
ExecStop=${CDIR}/stop.sh
Restart=on-failure
RestartSec=5
StandardOutput=append:${CDIR}/logs/pccs.log
StandardError=append:${CDIR}/logs/pccs.log
[Install]
WantedBy=multi-user.target
SERVEOF
sudo systemctl daemon-reload
sudo systemctl enable pccs 2>/dev/null || true
green " OK"
# ============================================================
# 启动应用
# 10: 启动
# ============================================================
blue "启动应用..."
blue "[10/10] 启动..."
"${CDIR}/stop.sh" 2>/dev/null || true
sleep 1
nohup bash "${CDIR}/start.sh" > "${CDIR}/logs/nohup.log" 2>&1 &
sleep 3
sleep 4
if curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:${APP_PORT}/" 2>/dev/null | grep -q '200\|404'; then
green " pccs 已启动: http://127.0.0.1:${APP_PORT}"
else
red " WARN: 无响应, 检查日志: tail -f ${CDIR}/logs/pccs.log"
red " WARN: check log: tail -50 ${CDIR}/logs/nohup.log"
tail -20 "${CDIR}/logs/nohup.log" 2>/dev/null
fi
echo ""
green "=========================================="
green " 部署完成!"
green " 访问: http://$(hostname -I | awk '{print $1}'):${APP_PORT}"
green " 启动: sudo systemctl start pccs"
green " 停止: sudo systemctl stop pccs"
green " 日志: tail -f ${CDIR}/logs/pccs.log"
green " 部署完成! http://$(hostname -I | awk '{print $1}'):${APP_PORT}"
green "=========================================="