diff --git a/build.sh b/build.sh index f240a85..c132a9d 100644 --- a/build.sh +++ b/build.sh @@ -1,313 +1,244 @@ #!/usr/bin/env bash # ============================================================ -# pccs build.sh — 一键部署脚本 (符合 web-application-spec) +# pccs build.sh — 一键部署 (符合部署目录规范) # -# 在部署节点上运行: -# cd /d/pccs/repos/pccs && bash build.sh +# 目标结构: +# /d/pccs/ +# ├── app/ ← 应用入口 +# ├── conf/ ← config.json +# ├── pkgs/ ← 所有模块 clone +# ├── wwwroot/ ← 软链接 +# ├── py3/ ← venv +# ├── logs/ +# ├── files/ +# └── scripts/ # -# 前提: -# - MariaDB 已安装, pccs 库已创建, test/test123 已授权 -# - /d 已作为 NFS 共享盘导出 -# - git, python3.10+, pip, npm 已安装 +# 运行: cd /d/pccs && bash build.sh # ============================================================ set -e -# --------------- 配置 --------------- -WORKDIR="$(cd "$(dirname "$0")/../.." && pwd)" -CDIR="${WORKDIR}" # 应用根目录 (/d/pccs) -PKGDIR="${CDIR}/repos" # 模块 clone 目录 -VENV="${CDIR}/venv" +# --------------- 路径 --------------- +# 假设 build.sh 放在 /d/pccs/build.sh (从 pccs repo 复制) +CDIR="$(cd "$(dirname "$0")" && pwd)" # /d/pccs +PKGDIR="${CDIR}/pkgs" +VENV="${CDIR}/py3" APPNAME="pccs" APP_PORT=9180 - -# Git 仓库基地址 GIT_BASE="https://git.opencomputing.cn/yumoqing" -# --------------- 颜色 --------------- red() { echo -e "\033[31m$1\033[0m"; } green() { echo -e "\033[32m$1\033[0m"; } blue() { echo -e "\033[34m$1\033[0m"; } blue "==========================================" -blue " pccs — 算力中心集群系统 部署" +blue " pccs 部署 — ${CDIR}" blue "==========================================" # ============================================================ -# Phase 0: 创建目录 +# 0: 目录 # ============================================================ mkdir -p "${CDIR}/app" "${CDIR}/conf" "${CDIR}/files" "${CDIR}/logs" \ - "${CDIR}/wwwroot/imgs" "${PKGDIR}" + "${CDIR}/wwwroot/imgs" "${CDIR}/scripts" "${PKGDIR}" + +# 复制 app/pccs.py 和 conf/config.json 到顶层 +if [ -d "${PKGDIR}/pccs" ]; then + cp -f "${PKGDIR}/pccs/app/pccs.py" "${CDIR}/app/pccs.py" 2>/dev/null || true + cp -f "${PKGDIR}/pccs/conf/config.json" "${CDIR}/conf/config.json" 2>/dev/null || true +fi # ============================================================ -# Phase 1: 安装系统依赖 +# 1: 系统依赖 # ============================================================ blue "[1/10] 系统依赖..." sudo apt-get update -qq -sudo apt-get install -y -qq python3-venv python3-pip python3-dev \ - build-essential nfs-common +sudo apt-get install -y -qq python3-venv python3-pip python3-dev build-essential nfs-common green " OK" # ============================================================ -# Phase 2: Python 虚拟环境 +# 2: venv # ============================================================ blue "[2/10] Python 虚拟环境..." python3 -m venv "${VENV}" --clear PIP="${VENV}/bin/pip" PYTHON="${VENV}/bin/python" -"${PIP}" install --upgrade pip setuptools wheel -q -# 配置 pip 国内镜像 (阿里云) "${PIP}" config set global.index-url https://mirrors.aliyun.com/pypi/simple/ 2>/dev/null || true "${PIP}" install --upgrade pip setuptools wheel -q -# 安装 xls2ddl (DDL 生成工具) -"${PIP}" install xls2ddl -q --timeout=30 2>/dev/null || { - red " WARN: xls2ddl install failed (PyPI unreachable?), skip" -} -green " OK" +"${PIP}" install xls2ddl -q --timeout=30 2>/dev/null || red " WARN: xls2ddl skip" +green " OK: ${VENV}" # ============================================================ -# Phase 3: Clone/Pull 所有模块 +# 3: Clone/Pull # ============================================================ -blue "[3/10] 同步代码仓库..." -ALL_REPOS=( - # 基础层 - apppublic sqlor ahserver - # 数据库层 - appbase rbac - # 前端 - bricks bricks_for_python - # 业务模块 - pcpool pcc storage_mgr image_mgr - # 主应用 - pccs -) -for repo in "${ALL_REPOS[@]}"; do +blue "[3/10] 同步仓库..." +ALL=(apppublic sqlor ahserver appbase rbac bricks bricks_for_python + pcpool pcc storage_mgr image_mgr pccs) +for repo in "${ALL[@]}"; do if [ -d "${PKGDIR}/${repo}/.git" ]; then (cd "${PKGDIR}/${repo}" && git pull origin main 2>/dev/null) || true else echo " clone ${repo}..." - git clone "${GIT_BASE}/${repo}.git" "${PKGDIR}/${repo}" 2>/dev/null || { - red " WARN: ${repo} clone failed" - } + git clone "${GIT_BASE}/${repo}.git" "${PKGDIR}/${repo}" 2>/dev/null || red " WARN: ${repo}" fi done + +# 确保 app/conf 文件存在 +cp -f "${PKGDIR}/pccs/app/pccs.py" "${CDIR}/app/pccs.py" 2>/dev/null || true +cp -f "${PKGDIR}/pccs/conf/config.json" "${CDIR}/conf/config.json" 2>/dev/null || true + +# 更新 config.json 路径占位符 +if [ -f "${CDIR}/conf/config.json" ]; then + sed -i "s|\$\[workdir\]\$|${CDIR}|g" "${CDIR}/conf/config.json" +fi green " OK" # ============================================================ -# Phase 4: pip install 所有模块 +# 4: pip install # ============================================================ -blue "[4/10] 安装 Python 模块..." -MODULES=( - apppublic sqlor ahserver - appbase rbac - bricks_for_python - pcpool pcc storage_mgr image_mgr -) +blue "[4/10] 安装模块..." +MODULES=(apppublic sqlor ahserver appbase rbac bricks_for_python + pcpool pcc storage_mgr image_mgr) for m in "${MODULES[@]}"; do mp="${PKGDIR}/${m}" - if [ -f "${mp}/setup.py" ] || [ -f "${mp}/setup.cfg" ] || [ -f "${mp}/pyproject.toml" ]; then - "${PIP}" install -e "${mp}" -q 2>&1 | tail -1 - fi + [ -f "${mp}/setup.py" ] || [ -f "${mp}/setup.cfg" ] || [ -f "${mp}/pyproject.toml" ] || continue + "${PIP}" install -e "${mp}" -q 2>&1 | tail -1 done green " OK" # ============================================================ -# Phase 5: 数据库 DDL — json2ddl 生成建表 SQL +# 5: DDL # ============================================================ -blue "[5/10] 数据库 DDL 生成..." -# 收集所有有 models/ 的模块 -DB_MODULES=(appbase rbac pcpool pcc storage_mgr image_mgr) - -for m in "${DB_MODULES[@]}"; do +blue "[5/10] DDL 建表..." +for m in appbase rbac pcpool pcc storage_mgr image_mgr; do mdir="${PKGDIR}/${m}/models" - if [ -d "$mdir" ]; then - has_json=$(ls "$mdir"/*.json 2>/dev/null | wc -l) - has_xlsx=$(ls "$mdir"/*.xlsx 2>/dev/null | wc -l) - if [ "$has_json" -gt 0 ]; then - echo " [${m}] json2ddl → DDL" - (cd "$mdir" && "${VENV}/bin/json2ddl" mysql . > /tmp/pccs_${m}.ddl.sql 2>/dev/null) || true - mysql -u test -ptest123 pccs < /tmp/pccs_${m}.ddl.sql 2>/dev/null || true + [ -d "$mdir" ] || continue + for jsonf in "$mdir"/*.json; do + [ -f "$jsonf" ] || continue + tbl=$(python3 -c "import json;d=json.load(open('$jsonf'));print(d['summary'][0]['name'])" 2>/dev/null) || continue + exists=$(mysql -u test -ptest123 pccs -sNe "SHOW TABLES LIKE '$tbl'" 2>/dev/null) + if [ -z "$exists" ]; then + echo " [${m}] create table ${tbl}..." + "${PYTHON}" -c " +import sys,json +sys.path[:0]=['${PKGDIR}/apppublic','${PKGDIR}/sqlor','${PKGDIR}/ahserver'] +from appPublic.jsonConfig import getConfig +from sqlor.dbpools import DBPools +from sqlor.ddl_template_mysql import DDLTemplateMySQL +import asyncio +async def go(): + config=getConfig('${CDIR}') + db=DBPools(config.databases) + ddl=DDLTemplateMySQL() + m=json.load(open('${jsonf}')) + sql=ddl.create_table(m['summary'][0]['name'],m.get('fields',[])) + async with db.sqlorContext('pccs') as s: + await s.sqlExe(sql,{}) + print(' done') +asyncio.run(go()) +" 2>&1 | tail -1 fi - if [ "$has_xlsx" -gt 0 ]; then - echo " [${m}] xls2ddl → DDL" - (cd "$mdir" && xls2ddl mysql . > /tmp/pccs_${m}_xlsx.ddl.sql 2>/dev/null) || true - mysql -u test -ptest123 pccs < /tmp/pccs_${m}_xlsx.ddl.sql 2>/dev/null || true - fi - fi + done done green " OK" # ============================================================ -# Phase 6: 导入初始数据 (data/ 目录) +# 6: xls2ui # ============================================================ -blue "[6/10] 初始数据导入..." -for m in "${DB_MODULES[@]}"; do - ddir="${PKGDIR}/${m}/data" - if [ -d "$ddir" ]; then - for xlsx in "$ddir"/*.xlsx; do - [ -f "$xlsx" ] || continue - echo " [${m}] dbloader → ${xlsx}" - "${VENV}/bin/dbloader" "${CDIR}" pccs "$xlsx" 2>/dev/null || true - done - fi -done -green " OK" - -# ============================================================ -# Phase 7: CRUD 生成 — xls2ui -# ============================================================ -blue "[7/10] CRUD 界面生成..." -for m in "${DB_MODULES[@]}"; do +blue "[6/10] CRUD 生成..." +for m in appbase rbac pcpool pcc storage_mgr image_mgr; do jdir="${PKGDIR}/${m}/json" mdir="${PKGDIR}/${m}/models" odir="${PKGDIR}/${m}/wwwroot" - if [ -d "$jdir" ] && [ -d "$mdir" ]; then - echo " [${m}] xls2ui..." - (cd "$jdir" && "${VENV}/bin/xls2ui" -m "$mdir" -o "$odir" "$m" *.json 2>/dev/null) || true - fi + [ -d "$jdir" ] && [ -d "$mdir" ] || continue + echo " [${m}] xls2ui..." + (cd "$jdir" && "${VENV}/bin/xls2ui" -m "$mdir" -o "$odir" "$m" *.json 2>/dev/null) || true done green " OK" # ============================================================ -# Phase 8: 创建 wwwroot 软链接 +# 7: wwwroot symlinks # ============================================================ -blue "[8/10] 创建 wwwroot 软链接..." -for m in "${DB_MODULES[@]}"; do - wdir="${PKGDIR}/${m}/wwwroot" - if [ -d "$wdir" ]; then - ln -sfn "$wdir" "${CDIR}/wwwroot/${m}" 2>/dev/null || true - fi +blue "[7/10] 软链接..." +for m in appbase rbac pcpool pcc storage_mgr image_mgr; do + [ -d "${PKGDIR}/${m}/wwwroot" ] || continue + ln -sfn "${PKGDIR}/${m}/wwwroot" "${CDIR}/wwwroot/${m}" 2>/dev/null || true done -# pccs 自身的 wwwroot -ln -sfn "${PKGDIR}/pccs/wwwroot" "${CDIR}/wwwroot/${APPNAME}" 2>/dev/null || true - -# Bricks 前端 +ln -sfn "${PKGDIR}/pccs/wwwroot" "${CDIR}/wwwroot/pccs" 2>/dev/null || true +# bricks if [ -f "${PKGDIR}/bricks/build.sh" ]; then - blue "[bricks] 构建前端..." - (cd "${PKGDIR}/bricks" && npm install --silent 2>/dev/null && bash build.sh 2>&1 | tail -3) || true + (cd "${PKGDIR}/bricks" && npm install --silent 2>/dev/null && bash build.sh 2>&1 | tail -2) || true ln -sfn "${PKGDIR}/bricks/dist" "${CDIR}/wwwroot/bricks" 2>/dev/null || true fi green " OK" # ============================================================ -# Phase 9: 种子应用数据 (appcode / RBAC) +# 8: 种子数据 # ============================================================ -blue "[9/10] 种子数据..." +blue "[8/10] 种子数据..." "${PYTHON}" - "${CDIR}" "${PKGDIR}" <<'PYEOF' -import sys, os, json - -CDIR = sys.argv[1] -PKGDIR = sys.argv[2] - -sys.path.insert(0, os.path.join(PKGDIR, 'apppublic')) -sys.path.insert(0, os.path.join(PKGDIR, 'sqlor')) -sys.path.insert(0, os.path.join(PKGDIR, 'ahserver')) - +import sys,os,json,asyncio,datetime +CDIR=sys.argv[1]; PKGDIR=sys.argv[2] +sys.path[:0]=[os.path.join(PKGDIR,'apppublic'),os.path.join(PKGDIR,'sqlor'),os.path.join(PKGDIR,'ahserver')] from appPublic.jsonConfig import getConfig from sqlor.dbpools import DBPools from appPublic.uniqueID import getID -import datetime, asyncio -APPCODES = [ - ('cluster_type', 'k8s', 'Kubernetes集群'), - ('cluster_type', 'slurm', 'Slurm集群'), - ('cluster_type', 'ray', 'Ray集群'), - ('cluster_status', 'deploying', '部署中'), - ('cluster_status', 'running', '运行中'), - ('cluster_status', 'failed', '失败'), - ('cluster_status', 'stopped', '已停止'), - ('cluster_status', 'destroyed', '已销毁'), - ('node_role', 'control', '控制节点'), - ('node_role', 'compute', '算力节点'), - ('node_role', 'storage', '存储节点'), - ('node_status', 'joining', '加入中'), - ('node_status', 'active', '活跃'), - ('node_status', 'draining', '排空中'), - ('node_status', 'removed', '已移除'), - ('node_type', 'control', '控制节点'), - ('node_type', 'compute', '算力节点'), - ('node_type', 'storage', '存储节点'), - ('storage_type', 'nfs', 'NFS'), - ('storage_type', 'glusterfs', 'GlusterFS'), - ('storage_type', 'cephfs', 'CephFS'), - ('storage_status', 'online', '在线'), - ('storage_status', 'offline', '离线'), - ('storage_status', 'maintenance', '维护中'), - ('storage_access_mode', 'rw', '读写'), - ('storage_access_mode', 'ro', '只读'), - ('export_status', 'active', '活跃'), - ('export_status', 'inactive', '停用'), - ('mount_status', 'unmounted', '未挂载'), - ('mount_status', 'mounted', '已挂载'), - ('mount_status', 'failed', '失败'), - ('mirror_source_type', 'public_mirror', '公共镜像'), - ('mirror_source_type', 'private_registry', '私有仓库'), - ('mirror_source_type', 'hub', '官方Hub'), - ('mirror_region', 'cn', '中国'), - ('mirror_region', 'us', '美国'), - ('mirror_region', 'eu', '欧洲'), - ('mirror_status', 'active', '活跃'), - ('mirror_status', 'inactive', '停用'), - ('mirror_status', 'error', '异常'), - ('registry_type', 'docker-registry', 'Docker Registry'), - ('registry_type', 'harbor', 'Harbor'), - ('registry_status', 'deploying', '部署中'), - ('registry_status', 'running', '运行中'), - ('registry_status', 'failed', '失败'), - ('registry_status', 'destroyed', '已销毁'), - ('image_type', 'vm', '虚拟机镜像'), - ('image_type', 'base', '基础镜像'), - ('image_type', 'app', '应用镜像'), - ('image_type', 'custom', '自定义镜像'), - ('sync_status', 'pending', '待同步'), - ('sync_status', 'syncing', '同步中'), - ('sync_status', 'synced', '已同步'), - ('sync_status', 'failed', '失败'), +CODES=[ + ('cluster_type','k8s','Kubernetes集群'),('cluster_type','slurm','Slurm集群'),('cluster_type','ray','Ray集群'), + ('cluster_status','deploying','部署中'),('cluster_status','running','运行中'),('cluster_status','failed','失败'), + ('cluster_status','stopped','已停止'),('cluster_status','destroyed','已销毁'), + ('node_role','control','控制节点'),('node_role','compute','算力节点'),('node_role','storage','存储节点'), + ('node_status','joining','加入中'),('node_status','active','活跃'),('node_status','draining','排空中'),('node_status','removed','已移除'), + ('node_type','control','控制节点'),('node_type','compute','算力节点'),('node_type','storage','存储节点'), + ('storage_type','nfs','NFS'),('storage_type','glusterfs','GlusterFS'),('storage_type','cephfs','CephFS'), + ('storage_status','online','在线'),('storage_status','offline','离线'),('storage_status','maintenance','维护中'), + ('storage_access_mode','rw','读写'),('storage_access_mode','ro','只读'), + ('export_status','active','活跃'),('export_status','inactive','停用'), + ('mount_status','unmounted','未挂载'),('mount_status','mounted','已挂载'),('mount_status','failed','失败'), + ('mirror_source_type','public_mirror','公共镜像'),('mirror_source_type','private_registry','私有仓库'),('mirror_source_type','hub','官方Hub'), + ('mirror_region','cn','中国'),('mirror_region','us','美国'),('mirror_region','eu','欧洲'), + ('mirror_status','active','活跃'),('mirror_status','inactive','停用'),('mirror_status','error','异常'), + ('registry_type','docker-registry','Docker Registry'),('registry_type','harbor','Harbor'), + ('registry_status','deploying','部署中'),('registry_status','running','运行中'),('registry_status','failed','失败'),('registry_status','destroyed','已销毁'), + ('image_type','vm','虚拟机镜像'),('image_type','base','基础镜像'),('image_type','app','应用镜像'),('image_type','custom','自定义镜像'), + ('sync_status','pending','待同步'),('sync_status','syncing','同步中'),('sync_status','synced','已同步'),('sync_status','failed','失败'), ] async def seed(): - config = getConfig(CDIR) - db = DBPools(config.databases) - now = datetime.datetime.now().isoformat() + config=getConfig(CDIR) + db=DBPools(config.databases) + now=datetime.datetime.now().isoformat() async with db.sqlorContext('pccs') as sor: - rows = await sor.R('appcodes_kv', {'parentid': 'cluster_type'}) + rows=await sor.R('appcodes_kv',{'parentid':'cluster_type'}) if rows: - print(' appcode 已存在, 跳过') + print(' appcode exists, skip') return - for parentid, k, v in APPCODES: - await sor.C('appcodes_kv', { - 'id': getID(), 'parentid': parentid, - 'k': k, 'v': v, 'created_at': now, - }) - print(f' 已写入 {len(APPCODES)} 条 appcode') - + for pid,k,v in CODES: + await sor.C('appcodes_kv',{'id':getID(),'parentid':pid,'k':k,'v':v,'created_at':now}) + print(f' seeded {len(CODES)} codes') asyncio.run(seed()) PYEOF green " OK" # ============================================================ -# Phase 10: 创建 systemd 服务 + 启动脚本 +# 9: 启动脚本 # ============================================================ -blue "[10/10] 服务配置..." - -# start.sh -cat > "${CDIR}/start.sh" < "${CDIR}/start.sh" </dev/null || true export PYTHONPATH="${PKGDIR}/ahserver:${PKGDIR}/apppublic:${PKGDIR}/sqlor:${PKGDIR}/appbase:${PKGDIR}/rbac:${PKGDIR}/bricks_for_python:\$PYTHONPATH" -exec "${VENV}/bin/python" "${CDIR}/app/${APPNAME}.py" 2>&1 | tee -a "${CDIR}/logs/${APPNAME}.log" -STARTEOF +exec "${VENV}/bin/python" "${CDIR}/app/pccs.py" +SHEOF chmod +x "${CDIR}/start.sh" -# stop.sh -cat > "${CDIR}/stop.sh" < "${CDIR}/stop.sh" </dev/null && echo "pccs stopped" || echo "pccs not running" -STOPEOF +pkill -f "${CDIR}/app/pccs.py" 2>/dev/null && echo "stopped" || echo "not running" +SHEOF chmod +x "${CDIR}/stop.sh" -# systemd service +# systemd sudo tee /etc/systemd/system/pccs.service > /dev/null </dev/null || true green " OK" # ============================================================ -# 启动应用 +# 10: 启动 # ============================================================ -blue "启动应用..." +blue "[10/10] 启动..." "${CDIR}/stop.sh" 2>/dev/null || true sleep 1 nohup bash "${CDIR}/start.sh" > "${CDIR}/logs/nohup.log" 2>&1 & -sleep 3 +sleep 4 if curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:${APP_PORT}/" 2>/dev/null | grep -q '200\|404'; then green " pccs 已启动: http://127.0.0.1:${APP_PORT}" else - red " WARN: 无响应, 检查日志: tail -f ${CDIR}/logs/pccs.log" + red " WARN: check log: tail -50 ${CDIR}/logs/nohup.log" + tail -20 "${CDIR}/logs/nohup.log" 2>/dev/null fi echo "" green "==========================================" -green " 部署完成!" -green " 访问: http://$(hostname -I | awk '{print $1}'):${APP_PORT}" -green " 启动: sudo systemctl start pccs" -green " 停止: sudo systemctl stop pccs" -green " 日志: tail -f ${CDIR}/logs/pccs.log" +green " 部署完成! http://$(hostname -I | awk '{print $1}'):${APP_PORT}" green "=========================================="