init: pcpool module skeleton — models, CRUD, API, load_path

This commit is contained in:
yumoqing 2026-08-07 11:23:18 +08:00
commit 76387d4323
16 changed files with 624 additions and 0 deletions

View File

@ -0,0 +1,59 @@
{
"tblname": "compute_node",
"alias": "compute_node_list",
"title": "算力节点",
"params": {
"browserfields": {
"pool_id": {
"title": "所属算力池",
"width": 150
},
"name": {
"title": "节点名称",
"width": 150
},
"ip_address": {
"title": "IP地址",
"width": 140
},
"cpu_cores": {
"title": "CPU核数",
"width": 80
},
"memory_gb": {
"title": "内存GB",
"width": 80
},
"gpu_count": {
"title": "GPU数",
"width": 70
},
"gpu_type": {
"title": "GPU型号",
"width": 100
},
"node_type": {
"title": "节点类型",
"width": 100
},
"status": {
"title": "状态",
"width": 100
}
},
"editexclouded": [
"id",
"resellerid",
"created_at",
"updated_at"
],
"toolbar": {
"tools": []
},
"binds": [],
"new_data_url": "{{entire_url('/pcpool/api/compute_node_create.dspy')}}",
"update_data_url": "{{entire_url('/pcpool/api/compute_node_update.dspy')}}",
"delete_data_url": "{{entire_url('/pcpool/api/compute_node_delete.dspy')}}",
"logined_userorgid": "resellerid"
}
}

View File

@ -0,0 +1,59 @@
{
"tblname": "compute_pool",
"alias": "compute_pool_list",
"title": "算力池",
"params": {
"browserfields": {
"name": {
"title": "池名称",
"width": 150
},
"pool_type": {
"title": "集群类型",
"width": 100
},
"total_cpu": {
"title": "总CPU核数",
"width": 100
},
"total_memory_gb": {
"title": "总内存GB",
"width": 100
},
"total_gpu": {
"title": "总GPU数",
"width": 80
},
"allocated_cpu": {
"title": "已分配CPU",
"width": 100
},
"allocated_memory_gb": {
"title": "已分配内存GB",
"width": 120
},
"allocated_gpu": {
"title": "已分配GPU",
"width": 100
},
"status": {
"title": "状态",
"width": 80
}
},
"editexclouded": [
"id",
"resellerid",
"created_at",
"updated_at"
],
"toolbar": {
"tools": []
},
"binds": [],
"new_data_url": "{{entire_url('/pcpool/api/compute_pool_create.dspy')}}",
"update_data_url": "{{entire_url('/pcpool/api/compute_pool_update.dspy')}}",
"delete_data_url": "{{entire_url('/pcpool/api/compute_pool_delete.dspy')}}",
"logined_userorgid": "resellerid"
}
}

129
models/compute_node.json Normal file
View File

@ -0,0 +1,129 @@
{
"summary": [
{
"name": "compute_node",
"title": "算力节点",
"primary": [
"id"
],
"catelog": "entity"
}
],
"fields": [
{
"name": "id",
"title": "id",
"type": "str",
"length": 32,
"nullable": "no"
},
{
"name": "pool_id",
"title": "所属算力池",
"type": "str",
"length": 32
},
{
"name": "name",
"title": "节点名称",
"type": "str",
"length": 128,
"nullable": "no"
},
{
"name": "ip_address",
"title": "IP地址",
"type": "str",
"length": 64,
"nullable": "no"
},
{
"name": "ssh_port",
"title": "SSH端口",
"type": "int",
"default": 22
},
{
"name": "ssh_user",
"title": "SSH用户",
"type": "str",
"length": 64
},
{
"name": "cpu_cores",
"title": "CPU核数",
"type": "int",
"default": 0
},
{
"name": "memory_gb",
"title": "内存GB",
"type": "int",
"default": 0
},
{
"name": "gpu_count",
"title": "GPU数量",
"type": "int",
"default": 0
},
{
"name": "gpu_type",
"title": "GPU型号",
"type": "str",
"length": 64
},
{
"name": "node_type",
"title": "节点类型(control/compute/storage)",
"type": "char",
"length": 16
},
{
"name": "status",
"title": "状态(available/allocated/maintenance/offline)",
"type": "char",
"length": 16,
"default": "available"
},
{
"name": "last_heartbeat",
"title": "最后心跳",
"type": "timestamp"
},
{
"name": "created_at",
"title": "创建时间",
"type": "timestamp",
"nullable": "no"
},
{
"name": "updated_at",
"title": "更新时间",
"type": "timestamp",
"nullable": "no"
}
],
"codes": [
{
"field": "pool_id",
"table": "compute_pool",
"valuefield": "id",
"textfield": "name"
},
{
"field": "node_type",
"table": "appcodes_kv",
"valuefield": "k",
"textfield": "v",
"cond": "parentid='node_type'"
},
{
"field": "status",
"table": "appcodes_kv",
"valuefield": "k",
"textfield": "v",
"cond": "parentid='node_status'"
}
]
}

119
models/compute_pool.json Normal file
View File

@ -0,0 +1,119 @@
{
"summary": [
{
"name": "compute_pool",
"title": "算力池",
"primary": [
"id"
],
"catelog": "entity"
}
],
"fields": [
{
"name": "id",
"title": "id",
"type": "str",
"length": 32,
"nullable": "no"
},
{
"name": "resellerid",
"title": "商户机构id",
"type": "str",
"length": 32,
"nullable": "no"
},
{
"name": "name",
"title": "池名称",
"type": "str",
"length": 128,
"nullable": "no"
},
{
"name": "pool_type",
"title": "集群类型(k8s/slurm/ray)",
"type": "char",
"length": 16,
"nullable": "no"
},
{
"name": "description",
"title": "描述",
"type": "str",
"length": 512
},
{
"name": "total_cpu",
"title": "总CPU核数",
"type": "int",
"default": 0
},
{
"name": "total_memory_gb",
"title": "总内存GB",
"type": "int",
"default": 0
},
{
"name": "total_gpu",
"title": "总GPU数",
"type": "int",
"default": 0
},
{
"name": "allocated_cpu",
"title": "已分配CPU",
"type": "int",
"default": 0
},
{
"name": "allocated_memory_gb",
"title": "已分配内存GB",
"type": "int",
"default": 0
},
{
"name": "allocated_gpu",
"title": "已分配GPU",
"type": "int",
"default": 0
},
{
"name": "status",
"title": "状态(1=启用 0=停用)",
"type": "char",
"length": 1,
"default": "1"
},
{
"name": "created_at",
"title": "创建时间",
"type": "timestamp",
"nullable": "no"
},
{
"name": "updated_at",
"title": "更新时间",
"type": "timestamp",
"nullable": "no"
}
],
"codes": [
{
"field": "pool_type",
"table": "appcodes_kv",
"valuefield": "k",
"textfield": "v",
"cond": "parentid='cluster_type'"
},
{
"field": "status",
"table": "appcodes_kv",
"valuefield": "k",
"textfield": "v",
"cond": "parentid='product_status'"
}
]
}

149
pcpool/__init__.py Normal file
View File

@ -0,0 +1,149 @@
"""
pcpool 算力中心算力池管理
- 算力节点池 CRUD
- 节点注册/注销/心跳
- 算力单元分配和回收
- 池资源统计
"""
import datetime
from appPublic.uniqueID import getID
from sqlor.dbpools import DBPools
MODULE_NAME = 'pccs'
async def node_heartbeat(request, params_kw):
"""节点心跳上报,更新 last_heartbeat"""
env = request._run_ns
node_id = params_kw.get('id') or params_kw.get('node_id', '')
if not node_id:
return {'status': 'error', 'message': 'Missing node id'}
dbname = env.get_module_dbname(MODULE_NAME)
now = datetime.datetime.now().isoformat()
async with DBPools().sqlorContext(dbname) as sor:
await sor.U('compute_node', {'id': node_id},
{'last_heartbeat': now, 'updated_at': now})
return {'status': 'ok'}
async def allocate_nodes(request, params_kw):
"""从算力池中分配节点给集群。
params: pool_id, cluster_id, role (control/compute), count, cpu, memory, gpu
返回分配的节点 ID 列表"""
env = request._run_ns
dbname = env.get_module_dbname(MODULE_NAME)
pool_id = params_kw.get('pool_id', '')
cluster_id = params_kw.get('cluster_id', '')
role = params_kw.get('role', 'compute')
count = int(params_kw.get('count', 1))
req_cpu = int(params_kw.get('cpu', 0))
req_mem = int(params_kw.get('memory', 0))
req_gpu = int(params_kw.get('gpu', 0))
if not pool_id:
return {'status': 'error', 'message': 'Missing pool_id'}
async with DBPools().sqlorContext(dbname) as sor:
# 查询可用节点
sql = """SELECT * FROM compute_node
WHERE pool_id = ${pool_id}$ AND status = 'available'
ORDER BY cpu_cores DESC, memory_gb DESC
LIMIT ${limit}$"""
available = await sor.sqlExe(sql, {'pool_id': pool_id, 'limit': count})
if len(available) < count:
return {'status': 'error',
'message': f'可用节点不足: 需要{count}, 可用{len(available)}'}
allocated = []
for node in available[:count]:
await sor.U('compute_node', {'id': node.id},
{'status': 'allocated', 'updated_at': datetime.datetime.now().isoformat()})
# 记录分配关系
cn_id = getID()
await sor.C('cluster_node', {
'id': cn_id, 'cluster_id': cluster_id,
'node_id': node.id, 'role': role, 'status': 'joining',
'assigned_at': datetime.datetime.now().isoformat()
})
allocated.append(node.id)
# 更新池已分配资源
if allocated:
await sor.sqlExe(
"""UPDATE compute_pool
SET allocated_cpu = allocated_cpu + ${cpu}$,
allocated_memory_gb = allocated_memory_gb + ${mem}$,
allocated_gpu = allocated_gpu + ${gpu}$,
updated_at = ${now}$
WHERE id = ${pool_id}$""",
{'cpu': req_cpu * len(allocated), 'mem': req_mem * len(allocated),
'gpu': req_gpu * len(allocated), 'pool_id': pool_id,
'now': datetime.datetime.now().isoformat()}
)
return {'status': 'ok', 'allocated_nodes': allocated}
async def release_nodes(request, params_kw):
"""回收节点回算力池。
params: node_ids (逗号分隔), pool_id"""
env = request._run_ns
dbname = env.get_module_dbname(MODULE_NAME)
node_ids_str = params_kw.get('node_ids', '')
if not node_ids_str:
return {'status': 'error', 'message': 'Missing node_ids'}
node_ids = [n.strip() for n in node_ids_str.split(',') if n.strip()]
pool_id = params_kw.get('pool_id', '')
now = datetime.datetime.now().isoformat()
async with DBPools().sqlorContext(dbname) as sor:
for nid in node_ids:
await sor.U('compute_node', {'id': nid},
{'status': 'available', 'updated_at': now})
# 清除分配记录
await sor.sqlExe(
"UPDATE cluster_node SET status='removed' WHERE node_id=${nid}$",
{'nid': nid})
# 回收资源计数
node_row = await sor.R('compute_node', {'id': nid})
if node_row:
n = node_row[0]
await sor.sqlExe(
"""UPDATE compute_pool
SET allocated_cpu = GREATEST(allocated_cpu - ${cpu}$, 0),
allocated_memory_gb = GREATEST(allocated_memory_gb - ${mem}$, 0),
allocated_gpu = GREATEST(allocated_gpu - ${gpu}$, 0),
updated_at = ${now}$
WHERE id = ${pool_id}$""",
{'cpu': n.cpu_cores, 'mem': n.memory_gb, 'gpu': n.gpu_count,
'pool_id': pool_id or n.pool_id, 'now': now})
return {'status': 'ok', 'released': len(node_ids)}
async def pool_stats(request, params_kw):
"""获取池资源统计"""
env = request._run_ns
dbname = env.get_module_dbname(MODULE_NAME)
pool_id = params_kw.get('pool_id', '')
async with DBPools().sqlorContext(dbname) as sor:
if pool_id:
pools = await sor.R('compute_pool', {'id': pool_id})
else:
pools = await sor.R('compute_pool', {})
if not pools:
return {'status': 'ok', 'data': []}
result = []
for p in pools:
nodes = await sor.R('compute_node', {'pool_id': p.id})
result.append({
'id': p.id, 'name': p.name, 'type': p.pool_type,
'total_cpu': p.total_cpu, 'allocated_cpu': p.allocated_cpu,
'total_memory_gb': p.total_memory_gb, 'allocated_memory_gb': p.allocated_memory_gb,
'total_gpu': p.total_gpu, 'allocated_gpu': p.allocated_gpu,
'node_count': len(nodes),
'available_nodes': sum(1 for n in nodes if n.status == 'available'),
'status': p.status
})
return {'status': 'ok', 'data': result}

68
scripts/load_path.py Normal file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""pcpool RBAC 权限管理"""
import subprocess, os, sys, json, glob
def find_sage_root():
for c in [os.path.expanduser("~/sage"), os.path.expanduser("~/repos/sage")]:
if os.path.isdir(os.path.join(c, "py3")): return c
return None
SAGE = find_sage_root()
if not SAGE: sys.exit("Sage root not found")
PY = os.path.join(SAGE, "py3", "bin", "python")
SET = os.path.join(SAGE, "set_role_perm.py")
JSON_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "json")
API_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "wwwroot", "api")
def load_cruds():
defs = []
for f in sorted(glob.glob(os.path.join(JSON_DIR, "*.json"))):
try:
d = json.load(open(f))
defs.append({"tblname": d["tblname"], "alias": d.get("alias", d["tblname"]), "subtables": d.get("params", {}).get("subtables", [])})
except: pass
return defs
def get_apis():
if not os.path.isdir(API_DIR): return []
return [f"/pcpool/api/{f}" for f in sorted(os.listdir(API_DIR)) if f.endswith(".dspy")]
cruds = load_cruds()
apis = get_apis()
PATHS_ANY = [
f"/pcpool/menu.ui",
]
PATHS_LOGINED = [
f"/pcpool",
f"/pcpool/index.ui",
]
for d in cruds:
PATHS_ANY.append(f"/pcpool/{d['alias']}")
PATHS_LOGINED.append(f"/pcpool/{d['alias']}/index.ui")
for act in ["get", "add", "update", "delete"]:
PATHS_LOGINED.append(f"/pcpool/{d['alias']}/{act}_{d['tblname']}.dspy")
for api in apis:
PATHS_LOGINED.append(api)
PATHS_OPERATOR = list(PATHS_LOGINED)
PATHS_ANY = list(dict.fromkeys(PATHS_ANY))
PATHS_LOGINED = list(dict.fromkeys(PATHS_LOGINED))
PATHS_OPERATOR = list(dict.fromkeys(PATHS_OPERATOR))
def reg(role, paths):
ok = 0
for p in paths:
r = subprocess.run([PY, SET, role, p], capture_output=True, text=True)
if r.returncode == 0: ok += 1
print(f" {role}: {ok}/{len(paths)}")
return ok
total = 0
print(f"{mod_name}: any={len(PATHS_ANY)} logined={len(PATHS_LOGINED)} operator={len(PATHS_OPERATOR)}")
total += reg("any", PATHS_ANY)
total += reg("logined", PATHS_LOGINED)
total += reg("reseller.operator", PATHS_OPERATOR)
print(f"Done. {total} entries.")

8
setup.json Normal file
View File

@ -0,0 +1,8 @@
{
"name": "pcpool",
"version": "0.1.0",
"description": "算力中心算力池管理节点池CRUD、节点注册/注销、算力单元分配回收",
"packages": ["pcpool"],
"install_requires": ["apppublic", "sqlor", "ahserver", "appbase"],
"python_requires": ">=3.10"
}

View File

@ -0,0 +1,3 @@
# compute_node create
result = {'status': 'ok', 'message': 'compute_node created'}
return result

View File

@ -0,0 +1,3 @@
# compute_node delete
result = {'status': 'ok', 'message': 'compute_node deleted'}
return result

View File

@ -0,0 +1,3 @@
# compute_node update
result = {'status': 'ok', 'message': 'compute_node updated'}
return result

View File

@ -0,0 +1,3 @@
# compute_pool create
result = {'status': 'ok', 'message': 'compute_pool created'}
return result

View File

@ -0,0 +1,3 @@
# compute_pool delete
result = {'status': 'ok', 'message': 'compute_pool deleted'}
return result

View File

@ -0,0 +1,3 @@
# compute_pool update
result = {'status': 'ok', 'message': 'compute_pool updated'}
return result

View File

@ -0,0 +1,4 @@
# 从算力池分配节点到集群
# params: pool_id, cluster_id, role(control/compute), count, cpu, memory, gpu
result = await allocate_nodes(request, params_kw)
return result if isinstance(result, dict) else {'status': 'error', 'message': str(result)}

View File

@ -0,0 +1,8 @@
# 节点心跳上报
node_id = params_kw.get('id', '')
if not node_id:
return {'status': 'error', 'message': 'Missing node id'}
import datetime
async with DBPools().sqlorContext('pccs') as sor:
await sor.U('compute_node', {'id': node_id}, {'last_heartbeat': datetime.datetime.now().isoformat(), 'updated_at': datetime.datetime.now().isoformat()})
return {'status': 'ok'}

View File

@ -0,0 +1,3 @@
# 回收节点回算力池
result = await release_nodes(request, params_kw)
return result if isinstance(result, dict) else {'status': 'error', 'message': str(result)}