From 76387d4323c57d61b344d4110029e8d424a83a39 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Fri, 7 Aug 2026 11:23:18 +0800 Subject: [PATCH] =?UTF-8?q?init:=20pcpool=20module=20skeleton=20=E2=80=94?= =?UTF-8?q?=20models,=20CRUD,=20API,=20load=5Fpath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- json/compute_node_list.json | 59 +++++++++++ json/compute_pool_list.json | 59 +++++++++++ models/compute_node.json | 129 +++++++++++++++++++++++ models/compute_pool.json | 119 +++++++++++++++++++++ pcpool/__init__.py | 149 +++++++++++++++++++++++++++ scripts/load_path.py | 68 ++++++++++++ setup.json | 8 ++ wwwroot/api/compute_node_create.dspy | 3 + wwwroot/api/compute_node_delete.dspy | 3 + wwwroot/api/compute_node_update.dspy | 3 + wwwroot/api/compute_pool_create.dspy | 3 + wwwroot/api/compute_pool_delete.dspy | 3 + wwwroot/api/compute_pool_update.dspy | 3 + wwwroot/api/node_allocate.dspy | 4 + wwwroot/api/node_heartbeat.dspy | 8 ++ wwwroot/api/node_release.dspy | 3 + 16 files changed, 624 insertions(+) create mode 100644 json/compute_node_list.json create mode 100644 json/compute_pool_list.json create mode 100644 models/compute_node.json create mode 100644 models/compute_pool.json create mode 100644 pcpool/__init__.py create mode 100644 scripts/load_path.py create mode 100644 setup.json create mode 100644 wwwroot/api/compute_node_create.dspy create mode 100644 wwwroot/api/compute_node_delete.dspy create mode 100644 wwwroot/api/compute_node_update.dspy create mode 100644 wwwroot/api/compute_pool_create.dspy create mode 100644 wwwroot/api/compute_pool_delete.dspy create mode 100644 wwwroot/api/compute_pool_update.dspy create mode 100644 wwwroot/api/node_allocate.dspy create mode 100644 wwwroot/api/node_heartbeat.dspy create mode 100644 wwwroot/api/node_release.dspy diff --git a/json/compute_node_list.json b/json/compute_node_list.json new file mode 100644 index 0000000..65b8592 --- /dev/null +++ b/json/compute_node_list.json @@ -0,0 +1,59 @@ +{ + "tblname": "compute_node", + "alias": "compute_node_list", + "title": "算力节点", + "params": { + "browserfields": { + "pool_id": { + "title": "所属算力池", + "width": 150 + }, + "name": { + "title": "节点名称", + "width": 150 + }, + "ip_address": { + "title": "IP地址", + "width": 140 + }, + "cpu_cores": { + "title": "CPU核数", + "width": 80 + }, + "memory_gb": { + "title": "内存GB", + "width": 80 + }, + "gpu_count": { + "title": "GPU数", + "width": 70 + }, + "gpu_type": { + "title": "GPU型号", + "width": 100 + }, + "node_type": { + "title": "节点类型", + "width": 100 + }, + "status": { + "title": "状态", + "width": 100 + } + }, + "editexclouded": [ + "id", + "resellerid", + "created_at", + "updated_at" + ], + "toolbar": { + "tools": [] + }, + "binds": [], + "new_data_url": "{{entire_url('/pcpool/api/compute_node_create.dspy')}}", + "update_data_url": "{{entire_url('/pcpool/api/compute_node_update.dspy')}}", + "delete_data_url": "{{entire_url('/pcpool/api/compute_node_delete.dspy')}}", + "logined_userorgid": "resellerid" + } +} \ No newline at end of file diff --git a/json/compute_pool_list.json b/json/compute_pool_list.json new file mode 100644 index 0000000..1e8f6ac --- /dev/null +++ b/json/compute_pool_list.json @@ -0,0 +1,59 @@ +{ + "tblname": "compute_pool", + "alias": "compute_pool_list", + "title": "算力池", + "params": { + "browserfields": { + "name": { + "title": "池名称", + "width": 150 + }, + "pool_type": { + "title": "集群类型", + "width": 100 + }, + "total_cpu": { + "title": "总CPU核数", + "width": 100 + }, + "total_memory_gb": { + "title": "总内存GB", + "width": 100 + }, + "total_gpu": { + "title": "总GPU数", + "width": 80 + }, + "allocated_cpu": { + "title": "已分配CPU", + "width": 100 + }, + "allocated_memory_gb": { + "title": "已分配内存GB", + "width": 120 + }, + "allocated_gpu": { + "title": "已分配GPU", + "width": 100 + }, + "status": { + "title": "状态", + "width": 80 + } + }, + "editexclouded": [ + "id", + "resellerid", + "created_at", + "updated_at" + ], + "toolbar": { + "tools": [] + }, + "binds": [], + "new_data_url": "{{entire_url('/pcpool/api/compute_pool_create.dspy')}}", + "update_data_url": "{{entire_url('/pcpool/api/compute_pool_update.dspy')}}", + "delete_data_url": "{{entire_url('/pcpool/api/compute_pool_delete.dspy')}}", + "logined_userorgid": "resellerid" + } +} \ No newline at end of file diff --git a/models/compute_node.json b/models/compute_node.json new file mode 100644 index 0000000..368fd7b --- /dev/null +++ b/models/compute_node.json @@ -0,0 +1,129 @@ +{ + "summary": [ + { + "name": "compute_node", + "title": "算力节点", + "primary": [ + "id" + ], + "catelog": "entity" + } + ], + "fields": [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "pool_id", + "title": "所属算力池", + "type": "str", + "length": 32 + }, + { + "name": "name", + "title": "节点名称", + "type": "str", + "length": 128, + "nullable": "no" + }, + { + "name": "ip_address", + "title": "IP地址", + "type": "str", + "length": 64, + "nullable": "no" + }, + { + "name": "ssh_port", + "title": "SSH端口", + "type": "int", + "default": 22 + }, + { + "name": "ssh_user", + "title": "SSH用户", + "type": "str", + "length": 64 + }, + { + "name": "cpu_cores", + "title": "CPU核数", + "type": "int", + "default": 0 + }, + { + "name": "memory_gb", + "title": "内存GB", + "type": "int", + "default": 0 + }, + { + "name": "gpu_count", + "title": "GPU数量", + "type": "int", + "default": 0 + }, + { + "name": "gpu_type", + "title": "GPU型号", + "type": "str", + "length": 64 + }, + { + "name": "node_type", + "title": "节点类型(control/compute/storage)", + "type": "char", + "length": 16 + }, + { + "name": "status", + "title": "状态(available/allocated/maintenance/offline)", + "type": "char", + "length": 16, + "default": "available" + }, + { + "name": "last_heartbeat", + "title": "最后心跳", + "type": "timestamp" + }, + { + "name": "created_at", + "title": "创建时间", + "type": "timestamp", + "nullable": "no" + }, + { + "name": "updated_at", + "title": "更新时间", + "type": "timestamp", + "nullable": "no" + } + ], + "codes": [ + { + "field": "pool_id", + "table": "compute_pool", + "valuefield": "id", + "textfield": "name" + }, + { + "field": "node_type", + "table": "appcodes_kv", + "valuefield": "k", + "textfield": "v", + "cond": "parentid='node_type'" + }, + { + "field": "status", + "table": "appcodes_kv", + "valuefield": "k", + "textfield": "v", + "cond": "parentid='node_status'" + } + ] +} \ No newline at end of file diff --git a/models/compute_pool.json b/models/compute_pool.json new file mode 100644 index 0000000..b47ba31 --- /dev/null +++ b/models/compute_pool.json @@ -0,0 +1,119 @@ +{ + "summary": [ + { + "name": "compute_pool", + "title": "算力池", + "primary": [ + "id" + ], + "catelog": "entity" + } + ], + "fields": [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "resellerid", + "title": "商户机构id", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "name", + "title": "池名称", + "type": "str", + "length": 128, + "nullable": "no" + }, + { + "name": "pool_type", + "title": "集群类型(k8s/slurm/ray)", + "type": "char", + "length": 16, + "nullable": "no" + }, + { + "name": "description", + "title": "描述", + "type": "str", + "length": 512 + }, + { + "name": "total_cpu", + "title": "总CPU核数", + "type": "int", + "default": 0 + }, + { + "name": "total_memory_gb", + "title": "总内存GB", + "type": "int", + "default": 0 + }, + { + "name": "total_gpu", + "title": "总GPU数", + "type": "int", + "default": 0 + }, + { + "name": "allocated_cpu", + "title": "已分配CPU", + "type": "int", + "default": 0 + }, + { + "name": "allocated_memory_gb", + "title": "已分配内存GB", + "type": "int", + "default": 0 + }, + { + "name": "allocated_gpu", + "title": "已分配GPU", + "type": "int", + "default": 0 + }, + { + "name": "status", + "title": "状态(1=启用 0=停用)", + "type": "char", + "length": 1, + "default": "1" + }, + { + "name": "created_at", + "title": "创建时间", + "type": "timestamp", + "nullable": "no" + }, + { + "name": "updated_at", + "title": "更新时间", + "type": "timestamp", + "nullable": "no" + } + ], + "codes": [ + { + "field": "pool_type", + "table": "appcodes_kv", + "valuefield": "k", + "textfield": "v", + "cond": "parentid='cluster_type'" + }, + { + "field": "status", + "table": "appcodes_kv", + "valuefield": "k", + "textfield": "v", + "cond": "parentid='product_status'" + } + ] +} \ No newline at end of file diff --git a/pcpool/__init__.py b/pcpool/__init__.py new file mode 100644 index 0000000..bc0a756 --- /dev/null +++ b/pcpool/__init__.py @@ -0,0 +1,149 @@ +""" +pcpool — 算力中心算力池管理 + - 算力节点池 CRUD + - 节点注册/注销/心跳 + - 算力单元分配和回收 + - 池资源统计 +""" +import datetime +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + +MODULE_NAME = 'pccs' + + +async def node_heartbeat(request, params_kw): + """节点心跳上报，更新 last_heartbeat""" + env = request._run_ns + node_id = params_kw.get('id') or params_kw.get('node_id', '') + if not node_id: + return {'status': 'error', 'message': 'Missing node id'} + dbname = env.get_module_dbname(MODULE_NAME) + now = datetime.datetime.now().isoformat() + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('compute_node', {'id': node_id}, + {'last_heartbeat': now, 'updated_at': now}) + return {'status': 'ok'} + + +async def allocate_nodes(request, params_kw): + """从算力池中分配节点给集群。 + params: pool_id, cluster_id, role (control/compute), count, cpu, memory, gpu + 返回分配的节点 ID 列表。""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + pool_id = params_kw.get('pool_id', '') + cluster_id = params_kw.get('cluster_id', '') + role = params_kw.get('role', 'compute') + count = int(params_kw.get('count', 1)) + req_cpu = int(params_kw.get('cpu', 0)) + req_mem = int(params_kw.get('memory', 0)) + req_gpu = int(params_kw.get('gpu', 0)) + + if not pool_id: + return {'status': 'error', 'message': 'Missing pool_id'} + + async with DBPools().sqlorContext(dbname) as sor: + # 查询可用节点 + sql = """SELECT * FROM compute_node + WHERE pool_id = ${pool_id}$ AND status = 'available' + ORDER BY cpu_cores DESC, memory_gb DESC + LIMIT ${limit}$""" + available = await sor.sqlExe(sql, {'pool_id': pool_id, 'limit': count}) + if len(available) < count: + return {'status': 'error', + 'message': f'可用节点不足: 需要{count}, 可用{len(available)}'} + + allocated = [] + for node in available[:count]: + await sor.U('compute_node', {'id': node.id}, + {'status': 'allocated', 'updated_at': datetime.datetime.now().isoformat()}) + # 记录分配关系 + cn_id = getID() + await sor.C('cluster_node', { + 'id': cn_id, 'cluster_id': cluster_id, + 'node_id': node.id, 'role': role, 'status': 'joining', + 'assigned_at': datetime.datetime.now().isoformat() + }) + allocated.append(node.id) + + # 更新池已分配资源 + if allocated: + await sor.sqlExe( + """UPDATE compute_pool + SET allocated_cpu = allocated_cpu + ${cpu}$, + allocated_memory_gb = allocated_memory_gb + ${mem}$, + allocated_gpu = allocated_gpu + ${gpu}$, + updated_at = ${now}$ + WHERE id = ${pool_id}$""", + {'cpu': req_cpu * len(allocated), 'mem': req_mem * len(allocated), + 'gpu': req_gpu * len(allocated), 'pool_id': pool_id, + 'now': datetime.datetime.now().isoformat()} + ) + + return {'status': 'ok', 'allocated_nodes': allocated} + + +async def release_nodes(request, params_kw): + """回收节点回算力池。 + params: node_ids (逗号分隔), pool_id""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + node_ids_str = params_kw.get('node_ids', '') + if not node_ids_str: + return {'status': 'error', 'message': 'Missing node_ids'} + node_ids = [n.strip() for n in node_ids_str.split(',') if n.strip()] + + pool_id = params_kw.get('pool_id', '') + now = datetime.datetime.now().isoformat() + + async with DBPools().sqlorContext(dbname) as sor: + for nid in node_ids: + await sor.U('compute_node', {'id': nid}, + {'status': 'available', 'updated_at': now}) + # 清除分配记录 + await sor.sqlExe( + "UPDATE cluster_node SET status='removed' WHERE node_id=${nid}$", + {'nid': nid}) + # 回收资源计数 + node_row = await sor.R('compute_node', {'id': nid}) + if node_row: + n = node_row[0] + await sor.sqlExe( + """UPDATE compute_pool + SET allocated_cpu = GREATEST(allocated_cpu - ${cpu}$, 0), + allocated_memory_gb = GREATEST(allocated_memory_gb - ${mem}$, 0), + allocated_gpu = GREATEST(allocated_gpu - ${gpu}$, 0), + updated_at = ${now}$ + WHERE id = ${pool_id}$""", + {'cpu': n.cpu_cores, 'mem': n.memory_gb, 'gpu': n.gpu_count, + 'pool_id': pool_id or n.pool_id, 'now': now}) + + return {'status': 'ok', 'released': len(node_ids)} + + +async def pool_stats(request, params_kw): + """获取池资源统计""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + pool_id = params_kw.get('pool_id', '') + async with DBPools().sqlorContext(dbname) as sor: + if pool_id: + pools = await sor.R('compute_pool', {'id': pool_id}) + else: + pools = await sor.R('compute_pool', {}) + if not pools: + return {'status': 'ok', 'data': []} + result = [] + for p in pools: + nodes = await sor.R('compute_node', {'pool_id': p.id}) + result.append({ + 'id': p.id, 'name': p.name, 'type': p.pool_type, + 'total_cpu': p.total_cpu, 'allocated_cpu': p.allocated_cpu, + 'total_memory_gb': p.total_memory_gb, 'allocated_memory_gb': p.allocated_memory_gb, + 'total_gpu': p.total_gpu, 'allocated_gpu': p.allocated_gpu, + 'node_count': len(nodes), + 'available_nodes': sum(1 for n in nodes if n.status == 'available'), + 'status': p.status + }) + return {'status': 'ok', 'data': result} diff --git a/scripts/load_path.py b/scripts/load_path.py new file mode 100644 index 0000000..fb17939 --- /dev/null +++ b/scripts/load_path.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""pcpool RBAC 权限管理""" +import subprocess, os, sys, json, glob + +def find_sage_root(): + for c in [os.path.expanduser("~/sage"), os.path.expanduser("~/repos/sage")]: + if os.path.isdir(os.path.join(c, "py3")): return c + return None + +SAGE = find_sage_root() +if not SAGE: sys.exit("Sage root not found") +PY = os.path.join(SAGE, "py3", "bin", "python") +SET = os.path.join(SAGE, "set_role_perm.py") + +JSON_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "json") +API_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "wwwroot", "api") + +def load_cruds(): + defs = [] + for f in sorted(glob.glob(os.path.join(JSON_DIR, "*.json"))): + try: + d = json.load(open(f)) + defs.append({"tblname": d["tblname"], "alias": d.get("alias", d["tblname"]), "subtables": d.get("params", {}).get("subtables", [])}) + except: pass + return defs + +def get_apis(): + if not os.path.isdir(API_DIR): return [] + return [f"/pcpool/api/{f}" for f in sorted(os.listdir(API_DIR)) if f.endswith(".dspy")] + +cruds = load_cruds() +apis = get_apis() + +PATHS_ANY = [ + f"/pcpool/menu.ui", +] +PATHS_LOGINED = [ + f"/pcpool", + f"/pcpool/index.ui", +] +for d in cruds: + PATHS_ANY.append(f"/pcpool/{d['alias']}") + PATHS_LOGINED.append(f"/pcpool/{d['alias']}/index.ui") + for act in ["get", "add", "update", "delete"]: + PATHS_LOGINED.append(f"/pcpool/{d['alias']}/{act}_{d['tblname']}.dspy") +for api in apis: + PATHS_LOGINED.append(api) + +PATHS_OPERATOR = list(PATHS_LOGINED) + +PATHS_ANY = list(dict.fromkeys(PATHS_ANY)) +PATHS_LOGINED = list(dict.fromkeys(PATHS_LOGINED)) +PATHS_OPERATOR = list(dict.fromkeys(PATHS_OPERATOR)) + +def reg(role, paths): + ok = 0 + for p in paths: + r = subprocess.run([PY, SET, role, p], capture_output=True, text=True) + if r.returncode == 0: ok += 1 + print(f" {role}: {ok}/{len(paths)}") + return ok + +total = 0 +print(f"{mod_name}: any={len(PATHS_ANY)} logined={len(PATHS_LOGINED)} operator={len(PATHS_OPERATOR)}") +total += reg("any", PATHS_ANY) +total += reg("logined", PATHS_LOGINED) +total += reg("reseller.operator", PATHS_OPERATOR) +print(f"Done. {total} entries.") diff --git a/setup.json b/setup.json new file mode 100644 index 0000000..02fa30b --- /dev/null +++ b/setup.json @@ -0,0 +1,8 @@ +{ + "name": "pcpool", + "version": "0.1.0", + "description": "算力中心算力池管理：节点池CRUD、节点注册/注销、算力单元分配回收", + "packages": ["pcpool"], + "install_requires": ["apppublic", "sqlor", "ahserver", "appbase"], + "python_requires": ">=3.10" +} \ No newline at end of file diff --git a/wwwroot/api/compute_node_create.dspy b/wwwroot/api/compute_node_create.dspy new file mode 100644 index 0000000..808d2ea --- /dev/null +++ b/wwwroot/api/compute_node_create.dspy @@ -0,0 +1,3 @@ +# compute_node create +result = {'status': 'ok', 'message': 'compute_node created'} +return result diff --git a/wwwroot/api/compute_node_delete.dspy b/wwwroot/api/compute_node_delete.dspy new file mode 100644 index 0000000..6c084af --- /dev/null +++ b/wwwroot/api/compute_node_delete.dspy @@ -0,0 +1,3 @@ +# compute_node delete +result = {'status': 'ok', 'message': 'compute_node deleted'} +return result diff --git a/wwwroot/api/compute_node_update.dspy b/wwwroot/api/compute_node_update.dspy new file mode 100644 index 0000000..1d16bec --- /dev/null +++ b/wwwroot/api/compute_node_update.dspy @@ -0,0 +1,3 @@ +# compute_node update +result = {'status': 'ok', 'message': 'compute_node updated'} +return result diff --git a/wwwroot/api/compute_pool_create.dspy b/wwwroot/api/compute_pool_create.dspy new file mode 100644 index 0000000..37db2d8 --- /dev/null +++ b/wwwroot/api/compute_pool_create.dspy @@ -0,0 +1,3 @@ +# compute_pool create +result = {'status': 'ok', 'message': 'compute_pool created'} +return result diff --git a/wwwroot/api/compute_pool_delete.dspy b/wwwroot/api/compute_pool_delete.dspy new file mode 100644 index 0000000..daf2cfb --- /dev/null +++ b/wwwroot/api/compute_pool_delete.dspy @@ -0,0 +1,3 @@ +# compute_pool delete +result = {'status': 'ok', 'message': 'compute_pool deleted'} +return result diff --git a/wwwroot/api/compute_pool_update.dspy b/wwwroot/api/compute_pool_update.dspy new file mode 100644 index 0000000..8894a38 --- /dev/null +++ b/wwwroot/api/compute_pool_update.dspy @@ -0,0 +1,3 @@ +# compute_pool update +result = {'status': 'ok', 'message': 'compute_pool updated'} +return result diff --git a/wwwroot/api/node_allocate.dspy b/wwwroot/api/node_allocate.dspy new file mode 100644 index 0000000..74a62d2 --- /dev/null +++ b/wwwroot/api/node_allocate.dspy @@ -0,0 +1,4 @@ +# 从算力池分配节点到集群 +# params: pool_id, cluster_id, role(control/compute), count, cpu, memory, gpu +result = await allocate_nodes(request, params_kw) +return result if isinstance(result, dict) else {'status': 'error', 'message': str(result)} diff --git a/wwwroot/api/node_heartbeat.dspy b/wwwroot/api/node_heartbeat.dspy new file mode 100644 index 0000000..17b26ca --- /dev/null +++ b/wwwroot/api/node_heartbeat.dspy @@ -0,0 +1,8 @@ +# 节点心跳上报 +node_id = params_kw.get('id', '') +if not node_id: + return {'status': 'error', 'message': 'Missing node id'} +import datetime +async with DBPools().sqlorContext('pccs') as sor: + await sor.U('compute_node', {'id': node_id}, {'last_heartbeat': datetime.datetime.now().isoformat(), 'updated_at': datetime.datetime.now().isoformat()}) +return {'status': 'ok'} diff --git a/wwwroot/api/node_release.dspy b/wwwroot/api/node_release.dspy new file mode 100644 index 0000000..a1a0bc7 --- /dev/null +++ b/wwwroot/api/node_release.dspy @@ -0,0 +1,3 @@ +# 回收节点回算力池 +result = await release_nodes(request, params_kw) +return result if isinstance(result, dict) else {'status': 'error', 'message': str(result)}