From 98193ce5a27e65c2925e8c633f5fe922dad76533 Mon Sep 17 00:00:00 2001 From: pccs Date: Wed, 12 Aug 2026 19:35:32 +0800 Subject: [PATCH] feat: menu/i18n/CRUD/permission/stats/model fixes --- .../lib/build/lib/build/lib/pcc/__init__.py | 264 ++++++++++++++++++ .../lib/build/lib/pcc/k8s_plugin/__init__.py | 221 +++++++++++++++ .../lib/build/lib/pcc/ray_plugin/__init__.py | 174 ++++++++++++ .../build/lib/pcc/slurm_plugin/__init__.py | 229 +++++++++++++++ .../lib/build/lib/build/lib/pcc/__init__.py | 264 ++++++++++++++++++ .../lib/build/lib/pcc/k8s_plugin/__init__.py | 221 +++++++++++++++ .../lib/build/lib/pcc/ray_plugin/__init__.py | 174 ++++++++++++ .../build/lib/pcc/slurm_plugin/__init__.py | 229 +++++++++++++++ .../build/lib/build/lib/scripts/load_path.py | 68 +++++ build/lib/build/lib/build/lib/pcc/__init__.py | 264 ++++++++++++++++++ .../lib/build/lib/pcc/k8s_plugin/__init__.py | 221 +++++++++++++++ .../lib/build/lib/pcc/ray_plugin/__init__.py | 174 ++++++++++++ .../build/lib/pcc/slurm_plugin/__init__.py | 229 +++++++++++++++ .../build/lib/build/lib/scripts/load_path.py | 68 +++++ build/lib/build/lib/pcc/__init__.py | 264 ++++++++++++++++++ .../lib/build/lib/pcc/k8s_plugin/__init__.py | 221 +++++++++++++++ .../lib/build/lib/pcc/ray_plugin/__init__.py | 174 ++++++++++++ .../build/lib/pcc/slurm_plugin/__init__.py | 229 +++++++++++++++ build/lib/build/lib/scripts/load_path.py | 68 +++++ build/lib/pcc/__init__.py | 264 ++++++++++++++++++ build/lib/pcc/k8s_plugin/__init__.py | 221 +++++++++++++++ build/lib/pcc/ray_plugin/__init__.py | 174 ++++++++++++ build/lib/pcc/slurm_plugin/__init__.py | 229 +++++++++++++++ build/lib/scripts/load_path.py | 68 +++++ i18n/en/msg.txt | 28 ++ i18n/zh/msg.txt | 28 ++ json/cluster_list.json | 24 +- json/cluster_node_list.json | 13 +- pcc.egg-info/PKG-INFO | 10 + pcc.egg-info/SOURCES.txt | 12 + pcc.egg-info/dependency_links.txt | 1 + pcc.egg-info/requires.txt | 5 + pcc.egg-info/top_level.txt | 7 + pcc/__init__.py | 22 ++ pyproject.toml | 19 ++ wwwroot/api/cluster_create.dspy | 19 +- wwwroot/api/cluster_delete.dspy | 9 +- wwwroot/api/cluster_node_create.dspy | 17 +- wwwroot/api/cluster_node_delete.dspy | 9 +- wwwroot/api/cluster_node_update.dspy | 12 +- wwwroot/api/cluster_status_all.dspy | 49 ++++ wwwroot/api/cluster_update.dspy | 13 +- wwwroot/cluster_list/add_cluster.dspy | 52 ++++ wwwroot/cluster_list/delete_cluster.dspy | 47 ++++ wwwroot/cluster_list/get_cluster.dspy | 145 ++++++++++ wwwroot/cluster_list/index.ui | 259 +++++++++++++++++ wwwroot/cluster_list/update_cluster.dspy | 70 +++++ .../cluster_node_list/add_cluster_node.dspy | 57 ++++ .../delete_cluster_node.dspy | 47 ++++ .../cluster_node_list/get_cluster_node.dspy | 107 +++++++ wwwroot/cluster_node_list/index.ui | 181 ++++++++++++ .../update_cluster_node.dspy | 75 +++++ 52 files changed, 6017 insertions(+), 32 deletions(-) create mode 100644 build/lib/build/lib/build/lib/build/lib/build/lib/pcc/__init__.py create mode 100644 build/lib/build/lib/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py create mode 100644 build/lib/build/lib/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py create mode 100644 build/lib/build/lib/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py create mode 100644 build/lib/build/lib/build/lib/build/lib/pcc/__init__.py create mode 100644 build/lib/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py create mode 100644 build/lib/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py create mode 100644 build/lib/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py create mode 100644 build/lib/build/lib/build/lib/build/lib/scripts/load_path.py create mode 100644 build/lib/build/lib/build/lib/pcc/__init__.py create mode 100644 build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py create mode 100644 build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py create mode 100644 build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py create mode 100644 build/lib/build/lib/build/lib/scripts/load_path.py create mode 100644 build/lib/build/lib/pcc/__init__.py create mode 100644 build/lib/build/lib/pcc/k8s_plugin/__init__.py create mode 100644 build/lib/build/lib/pcc/ray_plugin/__init__.py create mode 100644 build/lib/build/lib/pcc/slurm_plugin/__init__.py create mode 100644 build/lib/build/lib/scripts/load_path.py create mode 100644 build/lib/pcc/__init__.py create mode 100644 build/lib/pcc/k8s_plugin/__init__.py create mode 100644 build/lib/pcc/ray_plugin/__init__.py create mode 100644 build/lib/pcc/slurm_plugin/__init__.py create mode 100644 build/lib/scripts/load_path.py create mode 100644 i18n/en/msg.txt create mode 100644 i18n/zh/msg.txt create mode 100644 pcc.egg-info/PKG-INFO create mode 100644 pcc.egg-info/SOURCES.txt create mode 100644 pcc.egg-info/dependency_links.txt create mode 100644 pcc.egg-info/requires.txt create mode 100644 pcc.egg-info/top_level.txt create mode 100644 pyproject.toml create mode 100644 wwwroot/api/cluster_status_all.dspy create mode 100644 wwwroot/cluster_list/add_cluster.dspy create mode 100644 wwwroot/cluster_list/delete_cluster.dspy create mode 100644 wwwroot/cluster_list/get_cluster.dspy create mode 100644 wwwroot/cluster_list/index.ui create mode 100644 wwwroot/cluster_list/update_cluster.dspy create mode 100644 wwwroot/cluster_node_list/add_cluster_node.dspy create mode 100644 wwwroot/cluster_node_list/delete_cluster_node.dspy create mode 100644 wwwroot/cluster_node_list/get_cluster_node.dspy create mode 100644 wwwroot/cluster_node_list/index.ui create mode 100644 wwwroot/cluster_node_list/update_cluster_node.dspy diff --git a/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/__init__.py b/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/__init__.py new file mode 100644 index 0000000..8c44552 --- /dev/null +++ b/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/__init__.py @@ -0,0 +1,264 @@ +""" +pcc — 统一集群管理 (Pooled Computing Cluster) + 生命周期: deploy → run → stop → start → destroy + 插件: k8s_plugin, slurm_plugin, ray_plugin +""" +import datetime, json, asyncio, subprocess +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + +MODULE_NAME = 'pccs' + +_PLUGINS = { + 'k8s': 'pcc.k8s_plugin', + 'slurm': 'pcc.slurm_plugin', + 'ray': 'pcc.ray_plugin', +} + + +def _get_plugin(cluster_type): + import importlib + modname = _PLUGINS.get(cluster_type) + if not modname: + raise ValueError('Unknown cluster_type: ' + cluster_type) + return importlib.import_module(modname) + + +async def ssh_exec(host, port, user, cmd, timeout=120): + """异步 SSH 远程执行命令，返回 (exit_code, stdout, stderr)""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), '{user}@{host}', cmd + ] + proc = await asyncio.create_subprocess_exec( + *ssh_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout) + except asyncio.TimeoutError: + proc.kill() + return -1, '', f'SSH timeout after {timeout}s' + return proc.returncode, stdout.decode(), stderr.decode() + + +async def ssh_exec_bg(host, port, user, cmd): + """异步后台执行，不等待结果""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), user + '@' + host, + 'nohup bash -c "' + cmd.replace('"', '\\"') + '" > /dev/null 2>&1 &' + ] + subprocess.Popen(ssh_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +async def deploy_cluster(request, cluster_id): + """部署集群: 读取集群配置 → 分配节点 → 调用 plugin 部署""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': f'Cluster {cluster_id} not found'} + cluster = recs[0] + + # 更新状态为 deploying + await sor.U('cluster', {'id': cluster_id}, + {'status': 'deploying', 'updated_at': datetime.datetime.now().isoformat()}) + + # 获取已分配的节点 + node_recs = await sor.R('cluster_node', {'cluster_id': cluster_id}) + control_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'control'] + compute_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'compute'] + + control_nodes = [n[0] for n in control_nodes if n] + compute_nodes = [n[0] for n in compute_nodes if n] + + if not control_nodes: + return {'status': 'error', 'message': 'No control node assigned'} + + config = {} + if cluster.control_config: + try: + config = json.loads(cluster.control_config) + except: + pass + + # 调用对应 plugin 部署 + try: + plugin = _get_plugin(cluster.cluster_type) + deploy_log = await plugin.deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config) + except Exception as e: + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'failed', 'deploy_log': str(e), + 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'error', 'message': f'Deploy failed: {e}'} + + # 更新状态为 running + endpoint = deploy_log.get('endpoint', '') if isinstance(deploy_log, dict) else '' + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'endpoint': endpoint, + 'deploy_log': json.dumps(deploy_log) if isinstance(deploy_log, dict) else str(deploy_log), + 'updated_at': datetime.datetime.now().isoformat()}) + # 更新节点状态为 active + for n in control_nodes + compute_nodes: + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': n.id}, + {'status': 'active'}) + + return {'status': 'ok', 'message': 'Cluster deployed', 'endpoint': endpoint} + + +async def add_cluster_node(request, params_kw): + """向运行中集群动态添加节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + role = params_kw.get('role', 'compute') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + + node = await sor.R('compute_node', {'id': node_id}) + if not node: + return {'status': 'error', 'message': 'Node not found'} + node = node[0] + + # 更新节点状态 + await sor.U('compute_node', {'id': node_id}, + {'status': 'allocated', 'updated_at': datetime.datetime.now().isoformat()}) + + # 记录分配 + await sor.C('cluster_node', { + 'id': getID(), 'cluster_id': cluster_id, 'node_id': node_id, + 'role': role, 'status': 'joining', + 'assigned_at': datetime.datetime.now().isoformat() + }) + + # 调用 plugin 添加 + plugin = _get_plugin(cluster.cluster_type) + result = await plugin.add_node(env, cluster_id, node_id, role) + return result + + +async def remove_cluster_node(request, params_kw): + """从集群移除节点并回收""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + + plugin = _get_plugin(cluster[0].cluster_type) + result = await plugin.remove_node(env, cluster_id, node_id) + + # 回收节点 + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('compute_node', {'id': node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': node_id}, + {'status': 'removed'}) + + return result + + +async def cluster_status(request, cluster_id): + """获取集群运行状态""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + + # 从 plugin 获取实时状态 + try: + plugin = _get_plugin(cluster.cluster_type) + live_status = await plugin.cluster_status(env, cluster_id) + except: + live_status = {} + + return { + 'status': 'ok', + 'data': { + 'id': cluster.id, 'name': cluster.name, 'type': cluster.cluster_type, + 'version': cluster.version, 'endpoint': cluster.endpoint, + 'state': cluster.status, + 'nodes': [{'id': n.node_id, 'role': n.role, 'status': n.status} for n in nodes], + 'live': live_status + } + } + + +async def stop_cluster(request, cluster_id): + """停止集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'stopped', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster stopped'} + + +async def start_cluster(request, cluster_id): + """启动已停止的集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster started'} + + +async def destroy_cluster(request, cluster_id): + """销毁集群，回收所有节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + # 释放所有节点 + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + for n in nodes: + await sor.U('compute_node', {'id': n.node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'id': n.id}, {'status': 'removed'}) + await sor.U('cluster', {'id': cluster_id}, + {'status': 'destroyed', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster destroyed'} + +async def cluster_status_all(request, params_kw): + """获取所有集群状态统计""" + env = request._run_ns + dbname = env.get_module_dbname('pcc') + async with DBPools().sqlorContext(dbname) as sor: + clusters = await sor.R('cluster', {}) + if not clusters: + return {'status': 'ok', 'data': {'total': 0, 'running': 0, 'deploying': 0, 'failed': 0, 'stopped': 0}} + statuses = {} + for c in clusters: + s = getattr(c, 'status', 'unknown') + statuses[s] = statuses.get(s, 0) + 1 + return {'status': 'ok', 'data': { + 'total': len(clusters), + 'running': statuses.get('running', 0), + 'deploying': statuses.get('deploying', 0), + 'failed': statuses.get('failed', 0), + 'stopped': statuses.get('stopped', 0), + 'destroyed': statuses.get('destroyed', 0), + }} + diff --git a/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py b/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py new file mode 100644 index 0000000..b2bed1f --- /dev/null +++ b/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py @@ -0,0 +1,221 @@ +""" +k8s_plugin — K8s 集群管理 (基于 kubeadm) +- 控制节点: kubeadm init + CNI(flannel) +- 算力节点: kubeadm join +- 移除: kubectl drain + delete + kubeadm reset +- 状态: kubectl get nodes +""" +from pcc import ssh_exec + +K8S_VERSION = '1.29' +POD_CIDR = '10.244.0.0/16' +SVC_CIDR = '10.96.0.0/12' + + +async def _node_info(env, node_id): + """从 DB 获取节点信息""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_k8s(node): + """在节点上安装 K8s 基础组件 (containerd + kubeadm + kubelet + kubectl)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq apt-transport-https ca-certificates curl gpg && " + "curl -fsSL https://pkgs.k8s.io/core:/stable:/v{ver}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg && " + "echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v{ver}/deb/ /' > /etc/apt/sources.list.d/kubernetes.list && " + "apt-get update -qq && " + "apt-get install -y -qq kubelet kubeadm kubectl containerd && " + "apt-mark hold kubelet kubeadm kubectl && " + "mkdir -p /etc/containerd && " + "containerd config default > /etc/containerd/config.toml 2>/dev/null || true && " + "sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml && " + "systemctl restart containerd && systemctl enable kubelet" + ).format(ver=K8S_VERSION) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _install_cni(control_node, pod_cidr): + """在控制节点安装 CNI 网络插件""" + cmd = f"kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml" + return await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=120) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """K8s 集群部署：只部署控制节点。算力节点后续通过 add_node 加入。""" + pod_cidr = config.get('pod_cidr', POD_CIDR) + svc_cidr = config.get('svc_cidr', SVC_CIDR) + results = {'control_nodes': {}, 'compute_nodes': None, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 K8s 基础组件 + rc, out, err = await _install_k8s(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err}'} + continue + + # 2. kubeadm init（仅第一个控制节点） + init_cmd = ( + f"kubeadm init --pod-network-cidr={pod_cidr} --service-cidr={svc_cidr} " + f"--kubernetes-version=v{K8S_VERSION} --ignore-preflight-errors=all" + ) + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', init_cmd, timeout=300) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'init failed: {err[:200]}'} + continue + + # 3. 配置 kubectl + setup_cmd = "mkdir -p $HOME/.kube && cp -f /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config" + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', setup_cmd) + + # 4. 安装 CNI + await _install_cni(node, pod_cidr) + + # 5. 获取 join token（供后续算力节点使用） + rc2, token_out, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "kubeadm token create --print-join-command", timeout=30) + join_cmd = token_out.strip() if rc2 == 0 else '' + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'join_command': join_cmd, + 'endpoint': f'https://{node.ip_address}:6443' + } + results['endpoint'] = f'https://{node.ip_address}:6443' + + # 保存 join_command 到集群 config + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 K8s 集群添加算力节点：安装 k8s → kubeadm join""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 读取 join_command + from sqlor.dbpools import DBPools + from pcc import MODULE_NAME + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'Cluster not found'} + config = json.loads(recs[0].control_config or '{}') + join_cmd = '' + for cn_data in config.get('control_nodes', {}).values(): + if cn_data.get('join_command'): + join_cmd = cn_data['join_command'] + break + if not join_cmd: + return {'status': 'error', 'message': 'No join command found, cluster may not be deployed'} + + # 1. 安装 K8s + rc, out, err = await _install_k8s(node) + if rc != 0: + return {'status': 'error', 'message': f'k8s install failed: {err[:200]}'} + + # 2. kubeadm join + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', join_cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined K8s cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 K8s 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # drain + delete + reset + cmds = [ + f"kubectl drain {node.name} --ignore-daemonsets --delete-emptydir-data --timeout=60s", + f"kubectl delete node {node.name}", + "kubeadm reset -f", + ] + for cmd in cmds: + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + return {'status': 'ok', 'message': f'Node {node.name} removed'} + + +async def cluster_status(env, cluster_id): + """获取 K8s 集群状态""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + + control_id = recs[0].node_id + cnode = await _node_info(env, control_id) + if not cnode: + return {'error': 'Control node not found'} + + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'kubectl get nodes -o wide', timeout=30) + return {'status': 'ok', 'kubectl_nodes': out if rc == 0 else 'kubectl failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """K8s 算力单元分配: 创建 namespace + ResourceQuota""" + from pcc import ssh_exec as _ssh + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + + quota_yaml = ( + f"apiVersion: v1\\nkind: ResourceQuota\\nmetadata:\\n name: {unit_name}-quota\\n namespace: {unit_name}\\n" + f"spec:\\n hard:\\n requests.cpu: \\\"{cpu}\\\"\\n requests.memory: \\\"{memory}Gi\\\"\\n requests.nvidia.com/gpu: \\\"{gpu}\\\"" + ) + cmds = [ + f"kubectl create namespace {unit_name}", + f"echo '{quota_yaml}' | kubectl apply -f -", + ] + for cmd in cmds: + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'kubectl failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name}: ns + {cpu}C/{memory}G/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """K8s 算力单元回收: 删除 namespace""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + rc, _, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"kubectl delete namespace {unit_name} --timeout=60s") + if rc != 0: + return {'status': 'error', 'message': f'delete ns failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py b/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py new file mode 100644 index 0000000..e0bae45 --- /dev/null +++ b/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py @@ -0,0 +1,174 @@ +""" +ray_plugin — Ray 集群管理 +- 控制节点: ray start --head +- 算力节点: ray start --address=:6379 +- 移除: ray stop +- 状态: ray status +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_ray(node): + """安装 Ray""" + script = ( + "apt-get update -qq && apt-get install -y -qq python3 python3-pip && " + "pip3 install -q ray[default] 2>/dev/null || pip install -q ray[default]" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Ray 集群部署：仅部署 head 节点""" + head_port = config.get('head_port', 6379) + dashboard_port = config.get('dashboard_port', 8265) + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 Ray + rc, out, err = await _install_ray(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. ray start --head + cmd = ( + f"ray start --head --port={head_port} --dashboard-host=0.0.0.0 " + f"--dashboard-port={dashboard_port} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'start failed: {err2[:200]}'} + continue + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'head_address': f'{node.ip_address}:{head_port}', + 'dashboard': f'http://{node.ip_address}:{dashboard_port}' + } + results['endpoint'] = f'{node.ip_address}:{head_port}' + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Ray 集群添加 worker 节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取 head 节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + cluster_recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + head_node = await _node_info(env, recs[0].node_id) + if not head_node: + return {'status': 'error', 'message': 'Head node not found'} + + # 从 cluster config 读 head 地址 + config = json.loads(cluster_recs[0].control_config or '{}') if cluster_recs else {} + head_addr = config.get('endpoint', f'{head_node.ip_address}:6379') + + # 1. 安装 Ray + rc, _, err = await _install_ray(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. ray start --address + cmd = ( + f"ray start --address={head_addr} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined Ray cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Ray 集群移除 worker""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + rc, _, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', 'ray stop', timeout=60) + return {'status': 'ok', 'message': f'Node {node.name} removed from Ray'} + + +async def cluster_status(env, cluster_id): + """ray status""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'ray status', timeout=30) + return {'status': 'ok', 'ray_status': out if rc == 0 else 'ray status failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Ray 算力单元分配: 创建 detached actor / placement group 绑定资源标签""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 用 ray submit 提交一个资源占位 job + script = ( + "import ray; ray.init(address='auto'); " + "pg = ray.util.placement_group([{'CPU': " + str(cpu) + ", 'GPU': " + str(gpu) + "}], strategy='STRICT_SPREAD', name='" + unit_name + "'); " + "ray.get(pg.ready()); print('PG ready: " + unit_name + "')" + ) + py_cmd = f"python3 -c '{script}'" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=60) + if rc != 0: + return {'status': 'error', 'message': f'ray pg failed: {err[:200]}'} + return {'status': 'ok', 'message': f'PlacementGroup {unit_name}: {cpu}C/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """Ray 算力单元回收: 删除 placement group""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + py_cmd = f"python3 -c 'import ray; ray.init(address=\"auto\"); ray.util.remove_placement_group(ray.util.get_placement_group(\"{unit_name}\")); print(\"removed\")'" + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=30) + return {'status': 'ok', 'message': f'PlacementGroup {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py b/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py new file mode 100644 index 0000000..989c5a5 --- /dev/null +++ b/build/lib/build/lib/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py @@ -0,0 +1,229 @@ +""" +slurm_plugin — Slurm 集群管理 +- 控制节点: slurmctld + munge + NFS server +- 算力节点: slurmd + munge + NFS client +- 移除: drain node → scontrol delete → reset +- 状态: sinfo +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_slurm_deps(node): + """安装 Slurm 依赖 (munge + slurm)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq munge slurm-wlm slurm-client nfs-common nfs-kernel-server " + "&& systemctl enable munge && systemctl start munge" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _setup_munge(control_node, compute_node): + """复制 munge key 从控制节点到算力节点""" + cmd = f"scp -o StrictHostKeyChecking=no /etc/munge/munge.key {compute_node.ssh_user or 'root'}@{compute_node.ip_address}:/etc/munge/munge.key" + rc, _, err = await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=30) + if rc == 0: + await ssh_exec(compute_node.ip_address, compute_node.ssh_port or 22, + compute_node.ssh_user or 'root', + "chown munge:munge /etc/munge/munge.key && chmod 400 /etc/munge/munge.key && systemctl restart munge", timeout=30) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Slurm 集群部署：控制节点安装 slurmctld + munge""" + cluster_name = config.get('cluster_name', 'pccs-cluster') + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装依赖 + rc, out, err = await _install_slurm_deps(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. 生成 slurm.conf + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + conf = f"""ClusterName={cluster_name} +ControlMachine={node.ip_address} +SlurmUser=root +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/spool/slurmctld +SlurmdSpoolDir=/var/spool/slurmd +ReturnToService=1 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core +AccountingStorageType=accounting_storage/none +JobCompType=jobcomp/none +NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN +PartitionName=debug Nodes={node.name} Default=YES MaxTime=INFINITE State=UP +""" + # 写配置到控制节点 + write_cmd = f"cat > /etc/slurm/slurm.conf << 'SLURM_EOF'\n{conf}\nSLURM_EOF" + rc2, _, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', write_cmd, timeout=30) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'config write failed: {err2}'} + continue + + # 3. 启动 slurmctld + await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "mkdir -p /var/spool/slurmctld /var/spool/slurmd && " + "systemctl enable slurmctld && systemctl start slurmctld", timeout=60) + + results['control_nodes'][node.id] = {'status': 'ok', 'slurm_conf': conf} + results['endpoint'] = node.ip_address + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Slurm 集群添加算力节点：安装 slurmd + munge → 更新 slurm.conf""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'status': 'error', 'message': 'Control node not found'} + + # 1. 安装依赖 + rc, _, err = await _install_slurm_deps(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. 复制 munge key + await _setup_munge(cnode, node) + + # 3. 更新控制节点 slurm.conf 添加此节点 + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + add_line = f"NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN" + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"echo '{add_line}' >> /etc/slurm/slurm.conf && scontrol reconfigure", timeout=30) + + # 4. 启动 slurmd + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl enable slurmd && systemctl start slurmd", timeout=60) + + return {'status': 'ok', 'message': f'Node {node.name} joined Slurm cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Slurm 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control'}) + if recs: + cnode = await _node_info(env, recs[0].node_id) + if cnode: + # drain → remove from slurm.conf → scontrol reconfigure + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"scontrol update NodeName={node.name} State=DOWN Reason=removing && " + f"sed -i '/NodeName={node.name}/d' /etc/slurm/slurm.conf && " + "scontrol reconfigure", timeout=30) + + # stop slurmd on the node + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl stop slurmd && systemctl disable slurmd", timeout=30) + + return {'status': 'ok', 'message': f'Node {node.name} removed from Slurm'} + + +async def cluster_status(env, cluster_id): + """sinfo""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'sinfo', timeout=30) + return {'status': 'ok', 'sinfo': out if rc == 0 else 'sinfo failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Slurm 算力单元分配: 创建 partition + 关联节点""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 创建 partition + cmd = f"scontrol create PartitionName={unit_name} MaxNodes=UNLIMITED Default=NO MaxTime=UNLIMITED State=UP" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'scontrol failed: {err[:200]}'} + + # 分配 node 到 partition (通过 cluster_node 表找该集群下可用的 compute 节点) + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + assigned = await sor.sqlExe( + """SELECT cn.name FROM cluster_node cnn + JOIN compute_node cn ON cn.id = cnn.node_id + WHERE cnn.cluster_id=${cid}$ AND cnn.role='compute' AND cnn.status='active' + LIMIT 1""", + {'cid': cluster_id} + ) + if assigned: + node_name = assigned[0].name + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol update NodeName={node_name} Partition={unit_name}") + + return {'status': 'ok', 'message': f'Partition {unit_name} created'} + + +async def release_unit(env, cluster_id, unit_name): + """Slurm 算力单元回收: 删除 partition""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol delete PartitionName={unit_name}") + return {'status': 'ok', 'message': f'Partition {unit_name} deleted'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/build/lib/build/lib/pcc/__init__.py b/build/lib/build/lib/build/lib/build/lib/pcc/__init__.py new file mode 100644 index 0000000..8c44552 --- /dev/null +++ b/build/lib/build/lib/build/lib/build/lib/pcc/__init__.py @@ -0,0 +1,264 @@ +""" +pcc — 统一集群管理 (Pooled Computing Cluster) + 生命周期: deploy → run → stop → start → destroy + 插件: k8s_plugin, slurm_plugin, ray_plugin +""" +import datetime, json, asyncio, subprocess +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + +MODULE_NAME = 'pccs' + +_PLUGINS = { + 'k8s': 'pcc.k8s_plugin', + 'slurm': 'pcc.slurm_plugin', + 'ray': 'pcc.ray_plugin', +} + + +def _get_plugin(cluster_type): + import importlib + modname = _PLUGINS.get(cluster_type) + if not modname: + raise ValueError('Unknown cluster_type: ' + cluster_type) + return importlib.import_module(modname) + + +async def ssh_exec(host, port, user, cmd, timeout=120): + """异步 SSH 远程执行命令，返回 (exit_code, stdout, stderr)""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), '{user}@{host}', cmd + ] + proc = await asyncio.create_subprocess_exec( + *ssh_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout) + except asyncio.TimeoutError: + proc.kill() + return -1, '', f'SSH timeout after {timeout}s' + return proc.returncode, stdout.decode(), stderr.decode() + + +async def ssh_exec_bg(host, port, user, cmd): + """异步后台执行，不等待结果""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), user + '@' + host, + 'nohup bash -c "' + cmd.replace('"', '\\"') + '" > /dev/null 2>&1 &' + ] + subprocess.Popen(ssh_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +async def deploy_cluster(request, cluster_id): + """部署集群: 读取集群配置 → 分配节点 → 调用 plugin 部署""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': f'Cluster {cluster_id} not found'} + cluster = recs[0] + + # 更新状态为 deploying + await sor.U('cluster', {'id': cluster_id}, + {'status': 'deploying', 'updated_at': datetime.datetime.now().isoformat()}) + + # 获取已分配的节点 + node_recs = await sor.R('cluster_node', {'cluster_id': cluster_id}) + control_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'control'] + compute_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'compute'] + + control_nodes = [n[0] for n in control_nodes if n] + compute_nodes = [n[0] for n in compute_nodes if n] + + if not control_nodes: + return {'status': 'error', 'message': 'No control node assigned'} + + config = {} + if cluster.control_config: + try: + config = json.loads(cluster.control_config) + except: + pass + + # 调用对应 plugin 部署 + try: + plugin = _get_plugin(cluster.cluster_type) + deploy_log = await plugin.deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config) + except Exception as e: + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'failed', 'deploy_log': str(e), + 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'error', 'message': f'Deploy failed: {e}'} + + # 更新状态为 running + endpoint = deploy_log.get('endpoint', '') if isinstance(deploy_log, dict) else '' + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'endpoint': endpoint, + 'deploy_log': json.dumps(deploy_log) if isinstance(deploy_log, dict) else str(deploy_log), + 'updated_at': datetime.datetime.now().isoformat()}) + # 更新节点状态为 active + for n in control_nodes + compute_nodes: + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': n.id}, + {'status': 'active'}) + + return {'status': 'ok', 'message': 'Cluster deployed', 'endpoint': endpoint} + + +async def add_cluster_node(request, params_kw): + """向运行中集群动态添加节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + role = params_kw.get('role', 'compute') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + + node = await sor.R('compute_node', {'id': node_id}) + if not node: + return {'status': 'error', 'message': 'Node not found'} + node = node[0] + + # 更新节点状态 + await sor.U('compute_node', {'id': node_id}, + {'status': 'allocated', 'updated_at': datetime.datetime.now().isoformat()}) + + # 记录分配 + await sor.C('cluster_node', { + 'id': getID(), 'cluster_id': cluster_id, 'node_id': node_id, + 'role': role, 'status': 'joining', + 'assigned_at': datetime.datetime.now().isoformat() + }) + + # 调用 plugin 添加 + plugin = _get_plugin(cluster.cluster_type) + result = await plugin.add_node(env, cluster_id, node_id, role) + return result + + +async def remove_cluster_node(request, params_kw): + """从集群移除节点并回收""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + + plugin = _get_plugin(cluster[0].cluster_type) + result = await plugin.remove_node(env, cluster_id, node_id) + + # 回收节点 + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('compute_node', {'id': node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': node_id}, + {'status': 'removed'}) + + return result + + +async def cluster_status(request, cluster_id): + """获取集群运行状态""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + + # 从 plugin 获取实时状态 + try: + plugin = _get_plugin(cluster.cluster_type) + live_status = await plugin.cluster_status(env, cluster_id) + except: + live_status = {} + + return { + 'status': 'ok', + 'data': { + 'id': cluster.id, 'name': cluster.name, 'type': cluster.cluster_type, + 'version': cluster.version, 'endpoint': cluster.endpoint, + 'state': cluster.status, + 'nodes': [{'id': n.node_id, 'role': n.role, 'status': n.status} for n in nodes], + 'live': live_status + } + } + + +async def stop_cluster(request, cluster_id): + """停止集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'stopped', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster stopped'} + + +async def start_cluster(request, cluster_id): + """启动已停止的集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster started'} + + +async def destroy_cluster(request, cluster_id): + """销毁集群，回收所有节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + # 释放所有节点 + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + for n in nodes: + await sor.U('compute_node', {'id': n.node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'id': n.id}, {'status': 'removed'}) + await sor.U('cluster', {'id': cluster_id}, + {'status': 'destroyed', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster destroyed'} + +async def cluster_status_all(request, params_kw): + """获取所有集群状态统计""" + env = request._run_ns + dbname = env.get_module_dbname('pcc') + async with DBPools().sqlorContext(dbname) as sor: + clusters = await sor.R('cluster', {}) + if not clusters: + return {'status': 'ok', 'data': {'total': 0, 'running': 0, 'deploying': 0, 'failed': 0, 'stopped': 0}} + statuses = {} + for c in clusters: + s = getattr(c, 'status', 'unknown') + statuses[s] = statuses.get(s, 0) + 1 + return {'status': 'ok', 'data': { + 'total': len(clusters), + 'running': statuses.get('running', 0), + 'deploying': statuses.get('deploying', 0), + 'failed': statuses.get('failed', 0), + 'stopped': statuses.get('stopped', 0), + 'destroyed': statuses.get('destroyed', 0), + }} + diff --git a/build/lib/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py b/build/lib/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py new file mode 100644 index 0000000..b2bed1f --- /dev/null +++ b/build/lib/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py @@ -0,0 +1,221 @@ +""" +k8s_plugin — K8s 集群管理 (基于 kubeadm) +- 控制节点: kubeadm init + CNI(flannel) +- 算力节点: kubeadm join +- 移除: kubectl drain + delete + kubeadm reset +- 状态: kubectl get nodes +""" +from pcc import ssh_exec + +K8S_VERSION = '1.29' +POD_CIDR = '10.244.0.0/16' +SVC_CIDR = '10.96.0.0/12' + + +async def _node_info(env, node_id): + """从 DB 获取节点信息""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_k8s(node): + """在节点上安装 K8s 基础组件 (containerd + kubeadm + kubelet + kubectl)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq apt-transport-https ca-certificates curl gpg && " + "curl -fsSL https://pkgs.k8s.io/core:/stable:/v{ver}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg && " + "echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v{ver}/deb/ /' > /etc/apt/sources.list.d/kubernetes.list && " + "apt-get update -qq && " + "apt-get install -y -qq kubelet kubeadm kubectl containerd && " + "apt-mark hold kubelet kubeadm kubectl && " + "mkdir -p /etc/containerd && " + "containerd config default > /etc/containerd/config.toml 2>/dev/null || true && " + "sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml && " + "systemctl restart containerd && systemctl enable kubelet" + ).format(ver=K8S_VERSION) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _install_cni(control_node, pod_cidr): + """在控制节点安装 CNI 网络插件""" + cmd = f"kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml" + return await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=120) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """K8s 集群部署：只部署控制节点。算力节点后续通过 add_node 加入。""" + pod_cidr = config.get('pod_cidr', POD_CIDR) + svc_cidr = config.get('svc_cidr', SVC_CIDR) + results = {'control_nodes': {}, 'compute_nodes': None, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 K8s 基础组件 + rc, out, err = await _install_k8s(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err}'} + continue + + # 2. kubeadm init（仅第一个控制节点） + init_cmd = ( + f"kubeadm init --pod-network-cidr={pod_cidr} --service-cidr={svc_cidr} " + f"--kubernetes-version=v{K8S_VERSION} --ignore-preflight-errors=all" + ) + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', init_cmd, timeout=300) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'init failed: {err[:200]}'} + continue + + # 3. 配置 kubectl + setup_cmd = "mkdir -p $HOME/.kube && cp -f /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config" + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', setup_cmd) + + # 4. 安装 CNI + await _install_cni(node, pod_cidr) + + # 5. 获取 join token（供后续算力节点使用） + rc2, token_out, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "kubeadm token create --print-join-command", timeout=30) + join_cmd = token_out.strip() if rc2 == 0 else '' + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'join_command': join_cmd, + 'endpoint': f'https://{node.ip_address}:6443' + } + results['endpoint'] = f'https://{node.ip_address}:6443' + + # 保存 join_command 到集群 config + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 K8s 集群添加算力节点：安装 k8s → kubeadm join""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 读取 join_command + from sqlor.dbpools import DBPools + from pcc import MODULE_NAME + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'Cluster not found'} + config = json.loads(recs[0].control_config or '{}') + join_cmd = '' + for cn_data in config.get('control_nodes', {}).values(): + if cn_data.get('join_command'): + join_cmd = cn_data['join_command'] + break + if not join_cmd: + return {'status': 'error', 'message': 'No join command found, cluster may not be deployed'} + + # 1. 安装 K8s + rc, out, err = await _install_k8s(node) + if rc != 0: + return {'status': 'error', 'message': f'k8s install failed: {err[:200]}'} + + # 2. kubeadm join + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', join_cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined K8s cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 K8s 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # drain + delete + reset + cmds = [ + f"kubectl drain {node.name} --ignore-daemonsets --delete-emptydir-data --timeout=60s", + f"kubectl delete node {node.name}", + "kubeadm reset -f", + ] + for cmd in cmds: + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + return {'status': 'ok', 'message': f'Node {node.name} removed'} + + +async def cluster_status(env, cluster_id): + """获取 K8s 集群状态""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + + control_id = recs[0].node_id + cnode = await _node_info(env, control_id) + if not cnode: + return {'error': 'Control node not found'} + + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'kubectl get nodes -o wide', timeout=30) + return {'status': 'ok', 'kubectl_nodes': out if rc == 0 else 'kubectl failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """K8s 算力单元分配: 创建 namespace + ResourceQuota""" + from pcc import ssh_exec as _ssh + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + + quota_yaml = ( + f"apiVersion: v1\\nkind: ResourceQuota\\nmetadata:\\n name: {unit_name}-quota\\n namespace: {unit_name}\\n" + f"spec:\\n hard:\\n requests.cpu: \\\"{cpu}\\\"\\n requests.memory: \\\"{memory}Gi\\\"\\n requests.nvidia.com/gpu: \\\"{gpu}\\\"" + ) + cmds = [ + f"kubectl create namespace {unit_name}", + f"echo '{quota_yaml}' | kubectl apply -f -", + ] + for cmd in cmds: + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'kubectl failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name}: ns + {cpu}C/{memory}G/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """K8s 算力单元回收: 删除 namespace""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + rc, _, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"kubectl delete namespace {unit_name} --timeout=60s") + if rc != 0: + return {'status': 'error', 'message': f'delete ns failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py b/build/lib/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py new file mode 100644 index 0000000..e0bae45 --- /dev/null +++ b/build/lib/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py @@ -0,0 +1,174 @@ +""" +ray_plugin — Ray 集群管理 +- 控制节点: ray start --head +- 算力节点: ray start --address=:6379 +- 移除: ray stop +- 状态: ray status +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_ray(node): + """安装 Ray""" + script = ( + "apt-get update -qq && apt-get install -y -qq python3 python3-pip && " + "pip3 install -q ray[default] 2>/dev/null || pip install -q ray[default]" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Ray 集群部署：仅部署 head 节点""" + head_port = config.get('head_port', 6379) + dashboard_port = config.get('dashboard_port', 8265) + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 Ray + rc, out, err = await _install_ray(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. ray start --head + cmd = ( + f"ray start --head --port={head_port} --dashboard-host=0.0.0.0 " + f"--dashboard-port={dashboard_port} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'start failed: {err2[:200]}'} + continue + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'head_address': f'{node.ip_address}:{head_port}', + 'dashboard': f'http://{node.ip_address}:{dashboard_port}' + } + results['endpoint'] = f'{node.ip_address}:{head_port}' + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Ray 集群添加 worker 节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取 head 节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + cluster_recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + head_node = await _node_info(env, recs[0].node_id) + if not head_node: + return {'status': 'error', 'message': 'Head node not found'} + + # 从 cluster config 读 head 地址 + config = json.loads(cluster_recs[0].control_config or '{}') if cluster_recs else {} + head_addr = config.get('endpoint', f'{head_node.ip_address}:6379') + + # 1. 安装 Ray + rc, _, err = await _install_ray(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. ray start --address + cmd = ( + f"ray start --address={head_addr} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined Ray cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Ray 集群移除 worker""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + rc, _, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', 'ray stop', timeout=60) + return {'status': 'ok', 'message': f'Node {node.name} removed from Ray'} + + +async def cluster_status(env, cluster_id): + """ray status""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'ray status', timeout=30) + return {'status': 'ok', 'ray_status': out if rc == 0 else 'ray status failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Ray 算力单元分配: 创建 detached actor / placement group 绑定资源标签""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 用 ray submit 提交一个资源占位 job + script = ( + "import ray; ray.init(address='auto'); " + "pg = ray.util.placement_group([{'CPU': " + str(cpu) + ", 'GPU': " + str(gpu) + "}], strategy='STRICT_SPREAD', name='" + unit_name + "'); " + "ray.get(pg.ready()); print('PG ready: " + unit_name + "')" + ) + py_cmd = f"python3 -c '{script}'" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=60) + if rc != 0: + return {'status': 'error', 'message': f'ray pg failed: {err[:200]}'} + return {'status': 'ok', 'message': f'PlacementGroup {unit_name}: {cpu}C/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """Ray 算力单元回收: 删除 placement group""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + py_cmd = f"python3 -c 'import ray; ray.init(address=\"auto\"); ray.util.remove_placement_group(ray.util.get_placement_group(\"{unit_name}\")); print(\"removed\")'" + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=30) + return {'status': 'ok', 'message': f'PlacementGroup {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py b/build/lib/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py new file mode 100644 index 0000000..989c5a5 --- /dev/null +++ b/build/lib/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py @@ -0,0 +1,229 @@ +""" +slurm_plugin — Slurm 集群管理 +- 控制节点: slurmctld + munge + NFS server +- 算力节点: slurmd + munge + NFS client +- 移除: drain node → scontrol delete → reset +- 状态: sinfo +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_slurm_deps(node): + """安装 Slurm 依赖 (munge + slurm)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq munge slurm-wlm slurm-client nfs-common nfs-kernel-server " + "&& systemctl enable munge && systemctl start munge" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _setup_munge(control_node, compute_node): + """复制 munge key 从控制节点到算力节点""" + cmd = f"scp -o StrictHostKeyChecking=no /etc/munge/munge.key {compute_node.ssh_user or 'root'}@{compute_node.ip_address}:/etc/munge/munge.key" + rc, _, err = await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=30) + if rc == 0: + await ssh_exec(compute_node.ip_address, compute_node.ssh_port or 22, + compute_node.ssh_user or 'root', + "chown munge:munge /etc/munge/munge.key && chmod 400 /etc/munge/munge.key && systemctl restart munge", timeout=30) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Slurm 集群部署：控制节点安装 slurmctld + munge""" + cluster_name = config.get('cluster_name', 'pccs-cluster') + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装依赖 + rc, out, err = await _install_slurm_deps(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. 生成 slurm.conf + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + conf = f"""ClusterName={cluster_name} +ControlMachine={node.ip_address} +SlurmUser=root +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/spool/slurmctld +SlurmdSpoolDir=/var/spool/slurmd +ReturnToService=1 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core +AccountingStorageType=accounting_storage/none +JobCompType=jobcomp/none +NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN +PartitionName=debug Nodes={node.name} Default=YES MaxTime=INFINITE State=UP +""" + # 写配置到控制节点 + write_cmd = f"cat > /etc/slurm/slurm.conf << 'SLURM_EOF'\n{conf}\nSLURM_EOF" + rc2, _, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', write_cmd, timeout=30) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'config write failed: {err2}'} + continue + + # 3. 启动 slurmctld + await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "mkdir -p /var/spool/slurmctld /var/spool/slurmd && " + "systemctl enable slurmctld && systemctl start slurmctld", timeout=60) + + results['control_nodes'][node.id] = {'status': 'ok', 'slurm_conf': conf} + results['endpoint'] = node.ip_address + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Slurm 集群添加算力节点：安装 slurmd + munge → 更新 slurm.conf""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'status': 'error', 'message': 'Control node not found'} + + # 1. 安装依赖 + rc, _, err = await _install_slurm_deps(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. 复制 munge key + await _setup_munge(cnode, node) + + # 3. 更新控制节点 slurm.conf 添加此节点 + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + add_line = f"NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN" + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"echo '{add_line}' >> /etc/slurm/slurm.conf && scontrol reconfigure", timeout=30) + + # 4. 启动 slurmd + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl enable slurmd && systemctl start slurmd", timeout=60) + + return {'status': 'ok', 'message': f'Node {node.name} joined Slurm cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Slurm 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control'}) + if recs: + cnode = await _node_info(env, recs[0].node_id) + if cnode: + # drain → remove from slurm.conf → scontrol reconfigure + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"scontrol update NodeName={node.name} State=DOWN Reason=removing && " + f"sed -i '/NodeName={node.name}/d' /etc/slurm/slurm.conf && " + "scontrol reconfigure", timeout=30) + + # stop slurmd on the node + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl stop slurmd && systemctl disable slurmd", timeout=30) + + return {'status': 'ok', 'message': f'Node {node.name} removed from Slurm'} + + +async def cluster_status(env, cluster_id): + """sinfo""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'sinfo', timeout=30) + return {'status': 'ok', 'sinfo': out if rc == 0 else 'sinfo failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Slurm 算力单元分配: 创建 partition + 关联节点""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 创建 partition + cmd = f"scontrol create PartitionName={unit_name} MaxNodes=UNLIMITED Default=NO MaxTime=UNLIMITED State=UP" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'scontrol failed: {err[:200]}'} + + # 分配 node 到 partition (通过 cluster_node 表找该集群下可用的 compute 节点) + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + assigned = await sor.sqlExe( + """SELECT cn.name FROM cluster_node cnn + JOIN compute_node cn ON cn.id = cnn.node_id + WHERE cnn.cluster_id=${cid}$ AND cnn.role='compute' AND cnn.status='active' + LIMIT 1""", + {'cid': cluster_id} + ) + if assigned: + node_name = assigned[0].name + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol update NodeName={node_name} Partition={unit_name}") + + return {'status': 'ok', 'message': f'Partition {unit_name} created'} + + +async def release_unit(env, cluster_id, unit_name): + """Slurm 算力单元回收: 删除 partition""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol delete PartitionName={unit_name}") + return {'status': 'ok', 'message': f'Partition {unit_name} deleted'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/build/lib/build/lib/scripts/load_path.py b/build/lib/build/lib/build/lib/build/lib/scripts/load_path.py new file mode 100644 index 0000000..59b37c6 --- /dev/null +++ b/build/lib/build/lib/build/lib/build/lib/scripts/load_path.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""pcc RBAC 权限管理""" +import subprocess, os, sys, json, glob + +def find_sage_root(): + for c in [os.path.expanduser("~/sage"), os.path.expanduser("~/repos/sage")]: + if os.path.isdir(os.path.join(c, "py3")): return c + return None + +SAGE = find_sage_root() +if not SAGE: sys.exit("Sage root not found") +PY = os.path.join(SAGE, "py3", "bin", "python") +SET = os.path.join(SAGE, "set_role_perm.py") + +JSON_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "json") +API_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "wwwroot", "api") + +def load_cruds(): + defs = [] + for f in sorted(glob.glob(os.path.join(JSON_DIR, "*.json"))): + try: + d = json.load(open(f)) + defs.append({"tblname": d["tblname"], "alias": d.get("alias", d["tblname"]), "subtables": d.get("params", {}).get("subtables", [])}) + except: pass + return defs + +def get_apis(): + if not os.path.isdir(API_DIR): return [] + return [f"/pcc/api/{f}" for f in sorted(os.listdir(API_DIR)) if f.endswith(".dspy")] + +cruds = load_cruds() +apis = get_apis() + +PATHS_ANY = [ + f"/pcc/menu.ui", +] +PATHS_LOGINED = [ + f"/pcc", + f"/pcc/index.ui", +] +for d in cruds: + PATHS_ANY.append(f"/pcc/{d['alias']}") + PATHS_LOGINED.append(f"/pcc/{d['alias']}/index.ui") + for act in ["get", "add", "update", "delete"]: + PATHS_LOGINED.append(f"/pcc/{d['alias']}/{act}_{d['tblname']}.dspy") +for api in apis: + PATHS_LOGINED.append(api) + +PATHS_OPERATOR = list(PATHS_LOGINED) + +PATHS_ANY = list(dict.fromkeys(PATHS_ANY)) +PATHS_LOGINED = list(dict.fromkeys(PATHS_LOGINED)) +PATHS_OPERATOR = list(dict.fromkeys(PATHS_OPERATOR)) + +def reg(role, paths): + ok = 0 + for p in paths: + r = subprocess.run([PY, SET, role, p], capture_output=True, text=True) + if r.returncode == 0: ok += 1 + print(f" {role}: {ok}/{len(paths)}") + return ok + +total = 0 +print(f"{mod_name}: any={len(PATHS_ANY)} logined={len(PATHS_LOGINED)} operator={len(PATHS_OPERATOR)}") +total += reg("any", PATHS_ANY) +total += reg("logined", PATHS_LOGINED) +total += reg("reseller.operator", PATHS_OPERATOR) +print(f"Done. {total} entries.") diff --git a/build/lib/build/lib/build/lib/pcc/__init__.py b/build/lib/build/lib/build/lib/pcc/__init__.py new file mode 100644 index 0000000..8c44552 --- /dev/null +++ b/build/lib/build/lib/build/lib/pcc/__init__.py @@ -0,0 +1,264 @@ +""" +pcc — 统一集群管理 (Pooled Computing Cluster) + 生命周期: deploy → run → stop → start → destroy + 插件: k8s_plugin, slurm_plugin, ray_plugin +""" +import datetime, json, asyncio, subprocess +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + +MODULE_NAME = 'pccs' + +_PLUGINS = { + 'k8s': 'pcc.k8s_plugin', + 'slurm': 'pcc.slurm_plugin', + 'ray': 'pcc.ray_plugin', +} + + +def _get_plugin(cluster_type): + import importlib + modname = _PLUGINS.get(cluster_type) + if not modname: + raise ValueError('Unknown cluster_type: ' + cluster_type) + return importlib.import_module(modname) + + +async def ssh_exec(host, port, user, cmd, timeout=120): + """异步 SSH 远程执行命令，返回 (exit_code, stdout, stderr)""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), '{user}@{host}', cmd + ] + proc = await asyncio.create_subprocess_exec( + *ssh_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout) + except asyncio.TimeoutError: + proc.kill() + return -1, '', f'SSH timeout after {timeout}s' + return proc.returncode, stdout.decode(), stderr.decode() + + +async def ssh_exec_bg(host, port, user, cmd): + """异步后台执行，不等待结果""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), user + '@' + host, + 'nohup bash -c "' + cmd.replace('"', '\\"') + '" > /dev/null 2>&1 &' + ] + subprocess.Popen(ssh_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +async def deploy_cluster(request, cluster_id): + """部署集群: 读取集群配置 → 分配节点 → 调用 plugin 部署""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': f'Cluster {cluster_id} not found'} + cluster = recs[0] + + # 更新状态为 deploying + await sor.U('cluster', {'id': cluster_id}, + {'status': 'deploying', 'updated_at': datetime.datetime.now().isoformat()}) + + # 获取已分配的节点 + node_recs = await sor.R('cluster_node', {'cluster_id': cluster_id}) + control_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'control'] + compute_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'compute'] + + control_nodes = [n[0] for n in control_nodes if n] + compute_nodes = [n[0] for n in compute_nodes if n] + + if not control_nodes: + return {'status': 'error', 'message': 'No control node assigned'} + + config = {} + if cluster.control_config: + try: + config = json.loads(cluster.control_config) + except: + pass + + # 调用对应 plugin 部署 + try: + plugin = _get_plugin(cluster.cluster_type) + deploy_log = await plugin.deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config) + except Exception as e: + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'failed', 'deploy_log': str(e), + 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'error', 'message': f'Deploy failed: {e}'} + + # 更新状态为 running + endpoint = deploy_log.get('endpoint', '') if isinstance(deploy_log, dict) else '' + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'endpoint': endpoint, + 'deploy_log': json.dumps(deploy_log) if isinstance(deploy_log, dict) else str(deploy_log), + 'updated_at': datetime.datetime.now().isoformat()}) + # 更新节点状态为 active + for n in control_nodes + compute_nodes: + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': n.id}, + {'status': 'active'}) + + return {'status': 'ok', 'message': 'Cluster deployed', 'endpoint': endpoint} + + +async def add_cluster_node(request, params_kw): + """向运行中集群动态添加节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + role = params_kw.get('role', 'compute') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + + node = await sor.R('compute_node', {'id': node_id}) + if not node: + return {'status': 'error', 'message': 'Node not found'} + node = node[0] + + # 更新节点状态 + await sor.U('compute_node', {'id': node_id}, + {'status': 'allocated', 'updated_at': datetime.datetime.now().isoformat()}) + + # 记录分配 + await sor.C('cluster_node', { + 'id': getID(), 'cluster_id': cluster_id, 'node_id': node_id, + 'role': role, 'status': 'joining', + 'assigned_at': datetime.datetime.now().isoformat() + }) + + # 调用 plugin 添加 + plugin = _get_plugin(cluster.cluster_type) + result = await plugin.add_node(env, cluster_id, node_id, role) + return result + + +async def remove_cluster_node(request, params_kw): + """从集群移除节点并回收""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + + plugin = _get_plugin(cluster[0].cluster_type) + result = await plugin.remove_node(env, cluster_id, node_id) + + # 回收节点 + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('compute_node', {'id': node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': node_id}, + {'status': 'removed'}) + + return result + + +async def cluster_status(request, cluster_id): + """获取集群运行状态""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + + # 从 plugin 获取实时状态 + try: + plugin = _get_plugin(cluster.cluster_type) + live_status = await plugin.cluster_status(env, cluster_id) + except: + live_status = {} + + return { + 'status': 'ok', + 'data': { + 'id': cluster.id, 'name': cluster.name, 'type': cluster.cluster_type, + 'version': cluster.version, 'endpoint': cluster.endpoint, + 'state': cluster.status, + 'nodes': [{'id': n.node_id, 'role': n.role, 'status': n.status} for n in nodes], + 'live': live_status + } + } + + +async def stop_cluster(request, cluster_id): + """停止集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'stopped', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster stopped'} + + +async def start_cluster(request, cluster_id): + """启动已停止的集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster started'} + + +async def destroy_cluster(request, cluster_id): + """销毁集群，回收所有节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + # 释放所有节点 + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + for n in nodes: + await sor.U('compute_node', {'id': n.node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'id': n.id}, {'status': 'removed'}) + await sor.U('cluster', {'id': cluster_id}, + {'status': 'destroyed', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster destroyed'} + +async def cluster_status_all(request, params_kw): + """获取所有集群状态统计""" + env = request._run_ns + dbname = env.get_module_dbname('pcc') + async with DBPools().sqlorContext(dbname) as sor: + clusters = await sor.R('cluster', {}) + if not clusters: + return {'status': 'ok', 'data': {'total': 0, 'running': 0, 'deploying': 0, 'failed': 0, 'stopped': 0}} + statuses = {} + for c in clusters: + s = getattr(c, 'status', 'unknown') + statuses[s] = statuses.get(s, 0) + 1 + return {'status': 'ok', 'data': { + 'total': len(clusters), + 'running': statuses.get('running', 0), + 'deploying': statuses.get('deploying', 0), + 'failed': statuses.get('failed', 0), + 'stopped': statuses.get('stopped', 0), + 'destroyed': statuses.get('destroyed', 0), + }} + diff --git a/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py b/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py new file mode 100644 index 0000000..b2bed1f --- /dev/null +++ b/build/lib/build/lib/build/lib/pcc/k8s_plugin/__init__.py @@ -0,0 +1,221 @@ +""" +k8s_plugin — K8s 集群管理 (基于 kubeadm) +- 控制节点: kubeadm init + CNI(flannel) +- 算力节点: kubeadm join +- 移除: kubectl drain + delete + kubeadm reset +- 状态: kubectl get nodes +""" +from pcc import ssh_exec + +K8S_VERSION = '1.29' +POD_CIDR = '10.244.0.0/16' +SVC_CIDR = '10.96.0.0/12' + + +async def _node_info(env, node_id): + """从 DB 获取节点信息""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_k8s(node): + """在节点上安装 K8s 基础组件 (containerd + kubeadm + kubelet + kubectl)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq apt-transport-https ca-certificates curl gpg && " + "curl -fsSL https://pkgs.k8s.io/core:/stable:/v{ver}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg && " + "echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v{ver}/deb/ /' > /etc/apt/sources.list.d/kubernetes.list && " + "apt-get update -qq && " + "apt-get install -y -qq kubelet kubeadm kubectl containerd && " + "apt-mark hold kubelet kubeadm kubectl && " + "mkdir -p /etc/containerd && " + "containerd config default > /etc/containerd/config.toml 2>/dev/null || true && " + "sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml && " + "systemctl restart containerd && systemctl enable kubelet" + ).format(ver=K8S_VERSION) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _install_cni(control_node, pod_cidr): + """在控制节点安装 CNI 网络插件""" + cmd = f"kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml" + return await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=120) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """K8s 集群部署：只部署控制节点。算力节点后续通过 add_node 加入。""" + pod_cidr = config.get('pod_cidr', POD_CIDR) + svc_cidr = config.get('svc_cidr', SVC_CIDR) + results = {'control_nodes': {}, 'compute_nodes': None, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 K8s 基础组件 + rc, out, err = await _install_k8s(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err}'} + continue + + # 2. kubeadm init（仅第一个控制节点） + init_cmd = ( + f"kubeadm init --pod-network-cidr={pod_cidr} --service-cidr={svc_cidr} " + f"--kubernetes-version=v{K8S_VERSION} --ignore-preflight-errors=all" + ) + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', init_cmd, timeout=300) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'init failed: {err[:200]}'} + continue + + # 3. 配置 kubectl + setup_cmd = "mkdir -p $HOME/.kube && cp -f /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config" + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', setup_cmd) + + # 4. 安装 CNI + await _install_cni(node, pod_cidr) + + # 5. 获取 join token（供后续算力节点使用） + rc2, token_out, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "kubeadm token create --print-join-command", timeout=30) + join_cmd = token_out.strip() if rc2 == 0 else '' + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'join_command': join_cmd, + 'endpoint': f'https://{node.ip_address}:6443' + } + results['endpoint'] = f'https://{node.ip_address}:6443' + + # 保存 join_command 到集群 config + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 K8s 集群添加算力节点：安装 k8s → kubeadm join""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 读取 join_command + from sqlor.dbpools import DBPools + from pcc import MODULE_NAME + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'Cluster not found'} + config = json.loads(recs[0].control_config or '{}') + join_cmd = '' + for cn_data in config.get('control_nodes', {}).values(): + if cn_data.get('join_command'): + join_cmd = cn_data['join_command'] + break + if not join_cmd: + return {'status': 'error', 'message': 'No join command found, cluster may not be deployed'} + + # 1. 安装 K8s + rc, out, err = await _install_k8s(node) + if rc != 0: + return {'status': 'error', 'message': f'k8s install failed: {err[:200]}'} + + # 2. kubeadm join + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', join_cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined K8s cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 K8s 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # drain + delete + reset + cmds = [ + f"kubectl drain {node.name} --ignore-daemonsets --delete-emptydir-data --timeout=60s", + f"kubectl delete node {node.name}", + "kubeadm reset -f", + ] + for cmd in cmds: + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + return {'status': 'ok', 'message': f'Node {node.name} removed'} + + +async def cluster_status(env, cluster_id): + """获取 K8s 集群状态""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + + control_id = recs[0].node_id + cnode = await _node_info(env, control_id) + if not cnode: + return {'error': 'Control node not found'} + + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'kubectl get nodes -o wide', timeout=30) + return {'status': 'ok', 'kubectl_nodes': out if rc == 0 else 'kubectl failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """K8s 算力单元分配: 创建 namespace + ResourceQuota""" + from pcc import ssh_exec as _ssh + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + + quota_yaml = ( + f"apiVersion: v1\\nkind: ResourceQuota\\nmetadata:\\n name: {unit_name}-quota\\n namespace: {unit_name}\\n" + f"spec:\\n hard:\\n requests.cpu: \\\"{cpu}\\\"\\n requests.memory: \\\"{memory}Gi\\\"\\n requests.nvidia.com/gpu: \\\"{gpu}\\\"" + ) + cmds = [ + f"kubectl create namespace {unit_name}", + f"echo '{quota_yaml}' | kubectl apply -f -", + ] + for cmd in cmds: + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'kubectl failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name}: ns + {cpu}C/{memory}G/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """K8s 算力单元回收: 删除 namespace""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + rc, _, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"kubectl delete namespace {unit_name} --timeout=60s") + if rc != 0: + return {'status': 'error', 'message': f'delete ns failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py b/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py new file mode 100644 index 0000000..e0bae45 --- /dev/null +++ b/build/lib/build/lib/build/lib/pcc/ray_plugin/__init__.py @@ -0,0 +1,174 @@ +""" +ray_plugin — Ray 集群管理 +- 控制节点: ray start --head +- 算力节点: ray start --address=:6379 +- 移除: ray stop +- 状态: ray status +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_ray(node): + """安装 Ray""" + script = ( + "apt-get update -qq && apt-get install -y -qq python3 python3-pip && " + "pip3 install -q ray[default] 2>/dev/null || pip install -q ray[default]" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Ray 集群部署：仅部署 head 节点""" + head_port = config.get('head_port', 6379) + dashboard_port = config.get('dashboard_port', 8265) + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 Ray + rc, out, err = await _install_ray(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. ray start --head + cmd = ( + f"ray start --head --port={head_port} --dashboard-host=0.0.0.0 " + f"--dashboard-port={dashboard_port} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'start failed: {err2[:200]}'} + continue + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'head_address': f'{node.ip_address}:{head_port}', + 'dashboard': f'http://{node.ip_address}:{dashboard_port}' + } + results['endpoint'] = f'{node.ip_address}:{head_port}' + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Ray 集群添加 worker 节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取 head 节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + cluster_recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + head_node = await _node_info(env, recs[0].node_id) + if not head_node: + return {'status': 'error', 'message': 'Head node not found'} + + # 从 cluster config 读 head 地址 + config = json.loads(cluster_recs[0].control_config or '{}') if cluster_recs else {} + head_addr = config.get('endpoint', f'{head_node.ip_address}:6379') + + # 1. 安装 Ray + rc, _, err = await _install_ray(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. ray start --address + cmd = ( + f"ray start --address={head_addr} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined Ray cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Ray 集群移除 worker""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + rc, _, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', 'ray stop', timeout=60) + return {'status': 'ok', 'message': f'Node {node.name} removed from Ray'} + + +async def cluster_status(env, cluster_id): + """ray status""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'ray status', timeout=30) + return {'status': 'ok', 'ray_status': out if rc == 0 else 'ray status failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Ray 算力单元分配: 创建 detached actor / placement group 绑定资源标签""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 用 ray submit 提交一个资源占位 job + script = ( + "import ray; ray.init(address='auto'); " + "pg = ray.util.placement_group([{'CPU': " + str(cpu) + ", 'GPU': " + str(gpu) + "}], strategy='STRICT_SPREAD', name='" + unit_name + "'); " + "ray.get(pg.ready()); print('PG ready: " + unit_name + "')" + ) + py_cmd = f"python3 -c '{script}'" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=60) + if rc != 0: + return {'status': 'error', 'message': f'ray pg failed: {err[:200]}'} + return {'status': 'ok', 'message': f'PlacementGroup {unit_name}: {cpu}C/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """Ray 算力单元回收: 删除 placement group""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + py_cmd = f"python3 -c 'import ray; ray.init(address=\"auto\"); ray.util.remove_placement_group(ray.util.get_placement_group(\"{unit_name}\")); print(\"removed\")'" + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=30) + return {'status': 'ok', 'message': f'PlacementGroup {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py b/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py new file mode 100644 index 0000000..989c5a5 --- /dev/null +++ b/build/lib/build/lib/build/lib/pcc/slurm_plugin/__init__.py @@ -0,0 +1,229 @@ +""" +slurm_plugin — Slurm 集群管理 +- 控制节点: slurmctld + munge + NFS server +- 算力节点: slurmd + munge + NFS client +- 移除: drain node → scontrol delete → reset +- 状态: sinfo +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_slurm_deps(node): + """安装 Slurm 依赖 (munge + slurm)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq munge slurm-wlm slurm-client nfs-common nfs-kernel-server " + "&& systemctl enable munge && systemctl start munge" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _setup_munge(control_node, compute_node): + """复制 munge key 从控制节点到算力节点""" + cmd = f"scp -o StrictHostKeyChecking=no /etc/munge/munge.key {compute_node.ssh_user or 'root'}@{compute_node.ip_address}:/etc/munge/munge.key" + rc, _, err = await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=30) + if rc == 0: + await ssh_exec(compute_node.ip_address, compute_node.ssh_port or 22, + compute_node.ssh_user or 'root', + "chown munge:munge /etc/munge/munge.key && chmod 400 /etc/munge/munge.key && systemctl restart munge", timeout=30) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Slurm 集群部署：控制节点安装 slurmctld + munge""" + cluster_name = config.get('cluster_name', 'pccs-cluster') + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装依赖 + rc, out, err = await _install_slurm_deps(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. 生成 slurm.conf + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + conf = f"""ClusterName={cluster_name} +ControlMachine={node.ip_address} +SlurmUser=root +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/spool/slurmctld +SlurmdSpoolDir=/var/spool/slurmd +ReturnToService=1 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core +AccountingStorageType=accounting_storage/none +JobCompType=jobcomp/none +NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN +PartitionName=debug Nodes={node.name} Default=YES MaxTime=INFINITE State=UP +""" + # 写配置到控制节点 + write_cmd = f"cat > /etc/slurm/slurm.conf << 'SLURM_EOF'\n{conf}\nSLURM_EOF" + rc2, _, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', write_cmd, timeout=30) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'config write failed: {err2}'} + continue + + # 3. 启动 slurmctld + await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "mkdir -p /var/spool/slurmctld /var/spool/slurmd && " + "systemctl enable slurmctld && systemctl start slurmctld", timeout=60) + + results['control_nodes'][node.id] = {'status': 'ok', 'slurm_conf': conf} + results['endpoint'] = node.ip_address + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Slurm 集群添加算力节点：安装 slurmd + munge → 更新 slurm.conf""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'status': 'error', 'message': 'Control node not found'} + + # 1. 安装依赖 + rc, _, err = await _install_slurm_deps(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. 复制 munge key + await _setup_munge(cnode, node) + + # 3. 更新控制节点 slurm.conf 添加此节点 + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + add_line = f"NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN" + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"echo '{add_line}' >> /etc/slurm/slurm.conf && scontrol reconfigure", timeout=30) + + # 4. 启动 slurmd + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl enable slurmd && systemctl start slurmd", timeout=60) + + return {'status': 'ok', 'message': f'Node {node.name} joined Slurm cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Slurm 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control'}) + if recs: + cnode = await _node_info(env, recs[0].node_id) + if cnode: + # drain → remove from slurm.conf → scontrol reconfigure + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"scontrol update NodeName={node.name} State=DOWN Reason=removing && " + f"sed -i '/NodeName={node.name}/d' /etc/slurm/slurm.conf && " + "scontrol reconfigure", timeout=30) + + # stop slurmd on the node + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl stop slurmd && systemctl disable slurmd", timeout=30) + + return {'status': 'ok', 'message': f'Node {node.name} removed from Slurm'} + + +async def cluster_status(env, cluster_id): + """sinfo""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'sinfo', timeout=30) + return {'status': 'ok', 'sinfo': out if rc == 0 else 'sinfo failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Slurm 算力单元分配: 创建 partition + 关联节点""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 创建 partition + cmd = f"scontrol create PartitionName={unit_name} MaxNodes=UNLIMITED Default=NO MaxTime=UNLIMITED State=UP" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'scontrol failed: {err[:200]}'} + + # 分配 node 到 partition (通过 cluster_node 表找该集群下可用的 compute 节点) + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + assigned = await sor.sqlExe( + """SELECT cn.name FROM cluster_node cnn + JOIN compute_node cn ON cn.id = cnn.node_id + WHERE cnn.cluster_id=${cid}$ AND cnn.role='compute' AND cnn.status='active' + LIMIT 1""", + {'cid': cluster_id} + ) + if assigned: + node_name = assigned[0].name + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol update NodeName={node_name} Partition={unit_name}") + + return {'status': 'ok', 'message': f'Partition {unit_name} created'} + + +async def release_unit(env, cluster_id, unit_name): + """Slurm 算力单元回收: 删除 partition""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol delete PartitionName={unit_name}") + return {'status': 'ok', 'message': f'Partition {unit_name} deleted'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/build/lib/scripts/load_path.py b/build/lib/build/lib/build/lib/scripts/load_path.py new file mode 100644 index 0000000..59b37c6 --- /dev/null +++ b/build/lib/build/lib/build/lib/scripts/load_path.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""pcc RBAC 权限管理""" +import subprocess, os, sys, json, glob + +def find_sage_root(): + for c in [os.path.expanduser("~/sage"), os.path.expanduser("~/repos/sage")]: + if os.path.isdir(os.path.join(c, "py3")): return c + return None + +SAGE = find_sage_root() +if not SAGE: sys.exit("Sage root not found") +PY = os.path.join(SAGE, "py3", "bin", "python") +SET = os.path.join(SAGE, "set_role_perm.py") + +JSON_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "json") +API_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "wwwroot", "api") + +def load_cruds(): + defs = [] + for f in sorted(glob.glob(os.path.join(JSON_DIR, "*.json"))): + try: + d = json.load(open(f)) + defs.append({"tblname": d["tblname"], "alias": d.get("alias", d["tblname"]), "subtables": d.get("params", {}).get("subtables", [])}) + except: pass + return defs + +def get_apis(): + if not os.path.isdir(API_DIR): return [] + return [f"/pcc/api/{f}" for f in sorted(os.listdir(API_DIR)) if f.endswith(".dspy")] + +cruds = load_cruds() +apis = get_apis() + +PATHS_ANY = [ + f"/pcc/menu.ui", +] +PATHS_LOGINED = [ + f"/pcc", + f"/pcc/index.ui", +] +for d in cruds: + PATHS_ANY.append(f"/pcc/{d['alias']}") + PATHS_LOGINED.append(f"/pcc/{d['alias']}/index.ui") + for act in ["get", "add", "update", "delete"]: + PATHS_LOGINED.append(f"/pcc/{d['alias']}/{act}_{d['tblname']}.dspy") +for api in apis: + PATHS_LOGINED.append(api) + +PATHS_OPERATOR = list(PATHS_LOGINED) + +PATHS_ANY = list(dict.fromkeys(PATHS_ANY)) +PATHS_LOGINED = list(dict.fromkeys(PATHS_LOGINED)) +PATHS_OPERATOR = list(dict.fromkeys(PATHS_OPERATOR)) + +def reg(role, paths): + ok = 0 + for p in paths: + r = subprocess.run([PY, SET, role, p], capture_output=True, text=True) + if r.returncode == 0: ok += 1 + print(f" {role}: {ok}/{len(paths)}") + return ok + +total = 0 +print(f"{mod_name}: any={len(PATHS_ANY)} logined={len(PATHS_LOGINED)} operator={len(PATHS_OPERATOR)}") +total += reg("any", PATHS_ANY) +total += reg("logined", PATHS_LOGINED) +total += reg("reseller.operator", PATHS_OPERATOR) +print(f"Done. {total} entries.") diff --git a/build/lib/build/lib/pcc/__init__.py b/build/lib/build/lib/pcc/__init__.py new file mode 100644 index 0000000..8c44552 --- /dev/null +++ b/build/lib/build/lib/pcc/__init__.py @@ -0,0 +1,264 @@ +""" +pcc — 统一集群管理 (Pooled Computing Cluster) + 生命周期: deploy → run → stop → start → destroy + 插件: k8s_plugin, slurm_plugin, ray_plugin +""" +import datetime, json, asyncio, subprocess +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + +MODULE_NAME = 'pccs' + +_PLUGINS = { + 'k8s': 'pcc.k8s_plugin', + 'slurm': 'pcc.slurm_plugin', + 'ray': 'pcc.ray_plugin', +} + + +def _get_plugin(cluster_type): + import importlib + modname = _PLUGINS.get(cluster_type) + if not modname: + raise ValueError('Unknown cluster_type: ' + cluster_type) + return importlib.import_module(modname) + + +async def ssh_exec(host, port, user, cmd, timeout=120): + """异步 SSH 远程执行命令，返回 (exit_code, stdout, stderr)""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), '{user}@{host}', cmd + ] + proc = await asyncio.create_subprocess_exec( + *ssh_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout) + except asyncio.TimeoutError: + proc.kill() + return -1, '', f'SSH timeout after {timeout}s' + return proc.returncode, stdout.decode(), stderr.decode() + + +async def ssh_exec_bg(host, port, user, cmd): + """异步后台执行，不等待结果""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), user + '@' + host, + 'nohup bash -c "' + cmd.replace('"', '\\"') + '" > /dev/null 2>&1 &' + ] + subprocess.Popen(ssh_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +async def deploy_cluster(request, cluster_id): + """部署集群: 读取集群配置 → 分配节点 → 调用 plugin 部署""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': f'Cluster {cluster_id} not found'} + cluster = recs[0] + + # 更新状态为 deploying + await sor.U('cluster', {'id': cluster_id}, + {'status': 'deploying', 'updated_at': datetime.datetime.now().isoformat()}) + + # 获取已分配的节点 + node_recs = await sor.R('cluster_node', {'cluster_id': cluster_id}) + control_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'control'] + compute_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'compute'] + + control_nodes = [n[0] for n in control_nodes if n] + compute_nodes = [n[0] for n in compute_nodes if n] + + if not control_nodes: + return {'status': 'error', 'message': 'No control node assigned'} + + config = {} + if cluster.control_config: + try: + config = json.loads(cluster.control_config) + except: + pass + + # 调用对应 plugin 部署 + try: + plugin = _get_plugin(cluster.cluster_type) + deploy_log = await plugin.deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config) + except Exception as e: + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'failed', 'deploy_log': str(e), + 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'error', 'message': f'Deploy failed: {e}'} + + # 更新状态为 running + endpoint = deploy_log.get('endpoint', '') if isinstance(deploy_log, dict) else '' + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'endpoint': endpoint, + 'deploy_log': json.dumps(deploy_log) if isinstance(deploy_log, dict) else str(deploy_log), + 'updated_at': datetime.datetime.now().isoformat()}) + # 更新节点状态为 active + for n in control_nodes + compute_nodes: + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': n.id}, + {'status': 'active'}) + + return {'status': 'ok', 'message': 'Cluster deployed', 'endpoint': endpoint} + + +async def add_cluster_node(request, params_kw): + """向运行中集群动态添加节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + role = params_kw.get('role', 'compute') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + + node = await sor.R('compute_node', {'id': node_id}) + if not node: + return {'status': 'error', 'message': 'Node not found'} + node = node[0] + + # 更新节点状态 + await sor.U('compute_node', {'id': node_id}, + {'status': 'allocated', 'updated_at': datetime.datetime.now().isoformat()}) + + # 记录分配 + await sor.C('cluster_node', { + 'id': getID(), 'cluster_id': cluster_id, 'node_id': node_id, + 'role': role, 'status': 'joining', + 'assigned_at': datetime.datetime.now().isoformat() + }) + + # 调用 plugin 添加 + plugin = _get_plugin(cluster.cluster_type) + result = await plugin.add_node(env, cluster_id, node_id, role) + return result + + +async def remove_cluster_node(request, params_kw): + """从集群移除节点并回收""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + + plugin = _get_plugin(cluster[0].cluster_type) + result = await plugin.remove_node(env, cluster_id, node_id) + + # 回收节点 + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('compute_node', {'id': node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': node_id}, + {'status': 'removed'}) + + return result + + +async def cluster_status(request, cluster_id): + """获取集群运行状态""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + + # 从 plugin 获取实时状态 + try: + plugin = _get_plugin(cluster.cluster_type) + live_status = await plugin.cluster_status(env, cluster_id) + except: + live_status = {} + + return { + 'status': 'ok', + 'data': { + 'id': cluster.id, 'name': cluster.name, 'type': cluster.cluster_type, + 'version': cluster.version, 'endpoint': cluster.endpoint, + 'state': cluster.status, + 'nodes': [{'id': n.node_id, 'role': n.role, 'status': n.status} for n in nodes], + 'live': live_status + } + } + + +async def stop_cluster(request, cluster_id): + """停止集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'stopped', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster stopped'} + + +async def start_cluster(request, cluster_id): + """启动已停止的集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster started'} + + +async def destroy_cluster(request, cluster_id): + """销毁集群，回收所有节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + # 释放所有节点 + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + for n in nodes: + await sor.U('compute_node', {'id': n.node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'id': n.id}, {'status': 'removed'}) + await sor.U('cluster', {'id': cluster_id}, + {'status': 'destroyed', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster destroyed'} + +async def cluster_status_all(request, params_kw): + """获取所有集群状态统计""" + env = request._run_ns + dbname = env.get_module_dbname('pcc') + async with DBPools().sqlorContext(dbname) as sor: + clusters = await sor.R('cluster', {}) + if not clusters: + return {'status': 'ok', 'data': {'total': 0, 'running': 0, 'deploying': 0, 'failed': 0, 'stopped': 0}} + statuses = {} + for c in clusters: + s = getattr(c, 'status', 'unknown') + statuses[s] = statuses.get(s, 0) + 1 + return {'status': 'ok', 'data': { + 'total': len(clusters), + 'running': statuses.get('running', 0), + 'deploying': statuses.get('deploying', 0), + 'failed': statuses.get('failed', 0), + 'stopped': statuses.get('stopped', 0), + 'destroyed': statuses.get('destroyed', 0), + }} + diff --git a/build/lib/build/lib/pcc/k8s_plugin/__init__.py b/build/lib/build/lib/pcc/k8s_plugin/__init__.py new file mode 100644 index 0000000..b2bed1f --- /dev/null +++ b/build/lib/build/lib/pcc/k8s_plugin/__init__.py @@ -0,0 +1,221 @@ +""" +k8s_plugin — K8s 集群管理 (基于 kubeadm) +- 控制节点: kubeadm init + CNI(flannel) +- 算力节点: kubeadm join +- 移除: kubectl drain + delete + kubeadm reset +- 状态: kubectl get nodes +""" +from pcc import ssh_exec + +K8S_VERSION = '1.29' +POD_CIDR = '10.244.0.0/16' +SVC_CIDR = '10.96.0.0/12' + + +async def _node_info(env, node_id): + """从 DB 获取节点信息""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_k8s(node): + """在节点上安装 K8s 基础组件 (containerd + kubeadm + kubelet + kubectl)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq apt-transport-https ca-certificates curl gpg && " + "curl -fsSL https://pkgs.k8s.io/core:/stable:/v{ver}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg && " + "echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v{ver}/deb/ /' > /etc/apt/sources.list.d/kubernetes.list && " + "apt-get update -qq && " + "apt-get install -y -qq kubelet kubeadm kubectl containerd && " + "apt-mark hold kubelet kubeadm kubectl && " + "mkdir -p /etc/containerd && " + "containerd config default > /etc/containerd/config.toml 2>/dev/null || true && " + "sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml && " + "systemctl restart containerd && systemctl enable kubelet" + ).format(ver=K8S_VERSION) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _install_cni(control_node, pod_cidr): + """在控制节点安装 CNI 网络插件""" + cmd = f"kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml" + return await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=120) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """K8s 集群部署：只部署控制节点。算力节点后续通过 add_node 加入。""" + pod_cidr = config.get('pod_cidr', POD_CIDR) + svc_cidr = config.get('svc_cidr', SVC_CIDR) + results = {'control_nodes': {}, 'compute_nodes': None, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 K8s 基础组件 + rc, out, err = await _install_k8s(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err}'} + continue + + # 2. kubeadm init（仅第一个控制节点） + init_cmd = ( + f"kubeadm init --pod-network-cidr={pod_cidr} --service-cidr={svc_cidr} " + f"--kubernetes-version=v{K8S_VERSION} --ignore-preflight-errors=all" + ) + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', init_cmd, timeout=300) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'init failed: {err[:200]}'} + continue + + # 3. 配置 kubectl + setup_cmd = "mkdir -p $HOME/.kube && cp -f /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config" + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', setup_cmd) + + # 4. 安装 CNI + await _install_cni(node, pod_cidr) + + # 5. 获取 join token（供后续算力节点使用） + rc2, token_out, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "kubeadm token create --print-join-command", timeout=30) + join_cmd = token_out.strip() if rc2 == 0 else '' + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'join_command': join_cmd, + 'endpoint': f'https://{node.ip_address}:6443' + } + results['endpoint'] = f'https://{node.ip_address}:6443' + + # 保存 join_command 到集群 config + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 K8s 集群添加算力节点：安装 k8s → kubeadm join""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 读取 join_command + from sqlor.dbpools import DBPools + from pcc import MODULE_NAME + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'Cluster not found'} + config = json.loads(recs[0].control_config or '{}') + join_cmd = '' + for cn_data in config.get('control_nodes', {}).values(): + if cn_data.get('join_command'): + join_cmd = cn_data['join_command'] + break + if not join_cmd: + return {'status': 'error', 'message': 'No join command found, cluster may not be deployed'} + + # 1. 安装 K8s + rc, out, err = await _install_k8s(node) + if rc != 0: + return {'status': 'error', 'message': f'k8s install failed: {err[:200]}'} + + # 2. kubeadm join + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', join_cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined K8s cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 K8s 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # drain + delete + reset + cmds = [ + f"kubectl drain {node.name} --ignore-daemonsets --delete-emptydir-data --timeout=60s", + f"kubectl delete node {node.name}", + "kubeadm reset -f", + ] + for cmd in cmds: + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + return {'status': 'ok', 'message': f'Node {node.name} removed'} + + +async def cluster_status(env, cluster_id): + """获取 K8s 集群状态""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + + control_id = recs[0].node_id + cnode = await _node_info(env, control_id) + if not cnode: + return {'error': 'Control node not found'} + + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'kubectl get nodes -o wide', timeout=30) + return {'status': 'ok', 'kubectl_nodes': out if rc == 0 else 'kubectl failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """K8s 算力单元分配: 创建 namespace + ResourceQuota""" + from pcc import ssh_exec as _ssh + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + + quota_yaml = ( + f"apiVersion: v1\\nkind: ResourceQuota\\nmetadata:\\n name: {unit_name}-quota\\n namespace: {unit_name}\\n" + f"spec:\\n hard:\\n requests.cpu: \\\"{cpu}\\\"\\n requests.memory: \\\"{memory}Gi\\\"\\n requests.nvidia.com/gpu: \\\"{gpu}\\\"" + ) + cmds = [ + f"kubectl create namespace {unit_name}", + f"echo '{quota_yaml}' | kubectl apply -f -", + ] + for cmd in cmds: + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'kubectl failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name}: ns + {cpu}C/{memory}G/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """K8s 算力单元回收: 删除 namespace""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + rc, _, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"kubectl delete namespace {unit_name} --timeout=60s") + if rc != 0: + return {'status': 'error', 'message': f'delete ns failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/pcc/ray_plugin/__init__.py b/build/lib/build/lib/pcc/ray_plugin/__init__.py new file mode 100644 index 0000000..e0bae45 --- /dev/null +++ b/build/lib/build/lib/pcc/ray_plugin/__init__.py @@ -0,0 +1,174 @@ +""" +ray_plugin — Ray 集群管理 +- 控制节点: ray start --head +- 算力节点: ray start --address=:6379 +- 移除: ray stop +- 状态: ray status +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_ray(node): + """安装 Ray""" + script = ( + "apt-get update -qq && apt-get install -y -qq python3 python3-pip && " + "pip3 install -q ray[default] 2>/dev/null || pip install -q ray[default]" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Ray 集群部署：仅部署 head 节点""" + head_port = config.get('head_port', 6379) + dashboard_port = config.get('dashboard_port', 8265) + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 Ray + rc, out, err = await _install_ray(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. ray start --head + cmd = ( + f"ray start --head --port={head_port} --dashboard-host=0.0.0.0 " + f"--dashboard-port={dashboard_port} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'start failed: {err2[:200]}'} + continue + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'head_address': f'{node.ip_address}:{head_port}', + 'dashboard': f'http://{node.ip_address}:{dashboard_port}' + } + results['endpoint'] = f'{node.ip_address}:{head_port}' + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Ray 集群添加 worker 节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取 head 节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + cluster_recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + head_node = await _node_info(env, recs[0].node_id) + if not head_node: + return {'status': 'error', 'message': 'Head node not found'} + + # 从 cluster config 读 head 地址 + config = json.loads(cluster_recs[0].control_config or '{}') if cluster_recs else {} + head_addr = config.get('endpoint', f'{head_node.ip_address}:6379') + + # 1. 安装 Ray + rc, _, err = await _install_ray(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. ray start --address + cmd = ( + f"ray start --address={head_addr} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined Ray cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Ray 集群移除 worker""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + rc, _, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', 'ray stop', timeout=60) + return {'status': 'ok', 'message': f'Node {node.name} removed from Ray'} + + +async def cluster_status(env, cluster_id): + """ray status""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'ray status', timeout=30) + return {'status': 'ok', 'ray_status': out if rc == 0 else 'ray status failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Ray 算力单元分配: 创建 detached actor / placement group 绑定资源标签""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 用 ray submit 提交一个资源占位 job + script = ( + "import ray; ray.init(address='auto'); " + "pg = ray.util.placement_group([{'CPU': " + str(cpu) + ", 'GPU': " + str(gpu) + "}], strategy='STRICT_SPREAD', name='" + unit_name + "'); " + "ray.get(pg.ready()); print('PG ready: " + unit_name + "')" + ) + py_cmd = f"python3 -c '{script}'" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=60) + if rc != 0: + return {'status': 'error', 'message': f'ray pg failed: {err[:200]}'} + return {'status': 'ok', 'message': f'PlacementGroup {unit_name}: {cpu}C/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """Ray 算力单元回收: 删除 placement group""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + py_cmd = f"python3 -c 'import ray; ray.init(address=\"auto\"); ray.util.remove_placement_group(ray.util.get_placement_group(\"{unit_name}\")); print(\"removed\")'" + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=30) + return {'status': 'ok', 'message': f'PlacementGroup {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/pcc/slurm_plugin/__init__.py b/build/lib/build/lib/pcc/slurm_plugin/__init__.py new file mode 100644 index 0000000..989c5a5 --- /dev/null +++ b/build/lib/build/lib/pcc/slurm_plugin/__init__.py @@ -0,0 +1,229 @@ +""" +slurm_plugin — Slurm 集群管理 +- 控制节点: slurmctld + munge + NFS server +- 算力节点: slurmd + munge + NFS client +- 移除: drain node → scontrol delete → reset +- 状态: sinfo +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_slurm_deps(node): + """安装 Slurm 依赖 (munge + slurm)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq munge slurm-wlm slurm-client nfs-common nfs-kernel-server " + "&& systemctl enable munge && systemctl start munge" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _setup_munge(control_node, compute_node): + """复制 munge key 从控制节点到算力节点""" + cmd = f"scp -o StrictHostKeyChecking=no /etc/munge/munge.key {compute_node.ssh_user or 'root'}@{compute_node.ip_address}:/etc/munge/munge.key" + rc, _, err = await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=30) + if rc == 0: + await ssh_exec(compute_node.ip_address, compute_node.ssh_port or 22, + compute_node.ssh_user or 'root', + "chown munge:munge /etc/munge/munge.key && chmod 400 /etc/munge/munge.key && systemctl restart munge", timeout=30) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Slurm 集群部署：控制节点安装 slurmctld + munge""" + cluster_name = config.get('cluster_name', 'pccs-cluster') + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装依赖 + rc, out, err = await _install_slurm_deps(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. 生成 slurm.conf + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + conf = f"""ClusterName={cluster_name} +ControlMachine={node.ip_address} +SlurmUser=root +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/spool/slurmctld +SlurmdSpoolDir=/var/spool/slurmd +ReturnToService=1 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core +AccountingStorageType=accounting_storage/none +JobCompType=jobcomp/none +NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN +PartitionName=debug Nodes={node.name} Default=YES MaxTime=INFINITE State=UP +""" + # 写配置到控制节点 + write_cmd = f"cat > /etc/slurm/slurm.conf << 'SLURM_EOF'\n{conf}\nSLURM_EOF" + rc2, _, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', write_cmd, timeout=30) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'config write failed: {err2}'} + continue + + # 3. 启动 slurmctld + await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "mkdir -p /var/spool/slurmctld /var/spool/slurmd && " + "systemctl enable slurmctld && systemctl start slurmctld", timeout=60) + + results['control_nodes'][node.id] = {'status': 'ok', 'slurm_conf': conf} + results['endpoint'] = node.ip_address + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Slurm 集群添加算力节点：安装 slurmd + munge → 更新 slurm.conf""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'status': 'error', 'message': 'Control node not found'} + + # 1. 安装依赖 + rc, _, err = await _install_slurm_deps(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. 复制 munge key + await _setup_munge(cnode, node) + + # 3. 更新控制节点 slurm.conf 添加此节点 + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + add_line = f"NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN" + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"echo '{add_line}' >> /etc/slurm/slurm.conf && scontrol reconfigure", timeout=30) + + # 4. 启动 slurmd + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl enable slurmd && systemctl start slurmd", timeout=60) + + return {'status': 'ok', 'message': f'Node {node.name} joined Slurm cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Slurm 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control'}) + if recs: + cnode = await _node_info(env, recs[0].node_id) + if cnode: + # drain → remove from slurm.conf → scontrol reconfigure + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"scontrol update NodeName={node.name} State=DOWN Reason=removing && " + f"sed -i '/NodeName={node.name}/d' /etc/slurm/slurm.conf && " + "scontrol reconfigure", timeout=30) + + # stop slurmd on the node + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl stop slurmd && systemctl disable slurmd", timeout=30) + + return {'status': 'ok', 'message': f'Node {node.name} removed from Slurm'} + + +async def cluster_status(env, cluster_id): + """sinfo""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'sinfo', timeout=30) + return {'status': 'ok', 'sinfo': out if rc == 0 else 'sinfo failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Slurm 算力单元分配: 创建 partition + 关联节点""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 创建 partition + cmd = f"scontrol create PartitionName={unit_name} MaxNodes=UNLIMITED Default=NO MaxTime=UNLIMITED State=UP" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'scontrol failed: {err[:200]}'} + + # 分配 node 到 partition (通过 cluster_node 表找该集群下可用的 compute 节点) + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + assigned = await sor.sqlExe( + """SELECT cn.name FROM cluster_node cnn + JOIN compute_node cn ON cn.id = cnn.node_id + WHERE cnn.cluster_id=${cid}$ AND cnn.role='compute' AND cnn.status='active' + LIMIT 1""", + {'cid': cluster_id} + ) + if assigned: + node_name = assigned[0].name + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol update NodeName={node_name} Partition={unit_name}") + + return {'status': 'ok', 'message': f'Partition {unit_name} created'} + + +async def release_unit(env, cluster_id, unit_name): + """Slurm 算力单元回收: 删除 partition""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol delete PartitionName={unit_name}") + return {'status': 'ok', 'message': f'Partition {unit_name} deleted'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/build/lib/scripts/load_path.py b/build/lib/build/lib/scripts/load_path.py new file mode 100644 index 0000000..59b37c6 --- /dev/null +++ b/build/lib/build/lib/scripts/load_path.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""pcc RBAC 权限管理""" +import subprocess, os, sys, json, glob + +def find_sage_root(): + for c in [os.path.expanduser("~/sage"), os.path.expanduser("~/repos/sage")]: + if os.path.isdir(os.path.join(c, "py3")): return c + return None + +SAGE = find_sage_root() +if not SAGE: sys.exit("Sage root not found") +PY = os.path.join(SAGE, "py3", "bin", "python") +SET = os.path.join(SAGE, "set_role_perm.py") + +JSON_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "json") +API_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "wwwroot", "api") + +def load_cruds(): + defs = [] + for f in sorted(glob.glob(os.path.join(JSON_DIR, "*.json"))): + try: + d = json.load(open(f)) + defs.append({"tblname": d["tblname"], "alias": d.get("alias", d["tblname"]), "subtables": d.get("params", {}).get("subtables", [])}) + except: pass + return defs + +def get_apis(): + if not os.path.isdir(API_DIR): return [] + return [f"/pcc/api/{f}" for f in sorted(os.listdir(API_DIR)) if f.endswith(".dspy")] + +cruds = load_cruds() +apis = get_apis() + +PATHS_ANY = [ + f"/pcc/menu.ui", +] +PATHS_LOGINED = [ + f"/pcc", + f"/pcc/index.ui", +] +for d in cruds: + PATHS_ANY.append(f"/pcc/{d['alias']}") + PATHS_LOGINED.append(f"/pcc/{d['alias']}/index.ui") + for act in ["get", "add", "update", "delete"]: + PATHS_LOGINED.append(f"/pcc/{d['alias']}/{act}_{d['tblname']}.dspy") +for api in apis: + PATHS_LOGINED.append(api) + +PATHS_OPERATOR = list(PATHS_LOGINED) + +PATHS_ANY = list(dict.fromkeys(PATHS_ANY)) +PATHS_LOGINED = list(dict.fromkeys(PATHS_LOGINED)) +PATHS_OPERATOR = list(dict.fromkeys(PATHS_OPERATOR)) + +def reg(role, paths): + ok = 0 + for p in paths: + r = subprocess.run([PY, SET, role, p], capture_output=True, text=True) + if r.returncode == 0: ok += 1 + print(f" {role}: {ok}/{len(paths)}") + return ok + +total = 0 +print(f"{mod_name}: any={len(PATHS_ANY)} logined={len(PATHS_LOGINED)} operator={len(PATHS_OPERATOR)}") +total += reg("any", PATHS_ANY) +total += reg("logined", PATHS_LOGINED) +total += reg("reseller.operator", PATHS_OPERATOR) +print(f"Done. {total} entries.") diff --git a/build/lib/pcc/__init__.py b/build/lib/pcc/__init__.py new file mode 100644 index 0000000..8c44552 --- /dev/null +++ b/build/lib/pcc/__init__.py @@ -0,0 +1,264 @@ +""" +pcc — 统一集群管理 (Pooled Computing Cluster) + 生命周期: deploy → run → stop → start → destroy + 插件: k8s_plugin, slurm_plugin, ray_plugin +""" +import datetime, json, asyncio, subprocess +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + +MODULE_NAME = 'pccs' + +_PLUGINS = { + 'k8s': 'pcc.k8s_plugin', + 'slurm': 'pcc.slurm_plugin', + 'ray': 'pcc.ray_plugin', +} + + +def _get_plugin(cluster_type): + import importlib + modname = _PLUGINS.get(cluster_type) + if not modname: + raise ValueError('Unknown cluster_type: ' + cluster_type) + return importlib.import_module(modname) + + +async def ssh_exec(host, port, user, cmd, timeout=120): + """异步 SSH 远程执行命令，返回 (exit_code, stdout, stderr)""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), '{user}@{host}', cmd + ] + proc = await asyncio.create_subprocess_exec( + *ssh_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout) + except asyncio.TimeoutError: + proc.kill() + return -1, '', f'SSH timeout after {timeout}s' + return proc.returncode, stdout.decode(), stderr.decode() + + +async def ssh_exec_bg(host, port, user, cmd): + """异步后台执行，不等待结果""" + ssh_cmd = [ + 'ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes', + '-p', str(port), user + '@' + host, + 'nohup bash -c "' + cmd.replace('"', '\\"') + '" > /dev/null 2>&1 &' + ] + subprocess.Popen(ssh_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +async def deploy_cluster(request, cluster_id): + """部署集群: 读取集群配置 → 分配节点 → 调用 plugin 部署""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': f'Cluster {cluster_id} not found'} + cluster = recs[0] + + # 更新状态为 deploying + await sor.U('cluster', {'id': cluster_id}, + {'status': 'deploying', 'updated_at': datetime.datetime.now().isoformat()}) + + # 获取已分配的节点 + node_recs = await sor.R('cluster_node', {'cluster_id': cluster_id}) + control_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'control'] + compute_nodes = [await sor.R('compute_node', {'id': n.node_id}) for n in node_recs if n.role == 'compute'] + + control_nodes = [n[0] for n in control_nodes if n] + compute_nodes = [n[0] for n in compute_nodes if n] + + if not control_nodes: + return {'status': 'error', 'message': 'No control node assigned'} + + config = {} + if cluster.control_config: + try: + config = json.loads(cluster.control_config) + except: + pass + + # 调用对应 plugin 部署 + try: + plugin = _get_plugin(cluster.cluster_type) + deploy_log = await plugin.deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config) + except Exception as e: + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'failed', 'deploy_log': str(e), + 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'error', 'message': f'Deploy failed: {e}'} + + # 更新状态为 running + endpoint = deploy_log.get('endpoint', '') if isinstance(deploy_log, dict) else '' + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'endpoint': endpoint, + 'deploy_log': json.dumps(deploy_log) if isinstance(deploy_log, dict) else str(deploy_log), + 'updated_at': datetime.datetime.now().isoformat()}) + # 更新节点状态为 active + for n in control_nodes + compute_nodes: + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': n.id}, + {'status': 'active'}) + + return {'status': 'ok', 'message': 'Cluster deployed', 'endpoint': endpoint} + + +async def add_cluster_node(request, params_kw): + """向运行中集群动态添加节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + role = params_kw.get('role', 'compute') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + + node = await sor.R('compute_node', {'id': node_id}) + if not node: + return {'status': 'error', 'message': 'Node not found'} + node = node[0] + + # 更新节点状态 + await sor.U('compute_node', {'id': node_id}, + {'status': 'allocated', 'updated_at': datetime.datetime.now().isoformat()}) + + # 记录分配 + await sor.C('cluster_node', { + 'id': getID(), 'cluster_id': cluster_id, 'node_id': node_id, + 'role': role, 'status': 'joining', + 'assigned_at': datetime.datetime.now().isoformat() + }) + + # 调用 plugin 添加 + plugin = _get_plugin(cluster.cluster_type) + result = await plugin.add_node(env, cluster_id, node_id, role) + return result + + +async def remove_cluster_node(request, params_kw): + """从集群移除节点并回收""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + cluster_id = params_kw.get('cluster_id', '') + node_id = params_kw.get('node_id', '') + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + + plugin = _get_plugin(cluster[0].cluster_type) + result = await plugin.remove_node(env, cluster_id, node_id) + + # 回收节点 + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('compute_node', {'id': node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'cluster_id': cluster_id, 'node_id': node_id}, + {'status': 'removed'}) + + return result + + +async def cluster_status(request, cluster_id): + """获取集群运行状态""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + + async with DBPools().sqlorContext(dbname) as sor: + cluster = await sor.R('cluster', {'id': cluster_id}) + if not cluster: + return {'status': 'error', 'message': 'Cluster not found'} + cluster = cluster[0] + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + + # 从 plugin 获取实时状态 + try: + plugin = _get_plugin(cluster.cluster_type) + live_status = await plugin.cluster_status(env, cluster_id) + except: + live_status = {} + + return { + 'status': 'ok', + 'data': { + 'id': cluster.id, 'name': cluster.name, 'type': cluster.cluster_type, + 'version': cluster.version, 'endpoint': cluster.endpoint, + 'state': cluster.status, + 'nodes': [{'id': n.node_id, 'role': n.role, 'status': n.status} for n in nodes], + 'live': live_status + } + } + + +async def stop_cluster(request, cluster_id): + """停止集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'stopped', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster stopped'} + + +async def start_cluster(request, cluster_id): + """启动已停止的集群""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'status': 'running', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster started'} + + +async def destroy_cluster(request, cluster_id): + """销毁集群，回收所有节点""" + env = request._run_ns + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + # 释放所有节点 + nodes = await sor.R('cluster_node', {'cluster_id': cluster_id}) + for n in nodes: + await sor.U('compute_node', {'id': n.node_id}, + {'status': 'available', 'updated_at': datetime.datetime.now().isoformat()}) + await sor.U('cluster_node', {'id': n.id}, {'status': 'removed'}) + await sor.U('cluster', {'id': cluster_id}, + {'status': 'destroyed', 'updated_at': datetime.datetime.now().isoformat()}) + return {'status': 'ok', 'message': 'Cluster destroyed'} + +async def cluster_status_all(request, params_kw): + """获取所有集群状态统计""" + env = request._run_ns + dbname = env.get_module_dbname('pcc') + async with DBPools().sqlorContext(dbname) as sor: + clusters = await sor.R('cluster', {}) + if not clusters: + return {'status': 'ok', 'data': {'total': 0, 'running': 0, 'deploying': 0, 'failed': 0, 'stopped': 0}} + statuses = {} + for c in clusters: + s = getattr(c, 'status', 'unknown') + statuses[s] = statuses.get(s, 0) + 1 + return {'status': 'ok', 'data': { + 'total': len(clusters), + 'running': statuses.get('running', 0), + 'deploying': statuses.get('deploying', 0), + 'failed': statuses.get('failed', 0), + 'stopped': statuses.get('stopped', 0), + 'destroyed': statuses.get('destroyed', 0), + }} + diff --git a/build/lib/pcc/k8s_plugin/__init__.py b/build/lib/pcc/k8s_plugin/__init__.py new file mode 100644 index 0000000..b2bed1f --- /dev/null +++ b/build/lib/pcc/k8s_plugin/__init__.py @@ -0,0 +1,221 @@ +""" +k8s_plugin — K8s 集群管理 (基于 kubeadm) +- 控制节点: kubeadm init + CNI(flannel) +- 算力节点: kubeadm join +- 移除: kubectl drain + delete + kubeadm reset +- 状态: kubectl get nodes +""" +from pcc import ssh_exec + +K8S_VERSION = '1.29' +POD_CIDR = '10.244.0.0/16' +SVC_CIDR = '10.96.0.0/12' + + +async def _node_info(env, node_id): + """从 DB 获取节点信息""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_k8s(node): + """在节点上安装 K8s 基础组件 (containerd + kubeadm + kubelet + kubectl)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq apt-transport-https ca-certificates curl gpg && " + "curl -fsSL https://pkgs.k8s.io/core:/stable:/v{ver}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg && " + "echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v{ver}/deb/ /' > /etc/apt/sources.list.d/kubernetes.list && " + "apt-get update -qq && " + "apt-get install -y -qq kubelet kubeadm kubectl containerd && " + "apt-mark hold kubelet kubeadm kubectl && " + "mkdir -p /etc/containerd && " + "containerd config default > /etc/containerd/config.toml 2>/dev/null || true && " + "sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml && " + "systemctl restart containerd && systemctl enable kubelet" + ).format(ver=K8S_VERSION) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _install_cni(control_node, pod_cidr): + """在控制节点安装 CNI 网络插件""" + cmd = f"kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml" + return await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=120) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """K8s 集群部署：只部署控制节点。算力节点后续通过 add_node 加入。""" + pod_cidr = config.get('pod_cidr', POD_CIDR) + svc_cidr = config.get('svc_cidr', SVC_CIDR) + results = {'control_nodes': {}, 'compute_nodes': None, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 K8s 基础组件 + rc, out, err = await _install_k8s(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err}'} + continue + + # 2. kubeadm init（仅第一个控制节点） + init_cmd = ( + f"kubeadm init --pod-network-cidr={pod_cidr} --service-cidr={svc_cidr} " + f"--kubernetes-version=v{K8S_VERSION} --ignore-preflight-errors=all" + ) + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', init_cmd, timeout=300) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'init failed: {err[:200]}'} + continue + + # 3. 配置 kubectl + setup_cmd = "mkdir -p $HOME/.kube && cp -f /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config" + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', setup_cmd) + + # 4. 安装 CNI + await _install_cni(node, pod_cidr) + + # 5. 获取 join token（供后续算力节点使用） + rc2, token_out, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "kubeadm token create --print-join-command", timeout=30) + join_cmd = token_out.strip() if rc2 == 0 else '' + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'join_command': join_cmd, + 'endpoint': f'https://{node.ip_address}:6443' + } + results['endpoint'] = f'https://{node.ip_address}:6443' + + # 保存 join_command 到集群 config + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 K8s 集群添加算力节点：安装 k8s → kubeadm join""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 读取 join_command + from sqlor.dbpools import DBPools + from pcc import MODULE_NAME + dbname = env.get_module_dbname(MODULE_NAME) + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'Cluster not found'} + config = json.loads(recs[0].control_config or '{}') + join_cmd = '' + for cn_data in config.get('control_nodes', {}).values(): + if cn_data.get('join_command'): + join_cmd = cn_data['join_command'] + break + if not join_cmd: + return {'status': 'error', 'message': 'No join command found, cluster may not be deployed'} + + # 1. 安装 K8s + rc, out, err = await _install_k8s(node) + if rc != 0: + return {'status': 'error', 'message': f'k8s install failed: {err[:200]}'} + + # 2. kubeadm join + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', join_cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined K8s cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 K8s 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # drain + delete + reset + cmds = [ + f"kubectl drain {node.name} --ignore-daemonsets --delete-emptydir-data --timeout=60s", + f"kubectl delete node {node.name}", + "kubeadm reset -f", + ] + for cmd in cmds: + rc, out, err = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + return {'status': 'ok', 'message': f'Node {node.name} removed'} + + +async def cluster_status(env, cluster_id): + """获取 K8s 集群状态""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + + control_id = recs[0].node_id + cnode = await _node_info(env, control_id) + if not cnode: + return {'error': 'Control node not found'} + + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'kubectl get nodes -o wide', timeout=30) + return {'status': 'ok', 'kubectl_nodes': out if rc == 0 else 'kubectl failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """K8s 算力单元分配: 创建 namespace + ResourceQuota""" + from pcc import ssh_exec as _ssh + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + + quota_yaml = ( + f"apiVersion: v1\\nkind: ResourceQuota\\nmetadata:\\n name: {unit_name}-quota\\n namespace: {unit_name}\\n" + f"spec:\\n hard:\\n requests.cpu: \\\"{cpu}\\\"\\n requests.memory: \\\"{memory}Gi\\\"\\n requests.nvidia.com/gpu: \\\"{gpu}\\\"" + ) + cmds = [ + f"kubectl create namespace {unit_name}", + f"echo '{quota_yaml}' | kubectl apply -f -", + ] + for cmd in cmds: + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'kubectl failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name}: ns + {cpu}C/{memory}G/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """K8s 算力单元回收: 删除 namespace""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + rc, _, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"kubectl delete namespace {unit_name} --timeout=60s") + if rc != 0: + return {'status': 'error', 'message': f'delete ns failed: {err[:200]}'} + return {'status': 'ok', 'message': f'Unit {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/pcc/ray_plugin/__init__.py b/build/lib/pcc/ray_plugin/__init__.py new file mode 100644 index 0000000..e0bae45 --- /dev/null +++ b/build/lib/pcc/ray_plugin/__init__.py @@ -0,0 +1,174 @@ +""" +ray_plugin — Ray 集群管理 +- 控制节点: ray start --head +- 算力节点: ray start --address=:6379 +- 移除: ray stop +- 状态: ray status +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_ray(node): + """安装 Ray""" + script = ( + "apt-get update -qq && apt-get install -y -qq python3 python3-pip && " + "pip3 install -q ray[default] 2>/dev/null || pip install -q ray[default]" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Ray 集群部署：仅部署 head 节点""" + head_port = config.get('head_port', 6379) + dashboard_port = config.get('dashboard_port', 8265) + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装 Ray + rc, out, err = await _install_ray(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. ray start --head + cmd = ( + f"ray start --head --port={head_port} --dashboard-host=0.0.0.0 " + f"--dashboard-port={dashboard_port} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'start failed: {err2[:200]}'} + continue + + results['control_nodes'][node.id] = { + 'status': 'ok', + 'head_address': f'{node.ip_address}:{head_port}', + 'dashboard': f'http://{node.ip_address}:{dashboard_port}' + } + results['endpoint'] = f'{node.ip_address}:{head_port}' + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Ray 集群添加 worker 节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取 head 节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + cluster_recs = await sor.R('cluster', {'id': cluster_id}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + head_node = await _node_info(env, recs[0].node_id) + if not head_node: + return {'status': 'error', 'message': 'Head node not found'} + + # 从 cluster config 读 head 地址 + config = json.loads(cluster_recs[0].control_config or '{}') if cluster_recs else {} + head_addr = config.get('endpoint', f'{head_node.ip_address}:6379') + + # 1. 安装 Ray + rc, _, err = await _install_ray(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. ray start --address + cmd = ( + f"ray start --address={head_addr} --num-cpus={node.cpu_cores or 0} " + f"--num-gpus={node.gpu_count or 0}" + ) + rc2, out2, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', cmd, timeout=120) + if rc2 != 0: + return {'status': 'error', 'message': f'join failed: {err2[:200]}'} + + return {'status': 'ok', 'message': f'Node {node.name} joined Ray cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Ray 集群移除 worker""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + rc, _, _ = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', 'ray stop', timeout=60) + return {'status': 'ok', 'message': f'Node {node.name} removed from Ray'} + + +async def cluster_status(env, cluster_id): + """ray status""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'ray status', timeout=30) + return {'status': 'ok', 'ray_status': out if rc == 0 else 'ray status failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Ray 算力单元分配: 创建 detached actor / placement group 绑定资源标签""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 用 ray submit 提交一个资源占位 job + script = ( + "import ray; ray.init(address='auto'); " + "pg = ray.util.placement_group([{'CPU': " + str(cpu) + ", 'GPU': " + str(gpu) + "}], strategy='STRICT_SPREAD', name='" + unit_name + "'); " + "ray.get(pg.ready()); print('PG ready: " + unit_name + "')" + ) + py_cmd = f"python3 -c '{script}'" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=60) + if rc != 0: + return {'status': 'error', 'message': f'ray pg failed: {err[:200]}'} + return {'status': 'ok', 'message': f'PlacementGroup {unit_name}: {cpu}C/{gpu}GPU'} + + +async def release_unit(env, cluster_id, unit_name): + """Ray 算力单元回收: 删除 placement group""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + py_cmd = f"python3 -c 'import ray; ray.init(address=\"auto\"); ray.util.remove_placement_group(ray.util.get_placement_group(\"{unit_name}\")); print(\"removed\")'" + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', py_cmd, timeout=30) + return {'status': 'ok', 'message': f'PlacementGroup {unit_name} released'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/pcc/slurm_plugin/__init__.py b/build/lib/pcc/slurm_plugin/__init__.py new file mode 100644 index 0000000..989c5a5 --- /dev/null +++ b/build/lib/pcc/slurm_plugin/__init__.py @@ -0,0 +1,229 @@ +""" +slurm_plugin — Slurm 集群管理 +- 控制节点: slurmctld + munge + NFS server +- 算力节点: slurmd + munge + NFS client +- 移除: drain node → scontrol delete → reset +- 状态: sinfo +""" +from pcc import ssh_exec + + +async def _node_info(env, node_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('compute_node', {'id': node_id}) + return recs[0] if recs else None + + +async def _install_slurm_deps(node): + """安装 Slurm 依赖 (munge + slurm)""" + script = ( + "apt-get update -qq && " + "apt-get install -y -qq munge slurm-wlm slurm-client nfs-common nfs-kernel-server " + "&& systemctl enable munge && systemctl start munge" + ) + return await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', script, timeout=300) + + +async def _setup_munge(control_node, compute_node): + """复制 munge key 从控制节点到算力节点""" + cmd = f"scp -o StrictHostKeyChecking=no /etc/munge/munge.key {compute_node.ssh_user or 'root'}@{compute_node.ip_address}:/etc/munge/munge.key" + rc, _, err = await ssh_exec(control_node.ip_address, control_node.ssh_port or 22, + control_node.ssh_user or 'root', cmd, timeout=30) + if rc == 0: + await ssh_exec(compute_node.ip_address, compute_node.ssh_port or 22, + compute_node.ssh_user or 'root', + "chown munge:munge /etc/munge/munge.key && chmod 400 /etc/munge/munge.key && systemctl restart munge", timeout=30) + + +async def deploy_cluster(env, cluster_id, control_nodes, compute_nodes, config): + """Slurm 集群部署：控制节点安装 slurmctld + munge""" + cluster_name = config.get('cluster_name', 'pccs-cluster') + results = {'control_nodes': {}, 'endpoint': ''} + + for node in control_nodes: + # 1. 安装依赖 + rc, out, err = await _install_slurm_deps(node) + if rc != 0: + results['control_nodes'][node.id] = {'error': f'install failed: {err[:200]}'} + continue + + # 2. 生成 slurm.conf + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + conf = f"""ClusterName={cluster_name} +ControlMachine={node.ip_address} +SlurmUser=root +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/spool/slurmctld +SlurmdSpoolDir=/var/spool/slurmd +ReturnToService=1 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core +AccountingStorageType=accounting_storage/none +JobCompType=jobcomp/none +NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN +PartitionName=debug Nodes={node.name} Default=YES MaxTime=INFINITE State=UP +""" + # 写配置到控制节点 + write_cmd = f"cat > /etc/slurm/slurm.conf << 'SLURM_EOF'\n{conf}\nSLURM_EOF" + rc2, _, err2 = await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', write_cmd, timeout=30) + if rc2 != 0: + results['control_nodes'][node.id] = {'error': f'config write failed: {err2}'} + continue + + # 3. 启动 slurmctld + await ssh_exec(node.ip_address, node.ssh_port or 22, + node.ssh_user or 'root', + "mkdir -p /var/spool/slurmctld /var/spool/slurmd && " + "systemctl enable slurmctld && systemctl start slurmctld", timeout=60) + + results['control_nodes'][node.id] = {'status': 'ok', 'slurm_conf': conf} + results['endpoint'] = node.ip_address + + async with DBPools().sqlorContext(env.get_module_dbname('pccs')) as sor: + await sor.U('cluster', {'id': cluster_id}, + {'control_config': json.dumps(results)}) + return results + + +async def add_node(env, cluster_id, node_id, role='compute'): + """向 Slurm 集群添加算力节点：安装 slurmd + munge → 更新 slurm.conf""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': f'Node {node_id} not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'status': 'error', 'message': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'status': 'error', 'message': 'Control node not found'} + + # 1. 安装依赖 + rc, _, err = await _install_slurm_deps(node) + if rc != 0: + return {'status': 'error', 'message': f'install failed: {err[:200]}'} + + # 2. 复制 munge key + await _setup_munge(cnode, node) + + # 3. 更新控制节点 slurm.conf 添加此节点 + cpu = node.cpu_cores or 1 + mem = node.memory_gb or 4 + add_line = f"NodeName={node.name} CPUs={cpu} RealMemory={mem * 1024} State=UNKNOWN" + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"echo '{add_line}' >> /etc/slurm/slurm.conf && scontrol reconfigure", timeout=30) + + # 4. 启动 slurmd + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl enable slurmd && systemctl start slurmd", timeout=60) + + return {'status': 'ok', 'message': f'Node {node.name} joined Slurm cluster'} + + +async def remove_node(env, cluster_id, node_id): + """从 Slurm 集群移除节点""" + node = await _node_info(env, node_id) + if not node: + return {'status': 'error', 'message': 'Node not found'} + + # 获取控制节点 + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control'}) + if recs: + cnode = await _node_info(env, recs[0].node_id) + if cnode: + # drain → remove from slurm.conf → scontrol reconfigure + await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, cnode.ssh_user or 'root', + f"scontrol update NodeName={node.name} State=DOWN Reason=removing && " + f"sed -i '/NodeName={node.name}/d' /etc/slurm/slurm.conf && " + "scontrol reconfigure", timeout=30) + + # stop slurmd on the node + await ssh_exec(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + "systemctl stop slurmd && systemctl disable slurmd", timeout=30) + + return {'status': 'ok', 'message': f'Node {node.name} removed from Slurm'} + + +async def cluster_status(env, cluster_id): + """sinfo""" + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return {'error': 'No active control node'} + cnode = await _node_info(env, recs[0].node_id) + if not cnode: + return {'error': 'Control node not found'} + rc, out, _ = await ssh_exec(cnode.ip_address, cnode.ssh_port or 22, + cnode.ssh_user or 'root', 'sinfo', timeout=30) + return {'status': 'ok', 'sinfo': out if rc == 0 else 'sinfo failed'} + + +import json +from sqlor.dbpools import DBPools + + +async def allocate_unit(env, cluster_id, unit_name, cpu, memory, gpu=0): + """Slurm 算力单元分配: 创建 partition + 关联节点""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + # 创建 partition + cmd = f"scontrol create PartitionName={unit_name} MaxNodes=UNLIMITED Default=NO MaxTime=UNLIMITED State=UP" + rc, out, err = await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', cmd) + if rc != 0: + return {'status': 'error', 'message': f'scontrol failed: {err[:200]}'} + + # 分配 node 到 partition (通过 cluster_node 表找该集群下可用的 compute 节点) + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + assigned = await sor.sqlExe( + """SELECT cn.name FROM cluster_node cnn + JOIN compute_node cn ON cn.id = cnn.node_id + WHERE cnn.cluster_id=${cid}$ AND cnn.role='compute' AND cnn.status='active' + LIMIT 1""", + {'cid': cluster_id} + ) + if assigned: + node_name = assigned[0].name + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol update NodeName={node_name} Partition={unit_name}") + + return {'status': 'ok', 'message': f'Partition {unit_name} created'} + + +async def release_unit(env, cluster_id, unit_name): + """Slurm 算力单元回收: 删除 partition""" + node = await _find_control_node(env, cluster_id) + if not node: + return {'status': 'error', 'message': 'No control node found'} + from pcc import ssh_exec as _ssh + await _ssh(node.ip_address, node.ssh_port or 22, node.ssh_user or 'root', + f"scontrol delete PartitionName={unit_name}") + return {'status': 'ok', 'message': f'Partition {unit_name} deleted'} + + +async def _find_control_node(env, cluster_id): + from sqlor.dbpools import DBPools + dbname = env.get_module_dbname('pccs') + async with DBPools().sqlorContext(dbname) as sor: + recs = await sor.R('cluster_node', {'cluster_id': cluster_id, 'role': 'control', 'status': 'active'}) + if not recs: + return None + return await _node_info(env, recs[0].node_id) diff --git a/build/lib/scripts/load_path.py b/build/lib/scripts/load_path.py new file mode 100644 index 0000000..59b37c6 --- /dev/null +++ b/build/lib/scripts/load_path.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""pcc RBAC 权限管理""" +import subprocess, os, sys, json, glob + +def find_sage_root(): + for c in [os.path.expanduser("~/sage"), os.path.expanduser("~/repos/sage")]: + if os.path.isdir(os.path.join(c, "py3")): return c + return None + +SAGE = find_sage_root() +if not SAGE: sys.exit("Sage root not found") +PY = os.path.join(SAGE, "py3", "bin", "python") +SET = os.path.join(SAGE, "set_role_perm.py") + +JSON_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "json") +API_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "wwwroot", "api") + +def load_cruds(): + defs = [] + for f in sorted(glob.glob(os.path.join(JSON_DIR, "*.json"))): + try: + d = json.load(open(f)) + defs.append({"tblname": d["tblname"], "alias": d.get("alias", d["tblname"]), "subtables": d.get("params", {}).get("subtables", [])}) + except: pass + return defs + +def get_apis(): + if not os.path.isdir(API_DIR): return [] + return [f"/pcc/api/{f}" for f in sorted(os.listdir(API_DIR)) if f.endswith(".dspy")] + +cruds = load_cruds() +apis = get_apis() + +PATHS_ANY = [ + f"/pcc/menu.ui", +] +PATHS_LOGINED = [ + f"/pcc", + f"/pcc/index.ui", +] +for d in cruds: + PATHS_ANY.append(f"/pcc/{d['alias']}") + PATHS_LOGINED.append(f"/pcc/{d['alias']}/index.ui") + for act in ["get", "add", "update", "delete"]: + PATHS_LOGINED.append(f"/pcc/{d['alias']}/{act}_{d['tblname']}.dspy") +for api in apis: + PATHS_LOGINED.append(api) + +PATHS_OPERATOR = list(PATHS_LOGINED) + +PATHS_ANY = list(dict.fromkeys(PATHS_ANY)) +PATHS_LOGINED = list(dict.fromkeys(PATHS_LOGINED)) +PATHS_OPERATOR = list(dict.fromkeys(PATHS_OPERATOR)) + +def reg(role, paths): + ok = 0 + for p in paths: + r = subprocess.run([PY, SET, role, p], capture_output=True, text=True) + if r.returncode == 0: ok += 1 + print(f" {role}: {ok}/{len(paths)}") + return ok + +total = 0 +print(f"{mod_name}: any={len(PATHS_ANY)} logined={len(PATHS_LOGINED)} operator={len(PATHS_OPERATOR)}") +total += reg("any", PATHS_ANY) +total += reg("logined", PATHS_LOGINED) +total += reg("reseller.operator", PATHS_OPERATOR) +print(f"Done. {total} entries.") diff --git a/i18n/en/msg.txt b/i18n/en/msg.txt new file mode 100644 index 0000000..5832fec --- /dev/null +++ b/i18n/en/msg.txt @@ -0,0 +1,28 @@ +API入口地址: API Endpoint +入口: +分配时间: +创建时间: Created At +商户机构id: Reseller ID +所属算力池: Compute Pool +所属集群: Cluster +控制节点配置(JSON): +更新时间: Updated At +版本: +版本号: Version +状态: Status +状态(deploying/running/failed/stopped/destroyed): +状态(joining/active/draining/removed): +算力池: Compute Pool +算力节点: +算力节点配置(JSON): +节点: +节点分配: Node Assignment +角色: +角色(control/compute/storage): +部署日志(JSON): +集群: Cluster +集群名称: Cluster Name +集群管理: Cluster Management +集群类型: +集群类型(k8s/slurm/ray): +集群节点分配: diff --git a/i18n/zh/msg.txt b/i18n/zh/msg.txt new file mode 100644 index 0000000..14730a3 --- /dev/null +++ b/i18n/zh/msg.txt @@ -0,0 +1,28 @@ +API入口地址: API入口地址 +入口: 入口 +分配时间: 分配时间 +创建时间: 创建时间 +商户机构id: 商户机构id +所属算力池: 所属算力池 +所属集群: 所属集群 +控制节点配置(JSON): 控制节点配置(JSON) +更新时间: 更新时间 +版本: 版本 +版本号: 版本号 +状态: 状态 +状态(deploying/running/failed/stopped/destroyed): 状态(deploying/running/failed/stopped/destroyed) +状态(joining/active/draining/removed): 状态(joining/active/draining/removed) +算力池: 算力池 +算力节点: 算力节点 +算力节点配置(JSON): 算力节点配置(JSON) +节点: 节点 +节点分配: 节点分配 +角色: 角色 +角色(control/compute/storage): 角色(control/compute/storage) +部署日志(JSON): 部署日志(JSON) +集群: 集群 +集群名称: 集群名称 +集群管理: 集群管理 +集群类型: 集群类型 +集群类型(k8s/slurm/ray): 集群类型(k8s/slurm/ray) +集群节点分配: 集群节点分配 diff --git a/json/cluster_list.json b/json/cluster_list.json index 1d11645..9b4a00e 100644 --- a/json/cluster_list.json +++ b/json/cluster_list.json @@ -35,13 +35,23 @@ "created_at", "updated_at" ], - "toolbar": { - "tools": [] - }, "binds": [], - "new_data_url": "{{entire_url('/pcc/api/cluster_create.dspy')}}", - "update_data_url": "{{entire_url('/pcc/api/cluster_update.dspy')}}", - "delete_data_url": "{{entire_url('/pcc/api/cluster_delete.dspy')}}", - "logined_userorgid": "resellerid" + "logined_userorgid": "resellerid", + "editable": { + "new_data_url": "{{entire_url('/pcc/api/cluster_create.dspy')}}", + "update_data_url": "{{entire_url('/pcc/api/cluster_update.dspy')}}", + "delete_data_url": "{{entire_url('/pcc/api/cluster_delete.dspy')}}" + }, + "toolbar": { + "tools": [ + { + "name": "view_nodes", + "label": "查看节点", + "icon": "", + "type": "link", + "url": "/pcc/cluster_node_list/index.ui?cluster_id={{id}}" + } + ] + } } } \ No newline at end of file diff --git a/json/cluster_node_list.json b/json/cluster_node_list.json index 6c68bb8..bd1b465 100644 --- a/json/cluster_node_list.json +++ b/json/cluster_node_list.json @@ -27,13 +27,12 @@ "created_at", "updated_at" ], - "toolbar": { - "tools": [] - }, "binds": [], - "new_data_url": "{{entire_url('/cluster_node/api/cluster_node_create.dspy')}}", - "update_data_url": "{{entire_url('/cluster_node/api/cluster_node_update.dspy')}}", - "delete_data_url": "{{entire_url('/cluster_node/api/cluster_node_delete.dspy')}}", - "logined_userorgid": "resellerid" + "logined_userorgid": "resellerid", + "editable": { + "new_data_url": "{{entire_url('/cluster_node/api/cluster_node_create.dspy')}}", + "update_data_url": "{{entire_url('/cluster_node/api/cluster_node_update.dspy')}}", + "delete_data_url": "{{entire_url('/cluster_node/api/cluster_node_delete.dspy')}}" + } } } \ No newline at end of file diff --git a/pcc.egg-info/PKG-INFO b/pcc.egg-info/PKG-INFO new file mode 100644 index 0000000..077f763 --- /dev/null +++ b/pcc.egg-info/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 2.4 +Name: pcc +Version: 0.1.0 +Summary: 统一集群管理：K8s/Slurm/Ray 集群生命周期、节点分配、动态维护、监控 +Requires-Python: >=3.10 +Requires-Dist: apppublic +Requires-Dist: sqlor +Requires-Dist: ahserver +Requires-Dist: appbase +Requires-Dist: rbac diff --git a/pcc.egg-info/SOURCES.txt b/pcc.egg-info/SOURCES.txt new file mode 100644 index 0000000..cee7745 --- /dev/null +++ b/pcc.egg-info/SOURCES.txt @@ -0,0 +1,12 @@ +README.md +pyproject.toml +pcc/__init__.py +pcc.egg-info/PKG-INFO +pcc.egg-info/SOURCES.txt +pcc.egg-info/dependency_links.txt +pcc.egg-info/requires.txt +pcc.egg-info/top_level.txt +pcc/k8s_plugin/__init__.py +pcc/ray_plugin/__init__.py +pcc/slurm_plugin/__init__.py +scripts/load_path.py \ No newline at end of file diff --git a/pcc.egg-info/dependency_links.txt b/pcc.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/pcc.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/pcc.egg-info/requires.txt b/pcc.egg-info/requires.txt new file mode 100644 index 0000000..dd606cd --- /dev/null +++ b/pcc.egg-info/requires.txt @@ -0,0 +1,5 @@ +apppublic +sqlor +ahserver +appbase +rbac diff --git a/pcc.egg-info/top_level.txt b/pcc.egg-info/top_level.txt new file mode 100644 index 0000000..537efb3 --- /dev/null +++ b/pcc.egg-info/top_level.txt @@ -0,0 +1,7 @@ +build +i18n +json +models +pcc +scripts +wwwroot diff --git a/pcc/__init__.py b/pcc/__init__.py index 05f975d..8c44552 100644 --- a/pcc/__init__.py +++ b/pcc/__init__.py @@ -240,3 +240,25 @@ async def destroy_cluster(request, cluster_id): await sor.U('cluster', {'id': cluster_id}, {'status': 'destroyed', 'updated_at': datetime.datetime.now().isoformat()}) return {'status': 'ok', 'message': 'Cluster destroyed'} + +async def cluster_status_all(request, params_kw): + """获取所有集群状态统计""" + env = request._run_ns + dbname = env.get_module_dbname('pcc') + async with DBPools().sqlorContext(dbname) as sor: + clusters = await sor.R('cluster', {}) + if not clusters: + return {'status': 'ok', 'data': {'total': 0, 'running': 0, 'deploying': 0, 'failed': 0, 'stopped': 0}} + statuses = {} + for c in clusters: + s = getattr(c, 'status', 'unknown') + statuses[s] = statuses.get(s, 0) + 1 + return {'status': 'ok', 'data': { + 'total': len(clusters), + 'running': statuses.get('running', 0), + 'deploying': statuses.get('deploying', 0), + 'failed': statuses.get('failed', 0), + 'stopped': statuses.get('stopped', 0), + 'destroyed': statuses.get('destroyed', 0), + }} + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4ad7f1f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "pcc" +version = "0.1.0" +description = "统一集群管理：K8s/Slurm/Ray 集群生命周期、节点分配、动态维护、监控" +requires-python = ">=3.10" +dependencies = ["apppublic", "sqlor", "ahserver", "appbase", "rbac"] + +[tool.setuptools.package-dir] +"pcc" = "pcc" +"pcc.k8s_plugin" = "pcc/k8s_plugin" +"pcc.slurm_plugin" = "pcc/slurm_plugin" +"pcc.ray_plugin" = "pcc/ray_plugin" + +[tool.setuptools.packages.find] +where = ["."] diff --git a/wwwroot/api/cluster_create.dspy b/wwwroot/api/cluster_create.dspy index 09ccce7..19079eb 100644 --- a/wwwroot/api/cluster_create.dspy +++ b/wwwroot/api/cluster_create.dspy @@ -1,3 +1,16 @@ -# cluster create -result = {'status': 'ok', 'message': 'cluster created'} -return result +ns = params_kw.copy() +for k,v in list(ns.items()): + if v == 'NaN' or v == 'null': ns[k] = None + if k.endswith('_text'): ns.pop(k, None) +id = params_kw.get('id','') +if not id or len(str(id)) > 32: id = uuid() +ns['id'] = id +userorgid = await get_userorgid() +if not userorgid: + return {'widgettype':'Error','options':{'title':'Authorization Error','timeout':3,'cwidth':16,'cheight':9,'message':'Please login'}} +ns['resellerid'] = userorgid +ns['created_at'] = datetime.datetime.now().isoformat() +ns['updated_at'] = ns['created_at'] +async with DBPools().sqlorContext(get_module_dbname('pcc')) as sor: + await sor.C('cluster', ns) + return {'widgettype':'Message','options':{'cwidth':16,'cheight':9,'title':'Success','timeout':3,'message':'ok'}} diff --git a/wwwroot/api/cluster_delete.dspy b/wwwroot/api/cluster_delete.dspy index 85ba94d..c78905c 100644 --- a/wwwroot/api/cluster_delete.dspy +++ b/wwwroot/api/cluster_delete.dspy @@ -1,3 +1,6 @@ -# cluster delete -result = {'status': 'ok', 'message': 'cluster deleted'} -return result +ns = params_kw.copy() +if not ns.get('id'): + return {'widgettype':'Error','options':{'title':'Error','message':'missing id','cwidth':16,'cheight':9,'timeout':3}} +async with DBPools().sqlorContext(get_module_dbname('pcc')) as sor: + await sor.D('cluster', {'id': ns['id']}) + return {'widgettype':'Message','options':{'cwidth':16,'cheight':9,'title':'Success','timeout':3,'message':'ok'}} diff --git a/wwwroot/api/cluster_node_create.dspy b/wwwroot/api/cluster_node_create.dspy index 4439bb3..b16a3f3 100644 --- a/wwwroot/api/cluster_node_create.dspy +++ b/wwwroot/api/cluster_node_create.dspy @@ -1,3 +1,14 @@ -# cluster_node create -result = {'status': 'ok', 'message': 'cluster_node created'} -return result +ns = params_kw.copy() +for k,v in list(ns.items()): + if v == 'NaN' or v == 'null': ns[k] = None + if k.endswith('_text'): ns.pop(k, None) +id = params_kw.get('id','') +if not id or len(str(id)) > 32: id = uuid() +ns['id'] = id +userorgid = await get_userorgid() +if not userorgid: + return {'widgettype':'Error','options':{'title':'Authorization Error','timeout':3,'cwidth':16,'cheight':9,'message':'Please login'}} +ns['allocated_at'] = datetime.datetime.now().isoformat() +async with DBPools().sqlorContext(get_module_dbname('pcc')) as sor: + await sor.C('cluster_node', ns) + return {'widgettype':'Message','options':{'cwidth':16,'cheight':9,'title':'Success','timeout':3,'message':'ok'}} diff --git a/wwwroot/api/cluster_node_delete.dspy b/wwwroot/api/cluster_node_delete.dspy index 1b213e6..b70febb 100644 --- a/wwwroot/api/cluster_node_delete.dspy +++ b/wwwroot/api/cluster_node_delete.dspy @@ -1,3 +1,6 @@ -# cluster_node delete -result = {'status': 'ok', 'message': 'cluster_node deleted'} -return result +ns = params_kw.copy() +if not ns.get('id'): + return {'widgettype':'Error','options':{'title':'Error','message':'missing id','cwidth':16,'cheight':9,'timeout':3}} +async with DBPools().sqlorContext(get_module_dbname('pcc')) as sor: + await sor.D('cluster_node', {'id': ns['id']}) + return {'widgettype':'Message','options':{'cwidth':16,'cheight':9,'title':'Success','timeout':3,'message':'ok'}} diff --git a/wwwroot/api/cluster_node_update.dspy b/wwwroot/api/cluster_node_update.dspy index 9465c15..9415585 100644 --- a/wwwroot/api/cluster_node_update.dspy +++ b/wwwroot/api/cluster_node_update.dspy @@ -1,3 +1,9 @@ -# cluster_node update -result = {'status': 'ok', 'message': 'cluster_node updated'} -return result +ns = params_kw.copy() +for k,v in list(ns.items()): + if v == 'NaN' or v == 'null': ns[k] = None + if k.endswith('_text'): ns.pop(k, None) +if not ns.get('id'): + return {'widgettype':'Error','options':{'title':'Error','message':'missing id','cwidth':16,'cheight':9,'timeout':3}} +async with DBPools().sqlorContext(get_module_dbname('pcc')) as sor: + await sor.U('cluster_node', ns) + return {'widgettype':'Message','options':{'cwidth':16,'cheight':9,'title':'Success','timeout':3,'message':'ok'}} diff --git a/wwwroot/api/cluster_status_all.dspy b/wwwroot/api/cluster_status_all.dspy new file mode 100644 index 0000000..49116c2 --- /dev/null +++ b/wwwroot/api/cluster_status_all.dspy @@ -0,0 +1,49 @@ +env = request._run_ns +dbname = get_module_dbname('pcc') +try: + async with DBPools().sqlorContext(dbname) as sor: + clusters = await sor.R('cluster', {}) + total = len(clusters or []) + running = sum(1 for c in (clusters or []) if getattr(c,'status','') == 'running') + deploying = sum(1 for c in (clusters or []) if getattr(c,'status','') == 'deploying') + stopped = sum(1 for c in (clusters or []) if getattr(c,'status','') == 'stopped') + total_nodes = 0; total_cpu = 0; total_gpu = 0; total_mem = 0 + for c in (clusters or []): + nodes = await sor.R('cluster_node', {'cluster_id': c.id}) + total_nodes += len(nodes or []) + for n in (nodes or []): + total_cpu += int(getattr(n,'cpu_cores',0) or 0) + total_gpu += int(getattr(n,'gpu_count',0) or 0) + total_mem += int(getattr(n,'memory_gb',0) or 0) + return { + 'widgettype': 'HBox', + 'options': {'gap': '16px'}, + 'subwidgets': [ + {'widgettype': 'VBox', 'options': {'bgcolor': '#eff6ff', 'padding': '16px', 'width': '25%', 'border': '1px solid #dbeafe', 'borderRadius': '8px'}, + 'subwidgets': [ + {'widgettype': 'Text', 'options': {'otext': '集群', 'cfontsize': 0.7, 'color': '#64748b'}}, + {'widgettype': 'Text', 'options': {'otext': str(total) + ' 个', 'cfontsize': 2, 'fontWeight': 'bold', 'color': '#2563eb'}}, + {'widgettype': 'Text', 'options': {'otext': str(running) + '运行/' + str(deploying) + '部署/' + str(stopped) + '停', 'cfontsize': 0.7, 'color': '#94a3b8'}} + ]}, + {'widgettype': 'VBox', 'options': {'bgcolor': '#f5f3ff', 'padding': '16px', 'width': '25%', 'border': '1px solid #ede9fe', 'borderRadius': '8px'}, + 'subwidgets': [ + {'widgettype': 'Text', 'options': {'otext': '节点', 'cfontsize': 0.7, 'color': '#64748b'}}, + {'widgettype': 'Text', 'options': {'otext': str(total_nodes) + ' 个', 'cfontsize': 2, 'fontWeight': 'bold', 'color': '#7c3aed'}}, + {'widgettype': 'Text', 'options': {'otext': '已加入集群', 'cfontsize': 0.7, 'color': '#94a3b8'}} + ]}, + {'widgettype': 'VBox', 'options': {'bgcolor': '#ecfdf5', 'padding': '16px', 'width': '25%', 'border': '1px solid #d1fae5', 'borderRadius': '8px'}, + 'subwidgets': [ + {'widgettype': 'Text', 'options': {'otext': 'CPU', 'cfontsize': 0.7, 'color': '#64748b'}}, + {'widgettype': 'Text', 'options': {'otext': str(total_cpu) + ' 核', 'cfontsize': 2, 'fontWeight': 'bold', 'color': '#059669'}}, + {'widgettype': 'Text', 'options': {'otext': '集群总核数', 'cfontsize': 0.7, 'color': '#94a3b8'}} + ]}, + {'widgettype': 'VBox', 'options': {'bgcolor': '#fff7ed', 'padding': '16px', 'width': '25%', 'border': '1px solid #ffedd5', 'borderRadius': '8px'}, + 'subwidgets': [ + {'widgettype': 'Text', 'options': {'otext': 'GPU/内存', 'cfontsize': 0.7, 'color': '#64748b'}}, + {'widgettype': 'Text', 'options': {'otext': str(total_gpu) + ' GPU', 'cfontsize': 2, 'fontWeight': 'bold', 'color': '#ea580c'}}, + {'widgettype': 'Text', 'options': {'otext': str(total_mem) + ' GB 内存', 'cfontsize': 0.7, 'color': '#94a3b8'}} + ]} + ] + } +except: + return {'widgettype': 'Text', 'options': {'otext': '加载失败', 'color': '#dc2626'}} diff --git a/wwwroot/api/cluster_update.dspy b/wwwroot/api/cluster_update.dspy index 7928592..cf2e43b 100644 --- a/wwwroot/api/cluster_update.dspy +++ b/wwwroot/api/cluster_update.dspy @@ -1,3 +1,10 @@ -# cluster update -result = {'status': 'ok', 'message': 'cluster updated'} -return result +ns = params_kw.copy() +for k,v in list(ns.items()): + if v == 'NaN' or v == 'null': ns[k] = None + if k.endswith('_text'): ns.pop(k, None) +if not ns.get('id'): + return {'widgettype':'Error','options':{'title':'Error','message':'missing id','cwidth':16,'cheight':9,'timeout':3}} +ns['updated_at'] = datetime.datetime.now().isoformat() +async with DBPools().sqlorContext(get_module_dbname('pcc')) as sor: + await sor.U('cluster', ns) + return {'widgettype':'Message','options':{'cwidth':16,'cheight':9,'title':'Success','timeout':3,'message':'ok'}} diff --git a/wwwroot/cluster_list/add_cluster.dspy b/wwwroot/cluster_list/add_cluster.dspy new file mode 100644 index 0000000..0027734 --- /dev/null +++ b/wwwroot/cluster_list/add_cluster.dspy @@ -0,0 +1,52 @@ + +ns = params_kw.copy() +for k,v in ns.items(): + if v == 'NaN' or v == 'null': + ns[k] = None +id = params_kw.id +if not id or len(id) > 32: + id = uuid() +ns['id'] = id + + + +userorgid = await get_userorgid() +if not userorgid: + return { + "widgettype":"Error", + "options":{ + "title":"Authorization Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"Please login" + } + } +ns['resellerid'] = userorgid + +db = DBPools() +dbname = get_module_dbname('pcc') +async with db.sqlorContext(dbname) as sor: + r = await sor.C('cluster', ns.copy()) + return { + "widgettype":"Message", + "options":{ + "user_data":ns, + "cwidth":16, + "cheight":9, + "title":"Add Success", + "timeout":3, + "message":"ok" + } + } + +return { + "widgettype":"Error", + "options":{ + "title":"Add Error", + "cwidth":16, + "cheight":9, + "timeout":3, + "message":"failed" + } +} \ No newline at end of file diff --git a/wwwroot/cluster_list/delete_cluster.dspy b/wwwroot/cluster_list/delete_cluster.dspy new file mode 100644 index 0000000..ec4dc80 --- /dev/null +++ b/wwwroot/cluster_list/delete_cluster.dspy @@ -0,0 +1,47 @@ + +ns = { + 'id':params_kw['id'], +} + + +userorgid = await get_userorgid() +if not userorgid: + return { + "widgettype":"Error", + "options":{ + "title":"Authorization Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"Please login" + } + } +ns['resellerid'] = userorgid + +db = DBPools() +dbname = get_module_dbname('pcc') +async with db.sqlorContext(dbname) as sor: + r = await sor.D('cluster', ns) + debug('delete success'); + return { + "widgettype":"Message", + "options":{ + "title":"Delete Success", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"ok" + } + } + +debug('Delete failed'); +return { + "widgettype":"Error", + "options":{ + "title":"Delete Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"failed" + } +} \ No newline at end of file diff --git a/wwwroot/cluster_list/get_cluster.dspy b/wwwroot/cluster_list/get_cluster.dspy new file mode 100644 index 0000000..ef0fa3c --- /dev/null +++ b/wwwroot/cluster_list/get_cluster.dspy @@ -0,0 +1,145 @@ + +ns = params_kw.copy() + + +userorgid = await get_userorgid() +if not userorgid: + return { + "widgettype":"Error", + "options":{ + "title":"Authorization Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"Please login" + } + } +ns['resellerid'] = userorgid +ns['userorgid'] = userorgid + +debug(f'get_cluster.dspy:{ns=}') +if not ns.get('page'): + ns['page'] = 1 +if not ns.get('sort'): + + ns['sort'] = 'id' + + +sql = '''select a.*, b.pool_id_text, c.cluster_type_text, d.status_text +from (select * from cluster where 1=1 [[filterstr]]) a left join (select id as pool_id, + name as pool_id_text from compute_pool where 1 = 1) b on a.pool_id = b.pool_id left join (select k as cluster_type, + v as cluster_type_text from appcodes_kv where parentid='cluster_type') c on a.cluster_type = c.cluster_type left join (select k as status, + v as status_text from appcodes_kv where parentid='cluster_status') d on a.status = d.status''' + +filterjson = params_kw.get('data_filter') +if not filterjson: + fields = [ f['name'] for f in [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "resellerid", + "title": "商户机构id", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "pool_id", + "title": "所属算力池", + "type": "str", + "length": 32 + }, + { + "name": "name", + "title": "集群名称", + "type": "str", + "length": 128, + "nullable": "no" + }, + { + "name": "cluster_type", + "title": "集群类型(k8s/slurm/ray)", + "type": "char", + "length": 16, + "nullable": "no" + }, + { + "name": "version", + "title": "版本号", + "type": "str", + "length": 64 + }, + { + "name": "endpoint", + "title": "API入口地址", + "type": "str", + "length": 256 + }, + { + "name": "control_config", + "title": "控制节点配置(JSON)", + "type": "text" + }, + { + "name": "compute_config", + "title": "算力节点配置(JSON)", + "type": "text" + }, + { + "name": "status", + "title": "状态(deploying/running/failed/stopped/destroyed)", + "type": "char", + "length": 16, + "default": "deploying" + }, + { + "name": "deploy_log", + "title": "部署日志(JSON)", + "type": "text" + }, + { + "name": "created_at", + "title": "创建时间", + "type": "timestamp", + "nullable": "no" + }, + { + "name": "updated_at", + "title": "更新时间", + "type": "timestamp", + "nullable": "no" + } +] ] + filterjson = default_filterjson(fields, ns) +filterdic = ns.copy() +filterdic['filterstr'] = '' +filterdic['userorgid'] = '${userorgid}$' +filterdic['userid'] = '${userid}$' +if filterjson: + dbf = DBFilter(filterjson) + conds = dbf.gen(ns) + if conds: + ns.update(dbf.consts) + conds = f' and {conds}' + filterdic['filterstr'] = conds +ac = ArgsConvert('[[', ']]') +vars = ac.findAllVariables(sql) +NameSpace = {v:'${' + v + '}$' for v in vars if v != 'filterstr' } +filterdic.update(NameSpace) +sql = ac.convert(sql, filterdic) + +debug(f'{sql=}') +db = DBPools() +dbname = get_module_dbname('pcc') +async with db.sqlorContext(dbname) as sor: + r = await sor.sqlPaging(sql, ns) + return r +return { + "total":0, + "rows":[] +} \ No newline at end of file diff --git a/wwwroot/cluster_list/index.ui b/wwwroot/cluster_list/index.ui new file mode 100644 index 0000000..d5b5015 --- /dev/null +++ b/wwwroot/cluster_list/index.ui @@ -0,0 +1,259 @@ + +{ + "id":"cluster_tbl", + "widgettype":"Tabular", + "options":{ + "width":"100%", + "height":"100%", + + + "title":"集群", + + + + + "toolbar":{ + "tools": [ + { + "name": "view_nodes", + "label": "查看节点", + "icon": "", + "type": "link", + "url": "/pcc/cluster_node_list/index.ui?cluster_id={{id}}" + } + ] +}, + + "css":"card", + + + "editable":{ + + "new_data_url":"{{entire_url('add_cluster.dspy')}}", + + + "delete_data_url":"{{entire_url('delete_cluster.dspy')}}", + + + "update_data_url":"{{entire_url('update_cluster.dspy')}}" + + }, + + + "data_url":"{{entire_url('./get_cluster.dspy')}}", + + "data_method":"GET", + "data_params":{{json.dumps(params_kw, indent=4, ensure_ascii=False)}}, + "row_options":{ + + + + "browserfields": { + "name": { + "title": "集群名称", + "width": 150 + }, + "cluster_type": { + "title": "集群类型", + "width": 100 + }, + "pool_id": { + "title": "算力池", + "width": 150 + }, + "version": { + "title": "版本", + "width": 100 + }, + "endpoint": { + "title": "入口", + "width": 200 + }, + "status": { + "title": "状态", + "width": 100 + } +}, + + + "editexclouded":[ + "id", + "resellerid", + "created_at", + "updated_at" +], + + "fields":[ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32, + "nullable": "no", + "cwidth": 18, + "uitype": "str", + "datatype": "str", + "label": "id" + }, + { + "name": "resellerid", + "title": "商户机构id", + "type": "str", + "length": 32, + "nullable": "no", + "cwidth": 18, + "uitype": "str", + "datatype": "str", + "label": "商户机构id" + }, + { + "name": "pool_id", + "title": "所属算力池", + "type": "str", + "length": 32, + "label": "所属算力池", + "uitype": "code", + "valueField": "pool_id", + "textField": "pool_id_text", + "params": { + "dbname": "{{get_module_dbname('pcc')}}", + "table": "compute_pool", + "tblvalue": "id", + "tbltext": "name", + "valueField": "pool_id", + "textField": "pool_id_text" + }, + "dataurl": "{{entire_url('/appbase/get_code.dspy')}}" + }, + { + "name": "name", + "title": "集群名称", + "type": "str", + "length": 128, + "nullable": "no", + "cwidth": 18, + "uitype": "str", + "datatype": "str", + "label": "集群名称" + }, + { + "name": "cluster_type", + "title": "集群类型(k8s/slurm/ray)", + "type": "char", + "length": 16, + "nullable": "no", + "label": "集群类型(k8s/slurm/ray)", + "uitype": "code", + "valueField": "cluster_type", + "textField": "cluster_type_text", + "params": { + "dbname": "{{get_module_dbname('pcc')}}", + "table": "appcodes_kv", + "tblvalue": "k", + "tbltext": "v", + "valueField": "cluster_type", + "textField": "cluster_type_text", + "cond": "parentid='cluster_type'" + }, + "dataurl": "{{entire_url('/appbase/get_code.dspy')}}" + }, + { + "name": "version", + "title": "版本号", + "type": "str", + "length": 64, + "cwidth": 18, + "uitype": "str", + "datatype": "str", + "label": "版本号" + }, + { + "name": "endpoint", + "title": "API入口地址", + "type": "str", + "length": 256, + "cwidth": 18, + "uitype": "str", + "datatype": "str", + "label": "API入口地址" + }, + { + "name": "control_config", + "title": "控制节点配置(JSON)", + "type": "text", + "length": 0, + "uitype": "text", + "datatype": "text", + "label": "控制节点配置(JSON)" + }, + { + "name": "compute_config", + "title": "算力节点配置(JSON)", + "type": "text", + "length": 0, + "uitype": "text", + "datatype": "text", + "label": "算力节点配置(JSON)" + }, + { + "name": "status", + "title": "状态(deploying/running/failed/stopped/destroyed)", + "type": "char", + "length": 16, + "default": "deploying", + "label": "状态(deploying/running/failed/stopped/destroyed)", + "uitype": "code", + "valueField": "status", + "textField": "status_text", + "params": { + "dbname": "{{get_module_dbname('pcc')}}", + "table": "appcodes_kv", + "tblvalue": "k", + "tbltext": "v", + "valueField": "status", + "textField": "status_text", + "cond": "parentid='cluster_status'" + }, + "dataurl": "{{entire_url('/appbase/get_code.dspy')}}" + }, + { + "name": "deploy_log", + "title": "部署日志(JSON)", + "type": "text", + "length": 0, + "uitype": "text", + "datatype": "text", + "label": "部署日志(JSON)" + }, + { + "name": "created_at", + "title": "创建时间", + "type": "timestamp", + "nullable": "no", + "length": 0, + "uitype": "str", + "datatype": "timestamp", + "label": "创建时间" + }, + { + "name": "updated_at", + "title": "更新时间", + "type": "timestamp", + "nullable": "no", + "length": 0, + "uitype": "str", + "datatype": "timestamp", + "label": "更新时间" + } +] + }, + + + + "page_rows":160, + "cache_limit":5 + } + + ,"binds":[] + +} \ No newline at end of file diff --git a/wwwroot/cluster_list/update_cluster.dspy b/wwwroot/cluster_list/update_cluster.dspy new file mode 100644 index 0000000..c26cd4b --- /dev/null +++ b/wwwroot/cluster_list/update_cluster.dspy @@ -0,0 +1,70 @@ + +ns = params_kw.copy() +for k,v in ns.items(): + if v == 'NaN' or v == 'null': + ns[k] = None + + +userorgid = await get_userorgid() +if not userorgid: + return { + "widgettype":"Error", + "options":{ + "title":"Authorization Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"Please login" + } + } +ns['resellerid'] = userorgid + + + +db = DBPools() +dbname = get_module_dbname('pcc') +async with db.sqlorContext(dbname) as sor: + + ns1 = { + + "resellerid": userorgid, + + + "id": params_kw.id + } + recs = await sor.R('cluster', ns1) + if len(recs) < 1: + return { + "widgettype":"Error", + "options":{ + "title":"Update Error", + "cwidth":16, + "cheight":9, + "timeout":3, + "message":"Record no exist or with wrong ownership" + } + } + + r = await sor.U('cluster', ns) + debug('update success'); + return { + "widgettype":"Message", + "options":{ + "title":"Update Success", + "cwidth":16, + "cheight":9, + "timeout":3, + "message":"ok" + } + } + +return { + "widgettype":"Error", + "options":{ + "title":"Update Error", + "cwidth":16, + "cheight":9, + "timeout":3, + "message":"failed" + } +} \ No newline at end of file diff --git a/wwwroot/cluster_node_list/add_cluster_node.dspy b/wwwroot/cluster_node_list/add_cluster_node.dspy new file mode 100644 index 0000000..e26a55b --- /dev/null +++ b/wwwroot/cluster_node_list/add_cluster_node.dspy @@ -0,0 +1,57 @@ + +ns = params_kw.copy() +for k,v in ns.items(): + if v == 'NaN' or v == 'null': + ns[k] = None +id = params_kw.id +if not id or len(id) > 32: + id = uuid() +ns['id'] = id + + + +userorgid = await get_userorgid() +if not userorgid: + return { + "widgettype":"Error", + "options":{ + "title":"Authorization Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"Please login" + } + } +ns['resellerid'] = userorgid + +for k in list(ns.keys()): + if k.endswith('_at') or k.endswith('_time') or k == 'last_heartbeat': + v = ns.get(k, '') + if v == '' or v is None or v == 'None': + ns[k] = None +db = DBPools() +dbname = get_module_dbname('pcc') +async with db.sqlorContext(dbname) as sor: + r = await sor.C('cluster_node', ns.copy()) + return { + "widgettype":"Message", + "options":{ + "user_data":ns, + "cwidth":16, + "cheight":9, + "title":"Add Success", + "timeout":3, + "message":"ok" + } + } + +return { + "widgettype":"Error", + "options":{ + "title":"Add Error", + "cwidth":16, + "cheight":9, + "timeout":3, + "message":"failed" + } +} \ No newline at end of file diff --git a/wwwroot/cluster_node_list/delete_cluster_node.dspy b/wwwroot/cluster_node_list/delete_cluster_node.dspy new file mode 100644 index 0000000..045fcc7 --- /dev/null +++ b/wwwroot/cluster_node_list/delete_cluster_node.dspy @@ -0,0 +1,47 @@ + +ns = { + 'id':params_kw['id'], +} + + +userorgid = await get_userorgid() +if not userorgid: + return { + "widgettype":"Error", + "options":{ + "title":"Authorization Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"Please login" + } + } +ns['resellerid'] = userorgid + +db = DBPools() +dbname = get_module_dbname('pcc') +async with db.sqlorContext(dbname) as sor: + r = await sor.D('cluster_node', ns) + debug('delete success'); + return { + "widgettype":"Message", + "options":{ + "title":"Delete Success", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"ok" + } + } + +debug('Delete failed'); +return { + "widgettype":"Error", + "options":{ + "title":"Delete Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"failed" + } +} \ No newline at end of file diff --git a/wwwroot/cluster_node_list/get_cluster_node.dspy b/wwwroot/cluster_node_list/get_cluster_node.dspy new file mode 100644 index 0000000..2497214 --- /dev/null +++ b/wwwroot/cluster_node_list/get_cluster_node.dspy @@ -0,0 +1,107 @@ + +ns = params_kw.copy() + + +userorgid = await get_userorgid() +if not userorgid: + return { + "widgettype":"Error", + "options":{ + "title":"Authorization Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"Please login" + } + } +ns['resellerid'] = userorgid +ns['userorgid'] = userorgid + +debug(f'get_cluster_node.dspy:{ns=}') +if not ns.get('page'): + ns['page'] = 1 +if not ns.get('sort'): + + ns['sort'] = 'id' + + +sql = '''select a.*, b.cluster_id_text, c.node_id_text, d.role_text, e.status_text +from (select * from cluster_node where 1=1 [[filterstr]]) a left join (select id as cluster_id, + name as cluster_id_text from cluster where 1 = 1) b on a.cluster_id = b.cluster_id left join (select id as node_id, + name as node_id_text from compute_node where 1 = 1) c on a.node_id = c.node_id left join (select k as role, + v as role_text from appcodes_kv where parentid='node_role') d on a.role = d.role left join (select k as status, + v as status_text from appcodes_kv where parentid='node_status') e on a.status = e.status''' + +filterjson = params_kw.get('data_filter') +if not filterjson: + fields = [ f['name'] for f in [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "cluster_id", + "title": "所属集群", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "node_id", + "title": "算力节点", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "role", + "title": "角色(control/compute/storage)", + "type": "char", + "length": 16, + "nullable": "no" + }, + { + "name": "status", + "title": "状态(joining/active/draining/removed)", + "type": "char", + "length": 16, + "default": "joining" + }, + { + "name": "assigned_at", + "title": "分配时间", + "type": "timestamp", + "nullable": "no" + } +] ] + filterjson = default_filterjson(fields, ns) +filterdic = ns.copy() +filterdic['filterstr'] = '' +filterdic['userorgid'] = '${userorgid}$' +filterdic['userid'] = '${userid}$' +if filterjson: + dbf = DBFilter(filterjson) + conds = dbf.gen(ns) + if conds: + ns.update(dbf.consts) + conds = f' and {conds}' + filterdic['filterstr'] = conds +ac = ArgsConvert('[[', ']]') +vars = ac.findAllVariables(sql) +NameSpace = {v:'${' + v + '}$' for v in vars if v != 'filterstr' } +filterdic.update(NameSpace) +sql = ac.convert(sql, filterdic) + +debug(f'{sql=}') +db = DBPools() +dbname = get_module_dbname('pcc') +async with db.sqlorContext(dbname) as sor: + r = await sor.sqlPaging(sql, ns) + return r +return { + "total":0, + "rows":[] +} \ No newline at end of file diff --git a/wwwroot/cluster_node_list/index.ui b/wwwroot/cluster_node_list/index.ui new file mode 100644 index 0000000..ab0e7d0 --- /dev/null +++ b/wwwroot/cluster_node_list/index.ui @@ -0,0 +1,181 @@ + +{ + "id":"cluster_node_tbl", + "widgettype":"Tabular", + "options":{ + "width":"100%", + "height":"100%", + + + "title":"集群节点分配", + + + + + "css":"card", + + + "editable":{ + + "new_data_url":"{{entire_url('add_cluster_node.dspy')}}", + + + "delete_data_url":"{{entire_url('delete_cluster_node.dspy')}}", + + + "update_data_url":"{{entire_url('update_cluster_node.dspy')}}" + + }, + + + "data_url":"{{entire_url('./get_cluster_node.dspy')}}", + + "data_method":"GET", + "data_params":{{json.dumps(params_kw, indent=4, ensure_ascii=False)}}, + "row_options":{ + + + + "browserfields": { + "cluster_id": { + "title": "集群", + "width": 150 + }, + "node_id": { + "title": "节点", + "width": 150 + }, + "role": { + "title": "角色", + "width": 100 + }, + "status": { + "title": "状态", + "width": 100 + } +}, + + + "editexclouded":[ + "id", + "resellerid", + "created_at", + "updated_at" +], + + "fields":[ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32, + "nullable": "no", + "cwidth": 18, + "uitype": "str", + "datatype": "str", + "label": "id" + }, + { + "name": "cluster_id", + "title": "所属集群", + "type": "str", + "length": 32, + "nullable": "no", + "label": "所属集群", + "uitype": "code", + "valueField": "cluster_id", + "textField": "cluster_id_text", + "params": { + "dbname": "{{get_module_dbname('pcc')}}", + "table": "cluster", + "tblvalue": "id", + "tbltext": "name", + "valueField": "cluster_id", + "textField": "cluster_id_text" + }, + "dataurl": "{{entire_url('/appbase/get_code.dspy')}}" + }, + { + "name": "node_id", + "title": "算力节点", + "type": "str", + "length": 32, + "nullable": "no", + "label": "算力节点", + "uitype": "code", + "valueField": "node_id", + "textField": "node_id_text", + "params": { + "dbname": "{{get_module_dbname('pcc')}}", + "table": "compute_node", + "tblvalue": "id", + "tbltext": "name", + "valueField": "node_id", + "textField": "node_id_text" + }, + "dataurl": "{{entire_url('/appbase/get_code.dspy')}}" + }, + { + "name": "role", + "title": "角色(control/compute/storage)", + "type": "char", + "length": 16, + "nullable": "no", + "label": "角色(control/compute/storage)", + "uitype": "code", + "valueField": "role", + "textField": "role_text", + "params": { + "dbname": "{{get_module_dbname('pcc')}}", + "table": "appcodes_kv", + "tblvalue": "k", + "tbltext": "v", + "valueField": "role", + "textField": "role_text", + "cond": "parentid='node_role'" + }, + "dataurl": "{{entire_url('/appbase/get_code.dspy')}}" + }, + { + "name": "status", + "title": "状态(joining/active/draining/removed)", + "type": "char", + "length": 16, + "default": "joining", + "label": "状态(joining/active/draining/removed)", + "uitype": "code", + "valueField": "status", + "textField": "status_text", + "params": { + "dbname": "{{get_module_dbname('pcc')}}", + "table": "appcodes_kv", + "tblvalue": "k", + "tbltext": "v", + "valueField": "status", + "textField": "status_text", + "cond": "parentid='node_status'" + }, + "dataurl": "{{entire_url('/appbase/get_code.dspy')}}" + }, + { + "name": "assigned_at", + "title": "分配时间", + "type": "timestamp", + "nullable": "no", + "length": 0, + "uitype": "str", + "datatype": "timestamp", + "label": "分配时间" + } +] + }, + + + + "page_rows":160, + "cache_limit":5 + } + + ,"binds":[] + +} \ No newline at end of file diff --git a/wwwroot/cluster_node_list/update_cluster_node.dspy b/wwwroot/cluster_node_list/update_cluster_node.dspy new file mode 100644 index 0000000..865c94b --- /dev/null +++ b/wwwroot/cluster_node_list/update_cluster_node.dspy @@ -0,0 +1,75 @@ + +ns = params_kw.copy() +for k,v in ns.items(): + if v == 'NaN' or v == 'null': + ns[k] = None + + +userorgid = await get_userorgid() +if not userorgid: + return { + "widgettype":"Error", + "options":{ + "title":"Authorization Error", + "timeout":3, + "cwidth":16, + "cheight":9, + "message":"Please login" + } + } +ns['resellerid'] = userorgid + + + +for k in list(ns.keys()): + if k.endswith('_at') or k.endswith('_time') or k == 'last_heartbeat': + v = ns.get(k, '') + if v == '' or v is None or v == 'None': + ns[k] = None +db = DBPools() +dbname = get_module_dbname('pcc') +async with db.sqlorContext(dbname) as sor: + + ns1 = { + + "resellerid": userorgid, + + + "id": params_kw.id + } + recs = await sor.R('cluster_node', ns1) + if len(recs) < 1: + return { + "widgettype":"Error", + "options":{ + "title":"Update Error", + "cwidth":16, + "cheight":9, + "timeout":3, + "message":"Record no exist or with wrong ownership" + } + } + + r = await sor.U('cluster_node', ns) + debug('update success'); + return { + "widgettype":"Message", + "options":{ + "title":"Update Success", + "cwidth":16, + "cheight":9, + "timeout":3, + "message":"ok" + } + } + +return { + "widgettype":"Error", + "options":{ + "title":"Update Error", + "cwidth":16, + "cheight":9, + "timeout":3, + "message":"failed" + } +} \ No newline at end of file