584 lines
25 KiB
Python
584 lines
25 KiB
Python
"""
|
|
Hermes Agent Orchestrator - Enhanced with true workflow orchestration capabilities
|
|
Implements workflow parsing, parallel execution, and skill-based automation
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import uuid
|
|
from appPublic.uniqueID import getID
|
|
from typing import Dict, Any, List, Optional, Tuple
|
|
from datetime import datetime
|
|
from dataclasses import dataclass
|
|
|
|
# Import required dependencies
|
|
try:
|
|
from ahserver.serverenv import ServerEnv
|
|
from appPublic.worker import awaitify
|
|
from sqlor.dbpools import DBPools
|
|
except ImportError:
|
|
# For standalone testing
|
|
class ServerEnv:
|
|
def __init__(self):
|
|
pass
|
|
|
|
def awaitify(func):
|
|
async def wrapper(*args, **kwargs):
|
|
return func(*args, **kwargs)
|
|
return wrapper
|
|
|
|
class DBPools:
|
|
def __init__(self):
|
|
pass
|
|
|
|
def getConfig():
|
|
class Config:
|
|
databases = None
|
|
return Config()
|
|
|
|
@dataclass
|
|
class TaskDefinition:
|
|
"""Task definition structure for workflow execution"""
|
|
id: str
|
|
task_name: str
|
|
task_type: str
|
|
skill_name: Optional[str] = None
|
|
tool_name: Optional[str] = None
|
|
parameters: Dict[str, Any] = None
|
|
depends_on: Optional[str] = None
|
|
parallel_group: Optional[str] = None
|
|
timeout_seconds: int = 300
|
|
retry_count: int = 2
|
|
order_index: int = 0
|
|
|
|
@dataclass
|
|
class WorkflowDefinition:
|
|
"""Workflow definition structure"""
|
|
id: str
|
|
name: str
|
|
description: str = ""
|
|
workflow_type: str = "sequential"
|
|
max_concurrent_tasks: int = 3
|
|
timeout_seconds: int = 1800
|
|
retry_count: int = 2
|
|
tasks: List[TaskDefinition] = None
|
|
|
|
class HermesOrchestrator:
|
|
"""Core orchestrator implementation with workflow execution capabilities"""
|
|
|
|
def __init__(self, harnessed_agent_instance):
|
|
self.harnessed_agent = harnessed_agent_instance
|
|
|
|
def _get_current_user_id(self, context: Dict[str, Any]) -> str:
|
|
"""Get current user ID from request context"""
|
|
user_id = context.get('user_id') or context.get('userid')
|
|
if not user_id:
|
|
raise ValueError("User ID not found in context. User must be authenticated.")
|
|
return str(user_id)
|
|
|
|
async def create_workflow(self, name: str, description: str = "",
|
|
workflow_type: str = "sequential",
|
|
max_concurrent_tasks: int = 3,
|
|
timeout_seconds: int = 1800,
|
|
retry_count: int = 2,
|
|
context: Dict[str, Any] = None) -> Dict[str, Any]:
|
|
"""Create a new workflow definition"""
|
|
user_id = self._get_current_user_id(context) if context else "anonymous"
|
|
|
|
try:
|
|
workflow_id = getID()
|
|
env = ServerEnv()
|
|
|
|
dbname = env.get_module_dbname('harnessed_agent')
|
|
|
|
config = getConfig()
|
|
|
|
db = DBPools()
|
|
|
|
db.databases = config.databases
|
|
|
|
async with db.sqlorContext(dbname) as sor:
|
|
data = {
|
|
'id': workflow_id,
|
|
'user_id': user_id,
|
|
'name': name,
|
|
'description': description,
|
|
'workflow_type': workflow_type,
|
|
'max_concurrent_tasks': max_concurrent_tasks,
|
|
'timeout_seconds': timeout_seconds,
|
|
'retry_count': retry_count,
|
|
'status': 'active',
|
|
'created_at': datetime.now(),
|
|
'updated_at': datetime.now()
|
|
}
|
|
result = await sor.C('hermes_workflows', data)
|
|
return {"success": True, "workflow_id": workflow_id, "user_id": user_id}
|
|
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e), "user_id": user_id}
|
|
|
|
async def add_task_to_workflow(self, workflow_id: str, task_name: str,
|
|
task_type: str, skill_name: str = None,
|
|
tool_name: str = None, parameters: Dict[str, Any] = None,
|
|
depends_on: str = None, parallel_group: str = None,
|
|
timeout_seconds: int = 300, retry_count: int = 2,
|
|
order_index: int = 0,
|
|
context: Dict[str, Any] = None) -> Dict[str, Any]:
|
|
"""Add a task to an existing workflow"""
|
|
user_id = self._get_current_user_id(context) if context else "anonymous"
|
|
|
|
try:
|
|
# Verify workflow exists and belongs to user
|
|
env = ServerEnv()
|
|
|
|
dbname = env.get_module_dbname('harnessed_agent')
|
|
|
|
config = getConfig()
|
|
|
|
db = DBPools()
|
|
|
|
db.databases = config.databases
|
|
|
|
async with db.sqlorContext(dbname) as sor:
|
|
workflows = await sor.R('hermes_workflows', {
|
|
'id': workflow_id,
|
|
'user_id': user_id
|
|
})
|
|
if not workflows:
|
|
return {"success": False, "error": "Workflow not found or access denied"}
|
|
|
|
task_id = getID()
|
|
data = {
|
|
'id': task_id,
|
|
'user_id': user_id,
|
|
'workflow_id': workflow_id,
|
|
'task_name': task_name,
|
|
'task_type': task_type,
|
|
'skill_name': skill_name,
|
|
'tool_name': tool_name,
|
|
'parameters_json': json.dumps(parameters) if parameters else None,
|
|
'depends_on': depends_on,
|
|
'parallel_group': parallel_group,
|
|
'timeout_seconds': timeout_seconds,
|
|
'retry_count': retry_count,
|
|
'order_index': order_index,
|
|
'created_at': datetime.now(),
|
|
'updated_at': datetime.now()
|
|
}
|
|
result = await sor.C('hermes_tasks', data)
|
|
return {"success": True, "task_id": task_id, "workflow_id": workflow_id, "user_id": user_id}
|
|
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e), "user_id": user_id}
|
|
|
|
async def execute_workflow(self, workflow_id: str,
|
|
context: Dict[str, Any] = None) -> Dict[str, Any]:
|
|
"""Execute a complete workflow with proper orchestration"""
|
|
user_id = self._get_current_user_id(context) if context else "anonymous"
|
|
|
|
try:
|
|
# Load workflow definition
|
|
workflow_def = await self._load_workflow_definition(workflow_id, user_id)
|
|
if not workflow_def["success"]:
|
|
return workflow_def
|
|
|
|
workflow = workflow_def["workflow"]
|
|
|
|
# Execute based on workflow type
|
|
if workflow.workflow_type == "sequential":
|
|
result = await self._execute_sequential_workflow(workflow, user_id, context)
|
|
elif workflow.workflow_type == "parallel":
|
|
result = await self._execute_parallel_workflow(workflow, user_id, context)
|
|
elif workflow.workflow_type == "hybrid":
|
|
result = await self._execute_hybrid_workflow(workflow, user_id, context)
|
|
else:
|
|
return {"success": False, "error": f"Unknown workflow type: {workflow.workflow_type}"}
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e), "user_id": user_id}
|
|
|
|
async def _load_workflow_definition(self, workflow_id: str, user_id: str) -> Dict[str, Any]:
|
|
"""Load complete workflow definition with all tasks"""
|
|
try:
|
|
env = ServerEnv()
|
|
|
|
dbname = env.get_module_dbname('harnessed_agent')
|
|
|
|
config = getConfig()
|
|
|
|
db = DBPools()
|
|
|
|
db.databases = config.databases
|
|
|
|
async with db.sqlorContext(dbname) as sor:
|
|
# Load workflow
|
|
workflows = await sor.R('hermes_workflows', {
|
|
'id': workflow_id,
|
|
'user_id': user_id
|
|
})
|
|
if not workflows:
|
|
return {"success": False, "error": "Workflow not found"}
|
|
|
|
workflow_data = workflows[0]
|
|
|
|
# Load tasks
|
|
tasks = await sor.R('hermes_tasks', {
|
|
'workflow_id': workflow_id,
|
|
'user_id': user_id,
|
|
'sort': 'order_index asc'
|
|
})
|
|
|
|
# Convert to TaskDefinition objects
|
|
task_definitions = []
|
|
for task_data in tasks:
|
|
task_def = TaskDefinition(
|
|
id=task_data['id'],
|
|
task_name=task_data['task_name'],
|
|
task_type=task_data['task_type'],
|
|
skill_name=task_data.get('skill_name'),
|
|
tool_name=task_data.get('tool_name'),
|
|
parameters=json.loads(task_data['parameters_json']) if task_data.get('parameters_json') else {},
|
|
depends_on=task_data.get('depends_on'),
|
|
parallel_group=task_data.get('parallel_group'),
|
|
timeout_seconds=task_data['timeout_seconds'],
|
|
retry_count=task_data['retry_count'],
|
|
order_index=task_data['order_index']
|
|
)
|
|
task_definitions.append(task_def)
|
|
|
|
workflow_def = WorkflowDefinition(
|
|
id=workflow_data['id'],
|
|
name=workflow_data['name'],
|
|
description=workflow_data['description'],
|
|
workflow_type=workflow_data['workflow_type'],
|
|
max_concurrent_tasks=workflow_data['max_concurrent_tasks'],
|
|
timeout_seconds=workflow_data['timeout_seconds'],
|
|
retry_count=workflow_data['retry_count'],
|
|
tasks=task_definitions
|
|
)
|
|
|
|
return {"success": True, "workflow": workflow_def}
|
|
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
async def _execute_sequential_workflow(self, workflow: WorkflowDefinition,
|
|
user_id: str, context: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Execute workflow tasks sequentially"""
|
|
results = []
|
|
task_results = {}
|
|
|
|
for task in workflow.tasks:
|
|
# Check dependencies
|
|
if task.depends_on and task.depends_on not in task_results:
|
|
return {"success": False, "error": f"Dependency task {task.depends_on} not found", "user_id": user_id}
|
|
|
|
if task.depends_on and not task_results.get(task.depends_on, {}).get("success"):
|
|
return {"success": False, "error": f"Dependency task {task.depends_on} failed", "user_id": user_id}
|
|
|
|
# Execute task with retries
|
|
task_result = await self._execute_task_with_retries(task, user_id, context, workflow.retry_count)
|
|
task_results[task.id] = task_result
|
|
results.append(task_result)
|
|
|
|
if not task_result["success"]:
|
|
return {"success": False, "error": f"Task {task.task_name} failed: {task_result.get('error', 'Unknown error')}",
|
|
"results": results, "user_id": user_id}
|
|
|
|
return {"success": True, "results": results, "user_id": user_id}
|
|
|
|
async def _execute_parallel_workflow(self, workflow: WorkflowDefinition,
|
|
user_id: str, context: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Execute workflow tasks in parallel (up to max_concurrent_tasks)"""
|
|
semaphore = asyncio.Semaphore(workflow.max_concurrent_tasks)
|
|
results = []
|
|
task_futures = []
|
|
|
|
async def execute_task_limited(task):
|
|
async with semaphore:
|
|
return await self._execute_task_with_retries(task, user_id, context, workflow.retry_count)
|
|
|
|
# Create tasks for all workflow tasks
|
|
for task in workflow.tasks:
|
|
future = asyncio.create_task(execute_task_limited(task))
|
|
task_futures.append((task.id, future))
|
|
|
|
# Wait for all tasks to complete
|
|
for task_id, future in task_futures:
|
|
try:
|
|
result = await future
|
|
results.append(result)
|
|
if not result["success"]:
|
|
# Continue to let other tasks finish, but mark overall failure
|
|
pass
|
|
except Exception as e:
|
|
error_result = {"success": False, "error": str(e), "task_id": task_id}
|
|
results.append(error_result)
|
|
|
|
# Check if any task failed
|
|
any_failed = any(not r["success"] for r in results)
|
|
if any_failed:
|
|
return {"success": False, "results": results, "user_id": user_id}
|
|
else:
|
|
return {"success": True, "results": results, "user_id": user_id}
|
|
|
|
async def _execute_hybrid_workflow(self, workflow: WorkflowDefinition,
|
|
user_id: str, context: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Execute hybrid workflow with both sequential and parallel groups"""
|
|
# Group tasks by parallel_group
|
|
groups = {}
|
|
sequential_tasks = []
|
|
|
|
for task in workflow.tasks:
|
|
if task.parallel_group:
|
|
if task.parallel_group not in groups:
|
|
groups[task.parallel_group] = []
|
|
groups[task.parallel_group].append(task)
|
|
else:
|
|
sequential_tasks.append(task)
|
|
|
|
results = []
|
|
task_results = {}
|
|
|
|
# Execute sequential tasks first (including parallel groups as single units)
|
|
all_execution_units = []
|
|
|
|
# Add individual sequential tasks
|
|
for task in sequential_tasks:
|
|
all_execution_units.append(("sequential", task))
|
|
|
|
# Add parallel groups
|
|
for group_name, group_tasks in groups.items():
|
|
all_execution_units.append(("parallel_group", group_name, group_tasks))
|
|
|
|
# Sort by order_index of first task in each unit
|
|
def get_order_key(unit):
|
|
if unit[0] == "sequential":
|
|
return unit[1].order_index
|
|
else:
|
|
return min(task.order_index for task in unit[2])
|
|
|
|
all_execution_units.sort(key=get_order_key)
|
|
|
|
# Execute units in order
|
|
for unit in all_execution_units:
|
|
if unit[0] == "sequential":
|
|
task = unit[1]
|
|
# Check dependencies
|
|
if task.depends_on and task.depends_on not in task_results:
|
|
return {"success": False, "error": f"Dependency task {task.depends_on} not found", "user_id": user_id}
|
|
|
|
if task.depends_on and not task_results.get(task.depends_on, {}).get("success"):
|
|
return {"success": False, "error": f"Dependency task {task.depends_on} failed", "user_id": user_id}
|
|
|
|
task_result = await self._execute_task_with_retries(task, user_id, context, workflow.retry_count)
|
|
task_results[task.id] = task_result
|
|
results.append(task_result)
|
|
|
|
if not task_result["success"]:
|
|
return {"success": False, "error": f"Task {task.task_name} failed", "results": results, "user_id": user_id}
|
|
|
|
else: # parallel_group
|
|
group_name = unit[1]
|
|
group_tasks = unit[2]
|
|
|
|
# Check dependencies for all tasks in group
|
|
for task in group_tasks:
|
|
if task.depends_on and task.depends_on not in task_results:
|
|
return {"success": False, "error": f"Dependency task {task.depends_on} not found in group {group_name}", "user_id": user_id}
|
|
|
|
if task.depends_on and not task_results.get(task.depends_on, {}).get("success"):
|
|
return {"success": False, "error": f"Dependency task {task.depends_on} failed in group {group_name}", "user_id": user_id}
|
|
|
|
# Execute group in parallel
|
|
group_results = await self._execute_parallel_task_group(group_tasks, user_id, context, workflow.retry_count)
|
|
results.extend(group_results)
|
|
|
|
# Store individual task results
|
|
for i, task in enumerate(group_tasks):
|
|
task_results[task.id] = group_results[i]
|
|
|
|
# Check if any task in group failed
|
|
if any(not r["success"] for r in group_results):
|
|
return {"success": False, "error": f"Parallel group {group_name} failed", "results": results, "user_id": user_id}
|
|
|
|
return {"success": True, "results": results, "user_id": user_id}
|
|
|
|
async def _execute_parallel_task_group(self, tasks: List[TaskDefinition],
|
|
user_id: str, context: Dict[str, Any],
|
|
max_retries: int) -> List[Dict[str, Any]]:
|
|
"""Execute a group of tasks in parallel"""
|
|
semaphore = asyncio.Semaphore(len(tasks)) # Allow all tasks in group to run concurrently
|
|
|
|
async def execute_task_limited(task):
|
|
async with semaphore:
|
|
return await self._execute_task_with_retries(task, user_id, context, max_retries)
|
|
|
|
futures = [asyncio.create_task(execute_task_limited(task)) for task in tasks]
|
|
results = []
|
|
|
|
for future in futures:
|
|
try:
|
|
result = await future
|
|
results.append(result)
|
|
except Exception as e:
|
|
results.append({"success": False, "error": str(e)})
|
|
|
|
return results
|
|
|
|
async def _execute_task_with_retries(self, task: TaskDefinition,
|
|
user_id: str, context: Dict[str, Any],
|
|
max_retries: int) -> Dict[str, Any]:
|
|
"""Execute a single task with retry logic"""
|
|
execution_id = getID()
|
|
|
|
# Record execution start
|
|
await self._record_execution_start(execution_id, user_id, task, context)
|
|
|
|
last_error = None
|
|
for attempt in range(max_retries + 1):
|
|
try:
|
|
if attempt > 0:
|
|
# Wait before retry (exponential backoff)
|
|
await asyncio.sleep(2 ** attempt)
|
|
|
|
# Execute the actual task
|
|
result = await self._execute_single_task(task, user_id, context)
|
|
|
|
# Record successful execution
|
|
await self._record_execution_end(execution_id, user_id, "completed", result, None, attempt)
|
|
return result
|
|
|
|
except Exception as e:
|
|
last_error = str(e)
|
|
if attempt < max_retries:
|
|
continue
|
|
else:
|
|
# Record failed execution
|
|
await self._record_execution_end(execution_id, user_id, "failed", None, last_error, attempt)
|
|
return {"success": False, "error": last_error, "task_id": task.id, "attempts": attempt + 1}
|
|
|
|
# This should never be reached
|
|
return {"success": False, "error": "Unexpected execution state", "task_id": task.id}
|
|
|
|
async def _execute_single_task(self, task: TaskDefinition,
|
|
user_id: str, context: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Execute a single task based on its type"""
|
|
if task.task_type == "skill":
|
|
if not task.skill_name:
|
|
return {"success": False, "error": "Skill name required for skill task type"}
|
|
return await self.harnessed_agent.manage_skills("view", task.skill_name, context=context)
|
|
|
|
elif task.task_type == "tool":
|
|
if not task.tool_name:
|
|
return {"success": False, "error": "Tool name required for tool task type"}
|
|
return await self.harnessed_agent.execute_tool_call(task.tool_name, task.parameters or {}, context=context)
|
|
|
|
elif task.task_type == "memory":
|
|
# Memory operations require specific action parameter
|
|
action = task.parameters.get("action") if task.parameters else None
|
|
if not action:
|
|
return {"success": False, "error": "Memory action required (add/replace/remove)"}
|
|
return await self.harnessed_agent.manage_memory(
|
|
action,
|
|
task.parameters.get("target", "memory"),
|
|
task.parameters.get("content", ""),
|
|
task.parameters.get("old_text", ""),
|
|
context=context,
|
|
priority=task.parameters.get("priority")
|
|
)
|
|
|
|
elif task.task_type == "session_search":
|
|
query = task.parameters.get("query", "") if task.parameters else ""
|
|
limit = task.parameters.get("limit", 3) if task.parameters else 3
|
|
return await self.harnessed_agent.search_sessions(query, limit, context=context)
|
|
|
|
elif task.task_type == "custom":
|
|
# Custom script execution would go here
|
|
return {"success": True, "result": "Custom task executed", "task_id": task.id}
|
|
|
|
else:
|
|
return {"success": False, "error": f"Unknown task type: {task.task_type}"}
|
|
|
|
async def _record_execution_start(self, execution_id: str, user_id: str,
|
|
task: TaskDefinition, context: Dict[str, Any]):
|
|
"""Record execution start in database"""
|
|
try:
|
|
env = ServerEnv()
|
|
|
|
dbname = env.get_module_dbname('harnessed_agent')
|
|
|
|
config = getConfig()
|
|
|
|
db = DBPools()
|
|
|
|
db.databases = config.databases
|
|
|
|
async with db.sqlorContext(dbname) as sor:
|
|
data = {
|
|
'id': execution_id,
|
|
'user_id': user_id,
|
|
'workflow_id': task.workflow_id if hasattr(task, 'workflow_id') else "",
|
|
'task_id': task.id,
|
|
'execution_status': 'running',
|
|
'start_time': datetime.now(),
|
|
'created_at': datetime.now(),
|
|
'updated_at': datetime.now()
|
|
}
|
|
await sor.C('hermes_executions', data)
|
|
except Exception:
|
|
# Silently ignore recording errors
|
|
pass
|
|
|
|
async def _record_execution_end(self, execution_id: str, user_id: str, status: str,
|
|
result: Dict[str, Any], error: str, retry_count: int):
|
|
"""Record execution end in database"""
|
|
try:
|
|
env = ServerEnv()
|
|
|
|
dbname = env.get_module_dbname('harnessed_agent')
|
|
|
|
config = getConfig()
|
|
|
|
db = DBPools()
|
|
|
|
db.databases = config.databases
|
|
|
|
async with db.sqlorContext(dbname) as sor:
|
|
end_time = datetime.now()
|
|
data = {
|
|
'id': execution_id,
|
|
'user_id': user_id,
|
|
'execution_status': status,
|
|
'end_time': end_time,
|
|
'duration_seconds': None, # Will be calculated
|
|
'result_json': json.dumps(result) if result else None,
|
|
'error_message': error,
|
|
'retry_count': retry_count,
|
|
'updated_at': end_time
|
|
}
|
|
# Get start time to calculate duration
|
|
executions = await sor.R('hermes_executions', {'id': execution_id, 'user_id': user_id})
|
|
if executions and executions[0].get('start_time'):
|
|
start_time = executions[0]['start_time']
|
|
if isinstance(start_time, str):
|
|
start_time = datetime.fromisoformat(start_time.replace('Z', '+00:00'))
|
|
duration = (end_time - start_time).total_seconds()
|
|
data['duration_seconds'] = int(duration)
|
|
|
|
await sor.U('hermes_executions', data)
|
|
except Exception:
|
|
# Silently ignore recording errors
|
|
pass
|
|
|
|
# Global orchestrator instance
|
|
_orchestrator_instance = None
|
|
|
|
def get_hermes_orchestrator(harnessed_agent_instance):
|
|
"""Get or create the global orchestrator instance"""
|
|
global _orchestrator_instance
|
|
if _orchestrator_instance is None:
|
|
_orchestrator_instance = HermesOrchestrator(harnessed_agent_instance)
|
|
return _orchestrator_instance |