xiaohei/Agentswarm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
fb1eeeebfa9ad057c103a5a82c6faeae1af41909
Agentswarm/orchestrator/task_queue.py
T
FastheiandClaude Opus 4.8 1edf73aba4 fix(swarm/#70): 任务超时透传+提默认 / 事件时间线降噪 / 失败 termination_reason 准确 
真机端到端实测(#70)暴露三问题,本 PR 全部修复(仅 agent + orchestrator,不跨仓):

问题1【阻断】单任务超时只有 60s,生成类任务必挂
- agent/main.py: TASK_TIMEOUT_SECONDS 默认 60→300(仅对外部/独立启动 agent 生效)。
- agent_launcher.py: 新增 DEFAULT_TASK_TIMEOUT_SECONDS=300、_budget_duration_seconds、
  resolve_task_timeout(base=env 默认 300,与 run budget.duration_seconds 取较小);
  plan_launch_specs 把 TASK_TIMEOUT_SECONDS 透传进每个 agent env(非敏感,inline,
  k8s 不进 Secret)。

问题2【体验】事件时间线全是内部噪音(纯附加,未碰冻结契约)
- swarm_runtime.py: is_client_visible(=event_type∈FROZEN_CLIENT_EVENT_TYPES,单一真源);
  emit_event 给 envelope 加 metadata.client_visible 布尔 + 关键客户端事件回填可选
  payload.message(人话进度,仅取已有字段,不伪造)。task.heartbeat/retried/
  deployment.status_changed/timeline/budget 标 client_visible=false,仍持久化+回调
  但客户端据此过滤出时间线。冻结事件集/类型/sequence/artifact 形状一字未动。
- event-schema.md: 文档化两个附加字段 + 新增 §6.1,明确未解冻。

问题3【正确性】失败/超时 termination_reason 仍报 "tasks_completed"
- convergence.py: 新增 TIMEOUT/MAX_RETRIES_EXCEEDED/TASK_FAILED;classify_failure_reason
  按 timeout→max_retries→task_failed 取最具体(仅凭真实 per-task 信号);FAILED 分支
  再不会返回 tasks_completed(该 reason 仅用于成功),budget/rounds 仅在通用失败时才覆盖。
- task_queue.py: fail_task 永久失败时把 reason 落到 task.result({"success":false,"error":reason}),
  不覆盖已有结果,供 convergence 读取。
- main.py: compute_convergence_report 快照补 retry_count/max_retries。

测试:新增 test_resolve_task_timeout、扩 test-convergence(failed_timeout/max_retries/
generic + "FAILED 永不报 tasks_completed"不变量)。本地全过:test-agent-launcher /
test-convergence / test-runtime-contract / test-contract-freeze / test-merge-smoke /
test-workflow-e2e / test-security-boundary。

影响:agent + orchestrator + 文档;不动 Manager↔Swarm 冻结契约字段(问题2 纯附加)。
栈在 #64(agent_swarm git 注入)之上,#64 合并后本 PR base 自动转 main。

Closes #70

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-16 18:29:39 +08:00

582 lines
21 KiB
Python
Raw Blame History

"""Task queue management with dependency-aware failure recovery."""
import json
import os
import time
import uuid
import logging
from typing import Dict, List, Optional
from enum import Enum
from pydantic import BaseModel, Field
from .redis_client import redis_client
from .agent_registry import agent_registry, AgentStatus
logger = logging.getLogger(__name__)
def _retry_backoff_seconds(retry_count: int) -> float:
"""Delay before a failed task becomes dispatchable again (exponential, capped).
Defaults: 5s → 30s → 180s … capped at 300s. Spreads retries over minutes instead of burning
the whole budget in seconds, so a swarm's own agents (which may be cold-starting / waiting on
node scale-up) have time to register before the seed exhausts its retries (incident 2026-06-15).
Tunable via TASK_RETRY_BACKOFF_{BASE,FACTOR,CAP}.
"""
try:
base = float(os.getenv("TASK_RETRY_BACKOFF_BASE", "5") or 5)
factor = float(os.getenv("TASK_RETRY_BACKOFF_FACTOR", "6") or 6)
cap = float(os.getenv("TASK_RETRY_BACKOFF_CAP", "300") or 300)
except ValueError:
base, factor, cap = 5.0, 6.0, 300.0
exponent = max(0, retry_count - 1)
return min(cap, base * (factor ** exponent))
class TaskStatus(str, Enum):
"""Task status enumeration."""
PENDING = "pending"
ASSIGNED = "assigned"
IN_PROGRESS = "in_progress"
BLOCKED = "blocked"
COMPLETED = "completed"
FAILED = "failed"
CANCELLED = "cancelled"
class Task(BaseModel):
"""Task model."""
task_id: str
title: Optional[str] = None
description: str
status: TaskStatus
agent_role: str = "general"
required_capabilities: List[str] = Field(default_factory=list)
depends_on: List[str] = Field(default_factory=list)
parent_task_id: Optional[str] = None
root_task_id: Optional[str] = None
source: str = "manual"
assigned_agent_id: Optional[str] = None
created_at: float
started_at: Optional[float] = None
completed_at: Optional[float] = None
result: Optional[str] = None
blocked_reason: Optional[str] = None
child_task_ids: List[str] = Field(default_factory=list)
retry_count: int = 0
max_retries: int = 3
next_retry_at: Optional[float] = None
context: Dict = Field(default_factory=dict)
class TaskQueue:
"""Manages task assignment and reassignment with failure recovery."""
TASK_KEY_PREFIX = "task:"
PENDING_QUEUE_KEY = "queue:pending"
AGENT_TASK_KEY_PREFIX = "agent_task:"
def __init__(self):
pass
async def _save_task(self, task: Task):
"""Persist a task snapshot."""
key = f"{self.TASK_KEY_PREFIX}{task.task_id}"
await redis_client.set(key, task.model_dump_json())
async def create_task(
self,
description: str,
context: Optional[Dict] = None,
max_retries: int = 3,
task_id: Optional[str] = None,
title: Optional[str] = None,
agent_role: str = "general",
required_capabilities: Optional[List[str]] = None,
depends_on: Optional[List[str]] = None,
parent_task_id: Optional[str] = None,
root_task_id: Optional[str] = None,
source: str = "manual",
enqueue: bool = True,
) -> Task:
"""Create a new task and add to pending queue."""
task = Task(
task_id=task_id or str(uuid.uuid4()),
title=title,
description=description,
status=TaskStatus.PENDING,
agent_role=agent_role,
required_capabilities=required_capabilities or [],
depends_on=depends_on or [],
parent_task_id=parent_task_id,
root_task_id=root_task_id,
source=source,
created_at=time.time(),
max_retries=max_retries,
context=context or {},
)
await self._save_task(task)
if enqueue:
await redis_client.lpush(self.PENDING_QUEUE_KEY, task.task_id)
logger.info(f"Created task {task.task_id}: {description}")
return task
async def add_child_task(self, parent_task_id: str, child_task_id: str):
"""Register a child task on a parent task."""
parent = await self.get_task(parent_task_id)
if not parent:
return
if child_task_id not in parent.child_task_ids:
parent.child_task_ids.append(child_task_id)
await self._save_task(parent)
async def get_ready_pending_task(
self,
agent_capabilities: Optional[List[str]] = None,
) -> Optional[Task]:
"""Return and dequeue the next dispatchable task for an agent."""
pending_ids = await redis_client.lrange(self.PENDING_QUEUE_KEY, 0, -1)
capabilities = set(agent_capabilities or [])
for task_id in pending_ids:
task = await self.get_task(task_id)
if not task:
await self.remove_pending_task(task_id)
continue
if task.status != TaskStatus.PENDING:
await self.remove_pending_task(task_id)
continue
if not await self.is_task_ready(task):
continue
if not self.can_agent_run_task(task, capabilities):
continue
await self.remove_pending_task(task_id)
return task
return None
async def get_ready_pending_tasks(
self,
agent_capabilities: Optional[List[str]] = None,
) -> List[Task]:
"""Return ALL dispatchable tasks for an agent WITHOUT dequeuing any.
Candidate enumeration for the ACO decision engine (score-at-pull): the caller
scores/samples one and removes it via remove_pending_task. Dead/stale queue
entries are cleaned up the same way get_ready_pending_task does.
"""
pending_ids = await redis_client.lrange(self.PENDING_QUEUE_KEY, 0, -1)
capabilities = set(agent_capabilities or [])
candidates: List[Task] = []
for task_id in pending_ids:
task = await self.get_task(task_id)
if not task:
await self.remove_pending_task(task_id)
continue
if task.status != TaskStatus.PENDING:
await self.remove_pending_task(task_id)
continue
if not await self.is_task_ready(task):
continue
if not self.can_agent_run_task(task, capabilities):
continue
candidates.append(task)
return candidates
async def is_task_ready(self, task: Task) -> bool:
"""Return True when all dependencies are terminal and successful."""
if task.status != TaskStatus.PENDING:
return False
# Respect retry backoff: a task re-queued after a failure is not dispatchable until its
# next_retry_at has passed (see fail_task / _retry_backoff_seconds).
if task.next_retry_at and time.time() < task.next_retry_at:
return False
for dependency_id in task.depends_on:
dependency = await self.get_task(dependency_id)
if not dependency or dependency.status != TaskStatus.COMPLETED:
return False
return True
def can_agent_run_task(self, task: Task, capabilities: set[str]) -> bool:
"""Return whether an agent capability set satisfies task requirements."""
required = set(task.required_capabilities or [])
if not required:
return True
return required.issubset(capabilities)
async def assign_task(self, task_id: str, agent_id: str) -> bool:
"""Assign a task to an agent."""
# Get task
task = await self.get_task(task_id)
if not task:
logger.error(f"Task {task_id} not found")
return False
if task.status not in [TaskStatus.PENDING, TaskStatus.FAILED]:
logger.error(f"Task {task_id} cannot be assigned (status: {task.status})")
return False
if task.status == TaskStatus.PENDING and not await self.is_task_ready(task):
logger.error(f"Task {task_id} cannot be assigned before dependencies complete")
return False
# Verify agent is idle
agent = await agent_registry.get_agent(agent_id)
if not agent or agent.status != AgentStatus.IDLE:
logger.error(f"Agent {agent_id} is not available for task assignment")
return False
if not self.can_agent_run_task(task, set(agent.capabilities)):
logger.error(f"Agent {agent_id} does not satisfy task {task_id} capabilities")
return False
# Update task
task.status = TaskStatus.ASSIGNED
task.assigned_agent_id = agent_id
task.started_at = time.time()
task.blocked_reason = None
await self._save_task(task)
# Track agent's current task
agent_task_key = f"{self.AGENT_TASK_KEY_PREFIX}{agent_id}"
await redis_client.set(agent_task_key, task_id)
# Update agent status
await agent_registry.update_status(agent_id, AgentStatus.BUSY, task_id)
logger.info(f"Assigned task {task_id} to agent {agent_id}")
return True
async def start_task(self, task_id: str) -> bool:
"""Mark task as in progress."""
task = await self.get_task(task_id)
if not task:
return False
task.status = TaskStatus.IN_PROGRESS
task.blocked_reason = None
await self._save_task(task)
logger.info(f"Task {task_id} started")
return True
async def block_task(
self,
task_id: str,
reason: str = "",
release_agent: bool = True,
) -> Optional[Task]:
"""Mark a task blocked while waiting for delegated child work."""
task = await self.get_task(task_id)
if not task:
return None
if task.status in {TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED}:
return task
previous_agent_id = task.assigned_agent_id
task.status = TaskStatus.BLOCKED
task.blocked_reason = reason or "Waiting for delegated handoff work"
task.assigned_agent_id = None if release_agent else task.assigned_agent_id
await self.remove_pending_task(task_id)
await self._save_task(task)
if release_agent and previous_agent_id:
agent_task_key = f"{self.AGENT_TASK_KEY_PREFIX}{previous_agent_id}"
await redis_client.delete(agent_task_key)
await agent_registry.update_status(previous_agent_id, AgentStatus.IDLE)
logger.info(f"Task {task_id} blocked: {task.blocked_reason}")
return task
async def complete_task(self, task_id: str, result: str = None) -> bool:
"""Mark task as completed."""
task = await self.get_task(task_id)
if not task:
return False
if task.status == TaskStatus.CANCELLED:
logger.warning(f"Ignoring completion for cancelled task {task_id}")
return False
task.status = TaskStatus.COMPLETED
task.completed_at = time.time()
task.blocked_reason = None
if result:
task.result = result
await self._save_task(task)
# Clear agent's current task
if task.assigned_agent_id:
agent_task_key = f"{self.AGENT_TASK_KEY_PREFIX}{task.assigned_agent_id}"
await redis_client.delete(agent_task_key)
# Update agent to idle
await agent_registry.update_status(task.assigned_agent_id, AgentStatus.IDLE)
logger.info(f"Task {task_id} completed")
return True
async def fail_task(self, task_id: str, reason: str = "") -> bool:
"""Mark task as failed and handle retry logic."""
task = await self.get_task(task_id)
if not task:
return False
if task.status == TaskStatus.CANCELLED:
logger.warning(f"Ignoring failure for cancelled task {task_id}: {reason}")
return False
task.retry_count += 1
# Clear agent's current task
if task.assigned_agent_id:
agent_task_key = f"{self.AGENT_TASK_KEY_PREFIX}{task.assigned_agent_id}"
await redis_client.delete(agent_task_key)
# Update agent to idle
await agent_registry.update_status(task.assigned_agent_id, AgentStatus.IDLE)
# Check if we should retry
if task.retry_count < task.max_retries:
task.status = TaskStatus.PENDING
task.assigned_agent_id = None
task.started_at = None
# Backoff: gate re-dispatch until next_retry_at (is_task_ready enforces it) so retries
# spread over minutes rather than all firing within seconds.
delay = _retry_backoff_seconds(task.retry_count)
task.next_retry_at = time.time() + delay
# Re-add to pending queue
await redis_client.lpush(self.PENDING_QUEUE_KEY, task_id)
logger.warning(
f"Task {task_id} failed (retry {task.retry_count}/{task.max_retries}): {reason} "
f"(next retry in {delay:.0f}s)"
)
else:
task.status = TaskStatus.FAILED
task.completed_at = time.time()
# Persist the terminal failure reason on the task so downstream convergence (#70) can
# derive an accurate termination_reason (timeout / max_retries_exceeded / task_failed).
# Stored in the result envelope, mirroring the agent's failure payload shape
# ({"success": false, "error": ...}); does not overwrite a structured result if one
# already captured the cause.
if reason and not task.result:
task.result = json.dumps({"success": False, "error": reason})
logger.error(
f"Task {task_id} permanently failed after {task.retry_count} retries: {reason}"
)
task.blocked_reason = None if task.status == TaskStatus.PENDING else task.blocked_reason
await self._save_task(task)
return True
async def cancel_task(self, task_id: str, reason: str = "") -> bool:
"""Cancel a task without retrying it."""
task = await self.get_task(task_id)
if not task:
return False
if task.status in {TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED}:
logger.info(f"Task {task_id} already terminal ({task.status}); skip cancellation")
return False
if task.assigned_agent_id:
agent_task_key = f"{self.AGENT_TASK_KEY_PREFIX}{task.assigned_agent_id}"
await redis_client.delete(agent_task_key)
await self.remove_pending_task(task_id)
task.status = TaskStatus.CANCELLED
task.completed_at = time.time()
task.blocked_reason = None
if reason:
task.result = json.dumps({"cancelled": True, "reason": reason})
await self._save_task(task)
logger.info(f"Task {task_id} cancelled: {reason}")
return True
async def release_task(self, task_id: str, agent_id: Optional[str] = None) -> bool:
"""Return an assigned/in-progress task to the pending queue (e.g. agent rejected it).
Unlike fail_task this does not increment retry_count: a capacity rejection is not
a task failure, just a dispatch that needs to find a different agent.
"""
task = await self.get_task(task_id)
if not task:
return False
if task.status not in {TaskStatus.ASSIGNED, TaskStatus.IN_PROGRESS}:
return False
if agent_id and task.assigned_agent_id and task.assigned_agent_id != agent_id:
return False
previous_agent_id = task.assigned_agent_id
task.status = TaskStatus.PENDING
task.assigned_agent_id = None
task.started_at = None
task.blocked_reason = None
task.next_retry_at = None # a capacity release is not a failure — no backoff
await self._save_task(task)
await self.requeue_task(task_id)
if previous_agent_id:
agent_task_key = f"{self.AGENT_TASK_KEY_PREFIX}{previous_agent_id}"
await redis_client.delete(agent_task_key)
await agent_registry.update_status(previous_agent_id, AgentStatus.IDLE)
logger.info(f"Released task {task_id} back to pending queue")
return True
async def reopen_task(self, task_id: str) -> bool:
"""Re-open a completed task for another round (review loop rejected its result).
Resets the task to PENDING and requeues it without touching retry_count (a review
rejection is a quality decision, not a failure). The review-cycle budget in the
orchestrator bounds how many times this can happen.
"""
task = await self.get_task(task_id)
if not task:
return False
if task.status not in {TaskStatus.COMPLETED, TaskStatus.FAILED}:
return False
if task.assigned_agent_id:
agent_task_key = f"{self.AGENT_TASK_KEY_PREFIX}{task.assigned_agent_id}"
await redis_client.delete(agent_task_key)
task.status = TaskStatus.PENDING
task.assigned_agent_id = None
task.started_at = None
task.completed_at = None
task.blocked_reason = None
task.next_retry_at = None # a review reopen is not a failure — no backoff
await self._save_task(task)
await self.requeue_task(task_id)
logger.info(f"Re-opened task {task_id} for another review cycle")
return True
async def reassign_agent_tasks(self, failed_agent_id: str) -> List[str]:
"""Reassign all tasks from a failed agent back to pending queue."""
# Get agent's current task
agent_task_key = f"{self.AGENT_TASK_KEY_PREFIX}{failed_agent_id}"
task_id = await redis_client.get(agent_task_key)
reassigned_tasks = []
if task_id:
await self.fail_task(task_id, f"Agent {failed_agent_id} failed")
reassigned_tasks.append(task_id)
logger.info(
f"Reassigned {len(reassigned_tasks)} tasks from failed agent {failed_agent_id}"
)
return reassigned_tasks
async def recover_orphaned_tasks(
self,
active_agent_ids: set[str],
stale_after_seconds: int = 30,
) -> List[str]:
"""Recover assigned/in-progress tasks whose agents are no longer connected."""
current_time = time.time()
recoverable_statuses = {TaskStatus.ASSIGNED, TaskStatus.IN_PROGRESS}
recovered_tasks = []
for task in await self.get_all_tasks():
if task.status not in recoverable_statuses:
continue
if not task.assigned_agent_id:
continue
if task.assigned_agent_id in active_agent_ids:
continue
if task.started_at and current_time - task.started_at < stale_after_seconds:
continue
old_agent_id = task.assigned_agent_id
agent_task_key = f"{self.AGENT_TASK_KEY_PREFIX}{old_agent_id}"
await redis_client.delete(agent_task_key)
task.status = TaskStatus.PENDING
task.assigned_agent_id = None
task.started_at = None
if await self.is_task_ready(task):
await self._save_task(task)
await redis_client.lpush(self.PENDING_QUEUE_KEY, task.task_id)
else:
await self._save_task(task)
recovered_tasks.append(task.task_id)
logger.warning(
f"Recovered orphaned task {task.task_id} from inactive agent {old_agent_id}"
)
return recovered_tasks
async def get_task(self, task_id: str) -> Optional[Task]:
"""Get task by ID."""
key = f"{self.TASK_KEY_PREFIX}{task_id}"
data = await redis_client.get(key)
if not data:
return None
return Task.model_validate_json(data)
async def get_next_pending_task(self) -> Optional[Task]:
"""Get next pending task from queue."""
task_id = await redis_client.rpop(self.PENDING_QUEUE_KEY)
if not task_id:
return None
return await self.get_task(task_id)
async def requeue_task(self, task_id: str):
"""Put a task back on the pending queue."""
await redis_client.rpush(self.PENDING_QUEUE_KEY, task_id)
async def remove_pending_task(self, task_id: str):
"""Remove a task from the pending queue if present."""
await redis_client.lrem(self.PENDING_QUEUE_KEY, 0, task_id)
async def get_pending_count(self) -> int:
"""Get count of pending tasks."""
return await redis_client.llen(self.PENDING_QUEUE_KEY)
async def get_dependents(self, task_id: str) -> List[Task]:
"""Return tasks that directly depend on a task."""
return [
task for task in await self.get_all_tasks()
if task_id in task.depends_on
]
async def get_all_tasks(self, status: Optional[TaskStatus] = None) -> List[Task]:
"""Get all tasks, optionally filtered by status."""
pattern = f"{self.TASK_KEY_PREFIX}*"
keys = await redis_client.keys(pattern)
tasks = []
for key in keys:
data = await redis_client.get(key)
if data:
task = Task.model_validate_json(data)
if status is None or task.status == status:
tasks.append(task)
return tasks
# Global task queue instance
task_queue = TaskQueue()
Reference in New Issue View Git Blame Copy Permalink