fix(swarm): reap orphan agents on terminal run + retry backoff + surface task errors
孤儿 agent 事故修复(2026-06-15 swarm-69e470561bd7-seed 跨 workspace 执行)。 不改去中心化认领逻辑(swarm_dispatch 不动),仅治理 agent pod 生命周期与可观测性: - B1 终态回收: refresh_swarm_run_status 终态后 best-effort stop_launched 回收 pod+secret (此前仅 Manager stop 才回收,自然完成/失败的 run agent 残留 → 孤儿留在共享池抢别的 swarm 任务) - B2 防驱逐: agent pod 加 karpenter.sh/do-not-disrupt(AGENT_POD_ALLOW_DISRUPTION=1 可关) - stop_launched: backend=kubernetes 时按标签删,不再被内存集合 _k8s_swarms 门控(跨重启可靠) - C 重试退避: Task.next_retry_at + fail_task 指数退避 5→30→180s(cap 300, TASK_RETRY_BACKOFF_*), is_task_ready 门控;TASK_MAX_RETRIES 可配 - D 错误上浮: task_executor 失败时聚合 subtask error 到顶层 error(根治通用 "Task failed"), _execute_subtask 打印 LLM 响应片段 - 配额硬上限 16: _clamp_user_cap 契约测试全过: runtime-contract / merge-smoke / workflow-e2e / contract-freeze / max-agents-per-user / security-boundary。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
f9f1d9a6d5
commit
b394ce99d5
+32
-2
@@ -117,7 +117,7 @@ class TaskExecutor:
|
||||
|
||||
success = all(r["status"] in ["completed", "handed_off"] for r in results)
|
||||
awaiting_handoff = any(r["status"] == "handed_off" for r in results)
|
||||
return {
|
||||
payload = {
|
||||
"success": success,
|
||||
"task_id": task_id,
|
||||
"subtasks": results,
|
||||
@@ -125,6 +125,19 @@ class TaskExecutor:
|
||||
"agent_id": self.agent_id,
|
||||
"usage": self._usage_payload(time.time() - started_at),
|
||||
}
|
||||
if not success:
|
||||
# Surface the real per-subtask failure detail as a top-level `error` so the
|
||||
# orchestrator records WHY (agent/main.py falls back to a generic "Task failed"
|
||||
# when this key is absent — see incident 2026-06-15 swarm-69e470561bd7-seed).
|
||||
failed = [r for r in results if r.get("status") not in ("completed", "handed_off")]
|
||||
detail = "; ".join(
|
||||
f"{((r.get('subtask') or {}).get('description') or 'subtask')[:80]}: "
|
||||
f"{r.get('error') or r.get('summary') or 'no detail reported'}"
|
||||
for r in failed
|
||||
) or "subtask(s) failed without detail"
|
||||
payload["error"] = detail
|
||||
logger.error(f"Task {task_id} failed: {detail}")
|
||||
return payload
|
||||
except Exception as e:
|
||||
logger.error(f"Error executing task {task_id}: {e}")
|
||||
return {
|
||||
@@ -166,6 +179,7 @@ Return ONLY the JSON array, no other text."""
|
||||
}]
|
||||
|
||||
async def _execute_subtask(self, subtask: dict, task_id: str, context: dict, peer_collaboration_callback: Optional[Callable]) -> dict:
|
||||
content = ""
|
||||
try:
|
||||
description = subtask["description"]
|
||||
workspace_files = self._summarize_workspace()
|
||||
@@ -270,10 +284,26 @@ Return ONLY the JSON, no other text."""
|
||||
if apply_result["errors"]:
|
||||
result["status"] = "failed"
|
||||
result["error"] = "; ".join(apply_result["errors"])
|
||||
logger.warning(
|
||||
f"Subtask for task {task_id} failed applying file changes: {result['error']}"
|
||||
)
|
||||
else:
|
||||
# LLM returned 2xx but declared the subtask not completed: log its reason and a
|
||||
# bounded snippet of the model output so the failure is diagnosable from agent logs.
|
||||
logger.warning(
|
||||
"Subtask for task %s reported status=%r (error=%s summary=%s); llm_response[:500]=%s",
|
||||
task_id, result.get("status"), result.get("error"),
|
||||
result.get("summary"), (content or "").strip()[:500],
|
||||
)
|
||||
result["subtask"] = subtask
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Error executing subtask: {e}")
|
||||
# Parse failures / API errors land here; include a bounded model-output snippet
|
||||
# (model-generated text, no secrets) to explain why parsing/execution failed.
|
||||
logger.error(
|
||||
f"Error executing subtask for task {task_id}: {e}; "
|
||||
f"llm_response[:500]={(content or '').strip()[:500]}"
|
||||
)
|
||||
return {
|
||||
"subtask": subtask,
|
||||
"status": "failed",
|
||||
|
||||
Reference in New Issue
Block a user