Files
Agentswarm/agent/task_executor.py
T
gongzhiyongandClaude Opus 4.8 b394ce99d5 fix(swarm): reap orphan agents on terminal run + retry backoff + surface task errors
孤儿 agent 事故修复(2026-06-15 swarm-69e470561bd7-seed 跨 workspace 执行)。
不改去中心化认领逻辑(swarm_dispatch 不动),仅治理 agent pod 生命周期与可观测性:

- B1 终态回收: refresh_swarm_run_status 终态后 best-effort stop_launched 回收 pod+secret
  (此前仅 Manager stop 才回收,自然完成/失败的 run agent 残留 → 孤儿留在共享池抢别的 swarm 任务)
- B2 防驱逐: agent pod 加 karpenter.sh/do-not-disrupt(AGENT_POD_ALLOW_DISRUPTION=1 可关)
- stop_launched: backend=kubernetes 时按标签删,不再被内存集合 _k8s_swarms 门控(跨重启可靠)
- C 重试退避: Task.next_retry_at + fail_task 指数退避 5→30→180s(cap 300, TASK_RETRY_BACKOFF_*),
  is_task_ready 门控;TASK_MAX_RETRIES 可配
- D 错误上浮: task_executor 失败时聚合 subtask error 到顶层 error(根治通用 "Task failed"),
  _execute_subtask 打印 LLM 响应片段
- 配额硬上限 16: _clamp_user_cap

契约测试全过: runtime-contract / merge-smoke / workflow-e2e / contract-freeze /
max-agents-per-user / security-boundary。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-16 04:33:01 +08:00

549 lines
24 KiB
Python

"""Task execution engine with OpenAI-compatible LLM provider integration.
Merged from agent_swarm_v4:
- OpenAI-only provider configuration (OPENAI_*/MODEL_* env vars and custom base URLs)
- Peer-collaboration hook and specialist-role alignment prompting
- Preserves model invocation, handoff decision hooks, workspace summarization,
file-application behavior, and the Manager billing/audit usage attribution
(X-Agent/X-Agnet headers, usage payload, billing_source)
"""
import asyncio
import json
import logging
import os
import time
from pathlib import Path
from typing import Callable, Optional
from openai import AsyncOpenAI
from .handoff_logic import should_handoff
logger = logging.getLogger(__name__)
class TaskExecutor:
"""Executes tasks using a configurable OpenAI-compatible API."""
def __init__(self, agent_id: str, workspace_dir: str):
self.agent_id = agent_id
self.workspace_dir = workspace_dir
api_key = (
os.getenv("OPENAI_API_KEY")
or os.getenv("MODEL_API_KEY")
)
if not api_key:
raise ValueError("OPENAI_API_KEY environment variable not set")
self.model = (
os.getenv("OPENAI_MODEL")
or os.getenv("MODEL_NAME")
or os.getenv("MODEL_ID")
or "gpt-4o-mini"
)
self.api_base = (
os.getenv("OPENAI_API_BASE")
or os.getenv("MODEL_API_BASE")
or "https://api.openai.com/v1"
)
self.api_mode = "openai"
self.client = AsyncOpenAI(api_key=api_key, base_url=self.api_base)
self.client_type = "openai"
self.usage = self._empty_usage()
self.current_context: dict = {}
async def execute_task(
self,
task_id: str,
description: str,
context: dict,
handoff_callback: Optional[Callable] = None,
agent_capabilities: Optional[list[str]] = None,
peer_collaboration_callback: Optional[Callable] = None,
) -> dict:
logger.info(f"Executing task {task_id}: {description}")
try:
started_at = time.time()
self.current_context = context or {}
self.usage = self._empty_usage(context)
multi_agent_leaf_mode = self._multi_agent_leaf_mode(context)
allow_handoff = self._allow_dynamic_handoff(context)
if multi_agent_leaf_mode:
subtasks = [{
"description": description,
"complexity": "medium",
"estimated_time": 30,
"dependencies": [],
"required_capabilities": context.get("required_capabilities") or ["general"],
}]
elif self._subtask_handoff_enabled():
subtasks = await self._parse_task(description, context)
else:
subtasks = [{
"description": description,
"complexity": "medium",
"estimated_time": 30,
"dependencies": [],
"required_capabilities": ["general"],
}]
results = []
for subtask in subtasks:
if handoff_callback and self._subtask_handoff_enabled() and allow_handoff:
decision = should_handoff(
subtask,
self.agent_id,
agent_capabilities=agent_capabilities,
)
if decision.should_handoff:
await handoff_callback(
task_id=task_id,
subtask=subtask,
target_capabilities=decision.target_capabilities,
)
results.append({
"subtask": subtask,
"status": "handed_off",
"reason": decision.reason,
"target_capabilities": decision.target_capabilities,
})
continue
results.append(await self._execute_subtask(subtask, task_id, context, peer_collaboration_callback))
success = all(r["status"] in ["completed", "handed_off"] for r in results)
awaiting_handoff = any(r["status"] == "handed_off" for r in results)
payload = {
"success": success,
"task_id": task_id,
"subtasks": results,
"awaiting_handoff": awaiting_handoff,
"agent_id": self.agent_id,
"usage": self._usage_payload(time.time() - started_at),
}
if not success:
# Surface the real per-subtask failure detail as a top-level `error` so the
# orchestrator records WHY (agent/main.py falls back to a generic "Task failed"
# when this key is absent — see incident 2026-06-15 swarm-69e470561bd7-seed).
failed = [r for r in results if r.get("status") not in ("completed", "handed_off")]
detail = "; ".join(
f"{((r.get('subtask') or {}).get('description') or 'subtask')[:80]}: "
f"{r.get('error') or r.get('summary') or 'no detail reported'}"
for r in failed
) or "subtask(s) failed without detail"
payload["error"] = detail
logger.error(f"Task {task_id} failed: {detail}")
return payload
except Exception as e:
logger.error(f"Error executing task {task_id}: {e}")
return {
"success": False,
"task_id": task_id,
"error": str(e),
"agent_id": self.agent_id,
"usage": self._usage_payload(time.time() - started_at if "started_at" in locals() else 0),
}
finally:
self.current_context = {}
async def _parse_task(self, description: str, context: dict) -> list[dict]:
try:
prompt = f"""You are a task planning assistant. Break down the following programming task into concrete, actionable subtasks.
Task: {description}
Context: {json.dumps(context, indent=2)}
Return a JSON array of subtasks, where each subtask has:
- description: Clear description of what needs to be done
- complexity: \"low\", \"medium\", or \"high\"
- estimated_time: Estimated time in minutes
- dependencies: List of subtask indices this depends on (empty if none)
- required_capabilities: List of capabilities needed
Return ONLY the JSON array, no other text."""
content = await self._complete(prompt, max_tokens=2000)
return json.loads(self._strip_json_fence(content))
except Exception as e:
logger.error(f"Error parsing task: {e}")
return [{
"description": description,
"complexity": "medium",
"estimated_time": 30,
"dependencies": [],
"required_capabilities": ["general"],
}]
async def _execute_subtask(self, subtask: dict, task_id: str, context: dict, peer_collaboration_callback: Optional[Callable]) -> dict:
content = ""
try:
description = subtask["description"]
workspace_files = self._summarize_workspace()
workspace_context = self._collect_workspace_context()
user_prompt = context.get("user_prompt") or context.get("run_goal") or context.get("root_task_description") or ""
specialist_role = context.get("specialist_role", "general")
dependency_artifacts = context.get("dependency_artifacts") or []
implementation_artifacts = [
artifact for artifact in dependency_artifacts
if "implementation" in (artifact.get("task_id") or "")
]
testing_artifacts = [
artifact for artifact in dependency_artifacts
if "testing" in (artifact.get("task_id") or "")
]
peer_context = []
peer_agents = context.get("peer_agents") or []
should_consult_peers = specialist_role in {"testing", "documentation"}
if peer_collaboration_callback and peer_agents and should_consult_peers:
max_peer_consults = int((context or {}).get("max_peer_consults", 2) or 2)
preferred_roles = []
if specialist_role in {"testing", "documentation"}:
preferred_roles = ["implementation"]
ordered_peers = sorted(
peer_agents,
key=lambda peer: 0 if peer.get("role") in preferred_roles else 1,
)
for peer in ordered_peers[:max_peer_consults]:
try:
reply = await peer_collaboration_callback(
task_id=task_id,
target_agent_id=peer.get("agent_id"),
content=(
f"Specialist role: {specialist_role}. "
f"Please provide guidance and confirm behavior for: {description}. "
f"Original user request: {user_prompt}"
),
timeout_seconds=float((context or {}).get("peer_timeout_seconds", 10.0) or 10.0),
)
peer_context.append(
{
"agent_id": peer.get("agent_id"),
"role": peer.get("role"),
"content": reply.get("content", ""),
}
)
except Exception as exc:
logger.warning(f"Peer collaboration failed with {peer.get('agent_id')}: {exc}")
prompt = f"""You are a programming assistant working in a collaborative agent system.
Original user request:
{user_prompt}
Your specialist role:
{specialist_role}
Task: {description}
Dependency artifacts from other specialists:
{json.dumps(dependency_artifacts, indent=2)}
Implementation artifacts relevant to alignment:
{json.dumps(implementation_artifacts, indent=2)}
Testing artifacts relevant to alignment:
{json.dumps(testing_artifacts, indent=2)}
Workspace directory: {self.workspace_dir}
Current workspace files:
{json.dumps(workspace_files, indent=2)}
Relevant workspace file contents:
{json.dumps(workspace_context, indent=2)}
Peer specialist input:
{json.dumps(peer_context, indent=2)}
Alignment requirements:
- If your role is testing, align your tests with the implementation artifacts and their stated error semantics.
- If your role is documentation, align your docs with both implementation and testing artifacts.
- Do not invent behavior that conflicts with dependency artifacts unless you explicitly surface an error.
- Treat implementation artifacts as the source of truth for API behavior and exception semantics.
- If peer specialist input conflicts with implementation artifacts, prefer implementation semantics and explain the correction in your changes summary.
- If your role is documentation or testing, update only your specialist outputs to converge on implementation behavior unless the implementation artifact is clearly missing or contradictory.
Return your response as JSON with this structure:
{{
\"status\": \"completed\" or \"failed\",
\"summary\": \"Brief description of what was done\",
\"files\": [{{\"path\": \"relative/path.py\", \"action\": \"write\", \"content\": \"complete file content\"}}],
\"changes\": \"Detailed description of changes\",
\"error\": \"Error message if failed, null otherwise\"
}}
Return ONLY the JSON, no other text."""
content = await self._complete(prompt, max_tokens=4000)
result = self._parse_json_response(content)
if result.get("status") == "completed":
apply_result = await self._apply_file_changes(result.get("files", []))
result["files_modified"] = apply_result["files_modified"]
result["files_deleted"] = apply_result["files_deleted"]
if apply_result["errors"]:
result["status"] = "failed"
result["error"] = "; ".join(apply_result["errors"])
logger.warning(
f"Subtask for task {task_id} failed applying file changes: {result['error']}"
)
else:
# LLM returned 2xx but declared the subtask not completed: log its reason and a
# bounded snippet of the model output so the failure is diagnosable from agent logs.
logger.warning(
"Subtask for task %s reported status=%r (error=%s summary=%s); llm_response[:500]=%s",
task_id, result.get("status"), result.get("error"),
result.get("summary"), (content or "").strip()[:500],
)
result["subtask"] = subtask
return result
except Exception as e:
# Parse failures / API errors land here; include a bounded model-output snippet
# (model-generated text, no secrets) to explain why parsing/execution failed.
logger.error(
f"Error executing subtask for task {task_id}: {e}; "
f"llm_response[:500]={(content or '').strip()[:500]}"
)
return {
"subtask": subtask,
"status": "failed",
"error": str(e),
"summary": f"Failed to execute: {e}",
}
async def _complete(self, prompt: str, max_tokens: int) -> str:
extra_headers = self._model_attribution_headers()
response = await self.client.chat.completions.create(
model=self.model,
messages=[{"role": "user", "content": prompt}],
max_tokens=max_tokens,
extra_headers=extra_headers or None,
)
self._record_openai_usage(response)
return response.choices[0].message.content or ""
async def peer_reply(self, *, query: str, capabilities: list[str],
last_summary: Optional[str], max_tokens: Optional[int] = None) -> dict:
"""Compose a substantive, grounded reply to a peer agent's query (one bounded LLM call).
Returns {stance, content, evidence, refs}. `content` is what the requesting agent reads.
Bounded by PEER_CONSULT_MAX_TOKENS (default 500).
"""
if max_tokens is None:
max_tokens = int(os.getenv("PEER_CONSULT_MAX_TOKENS", "500"))
workspace_files = self._summarize_workspace()
workspace_context = self._collect_workspace_context(max_files=8)
prompt = f"""You are a specialist agent being consulted by a peer in a collaborative swarm.
Your capabilities: {', '.join(capabilities)}
Your latest completed work (summary): {last_summary or 'none'}
A peer asks:
{query}
Your workspace files:
{json.dumps(workspace_files, indent=2)}
Relevant workspace file contents:
{json.dumps(workspace_context, indent=2)}
Answer concisely and concretely, grounded in YOUR actual work/artifacts. Treat implementation
artifacts as the source of truth for behavior and exception semantics; if the peer's assumption
conflicts with your work, say so.
Return ONLY JSON:
{{\"stance\": \"agree|disagree|info\", \"content\": \"<concise answer to the peer>\", \"evidence\": \"<what in your work supports this>\", \"refs\": [\"relative/file/path\"]}}"""
content = await self._complete(prompt, max_tokens=max_tokens)
result = self._parse_json_response(content)
if not result.get("content"):
result["content"] = (content or "").strip()[:1000]
result.setdefault("stance", "info")
result.setdefault("evidence", "")
result.setdefault("refs", [])
return result
def _empty_usage(self, context: Optional[dict] = None) -> dict:
plan = ((context or {}).get("orchestration_plan") or {})
billing = plan.get("billing_context") or {}
return {
"model_id": self.model,
"model_tokens": 0,
"prompt_tokens": 0,
"completion_tokens": 0,
"model_cost_usd": 0.0,
"runtime_seconds": 0.0,
"billing_source": billing.get("provider") or os.getenv("BILLING_SOURCE", "unknown"),
}
def _record_openai_usage(self, response):
usage = getattr(response, "usage", None)
if not usage:
return
prompt_tokens = int(getattr(usage, "prompt_tokens", 0) or 0)
completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
total_tokens = int(getattr(usage, "total_tokens", 0) or prompt_tokens + completion_tokens)
self._add_usage(prompt_tokens, completion_tokens, total_tokens)
def _add_usage(self, prompt_tokens: int, completion_tokens: int, total_tokens: int):
self.usage["prompt_tokens"] += prompt_tokens
self.usage["completion_tokens"] += completion_tokens
self.usage["model_tokens"] += total_tokens
input_cost = float(os.getenv("MODEL_INPUT_COST_PER_1M", "0") or 0)
output_cost = float(os.getenv("MODEL_OUTPUT_COST_PER_1M", "0") or 0)
self.usage["model_cost_usd"] += (
prompt_tokens * input_cost / 1_000_000
+ completion_tokens * output_cost / 1_000_000
)
def _usage_payload(self, runtime_seconds: float) -> dict:
usage = dict(self.usage)
usage["runtime_seconds"] = runtime_seconds
return usage
def _model_attribution_headers(self) -> dict:
headers = {}
mapping = {
"manager_deployment_id": ["X-Agent-Manager-Deployment-ID", "X-Agnet-Manager-Deployment-ID"],
"swarm_id": ["X-Agent-Swarm-ID", "X-Agnet-Swarm-ID"],
"task_id": ["X-Agent-Task-ID", "X-Agnet-Task-ID"],
"agent_role": ["X-Agent-Agent-Role", "X-Agnet-Agent-Role"],
"correlation_id": ["X-Correlation-ID"],
"model_id": ["X-Agent-Model-ID", "X-Agnet-Model-ID"],
}
for key, header_names in mapping.items():
value = self.current_context.get(key)
if value:
for header in header_names:
headers[header] = str(value)
headers.setdefault("X-Agent-Model-ID", self.model)
headers.setdefault("X-Agnet-Model-ID", self.model)
return headers
def _subtask_handoff_enabled(self) -> bool:
return os.getenv("ENABLE_SUBTASK_HANDOFF", "false").lower() in {"1", "true", "yes"}
def _multi_agent_leaf_mode(self, context: Optional[dict]) -> bool:
return self._subtask_handoff_enabled() and (context or {}).get("workflow_mode") == "multi_agent"
def _allow_dynamic_handoff(self, context: Optional[dict]) -> bool:
if (context or {}).get("workflow_mode") != "multi_agent":
return True
return bool((context or {}).get("allow_handoff", False))
def _strip_json_fence(self, content: str) -> str:
content = content.strip()
if not content.startswith("```"):
return content
lines = content.split("\n")
if lines and lines[0].startswith("```"):
lines = lines[1:]
if lines and lines[-1].strip() == "```":
lines = lines[:-1]
return "\n".join(lines).strip()
def _parse_json_response(self, content: str) -> dict:
content = self._strip_json_fence(content)
try:
return json.loads(content)
except json.JSONDecodeError:
start = content.find("{")
end = content.rfind("}")
if start == -1 or end == -1 or end <= start:
raise
return json.loads(content[start:end + 1])
def _summarize_workspace(self, max_files: int = 80) -> list[str]:
root = Path(self.workspace_dir)
if not root.exists():
return []
ignored_dirs = {".git", "__pycache__", "node_modules", ".venv", "venv"}
files = []
for path in root.rglob("*"):
if len(files) >= max_files:
break
if not path.is_file():
continue
if any(part in ignored_dirs for part in path.relative_to(root).parts):
continue
files.append(str(path.relative_to(root)))
return sorted(files)
def _collect_workspace_context(self, max_files: int = 20, max_bytes_per_file: int = 6000) -> dict[str, str]:
root = Path(self.workspace_dir)
if not root.exists():
return {}
ignored_dirs = {".git", "__pycache__", "node_modules", ".venv", "venv"}
allowed_suffixes = {".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".md", ".txt", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".sh"}
context = {}
for path in sorted(root.rglob("*")):
if len(context) >= max_files:
break
if not path.is_file():
continue
relative = path.relative_to(root)
if any(part in ignored_dirs for part in relative.parts):
continue
if path.suffix and path.suffix.lower() not in allowed_suffixes:
continue
try:
data = path.read_bytes()[:max_bytes_per_file]
context[str(relative)] = data.decode("utf-8")
except UnicodeDecodeError:
continue
except Exception as e:
logger.warning(f"Failed to read workspace file {relative}: {e}")
return context
def _resolve_workspace_path(self, file_path: str) -> Path:
if not file_path or os.path.isabs(file_path):
raise ValueError(f"Invalid relative path: {file_path}")
root = Path(self.workspace_dir).resolve()
resolved = (root / file_path).resolve()
if root != resolved and root not in resolved.parents:
raise ValueError(f"Path escapes workspace: {file_path}")
return resolved
async def _apply_file_changes(self, files: list[dict]) -> dict:
result = {"files_modified": [], "files_deleted": [], "errors": []}
for file_change in files:
path = file_change.get("path")
action = file_change.get("action", "write")
try:
target = self._resolve_workspace_path(path)
if action == "write":
content = file_change.get("content")
if content is None:
raise ValueError(f"Missing content for {path}")
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content, encoding="utf-8")
result["files_modified"].append(path)
elif action == "delete":
if target.exists():
target.unlink()
result["files_deleted"].append(path)
else:
raise ValueError(f"Unsupported file action for {path}: {action}")
except Exception as e:
logger.error(f"Failed to apply file change {path}: {e}")
result["errors"].append(f"{path}: {e}")
return result
async def read_file(self, file_path: str) -> Optional[str]:
try:
full_path = os.path.join(self.workspace_dir, file_path)
with open(full_path, "r", encoding="utf-8") as f:
return f.read()
except Exception as e:
logger.error(f"Error reading file {file_path}: {e}")
return None
async def write_file(self, file_path: str, content: str) -> bool:
try:
full_path = self._resolve_workspace_path(file_path)
os.makedirs(os.path.dirname(full_path), exist_ok=True)
with open(full_path, "w", encoding="utf-8") as f:
f.write(content)
logger.info(f"Wrote file: {file_path}")
return True
except Exception as e:
logger.error(f"Error writing file {file_path}: {e}")
return False