Merge pull request #72 from xmindlab-heicode/fix/70-relanded

fix(swarm/#70): 超时透传+提默认 / 时间线降噪 / 失败reason准确(重新落到 main + 修 main 红测试)
This commit is contained in:
Fasthei
2026-06-16 18:31:24 +08:00
committed by GitHub
9 changed files with 262 additions and 15 deletions
+5 -1
View File
@@ -76,7 +76,11 @@ class Agent:
# agent's clone (see _execute_assignment / GitOperations.add_task_worktree), so concurrent tasks
# never share a checkout/index. Swarm parallelism = many agents × this per-agent concurrency.
MAX_CONCURRENT_TASKS = int(os.getenv("MAX_CONCURRENT_TASKS", "4"))
TASK_TIMEOUT_SECONDS = int(os.getenv("TASK_TIMEOUT_SECONDS", "60"))
# Per-task execution timeout. Default raised 60->300 (agent_swarm#70): 60s reliably killed
# generation-class tasks before the model finished. The Swarm launcher transmits an explicit
# TASK_TIMEOUT_SECONDS into each agent's env (capped at the run budget.duration_seconds), so
# this default only applies to externally-launched / standalone agents.
TASK_TIMEOUT_SECONDS = int(os.getenv("TASK_TIMEOUT_SECONDS", "300"))
PEER_REPLY_TIMEOUT_SECONDS = int(os.getenv("PEER_REPLY_TIMEOUT_SECONDS", "20"))
HEARTBEAT_INTERVAL_SECONDS = 15
+17 -2
View File
@@ -37,8 +37,14 @@
"occurred_at": "2026-06-08T...Z", // 事件时间
"correlation_id": "corr-...", // 追踪 ID(X-Correlation-ID)
"source": "heicode-swarm-runtime", // 事件来源标识
"metadata": { }, // 自定义元数据(不得含明文密钥)
"payload": { }, // 事件专属字段(见 §4)
"metadata": { // 自定义元数据(不得含明文密钥)
"client_visible": true // agent_swarm#70:是否客户端 cockpit 可见事件(= event_type ∈ FROZEN_CLIENT_EVENT_TYPES)。
// HM/客户端据此过滤内部噪音(heartbeat/retried/deployment.status_changed/timeline/budget),
// 无需硬编码冻结集。**仅展示层过滤标记,不改变冻结事件集/类型/sequence/artifact 形状。**
},
"payload": { // 事件专属字段(见 §4)
"message": "..." // agent_swarm#70:可选人话进度(仅客户端可见事件,缺省由运行时按 type+已有字段回填,不伪造新数据)
},
"artifact": { } // 可选;见 §3
}
```
@@ -118,4 +124,13 @@
剩余小项(不影响客户端 13 类冻结):
- `handoff.completed` 的 `from_role`/`to_role`:辅助事件,客户端 13 类用 `handoff.created`(已带 from/to);如 HM 仍消费 `handoff.completed`,在 handoff 完成处补两字段为后续小修。
### 6.1 agent_swarm#70 时间线降噪(附加,未触碰冻结契约)
回应 #70「事件时间线全是内部噪音」。**未改**冻结事件集 / 事件类型 / `sequence` / artifact 形状,仅两处**纯附加**增强(实现 `swarm_runtime.emit_event`,`is_client_visible` 为单一真源):
1. **`metadata.client_visible`(envelope,附加)**:每事件带布尔标记 = `event_type ∈ FROZEN_CLIENT_EVENT_TYPES`。客户端 cockpit 只渲染 `client_visible=true` 的事件,`task.heartbeat`/`task.retried`/`deployment.status_changed`/`timeline.updated`/`budget.alert` 等内部/调度事件 `client_visible=false`,仍持久化+回调(HM 控制面照常消费)但不入客户端时间线。这是**展示层过滤标记**,不改变运行时语义。
2. **`payload.message`(附加,可选)**:关键客户端可见事件(`task.created|claimed|running|completed|failed`、`swarm.completed|failed|stopped`)回填一句人话进度,仅取已有 payload 字段(role/title/summary/reason/termination_reason),**不伪造新数据**(规则 #9);emitter 已提供 `message` 时不覆盖。
> HM 侧无需改动即可消费(忽略 `metadata.client_visible`/`payload.message` 不影响契约);客户端可立即按 `metadata.client_visible` 过滤并展示 `payload.message`。如需把 `client_visible` 提升为 envelope 顶层字段或新增「内部事件」分类口径,属契约扩展,须与 HM 联调后另立工单(非本次范围)。
> 契约由 `scripts/test-contract-freeze.py` 守护(断言 sequence 单调、13 类 round-trip、artifact 形状、审批/停止真实发出、明文凭据脱敏);`scripts/test-workflow-e2e.py` 在全流程 e2e 中额外断言 `swarm.completed` + sequence 无空洞。
+41
View File
@@ -333,6 +333,44 @@ def resolve_git_grant(body: Dict[str, Any]) -> Optional[Dict[str, str]]:
return env
# ── Per-task execution timeout (agent_swarm#70) ────────────────────────────────
#: Default per-task timeout injected into launched agents. 60s (the old agent default) reliably
#: killed generation-class tasks before the model finished; raised to 300s. Overridable via
#: TASK_TIMEOUT_SECONDS on the orchestrator. Aligned (min) with the run's budget.duration_seconds
#: so a task can never outlive its run budget.
DEFAULT_TASK_TIMEOUT_SECONDS = 300
def _budget_duration_seconds(body: Dict[str, Any]) -> Optional[int]:
"""The run's wall-clock budget in seconds from the create body, or None when absent."""
budget = ((body.get("orchestration_plan") or {}).get("budget") or {})
raw = budget.get("duration_seconds") or budget.get("max_duration_seconds")
try:
val = int(raw)
except (TypeError, ValueError):
return None
return val if val > 0 else None
def resolve_task_timeout(body: Dict[str, Any]) -> int:
"""Per-task execution timeout (seconds) to inject into each launched agent (agent_swarm#70).
Base = ``TASK_TIMEOUT_SECONDS`` env (default ``DEFAULT_TASK_TIMEOUT_SECONDS`` = 300). When the
run declares ``budget.duration_seconds`` we take the **smaller** of the two so a single task can
never outlive the whole run budget. Always returns a positive int.
"""
try:
base = int(os.getenv("TASK_TIMEOUT_SECONDS", str(DEFAULT_TASK_TIMEOUT_SECONDS)))
except ValueError:
base = DEFAULT_TASK_TIMEOUT_SECONDS
if base <= 0:
base = DEFAULT_TASK_TIMEOUT_SECONDS
budget_duration = _budget_duration_seconds(body)
if budget_duration:
return min(base, budget_duration)
return base
def model_api_base() -> str:
return os.getenv("AGENT_OPENAI_API_BASE") or os.getenv("OPENAI_API_BASE") or "https://api.openai.com/v1"
@@ -434,6 +472,7 @@ def plan_launch_specs(
caps = pool_capabilities() or ["general"]
base = model_api_base()
mid = model_id(body)
task_timeout = resolve_task_timeout(body) # #70:透传执行超时,与 run budget.duration_seconds 取较小
specs: List[AgentLaunchSpec] = []
for i in range(count):
cap_csv = caps[i % len(caps)]
@@ -442,6 +481,8 @@ def plan_launch_specs(
"AGENT_ID": f"{run.swarm_id}-agent-{i+1}",
"AGENT_CAPABILITIES": cap_csv,
"OPENAI_API_BASE": base,
# Non-secret execution-timeout knob; inline like the other config env (never a Secret).
"TASK_TIMEOUT_SECONDS": str(task_timeout),
}
if model_key:
env["OPENAI_API_KEY"] = model_key
+67 -7
View File
@@ -56,6 +56,9 @@ class TerminationReason(str, Enum):
BUDGET_EXHAUSTED = "budget_exhausted" # token / cost / duration budget spent
MAX_ROUNDS_REACHED = "max_rounds_reached" # review/redo cycle cap hit
RISK_BLOCKED = "risk_blocked" # an unresolved blocking risk stopped the run
TIMEOUT = "timeout" # a task exhausted its execution timeout
MAX_RETRIES_EXCEEDED = "max_retries_exceeded" # a task failed after its retry budget
TASK_FAILED = "task_failed" # a task failed for another (non-timeout) reason
TASKS_COMPLETED = "tasks_completed" # fallback: all tasks terminal, nothing else to explain
@@ -180,6 +183,55 @@ def _task_result(task: Any) -> Dict[str, Any]:
return {}
def _task_field(task: Any, key: str) -> Any:
"""Read an arbitrary field off a task (dict or model)."""
if isinstance(task, dict):
return task.get(key)
return getattr(task, key, None)
def _is_timeout_failure(task: Any) -> bool:
"""Whether a failed task's recorded reason indicates an execution timeout.
The agent reports a timeout as ``result={"error": "timeout", ...}`` (agent/main.py); the
orchestrator persists that reason on the failed task. We match the canonical "timeout" token
(case-insensitive) in the task's result ``error``/``reason`` so we never guess.
"""
result = _task_result(task)
text = " ".join(
str(result.get(k) or "") for k in ("error", "reason", "summary")
).lower()
return "timeout" in text
def _is_max_retries_failure(task: Any) -> bool:
"""Whether a failed task exhausted its retry budget (retry_count >= max_retries)."""
try:
retry_count = int(_task_field(task, "retry_count") or 0)
max_retries = int(_task_field(task, "max_retries") or 0)
except (TypeError, ValueError):
return False
return max_retries > 0 and retry_count >= max_retries
def classify_failure_reason(tasks: List[Any]) -> TerminationReason:
"""Pick the most-specific termination reason for a FAILED run from its failed tasks.
Precedence (most specific first), derived only from real per-task signals (never fabricated):
1. ``timeout`` — any failed task whose recorded error is a timeout.
2. ``max_retries_exceeded`` — any failed task that exhausted its retry budget.
3. ``task_failed`` — a task failed for another reason (the honest generic fail).
Returns ``TASK_FAILED`` when no failed task is present (caller should only invoke this on a
FAILED run, but we stay safe).
"""
failed = [t for t in tasks if _task_status(t) in _TASK_TERMINAL_FAIL]
if any(_is_timeout_failure(t) for t in failed):
return TerminationReason.TIMEOUT
if any(_is_max_retries_failure(t) for t in failed):
return TerminationReason.MAX_RETRIES_EXCEEDED
return TerminationReason.TASK_FAILED
# --- conflict detectors ---------------------------------------------------
def detect_artifact_mismatch(tasks: List[Any]) -> List[Conflict]:
@@ -489,9 +541,13 @@ def evaluate_convergence(run_state: Dict[str, Any]) -> ConvergenceReport:
Decision order (first match wins) for terminal runs:
1. Any active/blocked task -> RUNNING (no termination_reason yet)
2. Any blocking risk / unresolved hard conflict -> BLOCKED, RISK_BLOCKED
3. Any failed task -> FAILED, BUDGET_EXHAUSTED if budget
spent else MAX_ROUNDS_REACHED if the
redo cap was hit else TASKS_COMPLETED
3. Any failed task -> FAILED, with the most accurate reason: TIMEOUT if a
failed task timed out, else MAX_RETRIES_EXCEEDED if a
failed task exhausted its retry budget, else
BUDGET_EXHAUSTED if budget spent, else
MAX_ROUNDS_REACHED if the redo cap was hit, else
TASK_FAILED (never TASKS_COMPLETED — that is a
success-only reason; see agent_swarm#70)
4. Budget exhausted (all tasks done) -> CONVERGED, BUDGET_EXHAUSTED
5. Review cap hit -> CONVERGED, MAX_ROUNDS_REACHED
6. Quality gate reached -> CONVERGED, QUALITY_REACHED
@@ -541,14 +597,18 @@ def evaluate_convergence(run_state: Dict[str, Any]) -> ConvergenceReport:
max_cycles = int(run_state.get("max_review_cycles", 0) or 0)
rounds_hit = max_cycles > 0 and cycles >= max_cycles
# 3. a failed task -> failed run, with the best available explanation.
# 3. a failed task -> failed run, with the best available explanation. A FAILED run must NEVER
# report `tasks_completed` (that reason is success-only and was the agent_swarm#70 bug):
# classify the failure (timeout / max_retries_exceeded / task_failed) and let a more specific
# resource cause (budget / rounds) win only when no concrete failure signal is present.
if any(s in _TASK_TERMINAL_FAIL for s in statuses):
if budget_state.get("exhausted"):
failure_reason = classify_failure_reason(tasks)
if failure_reason is TerminationReason.TASK_FAILED and budget_state.get("exhausted"):
reason = TerminationReason.BUDGET_EXHAUSTED
elif rounds_hit:
elif failure_reason is TerminationReason.TASK_FAILED and rounds_hit:
reason = TerminationReason.MAX_ROUNDS_REACHED
else:
reason = TerminationReason.TASKS_COMPLETED
reason = failure_reason
return _report(ConvergenceStatus.FAILED, reason)
# All tasks completed. Explain WHY it stopped, most-specific reason first.
+3
View File
@@ -452,6 +452,9 @@ async def compute_convergence_report(run, tasks, *, terminal: bool):
"status": t.status.value if hasattr(t.status, "value") else t.status,
"depends_on": t.depends_on,
"result": result,
# #70: retry counters let convergence classify a FAILED run as max_retries_exceeded.
"retry_count": t.retry_count,
"max_retries": t.max_retries,
})
plan = (run.request_body or {}).get("orchestration_plan") or {}
budget = plan.get("budget") or {}
+55
View File
@@ -111,6 +111,47 @@ FROZEN_CLIENT_EVENT_TYPES = (
# final state and the SSE stream can close — the client cockpit keys its terminal UI on these.
TERMINAL_CLIENT_EVENT_TYPES = frozenset({"swarm.completed", "swarm.failed", "swarm.stopped"})
_FROZEN_CLIENT_EVENT_SET = frozenset(FROZEN_CLIENT_EVENT_TYPES)
def is_client_visible(event_type: str) -> bool:
"""Whether an event type is part of the frozen client cockpit contract (vs internal noise).
The runtime stores/forwards many non-client event types (task.heartbeat, task.retried,
deployment.status_changed, timeline.updated, budget.alert, …). Only FROZEN_CLIENT_EVENT_TYPES
are rendered on the client timeline. This is the single source of truth both the runtime
(envelope `metadata.client_visible`, agent_swarm#70) and HM/clients filter on — no event is
removed, and the frozen type set is untouched.
"""
return event_type in _FROZEN_CLIENT_EVENT_SET
#: Human-readable message templates for key client-visible events (agent_swarm#70). Purely
#: additive: an optional `message` is set on the payload only when the emitter did not already
#: provide one. Templates pull from already-present payload fields, so no new data is fabricated.
def _client_message(event_type: str, payload: Dict[str, Any]) -> Optional[str]:
role = payload.get("agent_role")
title = payload.get("title") or payload.get("summary")
if event_type == "task.created":
return f"New task created: {title}" if title else "New task created"
if event_type == "task.claimed":
return f"{role} picked up a task" if role else "An agent picked up a task"
if event_type == "task.running":
return f"{role} is working on a task" if role else "An agent is working on a task"
if event_type == "task.completed":
return "Task completed"
if event_type == "task.failed":
reason = payload.get("reason")
return f"Task failed: {reason}" if reason else "Task failed"
if event_type == "swarm.completed":
return "Swarm run completed"
if event_type == "swarm.failed":
tr = payload.get("termination_reason")
return f"Swarm run failed ({tr})" if tr else "Swarm run failed"
if event_type == "swarm.stopped":
return "Swarm run stopped"
return None
class SwarmRuntime:
"""Stores swarm runs and emits Agent Manager callback events."""
@@ -651,6 +692,15 @@ class SwarmRuntime:
if isinstance(threshold, (int, float)):
redacted_payload["threshold_pct"] = threshold * 100 if threshold <= 1 else threshold
# Natural-language progress (agent_swarm#70): give client-visible events a human `message`
# so the cockpit timeline reads as progress instead of raw event types. Additive + optional;
# only set when the emitter did not already provide one. Internal/noise events get none.
client_visible = is_client_visible(event_type)
if client_visible and "message" not in redacted_payload:
message = _client_message(event_type, redacted_payload)
if message:
redacted_payload["message"] = message
event_id = f"evt_{uuid.uuid4().hex}"
# Per-swarm strictly-increasing sequence (agent_swarm#15.1): the client polls
# `events?after=<sequence>` and dedups/orders by it. INCR is atomic so concurrent emits on
@@ -669,6 +719,11 @@ class SwarmRuntime:
"occurred_at": occurred_at,
"correlation_id": run.correlation_id,
"source": self.runtime_source,
# client_visible (agent_swarm#70): explicit, type-derived flag so HM/clients can filter
# internal noise (heartbeat/retried/deployment.status_changed/timeline/budget) off the
# cockpit timeline without hard-coding the frozen set. Additive metadata field — does not
# touch the frozen envelope keys, event types, sequence, or artifact shape.
"metadata": {"client_visible": client_visible},
"payload": redacted_payload,
}
if redacted_artifact:
+7
View File
@@ -362,6 +362,13 @@ class TaskQueue:
else:
task.status = TaskStatus.FAILED
task.completed_at = time.time()
# Persist the terminal failure reason on the task so downstream convergence (#70) can
# derive an accurate termination_reason (timeout / max_retries_exceeded / task_failed).
# Stored in the result envelope, mirroring the agent's failure payload shape
# ({"success": false, "error": ...}); does not overwrite a structured result if one
# already captured the cause.
if reason and not task.result:
task.result = json.dumps({"success": False, "error": reason})
logger.error(
f"Task {task_id} permanently failed after {task.retry_count} retries: {reason}"
+29 -2
View File
@@ -96,7 +96,8 @@ def test_plan_specs():
specs = al.plan_launch_specs(run, body, connected_user_agents=8, limit=10, pool_size=3,
model_key="sk-test", orchestrator_url="ws://orch:8000", user_id="u-1",
git_env=git_env)
check("plan caps at limit (8 connected, cap 10 -> launch 2)", len(specs) == 2)
# [3,16] pod-count window (#66): MIN floor (3) wins over cap headroom (10-8=2) -> launch 3.
check("plan floors to MIN pool (8 connected, cap 10, headroom 2 -> floor 3)", len(specs) == 3)
s = specs[0]
check("spec env has ORCHESTRATOR_URL", s.env.get("ORCHESTRATOR_URL") == "ws://orch:8000")
check("spec env has model key (server-side injected)", s.env.get("OPENAI_API_KEY") == "sk-test")
@@ -106,6 +107,8 @@ def test_plan_specs():
check("spec env has GIT_USERNAME + GIT_PASSWORD", s.env.get("GIT_USERNAME") == "x-access-token" and s.env.get("GIT_PASSWORD") == "ghp_xyz")
check("spec has AGENT_ID + capabilities", bool(s.agent_id) and bool(s.env.get("AGENT_CAPABILITIES")))
check("agent ids unique", len({sp.agent_id for sp in specs}) == len(specs))
# #70: per-task timeout is transmitted into every launched agent's env (non-secret, inline).
check("spec env has TASK_TIMEOUT_SECONDS (transmitted, #70)", s.env.get("TASK_TIMEOUT_SECONDS") == str(al.DEFAULT_TASK_TIMEOUT_SECONDS))
# No key -> OPENAI_API_KEY omitted (not fabricated), no user -> HEICODE_USER_ID omitted, no git -> GIT_* omitted.
specs2 = al.plan_launch_specs(run, body, connected_user_agents=0, limit=10, pool_size=1,
model_key=None, orchestrator_url="ws://orch", user_id=None, git_env=None)
@@ -114,6 +117,30 @@ def test_plan_specs():
check("no git grant -> GIT_REPO_URL omitted", "GIT_REPO_URL" not in specs2[0].env)
def test_resolve_task_timeout():
# #70: per-task execution timeout transmitted to launched agents.
os.environ.pop("TASK_TIMEOUT_SECONDS", None)
check("default timeout = 300 (raised from 60)", al.resolve_task_timeout({}) == al.DEFAULT_TASK_TIMEOUT_SECONDS == 300)
# env override.
os.environ["TASK_TIMEOUT_SECONDS"] = "450"
check("env TASK_TIMEOUT_SECONDS honored", al.resolve_task_timeout({}) == 450)
# capped at run budget.duration_seconds (take the smaller).
body = {"orchestration_plan": {"budget": {"duration_seconds": 120}}}
check("timeout capped at budget.duration_seconds (smaller wins)", al.resolve_task_timeout(body) == 120)
body_big = {"orchestration_plan": {"budget": {"duration_seconds": 3600}}}
check("budget larger than base -> base wins", al.resolve_task_timeout(body_big) == 450)
os.environ.pop("TASK_TIMEOUT_SECONDS")
# max_duration_seconds alias also honored.
check("max_duration_seconds alias honored",
al.resolve_task_timeout({"orchestration_plan": {"budget": {"max_duration_seconds": 90}}}) == 90)
# bad/zero env -> falls back to default; absent budget -> default.
os.environ["TASK_TIMEOUT_SECONDS"] = "0"
check("non-positive env -> default", al.resolve_task_timeout({}) == al.DEFAULT_TASK_TIMEOUT_SECONDS)
os.environ["TASK_TIMEOUT_SECONDS"] = "notanint"
check("non-int env -> default", al.resolve_task_timeout({}) == al.DEFAULT_TASK_TIMEOUT_SECONDS)
os.environ.pop("TASK_TIMEOUT_SECONDS")
def test_resolve_git_grant():
# ── credential extraction (pure) ──
check("git creds: JSON git_username/git_password",
@@ -271,7 +298,7 @@ def main():
test_launch_count()
test_pool_window()
test_plan_specs()
test_git_launch_env()
test_resolve_task_timeout()
test_resolve_model_key()
test_resolve_git_grant()
test_azkv_resolver()
+38 -3
View File
@@ -151,14 +151,42 @@ terminal_cases = {
"expect_status": ConvergenceStatus.BLOCKED,
"expect_reason": TerminationReason.RISK_BLOCKED,
},
# a failed task -> FAILED, fallback reason when nothing else explains it
"failed_fallback": {
# a failed task with no specific signal -> FAILED, task_failed (NOT tasks_completed; #70 bug).
"failed_generic": {
"tasks": [
completed_task("t1", tests_passed=True),
{"task_id": "t2", "status": "failed", "depends_on": [], "result": {}},
],
"expect_status": ConvergenceStatus.FAILED,
"expect_reason": TerminationReason.TASKS_COMPLETED,
"expect_reason": TerminationReason.TASK_FAILED,
},
# a failed task whose recorded error is a timeout -> FAILED, timeout (#70).
"failed_timeout": {
"tasks": [
{"task_id": "t2", "status": "failed", "depends_on": [],
"result": {"success": False, "error": "timeout"}},
],
"expect_status": ConvergenceStatus.FAILED,
"expect_reason": TerminationReason.TIMEOUT,
},
# a failed task that exhausted its retry budget -> FAILED, max_retries_exceeded (#70).
"failed_max_retries": {
"tasks": [
{"task_id": "t2", "status": "failed", "depends_on": [], "result": {},
"retry_count": 3, "max_retries": 3},
],
"expect_status": ConvergenceStatus.FAILED,
"expect_reason": TerminationReason.MAX_RETRIES_EXCEEDED,
},
# timeout takes precedence over max_retries when both are present (#70 precedence).
"failed_timeout_over_retries": {
"tasks": [
{"task_id": "t2", "status": "failed", "depends_on": [],
"result": {"success": False, "error": "timeout"},
"retry_count": 3, "max_retries": 3},
],
"expect_status": ConvergenceStatus.FAILED,
"expect_reason": TerminationReason.TIMEOUT,
},
}
@@ -179,6 +207,13 @@ for label, case in terminal_cases.items():
check(f"[{label}] invariant: terminal -> reason set",
rep.termination_reason is not None)
# #70 invariant: a FAILED run must NEVER report tasks_completed (success-only reason).
for label, case in terminal_cases.items():
rep = evaluate_convergence(case)
if rep.status == ConvergenceStatus.FAILED:
check(f"[{label}] FAILED run never reports tasks_completed (#70)",
rep.termination_reason != TerminationReason.TASKS_COMPLETED)
# A still-running run must NOT carry a termination_reason.
running = evaluate_convergence({"tasks": [{"task_id": "x", "status": "in_progress", "depends_on": []}]})
check("running run is RUNNING", running.status == ConvergenceStatus.RUNNING)