xiaohei/Agentswarm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(#56): surface why a swarm run produces nothing (no silent black box)

Browse Source 
#56: a created run only seeds the task; if no expert agent connects, the run hangs
at `running` forever with no `agent.*` events and an empty /result — and the cause
(P-guard already detects NO_AGENTS_CONNECTED) was only in run.metadata["health"] /
/diagnostics, never on the surfaces HM/cockpit actually poll (/events, /result).

This makes a stuck run explain itself (no new Manager event type, no contract change):

- launch_swarm_agents now records run.metadata["agent_launch"]
  {backend, planned, launched, launched_ids, model_key_resolved, note}. The note
  pinpoints WHY there are 0 agents — e.g. AGENT_LAUNCH_BACKEND=none (no auto-launch),
  k8s launch failed (kubectl/RBAC + Pod Workload Identity, #16/#60 A.3), or the model
  key didn't resolve. No secret recorded — only a model_key_resolved bool.
- /result and /diagnostics now carry `health` (P-guard blockers) + `agent_launch`.
- assess_swarm_health emits ONE `timeline.updated` per distinct blocker-set (registered
  event; dedup by summary, reset when healthy) so /events and the cockpit (#39) show
  "swarm blocked: no_agents_connected" instead of silence. Still NO unregistered
  swarm.health event.

Scope: this surfaces the diagnosis. Actually executing a run still requires the
deployment to set AGENT_LAUNCH_BACKEND=kubernetes AND the #16/#60 A.3 Pod Workload
Identity / KV grant (infra, cross-team) — called out in the launch note.

Tests (scripts/test-swarm-guard.py): blocked run emits timeline.updated exactly once
(dedup) + still no swarm.health; /result + /diagnostics carry health w/ no_agents_connected;
launch_swarm_agents records the backend=none note. test-contract-freeze + test-agent-launcher
still green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
gongzhiyong
2026-06-14 16:52:44 +08:00
co-authored by Claude Opus 4.8
parent 9852df8cb4
commit 9aadc802df
2 changed files with 135 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
orchestrator/main.py
+77 -7
View File
@@ -467,16 +467,27 @@ async def assess_swarm_health(run, *, connected_agent_ids=None):
"connected_agent_caps": caps,
"budget_state": budget_state,
})
# INTERNAL state only. We deliberately do NOT route this through swarm_runtime.emit_event:
# `swarm.health` is not a registered Manager event, and emit_event can forward to a
# subscribe-all Manager callback (agent_callback.go). Stored on the run (+ a bounded log) so an
# operator/Manager can pull it; registering a Manager-facing health event is a separate contract
# change (event-schema.md).
# Stored on the run (+ a bounded log) so an operator/Manager can pull it via /diagnostics and
# /result. We deliberately do NOT introduce an unregistered `swarm.health` Manager event; instead
# the stall is surfaced ONCE per distinct blocker-set on the registered `timeline.updated` event
# (below), so a consumer watching /events and the cockpit (#39) sees *why* a run is stuck instead
# of silence — addressing agent_swarm#56.
report_dict = report.to_dict()
run.metadata["health"] = report_dict
if not report.healthy:
run.metadata["health_log"] = (run.metadata.get("health_log", [])[-49:] + [report_dict])
# Dedup by the report summary: emit on the first tick a (new) blocker-set appears; reset when the
# run becomes healthy again so a later re-block re-notifies. No raw content, only blocker reasons.
notify_sig = "" if report.healthy else report.summary
do_emit = bool(notify_sig) and run.metadata.get("health_notified_sig") != notify_sig
run.metadata["health_notified_sig"] = notify_sig
await swarm_runtime.save_run(run)
if do_emit:
await swarm_runtime.emit_event(run, "timeline.updated", payload={
"summary": report.summary,
"title": report.summary,
"blockers": [b["reason"] for b in report.blockers],
})
return report
@@ -1641,6 +1652,34 @@ async def finalize_parent_after_child(run, child_task, agent_id: str, success: b
)
def _launch_note(backend: str, planned: int, launched: int, model_key_resolved: bool) -> str:
"""Human-readable explanation of a launch outcome (agent_swarm#56).
Recorded on the run + surfaced in /result and /diagnostics so an operator immediately sees WHY
a run has no expert agents (the #56 black box), instead of a silent run stuck at `running`.
"""
if backend == "none":
return ("AGENT_LAUNCH_BACKEND=none — the runtime did NOT auto-launch agents. Either supply "
"agents externally, or set AGENT_LAUNCH_BACKEND=kubernetes (prod) / subprocess (dev). "
"With no agent connected the seeded task is never claimed (see health.blockers).")
if planned == 0:
return ("0 agents planned — the per-user cap (MAX_AGENTS_PER_USER) is already met by connected "
"agents, or AGENT_LAUNCH_POOL_SIZE is 0.")
if launched == 0:
note = (f"backend={backend}: planned {planned} but launched 0 — the launch backend failed "
f"(see orchestrator logs; for kubernetes verify kubectl/RBAC + Pod Workload Identity, "
f"agent_swarm#16 / #60 A.3).")
if not model_key_resolved:
note += (" Also: model key did NOT resolve from billing_context.secret_ref — launched "
"agents would start keyless (#60 A.3).")
return note
note = f"backend={backend}: launched {launched}/{planned} agent(s)."
if not model_key_resolved:
note += (" WARNING: model key did NOT resolve from billing_context.secret_ref — agents start "
"keyless and will error on model calls (#60 A.3).")
return note
async def launch_swarm_agents(run, body: Dict[str, Any]) -> None:
"""Swarm-owned agent launch (agent_swarm#16): launch the per-user expert pool for this run,
capped at MAX_AGENTS_PER_USER, with the model key resolved server-side from
@@ -1648,27 +1687,49 @@ async def launch_swarm_agents(run, body: Dict[str, Any]) -> None:
Manager-provided explicit agent breakdowns are honored as-is (those agents are provisioned by
the caller), so we only auto-launch the pool for the decentralized seed flow.
Records the launch outcome on run.metadata["agent_launch"] (backend / planned / launched /
model_key_resolved / note) — surfaced in /result + /diagnostics so a run with 0 expert agents
explains itself instead of hanging silently (agent_swarm#56). The note carries no secret — only
a bool for whether the model key resolved.
"""
if _manager_provided_agents(body):
run.metadata["agent_launch"] = {
"backend": "manager_provided", "planned": 0, "launched": 0, "launched_ids": [],
"note": "Manager provided explicit agents; the runtime did not auto-launch a pool.",
}
await swarm_runtime.save_run(run)
return
user_id = ((body.get("metadata") or {}).get("runtime_headers") or {}).get("x_user_id")
connected = manager.user_agent_count(user_id) if user_id else 0
backend = agent_launcher.launch_backend()
model_key = agent_launcher.resolve_model_key(body)
info: Dict[str, Any] = {
"backend": backend, "planned": 0, "launched": 0, "launched_ids": [],
"model_key_resolved": bool(model_key), "note": "",
}
try:
specs = agent_launcher.plan_launch_specs(
run, body,
connected_user_agents=connected,
limit=max_agents_per_user(),
pool_size=agent_launcher.desired_pool_size(),
model_key=agent_launcher.resolve_model_key(body),
model_key=model_key,
orchestrator_url=agent_launcher.orchestrator_ws_url(),
user_id=user_id,
)
info["planned"] = len(specs)
launched = await agent_launcher.launch(specs, swarm_id=run.swarm_id)
info["launched"] = len(launched)
info["launched_ids"] = launched
if launched:
run.metadata["launched_agents"] = launched
await swarm_runtime.save_run(run)
info["note"] = _launch_note(backend, len(specs), len(launched), bool(model_key))
except Exception as exc: # never fail run creation on launch
logger.warning("launch_swarm_agents failed for run %s: %s", run.swarm_id, exc)
info["note"] = f"launch raised: {exc}"
run.metadata["agent_launch"] = info
await swarm_runtime.save_run(run)
async def create_swarm_run_from_request(
@@ -2031,6 +2092,10 @@ async def build_runtime_diagnostics(run) -> Dict[str, Any]:
"attempts": callback_attempts,
},
"approvals": list(run.approvals.values()),
# P-guard health + agent-launch outcome: why a run can't make progress / has no expert agents
# (agent_swarm#56). `agent_launch` carries no secret (only a model_key_resolved bool).
"health": run.metadata.get("health"),
"agent_launch": run.metadata.get("agent_launch"),
"error_context": {
"task_failures": failures,
"callback_failures": [item for item in callback_attempts if item.get("status") == "failed"],
@@ -2326,6 +2391,11 @@ async def build_run_result(run) -> Dict[str, Any]:
"termination_reason": termination_reason,
"deliverable": build_run_deliverable(run, tasks),
"artifacts": collect_run_artifacts(run, tasks),
# Why a run produced nothing, surfaced on the result itself (agent_swarm#56): the P-guard
# health report (e.g. no_agents_connected) + the agent-launch outcome (backend/launched/note).
# Both are non-content descriptors; `agent_launch` carries no secret (only a resolved bool).
"health": run.metadata.get("health"),
"agent_launch": run.metadata.get("agent_launch"),
}
scripts/test-swarm-guard.py
+58
View File
@@ -134,9 +134,67 @@ async def test_wrapper():
sr_mod.SwarmRuntime.emit_event = orig_emit
# ---- #56: a stuck run explains itself (events + /result + launch note) ----
async def test_surface_56():
await redis_client.connect()
async def _noop_cb(self, *a, **k):
return None
sr_mod.SwarmRuntime._post_callback = _noop_cb
emitted = []
orig_emit = sr_mod.SwarmRuntime.emit_event
async def spy_emit(self, run, event_type, **k):
emitted.append(event_type)
return await orig_emit(self, run, event_type, **k)
sr_mod.SwarmRuntime.emit_event = spy_emit
body = {"mode": "swarm", "requirement": {"objective": "surface test"},
"callback": {"url": "http://127.0.0.1:9/cb", "subscribed_events": []},
"metadata": {"manager_deployment_id": "m-surface"}}
run, _ = await swarm_runtime.get_or_create_run(body=body, idempotency_key=None, correlation_id="cs")
seed = await task_queue.create_task(task_id=f"{run.swarm_id}-seed", description="x",
agent_role="general", required_capabilities=[], enqueue=True)
await swarm_runtime.attach_task(run, seed.task_id)
# pending seed + no agents → NO_AGENTS_CONNECTED; first tick emits timeline.updated, second dedups.
await orch.assess_swarm_health(run, connected_agent_ids=[])
await orch.assess_swarm_health(run, connected_agent_ids=[])
check("#56 blocked run emits timeline.updated exactly once (dedup)",
emitted.count("timeline.updated") == 1)
check("#56 still no unregistered swarm.health event", "swarm.health" not in emitted)
refreshed = await swarm_runtime.get_run(run.swarm_id)
result = await orch.build_run_result(refreshed)
health = result.get("health") or {}
check("#56 /result carries health (unhealthy)", health.get("healthy") is False)
check("#56 /result health names no_agents_connected",
any(b["reason"] == guard.Blocker.NO_AGENTS_CONNECTED.value for b in health.get("blockers", [])))
diag = await orch.build_runtime_diagnostics(refreshed)
check("#56 /diagnostics carries health", bool(diag.get("health")))
sr_mod.SwarmRuntime.emit_event = orig_emit
async def test_launch_outcome_56():
await redis_client.connect()
os.environ.pop("AGENT_LAUNCH_BACKEND", None) # default 'none'
body = {"mode": "swarm", "requirement": {"objective": "launch note"},
"callback": {"url": "http://127.0.0.1:9/cb", "subscribed_events": []},
"metadata": {"manager_deployment_id": "m-launch"}}
run, _ = await swarm_runtime.get_or_create_run(body=body, idempotency_key=None, correlation_id="cl")
await orch.launch_swarm_agents(run, body)
info = (await swarm_runtime.get_run(run.swarm_id)).metadata.get("agent_launch") or {}
check("#56 agent_launch recorded (backend none)", info.get("backend") == "none")
check("#56 agent_launch launched 0", info.get("launched") == 0)
check("#56 agent_launch note explains backend=none", "AGENT_LAUNCH_BACKEND=none" in (info.get("note") or ""))
async def main():
test_pure()
await test_wrapper()
await test_surface_56()
await test_launch_outcome_56()
print()
if failures:
print(f"{len(failures)} guard check(s) FAILED: {failures}")