gongzhiyongandClaude Opus 4.8
28a6c618ab
修正 P_rework 已知低估:此前 ReworkCount 仅由 retry_count>0 派生,漏了 cross_review/queen 质量门的重开(reopen_task 不增 retry_count——是质量决策非失败)。 - cross_review + queen_quality_gate 重开时累计 run.metadata["rework_reopens"] - run_collector:rework_count = retry 派生 + rework_reopens 测试 test-benchmark-collector/metrics 通过。影响:仅 benchmark 度量(P_rework 更准),不改运行路径。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
208 lines
10 KiB
Python
208 lines
10 KiB
Python
"""SwarmRunMetricsCollector — compute SwarmMetrics from a real swarm run.
|
|
|
|
Wires benchmark.metrics formulas to a run's tasks + event stream (orchestrator state).
|
|
Only metrics with real data sources are computed; the rest are returned as NaN and flagged
|
|
in `coverage` (False) — we do NOT fake a 0/100 score for uncollected metrics (rule #9).
|
|
|
|
Collected when the run exercises the relevant path (else NaN + coverage False):
|
|
- completion, collaboration, robustness → always, from tasks + handoff events
|
|
- cost → when the plan has a budget and tasks report usage
|
|
- governance → when the run had governed ops / approvals
|
|
- communication → when peers exchanged messages (request→reply rate, internal telemetry)
|
|
- reward → when the run was graded against a fixture (Q_quality real) + speed/cost present
|
|
- tau/eta/p_decision → when ENABLE_ACO_DISPATCH sampled assignments (recorded decisions)
|
|
|
|
Uncollected today (need work flagged in docs/benchmark/swarm-metrics-schema.md):
|
|
- gain → needs baselines (emergence-evaluation)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
from collections import Counter
|
|
|
|
from . import SwarmMetricsCollector
|
|
from ..metrics import (
|
|
SwarmMetrics, completion_score, collaboration_score, communication_score, cost_score,
|
|
robustness_score, governance_score, reward, speed_score, rework_penalty,
|
|
)
|
|
|
|
|
|
def _task_cost(task) -> float:
|
|
"""Extract model_cost_usd from a task's stored result, 0.0 if absent."""
|
|
result = getattr(task, "result", None)
|
|
if not result:
|
|
return 0.0
|
|
try:
|
|
data = json.loads(result) if isinstance(result, str) else result
|
|
return float((data.get("usage") or {}).get("model_cost_usd") or 0.0)
|
|
except Exception:
|
|
return 0.0
|
|
|
|
|
|
class SwarmRunMetricsCollector(SwarmMetricsCollector):
|
|
"""Collect SwarmMetrics for one swarm run (by swarm_id)."""
|
|
|
|
def __init__(self, swarm_id: str):
|
|
self.swarm_id = swarm_id
|
|
self.coverage: dict[str, bool] = {}
|
|
|
|
async def collect(self) -> SwarmMetrics:
|
|
# Lazy import: the collector reads live orchestrator state.
|
|
from orchestrator.swarm_runtime import swarm_runtime
|
|
from orchestrator.task_queue import task_queue, TaskStatus
|
|
|
|
run = await swarm_runtime.get_run(self.swarm_id)
|
|
if not run:
|
|
raise ValueError(f"run not found: {self.swarm_id}")
|
|
|
|
tasks = [t for t in [await task_queue.get_task(tid) for tid in run.task_ids] if t]
|
|
events = (await swarm_runtime.list_events(self.swarm_id, limit=10000)).get("events", [])
|
|
event_types = Counter(e.get("event_type") for e in events)
|
|
|
|
def status(t):
|
|
return t.status.value if hasattr(t.status, "value") else t.status
|
|
|
|
# --- s_completion (real) ---
|
|
total = len(tasks)
|
|
completed = sum(1 for t in tasks if status(t) == TaskStatus.COMPLETED.value)
|
|
s_completion = completion_score(completed, total)
|
|
self.coverage["s_completion"] = total > 0
|
|
|
|
# --- s_collaboration (real) ---
|
|
req = event_types.get("handoff.requested", 0)
|
|
comp = event_types.get("handoff.completed", 0)
|
|
handoff_success = (100.0 * comp / req) if req else 100.0
|
|
dep_tasks = [t for t in tasks if t.depends_on]
|
|
done_ids = {t.task_id for t in tasks if status(t) == TaskStatus.COMPLETED.value}
|
|
resolved = [t for t in dep_tasks if all(d in done_ids for d in t.depends_on)]
|
|
dep_resolution = (100.0 * len(resolved) / len(dep_tasks)) if dep_tasks else 100.0
|
|
per_agent = Counter(t.assigned_agent_id for t in tasks if t.assigned_agent_id)
|
|
if per_agent:
|
|
counts = list(per_agent.values())
|
|
workload_balance = 100.0 * (min(counts) / max(counts))
|
|
else:
|
|
workload_balance = 100.0
|
|
s_collaboration = collaboration_score(handoff_success, dep_resolution, workload_balance)
|
|
self.coverage["s_collaboration"] = total > 0
|
|
|
|
# --- s_robustness (real) ---
|
|
failures = [t for t in tasks if t.retry_count > 0 or status(t) == TaskStatus.FAILED.value]
|
|
recovered = [t for t in failures if status(t) == TaskStatus.COMPLETED.value]
|
|
s_robustness = robustness_score(len(recovered), len(failures))
|
|
self.coverage["s_robustness"] = True
|
|
|
|
# --- s_cost (real if budget + usage present) ---
|
|
plan = (run.request_body or {}).get("orchestration_plan") or {}
|
|
budget = plan.get("budget") or {}
|
|
max_cost = budget.get("max_cost_usd")
|
|
actual = sum(_task_cost(t) for t in tasks)
|
|
if isinstance(max_cost, (int, float)) and actual > 0:
|
|
s_cost = cost_score(float(max_cost), actual)
|
|
self.coverage["s_cost"] = True
|
|
else:
|
|
s_cost = math.nan
|
|
self.coverage["s_cost"] = False
|
|
|
|
# --- s_governance (real only if the run had governed ops / approvals) ---
|
|
approvals = list((run.approvals or {}).values())
|
|
if approvals:
|
|
compliant = sum(1 for a in approvals if a.get("decision") in ("approved", "rejected"))
|
|
s_governance = governance_score(compliant, len(approvals))
|
|
self.coverage["s_governance"] = True
|
|
else:
|
|
s_governance = math.nan
|
|
self.coverage["s_governance"] = False
|
|
|
|
# --- s_communication (real only if the run exchanged peer messages) ---
|
|
# successful = peer requests that received a matching reply (by correlation_id);
|
|
# total = distinct peer requests routed. A run with no peer collaboration → NaN.
|
|
collab = run.collaboration or {}
|
|
requests = set(collab.get("request_correlations") or [])
|
|
replies = set(collab.get("reply_correlations") or [])
|
|
if requests:
|
|
answered = len(requests & replies)
|
|
s_communication = communication_score(answered, len(requests))
|
|
self.coverage["s_communication"] = True
|
|
else:
|
|
s_communication = math.nan
|
|
self.coverage["s_communication"] = False
|
|
|
|
# --- reward (real only when the run was graded against a fixture: Group B) ---
|
|
# Needs Q_quality (fixture TestPassRate, masked-renormalized), a V_speed target_time, and a
|
|
# real E_cost. r_robust/g_gov derive from the run; p_rework from retries; p_risk from the
|
|
# approval risk levels (0 when no governed op was observed — a known under-approximation tied
|
|
# to the governance coverage gap, see docs/benchmark/metric-coverage-gaps.md).
|
|
quality = run.quality or {}
|
|
q_quality = quality.get("q_quality")
|
|
target_time = quality.get("target_time_seconds")
|
|
duration = max(0.0, float(run.updated_at or 0) - float(run.created_at or 0))
|
|
have_quality = isinstance(q_quality, (int, float)) and not math.isnan(q_quality)
|
|
have_speed = isinstance(target_time, (int, float)) and target_time and duration > 0
|
|
if have_quality and have_speed and self.coverage["s_cost"]:
|
|
# P_rework counts BOTH transient retries AND quality-driven review/queen reopens
|
|
# (reopen_task doesn't bump retry_count — it's a quality decision, not a failure;
|
|
# SC-10 fixes the prior under-count). Reopen tally accumulated on run.metadata.
|
|
rework_count = (sum(1 for t in tasks if getattr(t, "retry_count", 0) > 0)
|
|
+ int((run.metadata or {}).get("rework_reopens", 0) or 0))
|
|
risky = sum(1 for a in approvals
|
|
if str(a.get("risk_level", "")).lower() in {"high", "critical"})
|
|
p_risk = (100.0 * risky / len(approvals)) if approvals else 0.0
|
|
g_gov = governance_score(
|
|
sum(1 for a in approvals if a.get("decision") in ("approved", "rejected")),
|
|
len(approvals),
|
|
) # empty -> 100 (no governed-op violations observed)
|
|
reward_value = reward(
|
|
s_task=s_completion,
|
|
q_quality=float(q_quality),
|
|
v_speed=speed_score(float(target_time), duration),
|
|
e_cost=s_cost,
|
|
r_robust=s_robustness,
|
|
g_gov=g_gov,
|
|
p_risk=p_risk,
|
|
p_rework=rework_penalty(rework_count, total),
|
|
)
|
|
self.coverage["reward"] = True
|
|
else:
|
|
reward_value = math.nan
|
|
self.coverage["reward"] = False
|
|
|
|
# --- tau / eta / p_decision (real only when ACO dispatch recorded decisions: Group A) ---
|
|
# Run-level value = mean over the run's sampled assignments. p_decision uses the
|
|
# standard's §3.3 score (τ^α·η^β·100), recorded per decision as p_score. No decisions
|
|
# (flag off, or no ACO-dispatched task) → NaN.
|
|
decisions = [d for d in (run.decisions or [])
|
|
if isinstance(d.get("tau"), (int, float)) and isinstance(d.get("eta"), (int, float))]
|
|
if decisions:
|
|
tau = sum(d["tau"] for d in decisions) / len(decisions)
|
|
eta = sum(d["eta"] for d in decisions) / len(decisions)
|
|
p_dec = sum(float(d.get("p_score") or 0.0) for d in decisions) / len(decisions)
|
|
self.coverage["tau"] = self.coverage["eta"] = self.coverage["p_decision"] = True
|
|
else:
|
|
tau = eta = p_dec = math.nan
|
|
self.coverage["tau"] = self.coverage["eta"] = self.coverage["p_decision"] = False
|
|
|
|
# --- not yet collectable (see docs/benchmark/metric-coverage-gaps.md) ---
|
|
# s_gain needs baselines; s_swarm/g_e/g_e_cost/benchmark depend on it
|
|
# (any NaN component → NaN aggregate).
|
|
for k in ("s_gain", "s_swarm", "g_e", "g_e_cost", "benchmark"):
|
|
self.coverage[k] = False
|
|
|
|
return SwarmMetrics(
|
|
tau=tau,
|
|
eta=eta,
|
|
p_decision=p_dec,
|
|
reward=reward_value,
|
|
s_completion=s_completion,
|
|
s_gain=math.nan,
|
|
s_collaboration=s_collaboration,
|
|
s_communication=s_communication,
|
|
s_cost=s_cost,
|
|
s_robustness=s_robustness,
|
|
s_governance=s_governance,
|
|
s_swarm=math.nan,
|
|
g_e=math.nan,
|
|
g_e_cost=math.nan,
|
|
benchmark=math.nan,
|
|
)
|