"""Baseline comparison adapter — Agent 蜂群指标量化与标准 v2.0. Defines the shared `BenchmarkRunRecord` that BOTH the swarm and each baseline (Single / Chain / Sub-Agent / Strong) emit per benchmark run, plus the evaluator that computes G_E and G_E,c from two records. Pure/deterministic given records — the records themselves must be PRODUCED by the respective programs (instrumentation, esp. Quality, is pending; see docs/benchmark/baseline-record-schema.md). No record → no comparison (we never fabricate scores). """ from __future__ import annotations from dataclasses import dataclass from typing import Optional from ..metrics import ( quality_score, completion_score, speed_score, cost_score, robustness_score, emergence_gain, swarm_cost, cost_normalized_gain, BASE_COEFFICIENTS, ) SYSTEMS = {"swarm", "single", "chain", "sub", "strong"} # Quality sub-inputs that may legitimately be absent (masked-renormalized in quality_score, rule #9). _OPTIONAL_FIELDS = {"code_review_score", "user_acceptance"} @dataclass class BenchmarkRunRecord: """Unified per-run record (v2.0 §9 collection). Both swarm and baselines emit this. CodeReview/UserAcceptance are Optional: a runner that can only measure TestPassRate (the common case today — no reviewer/acceptance signal wired) leaves them None, and quality() renormalizes over the present inputs. None ≠ 0 (never fabricate a score, rule #9). """ system: str # one of SYSTEMS scenario: str # coding | refactoring | architecture | devops | bugfix task_set_id: str # identifies the shared task set (fairness: same id across systems) n_agent: int # agent count used by this system completed_tasks: int total_tasks: int test_pass_rate: float # 0..100 (Quality input — held-out fixture tests) budget_usd: float # planned cost actual_cost_usd: float # measured model cost model_tokens: int target_time_s: float # planned/target time actual_time_s: float # measured time recovered_failures: int total_failures: int code_review_score: Optional[float] = None # 0..100, None when not collected user_acceptance: Optional[float] = None # 0..100, None when not collected @classmethod def from_dict(cls, data: dict) -> "BenchmarkRunRecord": required = [f for f in cls.__dataclass_fields__ if f not in _OPTIONAL_FIELDS] missing = [f for f in required if f not in data] if missing: raise ValueError(f"benchmark record missing required fields: {missing}") if data["system"] not in SYSTEMS: raise ValueError(f"unknown system '{data['system']}' (expected one of {sorted(SYSTEMS)})") return cls(**{f: data[f] for f in cls.__dataclass_fields__ if f in data}) # --- derived quantities (from a single record) --- def quality(rec: BenchmarkRunRecord) -> float: """Q = 0.4·TestPass + 0.3·CodeReview + 0.3·UserAcceptance (v2.0 §4).""" return quality_score(rec.test_pass_rate, rec.code_review_score, rec.user_acceptance) def cost_efficiency(rec: BenchmarkRunRecord) -> float: """CostEfficiency = 100·Budget/ActualCost (v2.0 §4 E_cost口径).""" return cost_score(rec.budget_usd, rec.actual_cost_usd) def unified_metrics(rec: BenchmarkRunRecord) -> dict: """The protocol's unified collection (v2.0 §9): Completion / Quality / Cost / Time / Robustness.""" return { "completion": completion_score(rec.completed_tasks, rec.total_tasks), "quality": quality(rec), "cost_efficiency": cost_efficiency(rec), "speed": speed_score(rec.target_time_s, rec.actual_time_s), "robustness": robustness_score(rec.recovered_failures, rec.total_failures), } # --- comparison (swarm vs one baseline) --- def compare(swarm_rec: BenchmarkRunRecord, base_rec: BenchmarkRunRecord, *, symmetric_cost: bool = True) -> dict: """Compute G_E and G_E,c for the swarm against one baseline record. G_E = Q_swarm − Q_base G_E,c = (Q_swarm / C_swarm) − (Q_base / C_base), C_swarm = N_agent·(CostEfficiency/100+0.5) `symmetric_cost`: - True (DEFAULT — Benchmark Owner ratified correction, 2026-06-09): `C_base` uses the SAME formula as C_swarm with the baseline's own N_agent + CostEfficiency → both terms are quality-per-cost (comparable, non-degenerate). - False: literal v2.0 text — `C_base = 1.0` (scale-degenerate; kept for reference only). See docs/benchmark/cost-normalized-gain.md and OWNER-NOTE-cost-normalized-gain.md. """ if swarm_rec.system != "swarm": raise ValueError("swarm_rec.system must be 'swarm'") if swarm_rec.task_set_id != base_rec.task_set_id: raise ValueError("records must share the same task_set_id (fair comparison)") q_swarm, q_base = quality(swarm_rec), quality(base_rec) g_e = emergence_gain(q_swarm, q_base) c_swarm = swarm_cost(swarm_rec.n_agent, cost_efficiency(swarm_rec)) c_base = swarm_cost(base_rec.n_agent, cost_efficiency(base_rec)) if symmetric_cost else 1.0 g_e_c = cost_normalized_gain(q_swarm, c_swarm, q_base, c_base=c_base) return { "base_system": base_rec.system, "base_coefficient": BASE_COEFFICIENTS.get(base_rec.system), "cost_mode": "symmetric" if symmetric_cost else "literal_v2.0", "q_swarm": q_swarm, "q_base": q_base, "c_swarm": c_swarm, "c_base": c_base, "g_e": g_e, "g_e_cost": g_e_c, "raw_gain_positive": g_e > 0, "cost_normalized_positive": g_e_c > 0, } def evaluate(swarm_rec: BenchmarkRunRecord, baseline_recs: list[BenchmarkRunRecord], *, symmetric_cost: bool = True) -> dict: """Compare the swarm against all baselines. swarm_valid requires beating ALL on G_E and G_E,c.""" results = [compare(swarm_rec, b, symmetric_cost=symmetric_cost) for b in baseline_recs] swarm_valid = bool(results) and all(r["raw_gain_positive"] and r["cost_normalized_positive"] for r in results) return { "scenario": swarm_rec.scenario, "cost_mode": "symmetric" if symmetric_cost else "literal_v2.0", "comparisons": results, "swarm_valid": swarm_valid, }