Agentswarm/benchmark/baselines/comparison.py

"""Baseline comparison adapter — Agent 蜂群指标量化与标准 v2.0.

Defines the shared `BenchmarkRunRecord` that BOTH the swarm and each baseline (Single / Chain /
Sub-Agent / Strong) emit per benchmark run, plus the evaluator that computes G_E and G_E,c from
two records. Pure/deterministic given records — the records themselves must be PRODUCED by the
respective programs (instrumentation, esp. Quality, is pending; see
docs/benchmark/baseline-record-schema.md). No record → no comparison (we never fabricate scores).
"""
from __future__ import annotations

from dataclasses import dataclass
from typing import Optional

from ..metrics import (
    quality_score, completion_score, speed_score, cost_score, robustness_score,
    emergence_gain, swarm_cost, cost_normalized_gain, BASE_COEFFICIENTS,
)

SYSTEMS = {"swarm", "single", "chain", "sub", "strong"}

# Quality sub-inputs that may legitimately be absent (masked-renormalized in quality_score, rule #9).
_OPTIONAL_FIELDS = {"code_review_score", "user_acceptance"}


@dataclass
class BenchmarkRunRecord:
    """Unified per-run record (v2.0 §9 collection). Both swarm and baselines emit this.

    CodeReview/UserAcceptance are Optional: a runner that can only measure TestPassRate (the
    common case today — no reviewer/acceptance signal wired) leaves them None, and quality()
    renormalizes over the present inputs. None ≠ 0 (never fabricate a score, rule #9).
    """
    system: str            # one of SYSTEMS
    scenario: str          # coding | refactoring | architecture | devops | bugfix
    task_set_id: str       # identifies the shared task set (fairness: same id across systems)
    n_agent: int           # agent count used by this system
    completed_tasks: int
    total_tasks: int
    test_pass_rate: float       # 0..100  (Quality input — held-out fixture tests)
    budget_usd: float           # planned cost
    actual_cost_usd: float      # measured model cost
    model_tokens: int
    target_time_s: float        # planned/target time
    actual_time_s: float        # measured time
    recovered_failures: int
    total_failures: int
    code_review_score: Optional[float] = None  # 0..100, None when not collected
    user_acceptance: Optional[float] = None     # 0..100, None when not collected

    @classmethod
    def from_dict(cls, data: dict) -> "BenchmarkRunRecord":
        required = [f for f in cls.__dataclass_fields__ if f not in _OPTIONAL_FIELDS]
        missing = [f for f in required if f not in data]
        if missing:
            raise ValueError(f"benchmark record missing required fields: {missing}")
        if data["system"] not in SYSTEMS:
            raise ValueError(f"unknown system '{data['system']}' (expected one of {sorted(SYSTEMS)})")
        return cls(**{f: data[f] for f in cls.__dataclass_fields__ if f in data})


# --- derived quantities (from a single record) ---
def quality(rec: BenchmarkRunRecord) -> float:
    """Q = 0.4·TestPass + 0.3·CodeReview + 0.3·UserAcceptance (v2.0 §4)."""
    return quality_score(rec.test_pass_rate, rec.code_review_score, rec.user_acceptance)


def cost_efficiency(rec: BenchmarkRunRecord) -> float:
    """CostEfficiency = 100·Budget/ActualCost (v2.0 §4 E_cost口径)."""
    return cost_score(rec.budget_usd, rec.actual_cost_usd)


def unified_metrics(rec: BenchmarkRunRecord) -> dict:
    """The protocol's unified collection (v2.0 §9): Completion / Quality / Cost / Time / Robustness."""
    return {
        "completion": completion_score(rec.completed_tasks, rec.total_tasks),
        "quality": quality(rec),
        "cost_efficiency": cost_efficiency(rec),
        "speed": speed_score(rec.target_time_s, rec.actual_time_s),
        "robustness": robustness_score(rec.recovered_failures, rec.total_failures),
    }


# --- comparison (swarm vs one baseline) ---
def compare(swarm_rec: BenchmarkRunRecord, base_rec: BenchmarkRunRecord, *,
            symmetric_cost: bool = True) -> dict:
    """Compute G_E and G_E,c for the swarm against one baseline record.

    G_E   = Q_swarm − Q_base
    G_E,c = (Q_swarm / C_swarm) − (Q_base / C_base),  C_swarm = N_agent·(CostEfficiency/100+0.5)

    `symmetric_cost`:
      - True (DEFAULT — Benchmark Owner ratified correction, 2026-06-09): `C_base` uses the SAME
        formula as C_swarm with the baseline's own N_agent + CostEfficiency → both terms are
        quality-per-cost (comparable, non-degenerate).
      - False: literal v2.0 text — `C_base = 1.0` (scale-degenerate; kept for reference only).
      See docs/benchmark/cost-normalized-gain.md and OWNER-NOTE-cost-normalized-gain.md.
    """
    if swarm_rec.system != "swarm":
        raise ValueError("swarm_rec.system must be 'swarm'")
    if swarm_rec.task_set_id != base_rec.task_set_id:
        raise ValueError("records must share the same task_set_id (fair comparison)")

    q_swarm, q_base = quality(swarm_rec), quality(base_rec)
    g_e = emergence_gain(q_swarm, q_base)
    c_swarm = swarm_cost(swarm_rec.n_agent, cost_efficiency(swarm_rec))
    c_base = swarm_cost(base_rec.n_agent, cost_efficiency(base_rec)) if symmetric_cost else 1.0
    g_e_c = cost_normalized_gain(q_swarm, c_swarm, q_base, c_base=c_base)
    return {
        "base_system": base_rec.system,
        "base_coefficient": BASE_COEFFICIENTS.get(base_rec.system),
        "cost_mode": "symmetric" if symmetric_cost else "literal_v2.0",
        "q_swarm": q_swarm,
        "q_base": q_base,
        "c_swarm": c_swarm,
        "c_base": c_base,
        "g_e": g_e,
        "g_e_cost": g_e_c,
        "raw_gain_positive": g_e > 0,
        "cost_normalized_positive": g_e_c > 0,
    }


def evaluate(swarm_rec: BenchmarkRunRecord, baseline_recs: list[BenchmarkRunRecord], *,
             symmetric_cost: bool = True) -> dict:
    """Compare the swarm against all baselines. swarm_valid requires beating ALL on G_E and G_E,c."""
    results = [compare(swarm_rec, b, symmetric_cost=symmetric_cost) for b in baseline_recs]
    swarm_valid = bool(results) and all(r["raw_gain_positive"] and r["cost_normalized_positive"] for r in results)
    return {
        "scenario": swarm_rec.scenario,
        "cost_mode": "symmetric" if symmetric_cost else "literal_v2.0",
        "comparisons": results,
        "swarm_valid": swarm_valid,
    }