Agentswarm/scripts/test-benchmark-selfcert.py

"""Test SelfCertCollector + leaderboard (阶段1 自证采集器).

Hermetic, no Redis / no model. Verifies:
  - S_gain ≡ G_E vs the STRONGEST baseline (most conservative), g_e/g_e_cost wired.
  - Without a live run: collaboration/communication/governance NaN → S_swarm NaN,
    Benchmark_Agent NOT available, gaps list the honest blockers.
  - With a live run covering those: S_swarm + Reward real, but Benchmark_Agent STILL NaN
    because O (observability) has no formula in the standard → gaps == [O].
  - leaderboard builds and sorts; nothing is fabricated (NaN stays NaN).

Run:  python scripts/test-benchmark-selfcert.py
"""
import math
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from benchmark.baselines import BenchmarkRunRecord, quality
from benchmark.metrics import SwarmMetrics
from benchmark.collectors.selfcert_collector import SelfCertCollector
from benchmark.leaderboard import build_leaderboard, render_markdown

failures = []


def check(name, cond):
    print(("PASS" if cond else "FAIL"), "-", name)
    if not cond:
        failures.append(name)


def rec(system, *, test_pass_rate, n_agent, budget, actual, total=4, completed=4):
    return BenchmarkRunRecord(
        system=system, scenario="coding", task_set_id="coding-set-1", n_agent=n_agent,
        completed_tasks=completed, total_tasks=total, test_pass_rate=test_pass_rate,
        budget_usd=budget, actual_cost_usd=actual, model_tokens=1000,
        target_time_s=10.0, actual_time_s=8.0, recovered_failures=1, total_failures=2,
    )


def main():
    # swarm beats all baselines on raw quality; strongest baseline = 'strong' (Q=85).
    records = {
        "swarm":  rec("swarm",  test_pass_rate=90, n_agent=5, budget=15, actual=10),
        "single": rec("single", test_pass_rate=70, n_agent=1, budget=12, actual=10),
        "chain":  rec("chain",  test_pass_rate=75, n_agent=1, budget=12, actual=10),
        "sub":    rec("sub",    test_pass_rate=80, n_agent=2, budget=12, actual=10),
        "strong": rec("strong", test_pass_rate=85, n_agent=1, budget=12, actual=10),
    }

    # --- 1. no live run: gain wired, but S_swarm/benchmark honestly NaN ---
    res = SelfCertCollector(records).collect()
    m = res.metrics
    check("strongest baseline = strong", res.strongest_baseline == "strong")
    # S_gain ≡ G_E vs strongest = 90 - 85 = 5
    check("s_gain == G_E vs strongest (5.0)", round(m.s_gain, 4) == 5.0 and round(m.g_e, 4) == 5.0)
    check("g_e/g_e_cost covered", res.coverage["g_e"] and res.coverage["g_e_cost"])
    check("s_completion real (100)", round(m.s_completion, 1) == 100.0 and res.coverage["s_completion"])
    check("s_collaboration NaN (no live)", math.isnan(m.s_collaboration) and res.coverage["s_collaboration"] is False)
    check("s_swarm NaN (missing live components)", math.isnan(m.s_swarm) and res.coverage["s_swarm"] is False)
    check("benchmark NOT available", res.benchmark_available is False and math.isnan(m.benchmark))
    check("gaps mention O (no formula) and Gov", any("O " in g or "可观测性" in g for g in res.gaps)
          and any("Gov" in g for g in res.gaps))

    # --- 2. with a live run covering collaboration/communication/governance/reward ---
    live = SwarmMetrics(
        tau=math.nan, eta=math.nan, p_decision=math.nan, reward=72.0,
        s_completion=math.nan, s_gain=math.nan, s_collaboration=88.0, s_communication=92.0,
        s_cost=math.nan, s_robustness=math.nan, s_governance=100.0,
        s_swarm=math.nan, g_e=math.nan, g_e_cost=math.nan, benchmark=math.nan,
    )
    live_cov = {"s_collaboration": True, "s_communication": True, "s_governance": True, "reward": True}
    res2 = SelfCertCollector(records, live=live, live_coverage=live_cov).collect()
    m2 = res2.metrics
    check("with live: s_swarm REAL", not math.isnan(m2.s_swarm) and res2.coverage["s_swarm"])
    check("with live: reward REAL (72)", round(m2.reward, 1) == 72.0 and res2.coverage["reward"])
    check("with live: benchmark STILL NaN (O has no formula)",
          res2.benchmark_available is False and math.isnan(m2.benchmark))
    check("with live: ONLY remaining gap is O", len(res2.gaps) == 1 and ("O " in res2.gaps[0] or "可观测性" in res2.gaps[0]))

    # --- 3. offline-style: swarm == baselines → G_E=0, swarm not valid ---
    flat = {s: rec(s, test_pass_rate=80, n_agent=(5 if s == "swarm" else 1), budget=12, actual=10)
            for s in ("swarm", "single", "chain", "sub", "strong")}
    res3 = SelfCertCollector(flat).collect()
    check("flat quality → G_E=0", round(res3.metrics.g_e, 6) == 0.0)
    check("flat quality → swarm NOT valid", res3.swarm_valid is False)

    # --- 4. leaderboard builds + sorts (unavailable sinks) ---
    board = build_leaderboard([res, res2, res3])
    check("leaderboard has 3 rows", len(board["rows"]) == 3)
    check("leaderboard renders", "Benchmark Leaderboard" in render_markdown(board))
    check("benchmark_agent shown as None (unavailable)", all(r["benchmark_agent"] is None for r in board["rows"]))

    print()
    if failures:
        print(f"{len(failures)} self-cert check(s) FAILED: {failures}")
        return 1
    print("all benchmark self-cert checks passed")
    return 0


if __name__ == "__main__":
    sys.exit(main())