Agentswarm/benchmark/reports/__init__.py

"""Benchmark report (#22): turn per-system BenchmarkRunRecords into G_E / G_E,c / coverage /
confidence, plus a human-readable rendering.

What the report HONESTLY claims:
  - G_E, G_E,c per baseline + swarm_valid (from benchmark.baselines.evaluate).
  - per-system unified metrics (completion / quality / cost-efficiency / speed / robustness).
  - coverage: which Quality inputs were real (TestPassRate) vs masked-absent (CodeReview/UserAccept).
  - confidence: LOW unless many runs on a real backend (offline/1-run = pipeline validation only).

What it does NOT fabricate:
  - The full `Benchmark_Agent` aggregate needs S_swarm (with communication/governance), Reward,
    Observability — these come from a LIVE swarm run's SwarmMetrics, not from these topology
    records. So `benchmark_agent` is reported as NOT-AVAILABLE here, with the reason, rather than
    a guessed number. Wiring the live collector + multi-run + a real backend is the remaining step.
"""
from __future__ import annotations

import math
from typing import Dict, List

from ..baselines import BenchmarkRunRecord, evaluate, quality, unified_metrics


def _coverage(swarm: BenchmarkRunRecord) -> Dict[str, bool]:
    return {
        "completion": True,
        "quality.test_pass_rate": True,
        "quality.code_review_score": swarm.code_review_score is not None,
        "quality.user_acceptance": swarm.user_acceptance is not None,
        "cost_efficiency": swarm.actual_cost_usd > 0,
        "robustness": True,
        "g_e": True,            # computed vs baselines below
        "g_e_cost": True,
        # Requires a LIVE swarm run's SwarmMetrics (S_swarm components, Reward, Observability) —
        # not derivable from topology records alone.
        "benchmark_agent": False,
    }


def _confidence(backend_name: str, n_runs: int, swarm: BenchmarkRunRecord) -> Dict[str, object]:
    reasons = []
    level = "high"
    if backend_name != "openai":
        level = "none"
        reasons.append("offline backend: reference solution echoed, no real model differentiation")
    if n_runs < 3:
        level = "low" if level != "none" else level
        reasons.append(f"only {n_runs} run(s); no variance/significance (need ≥3)")
    if swarm.code_review_score is None or swarm.user_acceptance is None:
        reasons.append("Quality partial: CodeReview/UserAcceptance not collected (masked)")
    return {"level": level, "reasons": reasons}


def build_report(records: Dict[str, BenchmarkRunRecord], *, symmetric_cost: bool = True,
                 backend_name: str = "offline", n_runs: int = 1) -> dict:
    if "swarm" not in records:
        raise ValueError("records must include a 'swarm' record")
    swarm = records["swarm"]
    baselines = [r for s, r in records.items() if s != "swarm"]
    ev = evaluate(swarm, baselines, symmetric_cost=symmetric_cost)

    per_system = {}
    for system, rec in records.items():
        m = unified_metrics(rec)
        per_system[system] = {
            "n_agent": rec.n_agent,
            "completed": f"{rec.completed_tasks}/{rec.total_tasks}",
            "quality": round(quality(rec), 4),
            "actual_cost_usd": rec.actual_cost_usd,
            "cost_efficiency": round(m["cost_efficiency"], 4),
            "speed": round(m["speed"], 4),
            "robustness": round(m["robustness"], 4),
        }

    return {
        "task_set_id": swarm.task_set_id,
        "scenario": swarm.scenario,
        "backend": backend_name,
        "n_runs": n_runs,
        "cost_mode": ev["cost_mode"],
        "per_system": per_system,
        "comparisons": ev["comparisons"],
        "swarm_valid": ev["swarm_valid"],
        "coverage": _coverage(swarm),
        "confidence": _confidence(backend_name, n_runs, swarm),
        "benchmark_agent": {
            "available": False,
            "reason": "needs live swarm SwarmMetrics (S_swarm/Reward/Observability) + multi-run on a real backend",
        },
    }


def render_markdown(report: dict) -> str:
    lines: List[str] = []
    a = lines.append
    a(f"# Benchmark report — {report['scenario']} / `{report['task_set_id']}`")
    a("")
    a(f"- backend: **{report['backend']}**  · runs: **{report['n_runs']}**  · cost mode: {report['cost_mode']}")
    a(f"- **swarm_valid: {report['swarm_valid']}**  · confidence: **{report['confidence']['level']}**")
    if report["confidence"]["reasons"]:
        for r in report["confidence"]["reasons"]:
            a(f"  - ⚠️ {r}")
    a("")
    a("## Per-system")
    a("| system | n_agent | completed | quality | cost($) | cost_eff | speed | robustness |")
    a("|---|---|---|---|---|---|---|---|")
    for s, v in report["per_system"].items():
        a(f"| {s} | {v['n_agent']} | {v['completed']} | {v['quality']} | {v['actual_cost_usd']} "
          f"| {v['cost_efficiency']} | {v['speed']} | {v['robustness']} |")
    a("")
    a("## Swarm vs baselines")
    a("| baseline | G_E | G_E,c | G_E>0 | G_E,c>0 |")
    a("|---|---|---|---|---|")
    for c in report["comparisons"]:
        a(f"| {c['base_system']} | {round(c['g_e'], 4)} | {round(c['g_e_cost'], 4)} "
          f"| {c['raw_gain_positive']} | {c['cost_normalized_positive']} |")
    a("")
    ba = report["benchmark_agent"]
    a(f"## Benchmark_Agent: {'available' if ba['available'] else 'NOT AVAILABLE'}")
    if not ba["available"]:
        a(f"> {ba['reason']}")
    a("")
    a("## Coverage")
    for k, v in report["coverage"].items():
        a(f"- {'✅' if v else '🔴'} {k}")
    return "\n".join(lines)