"""Test the Group C benchmark runners + report + replay (offline, hermetic).

Proves the PIPELINE (taskset → 5 runners → records → evaluate → report → archive) end-to-end and,
crucially, that offline runs do NOT fabricate a swarm advantage. Real G_E needs a model key.
Run from agent_swarm_v6:
    pip install -r orchestrator/requirements.txt -r agent/requirements.txt
    python scripts/test-benchmark-runners.py
(grading executes code in the sandbox; this test self-confirms isolation, like test-sandbox.py.)
"""
import math
import os
import sys
import tempfile
from pathlib import Path

# The runners grade generated code in the fail-closed sandbox (orchestrator/sandbox.py). Confirm
# isolation for this hermetic test (CI runner is ephemeral) — same discipline as test-sandbox.py.
os.environ.setdefault("HEICODE_SANDBOX_ISOLATED", "1")
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from benchmark.tasksets import load_taskset
from benchmark.runners import run_all, RUNNERS, OfflineBackend
from benchmark.baselines import quality, BenchmarkRunRecord
from benchmark.reports import build_report, render_markdown
from benchmark.replay import save_archive

failures = []


def check(name, cond):
    print(("PASS" if cond else "FAIL"), "-", name)
    if not cond:
        failures.append(name)


taskset = load_taskset("coding-set-1")
backend = OfflineBackend(cost_per_call=0.002)
strong_backend = OfflineBackend(cost_per_call=0.01)
records = run_all(taskset, backend, strong_backend=strong_backend)

# --- all 5 systems produced a valid record ---
check("5 systems ran", set(records.keys()) == set(RUNNERS.keys()))
check("all records are BenchmarkRunRecord", all(isinstance(r, BenchmarkRunRecord) for r in records.values()))
check("all share the task_set_id", all(r.task_set_id == "coding-set-1" for r in records.values()))

# --- quality is REAL (graded by held-out tests), non-NaN ---
check("quality non-NaN for every system",
      all(not math.isnan(quality(r)) for r in records.values()))
# offline: reference solution passes the held-out tests -> 100 for all
check("offline quality = 100 (held-out tests pass)",
      all(quality(r) == 100.0 for r in records.values()))

# --- ANTI-FABRICATION: offline shows NO swarm quality advantage ---
check("offline G_E == 0 across baselines (no fabricated gain)",
      all(quality(records["swarm"]) - quality(records[b]) == 0.0 for b in ("single", "strong", "chain", "sub")))

# --- topology really differs (cost rises with agents/calls) ---
check("swarm costs >= single (more calls)", records["swarm"].actual_cost_usd >= records["single"].actual_cost_usd)
check("n_agent differs by topology",
      records["single"].n_agent == 1 and records["chain"].n_agent == 3 and records["sub"].n_agent == 4)

# --- report computes G_E / G_E,c / coverage / confidence honestly ---
report = build_report(records, backend_name="offline", n_runs=1)
check("report has a comparison per baseline", len(report["comparisons"]) == 4)
check("swarm_valid is False offline (honest, not fabricated)", report["swarm_valid"] is False)
check("G_E computed (not NaN) for each baseline",
      all(not math.isnan(c["g_e"]) for c in report["comparisons"]))
check("confidence = none offline", report["confidence"]["level"] == "none")
check("coverage: test_pass real, review/acceptance masked",
      report["coverage"]["quality.test_pass_rate"] is True
      and report["coverage"]["quality.code_review_score"] is False
      and report["coverage"]["quality.user_acceptance"] is False)
check("Benchmark_Agent reported NOT available (needs live metrics + real backend)",
      report["benchmark_agent"]["available"] is False)

# --- replay archive writes the artifacts ---
with tempfile.TemporaryDirectory() as d:
    path = save_archive(d, "run-1", records=records, report=report)
    p = Path(path)
    check("archive wrote records/report/md/meta",
          (p / "records.json").exists() and (p / "report.json").exists()
          and (p / "report.md").exists() and (p / "meta.json").exists())

# --- markdown renders without error ---
check("markdown renders", "Benchmark report" in render_markdown(report))

print()
if failures:
    print(f"{len(failures)} runner check(s) FAILED: {failures}")
    sys.exit(1)
print("all Group C benchmark runner checks passed")