"""Benchmark suite harness (#13/#21/#22): run a task set through all 5 systems → report.

Default backend is OFFLINE (hermetic, key-free) — it validates the whole pipeline but, by design,
shows NO swarm quality advantage (every system gets the same reference solution). REAL G_E numbers
require `--backend openai` with OPENAI_API_KEY set and a sufficiently large, frozen task set.

Grading executes generated code in the fail-closed sandbox, so HEICODE_SANDBOX_ISOLATED must be
set (only inside an isolated pod / ephemeral CI runner — see security-boundary §8.1).

Usage:
    HEICODE_SANDBOX_ISOLATED=1 python scripts/run-benchmark-suite.py --taskset coding-set-1 \
        [--backend offline|openai] [--archive runs/<label>] [--run-label <label>]
"""
import argparse
import sys
from pathlib import Path

# Markdown rendering uses ✅/🔴/⚠️; force UTF-8 so it prints on a Windows GBK console too.
try:
    sys.stdout.reconfigure(encoding="utf-8")
except Exception:
    pass

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from benchmark.tasksets import load_taskset, available_tasksets
from benchmark.runners import run_all, OfflineBackend, OpenAIBackend
from benchmark.reports import build_report, render_markdown
from benchmark.replay import save_archive


def main() -> int:
    ap = argparse.ArgumentParser(description="Run the benchmark suite (swarm + 4 baselines).")
    ap.add_argument("--taskset", default="coding-set-1", help=f"one of: {available_tasksets()}")
    ap.add_argument("--backend", choices=["offline", "openai"], default="offline")
    ap.add_argument("--n-runs", type=int, default=1)
    ap.add_argument("--archive", default=None, help="directory to write the replay archive into")
    ap.add_argument("--run-label", default="run-1", help="archive subdir name (deterministic)")
    args = ap.parse_args()

    taskset = load_taskset(args.taskset)

    if args.backend == "openai":
        backend = OpenAIBackend(price_per_1k_tokens=0.0006)
        strong_backend = OpenAIBackend(model="gpt-4o", price_per_1k_tokens=0.005)  # bigger model
    else:
        backend = OfflineBackend(cost_per_call=0.002)
        strong_backend = OfflineBackend(cost_per_call=0.01)  # "stronger" = costlier

    records = run_all(taskset, backend, strong_backend=strong_backend)
    report = build_report(records, backend_name=args.backend, n_runs=args.n_runs)

    print(render_markdown(report))
    if args.archive:
        path = save_archive(args.archive, args.run_label, records=records, report=report)
        print(f"\narchived → {path}")

    if args.backend == "offline":
        print("\nNOTE: offline backend — pipeline validation only, NOT a real comparison "
              "(no swarm advantage is or should be shown). Use --backend openai for real G_E.")
    return 0


if __name__ == "__main__":
    sys.exit(main())