Songhaoz666andClaude Opus 4.8
baa67350e6
在去中心化重构之上落地 benchmark 对比管线：5 个系统（single/strong/chain/sub_agent/swarm） 跑同一任务集、同一执行后端，产出统一 BenchmarkRunRecord → 评估器算 G_E/G_E,c → 报告 + 回放。 - benchmark/runners/：backend（Offline 确定性 / OpenAI 真实）+ base + 5 个 runner。各 runner 用 held-out fixture 测试在 Group B 沙箱里评分得 TestPassRate（权威，非自评）。 - benchmark/tasksets/：统一任务集 + 加载器（coding-set-1，1 个 fixture）。 - benchmark/reports/、benchmark/replay/：G_E/G_E,c/coverage/confidence + 归档。 - benchmark/baselines/comparison.py：BenchmarkRunRecord 的 CodeReview/UserAcceptance 改为 Optional（掩码归一，未采集即 None，规则 #9）。 - scripts/run-benchmark-suite.py harness + scripts/test-benchmark-runners.py。 与去中心化重构对齐：swarm runner 拓扑已**重指向去中心化流程**（种子→自选→自主分解→竞争→ 同伴交叉评审→收敛，calls=6/review=1），非旧 Master「分解→派发→单评审」。仍用同一离线后端 建模以保证公平对比（驱动活体编排器会换后端→记录不可比；活体全流程由 test-workflow-e2e 验证）。 沙箱适配：runner 评分走 fail-closed 沙箱（#24），故 test + CI 步骤设 HEICODE_SANDBOX_ISOLATED=1 （仅 CI/隔离 Pod）。 影响范围：agent_swarm（benchmark 层 + 测试 + docs + CI）。不碰 orchestrator 编排逻辑、 不改 Manager↔Swarm 契约、不影响 Client/计费/密钥/审计/发布链路。 诚实边界： - **离线后端只验证管线**：所有系统拿同一参考解 → quality 相同 → G_E=0、swarm_valid=False， 刻意不显示蜂群优势（反造假）。真实 G_E>0 需 --backend openai + 足量冻结任务集 + 多次运行。 - 故 Closes #21（运行器 + 统一记录已落地并产出合规非 NaN 记录）；Refs #20（仅 1/5 场景）、 Refs #22（评估器/报告/回放已建，但 Quality 仅 TestPassRate，CodeReview/UserAcceptance 缺）、 Refs #13（验收 EPIC，需真实 run 证明 Swarm>baselines，未满足）。 Closes #21 Refs #20 Refs #22 Refs #13 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
92 lines
4.2 KiB
Python
92 lines
4.2 KiB
Python
"""Test the Group C benchmark runners + report + replay (offline, hermetic).
|
|
|
|
Proves the PIPELINE (taskset → 5 runners → records → evaluate → report → archive) end-to-end and,
|
|
crucially, that offline runs do NOT fabricate a swarm advantage. Real G_E needs a model key.
|
|
Run from agent_swarm_v6:
|
|
pip install -r orchestrator/requirements.txt -r agent/requirements.txt
|
|
python scripts/test-benchmark-runners.py
|
|
(grading executes code in the sandbox; this test self-confirms isolation, like test-sandbox.py.)
|
|
"""
|
|
import math
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
# The runners grade generated code in the fail-closed sandbox (orchestrator/sandbox.py). Confirm
|
|
# isolation for this hermetic test (CI runner is ephemeral) — same discipline as test-sandbox.py.
|
|
os.environ.setdefault("HEICODE_SANDBOX_ISOLATED", "1")
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from benchmark.tasksets import load_taskset
|
|
from benchmark.runners import run_all, RUNNERS, OfflineBackend
|
|
from benchmark.baselines import quality, BenchmarkRunRecord
|
|
from benchmark.reports import build_report, render_markdown
|
|
from benchmark.replay import save_archive
|
|
|
|
failures = []
|
|
|
|
|
|
def check(name, cond):
|
|
print(("PASS" if cond else "FAIL"), "-", name)
|
|
if not cond:
|
|
failures.append(name)
|
|
|
|
|
|
taskset = load_taskset("coding-set-1")
|
|
backend = OfflineBackend(cost_per_call=0.002)
|
|
strong_backend = OfflineBackend(cost_per_call=0.01)
|
|
records = run_all(taskset, backend, strong_backend=strong_backend)
|
|
|
|
# --- all 5 systems produced a valid record ---
|
|
check("5 systems ran", set(records.keys()) == set(RUNNERS.keys()))
|
|
check("all records are BenchmarkRunRecord", all(isinstance(r, BenchmarkRunRecord) for r in records.values()))
|
|
check("all share the task_set_id", all(r.task_set_id == "coding-set-1" for r in records.values()))
|
|
|
|
# --- quality is REAL (graded by held-out tests), non-NaN ---
|
|
check("quality non-NaN for every system",
|
|
all(not math.isnan(quality(r)) for r in records.values()))
|
|
# offline: reference solution passes the held-out tests -> 100 for all
|
|
check("offline quality = 100 (held-out tests pass)",
|
|
all(quality(r) == 100.0 for r in records.values()))
|
|
|
|
# --- ANTI-FABRICATION: offline shows NO swarm quality advantage ---
|
|
check("offline G_E == 0 across baselines (no fabricated gain)",
|
|
all(quality(records["swarm"]) - quality(records[b]) == 0.0 for b in ("single", "strong", "chain", "sub")))
|
|
|
|
# --- topology really differs (cost rises with agents/calls) ---
|
|
check("swarm costs >= single (more calls)", records["swarm"].actual_cost_usd >= records["single"].actual_cost_usd)
|
|
check("n_agent differs by topology",
|
|
records["single"].n_agent == 1 and records["chain"].n_agent == 3 and records["sub"].n_agent == 4)
|
|
|
|
# --- report computes G_E / G_E,c / coverage / confidence honestly ---
|
|
report = build_report(records, backend_name="offline", n_runs=1)
|
|
check("report has a comparison per baseline", len(report["comparisons"]) == 4)
|
|
check("swarm_valid is False offline (honest, not fabricated)", report["swarm_valid"] is False)
|
|
check("G_E computed (not NaN) for each baseline",
|
|
all(not math.isnan(c["g_e"]) for c in report["comparisons"]))
|
|
check("confidence = none offline", report["confidence"]["level"] == "none")
|
|
check("coverage: test_pass real, review/acceptance masked",
|
|
report["coverage"]["quality.test_pass_rate"] is True
|
|
and report["coverage"]["quality.code_review_score"] is False
|
|
and report["coverage"]["quality.user_acceptance"] is False)
|
|
check("Benchmark_Agent reported NOT available (needs live metrics + real backend)",
|
|
report["benchmark_agent"]["available"] is False)
|
|
|
|
# --- replay archive writes the artifacts ---
|
|
with tempfile.TemporaryDirectory() as d:
|
|
path = save_archive(d, "run-1", records=records, report=report)
|
|
p = Path(path)
|
|
check("archive wrote records/report/md/meta",
|
|
(p / "records.json").exists() and (p / "report.json").exists()
|
|
and (p / "report.md").exists() and (p / "meta.json").exists())
|
|
|
|
# --- markdown renders without error ---
|
|
check("markdown renders", "Benchmark report" in render_markdown(report))
|
|
|
|
print()
|
|
if failures:
|
|
print(f"{len(failures)} runner check(s) FAILED: {failures}")
|
|
sys.exit(1)
|
|
print("all Group C benchmark runner checks passed")
|