Songhaoz666andClaude Opus 4.8
54cb327348
- 基准标准 v2.1：SwarmMetrics（15 字段）、τ/η/P_decision/reward 公式、对称 G_E,c（修正 C_base=1.0 退化）、Σλ=1.0 校验；新增基线对比与运行记录 schema；指标覆盖缺口分析；参考系数暂留为元数据（待量化）。 - 主控 Agent 实体（分解 / 评审决策 / 汇总）；事件契约修正（timeline.title、budget.threshold_pct、handoff 角色、task.released）+ 契约校验脚本。 - 实质性 LLM 对等回复（含降级回退）；集成契约（runtime / event / usage / audit / frontend / capability / security）；CLIENT_GUIDE 客户端指南；CI 工作流；治理与交付文档。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
87 lines
3.6 KiB
Python
87 lines
3.6 KiB
Python
"""Test the baseline comparison adapter (benchmark/baselines/comparison.py) — v2.0.
|
|
|
|
Run from agent_swarm_v6: python scripts/test-baseline-comparison.py
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from benchmark.baselines import BenchmarkRunRecord, quality, unified_metrics, compare, evaluate
|
|
|
|
failures = []
|
|
|
|
|
|
def check(name, cond):
|
|
print(("PASS" if cond else "FAIL"), "-", name)
|
|
if not cond:
|
|
failures.append(name)
|
|
|
|
|
|
swarm = BenchmarkRunRecord(
|
|
system="swarm", scenario="coding", task_set_id="coding-set-1", n_agent=5,
|
|
completed_tasks=10, total_tasks=10,
|
|
test_pass_rate=90, code_review_score=80, user_acceptance=85,
|
|
budget_usd=10, actual_cost_usd=5, model_tokens=50000,
|
|
target_time_s=600, actual_time_s=500, recovered_failures=1, total_failures=1,
|
|
)
|
|
sub = BenchmarkRunRecord(
|
|
system="sub", scenario="coding", task_set_id="coding-set-1", n_agent=3,
|
|
completed_tasks=9, total_tasks=10,
|
|
test_pass_rate=70, code_review_score=60, user_acceptance=65,
|
|
budget_usd=10, actual_cost_usd=6, model_tokens=42000,
|
|
target_time_s=600, actual_time_s=720, recovered_failures=1, total_failures=2,
|
|
)
|
|
|
|
# Quality: 0.4*90+0.3*80+0.3*85 = 85.5 ; 0.4*70+0.3*60+0.3*65 = 65.5
|
|
check("Q_swarm = 85.5", quality(swarm) == 85.5)
|
|
check("Q_sub = 65.5", quality(sub) == 65.5)
|
|
|
|
# DEFAULT is now symmetric C_base (Owner-ratified): C_base = n_base*(CostEff_base/100+0.5) = 3*(166.667/100+0.5) = 6.5
|
|
r = compare(swarm, sub)
|
|
check("default mode is symmetric", r["cost_mode"] == "symmetric" and r["c_base"] == 6.5)
|
|
check("G_E = 20.0", r["g_e"] == 20.0)
|
|
check("C_swarm = 12.5 (5*(200/100+0.5))", r["c_swarm"] == 12.5)
|
|
check("G_E,c symmetric = (85.5/12.5)-(65.5/6.5)", round(r["g_e_cost"], 4) == round((85.5 / 12.5) - (65.5 / 6.5), 4))
|
|
check("base_coefficient(sub) = 0.90", r["base_coefficient"] == 0.90)
|
|
check("raw gain positive", r["raw_gain_positive"] is True)
|
|
check("cost-normalized negative (5 agents not worth it here)", r["cost_normalized_positive"] is False)
|
|
|
|
ev = evaluate(swarm, [sub])
|
|
check("swarm_valid False (fails cost-normalized)", ev["swarm_valid"] is False)
|
|
|
|
# Literal v2.0 (reference only): C_base = 1.0
|
|
rl = compare(swarm, sub, symmetric_cost=False)
|
|
check("literal mode label + C_base=1.0", rl["cost_mode"] == "literal_v2.0" and rl["c_base"] == 1.0)
|
|
check("literal G_E,c = (85.5/12.5)-65.5", round(rl["g_e_cost"], 4) == round((85.5 / 12.5) - 65.5, 4))
|
|
|
|
um = unified_metrics(sub)
|
|
check("unified_metrics keys", set(um.keys()) == {"completion", "quality", "cost_efficiency", "speed", "robustness"})
|
|
check("unified completion 90", um["completion"] == 90.0)
|
|
|
|
# from_dict validation
|
|
ok = BenchmarkRunRecord.from_dict({f: getattr(sub, f) for f in BenchmarkRunRecord.__dataclass_fields__})
|
|
check("from_dict round-trips", ok == sub)
|
|
try:
|
|
BenchmarkRunRecord.from_dict({"system": "sub"})
|
|
check("from_dict rejects missing fields", False)
|
|
except ValueError:
|
|
check("from_dict rejects missing fields", True)
|
|
try:
|
|
compare(sub, swarm) # first arg must be system='swarm'
|
|
check("compare requires swarm record first", False)
|
|
except ValueError:
|
|
check("compare requires swarm record first", True)
|
|
try:
|
|
bad = BenchmarkRunRecord.from_dict({f: getattr(sub, f) for f in BenchmarkRunRecord.__dataclass_fields__} | {"task_set_id": "other"})
|
|
compare(swarm, bad)
|
|
check("compare requires same task_set_id", False)
|
|
except ValueError:
|
|
check("compare requires same task_set_id", True)
|
|
|
|
print()
|
|
if failures:
|
|
print(f"{len(failures)} comparison check(s) FAILED: {failures}")
|
|
sys.exit(1)
|
|
print("all baseline comparison checks passed (v2.0)")
|