FastheiandClaude Opus 4.8
94ef894ace
把采集器接进真实运行，让数据自己累积，为后续【经验标定】O/系数/S_gain 攒真实 用户数据（这些量只能由真实运行事后标定，不能先验写死）。本阶段不编任何公式。 - orchestrator/main.py：run 终态(completed/failed)在终态事件后调 capture_run_metrics。 只读 run 状态、try/except 包裹绝不失败 run；默认开，BENCHMARK_CAPTURE=0 可关。 - orchestrator/swarm_runtime.py：record_benchmark_metrics → 落 run.metadata['benchmark']。 - benchmark/collectors/capture.py：collect(run_collector) → 持久化 → 可选导出(to_thread)。 - benchmark/export/：MetricsExporter；默认 NoopExporter(无依赖/无凭据)； BENCHMARK_EXPORT_TARGET=blob 启用 Azure Blob(连接串或 Workload/Managed Identity， 凭据经环境注入，绝不写进代码)，归档 <scenario>/<swarm_id>.json。 诚实：缺项指标 NaN→null，不伪造(规则#9)。 验证：新增 test-benchmark-capture.py + 现有 collector/selfcert + 契约冒烟(runtime/ merge/freeze)全 PASS。 影响范围：agent_swarm。运行时新增**只读**终态钩子(不改派发/执行/契约/计费/审计字段)； 导出默认关，无密钥落地。Client/Manager/Agnet/CodeGW/发布链路不涉及。 依赖：benchmark/ 需在镜像内(quality.py 早已 import benchmark；由 #44 Dockerfile 修复覆盖)。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
99 lines
3.8 KiB
Python
99 lines
3.8 KiB
Python
"""Test 阶段2 benchmark data capture: terminal run → SwarmMetrics persisted to run.metadata.
|
|
|
|
Hermetic (REDIS_FAKE), default NoopExporter (no storage dependency). Verifies:
|
|
- capture_run_metrics persists run.metadata['benchmark'] with metrics + coverage.
|
|
- real metrics are numbers; uncollected ones are null (NaN→null, never fabricated).
|
|
- default export is a no-op and does not fail the capture.
|
|
|
|
Run: python scripts/test-benchmark-capture.py
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
os.environ["REDIS_FAKE"] = "1"
|
|
os.environ.pop("BENCHMARK_EXPORT_TARGET", None) # ensure default Noop
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from orchestrator.redis_client import redis_client
|
|
from orchestrator import swarm_runtime as sr_mod
|
|
from orchestrator.swarm_runtime import swarm_runtime
|
|
from orchestrator.task_queue import task_queue, TaskStatus
|
|
from benchmark.collectors.capture import capture_run_metrics
|
|
|
|
failures = []
|
|
|
|
|
|
def check(name, cond):
|
|
print(("PASS" if cond else "FAIL"), "-", name)
|
|
if not cond:
|
|
failures.append(name)
|
|
|
|
|
|
async def _noop(self, *a, **k):
|
|
return None
|
|
|
|
|
|
async def add_task(run, task_id, *, status, agent, cost=0.0, retry=0, depends_on=None):
|
|
t = await task_queue.create_task(task_id=task_id, description=task_id,
|
|
agent_role=task_id.split("-")[-1], depends_on=depends_on or [],
|
|
enqueue=False)
|
|
t.status = status
|
|
t.assigned_agent_id = agent
|
|
t.retry_count = retry
|
|
t.result = json.dumps({"usage": {"model_cost_usd": cost}})
|
|
await task_queue._save_task(t)
|
|
await swarm_runtime.attach_task(run, t.task_id)
|
|
return t
|
|
|
|
|
|
async def main():
|
|
await redis_client.connect()
|
|
sr_mod.SwarmRuntime._post_callback = _noop
|
|
|
|
body = {
|
|
"mode": "swarm",
|
|
"requirement": {"objective": "capture test"},
|
|
"orchestration_plan": {"budget": {"max_cost_usd": 10}},
|
|
"callback": {"url": "http://127.0.0.1:9/cb", "subscribed_events": []},
|
|
"metadata": {"manager_deployment_id": "m-cap", "scenario": "coding"},
|
|
}
|
|
run, _ = await swarm_runtime.get_or_create_run(body=body, idempotency_key=None, correlation_id="c")
|
|
await add_task(run, "t-implementation", status=TaskStatus.COMPLETED, agent="A", cost=2.0)
|
|
await add_task(run, "t-testing", status=TaskStatus.COMPLETED, agent="B", cost=3.0)
|
|
|
|
payload = await capture_run_metrics(run)
|
|
|
|
# persisted onto the run
|
|
reloaded = await swarm_runtime.get_run(run.swarm_id)
|
|
bench = (reloaded.metadata or {}).get("benchmark")
|
|
check("run.metadata['benchmark'] persisted", isinstance(bench, dict))
|
|
check("payload scenario carried", bench.get("scenario") == "coding")
|
|
check("metrics dict present", isinstance(bench.get("metrics"), dict))
|
|
check("coverage dict present", isinstance(bench.get("coverage"), dict))
|
|
# s_completion real (2/2*100 = 100), serialized as a number
|
|
check("s_completion real (100)", bench["metrics"].get("s_completion") == 100.0
|
|
and bench["coverage"].get("s_completion") is True)
|
|
# uncollected metric (gain) → null, coverage False (no fabrication)
|
|
check("s_gain null + coverage False", bench["metrics"].get("s_gain") is None
|
|
and bench["coverage"].get("s_gain") is False)
|
|
# JSON-serializable (would raise on NaN-as-float being non-serializable only with allow_nan=False;
|
|
# we converted NaN→None, so strict dumps must succeed)
|
|
try:
|
|
json.dumps(payload, allow_nan=False)
|
|
check("payload strictly JSON-serializable (no NaN)", True)
|
|
except ValueError:
|
|
check("payload strictly JSON-serializable (no NaN)", False)
|
|
|
|
print()
|
|
if failures:
|
|
print(f"{len(failures)} capture check(s) FAILED: {failures}")
|
|
sys.exit(1)
|
|
print("all benchmark capture checks passed")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|