FastheiandClaude Opus 4.8
27739af446
阶段3 数据落库：BENCHMARK_EXPORT_TARGET 支持逗号组合 cosmos,blob（默认 none）。 - CosmosExporter：Cosmos DB for NoSQL，一 run 一文档(id=swarm_id，分区键 /swarm_id)。 - BlobExporter：存储账户归档 <scenario>/<swarm_id>.json。 - get_exporters() 解析多 target；某 target 缺凭据/未知则跳过+告警，不影响其它与 run。 - capture 对每个 exporter 独立 to_thread 导出，互不影响。 - orchestrator/requirements：加 azure-cosmos/azure-storage-blob/azure-identity(懒导入)。 安全：所有连接串/AccountKey 仅从环境读(经 Secret/secret_ref 注入)， 绝不写进代码/日志/提交(组织安全规则)。 验证：新增 test-benchmark-export.py(target 解析/隔离/doc 成形) + capture/selfcert/ collector 回归全 PASS。 影响范围：agent_swarm benchmark 导出层 + 依赖；运行时只读钩子不变；导出默认关、无密钥落地。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
86 lines
3.2 KiB
Python
86 lines
3.2 KiB
Python
"""Test benchmark export target selection + doc shaping (no network).
|
|
|
|
Verifies:
|
|
- BENCHMARK_EXPORT_TARGET parsing: default→noop, comma list→multiple, unknown ignored.
|
|
- a misconfigured target (missing creds) is skipped, not fatal, and never empties the list.
|
|
- CosmosExporter.build_doc shapes a Cosmos doc (string id = swarm_id; partition field present).
|
|
- constructors do NOT connect (no creds touched until export()).
|
|
|
|
Run: python scripts/test-benchmark-export.py
|
|
"""
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from benchmark.export import get_exporters, NoopExporter, CosmosExporter, BlobExporter
|
|
|
|
failures = []
|
|
_ENV = ("BENCHMARK_EXPORT_TARGET", "BENCHMARK_COSMOS_CONNECTION_STRING",
|
|
"BENCHMARK_COSMOS_DATABASE", "BENCHMARK_COSMOS_CONTAINER",
|
|
"AZURE_STORAGE_CONNECTION_STRING", "BENCHMARK_BLOB_ACCOUNT_URL", "BENCHMARK_BLOB_CONTAINER")
|
|
|
|
|
|
def check(name, cond):
|
|
print(("PASS" if cond else "FAIL"), "-", name)
|
|
if not cond:
|
|
failures.append(name)
|
|
|
|
|
|
def reset(**env):
|
|
for k in _ENV:
|
|
os.environ.pop(k, None)
|
|
os.environ.update(env)
|
|
|
|
|
|
def names(exps):
|
|
return [e.name for e in exps]
|
|
|
|
|
|
def main():
|
|
# default → noop
|
|
reset()
|
|
check("default → [noop]", names(get_exporters()) == ["noop"])
|
|
|
|
# explicit none
|
|
reset(BENCHMARK_EXPORT_TARGET="none")
|
|
check("none → [noop]", names(get_exporters()) == ["noop"])
|
|
|
|
# cosmos + blob, both with creds present → both built
|
|
reset(BENCHMARK_EXPORT_TARGET="cosmos,blob",
|
|
BENCHMARK_COSMOS_CONNECTION_STRING="AccountEndpoint=https://x.documents.azure.com:443/;AccountKey=FAKE==;",
|
|
AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=https;AccountName=x;AccountKey=FAKE==;")
|
|
check("cosmos,blob → [cosmos, blob]", names(get_exporters()) == ["cosmos", "blob"])
|
|
|
|
# cosmos requested but NO connection string → skipped (not fatal); blob still built
|
|
reset(BENCHMARK_EXPORT_TARGET="cosmos,blob",
|
|
AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=https;AccountName=x;AccountKey=FAKE==;")
|
|
check("misconfigured cosmos skipped, blob kept", names(get_exporters()) == ["blob"])
|
|
|
|
# all targets misconfigured → never empty, falls back to noop
|
|
reset(BENCHMARK_EXPORT_TARGET="cosmos")
|
|
check("all misconfigured → [noop] fallback", names(get_exporters()) == ["noop"])
|
|
|
|
# unknown target ignored → noop fallback
|
|
reset(BENCHMARK_EXPORT_TARGET="clickhouse")
|
|
check("unknown target → [noop] fallback", names(get_exporters()) == ["noop"])
|
|
|
|
# CosmosExporter.build_doc: string id = swarm_id, partition field swarm_id retained
|
|
doc = CosmosExporter.build_doc({"swarm_id": "swarm-123", "scenario": "coding", "metrics": {"s_gain": None}})
|
|
check("cosmos doc id == swarm_id (str)", doc["id"] == "swarm-123" and isinstance(doc["id"], str))
|
|
check("cosmos doc keeps swarm_id partition field", doc["swarm_id"] == "swarm-123")
|
|
check("cosmos doc preserves null metric (no fabrication)", doc["metrics"]["s_gain"] is None)
|
|
|
|
reset()
|
|
print()
|
|
if failures:
|
|
print(f"{len(failures)} export check(s) FAILED: {failures}")
|
|
return 1
|
|
print("all benchmark export checks passed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|