Add report audit scenario

Extend the Agent standard matrix with a report-audit scenario so model input, output, handoff, and secret-safety evidence are tested instead of remaining narrative-only.

Constraint: The user requested another test pass and expanded Agent/swarm testing scenarios under docs/.

Rejected: Treating the model I/O report as untested documentation | it would leave the handoff and input/output evidence unguarded.

Confidence: high

Scope-risk: moderate

Directive: Keep model I/O reports under docs/ and redact secret-shaped values during export.

Tested: .venv/bin/python -u -B examples/run_standard_scenario_acceptance.py; .venv/bin/python -B -m unittest discover -s tests; .venv/bin/python -B -m py_compile swarm_minimal/*.py examples/*.py tests/*.py; .venv/bin/python -u -B examples/run_academic_standard_evaluation.py; git diff --check; docs secret-pattern scan.

Not-tested: Large-scale concurrent 3/5/7 worker load and external browser rendering were not run.

Co-authored-by: OmX <omx@oh-my-codex.dev>
This commit is contained in:
gongzhiyong
2026-05-16 15:17:47 +08:00
co-authored by OmX
parent e4997efeb1
commit d632fd9f64
13 changed files with 1919 additions and 21 deletions
+24 -3
View File
@@ -14,6 +14,8 @@ from swarm_minimal.local_env import load_project_env
RUN_IDS = [
"04c641d170fe4ea7aa3d882d9df37cca",
"c592a7ca4c0f4e02b93a1390b62d0af7",
"063632eeb17b45c197aa866066158667",
"3e8e58ae4e084bc8b90cf5c46f8992f3",
"78f189ccd1924ed0a4fb0a0a447ad449",
@@ -46,6 +48,12 @@ TEST_SCENARIOS = (
"purpose": "用真实 Azure PostgreSQL、Redis、Blob 和 NewAPI 跑 7 步连续推理链。",
"evidence": "本报告下方每个 live run 的 task.input / task.output / handoff 记录。",
},
{
"id": "S08",
"name": "model_io_report_audit",
"purpose": "验证本报告包含场景、输入、输出、交接证据,且没有明显真实密钥样式。",
"evidence": "`unittest tests.test_model_io_report_audit`",
},
)
OUTPUT_PATH = ROOT / "docs" / "MODEL_AGNET_IO_REPORT.zh-CN.md"
@@ -85,7 +93,7 @@ def build_report(store: PostgresRedisBlobSwarmStore) -> str:
sections.extend(
[
"",
"最新重跑结论:S01-S07 全部 PASS;最新 S07 live run 会排在下方第一个。",
"最新重跑结论:S01-S08 全部 PASS;最新 S07 live run 会排在下方第一个。",
"",
]
)
@@ -123,7 +131,7 @@ def render_run(convergence: dict[str, object], tasks: dict[str, dict[str, object
for index, observation in enumerate(convergence["observations"], start=1):
task_id = observation["task_id"]
task = tasks[task_id]
output = str(task.get("output") or "")
output = redact_sensitive_text(str(task.get("output") or ""))
model = infer_model(task, output)
lines.extend(
[
@@ -148,7 +156,7 @@ def render_run(convergence: dict[str, object], tasks: dict[str, dict[str, object
"#### 本次任务输入 task.input",
"",
"```text",
str(task["input"]).strip(),
redact_sensitive_text(str(task["input"])).strip(),
"```",
"",
"#### Agnet / 模型实际输出 task.output",
@@ -162,6 +170,19 @@ def render_run(convergence: dict[str, object], tasks: dict[str, dict[str, object
return lines
def redact_sensitive_text(text: str) -> str:
replacements = [
(r"sk-[A-Za-z0-9]{20,}", "sk-<redacted>"),
(r"AccountKey=[^;\s`]+", "AccountKey=<redacted>"),
(r"password=[^,;\s`]+", "password=<redacted>"),
(r"BEGIN [A-Z ]*PRIVATE KEY", "BEGIN <redacted> PRIVATE KEY"),
]
redacted = text
for pattern, replacement in replacements:
redacted = re.sub(pattern, replacement, redacted, flags=re.IGNORECASE)
return redacted
def handoff_description(goal: str, run_id: str, task: dict[str, object], output: str) -> str:
if goal.startswith("连续性长推理场景"):
match = re.search(r"chain_edge=([^;\\n]+)", output)
+1 -1
View File
@@ -91,7 +91,7 @@ def main() -> None:
},
"pass_condition": {
"local_academic_gate": "all A01-A05 checks pass",
"full_standard_gate": "local_academic_gate plus S07 live Azure/NewAPI scenario",
"full_standard_gate": "local_academic_gate plus S07 live Azure/NewAPI scenario and S08 model I/O report audit",
},
}
print(json.dumps(report, ensure_ascii=False, indent=2))
@@ -523,6 +523,8 @@ def no_required_nats_or_cosmos(text: str) -> bool:
"不引入",
"不使用",
"不依赖",
"不可",
"不可作为",
"无需",
"不要",
"不做",
@@ -532,6 +534,10 @@ def no_required_nats_or_cosmos(text: str) -> bool:
"未涉及",
"已排除",
"排除",
"反例",
"违反",
"拒绝",
"严重错误",
"误依赖",
"非必需",
"no ",
@@ -45,6 +45,15 @@ SCENARIOS = [
"command": [sys.executable, "-u", "-B", "examples/run_continuous_reasoning_acceptance.py"],
"parse_json": True,
},
{
"id": "S08",
"name": "model_io_report_audit",
"layer": "report-audit",
"given": "generated docs/MODEL_AGNET_IO_REPORT.zh-CN.md",
"when": "audit scenario coverage, task input/output sections, handoff evidence, and obvious secret patterns",
"then": "the report is human-auditable and does not contain obvious secret values",
"command": [sys.executable, "-B", "-m", "unittest", "tests.test_model_io_report_audit"],
},
]