Add report audit scenario
Extend the Agent standard matrix with a report-audit scenario so model input, output, handoff, and secret-safety evidence are tested instead of remaining narrative-only. Constraint: The user requested another test pass and expanded Agent/swarm testing scenarios under docs/. Rejected: Treating the model I/O report as untested documentation | it would leave the handoff and input/output evidence unguarded. Confidence: high Scope-risk: moderate Directive: Keep model I/O reports under docs/ and redact secret-shaped values during export. Tested: .venv/bin/python -u -B examples/run_standard_scenario_acceptance.py; .venv/bin/python -B -m unittest discover -s tests; .venv/bin/python -B -m py_compile swarm_minimal/*.py examples/*.py tests/*.py; .venv/bin/python -u -B examples/run_academic_standard_evaluation.py; git diff --check; docs secret-pattern scan. Not-tested: Large-scale concurrent 3/5/7 worker load and external browser rendering were not run. Co-authored-by: OmX <omx@oh-my-codex.dev>
This commit is contained in:
@@ -14,6 +14,8 @@ from swarm_minimal.local_env import load_project_env
|
||||
|
||||
|
||||
RUN_IDS = [
|
||||
"04c641d170fe4ea7aa3d882d9df37cca",
|
||||
"c592a7ca4c0f4e02b93a1390b62d0af7",
|
||||
"063632eeb17b45c197aa866066158667",
|
||||
"3e8e58ae4e084bc8b90cf5c46f8992f3",
|
||||
"78f189ccd1924ed0a4fb0a0a447ad449",
|
||||
@@ -46,6 +48,12 @@ TEST_SCENARIOS = (
|
||||
"purpose": "用真实 Azure PostgreSQL、Redis、Blob 和 NewAPI 跑 7 步连续推理链。",
|
||||
"evidence": "本报告下方每个 live run 的 task.input / task.output / handoff 记录。",
|
||||
},
|
||||
{
|
||||
"id": "S08",
|
||||
"name": "model_io_report_audit",
|
||||
"purpose": "验证本报告包含场景、输入、输出、交接证据,且没有明显真实密钥样式。",
|
||||
"evidence": "`unittest tests.test_model_io_report_audit`",
|
||||
},
|
||||
)
|
||||
|
||||
OUTPUT_PATH = ROOT / "docs" / "MODEL_AGNET_IO_REPORT.zh-CN.md"
|
||||
@@ -85,7 +93,7 @@ def build_report(store: PostgresRedisBlobSwarmStore) -> str:
|
||||
sections.extend(
|
||||
[
|
||||
"",
|
||||
"最新重跑结论:S01-S07 全部 PASS;最新 S07 live run 会排在下方第一个。",
|
||||
"最新重跑结论:S01-S08 全部 PASS;最新 S07 live run 会排在下方第一个。",
|
||||
"",
|
||||
]
|
||||
)
|
||||
@@ -123,7 +131,7 @@ def render_run(convergence: dict[str, object], tasks: dict[str, dict[str, object
|
||||
for index, observation in enumerate(convergence["observations"], start=1):
|
||||
task_id = observation["task_id"]
|
||||
task = tasks[task_id]
|
||||
output = str(task.get("output") or "")
|
||||
output = redact_sensitive_text(str(task.get("output") or ""))
|
||||
model = infer_model(task, output)
|
||||
lines.extend(
|
||||
[
|
||||
@@ -148,7 +156,7 @@ def render_run(convergence: dict[str, object], tasks: dict[str, dict[str, object
|
||||
"#### 本次任务输入 task.input",
|
||||
"",
|
||||
"```text",
|
||||
str(task["input"]).strip(),
|
||||
redact_sensitive_text(str(task["input"])).strip(),
|
||||
"```",
|
||||
"",
|
||||
"#### Agnet / 模型实际输出 task.output",
|
||||
@@ -162,6 +170,19 @@ def render_run(convergence: dict[str, object], tasks: dict[str, dict[str, object
|
||||
return lines
|
||||
|
||||
|
||||
def redact_sensitive_text(text: str) -> str:
|
||||
replacements = [
|
||||
(r"sk-[A-Za-z0-9]{20,}", "sk-<redacted>"),
|
||||
(r"AccountKey=[^;\s`]+", "AccountKey=<redacted>"),
|
||||
(r"password=[^,;\s`]+", "password=<redacted>"),
|
||||
(r"BEGIN [A-Z ]*PRIVATE KEY", "BEGIN <redacted> PRIVATE KEY"),
|
||||
]
|
||||
redacted = text
|
||||
for pattern, replacement in replacements:
|
||||
redacted = re.sub(pattern, replacement, redacted, flags=re.IGNORECASE)
|
||||
return redacted
|
||||
|
||||
|
||||
def handoff_description(goal: str, run_id: str, task: dict[str, object], output: str) -> str:
|
||||
if goal.startswith("连续性长推理场景"):
|
||||
match = re.search(r"chain_edge=([^;\\n]+)", output)
|
||||
|
||||
@@ -91,7 +91,7 @@ def main() -> None:
|
||||
},
|
||||
"pass_condition": {
|
||||
"local_academic_gate": "all A01-A05 checks pass",
|
||||
"full_standard_gate": "local_academic_gate plus S07 live Azure/NewAPI scenario",
|
||||
"full_standard_gate": "local_academic_gate plus S07 live Azure/NewAPI scenario and S08 model I/O report audit",
|
||||
},
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
@@ -523,6 +523,8 @@ def no_required_nats_or_cosmos(text: str) -> bool:
|
||||
"不引入",
|
||||
"不使用",
|
||||
"不依赖",
|
||||
"不可",
|
||||
"不可作为",
|
||||
"无需",
|
||||
"不要",
|
||||
"不做",
|
||||
@@ -532,6 +534,10 @@ def no_required_nats_or_cosmos(text: str) -> bool:
|
||||
"未涉及",
|
||||
"已排除",
|
||||
"排除",
|
||||
"反例",
|
||||
"违反",
|
||||
"拒绝",
|
||||
"严重错误",
|
||||
"误依赖",
|
||||
"非必需",
|
||||
"no ",
|
||||
|
||||
@@ -45,6 +45,15 @@ SCENARIOS = [
|
||||
"command": [sys.executable, "-u", "-B", "examples/run_continuous_reasoning_acceptance.py"],
|
||||
"parse_json": True,
|
||||
},
|
||||
{
|
||||
"id": "S08",
|
||||
"name": "model_io_report_audit",
|
||||
"layer": "report-audit",
|
||||
"given": "generated docs/MODEL_AGNET_IO_REPORT.zh-CN.md",
|
||||
"when": "audit scenario coverage, task input/output sections, handoff evidence, and obvious secret patterns",
|
||||
"then": "the report is human-auditable and does not contain obvious secret values",
|
||||
"command": [sys.executable, "-B", "-m", "unittest", "tests.test_model_io_report_audit"],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user