Files
fengqun/tests/test_standard_scenarios.py
T
gongzhiyongandOmX d632fd9f64 Add report audit scenario
Extend the Agent standard matrix with a report-audit scenario so model input, output, handoff, and secret-safety evidence are tested instead of remaining narrative-only.

Constraint: The user requested another test pass and expanded Agent/swarm testing scenarios under docs/.

Rejected: Treating the model I/O report as untested documentation | it would leave the handoff and input/output evidence unguarded.

Confidence: high

Scope-risk: moderate

Directive: Keep model I/O reports under docs/ and redact secret-shaped values during export.

Tested: .venv/bin/python -u -B examples/run_standard_scenario_acceptance.py; .venv/bin/python -B -m unittest discover -s tests; .venv/bin/python -B -m py_compile swarm_minimal/*.py examples/*.py tests/*.py; .venv/bin/python -u -B examples/run_academic_standard_evaluation.py; git diff --check; docs secret-pattern scan.

Not-tested: Large-scale concurrent 3/5/7 worker load and external browser rendering were not run.

Co-authored-by: OmX <omx@oh-my-codex.dev>
2026-05-16 15:17:47 +08:00

99 lines
5.4 KiB
Python

import unittest
from examples import run_continuous_reasoning_acceptance as continuous
from swarm_minimal.core import Agent, InMemorySwarmStore, SwarmCoordinator, Task, TaskStatus
class StandardScenarioTest(unittest.TestCase):
def test_continuity_markers_reject_missing_previous_step(self) -> None:
outputs_by_kind = {}
for index, step in enumerate(continuous.CHAIN_STEPS):
previous_marker = "START" if index == 0 else continuous.CHAIN_STEPS[index - 1]["marker"]
outputs_by_kind[step["capability"]] = f"{previous_marker} -> {step['marker']} 不变量 风险 下一步"
self.assertTrue(continuous.outputs_have_markers_and_links(outputs_by_kind))
broken = dict(outputs_by_kind)
broken["chain_step_04"] = "STEP-04 不变量 风险 下一步"
self.assertFalse(continuous.outputs_have_markers_and_links(broken))
def test_nats_and_cosmos_policy_rejects_required_dependencies_only(self) -> None:
self.assertTrue(continuous.no_required_nats_or_cosmos("MVP 不依赖 NATS/Cosmos,仅使用 PostgreSQL + Redis + Blob。"))
self.assertTrue(continuous.no_required_nats_or_cosmos("NATS 和 Cosmos 未在 MVP 中涉及。"))
self.assertTrue(continuous.no_required_nats_or_cosmos("NATS 和 Cosmos 不可作为 MVP 依赖。"))
self.assertTrue(continuous.no_required_nats_or_cosmos("反例:使用 NATS 作为必须依赖,违反验收标准。"))
self.assertFalse(continuous.no_required_nats_or_cosmos("MVP 必须依赖 NATS 才能完成任务队列。"))
self.assertFalse(continuous.no_required_nats_or_cosmos("需要引入 Cosmos 作为任务状态库。"))
def test_final_step_scores_above_intermediate_step(self) -> None:
final = (
"STEP-07 基于 STEP-06 不变量 风险 下一步 NEWAPI_MODEL 模型发现 验收 "
"./.venv/bin/python unittest "
+ " ".join(continuous.TARGET_FILES[:5])
)
step_six = (
"STEP-06 基于 STEP-05 不变量 风险 下一步 NEWAPI_MODEL 模型发现 复杂度 "
+ " ".join(continuous.TARGET_FILES[:4])
)
self.assertEqual(continuous.score_output(final, 6), 1.0)
self.assertLess(continuous.score_output(step_six, 5), continuous.score_output(final, 6))
def test_deterministic_seven_step_chain_updates_cursor_edges_and_summaries(self) -> None:
store = InMemorySwarmStore()
run_id = "deterministic-chain"
store.shared_state[f"run:{run_id}:goal"] = "standard continuous reasoning"
store.shared_state[f"run:{run_id}:status"] = "running"
store.shared_state[f"chain:{run_id}:cursor"] = "START"
agents = []
for index, step in enumerate(continuous.CHAIN_STEPS):
previous_marker = "START" if index == 0 else continuous.CHAIN_STEPS[index - 1]["marker"]
store.add_task(Task(kind=step["capability"], input=step["title"]))
def run(task, shared_state, step=step, index=index, previous_marker=previous_marker):
output = (
f"chain_edge={previous_marker}->{step['marker']} "
f"{step['marker']} 基于 {previous_marker} 不变量 风险 下一步 NEWAPI_MODEL 模型发现 "
+ " ".join(continuous.TARGET_FILES[:5])
)
shared_state[f"chain:{run_id}:{step['marker']}:summary"] = output[:180]
shared_state[f"chain:{run_id}:cursor"] = step["marker"]
shared_state[f"chain:{run_id}:edge:{previous_marker}->{step['marker']}"] = "done"
return output, 0.7 + index * 0.03
agents.append(Agent(id=f"deterministic-{index + 1}", capability=step["capability"], run=run))
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
self.assertEqual(result.completed_tasks, 7)
self.assertEqual(store.shared_state[f"chain:{run_id}:cursor"], "STEP-07")
for index, step in enumerate(continuous.CHAIN_STEPS):
previous_marker = "START" if index == 0 else continuous.CHAIN_STEPS[index - 1]["marker"]
self.assertEqual(store.shared_state[f"chain:{run_id}:edge:{previous_marker}->{step['marker']}"], "done")
self.assertIn(f"chain:{run_id}:{step['marker']}:summary", store.shared_state)
self.assertTrue(all(task.status == TaskStatus.DONE for task in store.tasks.values()))
def test_failed_agent_records_failed_task_and_negative_pheromone(self) -> None:
store = InMemorySwarmStore()
run_id = "failure-injection"
store.shared_state[f"run:{run_id}:goal"] = "failure injection"
store.add_task(Task(kind="ok", input="complete"))
store.add_task(Task(kind="fail", input="raise"))
agents = [
Agent(id="ok-agent", capability="ok", run=lambda task, _: ("ok output", 0.8)),
Agent(id="bad-agent", capability="fail", run=lambda task, _: (_ for _ in ()).throw(RuntimeError("forced failure"))),
]
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
failed = [task for task in store.tasks.values() if task.kind == "fail"][0]
self.assertEqual(failed.status, TaskStatus.FAILED)
self.assertIn("forced failure", failed.error or "")
self.assertLess(store.pheromones[failed.id], 0)
self.assertEqual(result.completed_tasks, 1)
if __name__ == "__main__":
unittest.main()