Extend the Agent standard matrix with a report-audit scenario so model input, output, handoff, and secret-safety evidence are tested instead of remaining narrative-only. Constraint: The user requested another test pass and expanded Agent/swarm testing scenarios under docs/. Rejected: Treating the model I/O report as untested documentation | it would leave the handoff and input/output evidence unguarded. Confidence: high Scope-risk: moderate Directive: Keep model I/O reports under docs/ and redact secret-shaped values during export. Tested: .venv/bin/python -u -B examples/run_standard_scenario_acceptance.py; .venv/bin/python -B -m unittest discover -s tests; .venv/bin/python -B -m py_compile swarm_minimal/*.py examples/*.py tests/*.py; .venv/bin/python -u -B examples/run_academic_standard_evaluation.py; git diff --check; docs secret-pattern scan. Not-tested: Large-scale concurrent 3/5/7 worker load and external browser rendering were not run. Co-authored-by: OmX <omx@oh-my-codex.dev>
99 lines
5.4 KiB
Python
99 lines
5.4 KiB
Python
import unittest
|
|
|
|
from examples import run_continuous_reasoning_acceptance as continuous
|
|
from swarm_minimal.core import Agent, InMemorySwarmStore, SwarmCoordinator, Task, TaskStatus
|
|
|
|
|
|
class StandardScenarioTest(unittest.TestCase):
|
|
def test_continuity_markers_reject_missing_previous_step(self) -> None:
|
|
outputs_by_kind = {}
|
|
for index, step in enumerate(continuous.CHAIN_STEPS):
|
|
previous_marker = "START" if index == 0 else continuous.CHAIN_STEPS[index - 1]["marker"]
|
|
outputs_by_kind[step["capability"]] = f"{previous_marker} -> {step['marker']} 不变量 风险 下一步"
|
|
|
|
self.assertTrue(continuous.outputs_have_markers_and_links(outputs_by_kind))
|
|
|
|
broken = dict(outputs_by_kind)
|
|
broken["chain_step_04"] = "STEP-04 不变量 风险 下一步"
|
|
self.assertFalse(continuous.outputs_have_markers_and_links(broken))
|
|
|
|
def test_nats_and_cosmos_policy_rejects_required_dependencies_only(self) -> None:
|
|
self.assertTrue(continuous.no_required_nats_or_cosmos("MVP 不依赖 NATS/Cosmos,仅使用 PostgreSQL + Redis + Blob。"))
|
|
self.assertTrue(continuous.no_required_nats_or_cosmos("NATS 和 Cosmos 未在 MVP 中涉及。"))
|
|
self.assertTrue(continuous.no_required_nats_or_cosmos("NATS 和 Cosmos 不可作为 MVP 依赖。"))
|
|
self.assertTrue(continuous.no_required_nats_or_cosmos("反例:使用 NATS 作为必须依赖,违反验收标准。"))
|
|
self.assertFalse(continuous.no_required_nats_or_cosmos("MVP 必须依赖 NATS 才能完成任务队列。"))
|
|
self.assertFalse(continuous.no_required_nats_or_cosmos("需要引入 Cosmos 作为任务状态库。"))
|
|
|
|
def test_final_step_scores_above_intermediate_step(self) -> None:
|
|
final = (
|
|
"STEP-07 基于 STEP-06 不变量 风险 下一步 NEWAPI_MODEL 模型发现 验收 "
|
|
"./.venv/bin/python unittest "
|
|
+ " ".join(continuous.TARGET_FILES[:5])
|
|
)
|
|
step_six = (
|
|
"STEP-06 基于 STEP-05 不变量 风险 下一步 NEWAPI_MODEL 模型发现 复杂度 "
|
|
+ " ".join(continuous.TARGET_FILES[:4])
|
|
)
|
|
|
|
self.assertEqual(continuous.score_output(final, 6), 1.0)
|
|
self.assertLess(continuous.score_output(step_six, 5), continuous.score_output(final, 6))
|
|
|
|
def test_deterministic_seven_step_chain_updates_cursor_edges_and_summaries(self) -> None:
|
|
store = InMemorySwarmStore()
|
|
run_id = "deterministic-chain"
|
|
store.shared_state[f"run:{run_id}:goal"] = "standard continuous reasoning"
|
|
store.shared_state[f"run:{run_id}:status"] = "running"
|
|
store.shared_state[f"chain:{run_id}:cursor"] = "START"
|
|
|
|
agents = []
|
|
for index, step in enumerate(continuous.CHAIN_STEPS):
|
|
previous_marker = "START" if index == 0 else continuous.CHAIN_STEPS[index - 1]["marker"]
|
|
store.add_task(Task(kind=step["capability"], input=step["title"]))
|
|
|
|
def run(task, shared_state, step=step, index=index, previous_marker=previous_marker):
|
|
output = (
|
|
f"chain_edge={previous_marker}->{step['marker']} "
|
|
f"{step['marker']} 基于 {previous_marker} 不变量 风险 下一步 NEWAPI_MODEL 模型发现 "
|
|
+ " ".join(continuous.TARGET_FILES[:5])
|
|
)
|
|
shared_state[f"chain:{run_id}:{step['marker']}:summary"] = output[:180]
|
|
shared_state[f"chain:{run_id}:cursor"] = step["marker"]
|
|
shared_state[f"chain:{run_id}:edge:{previous_marker}->{step['marker']}"] = "done"
|
|
return output, 0.7 + index * 0.03
|
|
|
|
agents.append(Agent(id=f"deterministic-{index + 1}", capability=step["capability"], run=run))
|
|
|
|
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
|
|
|
self.assertEqual(result.completed_tasks, 7)
|
|
self.assertEqual(store.shared_state[f"chain:{run_id}:cursor"], "STEP-07")
|
|
for index, step in enumerate(continuous.CHAIN_STEPS):
|
|
previous_marker = "START" if index == 0 else continuous.CHAIN_STEPS[index - 1]["marker"]
|
|
self.assertEqual(store.shared_state[f"chain:{run_id}:edge:{previous_marker}->{step['marker']}"], "done")
|
|
self.assertIn(f"chain:{run_id}:{step['marker']}:summary", store.shared_state)
|
|
self.assertTrue(all(task.status == TaskStatus.DONE for task in store.tasks.values()))
|
|
|
|
def test_failed_agent_records_failed_task_and_negative_pheromone(self) -> None:
|
|
store = InMemorySwarmStore()
|
|
run_id = "failure-injection"
|
|
store.shared_state[f"run:{run_id}:goal"] = "failure injection"
|
|
store.add_task(Task(kind="ok", input="complete"))
|
|
store.add_task(Task(kind="fail", input="raise"))
|
|
agents = [
|
|
Agent(id="ok-agent", capability="ok", run=lambda task, _: ("ok output", 0.8)),
|
|
Agent(id="bad-agent", capability="fail", run=lambda task, _: (_ for _ in ()).throw(RuntimeError("forced failure"))),
|
|
]
|
|
|
|
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
|
|
|
failed = [task for task in store.tasks.values() if task.kind == "fail"][0]
|
|
self.assertEqual(failed.status, TaskStatus.FAILED)
|
|
self.assertIn("forced failure", failed.error or "")
|
|
self.assertLess(store.pheromones[failed.id], 0)
|
|
self.assertEqual(result.completed_tasks, 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|