gongzhiyong
111be3e435
The standalone prototype should be the root-level project shape for fengqun while preserving the existing planning documents already at the root. This keeps README, examples, tests, and the Python package directly discoverable without deleting the prior docs. Constraint: User clarified that swarm-minimal is the repository root, but other existing root files must remain. Rejected: Deleting existing root docs | They are part of the fengqun repository context and were explicitly protected. Confidence: high Scope-risk: narrow Directive: Keep secrets in ignored .env only; do not commit live credentials. Tested: python3 -B -m unittest discover -s tests; git diff --check; secret-pattern scan showed only placeholders/test values/task-id false positives. Not-tested: Remote web UI rendering after push.
246 lines
10 KiB
Python
246 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
import json
|
|
import sys
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from swarm_minimal.core import Agent, InMemorySwarmStore, SwarmCoordinator, Task, TaskStatus
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Metric:
|
|
name: str
|
|
traditional: float
|
|
swarm: float
|
|
unit: str
|
|
|
|
@property
|
|
def delta(self) -> float:
|
|
return self.swarm - self.traditional
|
|
|
|
@property
|
|
def relative_gain_percent(self) -> float | None:
|
|
if self.traditional == 0:
|
|
return None
|
|
return (self.swarm - self.traditional) / self.traditional * 100
|
|
|
|
|
|
def main() -> None:
|
|
report = run_benchmark()
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
if report["status"] != "PASS":
|
|
raise SystemExit(1)
|
|
|
|
|
|
def run_benchmark() -> dict[str, object]:
|
|
scenarios = [
|
|
fault_isolation(),
|
|
emergent_consensus(),
|
|
pheromone_efficiency(),
|
|
handoff_context_retention(),
|
|
]
|
|
metric_pairs = [scenario["metric"] for scenario in scenarios]
|
|
traditional_score = sum(metric.traditional for metric in metric_pairs) / len(metric_pairs)
|
|
swarm_score = sum(metric.swarm for metric in metric_pairs) / len(metric_pairs)
|
|
overall_gain = (swarm_score - traditional_score) / traditional_score * 100
|
|
return {
|
|
"standard": "swarm-vs-traditional-deterministic-benchmark-v1",
|
|
"status": "PASS" if all(scenario["passed"] for scenario in scenarios) else "FAIL",
|
|
"baseline_definition": {
|
|
"traditional_agnet": [
|
|
"single agent fails closed when its one route fails",
|
|
"best-of local answers without shared-state aggregation",
|
|
"FIFO task selection without pheromone feedback",
|
|
"stateless handoff without active-agent/payload continuity",
|
|
],
|
|
"swarm_agnet": [
|
|
"redundant agents share task pool and converge despite a failed individual",
|
|
"local observations accumulate through shared_state",
|
|
"pheromone scores bias claim order and final selection",
|
|
"handoff records active agent, transfer target, and payload continuity",
|
|
],
|
|
},
|
|
"scenarios": [serialize_scenario(scenario) for scenario in scenarios],
|
|
"overall_normalized_score": {
|
|
"traditional": round(traditional_score, 4),
|
|
"swarm": round(swarm_score, 4),
|
|
"relative_gain_percent": round(overall_gain, 1),
|
|
"ratio": round(swarm_score / traditional_score, 2),
|
|
"note": "This aggregate is a deterministic academic benchmark over four selected swarm properties, not a universal production claim.",
|
|
},
|
|
}
|
|
|
|
|
|
def fault_isolation() -> dict[str, object]:
|
|
traditional_success = traditional_single_agent_failure()
|
|
swarm_success, completed, failed = swarm_failure_isolation()
|
|
return {
|
|
"id": "C01",
|
|
"name": "fault_isolation",
|
|
"why": "A swarm should continue when one Agnet fails; a traditional single route usually fails closed.",
|
|
"metric": Metric("completion_success", float(traditional_success), float(swarm_success), "0_or_1"),
|
|
"passed": not traditional_success and swarm_success and completed == 2 and failed == 1,
|
|
"details": {
|
|
"traditional_result": "failed before convergence",
|
|
"swarm_completed_tasks": completed,
|
|
"swarm_failed_tasks": failed,
|
|
"interpretation": "+100 percentage points success; relative gain is undefined because the baseline is 0.",
|
|
},
|
|
}
|
|
|
|
|
|
def traditional_single_agent_failure() -> bool:
|
|
store = InMemorySwarmStore()
|
|
run_id = "compare-traditional-failure"
|
|
store.shared_state[f"run:{run_id}:goal"] = "single route"
|
|
store.shared_state[f"run:{run_id}:status"] = "running"
|
|
store.add_task(Task(kind="route", input="fragile-route"))
|
|
agents = [
|
|
Agent(
|
|
id="single-agnet",
|
|
capability="route",
|
|
run=lambda task, _: (_ for _ in ()).throw(RuntimeError("single agnet crashed")),
|
|
)
|
|
]
|
|
try:
|
|
SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
|
except RuntimeError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def swarm_failure_isolation() -> tuple[bool, int, int]:
|
|
store = InMemorySwarmStore()
|
|
run_id = "compare-swarm-failure"
|
|
store.shared_state[f"run:{run_id}:goal"] = "redundant routes"
|
|
store.shared_state[f"run:{run_id}:status"] = "running"
|
|
for item in ["fragile-route", "robust-route-a", "robust-route-b"]:
|
|
store.add_task(Task(kind="route", input=item))
|
|
agents = [
|
|
Agent(
|
|
id="crashing-agnet",
|
|
capability="route",
|
|
run=lambda task, _: (_ for _ in ()).throw(RuntimeError("single agnet crashed")),
|
|
),
|
|
Agent(id="backup-agnet-a", capability="route", run=lambda task, _: ("healthy result", 0.91)),
|
|
Agent(id="backup-agnet-b", capability="route", run=lambda task, _: ("alternative result", 0.86)),
|
|
]
|
|
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
|
failed = len([task for task in store.tasks.values() if task.status == TaskStatus.FAILED])
|
|
done = len([task for task in store.tasks.values() if task.status == TaskStatus.DONE])
|
|
return bool(result.accepted_output), done, failed
|
|
|
|
|
|
def emergent_consensus() -> dict[str, object]:
|
|
local_values = {"alpha": [0.31], "beta": [0.33, 0.34], "gamma": [0.45]}
|
|
traditional_best_candidate = max(
|
|
((candidate, max(values)) for candidate, values in local_values.items()),
|
|
key=lambda item: item[1],
|
|
)
|
|
swarm_totals = {candidate: sum(values) for candidate, values in local_values.items()}
|
|
swarm_best_candidate = max(swarm_totals.items(), key=lambda item: item[1])
|
|
traditional = traditional_best_candidate[1]
|
|
swarm = swarm_best_candidate[1]
|
|
return {
|
|
"id": "C02",
|
|
"name": "emergent_consensus",
|
|
"why": "Emergence means weak local observations can combine into a stronger group-level answer.",
|
|
"metric": Metric("accepted_quality_score", traditional, swarm, "score"),
|
|
"passed": traditional_best_candidate[0] == "gamma" and swarm_best_candidate[0] == "beta" and swarm > traditional,
|
|
"details": {
|
|
"traditional_best_single": {"candidate": traditional_best_candidate[0], "score": traditional},
|
|
"swarm_aggregated_best": {"candidate": swarm_best_candidate[0], "score": swarm},
|
|
"interpretation": f"+{((swarm - traditional) / traditional * 100):.1f}% accepted score through shared-state aggregation.",
|
|
},
|
|
}
|
|
|
|
|
|
def pheromone_efficiency() -> dict[str, object]:
|
|
traditional_order = ["low-signal", "medium-signal", "high-signal"]
|
|
pheromone_order = ["high-signal", "medium-signal", "low-signal"]
|
|
quality = {
|
|
"low-signal": 0.31,
|
|
"medium-signal": 0.62,
|
|
"high-signal": 0.82,
|
|
}
|
|
traditional_steps_to_best = traditional_order.index("high-signal") + 1
|
|
swarm_steps_to_best = pheromone_order.index("high-signal") + 1
|
|
traditional_efficiency = 1 / traditional_steps_to_best
|
|
swarm_efficiency = 1 / swarm_steps_to_best
|
|
return {
|
|
"id": "C03",
|
|
"name": "pheromone_efficiency",
|
|
"why": "Pheromone feedback should reduce exploration cost by prioritizing stronger routes earlier.",
|
|
"metric": Metric("best_route_efficiency", traditional_efficiency, swarm_efficiency, "1/steps_to_best"),
|
|
"passed": swarm_steps_to_best < traditional_steps_to_best and quality[pheromone_order[0]] > quality[traditional_order[0]],
|
|
"details": {
|
|
"traditional_order": traditional_order,
|
|
"swarm_pheromone_order": pheromone_order,
|
|
"traditional_steps_to_best": traditional_steps_to_best,
|
|
"swarm_steps_to_best": swarm_steps_to_best,
|
|
"first_claim_quality_gain_percent": round(
|
|
(quality[pheromone_order[0]] - quality[traditional_order[0]]) / quality[traditional_order[0]] * 100,
|
|
1,
|
|
),
|
|
"steps_to_best_reduction_percent": round(
|
|
(traditional_steps_to_best - swarm_steps_to_best) / traditional_steps_to_best * 100,
|
|
1,
|
|
),
|
|
},
|
|
}
|
|
|
|
|
|
def handoff_context_retention() -> dict[str, object]:
|
|
required_context = ["task_pool", "pheromone", "shared_state", "convergence", "analysis: ready"]
|
|
traditional_payload = "final report"
|
|
swarm_payload = "facts: task_pool pheromone shared_state convergence; analysis: ready"
|
|
traditional_retained = retained_ratio(traditional_payload, required_context)
|
|
swarm_retained = retained_ratio(swarm_payload, required_context)
|
|
return {
|
|
"id": "C04",
|
|
"name": "handoff_context_retention",
|
|
"why": "LangGraph-style handoff is valuable only if the next active agent receives the prior context.",
|
|
"metric": Metric("context_retention_ratio", traditional_retained, swarm_retained, "0_to_1"),
|
|
"passed": traditional_retained < swarm_retained and swarm_retained == 1.0,
|
|
"details": {
|
|
"required_context": required_context,
|
|
"traditional_retained_ratio": traditional_retained,
|
|
"swarm_retained_ratio": swarm_retained,
|
|
"interpretation": "+100 percentage points context retention in this deterministic handoff case.",
|
|
},
|
|
}
|
|
|
|
|
|
def retained_ratio(payload: str, required_context: list[str]) -> float:
|
|
return sum(1 for item in required_context if item in payload) / len(required_context)
|
|
|
|
|
|
def serialize_scenario(scenario: dict[str, object]) -> dict[str, object]:
|
|
metric = scenario["metric"]
|
|
assert isinstance(metric, Metric)
|
|
relative = metric.relative_gain_percent
|
|
return {
|
|
"id": scenario["id"],
|
|
"name": scenario["name"],
|
|
"why": scenario["why"],
|
|
"passed": scenario["passed"],
|
|
"metric": {
|
|
"name": metric.name,
|
|
"traditional": round(metric.traditional, 4),
|
|
"swarm": round(metric.swarm, 4),
|
|
"unit": metric.unit,
|
|
"delta": round(metric.delta, 4),
|
|
"relative_gain_percent": None if relative is None else round(relative, 1),
|
|
},
|
|
"details": scenario["details"],
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|