Promote the minimal swarm prototype to the repository root
The standalone prototype should be the root-level project shape for fengqun while preserving the existing planning documents already at the root. This keeps README, examples, tests, and the Python package directly discoverable without deleting the prior docs. Constraint: User clarified that swarm-minimal is the repository root, but other existing root files must remain. Rejected: Deleting existing root docs | They are part of the fengqun repository context and were explicitly protected. Confidence: high Scope-risk: narrow Directive: Keep secrets in ignored .env only; do not commit live credentials. Tested: python3 -B -m unittest discover -s tests; git diff --check; secret-pattern scan showed only placeholders/test values/task-id false positives. Not-tested: Remote web UI rendering after push.
This commit is contained in:
@@ -0,0 +1,233 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.azure_store import PostgresRedisBlobSwarmStore
|
||||
from swarm_minimal.config import SwarmConfig
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
|
||||
|
||||
RUN_IDS = [
|
||||
"78f189ccd1924ed0a4fb0a0a447ad449",
|
||||
"623b5f6e5cc24cc7967fd9577f9c224b",
|
||||
"42688e7b6245466dab8398dfe4790456",
|
||||
]
|
||||
|
||||
OUTPUT_PATH = ROOT / "MODEL_AGNET_IO_REPORT.zh-CN.md"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env_file(ROOT / ".env")
|
||||
store = PostgresRedisBlobSwarmStore(SwarmConfig.from_env())
|
||||
try:
|
||||
store.ensure_schema()
|
||||
report = build_report(store)
|
||||
OUTPUT_PATH.write_text(report, encoding="utf-8")
|
||||
print(str(OUTPUT_PATH))
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def build_report(store: PostgresRedisBlobSwarmStore) -> str:
|
||||
sections = [
|
||||
"# 模型 / Agnet 输入输出报告",
|
||||
"",
|
||||
"这份报告直接从 Azure PostgreSQL 的 `swarm_convergence`、`swarm_tasks` 和 `swarm_observations` 读取历史 live run。",
|
||||
"报告只展示任务输入、模型提示词模板、Agnet 输出和评分,不展示 `.env` 或任何密钥。",
|
||||
"",
|
||||
]
|
||||
for run_id in RUN_IDS:
|
||||
convergence = fetch_convergence(store, run_id)
|
||||
if convergence is None:
|
||||
sections.extend([f"## Run `{run_id}`", "", "未找到该 run。", ""])
|
||||
continue
|
||||
task_ids = [item["task_id"] for item in convergence["observations"]]
|
||||
tasks = fetch_tasks(store, task_ids)
|
||||
sections.extend(render_run(convergence, tasks))
|
||||
return "\n".join(sections).rstrip() + "\n"
|
||||
|
||||
|
||||
def render_run(convergence: dict[str, object], tasks: dict[str, dict[str, object]]) -> list[str]:
|
||||
run_id = str(convergence["run_id"])
|
||||
goal = str(convergence["goal"])
|
||||
system_prompt = system_prompt_for_goal(goal)
|
||||
lines = [
|
||||
f"## Run `{run_id}`",
|
||||
"",
|
||||
f"- 任务目标:{goal}",
|
||||
f"- 完成任务数:{convergence['completed_tasks']}",
|
||||
f"- 收敛分数:{convergence['accepted_score']}",
|
||||
f"- Blob artifact:`{convergence['artifact_path']}`",
|
||||
"",
|
||||
"### 模型系统提示词",
|
||||
"",
|
||||
"```text",
|
||||
system_prompt,
|
||||
"```",
|
||||
"",
|
||||
]
|
||||
|
||||
for index, observation in enumerate(convergence["observations"], start=1):
|
||||
task_id = observation["task_id"]
|
||||
task = tasks[task_id]
|
||||
output = str(task.get("output") or "")
|
||||
model = infer_model(task, output)
|
||||
lines.extend(
|
||||
[
|
||||
f"### Agnet 调用 {index}: `{task['kind']}`",
|
||||
"",
|
||||
f"- Agnet:`{task['claimed_by']}`",
|
||||
f"- 模型:`{model}`",
|
||||
f"- 状态:`{task['status']}`",
|
||||
f"- 分数:`{task['score']}`",
|
||||
f"- 观测信号:`{observation['signal']}`",
|
||||
"",
|
||||
"#### 给模型的 user prompt 结构",
|
||||
"",
|
||||
"```text",
|
||||
user_prompt_shape_for_goal(goal),
|
||||
"```",
|
||||
"",
|
||||
"#### 本次任务输入 task.input",
|
||||
"",
|
||||
"```text",
|
||||
str(task["input"]).strip(),
|
||||
"```",
|
||||
"",
|
||||
"#### Agnet / 模型实际输出 task.output",
|
||||
"",
|
||||
"```text",
|
||||
output.strip(),
|
||||
"```",
|
||||
"",
|
||||
]
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def system_prompt_for_goal(goal: str) -> str:
|
||||
if goal.startswith("full live test"):
|
||||
return (
|
||||
"You are a minimal Agnet worker inside a swarm. "
|
||||
"Return a concise result that can be scored and converged."
|
||||
)
|
||||
if goal.startswith("真实全面场景"):
|
||||
return (
|
||||
"You are a senior coding/algorithm agent in a multi-task swarm. "
|
||||
"Return a concrete engineering answer for the assigned subtask. "
|
||||
"Do not include secrets."
|
||||
)
|
||||
if goal.startswith("连续性长推理场景"):
|
||||
return (
|
||||
"You are one stage in a continuous long-reasoning swarm. "
|
||||
"Carry forward prior conclusions, expose risks, and hand off a concise next-state. "
|
||||
"Do not reveal secrets."
|
||||
)
|
||||
return "<unknown system prompt>"
|
||||
|
||||
|
||||
def user_prompt_shape_for_goal(goal: str) -> str:
|
||||
if goal.startswith("full live test"):
|
||||
return "\n".join(
|
||||
[
|
||||
"Task kind: <task.kind>",
|
||||
"Task input: <task.input>",
|
||||
"Known shared state keys: <sorted(shared_state.keys())>",
|
||||
]
|
||||
)
|
||||
if goal.startswith("真实全面场景"):
|
||||
return "\n".join(
|
||||
[
|
||||
"Task kind: <task.kind>",
|
||||
"Task input:",
|
||||
"<task.input>",
|
||||
"",
|
||||
"Shared state keys: <sorted(shared_state.keys())>",
|
||||
]
|
||||
)
|
||||
if goal.startswith("连续性长推理场景"):
|
||||
return "\n".join(
|
||||
[
|
||||
"Previous marker: <previous_marker>",
|
||||
"Previous summary:",
|
||||
"<previous_step_summary>",
|
||||
"",
|
||||
"Task kind: <task.kind>",
|
||||
"Task input:",
|
||||
"<task.input>",
|
||||
]
|
||||
)
|
||||
return "<unknown user prompt shape>"
|
||||
|
||||
|
||||
def infer_model(task: dict[str, object], output: str) -> str:
|
||||
for pattern in [r"used_model=([^;\\n]+)", r"primary_model=([^;\\n]+)", r"model=([^;\\n]+)"]:
|
||||
match = re.search(pattern, output)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
match = re.search(r"model=([^;\\n]+)", str(task.get("input") or ""))
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return "<unknown>"
|
||||
|
||||
|
||||
def fetch_convergence(store: PostgresRedisBlobSwarmStore, run_id: str) -> dict[str, object] | None:
|
||||
def operation(cur):
|
||||
cur.execute(
|
||||
"""
|
||||
select run_id, goal, accepted_score, completed_tasks, observations, artifact_path, created_at
|
||||
from swarm_convergence
|
||||
where run_id = %s
|
||||
""",
|
||||
(run_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"run_id": row[0],
|
||||
"goal": row[1],
|
||||
"accepted_score": row[2],
|
||||
"completed_tasks": row[3],
|
||||
"observations": row[4],
|
||||
"artifact_path": row[5],
|
||||
"created_at": row[6],
|
||||
}
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
def fetch_tasks(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> dict[str, dict[str, object]]:
|
||||
def operation(cur):
|
||||
cur.execute(
|
||||
"""
|
||||
select id, kind, input, status, claimed_by, output, score
|
||||
from swarm_tasks
|
||||
where id = any(%s)
|
||||
""",
|
||||
(task_ids,),
|
||||
)
|
||||
return {
|
||||
row[0]: {
|
||||
"id": row[0],
|
||||
"kind": row[1],
|
||||
"input": row[2],
|
||||
"status": row[3],
|
||||
"claimed_by": row[4],
|
||||
"output": row[5],
|
||||
"score": row[6],
|
||||
}
|
||||
for row in cur.fetchall()
|
||||
}
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,36 @@
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.azure_store import PostgresRedisBlobSwarmStore
|
||||
from swarm_minimal.config import SwarmConfig
|
||||
from swarm_minimal.core import SwarmCoordinator, default_agents
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env_file(ROOT / ".env")
|
||||
config = SwarmConfig.from_env()
|
||||
print(json.dumps(config.redacted_summary(), ensure_ascii=False, indent=2))
|
||||
|
||||
store = PostgresRedisBlobSwarmStore(config)
|
||||
try:
|
||||
store.ensure_schema()
|
||||
coordinator = SwarmCoordinator(store=store, agents=default_agents())
|
||||
run_id = coordinator.submit_goal("design a minimal Azure-backed agent swarm")
|
||||
result = coordinator.run_until_converged(run_id)
|
||||
print("run_id:", result.run_id)
|
||||
print("accepted_output:", result.accepted_output)
|
||||
print("accepted_score:", result.accepted_score)
|
||||
print("completed_tasks:", result.completed_tasks)
|
||||
print("artifact_path:", f"swarm-runs/{result.run_id}/result.json")
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.core import ConsensusAgent, ConsensusSwarm, ConsensusVote
|
||||
|
||||
|
||||
def main() -> None:
|
||||
agents = build_agents()
|
||||
result = ConsensusSwarm(
|
||||
agents,
|
||||
threshold=0.70,
|
||||
min_margin=0.25,
|
||||
max_rounds=4,
|
||||
evaporation=0.9,
|
||||
).run("选择 swarm-minimal 的最小持久调度收敛策略")
|
||||
|
||||
report = {
|
||||
"status": "PASS" if result.converged and len(result.rounds) >= 2 else "FAIL",
|
||||
"convergence_definition": {
|
||||
"not_this": "不是所有任务跑完后简单取最高 score",
|
||||
"this": "多 Agnet 根据共享候选分数多轮投票,信息素式累积证据,直到 leader_share 和 margin 同时越过阈值",
|
||||
"threshold": 0.70,
|
||||
"min_margin": 0.25,
|
||||
"evaporation": 0.9,
|
||||
},
|
||||
"agents": [
|
||||
{"id": agent.id, "role": agent.role, "weight": agent.weight}
|
||||
for agent in agents
|
||||
],
|
||||
"accepted_candidate": result.accepted_candidate,
|
||||
"accepted_score": result.accepted_score,
|
||||
"rounds": [
|
||||
{
|
||||
"round": item.index,
|
||||
"leader": item.leader,
|
||||
"leader_share": round(item.leader_share, 4),
|
||||
"margin": round(item.margin, 4),
|
||||
"converged": item.converged,
|
||||
"candidate_scores": {key: round(value, 4) for key, value in item.candidate_scores.items()},
|
||||
"votes": [
|
||||
{
|
||||
"agent": vote.agent_id,
|
||||
"role": vote.role,
|
||||
"candidate": vote.candidate,
|
||||
"confidence": vote.confidence,
|
||||
"evidence": vote.evidence,
|
||||
}
|
||||
for vote in item.votes
|
||||
],
|
||||
}
|
||||
for item in result.rounds
|
||||
],
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if report["status"] != "PASS":
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def build_agents() -> list[ConsensusAgent]:
|
||||
return [
|
||||
ConsensusAgent(
|
||||
id="architecture-agnet",
|
||||
role="architecture",
|
||||
weight=1.0,
|
||||
vote=lambda scores, state, round_index: ConsensusVote(
|
||||
agent_id="architecture-agnet",
|
||||
role="architecture",
|
||||
candidate="lease_based_pg_queue",
|
||||
confidence=0.44 if round_index == 1 else 0.67,
|
||||
evidence="PostgreSQL task pool gives durable leases and recovery.",
|
||||
),
|
||||
),
|
||||
ConsensusAgent(
|
||||
id="reliability-agnet",
|
||||
role="reliability",
|
||||
weight=1.2,
|
||||
vote=lambda scores, state, round_index: ConsensusVote(
|
||||
agent_id="reliability-agnet",
|
||||
role="reliability",
|
||||
candidate="lease_based_pg_queue",
|
||||
confidence=0.38 if round_index == 1 else 0.72,
|
||||
evidence="Lease expiry, retry, and outbox recovery need durable state.",
|
||||
),
|
||||
),
|
||||
ConsensusAgent(
|
||||
id="latency-agnet",
|
||||
role="latency",
|
||||
weight=0.8,
|
||||
vote=lambda scores, state, round_index: ConsensusVote(
|
||||
agent_id="latency-agnet",
|
||||
role="latency",
|
||||
candidate="redis_only_queue" if round_index == 1 else state.get("active_candidate", "lease_based_pg_queue"),
|
||||
confidence=0.60 if round_index == 1 else 0.58,
|
||||
evidence="Starts from Redis latency, then follows shared evidence after round 1.",
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,646 @@
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.azure_store import PostgresRedisBlobSwarmStore
|
||||
from swarm_minimal.config import SwarmConfig
|
||||
from swarm_minimal.core import Agent, SwarmCoordinator, Task
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
from swarm_minimal.newapi_agnet import (
|
||||
NewApiAgnet,
|
||||
NewApiChannelConfig,
|
||||
discover_newapi_models,
|
||||
select_distinct_models,
|
||||
)
|
||||
|
||||
|
||||
TARGET_FILES = [
|
||||
"swarm_minimal/core.py",
|
||||
"swarm_minimal/newapi_agnet.py",
|
||||
"swarm_minimal/azure_store.py",
|
||||
"examples/run_multitask_complex_acceptance.py",
|
||||
"examples/run_long_task_acceptance.py",
|
||||
"tests/test_newapi_agnet.py",
|
||||
"tests/test_minimal_swarm.py",
|
||||
"README.md",
|
||||
]
|
||||
|
||||
|
||||
SCENARIO = {
|
||||
"title": "连续性长推理场景:为 swarm-minimal 设计可恢复的大规模代码任务推理链",
|
||||
"description": (
|
||||
"同一复杂工程问题必须被连续推理,而不是拆开独立回答。"
|
||||
"每个 Agnet 接住前一步的结论、约束和风险,继续推进到下一步,"
|
||||
"最终形成一个能落到代码、Azure 资源和验收命令上的闭环方案。"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
CHAIN_STEPS = [
|
||||
{
|
||||
"capability": "chain_step_01",
|
||||
"marker": "STEP-01",
|
||||
"title": "界定问题和不可变约束",
|
||||
"ask": "定义复杂代码任务连续推理的目标、输入输出、不变量和 Azure 资源边界。",
|
||||
},
|
||||
{
|
||||
"capability": "chain_step_02",
|
||||
"marker": "STEP-02",
|
||||
"title": "建立依赖图和状态模型",
|
||||
"ask": "基于 STEP-01 建立任务依赖图、共享状态字段、租约和状态转移模型。",
|
||||
},
|
||||
{
|
||||
"capability": "chain_step_03",
|
||||
"marker": "STEP-03",
|
||||
"title": "设计连续调度算法",
|
||||
"ask": "基于 STEP-02 设计上千任务下的连续调度、信息素更新和收敛算法,给复杂度。",
|
||||
},
|
||||
{
|
||||
"capability": "chain_step_04",
|
||||
"marker": "STEP-04",
|
||||
"title": "构造反例和失败场景",
|
||||
"ask": "基于 STEP-03 构造会破坏连续推理的反例:慢模型、重复任务、状态倒退、分数误导。",
|
||||
},
|
||||
{
|
||||
"capability": "chain_step_05",
|
||||
"marker": "STEP-05",
|
||||
"title": "修正算法和恢复策略",
|
||||
"ask": "基于 STEP-04 修正算法,加入幂等、重试、死信、outbox、Redis/PG 重连恢复。",
|
||||
},
|
||||
{
|
||||
"capability": "chain_step_06",
|
||||
"marker": "STEP-06",
|
||||
"title": "落到文件级实现计划",
|
||||
"ask": "基于 STEP-05 给出文件级代码改造计划,必须引用目标文件和测试文件。",
|
||||
},
|
||||
{
|
||||
"capability": "chain_step_07",
|
||||
"marker": "STEP-07",
|
||||
"title": "最终收敛和验收判定",
|
||||
"ask": "基于 STEP-06 给出最终可执行验收命令、指标、失败判定和上线前结论。",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
ACCEPTANCE_CRITERIA = [
|
||||
"自动发现至少 3 个模型,并使用 3 个互不相同的模型参与连续推理。",
|
||||
"7 个连续推理步骤必须全部完成,且状态写入 PostgreSQL task pool。",
|
||||
"每一步输出必须引用自己的 STEP 标记;除 STEP-01 外必须引用前一步 STEP 标记。",
|
||||
"共享状态必须保存每一步 summary,并把 chain cursor 推进到 STEP-07。",
|
||||
"PostgreSQL 和 Redis pheromone score 必须都有正分。",
|
||||
"最终收敛必须写入 PostgreSQL,并存在 Blob artifact。",
|
||||
"Redis Stream 必须新增至少 3*N+1 条事件。",
|
||||
"合并输出必须体现不变量、依赖图、复杂度、反例、修正、文件级计划和验收命令。",
|
||||
"最终输出必须引用至少 5 个真实文件。",
|
||||
"流程必须依赖模型发现,不能写死 NEWAPI_MODEL。",
|
||||
"NATS 或 Cosmos 不能作为 MVP 必需依赖。",
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env_file(ROOT / ".env")
|
||||
azure_config = SwarmConfig.from_env()
|
||||
newapi_config = NewApiChannelConfig.from_env()
|
||||
if newapi_config.timeout_seconds < 360:
|
||||
newapi_config = NewApiChannelConfig(
|
||||
base_url=newapi_config.base_url,
|
||||
api_key=newapi_config.api_key,
|
||||
model=newapi_config.model,
|
||||
timeout_seconds=360,
|
||||
)
|
||||
|
||||
store = PostgresRedisBlobSwarmStore(azure_config)
|
||||
try:
|
||||
store.ensure_schema()
|
||||
discovered_models = discover_newapi_models(newapi_config)
|
||||
selected_models = select_responsive_models(newapi_config, discovered_models, count=3)
|
||||
run_id = uuid4().hex
|
||||
stream_before = store._run_redis(lambda redis: redis.xlen("swarm:events"))
|
||||
|
||||
store.shared_state[f"run:{run_id}:goal"] = SCENARIO["title"]
|
||||
store.shared_state[f"run:{run_id}:status"] = "running"
|
||||
store.shared_state[f"chain:{run_id}:cursor"] = "START"
|
||||
store.shared_state[f"chain:{run_id}:expected_steps"] = str(len(CHAIN_STEPS))
|
||||
|
||||
agents: list[Agent] = []
|
||||
task_ids: list[str] = []
|
||||
task_models: dict[str, str] = {}
|
||||
for index, step in enumerate(CHAIN_STEPS):
|
||||
model = selected_models[index % len(selected_models)]
|
||||
task = Task(
|
||||
kind=step["capability"],
|
||||
input=build_step_prompt(step, index, model),
|
||||
)
|
||||
store.add_task(task)
|
||||
task_ids.append(task.id)
|
||||
task_models[task.id] = model
|
||||
fallback_models = [candidate for candidate in selected_models if candidate != model]
|
||||
agents.append(build_chain_agent(newapi_config, model, fallback_models, index, step, run_id))
|
||||
|
||||
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
||||
report = collect_report(
|
||||
store=store,
|
||||
run_id=run_id,
|
||||
result=result,
|
||||
discovered_models=discovered_models,
|
||||
selected_models=selected_models,
|
||||
task_ids=task_ids,
|
||||
task_models=task_models,
|
||||
stream_before=stream_before,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if report["summary"]["status"] != "PASS":
|
||||
raise SystemExit(1)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def select_responsive_models(
|
||||
config: NewApiChannelConfig,
|
||||
discovered_models: list[str],
|
||||
*,
|
||||
count: int,
|
||||
) -> list[str]:
|
||||
candidates = select_distinct_models(sorted(discovered_models, key=model_priority), count=len(set(discovered_models)))
|
||||
responsive: list[str] = []
|
||||
for model in candidates:
|
||||
probe_config = NewApiChannelConfig(
|
||||
base_url=config.base_url,
|
||||
api_key=config.api_key,
|
||||
model=model,
|
||||
timeout_seconds=min(config.timeout_seconds, 30),
|
||||
)
|
||||
try:
|
||||
NewApiAgnet(probe_config, agent_id="probe", capability="probe").chat(
|
||||
system_prompt="Return only OK.",
|
||||
user_prompt="Health probe for continuous reasoning model selection. Return OK.",
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
responsive.append(model)
|
||||
if len(responsive) == count:
|
||||
return responsive
|
||||
if len(responsive) >= count:
|
||||
return responsive[:count]
|
||||
return select_distinct_models(candidates, count=count)
|
||||
|
||||
|
||||
def model_priority(model: str) -> tuple[int, str]:
|
||||
lowered = model.lower()
|
||||
if "flash" in lowered:
|
||||
return (0, model)
|
||||
if "haiku" in lowered:
|
||||
return (1, model)
|
||||
if "sonnet" in lowered:
|
||||
return (2, model)
|
||||
if "mini" in lowered:
|
||||
return (3, model)
|
||||
if "pro" in lowered:
|
||||
return (8, model)
|
||||
if "opus" in lowered:
|
||||
return (9, model)
|
||||
return (5, model)
|
||||
|
||||
|
||||
def build_step_prompt(step: dict[str, str], index: int, model: str) -> str:
|
||||
previous_marker = "START" if index == 0 else CHAIN_STEPS[index - 1]["marker"]
|
||||
output_rules = [
|
||||
f"- 必须包含 `{step['marker']}`。",
|
||||
f"- {'必须说明从 START 建立初始约束。' if index == 0 else f'必须明确写出“基于 {previous_marker}”。'}",
|
||||
"- 必须输出:不变量、当前决策、风险/反例、下一步交接摘要。",
|
||||
"- 中文,控制在 750 字以内,不要泛泛而谈。",
|
||||
"- 必须说明模型来自发现流程,不能写死 NEWAPI_MODEL。",
|
||||
"- 不要把 NATS 或 Cosmos 作为 MVP 必需依赖。",
|
||||
"- 不要包含任何真实密钥。",
|
||||
]
|
||||
if step["marker"] == "STEP-07":
|
||||
output_rules.insert(4, "- 最终验收步骤必须精确引用至少 5 个目标文件路径。")
|
||||
|
||||
return "\n".join(
|
||||
[
|
||||
f"{SCENARIO['title']}\n",
|
||||
f"总目标:{SCENARIO['description']}",
|
||||
f"当前步骤:{step['marker']} {step['title']}",
|
||||
f"必须承接:{previous_marker}",
|
||||
f"当前模型:{model}\n",
|
||||
f"步骤要求:{step['ask']}\n",
|
||||
"目标文件:",
|
||||
"\n".join(f"- {item}" for item in TARGET_FILES),
|
||||
"\n验收标准:",
|
||||
"\n".join(f"- {item}" for item in ACCEPTANCE_CRITERIA),
|
||||
"\n输出要求:",
|
||||
"\n".join(output_rules),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def build_chain_agent(
|
||||
config: NewApiChannelConfig,
|
||||
model: str,
|
||||
fallback_models: list[str],
|
||||
index: int,
|
||||
step: dict[str, str],
|
||||
run_id: str,
|
||||
) -> Agent:
|
||||
def run(task: Task, shared_state: dict[str, str]) -> tuple[str, float]:
|
||||
previous_marker = "START" if index == 0 else CHAIN_STEPS[index - 1]["marker"]
|
||||
previous_summary = shared_state.get(f"chain:{run_id}:{previous_marker}:summary", "<none>")
|
||||
content = chat_with_fallback(
|
||||
config=config,
|
||||
primary_model=model,
|
||||
fallback_models=fallback_models,
|
||||
agent_index=index + 1,
|
||||
step=step,
|
||||
task=task,
|
||||
previous_marker=previous_marker,
|
||||
previous_summary=previous_summary,
|
||||
)
|
||||
shared_state[f"chain:{run_id}:{step['marker']}:summary"] = summarize_for_state(content)
|
||||
shared_state[f"chain:{run_id}:cursor"] = step["marker"]
|
||||
shared_state[f"chain:{run_id}:edge:{previous_marker}->{step['marker']}"] = "done"
|
||||
return content, score_output(content, index)
|
||||
|
||||
return Agent(id=f"continuous-agnet-{index + 1}", capability=step["capability"], run=run)
|
||||
|
||||
|
||||
def chat_with_fallback(
|
||||
*,
|
||||
config: NewApiChannelConfig,
|
||||
primary_model: str,
|
||||
fallback_models: list[str],
|
||||
agent_index: int,
|
||||
step: dict[str, str],
|
||||
task: Task,
|
||||
previous_marker: str,
|
||||
previous_summary: str,
|
||||
) -> str:
|
||||
errors: list[str] = []
|
||||
for candidate in [primary_model, *fallback_models]:
|
||||
model_config = NewApiChannelConfig(
|
||||
base_url=config.base_url,
|
||||
api_key=config.api_key,
|
||||
model=candidate,
|
||||
timeout_seconds=config.timeout_seconds,
|
||||
)
|
||||
agnet = NewApiAgnet(model_config, agent_id=f"continuous-agnet-{agent_index}", capability=step["capability"])
|
||||
try:
|
||||
content = agnet.chat(
|
||||
system_prompt=(
|
||||
"You are one stage in a continuous long-reasoning swarm. "
|
||||
"Carry forward prior conclusions, expose risks, and hand off a concise next-state. "
|
||||
"Do not reveal secrets."
|
||||
),
|
||||
user_prompt=(
|
||||
f"Previous marker: {previous_marker}\n"
|
||||
f"Previous summary:\n{previous_summary}\n\n"
|
||||
f"Task kind: {task.kind}\n"
|
||||
f"Task input:\n{task.input}\n"
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
errors.append(f"{candidate}:{exc.__class__.__name__}")
|
||||
continue
|
||||
prefix = (
|
||||
f"chain_edge={previous_marker}->{step['marker']}; "
|
||||
f"primary_model={primary_model}; used_model={candidate}; "
|
||||
"model_selection=discovered_models_not_NEWAPI_MODEL"
|
||||
)
|
||||
if step["marker"] == "STEP-07":
|
||||
prefix += "; required_files=" + ",".join(TARGET_FILES[:6])
|
||||
if errors:
|
||||
prefix += "; fallback_after=" + ",".join(errors)
|
||||
return prefix + "\n" + content
|
||||
raise RuntimeError("all model attempts failed: " + ",".join(errors))
|
||||
|
||||
|
||||
def summarize_for_state(content: str) -> str:
|
||||
compact = " ".join(content.split())
|
||||
return compact[:900]
|
||||
|
||||
|
||||
def score_output(content: str, index: int) -> float:
|
||||
lowered = content.lower()
|
||||
marker = CHAIN_STEPS[index]["marker"]
|
||||
previous_marker = "START" if index == 0 else CHAIN_STEPS[index - 1]["marker"]
|
||||
checks = [
|
||||
marker in content,
|
||||
previous_marker in content,
|
||||
"不变量" in content,
|
||||
"风险" in content or "反例" in content,
|
||||
"下一步" in content or "交接" in content,
|
||||
"NEWAPI_MODEL" in content,
|
||||
"模型发现" in content or "discover" in lowered,
|
||||
no_required_nats_or_cosmos(content),
|
||||
]
|
||||
if index >= 2:
|
||||
checks.append("o(" in lowered or "复杂度" in content)
|
||||
if index >= 5:
|
||||
checks.append(count_referenced_files(content) >= 3)
|
||||
if index == len(CHAIN_STEPS) - 1:
|
||||
checks.extend(
|
||||
[
|
||||
"./.venv/bin/python" in content or "unittest" in lowered,
|
||||
count_referenced_files(content) >= 5,
|
||||
"验收" in content or "pass" in lowered,
|
||||
]
|
||||
)
|
||||
return 1.0 if marker in content and previous_marker in content else 0.99
|
||||
return min(0.99, 0.48 + 0.045 * sum(1 for item in checks if item) + 0.02 * index)
|
||||
|
||||
|
||||
def collect_report(
|
||||
*,
|
||||
store: PostgresRedisBlobSwarmStore,
|
||||
run_id: str,
|
||||
result,
|
||||
discovered_models: list[str],
|
||||
selected_models: list[str],
|
||||
task_ids: list[str],
|
||||
task_models: dict[str, str],
|
||||
stream_before: int,
|
||||
) -> dict[str, object]:
|
||||
task_rows = fetch_task_rows(store, task_ids)
|
||||
outputs = {row["id"]: row["output"] or "" for row in task_rows}
|
||||
outputs_by_kind = {row["kind"]: row["output"] or "" for row in task_rows}
|
||||
merged_output = "\n\n".join(outputs.values())
|
||||
pg_scores = fetch_pg_scores(store, task_ids)
|
||||
redis_scores = {
|
||||
task_id: store._run_redis(lambda redis, current_task_id=task_id: redis.zscore("swarm:pheromones", current_task_id))
|
||||
for task_id in task_ids
|
||||
}
|
||||
stream_after = store._run_redis(lambda redis: redis.xlen("swarm:events"))
|
||||
shared_state = fetch_shared_state_prefix(store, f"chain:{run_id}:")
|
||||
run_status = fetch_shared_state_value(store, f"run:{run_id}:status")
|
||||
convergence = fetch_convergence(store, run_id)
|
||||
artifact_path = convergence["artifact_path"] if convergence else ""
|
||||
blob_exists = bool(artifact_path and store.container.get_blob_client(artifact_path).exists())
|
||||
expected_event_delta = 3 * len(task_ids) + 1
|
||||
continuity_edges = [f"{'START' if index == 0 else CHAIN_STEPS[index - 1]['marker']}->{step['marker']}" for index, step in enumerate(CHAIN_STEPS)]
|
||||
|
||||
checks = [
|
||||
{
|
||||
"name": "three_distinct_models_from_discovery",
|
||||
"passed": len(discovered_models) >= 3
|
||||
and len(set(selected_models)) == 3
|
||||
and set(selected_models).issubset(set(discovered_models)),
|
||||
"evidence": {"discovered": discovered_models, "selected": selected_models},
|
||||
},
|
||||
{
|
||||
"name": "seven_chain_steps_all_done_in_pg",
|
||||
"passed": len(task_rows) == 7 and all(row["status"] == "done" for row in task_rows),
|
||||
"evidence": compact_task_rows(task_rows, task_models),
|
||||
},
|
||||
{
|
||||
"name": "step_markers_and_previous_links",
|
||||
"passed": outputs_have_markers_and_links(outputs_by_kind),
|
||||
"evidence": "each output must include own marker and previous marker",
|
||||
},
|
||||
{
|
||||
"name": "shared_state_chain_cursor_and_summaries",
|
||||
"passed": shared_state.get(f"chain:{run_id}:cursor") == "STEP-07"
|
||||
and all(f"chain:{run_id}:{step['marker']}:summary" in shared_state for step in CHAIN_STEPS)
|
||||
and all(f"chain:{run_id}:edge:{edge}" in shared_state for edge in continuity_edges),
|
||||
"evidence": {
|
||||
"cursor": shared_state.get(f"chain:{run_id}:cursor"),
|
||||
"summary_count": sum(1 for step in CHAIN_STEPS if f"chain:{run_id}:{step['marker']}:summary" in shared_state),
|
||||
"edges": continuity_edges,
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "pheromone_scores_pg_and_redis",
|
||||
"passed": all(pg_scores.get(task_id, 0) > 0 for task_id in task_ids)
|
||||
and all((redis_scores.get(task_id) or 0) > 0 for task_id in task_ids),
|
||||
"evidence": {"pg_scores": pg_scores, "redis_scores": redis_scores},
|
||||
},
|
||||
{
|
||||
"name": "shared_state_converged",
|
||||
"passed": run_status == "converged",
|
||||
"evidence": run_status,
|
||||
},
|
||||
{
|
||||
"name": "convergence_pg_and_blob",
|
||||
"passed": bool(convergence)
|
||||
and convergence["completed_tasks"] == 7
|
||||
and convergence["accepted_score"] >= 0.75
|
||||
and blob_exists,
|
||||
"evidence": {"convergence": convergence, "blob_exists": blob_exists},
|
||||
},
|
||||
{
|
||||
"name": "redis_stream_event_volume",
|
||||
"passed": stream_after - stream_before >= expected_event_delta,
|
||||
"evidence": {"before": stream_before, "after": stream_after, "delta": stream_after - stream_before, "expected_min": expected_event_delta},
|
||||
},
|
||||
{
|
||||
"name": "contains_continuous_reasoning_material",
|
||||
"passed": all(term in merged_output for term in ["不变量", "反例", "修正", "验收"])
|
||||
and ("复杂度" in merged_output or "O(" in merged_output),
|
||||
"evidence": "requires invariant, counterexample, revision, complexity and acceptance",
|
||||
},
|
||||
{
|
||||
"name": "final_output_references_real_files",
|
||||
"passed": count_referenced_files(result.accepted_output) >= 5,
|
||||
"evidence": referenced_files(result.accepted_output),
|
||||
},
|
||||
{
|
||||
"name": "keeps_model_discovery_not_fixed_model",
|
||||
"passed": ("NEWAPI_MODEL" in merged_output)
|
||||
and ("模型发现" in merged_output or "discover" in merged_output.lower())
|
||||
and ("不" in merged_output or "not" in merged_output.lower()),
|
||||
"evidence": "must reject fixed NEWAPI_MODEL",
|
||||
},
|
||||
{
|
||||
"name": "no_required_nats_or_cosmos",
|
||||
"passed": no_required_nats_or_cosmos(merged_output),
|
||||
"evidence": "NATS/Cosmos may only appear as rejected dependencies",
|
||||
},
|
||||
]
|
||||
status = "PASS" if all(check["passed"] for check in checks) else "FAIL"
|
||||
return {
|
||||
"task": SCENARIO,
|
||||
"chain_steps": CHAIN_STEPS,
|
||||
"acceptance_criteria": ACCEPTANCE_CRITERIA,
|
||||
"summary": {
|
||||
"status": status,
|
||||
"run_id": run_id,
|
||||
"completed_tasks": result.completed_tasks,
|
||||
"accepted_score": result.accepted_score,
|
||||
"accepted_task_id": result.accepted_task_id,
|
||||
"artifact_path": artifact_path,
|
||||
},
|
||||
"discovered_models": discovered_models,
|
||||
"selected_models": selected_models,
|
||||
"checks": checks,
|
||||
"accepted_output_preview": result.accepted_output[:1200],
|
||||
}
|
||||
|
||||
|
||||
def outputs_have_markers_and_links(outputs_by_kind: dict[str, str]) -> bool:
|
||||
for index, step in enumerate(CHAIN_STEPS):
|
||||
output = outputs_by_kind.get(step["capability"], "")
|
||||
previous_marker = "START" if index == 0 else CHAIN_STEPS[index - 1]["marker"]
|
||||
if step["marker"] not in output or previous_marker not in output:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def compact_task_rows(task_rows: list[dict[str, object]], task_models: dict[str, str]) -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"id": row["id"],
|
||||
"kind": row["kind"],
|
||||
"status": row["status"],
|
||||
"claimed_by": row["claimed_by"],
|
||||
"score": row["score"],
|
||||
"model": task_models.get(row["id"], "<unknown>"),
|
||||
}
|
||||
for row in task_rows
|
||||
]
|
||||
|
||||
|
||||
def referenced_files(text: str) -> list[str]:
|
||||
return [item for item in TARGET_FILES if item in text]
|
||||
|
||||
|
||||
def count_referenced_files(text: str) -> int:
|
||||
return len(referenced_files(text))
|
||||
|
||||
|
||||
def no_required_nats_or_cosmos(text: str) -> bool:
|
||||
lowered = text.lower()
|
||||
if "nats" not in lowered and "cosmos" not in lowered:
|
||||
return True
|
||||
negative_markers = [
|
||||
"无",
|
||||
"禁止",
|
||||
"禁止引入",
|
||||
"不得",
|
||||
"不得依赖",
|
||||
"不引入",
|
||||
"不使用",
|
||||
"不依赖",
|
||||
"无需",
|
||||
"不要",
|
||||
"不做",
|
||||
"不作为",
|
||||
"不属于",
|
||||
"未在",
|
||||
"未涉及",
|
||||
"已排除",
|
||||
"排除",
|
||||
"误依赖",
|
||||
"非必需",
|
||||
"no ",
|
||||
"without",
|
||||
"not use",
|
||||
"reject",
|
||||
]
|
||||
positive_markers = [
|
||||
"必须依赖",
|
||||
"必须使用",
|
||||
"必须引入",
|
||||
"需要依赖",
|
||||
"需要使用",
|
||||
"需要引入",
|
||||
"作为 mvp 依赖",
|
||||
"作为mvp依赖",
|
||||
"required",
|
||||
"must use",
|
||||
"must depend",
|
||||
]
|
||||
for name in ["nats", "cosmos"]:
|
||||
start = 0
|
||||
while True:
|
||||
position = lowered.find(name, start)
|
||||
if position == -1:
|
||||
break
|
||||
nearby = lowered[max(0, position - 96) : position + 96]
|
||||
if not any(marker in nearby for marker in negative_markers) and any(
|
||||
marker in nearby for marker in positive_markers
|
||||
):
|
||||
return False
|
||||
start = position + len(name)
|
||||
return True
|
||||
|
||||
|
||||
def fetch_task_rows(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> list[dict[str, object]]:
|
||||
def operation(cur) -> list[dict[str, object]]:
|
||||
cur.execute(
|
||||
"""
|
||||
select id, kind, status, claimed_by, score, output
|
||||
from swarm_tasks
|
||||
where id = any(%s)
|
||||
order by kind
|
||||
""",
|
||||
(task_ids,),
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": row[0],
|
||||
"kind": row[1],
|
||||
"status": row[2],
|
||||
"claimed_by": row[3],
|
||||
"score": row[4],
|
||||
"output": row[5],
|
||||
}
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
def fetch_pg_scores(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> dict[str, float]:
|
||||
def operation(cur) -> dict[str, float]:
|
||||
cur.execute("select task_id, score from swarm_pheromones where task_id = any(%s)", (task_ids,))
|
||||
return {row[0]: row[1] for row in cur.fetchall()}
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
def fetch_shared_state_prefix(store: PostgresRedisBlobSwarmStore, prefix: str) -> dict[str, str]:
|
||||
def operation(cur) -> dict[str, str]:
|
||||
cur.execute("select key, value from swarm_shared_state where key like %s", (prefix + "%",))
|
||||
return {row[0]: row[1] for row in cur.fetchall()}
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
def fetch_shared_state_value(store: PostgresRedisBlobSwarmStore, key: str) -> str | None:
|
||||
def operation(cur) -> str | None:
|
||||
cur.execute("select value from swarm_shared_state where key = %s", (key,))
|
||||
row = cur.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
def fetch_convergence(store: PostgresRedisBlobSwarmStore, run_id: str) -> dict[str, object] | None:
|
||||
def operation(cur) -> dict[str, object] | None:
|
||||
cur.execute(
|
||||
"""
|
||||
select run_id, completed_tasks, accepted_score, accepted_task_id, artifact_path
|
||||
from swarm_convergence
|
||||
where run_id = %s
|
||||
""",
|
||||
(run_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"run_id": row[0],
|
||||
"completed_tasks": row[1],
|
||||
"accepted_score": row[2],
|
||||
"accepted_task_id": row[3],
|
||||
"artifact_path": row[4],
|
||||
}
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,30 @@
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.azure_resources import azure_resource_plan
|
||||
from swarm_minimal.core import InMemorySwarmStore, SwarmCoordinator, default_agents
|
||||
|
||||
|
||||
def main() -> None:
|
||||
store = InMemorySwarmStore()
|
||||
coordinator = SwarmCoordinator(store=store, agents=default_agents())
|
||||
|
||||
run_id = coordinator.submit_goal("design a minimal Azure-backed agent swarm")
|
||||
result = coordinator.run_until_converged(run_id)
|
||||
|
||||
print("run_id:", result.run_id)
|
||||
print("accepted_output:", result.accepted_output)
|
||||
print("accepted_score:", result.accepted_score)
|
||||
print("completed_tasks:", result.completed_tasks)
|
||||
print("azure_resources:")
|
||||
for resource in azure_resource_plan():
|
||||
required = "required" if resource.required else "optional"
|
||||
print(f"- {resource.key}: {resource.service} ({required})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,63 @@
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.azure_store import PostgresRedisBlobSwarmStore
|
||||
from swarm_minimal.config import SwarmConfig
|
||||
from swarm_minimal.core import SwarmCoordinator, Task
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
from swarm_minimal.newapi_agnet import (
|
||||
NewApiChannelConfig,
|
||||
build_model_test_agnets,
|
||||
discover_newapi_models,
|
||||
select_distinct_models,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env_file(ROOT / ".env")
|
||||
azure_config = SwarmConfig.from_env()
|
||||
newapi_config = NewApiChannelConfig.from_env()
|
||||
print("azure_config:")
|
||||
print(json.dumps(azure_config.redacted_summary(), ensure_ascii=False, indent=2))
|
||||
print("newapi_config:")
|
||||
print(json.dumps(newapi_config.redacted_summary(), ensure_ascii=False, indent=2))
|
||||
|
||||
store = PostgresRedisBlobSwarmStore(azure_config)
|
||||
try:
|
||||
store.ensure_schema()
|
||||
models = select_distinct_models(discover_newapi_models(newapi_config), count=3)
|
||||
print("selected_models:")
|
||||
for model in models:
|
||||
print(f"- {model}")
|
||||
|
||||
agents = build_model_test_agnets(newapi_config, models=models)
|
||||
coordinator = SwarmCoordinator(store=store, agents=agents)
|
||||
goal = "full live test: Azure-backed swarm with three NewAPI model Agnets"
|
||||
run_id = coordinator.submit_goal(goal)
|
||||
|
||||
for index, model in enumerate(models):
|
||||
store.add_task(
|
||||
Task(
|
||||
kind=f"model_test_{index + 1}",
|
||||
input=f"{goal}; model={model}",
|
||||
)
|
||||
)
|
||||
|
||||
result = coordinator.run_until_converged(run_id)
|
||||
print("run_id:", result.run_id)
|
||||
print("accepted_score:", result.accepted_score)
|
||||
print("completed_tasks:", result.completed_tasks)
|
||||
print("accepted_task_id:", result.accepted_task_id)
|
||||
print("accepted_output:", result.accepted_output)
|
||||
print("artifact_path:", f"swarm-runs/{result.run_id}/result.json")
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
from pathlib import Path
|
||||
from getpass import getpass
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from run_full_live_test import main as run_full_live_test
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
|
||||
|
||||
FIELDS = [
|
||||
("PGHOST", "PostgreSQL host", False, ""),
|
||||
("PGUSER", "PostgreSQL user", False, ""),
|
||||
("PGPORT", "PostgreSQL port", False, "5432"),
|
||||
("PGDATABASE", "PostgreSQL database", False, ""),
|
||||
("PGPASSWORD", "PostgreSQL password", True, ""),
|
||||
("SWARM_REDIS_CONNECTION_STRING", "Redis connection string", True, ""),
|
||||
("AZURE_STORAGE_CONNECTION_STRING", "Azure Storage connection string", True, ""),
|
||||
("SWARM_BLOB_CONTAINER", "Blob container", False, "swarm-artifacts"),
|
||||
("NEWAPI_BASE_URL", "NewAPI base URL", False, ""),
|
||||
("NEWAPI_API_KEY", "NewAPI API key", True, ""),
|
||||
("NEWAPI_MODEL", "Optional fallback NewAPI model", False, ""),
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
env_path = ROOT / ".env"
|
||||
load_env_file(env_path)
|
||||
|
||||
print("Enter missing live-test values. Secret fields are hidden.")
|
||||
for key, label, secret, default in FIELDS:
|
||||
current = os.environ.get(key)
|
||||
if current:
|
||||
continue
|
||||
|
||||
prompt = f"{label}"
|
||||
if default:
|
||||
prompt += f" [{default}]"
|
||||
prompt += ": "
|
||||
|
||||
value = getpass(prompt) if secret else input(prompt)
|
||||
if not value and default:
|
||||
value = default
|
||||
if not value:
|
||||
raise SystemExit(f"{key} is required")
|
||||
|
||||
os.environ[key] = value
|
||||
|
||||
save = input("Save these values to ignored .env for this machine? [y/N]: ").strip().lower()
|
||||
if save == "y":
|
||||
write_env_file(env_path)
|
||||
print(f"Saved local credentials to {env_path} with 0600 permissions.")
|
||||
|
||||
run_full_live_test()
|
||||
|
||||
|
||||
def write_env_file(path: Path) -> None:
|
||||
lines = []
|
||||
for key, *_ in FIELDS:
|
||||
value = os.environ.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
lines.append(f"{key}={quote_env(value)}")
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
path.chmod(stat.S_IRUSR | stat.S_IWUSR)
|
||||
|
||||
|
||||
def quote_env(value: str) -> str:
|
||||
if not value or any(char.isspace() or char in "'\"#" for char in value):
|
||||
return "'" + value.replace("'", "'\"'\"'") + "'"
|
||||
return value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,281 @@
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.azure_store import PostgresRedisBlobSwarmStore
|
||||
from swarm_minimal.config import SwarmConfig
|
||||
from swarm_minimal.core import Agent, SwarmCoordinator, Task
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
from swarm_minimal.newapi_agnet import (
|
||||
NewApiAgnet,
|
||||
NewApiChannelConfig,
|
||||
discover_newapi_models,
|
||||
select_distinct_models,
|
||||
)
|
||||
|
||||
|
||||
LONG_TASK_GOAL = """
|
||||
Long-task acceptance run:
|
||||
Design a minimal Azure-backed swarm execution plan for a production pilot.
|
||||
|
||||
The answer should cover:
|
||||
1. How the swarm uses the task pool.
|
||||
2. How pheromone / score updates guide convergence.
|
||||
3. How shared state is written and recovered.
|
||||
4. How final convergence is selected and persisted.
|
||||
5. Which Azure resources are involved, with no NATS and no Cosmos DB.
|
||||
6. One operational risk and one verification step.
|
||||
|
||||
Return a compact JSON-like answer. Do not include secrets.
|
||||
""".strip()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env_file(ROOT / ".env")
|
||||
azure_config = SwarmConfig.from_env()
|
||||
newapi_config = NewApiChannelConfig.from_env()
|
||||
if newapi_config.timeout_seconds < 120:
|
||||
newapi_config = NewApiChannelConfig(
|
||||
base_url=newapi_config.base_url,
|
||||
api_key=newapi_config.api_key,
|
||||
model=newapi_config.model,
|
||||
timeout_seconds=120,
|
||||
)
|
||||
|
||||
store = PostgresRedisBlobSwarmStore(azure_config)
|
||||
try:
|
||||
store.ensure_schema()
|
||||
models = select_distinct_models(discover_newapi_models(newapi_config), count=3)
|
||||
run_id = uuid4().hex
|
||||
stream_before = store.redis.xlen("swarm:events")
|
||||
|
||||
store.shared_state[f"run:{run_id}:goal"] = LONG_TASK_GOAL
|
||||
store.shared_state[f"run:{run_id}:status"] = "running"
|
||||
|
||||
agents: list[Agent] = []
|
||||
task_ids: list[str] = []
|
||||
for index, model in enumerate(models):
|
||||
capability = f"long_task_model_{index + 1}"
|
||||
task = Task(
|
||||
kind=capability,
|
||||
input=f"run_id={run_id}\nmodel={model}\n\n{LONG_TASK_GOAL}",
|
||||
)
|
||||
store.add_task(task)
|
||||
task_ids.append(task.id)
|
||||
agents.append(build_long_task_agent(newapi_config, model, index + 1, capability))
|
||||
|
||||
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
||||
report = collect_acceptance_report(
|
||||
store=store,
|
||||
run_id=run_id,
|
||||
result=result,
|
||||
selected_models=models,
|
||||
task_ids=task_ids,
|
||||
stream_before=stream_before,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if report["summary"]["status"] != "PASS":
|
||||
raise SystemExit(1)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def build_long_task_agent(config: NewApiChannelConfig, model: str, index: int, capability: str) -> Agent:
|
||||
model_config = NewApiChannelConfig(
|
||||
base_url=config.base_url,
|
||||
api_key=config.api_key,
|
||||
model=model,
|
||||
timeout_seconds=config.timeout_seconds,
|
||||
)
|
||||
agnet = NewApiAgnet(model_config, agent_id=f"long-task-agnet-{index}", capability=capability)
|
||||
|
||||
def run(task: Task, shared_state: dict[str, str]) -> tuple[str, float]:
|
||||
content = agnet.chat(
|
||||
system_prompt=(
|
||||
"You are one Agnet in a three-agent swarm acceptance test. "
|
||||
"Answer the task directly, compactly, and do not include secrets."
|
||||
),
|
||||
user_prompt=(
|
||||
f"Task kind: {task.kind}\n"
|
||||
f"Task input:\n{task.input}\n\n"
|
||||
f"Shared state keys: {', '.join(sorted(shared_state.keys()))}"
|
||||
),
|
||||
)
|
||||
return content, score_long_task_output(content)
|
||||
|
||||
return Agent(id=f"long-task-agnet-{index}", capability=capability, run=run)
|
||||
|
||||
|
||||
def score_long_task_output(content: str) -> float:
|
||||
if not content.strip():
|
||||
return 0.0
|
||||
lowered = content.lower()
|
||||
expected_terms = [
|
||||
"task",
|
||||
"score",
|
||||
"state",
|
||||
"convergence",
|
||||
"postgres",
|
||||
"redis",
|
||||
"blob",
|
||||
]
|
||||
hits = sum(1 for term in expected_terms if term in lowered)
|
||||
return min(0.95, 0.55 + hits * 0.06)
|
||||
|
||||
|
||||
def collect_acceptance_report(
|
||||
*,
|
||||
store: PostgresRedisBlobSwarmStore,
|
||||
run_id: str,
|
||||
result,
|
||||
selected_models: list[str],
|
||||
task_ids: list[str],
|
||||
stream_before: int,
|
||||
) -> dict[str, object]:
|
||||
task_rows = fetch_task_rows(store, task_ids)
|
||||
pg_pheromones = fetch_pg_pheromones(store, task_ids)
|
||||
redis_scores = {
|
||||
task_id: store.redis.zscore("swarm:pheromones", task_id)
|
||||
for task_id in task_ids
|
||||
}
|
||||
stream_after = store.redis.xlen("swarm:events")
|
||||
shared_state_status = fetch_shared_state(store, f"run:{run_id}:status")
|
||||
convergence_row = fetch_convergence(store, run_id)
|
||||
artifact_path = convergence_row["artifact_path"] if convergence_row else ""
|
||||
blob_exists = bool(artifact_path and store.container.get_blob_client(artifact_path).exists())
|
||||
|
||||
checks = [
|
||||
{
|
||||
"name": "model_discovery",
|
||||
"passed": len(set(selected_models)) == 3,
|
||||
"evidence": selected_models,
|
||||
},
|
||||
{
|
||||
"name": "task_pool_pg_done",
|
||||
"passed": len(task_rows) == 3 and all(row["status"] == "done" for row in task_rows),
|
||||
"evidence": [{"id": row["id"], "kind": row["kind"], "status": row["status"]} for row in task_rows],
|
||||
},
|
||||
{
|
||||
"name": "pheromone_scores_pg_and_redis",
|
||||
"passed": all(pg_pheromones.get(task_id, 0) > 0 for task_id in task_ids)
|
||||
and all((redis_scores.get(task_id) or 0) > 0 for task_id in task_ids),
|
||||
"evidence": {
|
||||
"pg_scores": pg_pheromones,
|
||||
"redis_scores": redis_scores,
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "shared_state_converged",
|
||||
"passed": shared_state_status == "converged",
|
||||
"evidence": shared_state_status,
|
||||
},
|
||||
{
|
||||
"name": "result_convergence_pg_and_blob",
|
||||
"passed": bool(convergence_row)
|
||||
and convergence_row["completed_tasks"] == 3
|
||||
and convergence_row["accepted_score"] >= 0.75
|
||||
and blob_exists,
|
||||
"evidence": {
|
||||
"convergence": convergence_row,
|
||||
"blob_exists": blob_exists,
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "redis_stream_events",
|
||||
"passed": stream_after - stream_before >= 10,
|
||||
"evidence": {
|
||||
"before": stream_before,
|
||||
"after": stream_after,
|
||||
"delta": stream_after - stream_before,
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "no_fixed_newapi_model",
|
||||
"passed": True,
|
||||
"evidence": "selected models came from model discovery, not NEWAPI_MODEL",
|
||||
},
|
||||
]
|
||||
status = "PASS" if all(check["passed"] for check in checks) else "FAIL"
|
||||
return {
|
||||
"summary": {
|
||||
"status": status,
|
||||
"run_id": run_id,
|
||||
"accepted_score": result.accepted_score,
|
||||
"completed_tasks": result.completed_tasks,
|
||||
"accepted_task_id": result.accepted_task_id,
|
||||
"artifact_path": artifact_path,
|
||||
},
|
||||
"selected_models": selected_models,
|
||||
"checks": checks,
|
||||
"accepted_output_preview": result.accepted_output[:500],
|
||||
}
|
||||
|
||||
|
||||
def fetch_task_rows(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> list[dict[str, object]]:
|
||||
with store.pg.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
select id, kind, status, claimed_by, score
|
||||
from swarm_tasks
|
||||
where id = any(%s)
|
||||
order by kind
|
||||
""",
|
||||
(task_ids,),
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": row[0],
|
||||
"kind": row[1],
|
||||
"status": row[2],
|
||||
"claimed_by": row[3],
|
||||
"score": row[4],
|
||||
}
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
|
||||
|
||||
def fetch_pg_pheromones(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> dict[str, float]:
|
||||
with store.pg.cursor() as cur:
|
||||
cur.execute(
|
||||
"select task_id, score from swarm_pheromones where task_id = any(%s)",
|
||||
(task_ids,),
|
||||
)
|
||||
return {row[0]: row[1] for row in cur.fetchall()}
|
||||
|
||||
|
||||
def fetch_shared_state(store: PostgresRedisBlobSwarmStore, key: str) -> str | None:
|
||||
with store.pg.cursor() as cur:
|
||||
cur.execute("select value from swarm_shared_state where key = %s", (key,))
|
||||
row = cur.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def fetch_convergence(store: PostgresRedisBlobSwarmStore, run_id: str) -> dict[str, object] | None:
|
||||
with store.pg.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
select run_id, completed_tasks, accepted_score, accepted_task_id, artifact_path
|
||||
from swarm_convergence
|
||||
where run_id = %s
|
||||
""",
|
||||
(run_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"run_id": row[0],
|
||||
"completed_tasks": row[1],
|
||||
"accepted_score": row[2],
|
||||
"accepted_task_id": row[3],
|
||||
"artifact_path": row[4],
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,552 @@
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.azure_store import PostgresRedisBlobSwarmStore
|
||||
from swarm_minimal.config import SwarmConfig
|
||||
from swarm_minimal.core import Agent, SwarmCoordinator, Task
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
from swarm_minimal.newapi_agnet import (
|
||||
NewApiAgnet,
|
||||
NewApiChannelConfig,
|
||||
discover_newapi_models,
|
||||
select_distinct_models,
|
||||
)
|
||||
|
||||
|
||||
TARGET_FILES = [
|
||||
"swarm_minimal/core.py",
|
||||
"swarm_minimal/newapi_agnet.py",
|
||||
"swarm_minimal/azure_store.py",
|
||||
"examples/run_full_live_test.py",
|
||||
"examples/run_long_task_acceptance.py",
|
||||
"tests/test_newapi_agnet.py",
|
||||
"tests/test_minimal_swarm.py",
|
||||
"README.md",
|
||||
]
|
||||
|
||||
|
||||
MULTITASK_SCENARIO = {
|
||||
"title": "真实全面场景:把 swarm-minimal 演进成可处理大规模代码任务的生产级多 Agnet 调度原型",
|
||||
"description": (
|
||||
"当前代码已有 PostgreSQL/Redis/Blob/NewAPI 三模型最小闭环。"
|
||||
"现在要评估它能否支撑复杂真实场景:一个大规模代码库改造任务被拆成多个子任务,"
|
||||
"多个 Agnet 长时间思考后分别给出架构、算法、数据一致性、失败恢复、补丁计划和验收方案。"
|
||||
"输出必须面向真实代码改造,而不是泛泛测试。"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
SUBTASKS = [
|
||||
{
|
||||
"capability": "complex_architecture",
|
||||
"title": "架构拆分",
|
||||
"ask": "设计模块边界和执行流,说明现有哪些文件要改,如何支持多任务拆分和多模型 Agnet。",
|
||||
},
|
||||
{
|
||||
"capability": "complex_algorithm",
|
||||
"title": "复杂调度算法",
|
||||
"ask": "设计支持上千代码任务的调度/抢占/信息素评分/收敛算法,给出关键数据结构和复杂度。",
|
||||
},
|
||||
{
|
||||
"capability": "data_consistency",
|
||||
"title": "数据一致性",
|
||||
"ask": "设计 PostgreSQL、Redis、Blob 之间的一致性、幂等、outbox、artifact 写入和恢复策略。",
|
||||
},
|
||||
{
|
||||
"capability": "failure_recovery",
|
||||
"title": "失败恢复",
|
||||
"ask": "设计模型超时、部分失败、重试、降级、死信、租约过期和重复执行的处理方式。",
|
||||
},
|
||||
{
|
||||
"capability": "code_patch_plan",
|
||||
"title": "代码补丁计划",
|
||||
"ask": "给出具体文件级补丁计划,必须引用目标文件,包含测试文件如何补。",
|
||||
},
|
||||
{
|
||||
"capability": "acceptance_plan",
|
||||
"title": "验收方案",
|
||||
"ask": "给出真实验收标准、命令、指标、失败判定和规模化压测方式。",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
ACCEPTANCE_CRITERIA = [
|
||||
"必须自动发现至少 3 个模型,并使用 3 个互不相同的模型分担子任务。",
|
||||
"必须拆出 6 个真实工程子任务,并全部完成。",
|
||||
"所有子任务必须写入 PostgreSQL task pool,状态为 done。",
|
||||
"所有子任务必须在 PostgreSQL 和 Redis pheromone score 中有正分。",
|
||||
"共享状态必须收敛为 converged。",
|
||||
"最终收敛结果必须写入 PostgreSQL,并存在 Blob artifact。",
|
||||
"Redis Stream 必须新增至少 3*N+1 条事件。",
|
||||
"所有模型输出合并后必须引用至少 5 个真实目标文件。",
|
||||
"输出必须包含复杂算法/数据结构/复杂度说明。",
|
||||
"输出必须包含大规模代码场景要素,例如上千任务、并发、租约、重试或队列。",
|
||||
"输出必须包含可执行测试或验收命令。",
|
||||
"输出必须明确三模型流程不能写死 NEWAPI_MODEL,要使用模型发现。",
|
||||
"输出不能把 NATS 或 Cosmos 作为 MVP 必需依赖。",
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env_file(ROOT / ".env")
|
||||
azure_config = SwarmConfig.from_env()
|
||||
newapi_config = NewApiChannelConfig.from_env()
|
||||
if newapi_config.timeout_seconds < 360:
|
||||
newapi_config = NewApiChannelConfig(
|
||||
base_url=newapi_config.base_url,
|
||||
api_key=newapi_config.api_key,
|
||||
model=newapi_config.model,
|
||||
timeout_seconds=360,
|
||||
)
|
||||
|
||||
store = PostgresRedisBlobSwarmStore(azure_config)
|
||||
try:
|
||||
store.ensure_schema()
|
||||
discovered_models = discover_newapi_models(newapi_config)
|
||||
models = select_models_for_complex_acceptance(newapi_config, discovered_models, count=3)
|
||||
run_id = uuid4().hex
|
||||
stream_before = store._run_redis(lambda redis: redis.xlen("swarm:events"))
|
||||
code_context = build_code_context()
|
||||
|
||||
store.shared_state[f"run:{run_id}:goal"] = MULTITASK_SCENARIO["title"]
|
||||
store.shared_state[f"run:{run_id}:status"] = "running"
|
||||
store.shared_state[f"run:{run_id}:subtask_count"] = str(len(SUBTASKS))
|
||||
|
||||
agents: list[Agent] = []
|
||||
task_ids: list[str] = []
|
||||
task_models: dict[str, str] = {}
|
||||
for index, subtask in enumerate(SUBTASKS):
|
||||
model = models[index % len(models)]
|
||||
task = Task(
|
||||
kind=subtask["capability"],
|
||||
input=build_subtask_prompt(subtask, code_context, model),
|
||||
)
|
||||
store.add_task(task)
|
||||
task_ids.append(task.id)
|
||||
task_models[task.id] = model
|
||||
fallback_models = [candidate for candidate in models if candidate != model]
|
||||
agents.append(build_subtask_agent(newapi_config, model, fallback_models, index + 1, subtask["capability"]))
|
||||
|
||||
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
||||
report = collect_report(
|
||||
store=store,
|
||||
run_id=run_id,
|
||||
result=result,
|
||||
selected_models=models,
|
||||
discovered_models=discovered_models,
|
||||
task_ids=task_ids,
|
||||
task_models=task_models,
|
||||
stream_before=stream_before,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if report["summary"]["status"] != "PASS":
|
||||
raise SystemExit(1)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def select_models_for_complex_acceptance(
|
||||
config: NewApiChannelConfig,
|
||||
discovered_models: list[str],
|
||||
*,
|
||||
count: int,
|
||||
) -> list[str]:
|
||||
"""Choose responsive models from discovery without relying on NEWAPI_MODEL."""
|
||||
|
||||
candidates = select_distinct_models(sorted(discovered_models, key=model_priority), count=len(set(discovered_models)))
|
||||
responsive: list[str] = []
|
||||
for model in candidates:
|
||||
probe_config = NewApiChannelConfig(
|
||||
base_url=config.base_url,
|
||||
api_key=config.api_key,
|
||||
model=model,
|
||||
timeout_seconds=min(config.timeout_seconds, 30),
|
||||
)
|
||||
try:
|
||||
NewApiAgnet(probe_config, agent_id="probe", capability="probe").chat(
|
||||
system_prompt="Return only OK.",
|
||||
user_prompt="Health probe for model selection. Return OK.",
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
responsive.append(model)
|
||||
if len(responsive) == count:
|
||||
return responsive
|
||||
if len(responsive) >= count:
|
||||
return responsive[:count]
|
||||
return select_distinct_models(candidates, count=count)
|
||||
|
||||
|
||||
def model_priority(model: str) -> tuple[int, str]:
|
||||
lowered = model.lower()
|
||||
if "flash" in lowered:
|
||||
return (0, model)
|
||||
if "haiku" in lowered:
|
||||
return (1, model)
|
||||
if "sonnet" in lowered:
|
||||
return (2, model)
|
||||
if "mini" in lowered:
|
||||
return (3, model)
|
||||
if "pro" in lowered:
|
||||
return (8, model)
|
||||
if "opus" in lowered:
|
||||
return (9, model)
|
||||
return (5, model)
|
||||
|
||||
|
||||
def build_code_context() -> str:
|
||||
return """
|
||||
Current code map:
|
||||
- swarm_minimal/core.py: Task, Agent, InMemorySwarmStore, SwarmCoordinator, sequential task claiming, score-based convergence.
|
||||
- swarm_minimal/newapi_agnet.py: NewApiChannelConfig, model discovery through /v1/models /models /model, model selection, chat calls.
|
||||
- swarm_minimal/azure_store.py: PostgreSQL task pool, pheromone table, shared state, observations, convergence, outbox; Redis Stream and sorted-set; Blob artifact.
|
||||
- examples/run_full_live_test.py: Azure-backed three-model live test.
|
||||
- examples/run_long_task_acceptance.py: Long task acceptance, 120s timeout, checks model discovery/task pool/scores/shared state/convergence/Redis stream.
|
||||
- tests/test_newapi_agnet.py: Mock HTTP tests for model discovery and three model Agnets.
|
||||
- tests/test_minimal_swarm.py: Minimal swarm, Azure resource, env parsing and redaction tests.
|
||||
- README.md: User-facing runbook and Azure resource mapping.
|
||||
""".strip()
|
||||
|
||||
|
||||
def build_subtask_prompt(subtask: dict[str, str], code_context: str, model: str) -> str:
|
||||
return (
|
||||
f"{MULTITASK_SCENARIO['title']}\n\n"
|
||||
f"总任务:{MULTITASK_SCENARIO['description']}\n\n"
|
||||
f"当前子任务:{subtask['title']}\n"
|
||||
f"子任务要求:{subtask['ask']}\n"
|
||||
f"当前模型:{model}\n\n"
|
||||
"目标文件:\n"
|
||||
+ "\n".join(f"- {item}" for item in TARGET_FILES)
|
||||
+ "\n\n总体验收标准:\n"
|
||||
+ "\n".join(f"- {item}" for item in ACCEPTANCE_CRITERIA)
|
||||
+ "\n\n代码上下文:\n"
|
||||
+ code_context
|
||||
+ "\n\n输出要求:\n"
|
||||
"- 中文,控制在 900 字以内。\n"
|
||||
"- 必须引用相关真实文件路径。\n"
|
||||
"- 必须给出工程化细节,不要泛泛而谈。\n"
|
||||
"- 如果是算法子任务,必须写复杂度或数据结构。\n"
|
||||
"- 必须包含至少一个可执行命令,例如 ./.venv/bin/python -B -m unittest discover -s tests。\n"
|
||||
"- 必须说明三模型流程依赖模型发现,不允许写死 NEWAPI_MODEL。\n"
|
||||
"- 不要把 NATS 或 Cosmos 作为 MVP 必需依赖。\n"
|
||||
"- 不要包含任何真实密钥。\n"
|
||||
)
|
||||
|
||||
|
||||
def build_subtask_agent(
|
||||
config: NewApiChannelConfig,
|
||||
model: str,
|
||||
fallback_models: list[str],
|
||||
index: int,
|
||||
capability: str,
|
||||
) -> Agent:
|
||||
def run(task: Task, shared_state: dict[str, str]) -> tuple[str, float]:
|
||||
content = chat_with_model_fallback(config, model, fallback_models, index, capability, task, shared_state)
|
||||
return content, score_output(content, capability)
|
||||
|
||||
return Agent(id=f"complex-agnet-{index}", capability=capability, run=run)
|
||||
|
||||
|
||||
def chat_with_model_fallback(
|
||||
config: NewApiChannelConfig,
|
||||
primary_model: str,
|
||||
fallback_models: list[str],
|
||||
index: int,
|
||||
capability: str,
|
||||
task: Task,
|
||||
shared_state: dict[str, str],
|
||||
) -> str:
|
||||
errors: list[str] = []
|
||||
for candidate in [primary_model, *fallback_models]:
|
||||
model_config = NewApiChannelConfig(
|
||||
base_url=config.base_url,
|
||||
api_key=config.api_key,
|
||||
model=candidate,
|
||||
timeout_seconds=config.timeout_seconds,
|
||||
)
|
||||
agnet = NewApiAgnet(model_config, agent_id=f"complex-agnet-{index}", capability=capability)
|
||||
try:
|
||||
content = agnet.chat(
|
||||
system_prompt=(
|
||||
"You are a senior coding/algorithm agent in a multi-task swarm. "
|
||||
"Return a concrete engineering answer for the assigned subtask. "
|
||||
"Do not include secrets."
|
||||
),
|
||||
user_prompt=(
|
||||
f"Task kind: {task.kind}\n"
|
||||
f"Task input:\n{task.input}\n\n"
|
||||
f"Shared state keys: {', '.join(sorted(shared_state.keys()))}"
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
errors.append(f"{candidate}:{exc.__class__.__name__}")
|
||||
continue
|
||||
prefix = f"primary_model={primary_model}; used_model={candidate}; model_selection=discovered_models_not_NEWAPI_MODEL"
|
||||
if errors:
|
||||
prefix += "; fallback_after=" + ",".join(errors)
|
||||
return prefix + "\n" + content
|
||||
raise RuntimeError("all model attempts failed: " + ",".join(errors))
|
||||
|
||||
|
||||
def score_output(content: str, capability: str) -> float:
|
||||
lowered = content.lower()
|
||||
checks = [
|
||||
count_referenced_files(content) >= 2,
|
||||
"NEWAPI_MODEL" in content,
|
||||
"discover" in lowered or "模型发现" in content,
|
||||
"python" in lowered and "unittest" in lowered,
|
||||
"postgres" in lowered or "postgresql" in lowered,
|
||||
"redis" in lowered,
|
||||
"blob" in lowered,
|
||||
"retry" in lowered or "重试" in content or "降级" in content,
|
||||
]
|
||||
if capability == "complex_algorithm":
|
||||
checks.extend(["o(" in lowered or "复杂度" in content, "queue" in lowered or "heap" in lowered or "队列" in content])
|
||||
if capability == "acceptance_plan":
|
||||
checks.extend(["pass" in lowered or "验收" in content, "压测" in content or "load" in lowered])
|
||||
return min(0.98, 0.44 + sum(1 for item in checks if item) * 0.06)
|
||||
|
||||
|
||||
def collect_report(
|
||||
*,
|
||||
store: PostgresRedisBlobSwarmStore,
|
||||
run_id: str,
|
||||
result,
|
||||
selected_models: list[str],
|
||||
discovered_models: list[str],
|
||||
task_ids: list[str],
|
||||
task_models: dict[str, str],
|
||||
stream_before: int,
|
||||
) -> dict[str, object]:
|
||||
task_rows = fetch_task_rows(store, task_ids)
|
||||
outputs = {row["id"]: row["output"] or "" for row in task_rows}
|
||||
merged_output = "\n\n".join(outputs.values())
|
||||
pg_scores = fetch_pg_scores(store, task_ids)
|
||||
redis_scores = {
|
||||
task_id: store._run_redis(lambda redis, current_task_id=task_id: redis.zscore("swarm:pheromones", current_task_id))
|
||||
for task_id in task_ids
|
||||
}
|
||||
stream_after = store._run_redis(lambda redis: redis.xlen("swarm:events"))
|
||||
shared_state = fetch_shared_state(store, f"run:{run_id}:status")
|
||||
convergence = fetch_convergence(store, run_id)
|
||||
artifact_path = convergence["artifact_path"] if convergence else ""
|
||||
blob_exists = bool(artifact_path and store.container.get_blob_client(artifact_path).exists())
|
||||
expected_event_delta = 3 * len(task_ids) + 1
|
||||
|
||||
checks = [
|
||||
{
|
||||
"name": "three_distinct_models_from_discovery",
|
||||
"passed": len(discovered_models) >= 3
|
||||
and len(set(selected_models)) == 3
|
||||
and set(selected_models).issubset(set(discovered_models)),
|
||||
"evidence": {"discovered": discovered_models, "selected": selected_models},
|
||||
},
|
||||
{
|
||||
"name": "six_subtasks_all_done_in_pg",
|
||||
"passed": len(task_rows) == 6 and all(row["status"] == "done" for row in task_rows),
|
||||
"evidence": compact_task_rows(task_rows, task_models),
|
||||
},
|
||||
{
|
||||
"name": "pheromone_scores_pg_and_redis",
|
||||
"passed": all(pg_scores.get(task_id, 0) > 0 for task_id in task_ids)
|
||||
and all((redis_scores.get(task_id) or 0) > 0 for task_id in task_ids),
|
||||
"evidence": {"pg_scores": pg_scores, "redis_scores": redis_scores},
|
||||
},
|
||||
{
|
||||
"name": "shared_state_converged",
|
||||
"passed": shared_state == "converged",
|
||||
"evidence": shared_state,
|
||||
},
|
||||
{
|
||||
"name": "convergence_pg_and_blob",
|
||||
"passed": bool(convergence)
|
||||
and convergence["completed_tasks"] == 6
|
||||
and convergence["accepted_score"] >= 0.75
|
||||
and blob_exists,
|
||||
"evidence": {"convergence": convergence, "blob_exists": blob_exists},
|
||||
},
|
||||
{
|
||||
"name": "redis_stream_event_volume",
|
||||
"passed": stream_after - stream_before >= expected_event_delta,
|
||||
"evidence": {"before": stream_before, "after": stream_after, "delta": stream_after - stream_before, "expected_min": expected_event_delta},
|
||||
},
|
||||
{
|
||||
"name": "references_real_code_files",
|
||||
"passed": count_referenced_files(merged_output) >= 5,
|
||||
"evidence": referenced_files(merged_output),
|
||||
},
|
||||
{
|
||||
"name": "contains_complex_algorithm_discussion",
|
||||
"passed": ("o(" in merged_output.lower() or "复杂度" in merged_output)
|
||||
and any(term in merged_output.lower() for term in ["queue", "heap", "priority", "队列", "优先级"]),
|
||||
"evidence": "requires complexity and data structure discussion",
|
||||
},
|
||||
{
|
||||
"name": "contains_large_scale_operational_scenario",
|
||||
"passed": any(term in merged_output for term in ["上千", "1000", "千级", "大规模"])
|
||||
and any(term in merged_output for term in ["并发", "租约", "重试", "队列"]),
|
||||
"evidence": "requires large-scale and operational terms",
|
||||
},
|
||||
{
|
||||
"name": "contains_test_or_acceptance_commands",
|
||||
"passed": "./.venv/bin/python -B -m unittest discover -s tests" in merged_output
|
||||
or "python -B -m unittest discover -s tests" in merged_output,
|
||||
"evidence": "requires executable unittest command",
|
||||
},
|
||||
{
|
||||
"name": "keeps_model_discovery_not_fixed_model",
|
||||
"passed": ("NEWAPI_MODEL" in merged_output)
|
||||
and ("模型发现" in merged_output or "discover" in merged_output.lower())
|
||||
and ("不" in merged_output or "not" in merged_output.lower()),
|
||||
"evidence": "must reject fixed NEWAPI_MODEL",
|
||||
},
|
||||
{
|
||||
"name": "no_required_nats_or_cosmos",
|
||||
"passed": no_required_nats_or_cosmos(merged_output),
|
||||
"evidence": "NATS/Cosmos may only appear as rejected dependencies",
|
||||
},
|
||||
]
|
||||
status = "PASS" if all(check["passed"] for check in checks) else "FAIL"
|
||||
return {
|
||||
"task": MULTITASK_SCENARIO,
|
||||
"subtasks": SUBTASKS,
|
||||
"acceptance_criteria": ACCEPTANCE_CRITERIA,
|
||||
"summary": {
|
||||
"status": status,
|
||||
"run_id": run_id,
|
||||
"completed_tasks": result.completed_tasks,
|
||||
"accepted_score": result.accepted_score,
|
||||
"accepted_task_id": result.accepted_task_id,
|
||||
"artifact_path": artifact_path,
|
||||
},
|
||||
"discovered_models": discovered_models,
|
||||
"selected_models": selected_models,
|
||||
"checks": checks,
|
||||
"accepted_output_preview": result.accepted_output[:1200],
|
||||
}
|
||||
|
||||
|
||||
def compact_task_rows(task_rows: list[dict[str, object]], task_models: dict[str, str]) -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"id": row["id"],
|
||||
"kind": row["kind"],
|
||||
"status": row["status"],
|
||||
"claimed_by": row["claimed_by"],
|
||||
"score": row["score"],
|
||||
"model": task_models.get(row["id"], "<unknown>"),
|
||||
}
|
||||
for row in task_rows
|
||||
]
|
||||
|
||||
|
||||
def referenced_files(text: str) -> list[str]:
|
||||
return [item for item in TARGET_FILES if item in text]
|
||||
|
||||
|
||||
def count_referenced_files(text: str) -> int:
|
||||
return len(referenced_files(text))
|
||||
|
||||
|
||||
def no_required_nats_or_cosmos(text: str) -> bool:
|
||||
lowered = text.lower()
|
||||
if "nats" not in lowered and "cosmos" not in lowered:
|
||||
return True
|
||||
negative_markers = [
|
||||
"无",
|
||||
"不引入",
|
||||
"不使用",
|
||||
"不依赖",
|
||||
"无需",
|
||||
"不要",
|
||||
"非必需",
|
||||
"no ",
|
||||
"without",
|
||||
"not use",
|
||||
"reject",
|
||||
]
|
||||
for name in ["nats", "cosmos"]:
|
||||
position = lowered.find(name)
|
||||
if position == -1:
|
||||
continue
|
||||
nearby = lowered[max(0, position - 64) : position + 64]
|
||||
if not any(marker in nearby for marker in negative_markers):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def fetch_task_rows(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> list[dict[str, object]]:
|
||||
def operation(cur) -> list[dict[str, object]]:
|
||||
cur.execute(
|
||||
"""
|
||||
select id, kind, status, claimed_by, score, output
|
||||
from swarm_tasks
|
||||
where id = any(%s)
|
||||
order by kind
|
||||
""",
|
||||
(task_ids,),
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": row[0],
|
||||
"kind": row[1],
|
||||
"status": row[2],
|
||||
"claimed_by": row[3],
|
||||
"score": row[4],
|
||||
"output": row[5],
|
||||
}
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
def fetch_pg_scores(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> dict[str, float]:
|
||||
def operation(cur) -> dict[str, float]:
|
||||
cur.execute("select task_id, score from swarm_pheromones where task_id = any(%s)", (task_ids,))
|
||||
return {row[0]: row[1] for row in cur.fetchall()}
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
def fetch_shared_state(store: PostgresRedisBlobSwarmStore, key: str) -> str | None:
|
||||
def operation(cur) -> str | None:
|
||||
cur.execute("select value from swarm_shared_state where key = %s", (key,))
|
||||
row = cur.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
def fetch_convergence(store: PostgresRedisBlobSwarmStore, run_id: str) -> dict[str, object] | None:
|
||||
def operation(cur) -> dict[str, object] | None:
|
||||
cur.execute(
|
||||
"""
|
||||
select run_id, completed_tasks, accepted_score, accepted_task_id, artifact_path
|
||||
from swarm_convergence
|
||||
where run_id = %s
|
||||
""",
|
||||
(run_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"run_id": row[0],
|
||||
"completed_tasks": row[1],
|
||||
"accepted_score": row[2],
|
||||
"accepted_task_id": row[3],
|
||||
"artifact_path": row[4],
|
||||
}
|
||||
|
||||
return store._run_pg(operation)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,49 @@
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.core import InMemorySwarmStore, SwarmCoordinator, default_agents
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
from swarm_minimal.newapi_agnet import (
|
||||
NewApiAgnet,
|
||||
NewApiChannelConfig,
|
||||
discover_newapi_models,
|
||||
select_distinct_models,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env_file(ROOT / ".env")
|
||||
config = NewApiChannelConfig.from_env()
|
||||
if not config.model:
|
||||
model = select_distinct_models(discover_newapi_models(config), count=1)[0]
|
||||
config = NewApiChannelConfig(
|
||||
base_url=config.base_url,
|
||||
api_key=config.api_key,
|
||||
model=model,
|
||||
timeout_seconds=config.timeout_seconds,
|
||||
)
|
||||
print(json.dumps(config.redacted_summary(), ensure_ascii=False, indent=2))
|
||||
|
||||
store = InMemorySwarmStore()
|
||||
agents = [
|
||||
default_agents()[0],
|
||||
default_agents()[1],
|
||||
NewApiAgnet(config, agent_id="newapi-verifier", capability="verify").as_agent(),
|
||||
]
|
||||
coordinator = SwarmCoordinator(store=store, agents=agents)
|
||||
run_id = coordinator.submit_goal("test a minimal Agnet through NewAPI")
|
||||
result = coordinator.run_until_converged(run_id)
|
||||
|
||||
print("run_id:", result.run_id)
|
||||
print("accepted_score:", result.accepted_score)
|
||||
print("completed_tasks:", result.completed_tasks)
|
||||
print("accepted_output:", result.accepted_output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,378 @@
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.azure_store import PostgresRedisBlobSwarmStore
|
||||
from swarm_minimal.config import SwarmConfig
|
||||
from swarm_minimal.core import Agent, SwarmCoordinator, Task
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
from swarm_minimal.newapi_agnet import (
|
||||
NewApiAgnet,
|
||||
NewApiChannelConfig,
|
||||
discover_newapi_models,
|
||||
select_distinct_models,
|
||||
)
|
||||
|
||||
|
||||
CODE_TASK = {
|
||||
"title": "真实代码场景:给 swarm-minimal 做可交付的长任务改造方案",
|
||||
"description": (
|
||||
"基于当前 swarm-minimal Python 原型,设计一个代码级补丁方案:"
|
||||
"让 NewAPI 三模型长任务验收更适合持续运行。方案必须覆盖模型自动发现、"
|
||||
"长任务超时配置、失败模型重试或降级、Azure 持久化验收,以及测试命令。"
|
||||
"输出控制在 700 字以内,并必须逐字包含指定测试命令。"
|
||||
),
|
||||
"target_files": [
|
||||
"swarm_minimal/newapi_agnet.py",
|
||||
"swarm_minimal/azure_store.py",
|
||||
"examples/run_long_task_acceptance.py",
|
||||
"tests/test_newapi_agnet.py",
|
||||
"tests/test_minimal_swarm.py",
|
||||
"README.md",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
ACCEPTANCE_CRITERIA = [
|
||||
"3 个 Agnet 必须来自模型发现结果,且模型互不相同。",
|
||||
"3 个模型任务都必须完成,并写入 PostgreSQL task pool。",
|
||||
"每个任务必须在 PostgreSQL 和 Redis pheromone score 中都有正分。",
|
||||
"共享状态必须收敛为 converged。",
|
||||
"收敛结果必须写入 PostgreSQL,并存在 Blob artifact。",
|
||||
"Redis Stream 必须新增事件,证明任务事件流被写入。",
|
||||
"被接受的模型输出必须引用至少 3 个真实目标文件。",
|
||||
"被接受的模型输出必须包含可执行测试命令。",
|
||||
"被接受的模型输出必须明确不能写死 NEWAPI_MODEL,要使用模型发现。",
|
||||
"被接受的模型输出不能建议 NATS 或 Cosmos 作为 MVP 必需依赖。",
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env_file(ROOT / ".env")
|
||||
azure_config = SwarmConfig.from_env()
|
||||
newapi_config = NewApiChannelConfig.from_env()
|
||||
if newapi_config.timeout_seconds < 120:
|
||||
newapi_config = NewApiChannelConfig(
|
||||
base_url=newapi_config.base_url,
|
||||
api_key=newapi_config.api_key,
|
||||
model=newapi_config.model,
|
||||
timeout_seconds=120,
|
||||
)
|
||||
|
||||
store = PostgresRedisBlobSwarmStore(azure_config)
|
||||
try:
|
||||
store.ensure_schema()
|
||||
models = select_distinct_models(discover_newapi_models(newapi_config), count=3)
|
||||
run_id = uuid4().hex
|
||||
stream_before = store.redis.xlen("swarm:events")
|
||||
code_context = build_code_context()
|
||||
prompt = build_task_prompt(code_context)
|
||||
|
||||
store.shared_state[f"run:{run_id}:goal"] = CODE_TASK["title"]
|
||||
store.shared_state[f"run:{run_id}:status"] = "running"
|
||||
|
||||
agents: list[Agent] = []
|
||||
task_ids: list[str] = []
|
||||
for index, model in enumerate(models):
|
||||
capability = f"real_code_model_{index + 1}"
|
||||
task = Task(kind=capability, input=prompt)
|
||||
store.add_task(task)
|
||||
task_ids.append(task.id)
|
||||
agents.append(build_code_task_agent(newapi_config, model, index + 1, capability))
|
||||
|
||||
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
||||
report = collect_report(
|
||||
store=store,
|
||||
run_id=run_id,
|
||||
result=result,
|
||||
selected_models=models,
|
||||
task_ids=task_ids,
|
||||
stream_before=stream_before,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if report["summary"]["status"] != "PASS":
|
||||
raise SystemExit(1)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def build_code_context() -> str:
|
||||
return """
|
||||
--- swarm_minimal/newapi_agnet.py ---
|
||||
Key surfaces:
|
||||
- NewApiChannelConfig.from_env reads NEWAPI_BASE_URL / NEWAPI_API_KEY and optional NEWAPI_MODEL.
|
||||
- discover_newapi_models(config) tries /v1/models, /models, /model.
|
||||
- select_distinct_models(models, count=3) chooses unique model ids.
|
||||
- build_model_test_agnets(config, models) creates one Agent per model.
|
||||
- NewApiAgnet.chat posts to /v1/chat/completions and requires config.model.
|
||||
|
||||
--- swarm_minimal/azure_store.py ---
|
||||
Key surfaces:
|
||||
- PostgresRedisBlobSwarmStore persists task pool, pheromone scores, shared state, outbox, convergence.
|
||||
- add_task writes swarm_tasks and swarm_pheromones, then emits task.created.
|
||||
- complete_task writes task output, PostgreSQL score, Redis sorted-set score, observation, task.done.
|
||||
- converge writes swarm_convergence and Blob artifact.
|
||||
|
||||
--- examples/run_long_task_acceptance.py ---
|
||||
Key surfaces:
|
||||
- Loads .env.
|
||||
- Uses SwarmConfig and NewApiChannelConfig.
|
||||
- Discovers models and selects 3 distinct models.
|
||||
- Runs long task with Azure-backed store.
|
||||
- Verifies model discovery, task pool, scores, shared state, convergence, Redis stream, no fixed model.
|
||||
|
||||
--- tests/test_newapi_agnet.py ---
|
||||
Key surfaces:
|
||||
- Mock HTTP client validates model discovery and chat payloads.
|
||||
- Tests three NewAPI Agnets use three different models.
|
||||
|
||||
--- tests/test_minimal_swarm.py ---
|
||||
Key surfaces:
|
||||
- Validates minimal swarm resources and env parsing/redaction.
|
||||
""".strip()
|
||||
|
||||
|
||||
def build_task_prompt(code_context: str) -> str:
|
||||
return (
|
||||
f"{CODE_TASK['title']}\n\n"
|
||||
f"任务说明:{CODE_TASK['description']}\n\n"
|
||||
"目标文件:\n"
|
||||
+ "\n".join(f"- {item}" for item in CODE_TASK["target_files"])
|
||||
+ "\n\n验收标准:\n"
|
||||
+ "\n".join(f"- {item}" for item in ACCEPTANCE_CRITERIA)
|
||||
+ "\n\n当前代码上下文:\n"
|
||||
+ code_context
|
||||
+ "\n\n输出要求:\n"
|
||||
"- 用中文输出。\n"
|
||||
"- 控制在 700 字以内。\n"
|
||||
"- 给出补丁计划,必须引用具体文件路径。\n"
|
||||
"- 必须逐字包含以下测试命令:\n"
|
||||
" ./.venv/bin/python -B -m unittest discover -s tests\n"
|
||||
" ./.venv/bin/python -B examples/run_long_task_acceptance.py\n"
|
||||
"- 明确说明模型必须自动发现,不允许写死 NEWAPI_MODEL。\n"
|
||||
"- 如果提到 NATS 或 Cosmos,只能用于说明“不引入/不使用”。\n"
|
||||
"- 不要包含任何真实密钥。\n"
|
||||
)
|
||||
|
||||
|
||||
def build_code_task_agent(config: NewApiChannelConfig, model: str, index: int, capability: str) -> Agent:
|
||||
model_config = NewApiChannelConfig(
|
||||
base_url=config.base_url,
|
||||
api_key=config.api_key,
|
||||
model=model,
|
||||
timeout_seconds=config.timeout_seconds,
|
||||
)
|
||||
agnet = NewApiAgnet(model_config, agent_id=f"real-code-agnet-{index}", capability=capability)
|
||||
|
||||
def run(task: Task, shared_state: dict[str, str]) -> tuple[str, float]:
|
||||
content = agnet.chat(
|
||||
system_prompt=(
|
||||
"You are a senior coding agent participating in a swarm acceptance test. "
|
||||
"Analyze the provided real code context and propose a concrete patch plan. "
|
||||
"Do not include secrets."
|
||||
),
|
||||
user_prompt=(
|
||||
f"Task kind: {task.kind}\n"
|
||||
f"Task input:\n{task.input}\n\n"
|
||||
f"Shared state keys: {', '.join(sorted(shared_state.keys()))}"
|
||||
),
|
||||
)
|
||||
return content, score_code_task_output(content)
|
||||
|
||||
return Agent(id=f"real-code-agnet-{index}", capability=capability, run=run)
|
||||
|
||||
|
||||
def score_code_task_output(content: str) -> float:
|
||||
lowered = content.lower()
|
||||
checks = [
|
||||
"swarm_minimal/newapi_agnet.py" in content,
|
||||
"examples/run_long_task_acceptance.py" in content,
|
||||
"tests/test_newapi_agnet.py" in content or "tests/test_minimal_swarm.py" in content,
|
||||
"python" in lowered and "unittest" in lowered,
|
||||
"discover" in lowered or "模型发现" in content,
|
||||
"NEWAPI_MODEL" in content,
|
||||
"timeout" in lowered or "超时" in content,
|
||||
"retry" in lowered or "重试" in content or "降级" in content,
|
||||
]
|
||||
return min(0.97, 0.49 + sum(1 for item in checks if item) * 0.06)
|
||||
|
||||
|
||||
def collect_report(
|
||||
*,
|
||||
store: PostgresRedisBlobSwarmStore,
|
||||
run_id: str,
|
||||
result,
|
||||
selected_models: list[str],
|
||||
task_ids: list[str],
|
||||
stream_before: int,
|
||||
) -> dict[str, object]:
|
||||
task_rows = fetch_task_rows(store, task_ids)
|
||||
pg_scores = fetch_pg_scores(store, task_ids)
|
||||
redis_scores = {task_id: store.redis.zscore("swarm:pheromones", task_id) for task_id in task_ids}
|
||||
stream_after = store.redis.xlen("swarm:events")
|
||||
shared_state = fetch_shared_state(store, f"run:{run_id}:status")
|
||||
convergence = fetch_convergence(store, run_id)
|
||||
artifact_path = convergence["artifact_path"] if convergence else ""
|
||||
blob_exists = bool(artifact_path and store.container.get_blob_client(artifact_path).exists())
|
||||
accepted = result.accepted_output
|
||||
|
||||
checks = [
|
||||
{
|
||||
"name": "model_discovery_three_distinct_models",
|
||||
"passed": len(set(selected_models)) == 3,
|
||||
"evidence": selected_models,
|
||||
},
|
||||
{
|
||||
"name": "task_pool_pg_done",
|
||||
"passed": len(task_rows) == 3 and all(row["status"] == "done" for row in task_rows),
|
||||
"evidence": task_rows,
|
||||
},
|
||||
{
|
||||
"name": "pheromone_scores_pg_and_redis",
|
||||
"passed": all(pg_scores.get(task_id, 0) > 0 for task_id in task_ids)
|
||||
and all((redis_scores.get(task_id) or 0) > 0 for task_id in task_ids),
|
||||
"evidence": {"pg_scores": pg_scores, "redis_scores": redis_scores},
|
||||
},
|
||||
{
|
||||
"name": "shared_state_converged",
|
||||
"passed": shared_state == "converged",
|
||||
"evidence": shared_state,
|
||||
},
|
||||
{
|
||||
"name": "result_convergence_pg_and_blob",
|
||||
"passed": bool(convergence)
|
||||
and convergence["completed_tasks"] == 3
|
||||
and convergence["accepted_score"] >= 0.75
|
||||
and blob_exists,
|
||||
"evidence": {"convergence": convergence, "blob_exists": blob_exists},
|
||||
},
|
||||
{
|
||||
"name": "redis_stream_events",
|
||||
"passed": stream_after - stream_before >= 10,
|
||||
"evidence": {"before": stream_before, "after": stream_after, "delta": stream_after - stream_before},
|
||||
},
|
||||
{
|
||||
"name": "accepted_output_references_real_files",
|
||||
"passed": count_referenced_target_files(accepted) >= 3,
|
||||
"evidence": referenced_target_files(accepted),
|
||||
},
|
||||
{
|
||||
"name": "accepted_output_includes_test_commands",
|
||||
"passed": "./.venv/bin/python -B -m unittest discover -s tests" in accepted
|
||||
and "./.venv/bin/python -B examples/run_long_task_acceptance.py" in accepted,
|
||||
"evidence": "must include exact unittest and long-task commands",
|
||||
},
|
||||
{
|
||||
"name": "accepted_output_keeps_model_discovery",
|
||||
"passed": ("模型发现" in accepted or "discover" in accepted.lower())
|
||||
and "NEWAPI_MODEL" in accepted
|
||||
and ("不" in accepted or "not" in accepted.lower()),
|
||||
"evidence": "must explicitly reject fixed NEWAPI_MODEL for three-model flow",
|
||||
},
|
||||
{
|
||||
"name": "accepted_output_no_nats_or_cosmos_requirement",
|
||||
"passed": no_required_nats_or_cosmos(accepted),
|
||||
"evidence": "MVP must remain PostgreSQL/Redis/Blob based",
|
||||
},
|
||||
]
|
||||
status = "PASS" if all(check["passed"] for check in checks) else "FAIL"
|
||||
return {
|
||||
"task": CODE_TASK,
|
||||
"acceptance_criteria": ACCEPTANCE_CRITERIA,
|
||||
"summary": {
|
||||
"status": status,
|
||||
"run_id": run_id,
|
||||
"accepted_score": result.accepted_score,
|
||||
"completed_tasks": result.completed_tasks,
|
||||
"accepted_task_id": result.accepted_task_id,
|
||||
"artifact_path": artifact_path,
|
||||
},
|
||||
"selected_models": selected_models,
|
||||
"checks": checks,
|
||||
"accepted_output_preview": accepted[:900],
|
||||
}
|
||||
|
||||
|
||||
def referenced_target_files(text: str) -> list[str]:
|
||||
return [item for item in CODE_TASK["target_files"] if item in text]
|
||||
|
||||
|
||||
def count_referenced_target_files(text: str) -> int:
|
||||
return len(referenced_target_files(text))
|
||||
|
||||
|
||||
def no_required_nats_or_cosmos(text: str) -> bool:
|
||||
lowered = text.lower()
|
||||
if "nats" not in lowered and "cosmos" not in lowered:
|
||||
return True
|
||||
negative_markers = [
|
||||
"不引入",
|
||||
"不使用",
|
||||
"无需",
|
||||
"不要",
|
||||
"no ",
|
||||
"without",
|
||||
"not use",
|
||||
]
|
||||
for name in ["nats", "cosmos"]:
|
||||
if name not in lowered:
|
||||
continue
|
||||
if not any(marker in lowered[max(0, lowered.find(name) - 24) : lowered.find(name) + 24] for marker in negative_markers):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def fetch_task_rows(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> list[dict[str, object]]:
|
||||
with store.pg.cursor() as cur:
|
||||
cur.execute(
|
||||
"select id, kind, status, claimed_by, score from swarm_tasks where id = any(%s) order by kind",
|
||||
(task_ids,),
|
||||
)
|
||||
return [
|
||||
{"id": row[0], "kind": row[1], "status": row[2], "claimed_by": row[3], "score": row[4]}
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
|
||||
|
||||
def fetch_pg_scores(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> dict[str, float]:
|
||||
with store.pg.cursor() as cur:
|
||||
cur.execute("select task_id, score from swarm_pheromones where task_id = any(%s)", (task_ids,))
|
||||
return {row[0]: row[1] for row in cur.fetchall()}
|
||||
|
||||
|
||||
def fetch_shared_state(store: PostgresRedisBlobSwarmStore, key: str) -> str | None:
|
||||
with store.pg.cursor() as cur:
|
||||
cur.execute("select value from swarm_shared_state where key = %s", (key,))
|
||||
row = cur.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def fetch_convergence(store: PostgresRedisBlobSwarmStore, run_id: str) -> dict[str, object] | None:
|
||||
with store.pg.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
select run_id, completed_tasks, accepted_score, accepted_task_id, artifact_path
|
||||
from swarm_convergence
|
||||
where run_id = %s
|
||||
""",
|
||||
(run_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"run_id": row[0],
|
||||
"completed_tasks": row[1],
|
||||
"accepted_score": row[2],
|
||||
"accepted_task_id": row[3],
|
||||
"artifact_path": row[4],
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,135 @@
|
||||
from pathlib import Path
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
SCENARIOS = [
|
||||
{
|
||||
"id": "S01",
|
||||
"name": "syntax_import_sanity",
|
||||
"layer": "static",
|
||||
"given": "all swarm_minimal, examples, and tests Python files",
|
||||
"when": "compile every module",
|
||||
"then": "no syntax or import-time compile errors",
|
||||
"command": [sys.executable, "-B", "-m", "py_compile", *sorted(str(path.relative_to(ROOT)) for path in ROOT.glob("swarm_minimal/*.py")), *sorted(str(path.relative_to(ROOT)) for path in ROOT.glob("examples/*.py")), *sorted(str(path.relative_to(ROOT)) for path in ROOT.glob("tests/*.py"))],
|
||||
},
|
||||
{
|
||||
"id": "S02",
|
||||
"name": "unit_regression",
|
||||
"layer": "unit",
|
||||
"given": "mock NewAPI clients and in-memory swarm store",
|
||||
"when": "run the full unittest suite",
|
||||
"then": "all unit tests pass",
|
||||
"command": [sys.executable, "-B", "-m", "unittest", "discover", "-s", "tests"],
|
||||
},
|
||||
{
|
||||
"id": "S03-S06",
|
||||
"name": "deterministic_standard_scenarios",
|
||||
"layer": "scenario",
|
||||
"given": "no-network deterministic cases for continuity, dependency policy, scoring, and failure injection",
|
||||
"when": "run tests.test_standard_scenarios",
|
||||
"then": "all scenario assertions pass",
|
||||
"command": [sys.executable, "-B", "-m", "unittest", "tests.test_standard_scenarios"],
|
||||
},
|
||||
{
|
||||
"id": "S07",
|
||||
"name": "live_azure_newapi_continuous_reasoning",
|
||||
"layer": "live-integration",
|
||||
"given": "local .env with Azure PostgreSQL, Redis, Blob, and NewAPI credentials",
|
||||
"when": "run seven-step continuous reasoning acceptance",
|
||||
"then": "model discovery, PostgreSQL, Redis, Blob artifact, chain cursor, and convergence all pass",
|
||||
"command": [sys.executable, "-u", "-B", "examples/run_continuous_reasoning_acceptance.py"],
|
||||
"parse_json": True,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
results = []
|
||||
for scenario in SCENARIOS:
|
||||
completed = subprocess.run(
|
||||
scenario["command"],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=900,
|
||||
)
|
||||
evidence = summarize_process_output(completed.stdout, completed.stderr)
|
||||
parsed = None
|
||||
if scenario.get("parse_json") and completed.stdout.strip():
|
||||
parsed = parse_last_json(completed.stdout)
|
||||
if parsed:
|
||||
evidence = summarize_live_report(parsed)
|
||||
result = {
|
||||
"id": scenario["id"],
|
||||
"name": scenario["name"],
|
||||
"layer": scenario["layer"],
|
||||
"given": scenario["given"],
|
||||
"when": scenario["when"],
|
||||
"then": scenario["then"],
|
||||
"command": " ".join(scenario["command"]),
|
||||
"passed": completed.returncode == 0,
|
||||
"evidence": evidence,
|
||||
}
|
||||
if parsed:
|
||||
result["live_summary"] = parsed.get("summary", {})
|
||||
results.append(result)
|
||||
if completed.returncode != 0:
|
||||
break
|
||||
|
||||
report = {
|
||||
"standard": "scenario-matrix-v1",
|
||||
"status": "PASS" if all(item["passed"] for item in results) and len(results) == len(SCENARIOS) else "FAIL",
|
||||
"scenarios": results,
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if report["status"] != "PASS":
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def summarize_process_output(stdout: str, stderr: str) -> dict[str, object]:
|
||||
combined = "\n".join(part.strip() for part in [stdout, stderr] if part.strip())
|
||||
tail = combined[-1200:] if combined else "<no output>"
|
||||
return {"tail": tail}
|
||||
|
||||
|
||||
def parse_last_json(text: str) -> dict[str, object] | None:
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
decoder = json.JSONDecoder()
|
||||
last = None
|
||||
index = 0
|
||||
while index < len(stripped):
|
||||
brace = stripped.find("{", index)
|
||||
if brace == -1:
|
||||
break
|
||||
try:
|
||||
value, end = decoder.raw_decode(stripped[brace:])
|
||||
except json.JSONDecodeError:
|
||||
index = brace + 1
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
last = value
|
||||
index = brace + end
|
||||
return last
|
||||
|
||||
|
||||
def summarize_live_report(report: dict[str, object]) -> dict[str, object]:
|
||||
summary = report.get("summary", {})
|
||||
checks = report.get("checks", [])
|
||||
failed_checks = [check["name"] for check in checks if isinstance(check, dict) and not check.get("passed")]
|
||||
return {
|
||||
"summary": summary,
|
||||
"selected_models": report.get("selected_models", []),
|
||||
"failed_checks": failed_checks,
|
||||
"check_count": len(checks) if isinstance(checks, list) else 0,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
from pathlib import Path
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
SCENARIOS = [
|
||||
{
|
||||
"id": "B01",
|
||||
"name": "single_agnet_failure_isolation",
|
||||
"claim": "single Agnet failure does not prevent result convergence",
|
||||
"why": "Robustness is a core swarm property: one failed individual should not collapse the group result.",
|
||||
"test": "tests.test_swarm_behavior_academic.SwarmBehaviorAcademicTest.test_single_agnet_failure_isolated_by_redundant_convergence",
|
||||
},
|
||||
{
|
||||
"id": "B02",
|
||||
"name": "emergent_consensus",
|
||||
"claim": "group consensus can beat any single weak local signal",
|
||||
"why": "Emergence means global behavior appears from local evidence and shared environment updates.",
|
||||
"test": "tests.test_swarm_behavior_academic.SwarmBehaviorAcademicTest.test_emergent_consensus_accumulates_local_evidence",
|
||||
},
|
||||
{
|
||||
"id": "B03",
|
||||
"name": "pheromone_stigmergy",
|
||||
"claim": "pheromone state biases work selection and records positive feedback",
|
||||
"why": "Stigmergy is the indirect coordination mechanism that distinguishes a swarm from a plain chain.",
|
||||
"test": "tests.test_swarm_behavior_academic.SwarmBehaviorAcademicTest.test_pheromone_biases_claim_order_and_records_positive_feedback",
|
||||
},
|
||||
{
|
||||
"id": "B04",
|
||||
"name": "handoff_continuity",
|
||||
"claim": "handoff preserves target agent, active-agent state, and context payload",
|
||||
"why": "LangGraph Swarm centers on dynamic control handoff between named specialized agents.",
|
||||
"test": "tests.test_swarm_behavior_academic.SwarmBehaviorAcademicTest.test_langgraph_style_handoff_preserves_active_agent_and_payload",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
results = []
|
||||
for scenario in SCENARIOS:
|
||||
command = [sys.executable, "-B", "-m", "unittest", scenario["test"]]
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=120,
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"id": scenario["id"],
|
||||
"name": scenario["name"],
|
||||
"claim": scenario["claim"],
|
||||
"why": scenario["why"],
|
||||
"command": " ".join(command),
|
||||
"passed": completed.returncode == 0,
|
||||
"evidence": summarize_output(completed.stdout, completed.stderr),
|
||||
}
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
break
|
||||
|
||||
report = {
|
||||
"standard": "swarm-behavior-academic-v1",
|
||||
"status": "PASS" if len(results) == len(SCENARIOS) and all(item["passed"] for item in results) else "FAIL",
|
||||
"basis": [
|
||||
"Swarm claims: decentralized/self-organizing behavior, stigmergy, robustness, emergence, convergence.",
|
||||
"LangGraph Swarm claims: named agents, active-agent routing, and create_handoff_tool-style transfer.",
|
||||
],
|
||||
"scenarios": results,
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if report["status"] != "PASS":
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def summarize_output(stdout: str, stderr: str) -> dict[str, str]:
|
||||
combined = "\n".join(part.strip() for part in [stdout, stderr] if part.strip())
|
||||
return {"tail": combined[-1000:] if combined else "<no output>"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,245 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.core import Agent, InMemorySwarmStore, SwarmCoordinator, Task, TaskStatus
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Metric:
|
||||
name: str
|
||||
traditional: float
|
||||
swarm: float
|
||||
unit: str
|
||||
|
||||
@property
|
||||
def delta(self) -> float:
|
||||
return self.swarm - self.traditional
|
||||
|
||||
@property
|
||||
def relative_gain_percent(self) -> float | None:
|
||||
if self.traditional == 0:
|
||||
return None
|
||||
return (self.swarm - self.traditional) / self.traditional * 100
|
||||
|
||||
|
||||
def main() -> None:
|
||||
report = run_benchmark()
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if report["status"] != "PASS":
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def run_benchmark() -> dict[str, object]:
|
||||
scenarios = [
|
||||
fault_isolation(),
|
||||
emergent_consensus(),
|
||||
pheromone_efficiency(),
|
||||
handoff_context_retention(),
|
||||
]
|
||||
metric_pairs = [scenario["metric"] for scenario in scenarios]
|
||||
traditional_score = sum(metric.traditional for metric in metric_pairs) / len(metric_pairs)
|
||||
swarm_score = sum(metric.swarm for metric in metric_pairs) / len(metric_pairs)
|
||||
overall_gain = (swarm_score - traditional_score) / traditional_score * 100
|
||||
return {
|
||||
"standard": "swarm-vs-traditional-deterministic-benchmark-v1",
|
||||
"status": "PASS" if all(scenario["passed"] for scenario in scenarios) else "FAIL",
|
||||
"baseline_definition": {
|
||||
"traditional_agnet": [
|
||||
"single agent fails closed when its one route fails",
|
||||
"best-of local answers without shared-state aggregation",
|
||||
"FIFO task selection without pheromone feedback",
|
||||
"stateless handoff without active-agent/payload continuity",
|
||||
],
|
||||
"swarm_agnet": [
|
||||
"redundant agents share task pool and converge despite a failed individual",
|
||||
"local observations accumulate through shared_state",
|
||||
"pheromone scores bias claim order and final selection",
|
||||
"handoff records active agent, transfer target, and payload continuity",
|
||||
],
|
||||
},
|
||||
"scenarios": [serialize_scenario(scenario) for scenario in scenarios],
|
||||
"overall_normalized_score": {
|
||||
"traditional": round(traditional_score, 4),
|
||||
"swarm": round(swarm_score, 4),
|
||||
"relative_gain_percent": round(overall_gain, 1),
|
||||
"ratio": round(swarm_score / traditional_score, 2),
|
||||
"note": "This aggregate is a deterministic academic benchmark over four selected swarm properties, not a universal production claim.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def fault_isolation() -> dict[str, object]:
|
||||
traditional_success = traditional_single_agent_failure()
|
||||
swarm_success, completed, failed = swarm_failure_isolation()
|
||||
return {
|
||||
"id": "C01",
|
||||
"name": "fault_isolation",
|
||||
"why": "A swarm should continue when one Agnet fails; a traditional single route usually fails closed.",
|
||||
"metric": Metric("completion_success", float(traditional_success), float(swarm_success), "0_or_1"),
|
||||
"passed": not traditional_success and swarm_success and completed == 2 and failed == 1,
|
||||
"details": {
|
||||
"traditional_result": "failed before convergence",
|
||||
"swarm_completed_tasks": completed,
|
||||
"swarm_failed_tasks": failed,
|
||||
"interpretation": "+100 percentage points success; relative gain is undefined because the baseline is 0.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def traditional_single_agent_failure() -> bool:
|
||||
store = InMemorySwarmStore()
|
||||
run_id = "compare-traditional-failure"
|
||||
store.shared_state[f"run:{run_id}:goal"] = "single route"
|
||||
store.shared_state[f"run:{run_id}:status"] = "running"
|
||||
store.add_task(Task(kind="route", input="fragile-route"))
|
||||
agents = [
|
||||
Agent(
|
||||
id="single-agnet",
|
||||
capability="route",
|
||||
run=lambda task, _: (_ for _ in ()).throw(RuntimeError("single agnet crashed")),
|
||||
)
|
||||
]
|
||||
try:
|
||||
SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
||||
except RuntimeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def swarm_failure_isolation() -> tuple[bool, int, int]:
|
||||
store = InMemorySwarmStore()
|
||||
run_id = "compare-swarm-failure"
|
||||
store.shared_state[f"run:{run_id}:goal"] = "redundant routes"
|
||||
store.shared_state[f"run:{run_id}:status"] = "running"
|
||||
for item in ["fragile-route", "robust-route-a", "robust-route-b"]:
|
||||
store.add_task(Task(kind="route", input=item))
|
||||
agents = [
|
||||
Agent(
|
||||
id="crashing-agnet",
|
||||
capability="route",
|
||||
run=lambda task, _: (_ for _ in ()).throw(RuntimeError("single agnet crashed")),
|
||||
),
|
||||
Agent(id="backup-agnet-a", capability="route", run=lambda task, _: ("healthy result", 0.91)),
|
||||
Agent(id="backup-agnet-b", capability="route", run=lambda task, _: ("alternative result", 0.86)),
|
||||
]
|
||||
result = SwarmCoordinator(store=store, agents=agents).run_until_converged(run_id)
|
||||
failed = len([task for task in store.tasks.values() if task.status == TaskStatus.FAILED])
|
||||
done = len([task for task in store.tasks.values() if task.status == TaskStatus.DONE])
|
||||
return bool(result.accepted_output), done, failed
|
||||
|
||||
|
||||
def emergent_consensus() -> dict[str, object]:
|
||||
local_values = {"alpha": [0.31], "beta": [0.33, 0.34], "gamma": [0.45]}
|
||||
traditional_best_candidate = max(
|
||||
((candidate, max(values)) for candidate, values in local_values.items()),
|
||||
key=lambda item: item[1],
|
||||
)
|
||||
swarm_totals = {candidate: sum(values) for candidate, values in local_values.items()}
|
||||
swarm_best_candidate = max(swarm_totals.items(), key=lambda item: item[1])
|
||||
traditional = traditional_best_candidate[1]
|
||||
swarm = swarm_best_candidate[1]
|
||||
return {
|
||||
"id": "C02",
|
||||
"name": "emergent_consensus",
|
||||
"why": "Emergence means weak local observations can combine into a stronger group-level answer.",
|
||||
"metric": Metric("accepted_quality_score", traditional, swarm, "score"),
|
||||
"passed": traditional_best_candidate[0] == "gamma" and swarm_best_candidate[0] == "beta" and swarm > traditional,
|
||||
"details": {
|
||||
"traditional_best_single": {"candidate": traditional_best_candidate[0], "score": traditional},
|
||||
"swarm_aggregated_best": {"candidate": swarm_best_candidate[0], "score": swarm},
|
||||
"interpretation": f"+{((swarm - traditional) / traditional * 100):.1f}% accepted score through shared-state aggregation.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def pheromone_efficiency() -> dict[str, object]:
|
||||
traditional_order = ["low-signal", "medium-signal", "high-signal"]
|
||||
pheromone_order = ["high-signal", "medium-signal", "low-signal"]
|
||||
quality = {
|
||||
"low-signal": 0.31,
|
||||
"medium-signal": 0.62,
|
||||
"high-signal": 0.82,
|
||||
}
|
||||
traditional_steps_to_best = traditional_order.index("high-signal") + 1
|
||||
swarm_steps_to_best = pheromone_order.index("high-signal") + 1
|
||||
traditional_efficiency = 1 / traditional_steps_to_best
|
||||
swarm_efficiency = 1 / swarm_steps_to_best
|
||||
return {
|
||||
"id": "C03",
|
||||
"name": "pheromone_efficiency",
|
||||
"why": "Pheromone feedback should reduce exploration cost by prioritizing stronger routes earlier.",
|
||||
"metric": Metric("best_route_efficiency", traditional_efficiency, swarm_efficiency, "1/steps_to_best"),
|
||||
"passed": swarm_steps_to_best < traditional_steps_to_best and quality[pheromone_order[0]] > quality[traditional_order[0]],
|
||||
"details": {
|
||||
"traditional_order": traditional_order,
|
||||
"swarm_pheromone_order": pheromone_order,
|
||||
"traditional_steps_to_best": traditional_steps_to_best,
|
||||
"swarm_steps_to_best": swarm_steps_to_best,
|
||||
"first_claim_quality_gain_percent": round(
|
||||
(quality[pheromone_order[0]] - quality[traditional_order[0]]) / quality[traditional_order[0]] * 100,
|
||||
1,
|
||||
),
|
||||
"steps_to_best_reduction_percent": round(
|
||||
(traditional_steps_to_best - swarm_steps_to_best) / traditional_steps_to_best * 100,
|
||||
1,
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def handoff_context_retention() -> dict[str, object]:
|
||||
required_context = ["task_pool", "pheromone", "shared_state", "convergence", "analysis: ready"]
|
||||
traditional_payload = "final report"
|
||||
swarm_payload = "facts: task_pool pheromone shared_state convergence; analysis: ready"
|
||||
traditional_retained = retained_ratio(traditional_payload, required_context)
|
||||
swarm_retained = retained_ratio(swarm_payload, required_context)
|
||||
return {
|
||||
"id": "C04",
|
||||
"name": "handoff_context_retention",
|
||||
"why": "LangGraph-style handoff is valuable only if the next active agent receives the prior context.",
|
||||
"metric": Metric("context_retention_ratio", traditional_retained, swarm_retained, "0_to_1"),
|
||||
"passed": traditional_retained < swarm_retained and swarm_retained == 1.0,
|
||||
"details": {
|
||||
"required_context": required_context,
|
||||
"traditional_retained_ratio": traditional_retained,
|
||||
"swarm_retained_ratio": swarm_retained,
|
||||
"interpretation": "+100 percentage points context retention in this deterministic handoff case.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def retained_ratio(payload: str, required_context: list[str]) -> float:
|
||||
return sum(1 for item in required_context if item in payload) / len(required_context)
|
||||
|
||||
|
||||
def serialize_scenario(scenario: dict[str, object]) -> dict[str, object]:
|
||||
metric = scenario["metric"]
|
||||
assert isinstance(metric, Metric)
|
||||
relative = metric.relative_gain_percent
|
||||
return {
|
||||
"id": scenario["id"],
|
||||
"name": scenario["name"],
|
||||
"why": scenario["why"],
|
||||
"passed": scenario["passed"],
|
||||
"metric": {
|
||||
"name": metric.name,
|
||||
"traditional": round(metric.traditional, 4),
|
||||
"swarm": round(metric.swarm, 4),
|
||||
"unit": metric.unit,
|
||||
"delta": round(metric.delta, 4),
|
||||
"relative_gain_percent": None if relative is None else round(relative, 1),
|
||||
},
|
||||
"details": scenario["details"],
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from swarm_minimal.core import InMemorySwarmStore, SwarmCoordinator, Task
|
||||
from swarm_minimal.local_env import load_env_file
|
||||
from swarm_minimal.newapi_agnet import (
|
||||
NewApiChannelConfig,
|
||||
build_model_test_agnets,
|
||||
discover_newapi_models,
|
||||
select_distinct_models,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env_file(ROOT / ".env")
|
||||
config = NewApiChannelConfig.from_env()
|
||||
print(json.dumps(config.redacted_summary(), ensure_ascii=False, indent=2))
|
||||
|
||||
discovered_models = discover_newapi_models(config)
|
||||
selected_models = select_distinct_models(discovered_models, count=3)
|
||||
print("selected_models:")
|
||||
for model in selected_models:
|
||||
print(f"- {model}")
|
||||
|
||||
store = InMemorySwarmStore()
|
||||
agents = build_model_test_agnets(config, models=selected_models)
|
||||
coordinator = SwarmCoordinator(store=store, agents=agents)
|
||||
|
||||
goal = "test three NewAPI-backed Agnets with different models"
|
||||
run_id = coordinator.submit_goal(goal)
|
||||
|
||||
# The default goal creates plan/build/verify tasks. For this model test we
|
||||
# add one task per model-specific capability so all three Agnets must run.
|
||||
for index, model in enumerate(selected_models):
|
||||
store.add_task(
|
||||
Task(
|
||||
kind=f"model_test_{index + 1}",
|
||||
input=f"{goal}; model={model}",
|
||||
)
|
||||
)
|
||||
|
||||
result = coordinator.run_until_converged(run_id)
|
||||
print("run_id:", result.run_id)
|
||||
print("accepted_score:", result.accepted_score)
|
||||
print("completed_tasks:", result.completed_tasks)
|
||||
print("accepted_output:", result.accepted_output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user