gongzhiyongandClaude Opus 4.8
1288fd19d7
按 juejin 协作聚合模型重构收敛(取代 best-of-N 选优): - 删蜂后选优(queen.py/test-queen.py) - 新增聚合节点 result_aggregator.py:共享池收集→同文件 LLM/AST 整合→沙箱验证→单次落 main - 质量驱动闭环:不达标打回迭代(AGGREGATE_ACCEPTANCE_THRESHOLD + MAX_REVIEW_CYCLES) - agent 停 git 工作分支,产出走 task.result.files 共享池(AGENT_GIT_PUSH_ENABLED 默认 false) - sandbox_runner 改用 pytest(原生支持 pytest 风格 class),修 stdlib runner 收集失败 - 文档同步重写为协作聚合模型 本地验证:产物仓单分支 main + 三函数完整 + pytest 12/12 pass_rate=100 一次达标。 影响:Swarm 收敛/聚合层;Manager/客户端契约不变(artifact字段/sequence/状态机;契约测试全过)。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
106 lines
5.2 KiB
Python
106 lines
5.2 KiB
Python
"""ResultAggregator(质量驱动协作聚合)测试 — 取代旧 test-queen(best-of-N 选优)。
|
|
|
|
覆盖文章模型 + ①②③闭环:收集共享池互补产出、AST 函数级整合(无 LLM 兜底不靠"最长版本")、
|
|
质量门决策(达标 accept / 不达标 bounce 打回 / 无法评分诚实不打回)、终态聚合 best-effort。
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
os.environ.setdefault("REDIS_FAKE", "1")
|
|
os.environ.setdefault("ALLOW_MEMORY_STORE", "1")
|
|
for k in ("OPENAI_API_KEY", "MODEL_API_KEY"): # 无 LLM,走 AST 兜底(确定性,不依赖外部模型)
|
|
os.environ.pop(k, None)
|
|
|
|
from orchestrator.result_aggregator import (
|
|
_strip_fence, _ast_merge_python, _merge_fallback, _decide,
|
|
_collect_per_path, merge_artifacts, finalize_run, _emit_patch_mode,
|
|
)
|
|
|
|
_failures = []
|
|
|
|
|
|
def check(name, cond):
|
|
print(("PASS" if cond else "FAIL") + " - " + name)
|
|
if not cond:
|
|
_failures.append(name)
|
|
|
|
|
|
class FakeTask:
|
|
def __init__(self, task_id, files):
|
|
self.task_id = task_id
|
|
self.assigned_agent_id = "agent-" + task_id
|
|
self.agent_role = "impl"
|
|
self.result = json.dumps({"files": files})
|
|
|
|
|
|
class FakeRun:
|
|
def __init__(self, metadata=None):
|
|
self.metadata = metadata or {}
|
|
self.swarm_id = "swarm-test"
|
|
|
|
|
|
# --- _strip_fence ---
|
|
check("strip fence python", _strip_fence("```python\nx = 1\n```") == "x = 1")
|
|
check("strip fence passthrough", _strip_fence("z = 3") == "z = 3")
|
|
|
|
# --- ① AST 函数级合并:各 agent 写不同函数 → 合并成含全部函数的一份(不靠最长版本) ---
|
|
v1 = "def is_palindrome(s):\n return s == s[::-1]\n"
|
|
v2 = "def count_vowels(s):\n return sum(c in 'aeiou' for c in s)\ndef reverse_words(s):\n return ' '.join(s.split()[::-1])\n"
|
|
merged_py = _ast_merge_python([v1, v2])
|
|
check("AST 合并含 is_palindrome", merged_py and "def is_palindrome" in merged_py)
|
|
check("AST 合并含 count_vowels", merged_py and "def count_vowels" in merged_py)
|
|
check("AST 合并含 reverse_words", merged_py and "def reverse_words" in merged_py)
|
|
check("AST 合并去重(同名一次)", merged_py and merged_py.count("def is_palindrome") == 1 if merged_py else False)
|
|
# 语法垃圾 → None(回退最长)
|
|
check("AST 解析全失败 → None", _ast_merge_python(["@#$%", "!!!"]) is None)
|
|
check("merge_fallback 非py取最长", _merge_fallback("a.txt", ["short", "longer text"]) == "longer text")
|
|
|
|
# --- _decide 决策门 ---
|
|
check("decide: 无阈值 → accept", _decide({"pass_rate": 10.0}, None, 0, 2) == "accept")
|
|
check("decide: 无法评分(None) → accept(诚实)", _decide({"pass_rate": None}, 80.0, 0, 2) == "accept")
|
|
check("decide: 达标 → accept", _decide({"pass_rate": 90.0}, 80.0, 0, 2) == "accept")
|
|
check("decide: 不达标且未到上限 → bounce", _decide({"pass_rate": 40.0}, 80.0, 0, 2) == "bounce")
|
|
check("decide: 不达标但达上限 → accept", _decide({"pass_rate": 40.0}, 80.0, 2, 2) == "accept")
|
|
|
|
# --- 收集共享池:同文件多版本都保留 ---
|
|
t1 = FakeTask("t1", [{"path": "stringutils.py", "content": v1, "action": "write"}])
|
|
t2 = FakeTask("t2", [{"path": "stringutils.py", "content": v2, "action": "write"}])
|
|
t3 = FakeTask("t3", [{"path": "test_stringutils.py", "content": "def test_x():\n assert True\n", "action": "write"}])
|
|
by_path = _collect_per_path([t1, t2, t3])
|
|
check("collect: 同文件两版本都保留", len(by_path.get("stringutils.py", [])) == 2)
|
|
|
|
# --- merge_artifacts:无 LLM 用 AST 整合三函数(不丢互补) ---
|
|
merged = asyncio.run(merge_artifacts([t1, t2, t3], "实现 stringutils"))
|
|
su = next(f for f in merged if f.path == "stringutils.py")
|
|
check("merge: 整合后含全部三函数", all(fn in su.content for fn in ("is_palindrome", "count_vowels", "reverse_words")))
|
|
|
|
# --- finalize_run:无阈值 → accept;无 git grant → 诚实不 promote ---
|
|
res = asyncio.run(finalize_run(FakeRun(), [t1, t2, t3], "实现 stringutils"))
|
|
check("finalize: decision=accept(无阈值)", res.get("decision") == "accept")
|
|
check("finalize: finalized=True", res.get("finalized") is True)
|
|
check("finalize: 无 git grant promoted=False", (res.get("promotion") or {}).get("reason") == "no_git_grant")
|
|
|
|
# --- finalize_run:无产物 → finalized=False ---
|
|
empty = asyncio.run(finalize_run(FakeRun(), [], "空"))
|
|
check("finalize: 无产物 finalized=False", empty.get("finalized") is False and empty.get("reason") == "no_artifacts")
|
|
|
|
# --- emit_patch 模式(SWE-bench):门控 + 无 grant 诚实失败 ---
|
|
check("emit_patch_mode: 默认关", _emit_patch_mode(FakeRun()) is False)
|
|
check("emit_patch_mode: per-run flag 开", _emit_patch_mode(FakeRun({"emit_patch": True})) is True)
|
|
os.environ["SWARM_EMIT_PATCH"] = "1"
|
|
check("emit_patch_mode: env 开", _emit_patch_mode(FakeRun()) is True)
|
|
patch_run = asyncio.run(finalize_run(FakeRun(), [t1, t2, t3], "实现 stringutils"))
|
|
check("emit_patch: mode=emit_patch", patch_run.get("mode") == "emit_patch")
|
|
check("emit_patch: 无 git grant patch=None", patch_run.get("patch") is None)
|
|
check("emit_patch: 无 grant reason=no_git_grant",
|
|
(patch_run.get("patch_meta") or {}).get("reason") == "no_git_grant")
|
|
os.environ.pop("SWARM_EMIT_PATCH", None)
|
|
|
|
if _failures:
|
|
print(f"\nFAILED: {len(_failures)} check(s): {_failures}")
|
|
raise SystemExit(1)
|
|
print("\nALL PASSED")
|