fix(swebench): 真实仓 patch 捕获 + agent 稳定性
#1 稳定性:HEARTBEAT_TIMEOUT 30→120(可配 AGENT_HEARTBEAT_TIMEOUT)——真实仓 clone/长 模型调用不再误杀 agent。 #2 正确性/忠实度: - 透传 GIT_BASE_COMMIT;agent clone 改为浅 fetch+checkout 该 commit(在正确基线上读/改, 且大仓也快);emit_patch 持久化 base_commit + _git_diff 对 SHA 走浅 fetch。 - agent 读文件预算可配并调大(AGENT_CTX_MAX_FILES=40/BYTES=40000),LLM 看到完整文件, 避免全量重写截断 → diff 失真。launcher 透传这些 env 给 agent pod。 import + runtime-contract/result-aggregator/security-boundary 测试全绿。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
74f5ce2f97
commit
fb1eeeebfa
+20
-2
@@ -79,15 +79,33 @@ class GitOperations:
|
|||||||
self._clear_workspace_dir(workspace)
|
self._clear_workspace_dir(workspace)
|
||||||
|
|
||||||
clone_url = self._authenticated_repo_url(repo_url)
|
clone_url = self._authenticated_repo_url(repo_url)
|
||||||
|
base_commit = (os.getenv("GIT_BASE_COMMIT") or "").strip()
|
||||||
|
|
||||||
|
if base_commit:
|
||||||
|
# SWE-bench/benchmark: work from the EXACT base_commit, not default HEAD. Shallow-fetch
|
||||||
|
# just that commit (fast + small even for huge repos) and check it out.
|
||||||
|
workspace.mkdir(parents=True, exist_ok=True)
|
||||||
|
steps = [
|
||||||
|
["git", "init", "-q", self.workspace_dir],
|
||||||
|
["git", "-C", self.workspace_dir, "remote", "add", "origin", clone_url],
|
||||||
|
["git", "-C", self.workspace_dir, "fetch", "--depth", "1", "origin", base_commit],
|
||||||
|
["git", "-C", self.workspace_dir, "checkout", "-q", "FETCH_HEAD"],
|
||||||
|
]
|
||||||
|
for step in steps:
|
||||||
|
r = await self._run_git_command(step)
|
||||||
|
if r.returncode != 0:
|
||||||
|
logger.error(f"base_commit shallow checkout failed at {step[1:4]}: {r.stderr}")
|
||||||
|
return False
|
||||||
|
logger.info(f"Checked out base_commit {base_commit[:12]} (shallow) at {self.workspace_dir}")
|
||||||
|
else:
|
||||||
result = await self._run_git_command([
|
result = await self._run_git_command([
|
||||||
"git", "clone", clone_url, self.workspace_dir
|
"git", "clone", clone_url, self.workspace_dir
|
||||||
])
|
])
|
||||||
|
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
logger.error(f"Git clone failed: {result.stderr}")
|
logger.error(f"Git clone failed: {result.stderr}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
logger.info(f"Successfully cloned repository to {self.workspace_dir}")
|
logger.info(f"Successfully cloned repository to {self.workspace_dir}")
|
||||||
|
|
||||||
await self._configure_git_user()
|
await self._configure_git_user()
|
||||||
await self._ensure_baseline_commit()
|
await self._ensure_baseline_commit()
|
||||||
return True
|
return True
|
||||||
|
|||||||
@@ -737,6 +737,10 @@ Return ONLY JSON:
|
|||||||
return sorted(files)
|
return sorted(files)
|
||||||
|
|
||||||
def _collect_workspace_context(self, max_files: int = 20, max_bytes_per_file: int = 6000) -> dict[str, str]:
|
def _collect_workspace_context(self, max_files: int = 20, max_bytes_per_file: int = 6000) -> dict[str, str]:
|
||||||
|
# Real-repo edits need the FULL file (the LLM rewrites complete content; truncated input →
|
||||||
|
# truncated/hallucinated rewrite → wrong diff). Raise budgets, env-tunable for big repos.
|
||||||
|
max_files = int(os.getenv("AGENT_CTX_MAX_FILES", str(max_files)))
|
||||||
|
max_bytes_per_file = int(os.getenv("AGENT_CTX_MAX_BYTES", str(max_bytes_per_file)))
|
||||||
root = Path(self.workspace_dir)
|
root = Path(self.workspace_dir)
|
||||||
if not root.exists():
|
if not root.exists():
|
||||||
return {}
|
return {}
|
||||||
|
|||||||
@@ -108,6 +108,14 @@ spec:
|
|||||||
# plan/分解 token 上限:降到 32768 避开模型网关 >32768 → 400(launcher 透传给 agent pod)
|
# plan/分解 token 上限:降到 32768 避开模型网关 >32768 → 400(launcher 透传给 agent pod)
|
||||||
- name: AGENT_PLAN_MAX_TOKENS
|
- name: AGENT_PLAN_MAX_TOKENS
|
||||||
value: "32768"
|
value: "32768"
|
||||||
|
# 心跳超时:真实仓 clone/长模型调用会让 agent >30s 不心跳被误杀,放宽到 120s
|
||||||
|
- name: AGENT_HEARTBEAT_TIMEOUT
|
||||||
|
value: "120"
|
||||||
|
# 真实仓上下文预算(launcher 透传给 agent pod):让 LLM 看到完整文件,避免重写截断
|
||||||
|
- name: AGENT_CTX_MAX_FILES
|
||||||
|
value: "40"
|
||||||
|
- name: AGENT_CTX_MAX_BYTES
|
||||||
|
value: "40000"
|
||||||
# 协作聚合质量门:pass_rate<70 打回迭代(对齐本地已验证收敛行为;未设=门关=不打回)
|
# 协作聚合质量门:pass_rate<70 打回迭代(对齐本地已验证收敛行为;未设=门关=不打回)
|
||||||
- name: AGGREGATE_ACCEPTANCE_THRESHOLD
|
- name: AGGREGATE_ACCEPTANCE_THRESHOLD
|
||||||
value: "70"
|
value: "70"
|
||||||
|
|||||||
@@ -322,6 +322,11 @@ def resolve_git_grant(body: Dict[str, Any]) -> Optional[Dict[str, str]]:
|
|||||||
branch = (meta.get("base_branch") or meta.get("branch") or "").strip()
|
branch = (meta.get("base_branch") or meta.get("branch") or "").strip()
|
||||||
if branch:
|
if branch:
|
||||||
env["GIT_BASE_BRANCH"] = branch
|
env["GIT_BASE_BRANCH"] = branch
|
||||||
|
# SWE-bench / benchmark: pin the exact commit the agent must work from. The agent shallow-fetches
|
||||||
|
# and checks out this commit so reads/edits are against the correct baseline (not default HEAD).
|
||||||
|
base_commit = (meta.get("base_commit") or grant.get("base_commit") or "").strip()
|
||||||
|
if base_commit:
|
||||||
|
env["GIT_BASE_COMMIT"] = base_commit
|
||||||
secret_ref = (grant.get("secret_ref") or grant.get("ref") or "").strip()
|
secret_ref = (grant.get("secret_ref") or grant.get("ref") or "").strip()
|
||||||
if secret_ref.startswith("azkv://"):
|
if secret_ref.startswith("azkv://"):
|
||||||
creds = _resolve_git_secret_ref(secret_ref)
|
creds = _resolve_git_secret_ref(secret_ref)
|
||||||
@@ -514,6 +519,11 @@ def plan_launch_specs(
|
|||||||
plan_max_tokens = os.getenv("AGENT_PLAN_MAX_TOKENS")
|
plan_max_tokens = os.getenv("AGENT_PLAN_MAX_TOKENS")
|
||||||
if plan_max_tokens:
|
if plan_max_tokens:
|
||||||
env["AGENT_PLAN_MAX_TOKENS"] = plan_max_tokens
|
env["AGENT_PLAN_MAX_TOKENS"] = plan_max_tokens
|
||||||
|
# 透传真实仓上下文预算(读多少文件/每文件多少字节)给 agent pod
|
||||||
|
for _ctx in ("AGENT_CTX_MAX_FILES", "AGENT_CTX_MAX_BYTES"):
|
||||||
|
_v = os.getenv(_ctx)
|
||||||
|
if _v:
|
||||||
|
env[_ctx] = _v
|
||||||
specs.append(AgentLaunchSpec(agent_id=env["AGENT_ID"], capabilities=cap_csv, env=env))
|
specs.append(AgentLaunchSpec(agent_id=env["AGENT_ID"], capabilities=cap_csv, env=env))
|
||||||
return specs
|
return specs
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
"""Agent registry with Redis-backed state management."""
|
"""Agent registry with Redis-backed state management."""
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import time
|
import time
|
||||||
import logging
|
import logging
|
||||||
from typing import Dict, List, Optional
|
from typing import Dict, List, Optional
|
||||||
@@ -30,7 +31,10 @@ class AgentMetadata(BaseModel):
|
|||||||
class AgentRegistry:
|
class AgentRegistry:
|
||||||
"""Manages agent registration and heartbeat tracking."""
|
"""Manages agent registration and heartbeat tracking."""
|
||||||
|
|
||||||
HEARTBEAT_TIMEOUT = 30 # seconds
|
# Seconds without a heartbeat before an agent is marked FAILED. 30s is too tight when an agent
|
||||||
|
# is busy cloning a large real repo or in a long model call (event loop can stall) — that spuriously
|
||||||
|
# failed real SWE tasks. Configurable; default raised to 120. Set AGENT_HEARTBEAT_TIMEOUT to tune.
|
||||||
|
HEARTBEAT_TIMEOUT = int(os.getenv("AGENT_HEARTBEAT_TIMEOUT", "120")) # seconds
|
||||||
AGENT_KEY_PREFIX = "agent:"
|
AGENT_KEY_PREFIX = "agent:"
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
|||||||
@@ -2004,6 +2004,8 @@ async def create_swarm_run_from_request(
|
|||||||
"repo_url": _meta.get("repo_url") or _git_grant.get("repo_url"),
|
"repo_url": _meta.get("repo_url") or _git_grant.get("repo_url"),
|
||||||
"secret_ref": _git_grant.get("secret_ref"),
|
"secret_ref": _git_grant.get("secret_ref"),
|
||||||
"base_branch": _meta.get("base_branch", "main"),
|
"base_branch": _meta.get("base_branch", "main"),
|
||||||
|
# benchmark: emit_patch must diff against the exact base_commit, not default HEAD
|
||||||
|
"base_commit": _meta.get("base_commit") or _git_grant.get("base_commit"),
|
||||||
}
|
}
|
||||||
await swarm_runtime.save_run(run)
|
await swarm_runtime.save_run(run)
|
||||||
|
|
||||||
|
|||||||
@@ -247,14 +247,26 @@ def _git_diff(env: Dict[str, str], base_ref: str, files: List[Any], swarm_id: st
|
|||||||
|
|
||||||
def git(*args, cwd=None):
|
def git(*args, cwd=None):
|
||||||
return subprocess.run(["git", *args], cwd=cwd or workdir,
|
return subprocess.run(["git", *args], cwd=cwd or workdir,
|
||||||
capture_output=True, text=True, timeout=300)
|
capture_output=True, text=True, timeout=600)
|
||||||
|
|
||||||
|
# Heuristic: a 40-hex (or short-hex) ref is a commit SHA → shallow-fetch just it (fast/small even
|
||||||
|
# for huge repos). A branch name → shallow clone that branch. Falls back to full clone on failure.
|
||||||
|
def _looks_like_sha(ref: str) -> bool:
|
||||||
|
return bool(ref) and len(ref) >= 7 and all(c in "0123456789abcdef" for c in ref.lower())
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 全量 clone(非 --depth 1):base_ref 可能是任意 base_commit,浅 clone 不含其历史。
|
if base_ref and _looks_like_sha(base_ref):
|
||||||
|
os.makedirs(repo_dir, exist_ok=True)
|
||||||
|
ok = (git("init", "-q", "repo").returncode == 0
|
||||||
|
and git("remote", "add", "origin", auth_url, cwd=repo_dir).returncode == 0
|
||||||
|
and git("fetch", "--depth", "1", "origin", base_ref, cwd=repo_dir).returncode == 0
|
||||||
|
and git("checkout", "-q", "FETCH_HEAD", cwd=repo_dir).returncode == 0)
|
||||||
|
if not ok:
|
||||||
|
return {"emitted": False, "reason": "shallow_checkout_failed"}
|
||||||
|
else:
|
||||||
if git("clone", auth_url, "repo").returncode != 0:
|
if git("clone", auth_url, "repo").returncode != 0:
|
||||||
return {"emitted": False, "reason": "clone_failed"}
|
return {"emitted": False, "reason": "clone_failed"}
|
||||||
if base_ref:
|
if base_ref and git("checkout", base_ref, cwd=repo_dir).returncode != 0:
|
||||||
if git("checkout", base_ref, cwd=repo_dir).returncode != 0:
|
|
||||||
return {"emitted": False, "reason": "checkout_failed"}
|
return {"emitted": False, "reason": "checkout_failed"}
|
||||||
for f in files:
|
for f in files:
|
||||||
dest = os.path.join(repo_dir, f.path)
|
dest = os.path.join(repo_dir, f.path)
|
||||||
|
|||||||
Reference in New Issue
Block a user