Songhaoz666andClaude Opus 4.8
a289495823
回应 Fasthei 的 Request changes 两个阻塞项：
1) 安全 / fail-closed 沙箱隔离（原仅靠 ENABLE_QUALITY_EVAL + 运维约定）：
- 新增第二道显式确认 HEICODE_SANDBOX_ISOLATED（断言运行在隔离 Pod 内）。
- sandbox.run_tests() 与 quality.evaluate_run_quality() 执行任何代码前调用
assert_isolated()，未确认即抛 SandboxIsolationError——不写文件、不起子进程。
- 启动期 assert_quality_eval_safe()：ENABLE_QUALITY_EVAL 开但隔离未确认 → 拒绝启动
（平台级硬失败，非运维口头约定）。
- 文档（security-boundary §8.1/§9、CLAUDE.md）与测试同步：test-sandbox/test-quality
先断言未确认时硬失败，再显式确认后继续。
2) #10 DecisionTrace 可回放（原仅存被选中任务的标量）：
- Decision 现记录完整重放上下文：整个候选集（每候选 tau/eta/weight/p_norm/dependents）、
alpha/beta/epsilon、seed、free_slots、total_weight、explore_draw、select_pick、
select_index、explored 分支。
- 新增 DecisionEngine.replay_decision(trace)：仅凭一条 trace（无 RNG/活体状态）复现被选任务；
test-decision-engine 断言「重放==实选」跨 50 次决策（探索+利用）成立。
- decision-engine.md §3.3 更新为可回放 DecisionTrace。
附：新增 docs/TESTING.md（reviewer 速查：依赖安装 + 每套测试命令，复审者此前因缺 fakeredis
未能跑到断言）。本地 11 项 gate 全绿。
影响范围：Swarm（orchestrator + 测试 + 文档）。不改 Manager↔Swarm 契约；新增开关
HEICODE_SANDBOX_ISOLATED（默认未设=拒绝执行）。仍非验收：gain/Benchmark_Agent 仍 NaN。
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
98 lines
3.5 KiB
Python
98 lines
3.5 KiB
Python
"""Test the in-pod code sandbox: fail-closed isolation, real execution, counting, timeout.
|
|
|
|
Hermetic, no model key. Run from agent_swarm_v6:
|
|
python scripts/test-sandbox.py
|
|
(no extra deps; this test sets HEICODE_SANDBOX_ISOLATED itself to confirm isolation.)
|
|
"""
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from orchestrator.sandbox import SandboxFile, SandboxIsolationError, run_tests
|
|
|
|
failures = []
|
|
|
|
|
|
def check(name, cond):
|
|
print(("PASS" if cond else "FAIL"), "-", name)
|
|
if not cond:
|
|
failures.append(name)
|
|
|
|
|
|
# --- 0. FAIL-CLOSED: with isolation NOT confirmed, the sandbox refuses to execute anything ---
|
|
os.environ.pop("HEICODE_SANDBOX_ISOLATED", None)
|
|
try:
|
|
run_tests([SandboxFile("calc.py", "x=1\n")], [SandboxFile("test_x.py", "def test_x():\n assert True\n")])
|
|
check("fail-closed without isolation confirmation", False)
|
|
except SandboxIsolationError:
|
|
check("fail-closed without isolation confirmation", True)
|
|
|
|
# Everything below requires explicit isolation confirmation (CI runner / pod manifest sets this).
|
|
os.environ["HEICODE_SANDBOX_ISOLATED"] = "1"
|
|
|
|
|
|
# --- 1. bare pytest-style tests against a generated module: 2/3 pass ---
|
|
impl = [SandboxFile("calc.py", "def add(a, b):\n return a + b\n")]
|
|
tests = [SandboxFile(
|
|
"test_calc.py",
|
|
"from calc import add\n"
|
|
"def test_add_pos():\n assert add(1, 2) == 3\n"
|
|
"def test_add_zero():\n assert add(0, 0) == 0\n"
|
|
"def test_wrong():\n assert add(1, 1) == 3\n",
|
|
)]
|
|
r = run_tests(impl, tests)
|
|
check("runs bare functions: total=3", r.total == 3)
|
|
check("counts passed=2", r.passed == 2)
|
|
check("counts failed=1", r.failed == 1)
|
|
check("pass_rate ~= 66.67", r.pass_rate is not None and round(r.pass_rate, 2) == 66.67)
|
|
|
|
# --- 2. unittest.TestCase style is also collected ---
|
|
ut = [SandboxFile(
|
|
"test_ut.py",
|
|
"import unittest\n"
|
|
"class T(unittest.TestCase):\n"
|
|
" def test_ok(self):\n self.assertEqual(2 + 2, 4)\n"
|
|
" def test_bad(self):\n self.assertEqual(2 + 2, 5)\n",
|
|
)]
|
|
r2 = run_tests([], ut)
|
|
check("unittest TestCase collected: total=2", r2.total == 2)
|
|
check("unittest passed=1 failed=1", r2.passed == 1 and r2.failed == 1)
|
|
|
|
# --- 3. import error in test counts as errored, not a crash ---
|
|
bad = [SandboxFile("test_imp.py", "import this_module_does_not_exist_xyz\n")]
|
|
r3 = run_tests([], bad)
|
|
check("import error -> errored>=1, no crash", r3.errored >= 1 and r3.error is None)
|
|
|
|
# --- 4. timeout is enforced and reported (no hang) ---
|
|
loop = [SandboxFile("test_loop.py", "def test_spin():\n while True:\n pass\n")]
|
|
r4 = run_tests([], loop, timeout_seconds=3)
|
|
check("infinite loop times out", r4.timed_out is True)
|
|
|
|
# --- 5. secrets are NOT visible to sandboxed code (env scrub) ---
|
|
import os as _os
|
|
_os.environ["OPENAI_API_KEY"] = "sk-should-not-leak"
|
|
_os.environ["AWS_SECRET_ACCESS_KEY"] = "should-not-leak"
|
|
leak = [SandboxFile(
|
|
"test_leak.py",
|
|
"import os\n"
|
|
"def test_no_openai():\n assert os.environ.get('OPENAI_API_KEY') is None\n"
|
|
"def test_no_aws():\n assert os.environ.get('AWS_SECRET_ACCESS_KEY') is None\n",
|
|
)]
|
|
r5 = run_tests([], leak)
|
|
check("env scrub: no secrets leak into sandbox", r5.total == 2 and r5.passed == 2)
|
|
|
|
# --- 6. path traversal is rejected ---
|
|
try:
|
|
run_tests([SandboxFile("../escape.py", "x=1")], [])
|
|
check("path traversal rejected", False)
|
|
except ValueError:
|
|
check("path traversal rejected", True)
|
|
|
|
print()
|
|
if failures:
|
|
print(f"{len(failures)} sandbox check(s) FAILED: {failures}")
|
|
sys.exit(1)
|
|
print("all sandbox checks passed")
|