Agentswarm/scripts/test-convergence.py

"""Hermetic unit tests for the swarm convergence protocol (Issue #12).

Module-level only: no WebSocket, no Redis, no model, no FastAPI. Exercises the
pure functions in orchestrator/convergence.py directly.

Run from agent_swarm_v6 (Windows, repo .venv):

    ..\\.venv\\Scripts\\python.exe scripts\\test-convergence.py

If the .venv is missing, it only needs the stdlib + the repo on sys.path:

    py -m venv ..\\.venv
    ..\\.venv\\Scripts\\python.exe -m pip install -r orchestrator\\requirements.txt
    ..\\.venv\\Scripts\\python.exe scripts\\test-convergence.py

(No third-party import is actually required by this test — convergence.py uses
only the stdlib — but the command above matches the repo convention.)
"""
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from orchestrator.convergence import (  # noqa: E402
    ConflictType,
    ConvergenceStatus,
    TerminationReason,
    detect_conflicts,
    evaluate_convergence,
    event_conflict_detected,
    event_conflict_resolved,
    event_consensus_updated,
    event_convergence_failed,
    event_convergence_reached,
    event_convergence_started,
    resolve_conflicts,
)

failures = []


def check(name, cond):
    print(("PASS" if cond else "FAIL"), "-", name)
    if not cond:
        failures.append(name)


# --- scenario: two conflicting artifacts; one conflict resolvable ---------
# task-a and task-b both wrote different content to src/app.py (artifact
# mismatch). The master critic REJECTED the work and flagged task-b for redo,
# which makes the review-disagreement conflict actionable (resolvable), while
# the artifact mismatch stays unresolved (review did not accept a winner).

run_state = {
    "tasks": [
        {
            "task_id": "task-a",
            "status": "completed",
            "depends_on": [],
            "result": {
                "files_modified": ["src/app.py"],
                "file_hashes": {"src/app.py": "hashAAA"},
                "tests_passed": True,
            },
        },
        {
            "task_id": "task-b",
            "status": "completed",
            "depends_on": [],
            "result": {
                "files_modified": ["src/app.py"],
                "file_hashes": {"src/app.py": "hashBBB"},
                "tests_passed": True,
            },
        },
    ],
    "review_verdict": {
        "accepted": False,
        "retry_tasks": ["task-b"],
        "summary": "Two versions of src/app.py disagree; redo task-b",
    },
    "budget": {"max_cost_usd": 10.0},
    "usage": {"total_cost_usd": 1.0},
}

conflicts = detect_conflicts(run_state["tasks"], run_state["review_verdict"])
types = {c.type for c in conflicts}
check("artifact mismatch detected", ConflictType.ARTIFACT_MISMATCH in types)
check("review disagreement detected", ConflictType.REVIEW_DISAGREEMENT in types)

resolved, unresolved = resolve_conflicts(conflicts, run_state["review_verdict"])
check("at least one conflict resolved (review disagreement closes)", len(resolved) >= 1)
check("review disagreement is among the resolved",
      any(c.type == ConflictType.REVIEW_DISAGREEMENT and c.resolved for c in resolved))
check("artifact mismatch stays unresolved (no accepted winner)",
      any(c.type == ConflictType.ARTIFACT_MISMATCH and not c.resolved for c in unresolved))
check("every resolved conflict carries a resolution note",
      all(c.resolution for c in resolved))

report = evaluate_convergence(run_state)
check("conflicting run is not silently CONVERGED (blocked by unresolved artifact)",
      report.status == ConvergenceStatus.BLOCKED)
check("blocked run carries a concrete termination_reason",
      report.termination_reason == TerminationReason.RISK_BLOCKED)
check("consensus < 100 when work is in conflict", report.consensus_score < 100.0)
check("report.to_dict() round-trips conflicts",
      len(report.to_dict()["conflicts"]) == len(conflicts))


# --- assert: every TERMINAL state carries a termination_reason ------------
# Build one run-state per intended terminal outcome and assert the reason.

def completed_task(tid, **result):
    return {"task_id": tid, "status": "completed", "depends_on": [], "result": result}


terminal_cases = {
    # clean completion -> tasks_completed fallback
    "tasks_completed": {
        "tasks": [completed_task("t1", tests_passed=True)],
        "expect_status": ConvergenceStatus.CONVERGED,
        "expect_reason": TerminationReason.TASKS_COMPLETED,
    },
    # quality gate reached
    "quality_reached": {
        "tasks": [completed_task("t1", tests_passed=True)],
        "quality": {"test_pass_rate": 1.0, "graded": True},
        "expect_status": ConvergenceStatus.CONVERGED,
        "expect_reason": TerminationReason.QUALITY_REACHED,
    },
    # budget exhausted
    "budget_exhausted": {
        "tasks": [completed_task("t1", tests_passed=True)],
        "budget": {"max_cost_usd": 5.0},
        "usage": {"total_cost_usd": 5.0},
        "expect_status": ConvergenceStatus.CONVERGED,
        "expect_reason": TerminationReason.BUDGET_EXHAUSTED,
    },
    # max rounds reached
    "max_rounds_reached": {
        "tasks": [completed_task("t1", tests_passed=True)],
        "review_cycles": 2,
        "max_review_cycles": 2,
        "expect_status": ConvergenceStatus.CONVERGED,
        "expect_reason": TerminationReason.MAX_ROUNDS_REACHED,
    },
    # risk blocked (explicit blocking input risk)
    "risk_blocked": {
        "tasks": [completed_task("t1", tests_passed=True)],
        "risks": [{"id": "r1", "description": "secret leak suspected", "blocking": True}],
        "expect_status": ConvergenceStatus.BLOCKED,
        "expect_reason": TerminationReason.RISK_BLOCKED,
    },
    # a failed task with no specific signal -> FAILED, task_failed (NOT tasks_completed; #70 bug).
    "failed_generic": {
        "tasks": [
            completed_task("t1", tests_passed=True),
            {"task_id": "t2", "status": "failed", "depends_on": [], "result": {}},
        ],
        "expect_status": ConvergenceStatus.FAILED,
        "expect_reason": TerminationReason.TASK_FAILED,
    },
    # a failed task whose recorded error is a timeout -> FAILED, timeout (#70).
    "failed_timeout": {
        "tasks": [
            {"task_id": "t2", "status": "failed", "depends_on": [],
             "result": {"success": False, "error": "timeout"}},
        ],
        "expect_status": ConvergenceStatus.FAILED,
        "expect_reason": TerminationReason.TIMEOUT,
    },
    # a failed task that exhausted its retry budget -> FAILED, max_retries_exceeded (#70).
    "failed_max_retries": {
        "tasks": [
            {"task_id": "t2", "status": "failed", "depends_on": [], "result": {},
             "retry_count": 3, "max_retries": 3},
        ],
        "expect_status": ConvergenceStatus.FAILED,
        "expect_reason": TerminationReason.MAX_RETRIES_EXCEEDED,
    },
    # timeout takes precedence over max_retries when both are present (#70 precedence).
    "failed_timeout_over_retries": {
        "tasks": [
            {"task_id": "t2", "status": "failed", "depends_on": [],
             "result": {"success": False, "error": "timeout"},
             "retry_count": 3, "max_retries": 3},
        ],
        "expect_status": ConvergenceStatus.FAILED,
        "expect_reason": TerminationReason.TIMEOUT,
    },
}

for label, case in terminal_cases.items():
    rep = evaluate_convergence(case)
    check(f"[{label}] terminal status == {case['expect_status'].value}",
          rep.status == case["expect_status"])
    check(f"[{label}] is_terminal()", rep.is_terminal())
    check(f"[{label}] carries a non-None termination_reason",
          rep.termination_reason is not None)
    check(f"[{label}] termination_reason == {case['expect_reason'].value}",
          rep.termination_reason == case["expect_reason"])

# Invariant: EVERY terminal report has a reason; non-terminal has none.
for label, case in terminal_cases.items():
    rep = evaluate_convergence(case)
    if rep.is_terminal():
        check(f"[{label}] invariant: terminal -> reason set",
              rep.termination_reason is not None)

# #70 invariant: a FAILED run must NEVER report tasks_completed (success-only reason).
for label, case in terminal_cases.items():
    rep = evaluate_convergence(case)
    if rep.status == ConvergenceStatus.FAILED:
        check(f"[{label}] FAILED run never reports tasks_completed (#70)",
              rep.termination_reason != TerminationReason.TASKS_COMPLETED)

# A still-running run must NOT carry a termination_reason.
running = evaluate_convergence({"tasks": [{"task_id": "x", "status": "in_progress", "depends_on": []}]})
check("running run is RUNNING", running.status == ConvergenceStatus.RUNNING)
check("running run has no termination_reason", running.termination_reason is None)
check("running run is not terminal", not running.is_terminal())


# --- dependency inconsistency detector ------------------------------------
dep_state = {
    "tasks": [
        {"task_id": "parent", "status": "completed", "depends_on": ["child"], "result": {}},
        {"task_id": "child", "status": "failed", "depends_on": [], "result": {}},
    ],
}
dep_conflicts = detect_conflicts(dep_state["tasks"])
check("dependency inconsistency detected",
      any(c.type == ConflictType.DEPENDENCY_INCONSISTENCY for c in dep_conflicts))


# --- test failure detector ------------------------------------------------
tf_conflicts = detect_conflicts([completed_task("t1", tests_passed=False)])
check("test failure detected",
      any(c.type == ConflictType.TEST_FAILURE for c in tf_conflicts))
check("no test signal is not a failure",
      not any(c.type == ConflictType.TEST_FAILURE
              for c in detect_conflicts([completed_task("t1")])))


# --- event builders -------------------------------------------------------
et, _ = event_convergence_started(run_state)
check("convergence.started event type", et == "convergence.started")
et, _ = event_conflict_detected(conflicts[0])
check("conflict.detected event type", et == "conflict.detected")
et, _ = event_conflict_resolved(resolved[0])
check("conflict.resolved event type", et == "conflict.resolved")
et, payload = event_consensus_updated(report)
check("consensus.updated event type + score", et == "consensus.updated" and "consensus_score" in payload)
et, payload = event_convergence_failed(report)
check("convergence.failed carries termination_reason",
      et == "convergence.failed" and payload["termination_reason"] == "risk_blocked")

clean = evaluate_convergence(terminal_cases["tasks_completed"])
et, payload = event_convergence_reached(clean)
check("convergence.reached carries termination_reason",
      et == "convergence.reached" and payload["termination_reason"] == "tasks_completed")


print()
if failures:
    print(f"{len(failures)} convergence check(s) FAILED: {failures}")
    sys.exit(1)
print("all convergence protocol checks passed")