fengqun/examples/run_academic_standard_evaluation.py

from __future__ import annotations

from pathlib import Path
import json
import subprocess
import sys


ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from swarm_minimal.academic_evaluation import (  # noqa: E402
    ACADEMIC_STANDARD_SOURCES,
    ALGORITHMS_USED,
    assess_markov_process_fit,
)
from swarm_minimal.local_env import find_project_env  # noqa: E402


CHECKS = [
    {
        "id": "A01",
        "name": "static_compile",
        "layer": "static",
        "command": [
            sys.executable,
            "-B",
            "-m",
            "py_compile",
            *sorted(str(path.relative_to(ROOT)) for path in ROOT.glob("swarm_minimal/*.py")),
            *sorted(str(path.relative_to(ROOT)) for path in ROOT.glob("examples/*.py")),
            *sorted(str(path.relative_to(ROOT)) for path in ROOT.glob("tests/*.py")),
        ],
    },
    {
        "id": "A02",
        "name": "unit_and_deterministic_scenarios",
        "layer": "unit-scenario",
        "command": [sys.executable, "-B", "-m", "unittest", "discover", "-s", "tests"],
    },
    {
        "id": "A03",
        "name": "swarm_behavior_acceptance",
        "layer": "behavior",
        "command": [sys.executable, "-u", "-B", "examples/run_swarm_behavior_acceptance.py"],
        "parse_json": True,
    },
    {
        "id": "A04",
        "name": "swarm_vs_traditional_benchmark",
        "layer": "benchmark",
        "command": [sys.executable, "-u", "-B", "examples/run_swarm_vs_traditional_benchmark.py"],
        "parse_json": True,
    },
    {
        "id": "A05",
        "name": "consensus_convergence_acceptance",
        "layer": "consensus",
        "command": [sys.executable, "-u", "-B", "examples/run_consensus_convergence_acceptance.py"],
        "parse_json": True,
    },
]


def main() -> None:
    results = [run_check(check) for check in CHECKS]
    env_path = find_project_env(ROOT)
    live_ready = env_path is not None
    markov = assess_markov_process_fit()
    report = {
        "standard": "academic-standard-evaluation-v1",
        "status": "PASS" if all(item["passed"] for item in results) else "FAIL",
        "scope": {
            "deterministic_local": True,
            "live_external_github_code": "ready" if live_ready else "blocked_missing_.env",
            "env_path": str(env_path.relative_to(ROOT)) if env_path else None,
            "live_note": (
                "S07 external GitHub code reasoning requires Azure PostgreSQL, Redis, Blob and NewAPI credentials in an ignored .env."
            ),
        },
        "standards": ACADEMIC_STANDARD_SOURCES,
        "checks": results,
        "algorithms_used": ALGORITHMS_USED,
        "markov_process_assessment": {
            "markov_style_state_machine": markov.markov_style_state_machine,
            "formal_markov_process": markov.formal_markov_process,
            "formal_markov_decision_process": markov.formal_markov_decision_process,
            "sufficient_state": markov.sufficient_state,
            "limiting_factors": markov.limiting_factors,
            "conclusion": markov.conclusion,
        },
        "pass_condition": {
            "local_academic_gate": "all A01-A05 checks pass",
            "full_standard_gate": "local_academic_gate plus S07 live external GitHub code scenario and S08 model I/O report audit",
        },
    }
    print(json.dumps(report, ensure_ascii=False, indent=2))
    if report["status"] != "PASS":
        raise SystemExit(1)


def run_check(check: dict[str, object]) -> dict[str, object]:
    command = check["command"]
    assert isinstance(command, list)
    completed = subprocess.run(
        command,
        cwd=ROOT,
        text=True,
        capture_output=True,
        timeout=900,
    )
    parsed = parse_last_json(completed.stdout) if check.get("parse_json") else None
    evidence: dict[str, object] = {"tail": summarize_process_output(completed.stdout, completed.stderr)}
    if parsed:
        evidence = summarize_json(parsed)
    return {
        "id": check["id"],
        "name": check["name"],
        "layer": check["layer"],
        "command": " ".join(command),
        "passed": completed.returncode == 0,
        "evidence": evidence,
    }


def summarize_process_output(stdout: str, stderr: str) -> str:
    combined = "\n".join(part.strip() for part in [stdout, stderr] if part.strip())
    return combined[-1200:] if combined else "<no output>"


def parse_last_json(text: str) -> dict[str, object] | None:
    stripped = text.strip()
    if not stripped:
        return None
    decoder = json.JSONDecoder()
    last = None
    index = 0
    while index < len(stripped):
        brace = stripped.find("{", index)
        if brace == -1:
            break
        try:
            value, end = decoder.raw_decode(stripped[brace:])
        except json.JSONDecodeError:
            index = brace + 1
            continue
        if isinstance(value, dict):
            last = value
        index = brace + end
    return last


def summarize_json(value: dict[str, object]) -> dict[str, object]:
    summary: dict[str, object] = {"status": value.get("status")}
    if "standard" in value:
        summary["standard"] = value["standard"]
    if "overall_normalized_score" in value:
        summary["overall_normalized_score"] = value["overall_normalized_score"]
    if "accepted_candidate" in value:
        summary["accepted_candidate"] = value["accepted_candidate"]
    if "rounds" in value and isinstance(value["rounds"], list):
        summary["round_count"] = len(value["rounds"])
    if "scenarios" in value and isinstance(value["scenarios"], list):
        summary["scenario_count"] = len(value["scenarios"])
        summary["failed_scenarios"] = [
            item.get("id")
            for item in value["scenarios"]
            if isinstance(item, dict) and not item.get("passed", False)
        ]
    return summary


if __name__ == "__main__":
    main()