"""Unified benchmark task sets (#20) — the shared, frozen tasks every system runs. Fairness rule (standard §9.1 / baseline-comparison §3): swarm and every baseline run the SAME task_set_id under the SAME model gateway. A task set groups tasks of one scenario; each task references a fixture (objective + HELD-OUT acceptance tests + offline reference solution). Layout: benchmark/tasksets//taskset.json `taskset.json`: { "task_set_id": "coding-set-1", "scenario": "coding", # coding | refactoring | architecture | devops | bugfix "tasks": [ { "id": "add_function", "fixture": "add_function" } ] } Coverage is honest: only scenarios with real fixtures are loadable. The other four scenarios (refactoring/architecture/devops/bugfix) are declared TODO in this package's README until their fixtures + held-out tests exist — we do not ship empty task sets that would silently pass. """ from __future__ import annotations import json from dataclasses import dataclass, field from pathlib import Path from typing import List from ..fixtures import Fixture, load_fixture _TASKSET_ROOT = Path(__file__).resolve().parent SCENARIOS = {"coding", "refactoring", "architecture", "devops", "bugfix"} @dataclass class TaskSetItem: id: str fixture_id: str @property def fixture(self) -> Fixture: return load_fixture(self.fixture_id) @dataclass class TaskSet: task_set_id: str scenario: str tasks: List[TaskSetItem] = field(default_factory=list) def available_tasksets() -> List[str]: return sorted( p.name for p in _TASKSET_ROOT.iterdir() if p.is_dir() and (p / "taskset.json").exists() ) def load_taskset(task_set_id: str) -> TaskSet: meta_path = _TASKSET_ROOT / task_set_id / "taskset.json" if not meta_path.exists(): raise FileNotFoundError(f"unknown task set: {task_set_id}") meta = json.loads(meta_path.read_text(encoding="utf-8")) scenario = meta.get("scenario", "") if scenario not in SCENARIOS: raise ValueError(f"task set {task_set_id} has unknown scenario '{scenario}'") tasks = [TaskSetItem(id=t["id"], fixture_id=t["fixture"]) for t in meta.get("tasks", [])] if not tasks: raise ValueError(f"task set {task_set_id} has no tasks (refusing to ship an empty set)") return TaskSet(task_set_id=meta["task_set_id"], scenario=scenario, tasks=tasks)