fengqun/examples/export_model_agnet_io_report.py

from __future__ import annotations

from pathlib import Path
import re
import sys


ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from swarm_minimal.azure_store import PostgresRedisBlobSwarmStore
from swarm_minimal.config import SwarmConfig
from swarm_minimal.local_env import load_project_env


LIVE_GOAL_PREFIX = "外部 GitHub 代码场景：审查 fastapi/fastapi"
RUN_IDS: list[str] = []
EXTERNAL_REPO = "fastapi/fastapi"
EXTERNAL_REPO_URL = "https://github.com/fastapi/fastapi"
EXTERNAL_REPO_COMMIT = "ecace740f3eaccb1aba152cf1de79477095c56f4"

TARGET_FILES = (
    "fastapi/routing.py",
    "fastapi/dependencies/utils.py",
    "fastapi/openapi/utils.py",
    "fastapi/params.py",
    "fastapi/encoders.py",
    "fastapi/applications.py",
    "tests/test_serialize_response_model.py",
    "tests/test_response_model_data_filter.py",
)

CHAIN_STEP_DETAILS = {
    "chain_step_01": {
        "marker": "STEP-01",
        "title": "界定问题和不可变约束",
        "role": "把被测对象锁定为外部 FastAPI 仓库，定义输入、输出、不变量、禁止自测边界和下一步交接摘要。",
        "expected_output": "外部仓库边界、不变量、风险和 STEP-02 交接。",
    },
    "chain_step_02": {
        "marker": "STEP-02",
        "title": "建立依赖图和状态模型",
        "role": "承接 STEP-01，建立 routing、dependencies、openapi、encoders、测试文件之间的依赖关系。",
        "expected_output": "FastAPI response_model、依赖注入、OpenAPI schema 和响应序列化的依赖图。",
    },
    "chain_step_03": {
        "marker": "STEP-03",
        "title": "定位跨文件风险路径",
        "role": "承接 STEP-02，定位 response_model、Depends、参数 metadata、jsonable_encoder、OpenAPI schema 之间的漂移风险。",
        "expected_output": "跨文件风险路径、复杂度和 STEP-04 反例构造入口。",
    },
    "chain_step_04": {
        "marker": "STEP-04",
        "title": "构造反例和失败场景",
        "role": "承接 STEP-03，构造响应过滤、默认值、nullable、依赖参数和 OpenAPI schema 不一致的反例。",
        "expected_output": "可触发失败的反例、预期破坏点和 STEP-05 修正入口。",
    },
    "chain_step_05": {
        "marker": "STEP-05",
        "title": "修正算法和恢复策略",
        "role": "承接 STEP-04，给出应修改的 FastAPI 模块、兼容性策略和 Starlette/Pydantic 交互保护。",
        "expected_output": "修正算法、兼容策略、恢复策略和 STEP-06 文件级计划入口。",
    },
    "chain_step_06": {
        "marker": "STEP-06",
        "title": "落到文件级实现计划",
        "role": "承接 STEP-05，把修正策略落到具体源码文件和测试文件。",
        "expected_output": "文件级补丁计划、目标测试文件和 STEP-07 验收入口。",
    },
    "chain_step_07": {
        "marker": "STEP-07",
        "title": "最终收敛和验收判定",
        "role": "承接 STEP-06，给出可执行验收命令、指标、失败判定和可合并结论。",
        "expected_output": "最终验收命令、失败判定、文件引用和可合并结论。",
    },
}

TEST_SCENARIOS = (
    {
        "id": "S01",
        "name": "syntax_import_sanity",
        "purpose": "验证所有 Python 模块可编译，排除语法和导入层错误。",
        "evidence": "`py_compile swarm_minimal/*.py examples/*.py tests/*.py`",
    },
    {
        "id": "S02",
        "name": "unit_regression",
        "purpose": "验证内存蜂群、NewAPI mock、配置脱敏和基础收敛行为。",
        "evidence": "`unittest discover -s tests`",
    },
    {
        "id": "S03-S06",
        "name": "deterministic_standard_scenarios",
        "purpose": "验证链路连续性、依赖边界、最终评分和失败注入。",
        "evidence": "`unittest tests.test_standard_scenarios`",
    },
    {
        "id": "S07",
        "name": "live_external_github_code_reasoning",
        "purpose": "用真实 Azure PostgreSQL、Redis、Blob 和 NewAPI 对外部 GitHub 项目 fastapi/fastapi 跑 7 步代码推理链。",
        "evidence": "本报告下方每个 live run 的 task.input / task.output / handoff 记录。",
    },
    {
        "id": "S08",
        "name": "model_io_report_audit",
        "purpose": "验证本报告包含场景、输入、输出、交接证据，且没有明显真实密钥样式。",
        "evidence": "`unittest tests.test_model_io_report_audit`",
    },
)

OUTPUT_PATH = ROOT / "docs" / "MODEL_AGNET_IO_REPORT.zh-CN.md"


def main() -> None:
    load_project_env(ROOT)
    store = PostgresRedisBlobSwarmStore(SwarmConfig.from_env())
    try:
        store.ensure_schema()
        report = build_report(store)
        OUTPUT_PATH.parent.mkdir(exist_ok=True)
        OUTPUT_PATH.write_text(report, encoding="utf-8")
        print(str(OUTPUT_PATH))
    finally:
        store.close()


def build_report(store: PostgresRedisBlobSwarmStore) -> str:
    sections = [
        "# 模型 / Agnet 输入输出报告",
        "",
        "这份报告直接从 Azure PostgreSQL 的 `swarm_convergence`、`swarm_tasks` 和 `swarm_observations` 读取最新 live run。",
        "报告解释 S07 外部 GitHub 代码场景：被测项目是什么、本仓库做了什么测试工作、不同 Agnet 如何接手，以及最终如何收敛。",
        "报告不展示 `.env` 或任何密钥；模型原始输出保存在 PostgreSQL，本页只给中文审计摘要和必要证据前缀，避免纯英文原文淹没测试结论。",
        "",
        "## 场景说明：FastAPI 是什么项目",
        "",
        f"- 被测项目：`{EXTERNAL_REPO}`，固定 commit `{EXTERNAL_REPO_COMMIT}`。",
        "- 项目用途：FastAPI 是一个 Python API/Web 框架，用 Python 类型标注、Pydantic 数据模型和 Starlette 路由能力来构建 HTTP API，并自动生成 OpenAPI 文档。",
        "- 本轮为什么选它：响应模型过滤、依赖注入、OpenAPI schema 生成和 JSON 序列化横跨多个文件，适合验证蜂群 Agent 是否能处理复杂外部代码上下文。",
        "- 本轮不是什么：不是把当前 `fengqun` 仓库当成被测代码，也不是向 FastAPI 上游提交了真实补丁。这里验证的是蜂群对外部复杂项目的任务分解、模型交接、修正方案生成和收敛能力。",
        "",
        "## 本轮实际做了什么代码 / 测试工作",
        "",
        "- 把 S07 live 场景从“用本仓库验证自己”替换为外部 GitHub 项目 `fastapi/fastapi`。",
        "- 固定外部 commit 和目标文件范围，避免模型把本仓库源码当成被测对象。",
        "- 通过 NewAPI 模型发现流程选择至少 3 个不同模型，而不是写死 `NEWAPI_MODEL`。",
        "- 创建 7 个连续 Agnet 步骤：问题边界、依赖图、风险路径、反例、修正策略、文件级计划、最终验收。",
        "- 用 Azure PostgreSQL 保存 task pool、shared state、observation、convergence；用 Redis 保存事件流和 pheromone score；用 Blob 保存最终 artifact。",
        "- 对拒答、角色拒绝、偏题、本仓库漂移和交接断裂输出加入质量门：先扣分，再重试，必要时切换 fallback 模型。",
        "- 在最高分收敛前加入三角色多轮质量共识：交接连续性审查、输出质量审查和最终收敛审查必须共同接受。",
        "- 新增/更新报告审计测试，要求报告能说清场景、输入、输出、交接证据，并且不能泄露明显密钥样式。",
        "",
        "## 不同 Agnet 的工作怎么实现",
        "",
        "| Agnet 步骤 | capability | 职责 | 期望产出 |",
        "| --- | --- | --- | --- |",
    ]
    for kind, detail in CHAIN_STEP_DETAILS.items():
        sections.append(
            f"| {detail['marker']} | `{kind}` | {detail['role']} | {detail['expected_output']} |"
        )
    sections.extend(
        [
            "",
            "实现上，每个 Agnet 是一个 `Agent(id, capability, run)`：`SwarmCoordinator` 只会把 kind 匹配的 task 分给对应 capability。",
            "每个 `run` 会读取上一阶段 `chain:{run_id}:{STEP}:summary`，把它放进下一次模型调用的 `Previous summary`，再把本阶段摘要写回 shared state。",
            "所以这里的“接手”不是文档口头描述，而是由 task pool、shared state、Redis event 和 PostgreSQL observation 共同记录。",
            "",
            "## 收敛过程如何发生",
            "",
            "1. 初始化 run：写入 `run:{run_id}:goal/status`，创建 7 个 task，写入 PostgreSQL task pool。",
            "2. 模型选择：从 NewAPI `/models` 类接口发现模型，筛选可响应模型，再给 7 个步骤轮转分配。",
            "3. 任务执行：每个 Agnet claim 自己的 task，带上上一阶段 summary 调用模型。",
            "4. 质量补救：如果输出出现拒答、角色拒绝、偏题、本仓库漂移或交接断裂，先重试；仍不合格时切换 fallback 模型接手当前步骤。",
            "5. 状态推进：每步通过质量门后写入 `chain:{run_id}:{STEP}:summary`、`chain:{run_id}:edge:上一步->当前步` 和 `chain:{run_id}:cursor`。",
            "6. 评分沉淀：每个 task 的 score 写入 PostgreSQL 和 Redis pheromone sorted set，同时写入 Redis Stream 事件。",
            "7. 共识质量门：三类审查 Agnet 对交接连续性、输出质量和最终验收进行多轮投票，第一轮只形成候选，第二轮或以后达成接受才算通过。",
            "8. 结果收敛：通过质量门和共识门后，`SwarmCoordinator.converge` 从已完成 task 中选择最高分输出作为 accepted output，写入 `swarm_convergence`，并上传 Blob artifact。",
            "",
            "## 本轮测试场景补充",
            "",
            "本报告重点解释 S07 外部 GitHub 代码场景里的模型输入输出，但它属于完整标准矩阵的一部分；本轮重跑覆盖以下场景：",
            "",
            "| ID | 场景 | 测试目的 | 证据入口 |",
            "| --- | --- | --- | --- |",
        ]
    )
    for scenario in TEST_SCENARIOS:
        sections.append(
            f"| {scenario['id']} | {scenario['name']} | {scenario['purpose']} | {scenario['evidence']} |"
        )
    sections.extend(
        [
            "",
            "最新重跑结论：S01-S08 全部 PASS；最新 S07 外部 GitHub live run 会排在下方第一个。",
            "",
        ]
    )
    run_ids = fetch_latest_run_ids(store) or RUN_IDS
    if not run_ids:
        sections.extend(["## Run", "", "未找到外部 GitHub 代码场景的 live run。", ""])
    for run_id in run_ids:
        convergence = fetch_convergence(store, run_id)
        if convergence is None:
            sections.extend([f"## Run `{run_id}`", "", "未找到该 run。", ""])
            continue
        task_ids = [item["task_id"] for item in convergence["observations"]]
        tasks = fetch_tasks(store, task_ids)
        sections.extend(render_run(convergence, tasks))
    return "\n".join(line.rstrip() for line in "\n".join(sections).splitlines()).rstrip() + "\n"


def render_run(convergence: dict[str, object], tasks: dict[str, dict[str, object]]) -> list[str]:
    run_id = str(convergence["run_id"])
    goal = str(convergence["goal"])
    system_prompt = system_prompt_for_goal(goal)
    lines = [
        f"## Run `{run_id}`",
        "",
        f"- 任务目标：{goal}",
        f"- 完成任务数：{convergence['completed_tasks']}",
        f"- 收敛分数：{convergence['accepted_score']}",
        f"- Blob artifact：`{convergence['artifact_path']}`",
        "",
        "### 模型系统提示词",
        "",
        "```text",
        system_prompt,
        "```",
        "",
        "### 本次 run 的 Agnet 分工与输出质量",
        "",
        "| 调用 | Agnet | capability | 模型 | 任务职责 | 输出质量备注 |",
        "| --- | --- | --- | --- | --- | --- |",
    ]
    for index, observation in enumerate(convergence["observations"], start=1):
        task_id = observation["task_id"]
        task = tasks[task_id]
        output = redact_sensitive_text(str(task.get("output") or ""))
        detail = step_detail(str(task["kind"]))
        model = infer_model(task, output)
        lines.append(
            f"| {index} | `{task['claimed_by']}` | `{task['kind']}` | `{model}` | {detail['role']} | {output_quality_note(output, detail)} |"
        )
    lines.extend([""])

    for index, observation in enumerate(convergence["observations"], start=1):
        task_id = observation["task_id"]
        task = tasks[task_id]
        output = redact_sensitive_text(str(task.get("output") or ""))
        model = infer_model(task, output)
        detail = step_detail(str(task["kind"]))
        lines.extend(
            [
                f"### Agnet 调用 {index}: `{task['kind']}`",
                "",
                f"- Agnet：`{task['claimed_by']}`",
                f"- 模型：`{model}`",
                f"- 状态：`{task['status']}`",
                f"- 分数：`{task['score']}`",
                f"- 观测信号：`{observation['signal']}`",
                f"- 本步职责：{detail['role']}",
                f"- 期望产出：{detail['expected_output']}",
                "",
                "#### 给模型的 user prompt 结构",
                "",
                "```text",
                user_prompt_shape_for_goal(goal),
                "```",
                "",
                "#### 接手 / 交接机制",
                "",
                handoff_description(goal, run_id, task, output),
                "",
                "#### 本次任务输入 task.input",
                "",
                "```text",
                redact_sensitive_text(str(task["input"])).strip(),
                "```",
                "",
                "#### Agnet / 模型实际输出 task.output（审计摘要）",
                "",
                render_output_audit(run_id, task, output, detail),
                "",
            ]
        )
    return lines


def step_detail(kind: str) -> dict[str, str]:
    return CHAIN_STEP_DETAILS.get(
        kind,
        {
            "marker": kind,
            "title": kind,
            "role": "未知步骤，报告无法从 capability 映射到固定职责。",
            "expected_output": "未知产出。",
        },
    )


def render_output_audit(
    run_id: str,
    task: dict[str, object],
    output: str,
    detail: dict[str, str],
) -> str:
    prefix = first_line(output)
    edge = chain_edge_from_output(output)
    marker = detail["marker"]
    bullets = [
        "- 原始来源：PostgreSQL `swarm_tasks.output`。本报告不全文粘贴原文，而是给中文审计摘要；需要原文时可用 run_id 和 task_id 回查数据库。",
        f"- task_id：`{task['id']}`",
        f"- 关键前缀：`{prefix}`",
        f"- 链路边：`{edge}`；对应 shared state 键 `chain:{run_id}:edge:{edge}`。",
        f"- STEP 标记检查：`{marker}` {'存在' if marker in output else '缺失'}。",
        f"- 内容质量：{output_quality_note(output, detail)}",
        f"- 输出归纳：{output_summary(output, detail)}",
    ]
    if looks_like_boundary_refusal(output):
        bullets.append(
            "- 处理结论：该步暴露了异构模型在角色约束上的风险；它可作为链路/边界测试证据，但不应被当作充分的 FastAPI 技术修正内容。"
        )
    else:
        bullets.append(
            "- 处理结论：该步输出进入下一阶段 summary，并参与最终 score、pheromone 和 convergence 计算。"
        )
    return "\n".join(bullets)


def first_line(text: str) -> str:
    stripped = text.strip()
    if not stripped:
        return "<empty>"
    return stripped.splitlines()[0][:900]


def chain_edge_from_output(text: str) -> str:
    match = re.search(r"chain_edge=([^;\n]+)", text)
    return match.group(1).strip() if match else "<missing>"


def looks_like_boundary_refusal(text: str) -> bool:
    lowered = text.lower()
    refusal_markers = [
        "i appreciate the detailed context",
        "i need to clarify my actual role",
        "what i can actually do",
        "what i cannot do",
        "fake \"previous context\"",
        "fabricated context",
        "not a component in a multi-step reasoning swarm",
        "quality_gate_failed",
    ]
    return any(marker in lowered for marker in refusal_markers)


def output_quality_note(output: str, detail: dict[str, str]) -> str:
    if looks_like_boundary_refusal(output):
        return "模型输出主要是能力边界/拒绝式说明，未充分完成本步技术职责，标记为输出质量风险。"
    marker = detail["marker"]
    if marker not in output:
        return "未找到本步 STEP 标记，交接证据不足。"
    if "fastapi/fastapi" not in output:
        return "缺少外部仓库标识，存在目标漂移风险。"
    return "输出包含本步 STEP、外部仓库标识和交接信息，可作为本阶段审计证据。"


def output_summary(output: str, detail: dict[str, str]) -> str:
    if looks_like_boundary_refusal(output):
        return (
            f"该步原本应产出“{detail['expected_output']}”，但实际更像模型安全/能力边界声明；"
            "这说明当前评分标准还需要增加“非任务型输出”扣分。"
        )
    summaries = {
        "STEP-01": "建立了 FastAPI 外部仓库边界、禁止自测约束、目标文件范围、不变量、风险和下一步依赖图交接。",
        "STEP-02": "围绕 FastAPI 路由、依赖注入、OpenAPI 与响应序列化建立依赖图和状态模型。",
        "STEP-03": "定位 response_model、Depends、jsonable_encoder 与 OpenAPI schema 之间可能发生漂移的跨文件路径，并给出复杂度视角。",
        "STEP-04": "构造响应过滤、默认值、nullable、依赖参数和 OpenAPI schema 不一致的反例，为修正策略提供失败样本。",
        "STEP-05": "给出响应序列化与 OpenAPI/依赖注入交互的修正策略和兼容保护思路。",
        "STEP-06": "把修正策略落到 FastAPI 源码文件和测试文件，形成文件级补丁计划。",
        "STEP-07": "给出最终验收命令、失败判定、文件引用和可合并结论，并作为最高分输出进入收敛。",
    }
    return summaries.get(detail["marker"], detail["expected_output"])


def redact_sensitive_text(text: str) -> str:
    replacements = [
        (r"sk-[A-Za-z0-9]{20,}", "sk-<redacted>"),
        (r"AccountKey=[^;\s`]+", "AccountKey=<redacted>"),
        (r"password=[^,;\s`]+", "password=<redacted>"),
        (r"BEGIN [A-Z ]*PRIVATE KEY", "BEGIN <redacted> PRIVATE KEY"),
    ]
    redacted = text
    for pattern, replacement in replacements:
        redacted = re.sub(pattern, replacement, redacted, flags=re.IGNORECASE)
    return redacted


def handoff_description(goal: str, run_id: str, task: dict[str, object], output: str) -> str:
    if is_live_code_goal(goal):
        match = re.search(r"chain_edge=([^;\n]+)", output)
        edge = match.group(1).strip() if match else "<missing>"
        marker = edge.split("->")[-1] if "->" in edge else str(task.get("kind") or "<unknown>")
        return "\n".join(
            [
                f"- 当前输出前缀记录 `chain_edge={edge}`，证明本 Agnet 承接了上一阶段。",
                "- 调用前，wrapper 会把 `Previous marker` 和 `Previous summary` 放入 user prompt。",
                f"- 执行后，wrapper 把输出摘要写入 `chain:{run_id}:{marker}:summary`。",
                f"- 同时推进 `chain:{run_id}:cursor`，并写入 `chain:{run_id}:edge:{edge}=done`。",
                "- 下一个 Agnet 读取这个 summary 和 edge 后继续执行，所以接手不是靠口头描述，而是靠共享状态字段完成。",
            ]
        )
    if goal.startswith("真实全面场景"):
        return "\n".join(
            [
                "- 每个 Agnet 从 PostgreSQL 任务池 claim 自己的子任务。",
                "- 共享状态键列表作为上下文输入，让后续任务能看到已有 run/task/agent 状态。",
                "- 完成后写回 task output、score、observation 和 Redis Stream 事件，供收敛阶段读取。",
            ]
        )
    return "\n".join(
        [
            "- Agnet 从共享任务池 claim 与自身 capability 匹配的任务。",
            "- 完成后写回 output、score、observation、heartbeat 和 pheromone 分数。",
            "- 后续 Agnet 通过 shared state、task status 和 pheromone score 感知前序执行结果。",
        ]
    )


def system_prompt_for_goal(goal: str) -> str:
    if goal.startswith("full live test"):
        return (
            "You are a minimal Agnet worker inside a swarm. "
            "Return a concise result that can be scored and converged."
        )
    if goal.startswith("真实全面场景"):
        return (
            "You are a senior coding/algorithm agent in a multi-task swarm. "
            "Return a concrete engineering answer for the assigned subtask. "
            "Do not include secrets."
        )
    if is_live_code_goal(goal):
        return (
            "You are one stage in a continuous long-reasoning swarm. "
            "Carry forward prior conclusions, expose risks, and hand off a concise next-state. "
            "Do not reveal secrets."
        )
    return "<unknown system prompt>"


def user_prompt_shape_for_goal(goal: str) -> str:
    if goal.startswith("full live test"):
        return "\n".join(
            [
                "Task kind: <task.kind>",
                "Task input: <task.input>",
                "Known shared state keys: <sorted(shared_state.keys())>",
            ]
        )
    if goal.startswith("真实全面场景"):
        return "\n".join(
            [
                "Task kind: <task.kind>",
                "Task input:",
                "<task.input>",
                "",
                "Shared state keys: <sorted(shared_state.keys())>",
            ]
        )
    if is_live_code_goal(goal):
        return "\n".join(
            [
                "Previous marker: <previous_marker>",
                "Previous summary:",
                "<previous_step_summary>",
                "",
                "Task kind: <task.kind>",
                "Task input:",
                "<task.input>",
            ]
        )
    return "<unknown user prompt shape>"


def is_live_code_goal(goal: str) -> bool:
    return goal.startswith(LIVE_GOAL_PREFIX) or goal.startswith("连续性长推理场景")


def fetch_latest_run_ids(store: PostgresRedisBlobSwarmStore, *, limit: int = 1) -> list[str]:
    def operation(cur):
        cur.execute(
            """
            select run_id
            from swarm_convergence
            where goal like %s
            order by created_at desc
            limit %s
            """,
            (LIVE_GOAL_PREFIX + "%", limit),
        )
        return [row[0] for row in cur.fetchall()]

    return store._run_pg(operation)


def infer_model(task: dict[str, object], output: str) -> str:
    for pattern in [r"used_model=([^;\n]+)", r"primary_model=([^;\n]+)", r"model=([^;\n]+)"]:
        match = re.search(pattern, output)
        if match:
            return match.group(1).strip()
    match = re.search(r"model=([^;\n]+)", str(task.get("input") or ""))
    if match:
        return match.group(1).strip()
    return "<unknown>"


def fetch_convergence(store: PostgresRedisBlobSwarmStore, run_id: str) -> dict[str, object] | None:
    def operation(cur):
        cur.execute(
            """
            select run_id, goal, accepted_score, completed_tasks, observations, artifact_path, created_at
            from swarm_convergence
            where run_id = %s
            """,
            (run_id,),
        )
        row = cur.fetchone()
        if row is None:
            return None
        return {
            "run_id": row[0],
            "goal": row[1],
            "accepted_score": row[2],
            "completed_tasks": row[3],
            "observations": row[4],
            "artifact_path": row[5],
            "created_at": row[6],
        }

    return store._run_pg(operation)


def fetch_tasks(store: PostgresRedisBlobSwarmStore, task_ids: list[str]) -> dict[str, dict[str, object]]:
    def operation(cur):
        cur.execute(
            """
            select id, kind, input, status, claimed_by, output, score
            from swarm_tasks
            where id = any(%s)
            """,
            (task_ids,),
        )
        return {
            row[0]: {
                "id": row[0],
                "kind": row[1],
                "input": row[2],
                "status": row[3],
                "claimed_by": row[4],
                "output": row[5],
                "score": row[6],
            }
            for row in cur.fetchall()
        }

    return store._run_pg(operation)


if __name__ == "__main__":
    main()