Files
Agentswarm/orchestrator/cross_review.py
T
Songhaoz666andClaude Opus 4.8 a7d2bb2870 评审/返工时间线对客户端可见:review/rework 4 类脱敏事件纳入冻结集(Refs #34)
文档:event-schema §4 增 4 类(标 ⭐ + 脱敏说明)、header 13→17;frontend-event-api 评审/返工时间线行推进为已定义 + header 注明部分推进 + 剩余跨仓项(HM 注册、cockpit 渲染、仅脱敏摘要);review-loop-protocol §3.2 由"事件不进 Manager 流"更正为"已脱敏外发"。

测试:新增 scripts/test-review-timeline-events.py(单元投影脱敏 + run_cross_review 真实站点发出 + 断言无 evidence/summary 泄漏 + 4 类在冻结集);test-contract-freeze 的 13-精确断言改为"13 核心为子集"(因 #34 扩到 17)。接入 CI。本地全绿(含 e2e/cross-review 回归)。

影响范围:仅 agent_swarm(orchestrator + docs + 测试 + CI)。Manager:回调新增 4 类(向后兼容;HM agent_callback.go 需登记方可对客户端暴露)。密钥/内容:脱敏投影不含 evidence/summary/原文/密钥。不改契约鉴权/计费/审批链。

Refs #34

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-11 17:10:57 +08:00

534 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Cross-review protocol — multi-reviewer swarm validation closure (issue #11).
WHY THIS EXISTS
---------------
Today's "review loop" (``orchestrator/main.py:maybe_run_review_cycle`` +
``master_agent.review_and_decide`` + ``planner.review``) is a *single*-critic gate: the
Master judges the combined result pass/fail and reopens whatever ``retry_tasks`` it names.
Functionally that is equivalent to a Supervisor Retry — one authority decides, the workers
redo. There is no second, independent opinion; no recorded disagreement; no structured
attribution of *why* the rework was needed and *who* introduced the fault. So the loop never
forms a swarm-style cross-validation closure, and it cannot feed a faithful ``P_rework`` input
to the benchmark (``docs/benchmark/swarm-metrics-schema.md`` §2).
WHAT THIS MODULE ADDS
---------------------
A pure, side-effect-free protocol layer (no Redis, no WebSocket, no model calls):
* ``ReviewDecision`` — one reviewer's structured verdict with evidence + rework target.
* ``aggregate_reviews`` — combine >= 2 independent reviewers, detect disagreement, and
arbitrate (majority, then reviewer-weight tie-break), recording
the disagreement rather than hiding it.
* ``ReworkAttribution`` + ``classify_rework`` — attribute a rework to its root cause and
source task/agent, and classify it into a ``ReworkCategory``
usable as a ``P_rework`` input.
* event payload builders — ``review.started`` / ``review.decision_made`` /
``rework.requested`` / ``rework.completed`` shaped like the
existing ``swarm_runtime.emit_event`` payloads.
This module is INTENTIONALLY NOT WIRED into the live finalize path. Integration (how it would
replace/augment the Master review, behind ``ENABLE_CROSS_REVIEW``) is documented in
``docs/swarm/review-loop-protocol.md`` under "集成说明(Integration notes)". See that doc for
the honest non-integration statement (org honesty rule #9).
"""
from __future__ import annotations
import logging
from collections import Counter
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional
logger = logging.getLogger(__name__)
# --------------------------------------------------------------------------------------
# Rework taxonomy (P_rework input — benchmark schema §2)
# --------------------------------------------------------------------------------------
class ReworkCategory(str, Enum):
"""Why a piece of work had to be redone.
These categories partition the ``ReworkCount`` that feeds ``P_rework =
ReworkCount/TotalTasks*100`` (``docs/benchmark/swarm-metrics-schema.md`` §2). Splitting by
cause lets the collector attribute rework to the phase that introduced it instead of
treating every redo as an undifferentiated retry.
"""
REQUIREMENT = "requirement" # objective / spec misunderstanding
IMPLEMENTATION = "implementation" # impl defect: wrong/missing behavior
TEST = "test" # test defect: wrong assertions / framework mismatch
DOC = "doc" # documentation drift vs. implementation
COLLABORATION = "collaboration" # cross-specialist inconsistency / handoff gap
UNKNOWN = "unknown" # no signal — do NOT fabricate a cause (honesty rule #9)
# Keyword signals per category. Deterministic and explainable: the same heuristic family the
# existing planner._heuristic_consistency_check uses (e.g. pytest/unittest, ValueError clash).
_CATEGORY_SIGNALS: Dict[ReworkCategory, tuple] = {
ReworkCategory.REQUIREMENT: ("requirement", "objective", "scope", "misunderstood", "spec"),
ReworkCategory.IMPLEMENTATION: ("implementation", "impl", "bug", "logic", "incorrect behavior",
"wrong output", "raise", "exception"),
ReworkCategory.TEST: ("test", "pytest", "unittest", "assert", "coverage", "fixture"),
ReworkCategory.DOC: ("doc", "documentation", "readme", "docstring", "example"),
ReworkCategory.COLLABORATION: ("conflict", "inconsistent", "mismatch", "disagree",
"handoff", "between specialists", "across specialists"),
}
# --------------------------------------------------------------------------------------
# A single reviewer's verdict
# --------------------------------------------------------------------------------------
@dataclass
class ReviewDecision:
"""One independent reviewer's structured verdict on a swarm artifact.
Unlike today's Master verdict (a bare ``{accepted, summary, retry_tasks}`` dict), this
carries the *evidence* the verdict rests on, which acceptance *criteria* failed, which tasks
are *affected*, the *recommended rework*, and the *reviewer identity* — so >= 2 reviewers can
be cross-checked and any disagreement can be located and recorded.
"""
verdict: str # "pass" | "fail"
reviewer_agent_id: str
evidence: List[str] = field(default_factory=list)
failed_criteria: List[str] = field(default_factory=list)
affected_tasks: List[str] = field(default_factory=list)
recommended_rework: List[str] = field(default_factory=list)
confidence: float = 1.0 # reviewer self-confidence in [0,1]
weight: float = 1.0 # arbitration weight (e.g. seniority/role trust)
summary: str = ""
def __post_init__(self) -> None:
normalized = (self.verdict or "").strip().lower()
if normalized not in {"pass", "fail"}:
raise ValueError(f"verdict must be 'pass' or 'fail', got {self.verdict!r}")
self.verdict = normalized
if not self.reviewer_agent_id:
raise ValueError("reviewer_agent_id is required")
try:
self.confidence = max(0.0, min(1.0, float(self.confidence)))
except (TypeError, ValueError):
self.confidence = 1.0
try:
self.weight = max(0.0, float(self.weight))
except (TypeError, ValueError):
self.weight = 1.0
@property
def passed(self) -> bool:
return self.verdict == "pass"
def to_dict(self) -> Dict[str, Any]:
return {
"verdict": self.verdict,
"reviewer_agent_id": self.reviewer_agent_id,
"evidence": list(self.evidence),
"failed_criteria": list(self.failed_criteria),
"affected_tasks": list(self.affected_tasks),
"recommended_rework": list(self.recommended_rework),
"confidence": self.confidence,
"weight": self.weight,
"summary": self.summary,
}
# --------------------------------------------------------------------------------------
# Rework attribution (who/what caused the redo)
# --------------------------------------------------------------------------------------
@dataclass
class ReworkAttribution:
"""Attribution record for one requested rework.
Answers *why* (``rework_reason``), *what kind* (``root_cause`` -> ``ReworkCategory``), and
*whose work introduced it* (``source_task_id`` / ``introduced_by_agent_id``). This is the
structured input the benchmark collector needs to compute a meaningful ``P_rework`` and to
split rework by phase instead of by blind retry count.
"""
target_task_id: str # task being reopened / redone
rework_reason: str # human-readable cause
root_cause: ReworkCategory = ReworkCategory.UNKNOWN
source_task_id: Optional[str] = None # upstream task that introduced the fault
introduced_by_agent_id: Optional[str] = None # agent whose output introduced the fault
detected_by_agent_id: Optional[str] = None # reviewer who flagged it
evidence: List[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
return {
"target_task_id": self.target_task_id,
"rework_reason": self.rework_reason,
"root_cause": self.root_cause.value,
"source_task_id": self.source_task_id,
"introduced_by_agent_id": self.introduced_by_agent_id,
"detected_by_agent_id": self.detected_by_agent_id,
"evidence": list(self.evidence),
}
def classify_rework(
reason: str,
failed_criteria: Optional[List[str]] = None,
*,
disagreement: bool = False,
) -> ReworkCategory:
"""Classify a rework into a :class:`ReworkCategory` from its textual signals.
Deterministic keyword scoring (no model). When two reviewers disagreed on the same
artifact, the rework is biased toward ``COLLABORATION`` because a cross-specialist
inconsistency is the most common root of reviewer disagreement. Returns ``UNKNOWN`` when no
signal matches — it never fabricates a cause (honesty rule #9).
"""
text = " ".join([reason or ""] + list(failed_criteria or [])).lower()
if not text.strip():
return ReworkCategory.COLLABORATION if disagreement else ReworkCategory.UNKNOWN
scores: Counter = Counter()
for category, signals in _CATEGORY_SIGNALS.items():
for signal in signals:
if signal in text:
scores[category] += 1
if disagreement:
# A recorded disagreement is itself evidence of a collaboration/consistency gap, so it
# both adds a vote AND wins ties: when reviewers split, the cross-specialist
# inconsistency is the root we want surfaced over any single-phase signal.
scores[ReworkCategory.COLLABORATION] += 1
if not scores:
return ReworkCategory.UNKNOWN
def _rank(kv):
category, score = kv
# Highest score wins. On a tie, prefer COLLABORATION when reviewers disagreed; otherwise
# fall back to a stable category order for reproducibility.
collab_tiebreak = 1 if (disagreement and category is ReworkCategory.COLLABORATION) else 0
return (score, collab_tiebreak, -list(ReworkCategory).index(category))
best = max(scores.items(), key=_rank)
return best[0]
def build_rework_attributions(
verdict: "AggregatedVerdict",
*,
task_owner: Optional[Dict[str, str]] = None,
) -> List[ReworkAttribution]:
"""Build one :class:`ReworkAttribution` per task the arbitrated verdict wants redone.
``task_owner`` maps task_id -> agent_id so the introducing agent can be attributed; when a
task's owner is unknown the field stays ``None`` rather than being guessed.
"""
task_owner = task_owner or {}
attributions: List[ReworkAttribution] = []
# Prefer the failing reviewers' evidence/criteria as the reason source.
failing = [d for d in verdict.decisions if not d.passed]
reason_bits = []
failed_criteria: List[str] = []
detected_by: Optional[str] = None
evidence: List[str] = []
for d in failing:
if d.summary:
reason_bits.append(d.summary)
failed_criteria.extend(d.failed_criteria)
evidence.extend(d.evidence)
detected_by = detected_by or d.reviewer_agent_id
reason = "; ".join(reason_bits) or verdict.summary or "rework requested by cross-review"
for task_id in verdict.rework_targets:
category = classify_rework(reason, failed_criteria, disagreement=verdict.disagreement)
attributions.append(
ReworkAttribution(
target_task_id=task_id,
rework_reason=reason,
root_cause=category,
source_task_id=task_id,
introduced_by_agent_id=task_owner.get(task_id),
detected_by_agent_id=detected_by,
evidence=list(dict.fromkeys(evidence)), # de-dup, keep order
)
)
return attributions
# --------------------------------------------------------------------------------------
# Aggregated, arbitrated verdict across >= 2 reviewers
# --------------------------------------------------------------------------------------
@dataclass
class AggregatedVerdict:
"""The cross-review outcome after combining and arbitrating independent reviewers."""
accepted: bool
method: str # arbitration method actually used
disagreement: bool # reviewers did not unanimously agree
pass_votes: int
fail_votes: int
rework_targets: List[str] = field(default_factory=list)
decisions: List[ReviewDecision] = field(default_factory=list)
summary: str = ""
def to_dict(self) -> Dict[str, Any]:
return {
"accepted": self.accepted,
"method": self.method,
"disagreement": self.disagreement,
"pass_votes": self.pass_votes,
"fail_votes": self.fail_votes,
"rework_targets": list(self.rework_targets),
"summary": self.summary,
"decisions": [d.to_dict() for d in self.decisions],
}
def aggregate_reviews(
reviews: List[ReviewDecision],
*,
method: str = "majority",
) -> AggregatedVerdict:
"""Combine >= 2 independent reviewers into one arbitrated verdict.
This is the core difference from today's single-critic gate. It:
1. Requires at least two reviewers (a single reviewer is a Supervisor Retry, not a
cross-review) — raises ``ValueError`` otherwise.
2. Detects *disagreement* (reviewers split on pass/fail) and records it explicitly.
3. Arbitrates:
- ``method="majority"``: more fails than passes -> reject. A pass/fail tie is
resolved conservatively as a rejection (safety-biased: never silently accept a
split decision).
- ``method="weighted"``: compares Σ(weight·confidence) of pass vs. fail reviewers; the
heavier side wins, ties -> reject.
4. Unions the ``recommended_rework``/``affected_tasks`` of the *failing* reviewers into the
rework target set, so every concern raised is acted on.
Acceptance requires no rework targets AND a non-reject arbitration outcome.
"""
if len(reviews) < 2:
raise ValueError("cross-review requires at least 2 independent reviewers")
pass_votes = sum(1 for d in reviews if d.passed)
fail_votes = len(reviews) - pass_votes
disagreement = pass_votes > 0 and fail_votes > 0
if method == "weighted":
pass_w = sum(d.weight * d.confidence for d in reviews if d.passed)
fail_w = sum(d.weight * d.confidence for d in reviews if not d.passed)
# Tie or fail-heavy -> reject (safety bias).
accepted = pass_w > fail_w
used_method = "weighted"
else:
# Majority; tie -> reject (safety bias).
accepted = pass_votes > fail_votes
used_method = "majority"
rework_targets: List[str] = []
for d in reviews:
if d.passed:
continue
for tid in list(d.recommended_rework) + list(d.affected_tasks):
if tid and tid not in rework_targets:
rework_targets.append(tid)
# A reject with no nameable target still must not be silently accepted; surface it so the
# caller can decide (the live loop would, e.g., reopen all completed tasks or stop).
if not accepted and not rework_targets:
rework_targets = [] # explicit: empty target set, accepted stays False
accepted = accepted and not rework_targets
if disagreement:
summary = (
f"reviewers disagreed ({pass_votes} pass / {fail_votes} fail); "
f"arbitrated by {used_method} -> {'accept' if accepted else 'reject'}"
)
else:
summary = (
f"reviewers unanimous ({pass_votes} pass / {fail_votes} fail) -> "
f"{'accept' if accepted else 'reject'}"
)
logger.info("[cross_review] %s; rework_targets=%s", summary, rework_targets)
return AggregatedVerdict(
accepted=accepted,
method=used_method,
disagreement=disagreement,
pass_votes=pass_votes,
fail_votes=fail_votes,
rework_targets=rework_targets,
decisions=list(reviews),
summary=summary,
)
# --------------------------------------------------------------------------------------
# Event payload builders (shape matches swarm_runtime.emit_event payloads)
# --------------------------------------------------------------------------------------
def review_started_payload(
run_id: str,
*,
reviewer_agent_ids: List[str],
artifact_task_ids: List[str],
cycle: int = 0,
) -> Dict[str, Any]:
"""Payload for a ``review.started`` event (cross-review round begins)."""
return {
"swarm_id": run_id,
"phase": "Review",
"review_kind": "cross_review",
"cycle": cycle,
"reviewer_agent_ids": list(reviewer_agent_ids),
"artifact_task_ids": list(artifact_task_ids),
"reviewer_count": len(reviewer_agent_ids),
"summary": f"Cross-review round {cycle} started with {len(reviewer_agent_ids)} reviewers",
}
def review_decision_made_payload(
run_id: str,
verdict: AggregatedVerdict,
*,
cycle: int = 0,
) -> Dict[str, Any]:
"""Payload for a ``review.decision_made`` event (arbitrated verdict produced)."""
return {
"swarm_id": run_id,
"phase": "Review",
"review_kind": "cross_review",
"cycle": cycle,
"accepted": verdict.accepted,
"arbitration_method": verdict.method,
"disagreement": verdict.disagreement,
"pass_votes": verdict.pass_votes,
"fail_votes": verdict.fail_votes,
"rework_targets": list(verdict.rework_targets),
"reviews": [d.to_dict() for d in verdict.decisions],
"summary": verdict.summary,
}
def rework_requested_payload(
run_id: str,
attribution: ReworkAttribution,
*,
cycle: int = 0,
) -> Dict[str, Any]:
"""Payload for a ``rework.requested`` event (one task sent back for redo)."""
return {
"swarm_id": run_id,
"phase": "Review",
"cycle": cycle,
"task_id": attribution.target_task_id,
"rework_reason": attribution.rework_reason,
"root_cause": attribution.root_cause.value,
"source_task_id": attribution.source_task_id,
"introduced_by_agent_id": attribution.introduced_by_agent_id,
"detected_by_agent_id": attribution.detected_by_agent_id,
"evidence": list(attribution.evidence),
"summary": f"Rework requested for {attribution.target_task_id} "
f"({attribution.root_cause.value})",
}
def rework_completed_payload(
run_id: str,
attribution: ReworkAttribution,
*,
cycle: int = 0,
succeeded: bool = True,
) -> Dict[str, Any]:
"""Payload for a ``rework.completed`` event (a redone task reached a terminal state)."""
return {
"swarm_id": run_id,
"phase": "Review",
"cycle": cycle,
"task_id": attribution.target_task_id,
"root_cause": attribution.root_cause.value,
"introduced_by_agent_id": attribution.introduced_by_agent_id,
"status": "completed" if succeeded else "failed",
"summary": f"Rework {'completed' if succeeded else 'failed'} for "
f"{attribution.target_task_id}",
}
# ── Client-visible REDACTED projections (issue #34) ───────────────────────────
# The payload builders above are the rich INTERNAL telemetry shape. The desktop cockpit
# review/rework timeline must show only non-content metadata, so the client-facing events carry a
# strict allowlist: reviewer identity/verdict/failed_criteria/affected_tasks/recommended_rework/
# root_cause (+ safe scalars: cycle, counts, accepted, disagreement, method, status). They DROP all
# free text — `evidence`, `rework_reason`, and human-readable `summary` — to honor
# security-boundary (no prompt/code/content) and the #34 DoD. These four are the event types added
# to FROZEN_CLIENT_EVENT_TYPES; the emitter (`main.run_cross_review`) sends ONLY these projections.
# The four review/rework event types promoted to the client-visible set (#34).
CLIENT_REVIEW_EVENT_TYPES = (
"review.started",
"review.decision_made",
"rework.requested",
"rework.completed",
)
def review_started_client_payload(run_id: str, reviewer_agent_ids: List[str], *, cycle: int = 0) -> Dict[str, Any]:
"""Redacted `review.started` (a cross-review round begins). Identity + count only."""
return {
"swarm_id": run_id,
"phase": "Review",
"review_kind": "cross_review",
"cycle": cycle,
"reviewer_agent_ids": list(reviewer_agent_ids),
"reviewer_count": len(reviewer_agent_ids),
}
def review_decision_client_payload(run_id: str, verdict: "AggregatedVerdict", *, cycle: int = 0) -> Dict[str, Any]:
"""Redacted `review.decision_made`: arbitration outcome + per-reviewer verdict/criteria only.
Drops each ReviewDecision's free-text `evidence`/`summary` and the verdict `summary`.
"""
return {
"swarm_id": run_id,
"phase": "Review",
"review_kind": "cross_review",
"cycle": cycle,
"accepted": verdict.accepted,
"arbitration_method": verdict.method,
"disagreement": verdict.disagreement,
"pass_votes": verdict.pass_votes,
"fail_votes": verdict.fail_votes,
"rework_targets": list(verdict.rework_targets),
"reviewers": [
{
"reviewer_agent_id": d.reviewer_agent_id,
"verdict": d.verdict,
"failed_criteria": list(d.failed_criteria),
"affected_tasks": list(d.affected_tasks),
"recommended_rework": list(d.recommended_rework),
}
for d in verdict.decisions
],
}
def rework_requested_client_payload(run_id: str, attribution: ReworkAttribution, *, cycle: int = 0) -> Dict[str, Any]:
"""Redacted `rework.requested`: target + root_cause + attribution ids only (no reason/evidence)."""
return {
"swarm_id": run_id,
"phase": "Review",
"cycle": cycle,
"task_id": attribution.target_task_id,
"root_cause": attribution.root_cause.value,
"source_task_id": attribution.source_task_id,
"introduced_by_agent_id": attribution.introduced_by_agent_id,
"detected_by_agent_id": attribution.detected_by_agent_id,
}
def rework_completed_client_payload(run_id: str, *, task_id: str, root_cause: str, cycle: int = 0) -> Dict[str, Any]:
"""Redacted `rework.completed`: which task's rework converged + its root_cause."""
return {
"swarm_id": run_id,
"phase": "Review",
"cycle": cycle,
"task_id": task_id,
"root_cause": root_cause,
"status": "completed",
}