fix(soak): 合成观察那两档不该逐字比对，判据写错了

193 次真实运行报了 9 处击穿，核下来是判据的错不是库的错：动作被拒绝或环境故障时，库
刻意不把执行器给的观察回填进历史，而是换成合成观察那一段。两个字段承载的本来就不是同
一件事——一个是执行器原文的留痕，一个是真正喂给模型的文本。

改判据时顺着源码发现被替换的是三列不是一列：观察文本、是否合成、截断字符数在那两档下
全部由库填。所以旧判据在环境故障那一档上必然也会误报，只是这 193 次里没撞上真的环境
故障；截断数那一列则是恰好两边都是 0，潜伏着没炸。

现在 executed 档三列仍然逐字比对，另两档改成断言步记录的「是否合成」标记确实立起来了——
库既然替换了观察，不立这个标记才是真出了问题。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tools/soak/scoreboard.py

@@ -57,6 +57,7 @@ from polyloop.serialization import (
 )
 from polyloop.stores import RECORD_KEY
 from polyloop.types import (
+    ActionStatus,
     Intent,
     IntentKind,
     ModelCallResult,
@@ -718,6 +719,10 @@ def _check_outcome_agrees_with_step(facts: RunFacts, config: CheckConfig) -> Che
 
     两者一次原子落地，所以它们不可能来自两次不同的执行。对不上说明装配那一层把某一侧
     填错了，而这种错在轨迹里完全看不出来——两个字段各自都合法。
+
+    **观察那三列只在 `executed` 一档上是原样透传，另外两档不是**，判据必须跟着分档，
+    理由写在 `Invariant` 的说明里（那段会进报告）。这条一开始按「三档都逐字相同」写，
+    在 193 次真实运行上报了 9 处击穿，核下来全是判据错、不是库错。
     """
     del config
     breaches: list[Evidence] = []
@@ -735,24 +740,31 @@ def _check_outcome_agrees_with_step(facts: RunFacts, config: CheckConfig) -> Che
                     )
                 )
             continue
-        pairs = (
+        passthrough = outcome.status is ActionStatus.EXECUTED
+        pairs = [
             ("action_status", outcome.status, record.step.action_status),
             (
                 "env_reported_completion",
                 outcome.env_reported_completion,
                 record.step.env_reported_completion,
             ),
-            (
-                "observation_is_synthetic",
-                outcome.observation_is_synthetic,
-                record.step.observation_is_synthetic,
-            ),
-            (
-                "observation_truncated_chars",
-                outcome.observation_truncated_chars,
-                record.step.observation_truncated_chars,
-            ),
-        )
+        ]
+        if passthrough:
+            # 只有这一档，步记录上的观察三列是执行器返回值的原样透传。
+            pairs.append(
+                (
+                    "observation_is_synthetic",
+                    outcome.observation_is_synthetic,
+                    record.step.observation_is_synthetic,
+                )
+            )
+            pairs.append(
+                (
+                    "observation_truncated_chars",
+                    outcome.observation_truncated_chars,
+                    record.step.observation_truncated_chars,
+                )
+            )
         for name, from_outcome, from_step in pairs:
             if from_outcome != from_step:
                 breaches.append(
@@ -764,17 +776,31 @@ def _check_outcome_agrees_with_step(facts: RunFacts, config: CheckConfig) -> Che
                         actual=f"动作结果 {from_outcome!r}，步记录 {from_step!r}",
                     )
                 )
-        if outcome.observation != record.step.observation:
+        if passthrough:
+            if outcome.observation != record.step.observation:
+                breaches.append(
+                    Evidence(
+                        run_id=facts.run_id,
+                        line_no=number,
+                        step_idx=record.step.step_idx,
+                        expected="executed 档下步记录的 observation 与动作结果的同名字段逐字相同",
+                        actual=(
+                            f"文本不同（长度 {len(outcome.observation)} vs "
+                            f"{len(record.step.observation)}）"
+                        ),
+                    )
+                )
+        elif not record.step.observation_is_synthetic:
             breaches.append(
                 Evidence(
                     run_id=facts.run_id,
                     line_no=number,
                     step_idx=record.step.step_idx,
-                    expected="步记录的 observation 与动作结果的同名字段相同",
-                    actual=(
-                        f"文本不同（长度 {len(outcome.observation)} vs "
-                        f"{len(record.step.observation)}）"
+                    expected=(
+                        f"{outcome.status.value} 档下库替换了回填进历史的观察，"
+                        "所以步记录的 observation_is_synthetic 为 True"
                     ),
+                    actual="observation_is_synthetic = False，那段观察没有被标成合成的",
                 )
             )
     return CheckOutcome(breaches=tuple(_cap(breaches, facts.run_id)))
@@ -1103,7 +1129,15 @@ INVARIANTS: tuple[Invariant, ...] = (
     ),
     Invariant(
         name="动作结果与步记录一致",
-        description="同一条 step_completed 里，动作结果与步记录的同名字段说的是同一件事。",
+        description=(
+            "同一条 step_completed 里，动作结果与步记录说的是同一件事。状态与完成标记这两项"
+            "在三档下都必须相同。观察那几列只在 executed 一档下逐字相同，因为只有那一档是"
+            "执行器返回值的原样透传：动作被拒绝与环境故障这两档，库刻意不把执行器给的观察"
+            "回填进历史，换成合成的那一段——执行器那段是「模型看得见的东西」，每次现造的话"
+            "同一份配置跑出来的两次运行在模型看来其实不同。执行器的原文没有丢，它就留在同一"
+            "条记录的动作结果里。所以这两档改判另一件事：库既然替换了观察，就必须把步记录的 "
+            "observation_is_synthetic 立起来，不立才是真出了问题。"
+        ),
         check=_check_outcome_agrees_with_step,
     ),
     Invariant(

tools/soak/tests/test_scoreboard.py

@@ -221,6 +221,41 @@ def edit_log(path: Path, edit: Callable[[list[dict]], list[dict]]) -> None:
     )
 
 
+def reshape_step(
+    path: Path,
+    *,
+    step_idx: int,
+    status: str,
+    outcome_observation: str,
+    step_observation: str,
+    outcome_is_synthetic: bool = False,
+    step_is_synthetic: bool = False,
+    outcome_truncated: int = 0,
+    step_truncated: int = 0,
+) -> None:
+    """把某一步改写成「执行器那侧与步记录那侧的观察不是同一段」的样子。
+
+    `build_records` 造出来的每一步都是 executed 且两侧逐字相同，而真实产物里被拒绝的动作
+    与环境故障那两档不长这样——库会替换回填进历史的观察。反例都从这里造。
+    """
+
+    def edit(items: list[dict]) -> list[dict]:
+        for item in items:
+            if item["record"] != "step_completed" or item["step"]["step_idx"] != step_idx:
+                continue
+            item["action_outcome"]["status"] = status
+            item["action_outcome"]["observation"] = outcome_observation
+            item["action_outcome"]["observation_is_synthetic"] = outcome_is_synthetic
+            item["action_outcome"]["observation_truncated_chars"] = outcome_truncated
+            item["step"]["action_status"] = status
+            item["step"]["observation"] = step_observation
+            item["step"]["observation_is_synthetic"] = step_is_synthetic
+            item["step"]["observation_truncated_chars"] = step_truncated
+        return items
+
+    edit_log(path, edit)
+
+
 def verdict_of(scoreboard, name: str) -> Verdict:
     for item in scoreboard.invariants:
         if item.name == name:
@@ -464,6 +499,101 @@ def test_outcome_disagreeing_with_step_record_is_a_breach(tmp_path: Path) -> Non
     assert_breached(evaluate(tmp_path), "动作结果与步记录一致")
 
 
+def test_executed_step_with_a_different_observation_is_a_breach(tmp_path: Path) -> None:
+    """executed 档是原样透传，两侧观察必须逐字相同。"""
+    materialize(tmp_path)
+    reshape_step(
+        tmp_path / "soak-0001.jsonl",
+        step_idx=0,
+        status="executed",
+        outcome_observation="环境返回的原文",
+        step_observation="换了一段别的",
+    )
+    assert_breached(evaluate(tmp_path), "动作结果与步记录一致")
+
+
+@pytest.mark.parametrize(
+    ("status", "executor_text", "executor_is_synthetic"),
+    [
+        (
+            "not_executed",
+            "工具不存在：'final_answer'，本次可见的是 "
+            "['read_document', 'grep_document', 'write_note']",
+            True,
+        ),
+        ("env_error", "容器没了：connection refused", False),
+    ],
+)
+def test_replaced_observation_is_not_a_breach(
+    tmp_path: Path, status: str, executor_text: str, executor_is_synthetic: bool
+) -> None:
+    """未执行与环境故障两档，库刻意换掉回填进历史的观察，两侧文本不同是正常的。
+
+    数据形状照 `tools/soak/runs/full/govdoc-15-execute.jsonl` 第 36 行那条真实记录造：
+    执行器那侧留的是「工具不存在」的原文，步记录那侧是合成的那段提示。截断数也一起换掉
+    （库在这两档下一律填 0），所以它同样不该被比对。
+    """
+    materialize(tmp_path)
+    reshape_step(
+        tmp_path / "soak-0001.jsonl",
+        step_idx=0,
+        status=status,
+        outcome_observation=executor_text,
+        step_observation=(
+            "[动作被拒绝，这一步没有执行任何工具]\n"
+            "请对照工具清单检查工具名与参数，然后重新输出一个 JSON 对象。"
+        ),
+        outcome_is_synthetic=executor_is_synthetic,
+        step_is_synthetic=True,
+        outcome_truncated=40,
+        step_truncated=0,
+    )
+    assert verdict_of(evaluate(tmp_path), "动作结果与步记录一致") is Verdict.PASSED
+
+
+@pytest.mark.parametrize("status", ["not_executed", "env_error"])
+def test_replaced_observation_without_the_synthetic_flag_is_a_breach(
+    tmp_path: Path, status: str
+) -> None:
+    """库既然替换了观察，就必须把 observation_is_synthetic 立起来，不立才是真出了问题。"""
+    materialize(tmp_path)
+    reshape_step(
+        tmp_path / "soak-0001.jsonl",
+        step_idx=0,
+        status=status,
+        outcome_observation="执行器给的原文",
+        step_observation="库换上去的那一段",
+        outcome_is_synthetic=False,
+        step_is_synthetic=False,
+    )
+    assert_breached(evaluate(tmp_path), "动作结果与步记录一致")
+
+
+@pytest.mark.parametrize("status", ["not_executed", "env_error"])
+def test_status_still_has_to_agree_in_every_branch(tmp_path: Path, status: str) -> None:
+    """状态与完成标记这两项三档下都比：观察分档，它们不分。"""
+    materialize(tmp_path)
+    reshape_step(
+        tmp_path / "soak-0001.jsonl",
+        step_idx=0,
+        status=status,
+        outcome_observation="执行器给的原文",
+        step_observation="库换上去的那一段",
+        outcome_is_synthetic=True,
+        step_is_synthetic=True,
+    )
+    edit_log(
+        tmp_path / "soak-0001.jsonl",
+        lambda items: [
+            {**item, "step": {**item["step"], "action_status": "executed"}}
+            if item["record"] == "step_completed" and item["step"]["step_idx"] == 0
+            else item
+            for item in items
+        ],
+    )
+    assert_breached(evaluate(tmp_path), "动作结果与步记录一致")
+
+
 def test_prompt_chars_going_backwards_is_a_breach(tmp_path: Path) -> None:
     materialize(tmp_path, steps=3)