fix(soak): 按 Codex 对抗审查加固故障注入的判据

九类实跑全过之后让 Codex 专门找「判据其实验不到它声称要验的东西」，它报的每条都带具体失败场景。这次的绿不是假的——实跑数据里这些判据都有料可判——但它们在别的输入下会假绿。真空成立三处改成无法判定：空日志的「全都解得开」、零步的「没有 env_error」、前后都空的「审计账没变」。上一版实跑里最后那条正是 0 vs 0 通过的。「环境是在完整一步之后坏的」原本只有通过和无法判定两档，没有击穿分支——那条判的是注入器自己，它对外宣称在第 N 次执行之后动手，整类故障的结论都建立在这句话上。提示词那两条原本只读库写进日志的数，而这套东西反复强调不能拿库的自述验库的行为——它在自己的核心主张上破了例。现在包一层模型客户端记下每次真正发出去的消息字符数，跑完逐步对账。字符数口径与库的算法对拍过，不然会全程假击穿。绝不重放那条补了两个角度：崩溃时账上那几条续跑之后要逐行原样还在（旧条目被改写、被抹掉原来一路绿灯），以及续跑段里不许出现与崩溃前完全相同的条目。Codex 提的「同一文件不同内容」在 b 档已被条数判据挡住，a 档不能加同样的规则——那一档续跑本来就该接着跑，模型再写一次是正常行为，加了会变成随机红。取消补了环境侧的静默判据与事件文件完整性。容器里已经发出去的执行在客户端计数上不留痕，这个盲区如实写进已知缺口，没假装验到；崩溃形态只能落在写入调用返回之后，同样记下来。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-11 11:53:00 -04:00
parent e590bea70e
commit 5a4d13b0f2
2 changed files with 633 additions and 19 deletions
@@ -61,16 +61,20 @@ from tools.soak.faults import (
    JsonlEventSink,
    KillTiming,
    LogRead,
+    PromptSizeRecordingClient,
    SelfKillingStore,
    build_context_overflow_budget,
    build_parser,
    check_all_steps_parse_failed,
    check_at_least_one_step,
+    check_audit_prefix_preserved,
    check_audit_unchanged,
    check_cancelled_raised,
    check_crash_prefix_preserved,
    check_env_broken_after_a_full_step,
+    check_env_quiet_after_cancel,
    check_env_untouched,
+    check_events_file_intact,
    check_executed_action_count,
    check_intents_settled,
    check_last_observation_is_synthetic,
@@ -78,7 +82,9 @@ from tools.soak.faults import (
    check_lease_returned,
    check_log_readable,
    check_never_action_not_replayed,
+    check_no_cross_segment_replay,
    check_no_env_error_step,
+    check_prompt_chars_match_what_was_sent,
    check_prompt_chars_monotonic,
    check_prompt_reached_max_prompt_chars,
    check_resume_made_progress,
@@ -1573,23 +1579,42 @@ def test_steps_before_last_all_executed_is_undetermined_with_a_single_step() ->


 def test_env_broken_after_a_full_step_passes() -> None:
-    outcome = check_env_broken_after_a_full_step(broken=True, executions_before_break=1)
+    outcome = check_env_broken_after_a_full_step(
+        broken=True, executions_before_break=1, expected_after=1
+    )
    assert outcome.status is CriterionStatus.PASSED


 def test_env_broken_after_a_full_step_is_undetermined_when_it_never_broke() -> None:
-    outcome = check_env_broken_after_a_full_step(broken=False, executions_before_break=0)
+    outcome = check_env_broken_after_a_full_step(
+        broken=False, executions_before_break=0, expected_after=1
+    )
    assert outcome.status is CriterionStatus.UNDETERMINED
    assert "一次都没被弄坏" in outcome.evidence


-def test_env_broken_after_a_full_step_is_undetermined_when_it_broke_too_early() -> None:
-    """第一次执行之前就打死容器的话，压到的是初始化而不是动作执行接缝。"""
-    outcome = check_env_broken_after_a_full_step(broken=True, executions_before_break=0)
-    assert outcome.status is CriterionStatus.UNDETERMINED
+def test_env_broken_after_a_full_step_breaches_when_it_broke_too_early() -> None:
+    """第一次执行之前就打死容器的话，压到的是初始化而不是动作执行接缝。
+
+    **这是击穿不是无法判定**：注入器对外宣称在第 1 次执行之后动手，整类故障的结论都建立在
+    那句话上。真在别的时刻动手的话，这一类压到的是另一件事，而报告上仍然写着它通过了。
+    """
+    outcome = check_env_broken_after_a_full_step(
+        broken=True, executions_before_break=0, expected_after=1
+    )
+    assert outcome.status is CriterionStatus.BREACHED
    assert "初始化" in outcome.evidence


+def test_env_broken_after_a_full_step_breaches_when_it_broke_too_late() -> None:
+    """比声明的晚动手同样是击穿：压到的仍然不是它声称的那个时刻。"""
+    outcome = check_env_broken_after_a_full_step(
+        broken=True, executions_before_break=3, expected_after=1
+    )
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "比声明的晚" in outcome.evidence
+
+
 # ---------------------------------------------------------------------------
 # 十八、环境故障：杀容器那段编排里的纯函数与执行器包装
 # ---------------------------------------------------------------------------
@@ -1727,3 +1752,276 @@ def test_parser_accepts_the_new_faults() -> None:
        ]
    )
    assert args.fault == ["context_overflow", "env_error"]
+
+
+# ---------------------------------------------------------------------------
+# 二十、真空成立那一类：空数据不许报通过
+# ---------------------------------------------------------------------------
+
+
+def test_log_readable_is_undetermined_on_an_empty_log() -> None:
+    """「零条记录全都解得开」是一句真空成立的话，它在报告上和真验过一模一样。"""
+    outcome = check_log_readable(LogRead(payloads=(), torn=False, bad_lines=()))
+    assert outcome.status is CriterionStatus.UNDETERMINED
+
+
+def test_log_readable_is_undetermined_when_the_file_is_missing(tmp_path: Path) -> None:
+    """日志文件根本不在时 `read_log` 给的就是空读数，这条链路要连得上。"""
+    outcome = check_log_readable(read_log(tmp_path / "nope.jsonl"))
+    assert outcome.status is CriterionStatus.UNDETERMINED
+
+
+def test_no_env_error_step_is_undetermined_without_steps() -> None:
+    outcome = check_no_env_error_step(as_read(intent()))
+    assert outcome.status is CriterionStatus.UNDETERMINED
+
+
+def test_audit_unchanged_is_undetermined_when_both_sides_are_empty() -> None:
+    """上一版实跑里这条正是 0 比 0 通过的：没有副作用就没有「没被重放」可验。"""
+    outcome = check_audit_unchanged(before=[], after=[])
+    assert outcome.status is CriterionStatus.UNDETERMINED
+
+
+def test_audit_unchanged_still_breaches_when_only_after_has_entries() -> None:
+    """崩溃时是空的、续跑之后长出条目：那是续跑执行了副作用，仍然是击穿。"""
+    outcome = check_audit_unchanged(before=[], after=["write_note\tevidence.md\taaa"])
+    assert outcome.status is CriterionStatus.BREACHED
+
+
+# ---------------------------------------------------------------------------
+# 二十一、环境账的前缀与跨崩溃边界的重放
+# ---------------------------------------------------------------------------
+
+
+def test_audit_prefix_preserved_passes_when_resume_only_appends() -> None:
+    before = ["write_note\tplan.md\taaa"]
+    after = [*before, "write_note\tevidence.md\tbbb"]
+    assert check_audit_prefix_preserved(before=before, after=after).status is (
+        CriterionStatus.PASSED
+    )
+
+
+def test_audit_prefix_preserved_breaches_when_an_old_entry_is_rewritten() -> None:
+    """条数不变、去重也不变，只有这条看得见：已经落地的那次副作用被改写了。"""
+    outcome = check_audit_prefix_preserved(
+        before=["write_note\tevidence.md\taaa"], after=["write_note\tevidence.md\tbbb"]
+    )
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "第 0 条" in outcome.evidence
+
+
+def test_audit_prefix_preserved_breaches_when_entries_disappear() -> None:
+    outcome = check_audit_prefix_preserved(before=["write_note\tplan.md\taaa"], after=[])
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "被抹掉" in outcome.evidence
+
+
+def test_audit_prefix_preserved_is_undetermined_without_a_crash_side_ledger() -> None:
+    assert check_audit_prefix_preserved(before=[], after=[]).status is (
+        CriterionStatus.UNDETERMINED
+    )
+
+
+def test_no_cross_segment_replay_breaches_on_a_repeat_after_the_crash() -> None:
+    """跨越崩溃边界的重复就是重放的签名。"""
+    before = ["write_note\tevidence.md\taaa"]
+    after = [*before, "write_note\tevidence.md\taaa"]
+    outcome = check_no_cross_segment_replay(before=before, after=after)
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "aaa" in outcome.evidence
+
+
+def test_no_cross_segment_replay_allows_new_content_after_the_crash() -> None:
+    """续跑写了别的内容不算重放：时机 A 下崩溃点之后本来就该接着跑。"""
+    before = ["write_note\tevidence.md\taaa"]
+    after = [*before, "write_note\tevidence.md\tbbb"]
+    assert check_no_cross_segment_replay(before=before, after=after).status is (
+        CriterionStatus.PASSED
+    )
+
+
+def test_no_cross_segment_replay_passes_when_resume_wrote_nothing() -> None:
+    before = ["write_note\tevidence.md\taaa"]
+    assert check_no_cross_segment_replay(before=before, after=list(before)).status is (
+        CriterionStatus.PASSED
+    )
+
+
+def test_no_cross_segment_replay_is_undetermined_without_a_crash_side_ledger() -> None:
+    outcome = check_no_cross_segment_replay(before=[], after=["write_note\tplan.md\taaa"])
+    assert outcome.status is CriterionStatus.UNDETERMINED
+
+
+# ---------------------------------------------------------------------------
+# 二十二、提示词：库记的数与它真正发出去的对账
+# ---------------------------------------------------------------------------
+
+
+class _SpyModelClient:
+    """记下每次收到的调用，`parameters()` 有自己的取值，用来验包装层原样转发。"""
+
+    def __init__(self) -> None:
+        self.calls: list[object] = []
+
+    def parameters(self) -> dict[str, str]:
+        return {"scope": "llm", "sources": "src:prov:model"}
+
+    async def call(self, call: object) -> ModelReply:
+        self.calls.append(call)
+        return ModelReply(call_id="c0", content="x", thinking="")
+
+
+def a_model_call(*, call_index: int, texts: tuple[str, ...]):
+    from polyloop.ports import ModelCall
+
+    return ModelCall(
+        messages=tuple(Message(role=Role.USER, content=(TextBlock(text=text),)) for text in texts),
+        call_index=call_index,
+        run_id=RUN_ID,
+        result_id=f"r{call_index}",
+        binding={},
+    )
+
+
+async def test_prompt_size_recorder_counts_what_was_sent() -> None:
+    inner = _SpyModelClient()
+    recorder = PromptSizeRecordingClient(inner=inner)
+    await recorder.call(a_model_call(call_index=0, texts=("abc", "de")))
+    await recorder.call(a_model_call(call_index=1, texts=("abcdef",)))
+    assert recorder.chars_by_call_index == {0: 5, 1: 6}
+    assert len(inner.calls) == 2
+
+
+def test_prompt_size_recorder_forwards_parameters_verbatim() -> None:
+    """加一个键会让续跑报假的参数漂移。"""
+    inner = _SpyModelClient()
+    assert PromptSizeRecordingClient(inner=inner).parameters() == inner.parameters()
+
+
+def test_prompt_chars_match_what_was_sent_passes() -> None:
+    read = as_read(
+        step_completed(step_idx=0, prompt_chars=100),
+        step_completed(step_idx=1, prompt_chars=250),
+    )
+    outcome = check_prompt_chars_match_what_was_sent(read, observed={0: 100, 1: 250})
+    assert outcome.status is CriterionStatus.PASSED
+
+
+def test_prompt_chars_match_what_was_sent_breaches_on_a_fabricated_value() -> None:
+    """库静默截断了历史，却仍把 prompt_chars 记成一路不减的自述值。
+
+    单调性那条与撞上限那条都只读这一列，两条都会通过；只有和真正发出去的字符数对账才看得见。
+    """
+    read = as_read(
+        step_completed(step_idx=0, prompt_chars=100),
+        step_completed(step_idx=1, prompt_chars=250),
+    )
+    assert check_prompt_chars_monotonic(read).status is CriterionStatus.PASSED
+    outcome = check_prompt_chars_match_what_was_sent(read, observed={0: 100, 1: 40})
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "第 1 步记的是 250 字符，实际发出去 40 字符" in outcome.evidence
+
+
+def test_prompt_chars_match_what_was_sent_is_undetermined_without_steps() -> None:
+    outcome = check_prompt_chars_match_what_was_sent(as_read(intent()), observed={0: 10})
+    assert outcome.status is CriterionStatus.UNDETERMINED
+
+
+def test_prompt_chars_match_what_was_sent_is_undetermined_without_calls() -> None:
+    read = as_read(step_completed(step_idx=0, prompt_chars=100))
+    assert check_prompt_chars_match_what_was_sent(read, observed={}).status is (
+        CriterionStatus.UNDETERMINED
+    )
+
+
+def test_prompt_chars_match_what_was_sent_is_undetermined_when_nothing_lines_up() -> None:
+    read = as_read(step_completed(step_idx=0, prompt_chars=100))
+    outcome = check_prompt_chars_match_what_was_sent(read, observed={7: 100})
+    assert outcome.status is CriterionStatus.UNDETERMINED
+
+
+# ---------------------------------------------------------------------------
+# 二十三、取消：环境侧安静下来，事件文件完整
+# ---------------------------------------------------------------------------
+
+
+def test_env_quiet_after_cancel_passes_when_both_counters_stand_still() -> None:
+    outcome = check_env_quiet_after_cancel(
+        dispatched_before=1, dispatched_after=1, completed_before=0, completed_after=0, waited_s=3.0
+    )
+    assert outcome.status is CriterionStatus.PASSED
+
+
+def test_env_quiet_after_cancel_breaches_when_more_work_is_dispatched() -> None:
+    """库在取消之后还往环境派活。"""
+    outcome = check_env_quiet_after_cancel(
+        dispatched_before=1, dispatched_after=2, completed_before=0, completed_after=0, waited_s=3.0
+    )
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "还在往环境派活" in outcome.evidence
+
+
+def test_env_quiet_after_cancel_breaches_when_an_execution_lands_late() -> None:
+    """动作已经发到容器，取消之后仍然跑完并改了环境状态。"""
+    outcome = check_env_quiet_after_cancel(
+        dispatched_before=1, dispatched_after=1, completed_before=0, completed_after=1, waited_s=3.0
+    )
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "跑完了" in outcome.evidence
+
+
+def test_env_quiet_after_cancel_breaches_when_a_counter_goes_backwards() -> None:
+    outcome = check_env_quiet_after_cancel(
+        dispatched_before=2, dispatched_after=1, completed_before=1, completed_after=1, waited_s=3.0
+    )
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "倒退" in outcome.evidence
+
+
+def test_events_file_intact_passes(tmp_path: Path) -> None:
+    path = tmp_path / f"{RUN_ID}.events.jsonl"
+    path.write_text(
+        '{"kind":"step_finished","run_id":"r","step_idx":0}\n'
+        '{"kind":"step_finished","run_id":"r","step_idx":1}\n',
+        encoding="utf-8",
+    )
+    outcome = check_events_file_intact(path)
+    assert outcome.status is CriterionStatus.PASSED
+    assert "2 行" in outcome.evidence
+
+
+def test_events_file_intact_breaches_on_a_torn_tail(tmp_path: Path) -> None:
+    """取消穿过了一次事件写入的中途。库有宽限期，所以这算击穿而不是「没发生过」。"""
+    path = tmp_path / f"{RUN_ID}.events.jsonl"
+    path.write_text(
+        '{"kind":"step_finished","run_id":"r","step_idx":0}\n{"kind":"step_fin',
+        encoding="utf-8",
+    )
+    outcome = check_events_file_intact(path)
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "没有被换行终结" in outcome.evidence
+
+
+def test_events_file_intact_breaches_on_a_broken_line(tmp_path: Path) -> None:
+    path = tmp_path / f"{RUN_ID}.events.jsonl"
+    path.write_text('{"kind":"step_finished"}\n不是 JSON\n', encoding="utf-8")
+    outcome = check_events_file_intact(path)
+    assert outcome.status is CriterionStatus.BREACHED
+    assert "[2]" in outcome.evidence
+
+
+def test_events_file_intact_breaches_on_a_non_object_line(tmp_path: Path) -> None:
+    path = tmp_path / f"{RUN_ID}.events.jsonl"
+    path.write_text('{"kind":"step_finished"}\n[1,2,3]\n', encoding="utf-8")
+    assert check_events_file_intact(path).status is CriterionStatus.BREACHED
+
+
+def test_events_file_intact_is_undetermined_without_a_file(tmp_path: Path) -> None:
+    outcome = check_events_file_intact(tmp_path / "nope.events.jsonl")
+    assert outcome.status is CriterionStatus.UNDETERMINED
+
+
+def test_events_file_intact_is_undetermined_on_an_empty_file(tmp_path: Path) -> None:
+    path = tmp_path / f"{RUN_ID}.events.jsonl"
+    path.write_bytes(b"")
+    assert check_events_file_intact(path).status is CriterionStatus.UNDETERMINED