diff --git a/tools/soak/scoreboard.py b/tools/soak/scoreboard.py index caa9bfa..085ecac 100644 --- a/tools/soak/scoreboard.py +++ b/tools/soak/scoreboard.py @@ -57,6 +57,7 @@ from polyloop.serialization import ( ) from polyloop.stores import RECORD_KEY from polyloop.types import ( + ActionStatus, Intent, IntentKind, ModelCallResult, @@ -718,6 +719,10 @@ def _check_outcome_agrees_with_step(facts: RunFacts, config: CheckConfig) -> Che 两者一次原子落地，所以它们不可能来自两次不同的执行。对不上说明装配那一层把某一侧 填错了，而这种错在轨迹里完全看不出来——两个字段各自都合法。 + + **观察那三列只在 `executed` 一档上是原样透传，另外两档不是**，判据必须跟着分档， + 理由写在 `Invariant` 的说明里（那段会进报告）。这条一开始按「三档都逐字相同」写， + 在 193 次真实运行上报了 9 处击穿，核下来全是判据错、不是库错。 """ del config breaches: list[Evidence] = [] @@ -735,24 +740,31 @@ def _check_outcome_agrees_with_step(facts: RunFacts, config: CheckConfig) -> Che ) ) continue - pairs = ( + passthrough = outcome.status is ActionStatus.EXECUTED + pairs = [ ("action_status", outcome.status, record.step.action_status), ( "env_reported_completion", outcome.env_reported_completion, record.step.env_reported_completion, ), - ( - "observation_is_synthetic", - outcome.observation_is_synthetic, - record.step.observation_is_synthetic, - ), - ( - "observation_truncated_chars", - outcome.observation_truncated_chars, - record.step.observation_truncated_chars, - ), - ) + ] + if passthrough: + # 只有这一档，步记录上的观察三列是执行器返回值的原样透传。 + pairs.append( + ( + "observation_is_synthetic", + outcome.observation_is_synthetic, + record.step.observation_is_synthetic, + ) + ) + pairs.append( + ( + "observation_truncated_chars", + outcome.observation_truncated_chars, + record.step.observation_truncated_chars, + ) + ) for name, from_outcome, from_step in pairs: if from_outcome != from_step: breaches.append( @@ -764,17 +776,31 @@ def _check_outcome_agrees_with_step(facts: RunFacts, config: CheckConfig) -> Che actual=f"动作结果 {from_outcome!r}，步记录 {from_step!r}", ) ) - if outcome.observation != record.step.observation: + if passthrough: + if outcome.observation != record.step.observation: + breaches.append( + Evidence( + run_id=facts.run_id, + line_no=number, + step_idx=record.step.step_idx, + expected="executed 档下步记录的 observation 与动作结果的同名字段逐字相同", + actual=( + f"文本不同（长度 {len(outcome.observation)} vs " + f"{len(record.step.observation)}）" + ), + ) + ) + elif not record.step.observation_is_synthetic: breaches.append( Evidence( run_id=facts.run_id, line_no=number, step_idx=record.step.step_idx, - expected="步记录的 observation 与动作结果的同名字段相同", - actual=( - f"文本不同（长度 {len(outcome.observation)} vs " - f"{len(record.step.observation)}）" + expected=( + f"{outcome.status.value} 档下库替换了回填进历史的观察，" + "所以步记录的 observation_is_synthetic 为 True" ), + actual="observation_is_synthetic = False，那段观察没有被标成合成的", ) ) return CheckOutcome(breaches=tuple(_cap(breaches, facts.run_id))) @@ -1103,7 +1129,15 @@ INVARIANTS: tuple[Invariant, ...] = ( ), Invariant( name="动作结果与步记录一致", - description="同一条 step_completed 里，动作结果与步记录的同名字段说的是同一件事。", + description=( + "同一条 step_completed 里，动作结果与步记录说的是同一件事。状态与完成标记这两项" + "在三档下都必须相同。观察那几列只在 executed 一档下逐字相同，因为只有那一档是" + "执行器返回值的原样透传：动作被拒绝与环境故障这两档，库刻意不把执行器给的观察" + "回填进历史，换成合成的那一段——执行器那段是「模型看得见的东西」，每次现造的话" + "同一份配置跑出来的两次运行在模型看来其实不同。执行器的原文没有丢，它就留在同一" + "条记录的动作结果里。所以这两档改判另一件事：库既然替换了观察，就必须把步记录的 " + "observation_is_synthetic 立起来，不立才是真出了问题。" + ), check=_check_outcome_agrees_with_step, ), Invariant( diff --git a/tools/soak/tests/test_scoreboard.py b/tools/soak/tests/test_scoreboard.py index 3f19b20..6fcb8d6 100644 --- a/tools/soak/tests/test_scoreboard.py +++ b/tools/soak/tests/test_scoreboard.py @@ -221,6 +221,41 @@ def edit_log(path: Path, edit: Callable[[list[dict]], list[dict]]) -> None: ) +def reshape_step( + path: Path, + *, + step_idx: int, + status: str, + outcome_observation: str, + step_observation: str, + outcome_is_synthetic: bool = False, + step_is_synthetic: bool = False, + outcome_truncated: int = 0, + step_truncated: int = 0, +) -> None: + """把某一步改写成「执行器那侧与步记录那侧的观察不是同一段」的样子。 + + `build_records` 造出来的每一步都是 executed 且两侧逐字相同，而真实产物里被拒绝的动作 + 与环境故障那两档不长这样——库会替换回填进历史的观察。反例都从这里造。 + """ + + def edit(items: list[dict]) -> list[dict]: + for item in items: + if item["record"] != "step_completed" or item["step"]["step_idx"] != step_idx: + continue + item["action_outcome"]["status"] = status + item["action_outcome"]["observation"] = outcome_observation + item["action_outcome"]["observation_is_synthetic"] = outcome_is_synthetic + item["action_outcome"]["observation_truncated_chars"] = outcome_truncated + item["step"]["action_status"] = status + item["step"]["observation"] = step_observation + item["step"]["observation_is_synthetic"] = step_is_synthetic + item["step"]["observation_truncated_chars"] = step_truncated + return items + + edit_log(path, edit) + + def verdict_of(scoreboard, name: str) -> Verdict: for item in scoreboard.invariants: if item.name == name: @@ -464,6 +499,101 @@ def test_outcome_disagreeing_with_step_record_is_a_breach(tmp_path: Path) -> Non assert_breached(evaluate(tmp_path), "动作结果与步记录一致") +def test_executed_step_with_a_different_observation_is_a_breach(tmp_path: Path) -> None: + """executed 档是原样透传，两侧观察必须逐字相同。""" + materialize(tmp_path) + reshape_step( + tmp_path / "soak-0001.jsonl", + step_idx=0, + status="executed", + outcome_observation="环境返回的原文", + step_observation="换了一段别的", + ) + assert_breached(evaluate(tmp_path), "动作结果与步记录一致") + + +@pytest.mark.parametrize( + ("status", "executor_text", "executor_is_synthetic"), + [ + ( + "not_executed", + "工具不存在：'final_answer'，本次可见的是 " + "['read_document', 'grep_document', 'write_note']", + True, + ), + ("env_error", "容器没了：connection refused", False), + ], +) +def test_replaced_observation_is_not_a_breach( + tmp_path: Path, status: str, executor_text: str, executor_is_synthetic: bool +) -> None: + """未执行与环境故障两档，库刻意换掉回填进历史的观察，两侧文本不同是正常的。 + + 数据形状照 `tools/soak/runs/full/govdoc-15-execute.jsonl` 第 36 行那条真实记录造： + 执行器那侧留的是「工具不存在」的原文，步记录那侧是合成的那段提示。截断数也一起换掉 + （库在这两档下一律填 0），所以它同样不该被比对。 + """ + materialize(tmp_path) + reshape_step( + tmp_path / "soak-0001.jsonl", + step_idx=0, + status=status, + outcome_observation=executor_text, + step_observation=( + "[动作被拒绝，这一步没有执行任何工具]\n" + "请对照工具清单检查工具名与参数，然后重新输出一个 JSON 对象。" + ), + outcome_is_synthetic=executor_is_synthetic, + step_is_synthetic=True, + outcome_truncated=40, + step_truncated=0, + ) + assert verdict_of(evaluate(tmp_path), "动作结果与步记录一致") is Verdict.PASSED + + +@pytest.mark.parametrize("status", ["not_executed", "env_error"]) +def test_replaced_observation_without_the_synthetic_flag_is_a_breach( + tmp_path: Path, status: str +) -> None: + """库既然替换了观察，就必须把 observation_is_synthetic 立起来，不立才是真出了问题。""" + materialize(tmp_path) + reshape_step( + tmp_path / "soak-0001.jsonl", + step_idx=0, + status=status, + outcome_observation="执行器给的原文", + step_observation="库换上去的那一段", + outcome_is_synthetic=False, + step_is_synthetic=False, + ) + assert_breached(evaluate(tmp_path), "动作结果与步记录一致") + + +@pytest.mark.parametrize("status", ["not_executed", "env_error"]) +def test_status_still_has_to_agree_in_every_branch(tmp_path: Path, status: str) -> None: + """状态与完成标记这两项三档下都比：观察分档，它们不分。""" + materialize(tmp_path) + reshape_step( + tmp_path / "soak-0001.jsonl", + step_idx=0, + status=status, + outcome_observation="执行器给的原文", + step_observation="库换上去的那一段", + outcome_is_synthetic=True, + step_is_synthetic=True, + ) + edit_log( + tmp_path / "soak-0001.jsonl", + lambda items: [ + {**item, "step": {**item["step"], "action_status": "executed"}} + if item["record"] == "step_completed" and item["step"]["step_idx"] == 0 + else item + for item in items + ], + ) + assert_breached(evaluate(tmp_path), "动作结果与步记录一致") + + def test_prompt_chars_going_backwards_is_a_breach(tmp_path: Path) -> None: materialize(tmp_path, steps=3)