fix(soak): 合成观察那两档不该逐字比对,判据写错了
193 次真实运行报了 9 处击穿,核下来是判据的错不是库的错:动作被拒绝或环境故障时,库 刻意不把执行器给的观察回填进历史,而是换成合成观察那一段。两个字段承载的本来就不是同 一件事——一个是执行器原文的留痕,一个是真正喂给模型的文本。 改判据时顺着源码发现被替换的是三列不是一列:观察文本、是否合成、截断字符数在那两档下 全部由库填。所以旧判据在环境故障那一档上必然也会误报,只是这 193 次里没撞上真的环境 故障;截断数那一列则是恰好两边都是 0,潜伏着没炸。 现在 executed 档三列仍然逐字比对,另两档改成断言步记录的「是否合成」标记确实立起来了—— 库既然替换了观察,不立这个标记才是真出了问题。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+52
-18
@@ -57,6 +57,7 @@ from polyloop.serialization import (
|
||||
)
|
||||
from polyloop.stores import RECORD_KEY
|
||||
from polyloop.types import (
|
||||
ActionStatus,
|
||||
Intent,
|
||||
IntentKind,
|
||||
ModelCallResult,
|
||||
@@ -718,6 +719,10 @@ def _check_outcome_agrees_with_step(facts: RunFacts, config: CheckConfig) -> Che
|
||||
|
||||
两者一次原子落地,所以它们不可能来自两次不同的执行。对不上说明装配那一层把某一侧
|
||||
填错了,而这种错在轨迹里完全看不出来——两个字段各自都合法。
|
||||
|
||||
**观察那三列只在 `executed` 一档上是原样透传,另外两档不是**,判据必须跟着分档,
|
||||
理由写在 `Invariant` 的说明里(那段会进报告)。这条一开始按「三档都逐字相同」写,
|
||||
在 193 次真实运行上报了 9 处击穿,核下来全是判据错、不是库错。
|
||||
"""
|
||||
del config
|
||||
breaches: list[Evidence] = []
|
||||
@@ -735,24 +740,31 @@ def _check_outcome_agrees_with_step(facts: RunFacts, config: CheckConfig) -> Che
|
||||
)
|
||||
)
|
||||
continue
|
||||
pairs = (
|
||||
passthrough = outcome.status is ActionStatus.EXECUTED
|
||||
pairs = [
|
||||
("action_status", outcome.status, record.step.action_status),
|
||||
(
|
||||
"env_reported_completion",
|
||||
outcome.env_reported_completion,
|
||||
record.step.env_reported_completion,
|
||||
),
|
||||
(
|
||||
"observation_is_synthetic",
|
||||
outcome.observation_is_synthetic,
|
||||
record.step.observation_is_synthetic,
|
||||
),
|
||||
(
|
||||
"observation_truncated_chars",
|
||||
outcome.observation_truncated_chars,
|
||||
record.step.observation_truncated_chars,
|
||||
),
|
||||
)
|
||||
]
|
||||
if passthrough:
|
||||
# 只有这一档,步记录上的观察三列是执行器返回值的原样透传。
|
||||
pairs.append(
|
||||
(
|
||||
"observation_is_synthetic",
|
||||
outcome.observation_is_synthetic,
|
||||
record.step.observation_is_synthetic,
|
||||
)
|
||||
)
|
||||
pairs.append(
|
||||
(
|
||||
"observation_truncated_chars",
|
||||
outcome.observation_truncated_chars,
|
||||
record.step.observation_truncated_chars,
|
||||
)
|
||||
)
|
||||
for name, from_outcome, from_step in pairs:
|
||||
if from_outcome != from_step:
|
||||
breaches.append(
|
||||
@@ -764,17 +776,31 @@ def _check_outcome_agrees_with_step(facts: RunFacts, config: CheckConfig) -> Che
|
||||
actual=f"动作结果 {from_outcome!r},步记录 {from_step!r}",
|
||||
)
|
||||
)
|
||||
if outcome.observation != record.step.observation:
|
||||
if passthrough:
|
||||
if outcome.observation != record.step.observation:
|
||||
breaches.append(
|
||||
Evidence(
|
||||
run_id=facts.run_id,
|
||||
line_no=number,
|
||||
step_idx=record.step.step_idx,
|
||||
expected="executed 档下步记录的 observation 与动作结果的同名字段逐字相同",
|
||||
actual=(
|
||||
f"文本不同(长度 {len(outcome.observation)} vs "
|
||||
f"{len(record.step.observation)})"
|
||||
),
|
||||
)
|
||||
)
|
||||
elif not record.step.observation_is_synthetic:
|
||||
breaches.append(
|
||||
Evidence(
|
||||
run_id=facts.run_id,
|
||||
line_no=number,
|
||||
step_idx=record.step.step_idx,
|
||||
expected="步记录的 observation 与动作结果的同名字段相同",
|
||||
actual=(
|
||||
f"文本不同(长度 {len(outcome.observation)} vs "
|
||||
f"{len(record.step.observation)})"
|
||||
expected=(
|
||||
f"{outcome.status.value} 档下库替换了回填进历史的观察,"
|
||||
"所以步记录的 observation_is_synthetic 为 True"
|
||||
),
|
||||
actual="observation_is_synthetic = False,那段观察没有被标成合成的",
|
||||
)
|
||||
)
|
||||
return CheckOutcome(breaches=tuple(_cap(breaches, facts.run_id)))
|
||||
@@ -1103,7 +1129,15 @@ INVARIANTS: tuple[Invariant, ...] = (
|
||||
),
|
||||
Invariant(
|
||||
name="动作结果与步记录一致",
|
||||
description="同一条 step_completed 里,动作结果与步记录的同名字段说的是同一件事。",
|
||||
description=(
|
||||
"同一条 step_completed 里,动作结果与步记录说的是同一件事。状态与完成标记这两项"
|
||||
"在三档下都必须相同。观察那几列只在 executed 一档下逐字相同,因为只有那一档是"
|
||||
"执行器返回值的原样透传:动作被拒绝与环境故障这两档,库刻意不把执行器给的观察"
|
||||
"回填进历史,换成合成的那一段——执行器那段是「模型看得见的东西」,每次现造的话"
|
||||
"同一份配置跑出来的两次运行在模型看来其实不同。执行器的原文没有丢,它就留在同一"
|
||||
"条记录的动作结果里。所以这两档改判另一件事:库既然替换了观察,就必须把步记录的 "
|
||||
"observation_is_synthetic 立起来,不立才是真出了问题。"
|
||||
),
|
||||
check=_check_outcome_agrees_with_step,
|
||||
),
|
||||
Invariant(
|
||||
|
||||
Reference in New Issue
Block a user