fix(soak): 崩溃改成子进程内部的确定性自杀，外部 SIGKILL 抢不到那个窗口
第一次实跑的结果：时机 A（一步完整落地之后崩）三次都没命中，每次都是「发信号与子进程停笔 之间又写进了记录」。原因是库写完步记录之后紧接着就写下一条意图，中间只有内存计算，窗口 窄到外部信号挤不进去。这个观察本身留在模块 docstring 里——它说明自然崩溃几乎总是落在 「有意图没结果」那一态上。 改成在子进程里包一层存储，在写入落盘返回之后按条件调 os._exit(137)。os._exit 不跑 finally、不跑 atexit、不 flush，对磁盘的效果与 SIGKILL 等价，而 JsonlRunStore 本来就 写完即 fsync，没有未刷缓冲要指望退出时替它写。外部 SIGKILL 那条路降级成兜底，没有删。 自杀条件都要求工作区审计账已经非空，即那个声明绝不重放的写入真的执行过。上一次实跑里 最硬的那条判据（绝不重放的动作没被执行两次）报的是无法判定，就是因为崩得太早、审计账 是空的，去重比对真空成立——判定器诚实地报了无法判定而不是绿，现在给它补上实料。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+283
-11
@@ -6,8 +6,12 @@
|
||||
违反的输入验它说击穿。
|
||||
|
||||
子进程编排那部分拆出了两个纯函数（`should_kill` 判时机到没到、`terminated_prefix` 截已终结
|
||||
前缀），它们不碰进程也不碰模型，直接单独测。真起子进程那两条用的是一个只会往文件里写几行
|
||||
前缀），它们不碰进程也不碰模型，直接单独测。真起子进程那几条用的是一个只会往文件里写几行
|
||||
JSON 的假子进程，跑完不到两秒。
|
||||
|
||||
确定性自杀那条路（`SelfKillingStore`）的测试**把 `os._exit` 换成一个抛哨兵异常的替身**：真调
|
||||
`os._exit` 会把跑测试的 pytest 进程一起带走，整次收集连一行结果都留不下。替身同时让「死之前
|
||||
那条记录有没有先写进内层存储」变得可断言——顺序反了的话，崩溃现场就少一条本该已经落地的记录。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,9 +23,24 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from polyloop.ports import EventKind, InvalidDecision
|
||||
from polyloop.types import ModelReply, RunResult, StopReason
|
||||
from polyloop.ports import EventKind, InvalidDecision, RunLog
|
||||
from polyloop.types import (
|
||||
ActionOutcome,
|
||||
ActionStatus,
|
||||
Intent,
|
||||
IntentKind,
|
||||
ModelCallResult,
|
||||
ModelReply,
|
||||
ReplayPolicy,
|
||||
RunFinished,
|
||||
RunResult,
|
||||
RunStarted,
|
||||
StepCompleted,
|
||||
StepRecord,
|
||||
StopReason,
|
||||
)
|
||||
from tools.soak.faults import (
|
||||
CRASH_EXIT_CODE,
|
||||
AlwaysInvalidParser,
|
||||
CallGuard,
|
||||
Criterion,
|
||||
@@ -30,6 +49,7 @@ from tools.soak.faults import (
|
||||
JsonlEventSink,
|
||||
KillTiming,
|
||||
LogRead,
|
||||
SelfKillingStore,
|
||||
build_parser,
|
||||
check_all_steps_parse_failed,
|
||||
check_audit_unchanged,
|
||||
@@ -59,6 +79,7 @@ from tools.soak.faults import (
|
||||
terminated_prefix,
|
||||
write_sidecars,
|
||||
)
|
||||
from tools.soak.scenarios.govdoc import AUDIT_LOG_NAME
|
||||
|
||||
RUN_ID = "fault-test-0"
|
||||
|
||||
@@ -596,11 +617,195 @@ def test_should_kill_on_empty_log() -> None:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 十二、子进程编排：真起一个假子进程
|
||||
# 十二、确定性自杀：写入落盘之后按时机把自己打死
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _ExitCalledError(Exception):
|
||||
"""`os._exit` 的替身抛的哨兵。真调 `os._exit` 会把 pytest 一起带走。"""
|
||||
|
||||
def __init__(self, code: int) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _exit_sentinel(code: int) -> None:
|
||||
raise _ExitCalledError(code)
|
||||
|
||||
|
||||
class _RecordingStore:
|
||||
"""记下每一次写入的假存储。`parameters()` 报的键与 `JsonlRunStore` 一致。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[str] = []
|
||||
|
||||
def parameters(self) -> dict[str, str]:
|
||||
return {"kind": "jsonl"}
|
||||
|
||||
async def read_log(self, run_id: str) -> RunLog:
|
||||
self.calls.append("read_log")
|
||||
return RunLog()
|
||||
|
||||
async def write_run_started(self, record: RunStarted) -> None:
|
||||
self.calls.append("run_started")
|
||||
|
||||
async def write_intent(self, record: Intent) -> None:
|
||||
self.calls.append("intent")
|
||||
|
||||
async def write_model_call_result(self, record: ModelCallResult) -> None:
|
||||
self.calls.append("model_call_result")
|
||||
|
||||
async def write_step_completed(self, record: StepCompleted) -> None:
|
||||
self.calls.append("step_completed")
|
||||
|
||||
async def write_run_finished(self, record: RunFinished) -> None:
|
||||
self.calls.append("run_finished")
|
||||
|
||||
|
||||
def an_intent(policy: ReplayPolicy = ReplayPolicy.NEVER) -> Intent:
|
||||
return Intent(
|
||||
run_id=RUN_ID,
|
||||
kind=IntentKind.MODEL_CALL,
|
||||
call_index=0,
|
||||
result_id="r0",
|
||||
replay_policy=policy,
|
||||
)
|
||||
|
||||
|
||||
def a_step_completed() -> StepCompleted:
|
||||
return StepCompleted(
|
||||
run_id=RUN_ID,
|
||||
result_id="a0",
|
||||
action_outcome=ActionOutcome(
|
||||
status=ActionStatus.EXECUTED,
|
||||
observation="o",
|
||||
observation_is_synthetic=False,
|
||||
env_reported_completion=False,
|
||||
observation_truncated_chars=0,
|
||||
),
|
||||
step=StepRecord(
|
||||
step_idx=0,
|
||||
raw_output="x",
|
||||
content_chars=1,
|
||||
thinking_chars=0,
|
||||
action="a",
|
||||
parse_ok=True,
|
||||
parse_error=None,
|
||||
observation="o",
|
||||
observation_is_synthetic=False,
|
||||
observation_truncated_chars=0,
|
||||
prompt_chars=10,
|
||||
call_id="c0",
|
||||
step_wall_ms=1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def wrapped(
|
||||
inner: _RecordingStore, *, timing: KillTiming, workspace: Path, audited: bool
|
||||
) -> SelfKillingStore:
|
||||
"""包一层，并按需要让工作区的审计账非空。"""
|
||||
workspace.mkdir(parents=True, exist_ok=True)
|
||||
if audited:
|
||||
(workspace / AUDIT_LOG_NAME).write_text("write_note\tevidence.md\taaa\n", encoding="utf-8")
|
||||
return SelfKillingStore(
|
||||
inner=inner, timing=timing, workspace=workspace, exit_now=_exit_sentinel
|
||||
)
|
||||
|
||||
|
||||
async def test_self_kill_after_step_when_the_audit_is_not_empty(tmp_path: Path) -> None:
|
||||
"""时机 A：步记录落盘之后当场死。**死之前那条记录必须已经写进内层存储**。"""
|
||||
inner = _RecordingStore()
|
||||
store = wrapped(inner, timing=KillTiming.AFTER_STEP, workspace=tmp_path, audited=True)
|
||||
with pytest.raises(_ExitCalledError) as caught:
|
||||
await store.write_step_completed(a_step_completed())
|
||||
assert caught.value.code == CRASH_EXIT_CODE
|
||||
assert inner.calls == ["step_completed"]
|
||||
|
||||
|
||||
async def test_self_kill_after_step_waits_for_a_real_write_note(tmp_path: Path) -> None:
|
||||
"""审计账还是空的就不死：那时崩掉，最硬那条判据只能真空成立。"""
|
||||
inner = _RecordingStore()
|
||||
store = wrapped(inner, timing=KillTiming.AFTER_STEP, workspace=tmp_path, audited=False)
|
||||
await store.write_step_completed(a_step_completed())
|
||||
assert inner.calls == ["step_completed"]
|
||||
|
||||
|
||||
async def test_self_kill_after_step_ignores_intents(tmp_path: Path) -> None:
|
||||
inner = _RecordingStore()
|
||||
store = wrapped(inner, timing=KillTiming.AFTER_STEP, workspace=tmp_path, audited=True)
|
||||
await store.write_intent(an_intent())
|
||||
assert inner.calls == ["intent"]
|
||||
|
||||
|
||||
async def test_self_kill_at_intent_on_a_never_intent(tmp_path: Path) -> None:
|
||||
inner = _RecordingStore()
|
||||
store = wrapped(inner, timing=KillTiming.AT_INTENT, workspace=tmp_path, audited=True)
|
||||
with pytest.raises(_ExitCalledError) as caught:
|
||||
await store.write_intent(an_intent(ReplayPolicy.NEVER))
|
||||
assert caught.value.code == CRASH_EXIT_CODE
|
||||
assert inner.calls == ["intent"]
|
||||
|
||||
|
||||
async def test_self_kill_at_intent_skips_a_safe_intent(tmp_path: Path) -> None:
|
||||
"""`safe` 那条意图崩了也没用：续跑会重放动作接着跑，停止原因不是状态未知。"""
|
||||
inner = _RecordingStore()
|
||||
store = wrapped(inner, timing=KillTiming.AT_INTENT, workspace=tmp_path, audited=True)
|
||||
await store.write_intent(an_intent(ReplayPolicy.SAFE))
|
||||
assert inner.calls == ["intent"]
|
||||
|
||||
|
||||
async def test_self_kill_at_intent_waits_for_a_real_write_note(tmp_path: Path) -> None:
|
||||
inner = _RecordingStore()
|
||||
store = wrapped(inner, timing=KillTiming.AT_INTENT, workspace=tmp_path, audited=False)
|
||||
await store.write_intent(an_intent(ReplayPolicy.NEVER))
|
||||
assert inner.calls == ["intent"]
|
||||
|
||||
|
||||
async def test_self_kill_at_intent_ignores_step_records(tmp_path: Path) -> None:
|
||||
inner = _RecordingStore()
|
||||
store = wrapped(inner, timing=KillTiming.AT_INTENT, workspace=tmp_path, audited=True)
|
||||
await store.write_step_completed(a_step_completed())
|
||||
assert inner.calls == ["step_completed"]
|
||||
|
||||
|
||||
async def test_self_kill_never_fires_on_the_other_three_writes(tmp_path: Path) -> None:
|
||||
"""开始、模型调用结果、结束这三处一律不死：它们不是任何一个时机的定义点。"""
|
||||
for timing in KillTiming:
|
||||
inner = _RecordingStore()
|
||||
store = wrapped(inner, timing=timing, workspace=tmp_path, audited=True)
|
||||
await store.write_run_started(RunStarted(run_id=RUN_ID, parameter_snapshot={}))
|
||||
await store.write_model_call_result(
|
||||
ModelCallResult(run_id=RUN_ID, result_id="r0", reply=None, failure="炸了")
|
||||
)
|
||||
await store.write_run_finished(
|
||||
RunFinished(
|
||||
run_id=RUN_ID,
|
||||
result=RunResult(
|
||||
run_id=RUN_ID,
|
||||
stop_reason=StopReason.STEP_BUDGET,
|
||||
final_answer=None,
|
||||
steps=(),
|
||||
),
|
||||
)
|
||||
)
|
||||
assert await store.read_log(RUN_ID) == RunLog()
|
||||
assert inner.calls == ["run_started", "model_call_result", "run_finished", "read_log"]
|
||||
|
||||
|
||||
def test_self_kill_forwards_parameters_verbatim(tmp_path: Path) -> None:
|
||||
"""包装层不许往参数快照里加自己的键：父进程续跑用的是没包过的存储，加了就报假漂移。"""
|
||||
inner = _RecordingStore()
|
||||
store = wrapped(inner, timing=KillTiming.AFTER_STEP, workspace=tmp_path, audited=True)
|
||||
assert store.parameters() == inner.parameters()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 十三、子进程编排：真起一个假子进程
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: 假子进程：往指定文件里逐条追加记录，中间留出足够父进程轮询到的间隔，然后一直睡着不退出。
|
||||
#: 它不打模型、不起容器，只是一个会按顺序写文件的东西。
|
||||
#: 它不打模型、不起容器，只是一个会按顺序写文件的东西。走的是外部 SIGKILL 那条兜底路径。
|
||||
_FAKE_CHILD = """
|
||||
import json, sys, time
|
||||
path = sys.argv[1]
|
||||
@@ -613,6 +818,7 @@ with open(path, "a", encoding="utf-8") as handle:
|
||||
time.sleep(30)
|
||||
"""
|
||||
|
||||
#: 写完就正常退出，一次都不自杀。
|
||||
_FAKE_CHILD_EXITS = """
|
||||
import json, sys
|
||||
path = sys.argv[1]
|
||||
@@ -622,8 +828,72 @@ with open(path, "a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(record) + "\\n")
|
||||
"""
|
||||
|
||||
#: 写完就按崩溃退出码把自己打死，模拟 `SelfKillingStore` 那条主路径。
|
||||
_FAKE_CHILD_SELF_KILL = """
|
||||
import json, os, sys
|
||||
path = sys.argv[1]
|
||||
records = json.loads(sys.argv[2])
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(json.dumps(record) + "\\n")
|
||||
handle.flush()
|
||||
print("子进程说了句话")
|
||||
sys.stdout.flush()
|
||||
os._exit(int(sys.argv[3]))
|
||||
"""
|
||||
|
||||
async def test_spawn_and_kill_hits_the_after_step_timing(tmp_path: Path) -> None:
|
||||
|
||||
async def test_spawn_and_kill_accepts_a_self_killed_child(tmp_path: Path) -> None:
|
||||
"""主路径：子进程按崩溃退出码自杀，且日志尾部形态对得上，算命中。"""
|
||||
log_path = tmp_path / f"{RUN_ID}.jsonl"
|
||||
records = [intent(), model_result(), step_completed()]
|
||||
outcome = await spawn_and_kill(
|
||||
argv=[
|
||||
sys.executable,
|
||||
"-c",
|
||||
_FAKE_CHILD_SELF_KILL,
|
||||
str(log_path),
|
||||
json.dumps(records),
|
||||
str(CRASH_EXIT_CODE),
|
||||
],
|
||||
log_path=log_path,
|
||||
timing=KillTiming.AFTER_STEP,
|
||||
after_steps=1,
|
||||
child_log_path=tmp_path / f"{RUN_ID}.child.log",
|
||||
timeout_s=20.0,
|
||||
)
|
||||
assert outcome.hit is True
|
||||
assert outcome.exit_code == CRASH_EXIT_CODE
|
||||
assert "自杀" in outcome.reason
|
||||
assert outcome.steps == 1
|
||||
assert outcome.snapshot == log_path.read_bytes()
|
||||
assert "子进程说了句话" in (tmp_path / f"{RUN_ID}.child.log").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
async def test_spawn_and_kill_rejects_a_self_kill_at_the_wrong_tail(tmp_path: Path) -> None:
|
||||
"""自杀了但尾部形态不对：报没命中，不许因为退出码对就当成命中。"""
|
||||
log_path = tmp_path / f"{RUN_ID}.jsonl"
|
||||
records = [intent(), model_result()]
|
||||
outcome = await spawn_and_kill(
|
||||
argv=[
|
||||
sys.executable,
|
||||
"-c",
|
||||
_FAKE_CHILD_SELF_KILL,
|
||||
str(log_path),
|
||||
json.dumps(records),
|
||||
str(CRASH_EXIT_CODE),
|
||||
],
|
||||
log_path=log_path,
|
||||
timing=KillTiming.AFTER_STEP,
|
||||
after_steps=1,
|
||||
timeout_s=20.0,
|
||||
)
|
||||
assert outcome.hit is False
|
||||
assert "尾部不是时机" in outcome.reason
|
||||
|
||||
|
||||
async def test_spawn_and_kill_still_falls_back_to_sigkill(tmp_path: Path) -> None:
|
||||
"""兜底路径没删：子进程一直不自杀时，父进程仍然会在尾部形态对上的那一刻杀掉它。"""
|
||||
log_path = tmp_path / f"{RUN_ID}.jsonl"
|
||||
records = [intent(), model_result(), step_completed()]
|
||||
outcome = await spawn_and_kill(
|
||||
@@ -634,13 +904,14 @@ async def test_spawn_and_kill_hits_the_after_step_timing(tmp_path: Path) -> None
|
||||
timeout_s=20.0,
|
||||
)
|
||||
assert outcome.hit is True
|
||||
assert "兜底" in outcome.reason
|
||||
assert outcome.steps == 1
|
||||
assert outcome.model_calls == 1
|
||||
assert outcome.snapshot == log_path.read_bytes()
|
||||
|
||||
|
||||
async def test_spawn_and_kill_reports_a_miss_when_the_child_exits(tmp_path: Path) -> None:
|
||||
"""子进程在命中时机之前就退出了要报出来，不许悄悄当成命中。"""
|
||||
async def test_spawn_and_kill_reports_a_miss_when_the_child_exits_normally(tmp_path: Path) -> None:
|
||||
"""子进程正常跑完了要报出来，不许悄悄当成命中。"""
|
||||
log_path = tmp_path / f"{RUN_ID}.jsonl"
|
||||
records = [intent(), model_result()]
|
||||
outcome = await spawn_and_kill(
|
||||
@@ -651,7 +922,8 @@ async def test_spawn_and_kill_reports_a_miss_when_the_child_exits(tmp_path: Path
|
||||
timeout_s=20.0,
|
||||
)
|
||||
assert outcome.hit is False
|
||||
assert "退出" in outcome.reason
|
||||
assert outcome.exit_code == 0
|
||||
assert f"不是按时机自杀的 {CRASH_EXIT_CODE}" in outcome.reason
|
||||
|
||||
|
||||
async def test_spawn_and_kill_reports_a_miss_on_timeout(tmp_path: Path) -> None:
|
||||
@@ -664,11 +936,11 @@ async def test_spawn_and_kill_reports_a_miss_on_timeout(tmp_path: Path) -> None:
|
||||
timeout_s=0.5,
|
||||
)
|
||||
assert outcome.hit is False
|
||||
assert "没命中时机" in outcome.reason
|
||||
assert "仍没崩在时机" in outcome.reason
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 十三、事件出口、护栏、sidecar、命令行
|
||||
# 十四、事件出口、护栏、sidecar、命令行
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user