iomgaa/PolyLoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix(soak): 父子两侧的崩溃条件对齐，并给自杀留一个窗口

Browse Source 
上一轮实跑里确定性自杀一次都没走到——父进程一看见日志尾部形态对上就发信号，而自杀条件
比它严一档（还要求审计账非空），于是外部信号永远抢先，自杀路径成了死代码，崩溃点又变回
碰运气。表现是崩得太早：动作还没执行过，最硬的那条判据没有实料可判。

两处对齐：父进程的命中条件也要求审计账非空（做成必传参数，给默认值等于让某个调用点静默
跳过这一条，而这正是这次出问题的方式）；兜底 SIGKILL 之前先等两秒看子进程是不是自己以
137 退出。

实跑结果：两档都走确定性自杀路径，都崩在「写入执行过之后」，绝不重放那条判据在两档都有
实料——审计账 1 条、去重后仍 1 条，续跑前后也都是 1 条。整套七类击穿 0 条、无法判定 0 条。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
iomgaa
2026-08-11 10:41:27 -04:00
parent a641a1fc11
commit 5d5371792d
2 changed files with 237 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/soak/tests/test_faults.py
+146 -45
View File
@@ -588,32 +588,69 @@ def test_lease_returned_passes_and_breaches() -> None:
def test_should_kill_after_step_waits_for_enough_steps() -> None:
read = as_read(intent(), model_result(), step_completed())
assert should_kill(read, timing=KillTiming.AFTER_STEP, after_steps=2) is False
assert should_kill(read, timing=KillTiming.AFTER_STEP, after_steps=1) is True
assert (
should_kill(read, timing=KillTiming.AFTER_STEP, after_steps=2, audit_is_not_empty=True)
is False
)
assert (
should_kill(read, timing=KillTiming.AFTER_STEP, after_steps=1, audit_is_not_empty=True)
is True
)
def test_should_kill_after_step_rejects_trailing_intent() -> None:
read = as_read(step_completed(), intent(call_index=1, result_id="r1"))
assert should_kill(read, timing=KillTiming.AFTER_STEP, after_steps=1) is False
assert (
should_kill(read, timing=KillTiming.AFTER_STEP, after_steps=1, audit_is_not_empty=True)
is False
)
def test_should_kill_at_intent_requires_never_policy() -> None:
"""`safe` 那条意图不算命中：悬在它上面续跑会重放动作接着跑，停止原因不是状态未知。"""
never = as_read(step_completed(), intent(call_index=1, result_id="r1", replay_policy="never"))
safe = as_read(step_completed(), intent(call_index=1, result_id="r1", replay_policy="safe"))
assert should_kill(never, timing=KillTiming.AT_INTENT, after_steps=1) is True
assert should_kill(safe, timing=KillTiming.AT_INTENT, after_steps=1) is False
assert (
should_kill(never, timing=KillTiming.AT_INTENT, after_steps=1, audit_is_not_empty=True)
is True
)
assert (
should_kill(safe, timing=KillTiming.AT_INTENT, after_steps=1, audit_is_not_empty=True)
is False
)
def test_should_kill_at_intent_rejects_trailing_step() -> None:
read = as_read(step_completed())
assert should_kill(read, timing=KillTiming.AT_INTENT, after_steps=0) is False
assert (
should_kill(read, timing=KillTiming.AT_INTENT, after_steps=0, audit_is_not_empty=True)
is False
)
def test_should_kill_on_empty_log() -> None:
empty = LogRead(payloads=(), torn=False, bad_lines=())
assert should_kill(empty, timing=KillTiming.AFTER_STEP, after_steps=0) is False
assert should_kill(empty, timing=KillTiming.AT_INTENT, after_steps=0) is False
assert (
should_kill(empty, timing=KillTiming.AFTER_STEP, after_steps=0, audit_is_not_empty=True)
is False
)
assert (
should_kill(empty, timing=KillTiming.AT_INTENT, after_steps=0, audit_is_not_empty=True)
is False
)
def test_should_kill_requires_a_non_empty_audit() -> None:
"""父进程的条件必须与子进程的自杀条件逐字对齐，包括审计账那一项。
松一档的后果实测过：父进程每次都在子进程走到自杀那一行之前抢先发信号，自杀路径成了死代码，
而崩溃点落在哪儿又变回碰运气——其中一次就崩在审计账还是空的时候，最硬那条判据只能真空成立。
"""
after_step = as_read(intent(), model_result(), step_completed())
at_intent = as_read(step_completed(), intent(call_index=1, result_id="r1"))
for read, timing in ((after_step, KillTiming.AFTER_STEP), (at_intent, KillTiming.AT_INTENT)):
assert should_kill(read, timing=timing, after_steps=1, audit_is_not_empty=True) is True
assert should_kill(read, timing=timing, after_steps=1, audit_is_not_empty=False) is False
# ---------------------------------------------------------------------------
@@ -707,7 +744,7 @@ def wrapped(
"""包一层，并按需要让工作区的审计账非空。"""
workspace.mkdir(parents=True, exist_ok=True)
if audited:
(workspace / AUDIT_LOG_NAME).write_text("write_note\tevidence.md\taaa\n", encoding="utf-8")
seed_audit(workspace)
return SelfKillingStore(
inner=inner, timing=timing, workspace=workspace, exit_now=_exit_sentinel
)
@@ -828,9 +865,10 @@ with open(path, "a", encoding="utf-8") as handle:
handle.write(json.dumps(record) + "\\n")
"""
#: 写完就按崩溃退出码把自己打死，模拟 `SelfKillingStore` 那条主路径。
#: 写完就按给定的退出码把自己打死，模拟 `SelfKillingStore` 那条主路径。第四个参数是写完之后
#: 先睡多久再死——用来验父进程真的留了自杀窗口，而不是看见条件成立就立刻开枪。
_FAKE_CHILD_SELF_KILL = """
import json, os, sys
import json, os, sys, time
path = sys.argv[1]
records = json.loads(sys.argv[2])
with open(path, "a", encoding="utf-8") as handle:
@@ -839,24 +877,39 @@ with open(path, "a", encoding="utf-8") as handle:
handle.flush()
print("子进程说了句话")
sys.stdout.flush()
time.sleep(float(sys.argv[4]))
os._exit(int(sys.argv[3]))
"""
def seed_audit(workspace: Path) -> Path:
"""让工作区的审计账非空。父子两侧的命中条件都要求它。"""
workspace.mkdir(parents=True, exist_ok=True)
(workspace / AUDIT_LOG_NAME).write_text("write_note\tevidence.md\taaa\n", encoding="utf-8")
return workspace
def self_kill_argv(log_path: Path, records: list[dict[str, object]], *, code: int, sleep: float):
return [
sys.executable,
"-c",
_FAKE_CHILD_SELF_KILL,
str(log_path),
json.dumps(records),
str(code),
str(sleep),
]
async def test_spawn_and_kill_accepts_a_self_killed_child(tmp_path: Path) -> None:
"""主路径：子进程按崩溃退出码自杀，且日志尾部形态对得上，算命中。"""
log_path = tmp_path / f"{RUN_ID}.jsonl"
workspace = seed_audit(tmp_path / "ws")
records = [intent(), model_result(), step_completed()]
outcome = await spawn_and_kill(
argv=[
sys.executable,
"-c",
_FAKE_CHILD_SELF_KILL,
str(log_path),
json.dumps(records),
str(CRASH_EXIT_CODE),
],
argv=self_kill_argv(log_path, records, code=CRASH_EXIT_CODE, sleep=0.0),
log_path=log_path,
workspace=workspace,
timing=KillTiming.AFTER_STEP,
after_steps=1,
child_log_path=tmp_path / f"{RUN_ID}.child.log",
@@ -870,20 +923,82 @@ async def test_spawn_and_kill_accepts_a_self_killed_child(tmp_path: Path) -> Non
assert "子进程说了句话" in (tmp_path / f"{RUN_ID}.child.log").read_text(encoding="utf-8")
async def test_spawn_and_kill_waits_out_the_self_kill_grace(tmp_path: Path) -> None:
"""条件对上之后子进程还要过一会儿才死：父进程必须等它，不许抢先开枪。
抢先的后果不是「杀错了」，是自杀那条路永远走不到，而崩溃点落在哪儿又变回碰运气。
"""
log_path = tmp_path / f"{RUN_ID}.jsonl"
workspace = seed_audit(tmp_path / "ws")
records = [intent(), model_result(), step_completed()]
outcome = await spawn_and_kill(
argv=self_kill_argv(log_path, records, code=CRASH_EXIT_CODE, sleep=0.4),
log_path=log_path,
workspace=workspace,
timing=KillTiming.AFTER_STEP,
after_steps=1,
self_kill_grace_s=5.0,
timeout_s=20.0,
)
assert outcome.hit is True
assert outcome.exit_code == CRASH_EXIT_CODE
assert "自杀" in outcome.reason
async def test_spawn_and_kill_falls_back_after_the_grace_runs_out(tmp_path: Path) -> None:
"""窗口用完子进程还活着，才轮到兜底 SIGKILL。这条路径没被删。"""
log_path = tmp_path / f"{RUN_ID}.jsonl"
workspace = seed_audit(tmp_path / "ws")
records = [intent(), model_result(), step_completed()]
outcome = await spawn_and_kill(
argv=self_kill_argv(log_path, records, code=CRASH_EXIT_CODE, sleep=30.0),
log_path=log_path,
workspace=workspace,
timing=KillTiming.AFTER_STEP,
after_steps=1,
self_kill_grace_s=0.2,
timeout_s=20.0,
)
assert outcome.hit is True
assert "兜底" in outcome.reason
assert outcome.exit_code == -9
assert outcome.steps == 1
assert outcome.model_calls == 1
assert outcome.snapshot == log_path.read_bytes()
async def test_spawn_and_kill_needs_a_non_empty_audit(tmp_path: Path) -> None:
"""日志尾部形态对了但审计账是空的：不算命中，等到超时为止。
这一条守的正是上一版实测出来的毛病——那次崩在第 1 步、审计账 0 条，最硬那条判据只能报
无法判定，而报告上看起来只是「少了一条」。
"""
log_path = tmp_path / f"{RUN_ID}.jsonl"
workspace = tmp_path / "ws"
workspace.mkdir()
records = [intent(), model_result(), step_completed()]
outcome = await spawn_and_kill(
argv=[sys.executable, "-c", _FAKE_CHILD, str(log_path), json.dumps(records)],
log_path=log_path,
workspace=workspace,
timing=KillTiming.AFTER_STEP,
after_steps=1,
self_kill_grace_s=0.2,
timeout_s=0.6,
)
assert outcome.hit is False
assert "仍没崩在时机" in outcome.reason
async def test_spawn_and_kill_rejects_a_self_kill_at_the_wrong_tail(tmp_path: Path) -> None:
"""自杀了但尾部形态不对：报没命中，不许因为退出码对就当成命中。"""
log_path = tmp_path / f"{RUN_ID}.jsonl"
workspace = seed_audit(tmp_path / "ws")
records = [intent(), model_result()]
outcome = await spawn_and_kill(
argv=[
sys.executable,
"-c",
_FAKE_CHILD_SELF_KILL,
str(log_path),
json.dumps(records),
str(CRASH_EXIT_CODE),
],
argv=self_kill_argv(log_path, records, code=CRASH_EXIT_CODE, sleep=0.0),
log_path=log_path,
workspace=workspace,
timing=KillTiming.AFTER_STEP,
after_steps=1,
timeout_s=20.0,
@@ -892,31 +1007,15 @@ async def test_spawn_and_kill_rejects_a_self_kill_at_the_wrong_tail(tmp_path: Pa
assert "尾部不是时机" in outcome.reason
async def test_spawn_and_kill_still_falls_back_to_sigkill(tmp_path: Path) -> None:
"""兜底路径没删：子进程一直不自杀时，父进程仍然会在尾部形态对上的那一刻杀掉它。"""
log_path = tmp_path / f"{RUN_ID}.jsonl"
records = [intent(), model_result(), step_completed()]
outcome = await spawn_and_kill(
argv=[sys.executable, "-c", _FAKE_CHILD, str(log_path), json.dumps(records)],
log_path=log_path,
timing=KillTiming.AFTER_STEP,
after_steps=1,
timeout_s=20.0,
)
assert outcome.hit is True
assert "兜底" in outcome.reason
assert outcome.steps == 1
assert outcome.model_calls == 1
assert outcome.snapshot == log_path.read_bytes()
async def test_spawn_and_kill_reports_a_miss_when_the_child_exits_normally(tmp_path: Path) -> None:
"""子进程正常跑完了要报出来，不许悄悄当成命中。"""
log_path = tmp_path / f"{RUN_ID}.jsonl"
workspace = seed_audit(tmp_path / "ws")
records = [intent(), model_result()]
outcome = await spawn_and_kill(
argv=[sys.executable, "-c", _FAKE_CHILD_EXITS, str(log_path), json.dumps(records)],
log_path=log_path,
workspace=workspace,
timing=KillTiming.AFTER_STEP,
after_steps=1,
timeout_s=20.0,
@@ -928,9 +1027,11 @@ async def test_spawn_and_kill_reports_a_miss_when_the_child_exits_normally(tmp_p
async def test_spawn_and_kill_reports_a_miss_on_timeout(tmp_path: Path) -> None:
log_path = tmp_path / f"{RUN_ID}.jsonl"
workspace = seed_audit(tmp_path / "ws")
outcome = await spawn_and_kill(
argv=[sys.executable, "-c", _FAKE_CHILD, str(log_path), json.dumps([intent()])],
log_path=log_path,
workspace=workspace,
timing=KillTiming.AFTER_STEP,
after_steps=1,
timeout_s=0.5,