"""故障注入判据的测试。不打真实模型、不起容器，全部用构造出来的输入。 **每一条判据都要有一个「构造出违反它的输入 → 判据确实报击穿」的用例。** 一个永远返回通过的 判据比没有判据更糟：它会让所有人以为这些不变量被守着，而它什么都没守，而且这件事在压测报告 上看起来是一整片绿。所以下面每条判据都成对出现——一条喂它合规的输入验它说通过，一条喂它明确 违反的输入验它说击穿。 子进程编排那部分拆出了两个纯函数（`should_kill` 判时机到没到、`terminated_prefix` 截已终结 前缀），它们不碰进程也不碰模型，直接单独测。真起子进程那两条用的是一个只会往文件里写几行 JSON 的假子进程，跑完不到两秒。 """ from __future__ import annotations import asyncio import json import sys from pathlib import Path import pytest from polyloop.ports import EventKind, InvalidDecision from polyloop.types import ModelReply, RunResult, StopReason from tools.soak.faults import ( AlwaysInvalidParser, CallGuard, Criterion, CriterionStatus, FaultReport, JsonlEventSink, KillTiming, LogRead, build_parser, check_all_steps_parse_failed, check_audit_unchanged, check_cancelled_raised, check_crash_prefix_preserved, check_env_untouched, check_executed_action_count, check_intents_settled, check_lease_returned, check_log_readable, check_never_action_not_replayed, check_no_env_error_step, check_resume_made_progress, check_step_count, check_step_indices_dense, check_stop_reason, count_model_calls, guarded, main, parse_audit_line, parse_terminated, read_log, should_kill, spawn_and_kill, stop_reason_of, tagged, terminated_prefix, write_sidecars, ) RUN_ID = "fault-test-0" # --------------------------------------------------------------------------- # 造记录：日志行的形状照 `polyloop.serialization.encode` 加 `polyloop.stores` 的类型标签 # --------------------------------------------------------------------------- def intent( *, kind: str = "model_call", call_index: int = 0, result_id: str = "r0", replay_policy: str = "never", ) -> dict[str, object]: return { "record": "intent", "run_id": RUN_ID, "kind": kind, "call_index": call_index, "result_id": result_id, "replay_policy": replay_policy, } def model_result(*, result_id: str = "r0") -> dict[str, object]: return { "record": "model_call_result", "run_id": RUN_ID, "result_id": result_id, "reply": {"call_id": "c0", "content": "x", "thinking": ""}, "failure": None, } def step_completed( *, step_idx: int = 0, result_id: str | None = "a0", status: str | None = "executed", parse_ok: bool = True, action_status: str | None = "executed", ) -> dict[str, object]: outcome = ( None if status is None else { "status": status, "observation": "o", "observation_is_synthetic": False, "env_reported_completion": False, "observation_truncated_chars": 0, } ) return { "record": "step_completed", "run_id": RUN_ID, "result_id": result_id, "action_outcome": outcome, "step": { "step_idx": step_idx, "raw_output": "x", "content_chars": 1, "thinking_chars": 0, "action": None, "parse_ok": parse_ok, "parse_error": None if parse_ok else "解释不了", "observation": "o", "observation_is_synthetic": False, "observation_truncated_chars": 0, "prompt_chars": 10, "call_id": "c0", "step_wall_ms": 1, "tool_name": None, "tool_arguments": None, "action_status": action_status, "env_reported_completion": False, "schema_version": 1, }, } def run_finished(*, stop_reason: str = "task_completed") -> dict[str, object]: return { "record": "run_finished", "run_id": RUN_ID, "result": { "run_id": RUN_ID, "stop_reason": stop_reason, "final_answer": None, "steps": [], "schema_version": 1, "event_delivery_failures": 0, }, } def as_bytes(*payloads: dict[str, object]) -> bytes: return "".join(json.dumps(item, ensure_ascii=False) + "\n" for item in payloads).encode("utf-8") def as_read(*payloads: dict[str, object]) -> LogRead: return parse_terminated(as_bytes(*payloads)) # --------------------------------------------------------------------------- # 一、判据本身的形状 # --------------------------------------------------------------------------- def test_criterion_requires_evidence() -> None: """没有证据的判据构造不出来。一条只会说「有问题」的判据等于没有判据。""" with pytest.raises(ValueError, match="没有给证据"): Criterion(name="x", status=CriterionStatus.PASSED, evidence=" ") def test_criterion_requires_name() -> None: with pytest.raises(ValueError, match="必须有名字"): Criterion(name=" ", status=CriterionStatus.PASSED, evidence="有证据") # --------------------------------------------------------------------------- # 二、日志读取 # --------------------------------------------------------------------------- def test_terminated_prefix_cuts_at_last_newline() -> None: assert terminated_prefix(b'{"a":1}\n{"b":2}\n') == b'{"a":1}\n{"b":2}\n' assert terminated_prefix(b'{"a":1}\n{"b":') == b'{"a":1}\n' assert terminated_prefix(b'{"a":') == b"" def test_parse_terminated_drops_torn_tail() -> None: raw = as_bytes(intent()) + b'{"record":"inte' read = parse_terminated(raw) assert read.torn is True assert len(read.payloads) == 1 assert read.bad_lines == () def test_parse_terminated_flags_bad_terminated_line() -> None: raw = as_bytes(intent()) + b"not json at all\n" read = parse_terminated(raw) assert read.torn is False assert read.bad_lines == (2,) def test_check_log_readable_breaches_on_bad_line() -> None: read = parse_terminated(as_bytes(intent()) + b"{}\n") outcome = check_log_readable(read) assert outcome.status is CriterionStatus.BREACHED def test_check_log_readable_passes_on_clean_log() -> None: assert check_log_readable(as_read(intent())).status is CriterionStatus.PASSED def test_read_log_of_missing_file_is_empty() -> None: assert read_log(Path("/nonexistent/nope.jsonl")).payloads == () def test_tagged_and_counters() -> None: read = as_read( intent(kind="model_call", result_id="r0"), intent(kind="action", result_id="a0"), run_finished(stop_reason="step_budget"), ) assert len(tagged(read, "intent")) == 2 assert count_model_calls(read) == 1 assert stop_reason_of(read) == "step_budget" def test_stop_reason_of_without_finished_record() -> None: assert stop_reason_of(as_read(intent())) is None # --------------------------------------------------------------------------- # 三、崩溃续跑：字节前缀 # --------------------------------------------------------------------------- def test_prefix_preserved_passes_when_resume_only_appends() -> None: crashed = as_bytes(intent(), model_result()) final = crashed + as_bytes(step_completed()) outcome = check_crash_prefix_preserved(crashed=crashed, final=final) assert outcome.status is CriterionStatus.PASSED def test_prefix_preserved_breaches_when_one_byte_changed() -> None: """前缀里改一个字节就必须报击穿。 这是这条判据存在的全部理由：解析出来的对象可能仍然等价（改的是空格、字段顺序、数字的 表示法），而承诺是更硬的那一条——已经写下去的字节不许再动。 """ crashed = as_bytes(intent(), model_result()) mutated = bytearray(crashed) mutated[5] = mutated[5] ^ 0x01 final = bytes(mutated) + as_bytes(step_completed()) outcome = check_crash_prefix_preserved(crashed=crashed, final=final) assert outcome.status is CriterionStatus.BREACHED assert "第 5 字节" in outcome.evidence def test_prefix_preserved_breaches_when_final_is_shorter() -> None: crashed = as_bytes(intent(), model_result()) outcome = check_crash_prefix_preserved(crashed=crashed, final=crashed[:10]) assert outcome.status is CriterionStatus.BREACHED def test_prefix_preserved_ignores_torn_tail() -> None: """崩溃快照末尾那段没被换行终结的字节不参与比对：那次写从来没算数。""" crashed = as_bytes(intent()) + b'{"record":"model_ca' final = as_bytes(intent()) + b'{"record":"model_ca' + as_bytes(model_result()) assert check_crash_prefix_preserved(crashed=crashed, final=final).status is ( CriterionStatus.PASSED ) def test_prefix_preserved_undetermined_without_any_terminated_record() -> None: outcome = check_crash_prefix_preserved(crashed=b'{"half', final=b'{"half"}\n') assert outcome.status is CriterionStatus.UNDETERMINED # --------------------------------------------------------------------------- # 四、崩溃续跑：步号 # --------------------------------------------------------------------------- def test_step_indices_dense_passes() -> None: read = as_read(*(step_completed(step_idx=index) for index in range(4))) assert check_step_indices_dense(read).status is CriterionStatus.PASSED def test_step_indices_dense_breaches_on_gap() -> None: read = as_read(step_completed(step_idx=0), step_completed(step_idx=2)) assert check_step_indices_dense(read).status is CriterionStatus.BREACHED def test_step_indices_dense_breaches_on_repeat() -> None: """同一个步号出现两次：续跑把已经落地的那一步又跑了一遍。""" read = as_read(step_completed(step_idx=0), step_completed(step_idx=0)) assert check_step_indices_dense(read).status is CriterionStatus.BREACHED def test_step_indices_dense_undetermined_without_steps() -> None: assert check_step_indices_dense(as_read(intent())).status is CriterionStatus.UNDETERMINED # --------------------------------------------------------------------------- # 五、崩溃续跑：意图有没有归宿 # --------------------------------------------------------------------------- def test_intents_settled_passes_when_all_have_results() -> None: read = as_read( intent(kind="model_call", result_id="r0"), model_result(result_id="r0"), intent(kind="action", result_id="a0"), step_completed(result_id="a0"), ) assert check_intents_settled(read).status is CriterionStatus.PASSED def test_intents_settled_allows_one_dangling_at_the_end() -> None: """崩溃点上那条悬空的意图是合法的：时机 B 下它就是崩溃点本身。""" read = as_read( intent(kind="model_call", result_id="r0"), model_result(result_id="r0"), intent(kind="action", result_id="a0"), step_completed(result_id="a0"), intent(kind="model_call", call_index=1, result_id="r1"), run_finished(stop_reason="resume_state_unknown"), ) assert check_intents_settled(read).status is CriterionStatus.PASSED def test_intents_settled_breaches_on_dangling_in_the_middle() -> None: """中间悬空说明有一步的执行状态被跳过去了，后面的步建立在一个没有答案的问题上。""" read = as_read( intent(kind="model_call", result_id="r0"), intent(kind="model_call", call_index=1, result_id="r1"), model_result(result_id="r1"), ) outcome = check_intents_settled(read) assert outcome.status is CriterionStatus.BREACHED assert "[0]" in outcome.evidence def test_intents_settled_undetermined_without_intents() -> None: assert check_intents_settled(as_read(run_finished())).status is CriterionStatus.UNDETERMINED # --------------------------------------------------------------------------- # 六、崩溃续跑：审计账 # --------------------------------------------------------------------------- def test_parse_audit_line() -> None: assert parse_audit_line("write_note\tplan.md\tabc123") == ("write_note", "plan.md", "abc123") assert parse_audit_line("write_note\tplan.md") is None assert parse_audit_line("write_note\t\tabc123") is None def test_never_action_not_replayed_passes_on_distinct_writes() -> None: audit = ["write_note\tplan.md\taaa", "write_note\tevidence.md\tbbb"] assert check_never_action_not_replayed(audit).status is CriterionStatus.PASSED def test_never_action_not_replayed_breaches_on_duplicate() -> None: """同一个 (文件名, 内容摘要) 出现两次就是重放。""" audit = [ "write_note\tevidence.md\tbbb", "write_note\tplan.md\taaa", "write_note\tevidence.md\tbbb", ] outcome = check_never_action_not_replayed(audit) assert outcome.status is CriterionStatus.BREACHED assert "bbb" in outcome.evidence def test_never_action_not_replayed_undetermined_on_empty_audit() -> None: """一条副作用都没执行过时不许报「通过」——那次跑根本没验到这条不变量。""" assert check_never_action_not_replayed([]).status is CriterionStatus.UNDETERMINED def test_never_action_not_replayed_undetermined_on_malformed_audit() -> None: assert check_never_action_not_replayed(["坏行"]).status is CriterionStatus.UNDETERMINED def test_audit_unchanged_passes() -> None: audit = ["write_note\tplan.md\taaa"] assert check_audit_unchanged(before=audit, after=list(audit)).status is CriterionStatus.PASSED def test_audit_unchanged_breaches_when_resume_adds_a_line() -> None: before = ["write_note\tplan.md\taaa"] after = [*before, "write_note\tplan.md\taaa"] outcome = check_audit_unchanged(before=before, after=after) assert outcome.status is CriterionStatus.BREACHED assert "多出 1 条" in outcome.evidence def test_audit_unchanged_breaches_when_prefix_rewritten() -> None: outcome = check_audit_unchanged( before=["write_note\tplan.md\taaa"], after=["write_note\tplan.md\tzzz"] ) assert outcome.status is CriterionStatus.BREACHED # --------------------------------------------------------------------------- # 七、崩溃续跑：续跑有没有往下走 # --------------------------------------------------------------------------- def test_resume_made_progress_passes() -> None: assert check_resume_made_progress(crashed_steps=2, final_steps=5).status is ( CriterionStatus.PASSED ) def test_resume_made_progress_breaches_when_stalled() -> None: assert check_resume_made_progress(crashed_steps=2, final_steps=2).status is ( CriterionStatus.BREACHED ) # --------------------------------------------------------------------------- # 八、停止原因与步数 # --------------------------------------------------------------------------- def test_stop_reason_passes() -> None: read = as_read(run_finished(stop_reason="step_budget")) assert check_stop_reason(read, "step_budget").status is CriterionStatus.PASSED def test_stop_reason_breaches_on_wrong_value() -> None: read = as_read(run_finished(stop_reason="env_error")) outcome = check_stop_reason(read, "step_budget") assert outcome.status is CriterionStatus.BREACHED assert "env_error" in outcome.evidence def test_stop_reason_breaches_without_finished_record() -> None: """取消也要留结束记录，不然恢复会把它当成可以续跑。缺记录是击穿，不是判不了。""" outcome = check_stop_reason(as_read(step_completed()), "cancelled") assert outcome.status is CriterionStatus.BREACHED def test_step_count_passes_and_breaches() -> None: read = as_read(*(step_completed(step_idx=index) for index in range(3))) assert check_step_count(read, expected=3, name="n").status is CriterionStatus.PASSED assert check_step_count(read, expected=2, name="n").status is CriterionStatus.BREACHED def test_executed_action_count_passes_and_breaches() -> None: read = as_read( step_completed(step_idx=0, status="executed"), step_completed(step_idx=1, status="executed"), step_completed(step_idx=2, status="not_executed"), ) assert check_executed_action_count(read, expected=2).status is CriterionStatus.PASSED assert check_executed_action_count(read, expected=3).status is CriterionStatus.BREACHED def test_no_env_error_step_passes_and_breaches() -> None: clean = as_read(step_completed(step_idx=0, status="executed")) assert check_no_env_error_step(clean).status is CriterionStatus.PASSED dirty = as_read( step_completed(step_idx=0, status="executed"), step_completed(step_idx=1, status="env_error"), ) outcome = check_no_env_error_step(dirty) assert outcome.status is CriterionStatus.BREACHED assert "[1]" in outcome.evidence # --------------------------------------------------------------------------- # 九、解析失败连击 # --------------------------------------------------------------------------- def test_all_steps_parse_failed_passes() -> None: read = as_read( *( step_completed( step_idx=index, result_id=None, status=None, parse_ok=False, action_status=None ) for index in range(3) ) ) assert check_all_steps_parse_failed(read).status is CriterionStatus.PASSED def test_all_steps_parse_failed_breaches_when_a_step_parsed() -> None: read = as_read( step_completed(step_idx=0, result_id=None, status=None, parse_ok=False, action_status=None), step_completed(step_idx=1, parse_ok=True), ) assert check_all_steps_parse_failed(read).status is CriterionStatus.BREACHED def test_all_steps_parse_failed_breaches_when_an_action_was_dispatched() -> None: """解析失败那一步不许带动作状态——带了就说明有动作被分发过。""" read = as_read( step_completed(step_idx=0, parse_ok=False, action_status="executed"), ) assert check_all_steps_parse_failed(read).status is CriterionStatus.BREACHED def test_all_steps_parse_failed_undetermined_without_steps() -> None: assert check_all_steps_parse_failed(as_read(intent())).status is CriterionStatus.UNDETERMINED def test_env_untouched_passes_and_breaches() -> None: assert check_env_untouched(executions=0, source="假环境").status is CriterionStatus.PASSED outcome = check_env_untouched(executions=1, source="假环境") assert outcome.status is CriterionStatus.BREACHED def test_always_invalid_parser_is_sync_and_never_raises() -> None: parser = AlwaysInvalidParser() parsed = parser.parse(ModelReply(call_id=None, content="```python\nprint(1)\n```", thinking="")) assert isinstance(parsed.decision, InvalidDecision) assert parsed.decision.explanation.strip() # 契约：回填历史的那段不许比模型原文长。 assert len(parsed.history_text) <= len("```python\nprint(1)\n```") assert parser.parameters()["kind"] == "always_invalid" # --------------------------------------------------------------------------- # 十、取消 # --------------------------------------------------------------------------- def test_cancelled_raised_passes() -> None: outcome = check_cancelled_raised(raised=asyncio.CancelledError()) assert outcome.status is CriterionStatus.PASSED def test_cancelled_raised_breaches_when_swallowed() -> None: """取消被吞掉之后返回一个结果，调用方的结构化并发就断了。""" assert check_cancelled_raised(raised=None).status is CriterionStatus.BREACHED def test_cancelled_raised_breaches_on_other_exception() -> None: outcome = check_cancelled_raised(raised=RuntimeError("别的错")) assert outcome.status is CriterionStatus.BREACHED assert "RuntimeError" in outcome.evidence def test_lease_returned_passes_and_breaches() -> None: assert check_lease_returned(borrowed=True, timeout_s=1.0, pool_size=1).status is ( CriterionStatus.PASSED ) assert check_lease_returned(borrowed=False, timeout_s=1.0, pool_size=1).status is ( CriterionStatus.BREACHED ) # --------------------------------------------------------------------------- # 十一、子进程编排：时机判定这个纯函数 # --------------------------------------------------------------------------- def test_should_kill_after_step_waits_for_enough_steps() -> None: read = as_read(intent(), model_result(), step_completed()) assert should_kill(read, timing=KillTiming.AFTER_STEP, after_steps=2) is False assert should_kill(read, timing=KillTiming.AFTER_STEP, after_steps=1) is True def test_should_kill_after_step_rejects_trailing_intent() -> None: read = as_read(step_completed(), intent(call_index=1, result_id="r1")) assert should_kill(read, timing=KillTiming.AFTER_STEP, after_steps=1) is False def test_should_kill_at_intent_requires_never_policy() -> None: """`safe` 那条意图不算命中：悬在它上面续跑会重放动作接着跑，停止原因不是状态未知。""" never = as_read(step_completed(), intent(call_index=1, result_id="r1", replay_policy="never")) safe = as_read(step_completed(), intent(call_index=1, result_id="r1", replay_policy="safe")) assert should_kill(never, timing=KillTiming.AT_INTENT, after_steps=1) is True assert should_kill(safe, timing=KillTiming.AT_INTENT, after_steps=1) is False def test_should_kill_at_intent_rejects_trailing_step() -> None: read = as_read(step_completed()) assert should_kill(read, timing=KillTiming.AT_INTENT, after_steps=0) is False def test_should_kill_on_empty_log() -> None: empty = LogRead(payloads=(), torn=False, bad_lines=()) assert should_kill(empty, timing=KillTiming.AFTER_STEP, after_steps=0) is False assert should_kill(empty, timing=KillTiming.AT_INTENT, after_steps=0) is False # --------------------------------------------------------------------------- # 十二、子进程编排：真起一个假子进程 # --------------------------------------------------------------------------- #: 假子进程：往指定文件里逐条追加记录，中间留出足够父进程轮询到的间隔，然后一直睡着不退出。 #: 它不打模型、不起容器，只是一个会按顺序写文件的东西。 _FAKE_CHILD = """ import json, sys, time path = sys.argv[1] records = json.loads(sys.argv[2]) with open(path, "a", encoding="utf-8") as handle: for record in records: handle.write(json.dumps(record) + "\\n") handle.flush() time.sleep(0.05) time.sleep(30) """ _FAKE_CHILD_EXITS = """ import json, sys path = sys.argv[1] records = json.loads(sys.argv[2]) with open(path, "a", encoding="utf-8") as handle: for record in records: handle.write(json.dumps(record) + "\\n") """ async def test_spawn_and_kill_hits_the_after_step_timing(tmp_path: Path) -> None: log_path = tmp_path / f"{RUN_ID}.jsonl" records = [intent(), model_result(), step_completed()] outcome = await spawn_and_kill( argv=[sys.executable, "-c", _FAKE_CHILD, str(log_path), json.dumps(records)], log_path=log_path, timing=KillTiming.AFTER_STEP, after_steps=1, timeout_s=20.0, ) assert outcome.hit is True assert outcome.steps == 1 assert outcome.model_calls == 1 assert outcome.snapshot == log_path.read_bytes() async def test_spawn_and_kill_reports_a_miss_when_the_child_exits(tmp_path: Path) -> None: """子进程在命中时机之前就退出了要报出来，不许悄悄当成命中。""" log_path = tmp_path / f"{RUN_ID}.jsonl" records = [intent(), model_result()] outcome = await spawn_and_kill( argv=[sys.executable, "-c", _FAKE_CHILD_EXITS, str(log_path), json.dumps(records)], log_path=log_path, timing=KillTiming.AFTER_STEP, after_steps=1, timeout_s=20.0, ) assert outcome.hit is False assert "退出" in outcome.reason async def test_spawn_and_kill_reports_a_miss_on_timeout(tmp_path: Path) -> None: log_path = tmp_path / f"{RUN_ID}.jsonl" outcome = await spawn_and_kill( argv=[sys.executable, "-c", _FAKE_CHILD, str(log_path), json.dumps([intent()])], log_path=log_path, timing=KillTiming.AFTER_STEP, after_steps=1, timeout_s=0.5, ) assert outcome.hit is False assert "没命中时机" in outcome.reason # --------------------------------------------------------------------------- # 十三、事件出口、护栏、sidecar、命令行 # --------------------------------------------------------------------------- class _FakeEvent: """只带事件出口会读的那三个字段。""" def __init__(self, step_idx: int) -> None: self.kind = EventKind.STEP_FINISHED self.run_id = RUN_ID self.step = type("Step", (), {"step_idx": step_idx})() async def test_event_sink_writes_one_line_per_event(tmp_path: Path) -> None: sink = JsonlEventSink(tmp_path / f"{RUN_ID}.events.jsonl") await sink.emit(_FakeEvent(0)) # type: ignore[arg-type] await sink.emit(_FakeEvent(1)) # type: ignore[arg-type] lines = (tmp_path / f"{RUN_ID}.events.jsonl").read_text(encoding="utf-8").splitlines() assert [json.loads(line)["step_idx"] for line in lines] == [0, 1] assert sink.delivered == 2 def test_event_sink_parameters_carry_no_path() -> None: """路径进参数快照会让续跑报一次假的参数漂移。""" sink = JsonlEventSink("/tmp/whatever.jsonl") assert sink.parameters() == {"kind": "jsonl_events"} def test_call_guard_accounting() -> None: guard = CallGuard(limit=10) guard.charge(4) assert guard.remaining == 6 assert guard.affordable(6) is True assert guard.affordable(7) is False def test_write_sidecars_shape(tmp_path: Path) -> None: result = RunResult( run_id=RUN_ID, stop_reason=StopReason.STEP_BUDGET, final_answer=None, steps=() ) write_sidecars( runs_dir=tmp_path, run_id=RUN_ID, scenario="appworld", fault="step_budget", result=result, wall_ms=12, model_calls=3, sink_failures=0, env_executions=3, task_id="t0", phase=None, resumed_from_step=None, ) meta = json.loads((tmp_path / f"{RUN_ID}.meta.json").read_text(encoding="utf-8")) assert meta["fault"] == "step_budget" assert meta["scenario"] == "appworld" assert meta["success"] is None assert set(meta) == { "scenario", "task_id", "phase", "wall_ms", "model_calls", "sink_failures", "env_executions", "fault", "success", "resumed_from_step", } stored = json.loads((tmp_path / f"{RUN_ID}.result.json").read_text(encoding="utf-8")) assert stored["stop_reason"] == "step_budget" def test_write_sidecars_omits_result_when_there_is_none(tmp_path: Path) -> None: """取消那条路上 `run` 不返回结果，硬造一份是伪造。""" write_sidecars( runs_dir=tmp_path, run_id=RUN_ID, scenario="appworld", fault="cancel_model", result=None, wall_ms=1, model_calls=2, sink_failures=0, env_executions=1, task_id="t0", phase=None, resumed_from_step=None, ) assert not (tmp_path / f"{RUN_ID}.result.json").exists() assert (tmp_path / f"{RUN_ID}.meta.json").exists() async def test_guarded_turns_a_crash_into_undetermined() -> None: """某一类跑到一半炸了，前面几类已经花钱跑出来的结论不能跟着丢。""" async def boom() -> FaultReport: raise RuntimeError("容器起不来") report = await guarded("cancel_env", boom()) assert report.fault == "cancel_env" assert report.undetermineds assert "容器起不来" in report.undetermineds[0].evidence assert not report.breaches async def test_guarded_passes_a_normal_report_through() -> None: async def fine() -> FaultReport: return FaultReport(fault="x", criteria=(check_env_untouched(executions=0, source="假"),)) report = await guarded("x", fine()) assert not report.breaches assert not report.undetermineds def test_main_requires_budget_calls(tmp_path: Path) -> None: with pytest.raises(SystemExit): main(["--runs-dir", str(tmp_path)]) def test_main_rejects_unknown_fault(tmp_path: Path) -> None: with pytest.raises(SystemExit): main(["--runs-dir", str(tmp_path), "--budget-calls", "10", "--fault", "不存在"]) def test_parser_accepts_repeated_fault_flags() -> None: args = build_parser().parse_args( [ "--runs-dir", "x", "--budget-calls", "10", "--fault", "cancel_model", "--fault", "cancel_env", ] ) assert args.fault == ["cancel_model", "cancel_env"]