fix: isolate gate baseline-arm INFRA errors from BaselineCache (algo #6)
This commit is contained in:
@@ -505,6 +505,76 @@ async def test_baseline_cache_hit(tmp_path: Path) -> None:
|
||||
log.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None:
|
||||
"""基线臂 INFRA error 的 unit 不写入 BaselineCache(不永久污染),且从有效单元排除。"""
|
||||
from app.harness.gate_ladder import skill_hash
|
||||
from app.harness.question_units import build_units
|
||||
from app.harness.validate import _resolve_baseline_block
|
||||
|
||||
workspace = _setup_workspace(tmp_path)
|
||||
log = _make_log(workspace)
|
||||
questions = _make_questions(2) # q0 干净, q1 INFRA error
|
||||
units = build_units(questions)
|
||||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||
s_hash = skill_hash("baseline skill content")
|
||||
|
||||
async def mock_fn(qs, *, run_id, skills_dir):
|
||||
for q in qs:
|
||||
is_err = q.question_id == "q1"
|
||||
log.insert(
|
||||
"predictions",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"video_id": "v0",
|
||||
"question_id": q.question_id,
|
||||
"task_type": "temporal",
|
||||
"prediction": "" if is_err else "A",
|
||||
"answer": "A",
|
||||
"evidence": "",
|
||||
"reasoning": "",
|
||||
"steps_used": 1,
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 10,
|
||||
"stop_reason": "error" if is_err else "completed",
|
||||
"steps_json": "[]",
|
||||
},
|
||||
)
|
||||
return InferenceResult(
|
||||
run_id=run_id,
|
||||
accuracy=0.5,
|
||||
total=2,
|
||||
correct=1,
|
||||
per_task_type={},
|
||||
steps_mean=1.0,
|
||||
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||||
stop_reason_counts={"completed": 1, "error": 1},
|
||||
)
|
||||
|
||||
try:
|
||||
b_units, valid_units, _errors_inc, _denom_inc = await _resolve_baseline_block(
|
||||
units=units,
|
||||
task_type="temporal",
|
||||
s_hash=s_hash,
|
||||
prompts_version="p1",
|
||||
baseline_cache=cache,
|
||||
base_skills_dir=workspace / "skills" / "v1",
|
||||
run_inference=mock_fn,
|
||||
log=log,
|
||||
run_id="step1_gate_b0_base",
|
||||
)
|
||||
# q1 是 INFRA:不写缓存、不入 b_units、不在有效单元里
|
||||
assert cache.get("temporal", s_hash, "p1", "q1") is None
|
||||
assert "q1" not in b_units
|
||||
assert all(u.unit_id != "q1" for u in valid_units)
|
||||
# q0 干净:正常缓存并入 b_units/valid_units
|
||||
assert cache.get("temporal", s_hash, "p1", "q0") is True
|
||||
assert b_units["q0"] is True
|
||||
assert any(u.unit_id == "q0" for u in valid_units)
|
||||
finally:
|
||||
log.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_last_block_terminal(tmp_path: Path) -> None:
|
||||
"""单块 + n_remaining=0 → 终态判定(provisional 或 inertia),非 continue。"""
|
||||
|
||||
Reference in New Issue
Block a user